1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2015 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
25 #include "diagnostic-core.h"
27 /* Include insn-config.h before expr.h so that HAVE_conditional_move
28 is properly defined. */
29 #include "insn-config.h"
34 #include "double-int.h"
41 #include "tree-hasher.h"
42 #include "stor-layout.h"
43 #include "stringpool.h"
47 #include "hard-reg-set.h"
51 #include "statistics.h"
53 #include "fixed-value.h"
61 #include "insn-codes.h"
68 #include "dominance.h"
70 #include "basic-block.h"
73 struct target_optabs default_target_optabs
;
74 struct target_libfuncs default_target_libfuncs
;
75 struct target_optabs
*this_fn_optabs
= &default_target_optabs
;
77 struct target_optabs
*this_target_optabs
= &default_target_optabs
;
78 struct target_libfuncs
*this_target_libfuncs
= &default_target_libfuncs
;
81 #define libfunc_hash \
82 (this_target_libfuncs->x_libfunc_hash)
84 static void prepare_float_lib_cmp (rtx
, rtx
, enum rtx_code
, rtx
*,
86 static rtx
expand_unop_direct (machine_mode
, optab
, rtx
, rtx
, int);
87 static void emit_libcall_block_1 (rtx_insn
*, rtx
, rtx
, rtx
, bool);
89 /* Debug facility for use in GDB. */
90 void debug_optab_libfuncs (void);
92 /* Prefixes for the current version of decimal floating point (BID vs. DPD) */
93 #if ENABLE_DECIMAL_BID_FORMAT
94 #define DECIMAL_PREFIX "bid_"
96 #define DECIMAL_PREFIX "dpd_"
99 /* Used for libfunc_hash. */
102 libfunc_hasher::hash (libfunc_entry
*e
)
104 return ((e
->mode1
+ e
->mode2
* NUM_MACHINE_MODES
) ^ e
->op
);
107 /* Used for libfunc_hash. */
110 libfunc_hasher::equal (libfunc_entry
*e1
, libfunc_entry
*e2
)
112 return e1
->op
== e2
->op
&& e1
->mode1
== e2
->mode1
&& e1
->mode2
== e2
->mode2
;
115 /* Return libfunc corresponding operation defined by OPTAB converting
116 from MODE2 to MODE1. Trigger lazy initialization if needed, return NULL
117 if no libfunc is available. */
119 convert_optab_libfunc (convert_optab optab
, machine_mode mode1
,
122 struct libfunc_entry e
;
123 struct libfunc_entry
**slot
;
125 /* ??? This ought to be an assert, but not all of the places
126 that we expand optabs know about the optabs that got moved
128 if (!(optab
>= FIRST_CONV_OPTAB
&& optab
<= LAST_CONVLIB_OPTAB
))
134 slot
= libfunc_hash
->find_slot (&e
, NO_INSERT
);
137 const struct convert_optab_libcall_d
*d
138 = &convlib_def
[optab
- FIRST_CONV_OPTAB
];
140 if (d
->libcall_gen
== NULL
)
143 d
->libcall_gen (optab
, d
->libcall_basename
, mode1
, mode2
);
144 slot
= libfunc_hash
->find_slot (&e
, NO_INSERT
);
148 return (*slot
)->libfunc
;
151 /* Return libfunc corresponding operation defined by OPTAB in MODE.
152 Trigger lazy initialization if needed, return NULL if no libfunc is
155 optab_libfunc (optab optab
, machine_mode mode
)
157 struct libfunc_entry e
;
158 struct libfunc_entry
**slot
;
160 /* ??? This ought to be an assert, but not all of the places
161 that we expand optabs know about the optabs that got moved
163 if (!(optab
>= FIRST_NORM_OPTAB
&& optab
<= LAST_NORMLIB_OPTAB
))
169 slot
= libfunc_hash
->find_slot (&e
, NO_INSERT
);
172 const struct optab_libcall_d
*d
173 = &normlib_def
[optab
- FIRST_NORM_OPTAB
];
175 if (d
->libcall_gen
== NULL
)
178 d
->libcall_gen (optab
, d
->libcall_basename
, d
->libcall_suffix
, mode
);
179 slot
= libfunc_hash
->find_slot (&e
, NO_INSERT
);
183 return (*slot
)->libfunc
;
187 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
188 the result of operation CODE applied to OP0 (and OP1 if it is a binary
191 If the last insn does not set TARGET, don't do anything, but return 1.
193 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
194 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
195 try again, ensuring that TARGET is not one of the operands. */
198 add_equal_note (rtx_insn
*insns
, rtx target
, enum rtx_code code
, rtx op0
, rtx op1
)
204 gcc_assert (insns
&& INSN_P (insns
) && NEXT_INSN (insns
));
206 if (GET_RTX_CLASS (code
) != RTX_COMM_ARITH
207 && GET_RTX_CLASS (code
) != RTX_BIN_ARITH
208 && GET_RTX_CLASS (code
) != RTX_COMM_COMPARE
209 && GET_RTX_CLASS (code
) != RTX_COMPARE
210 && GET_RTX_CLASS (code
) != RTX_UNARY
)
213 if (GET_CODE (target
) == ZERO_EXTRACT
)
216 for (last_insn
= insns
;
217 NEXT_INSN (last_insn
) != NULL_RTX
;
218 last_insn
= NEXT_INSN (last_insn
))
221 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
222 a value changing in the insn, so the note would be invalid for CSE. */
223 if (reg_overlap_mentioned_p (target
, op0
)
224 || (op1
&& reg_overlap_mentioned_p (target
, op1
)))
227 && (rtx_equal_p (target
, op0
)
228 || (op1
&& rtx_equal_p (target
, op1
))))
230 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
231 over expanding it as temp = MEM op X, MEM = temp. If the target
232 supports MEM = MEM op X instructions, it is sometimes too hard
233 to reconstruct that form later, especially if X is also a memory,
234 and due to multiple occurrences of addresses the address might
235 be forced into register unnecessarily.
236 Note that not emitting the REG_EQUIV note might inhibit
237 CSE in some cases. */
238 set
= single_set (last_insn
);
240 && GET_CODE (SET_SRC (set
)) == code
241 && MEM_P (SET_DEST (set
))
242 && (rtx_equal_p (SET_DEST (set
), XEXP (SET_SRC (set
), 0))
243 || (op1
&& rtx_equal_p (SET_DEST (set
),
244 XEXP (SET_SRC (set
), 1)))))
250 set
= set_for_reg_notes (last_insn
);
254 if (! rtx_equal_p (SET_DEST (set
), target
)
255 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
256 && (GET_CODE (SET_DEST (set
)) != STRICT_LOW_PART
257 || ! rtx_equal_p (XEXP (SET_DEST (set
), 0), target
)))
260 if (GET_RTX_CLASS (code
) == RTX_UNARY
)
270 if (GET_MODE (op0
) != VOIDmode
&& GET_MODE (target
) != GET_MODE (op0
))
272 note
= gen_rtx_fmt_e (code
, GET_MODE (op0
), copy_rtx (op0
));
273 if (GET_MODE_SIZE (GET_MODE (op0
))
274 > GET_MODE_SIZE (GET_MODE (target
)))
275 note
= simplify_gen_unary (TRUNCATE
, GET_MODE (target
),
276 note
, GET_MODE (op0
));
278 note
= simplify_gen_unary (ZERO_EXTEND
, GET_MODE (target
),
279 note
, GET_MODE (op0
));
284 note
= gen_rtx_fmt_e (code
, GET_MODE (target
), copy_rtx (op0
));
288 note
= gen_rtx_fmt_ee (code
, GET_MODE (target
), copy_rtx (op0
), copy_rtx (op1
));
290 set_unique_reg_note (last_insn
, REG_EQUAL
, note
);
295 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
296 for a widening operation would be. In most cases this would be OP0, but if
297 that's a constant it'll be VOIDmode, which isn't useful. */
300 widened_mode (machine_mode to_mode
, rtx op0
, rtx op1
)
302 machine_mode m0
= GET_MODE (op0
);
303 machine_mode m1
= GET_MODE (op1
);
306 if (m0
== VOIDmode
&& m1
== VOIDmode
)
308 else if (m0
== VOIDmode
|| GET_MODE_SIZE (m0
) < GET_MODE_SIZE (m1
))
313 if (GET_MODE_SIZE (result
) > GET_MODE_SIZE (to_mode
))
319 /* Like optab_handler, but for widening_operations that have a
320 TO_MODE and a FROM_MODE. */
323 widening_optab_handler (optab op
, machine_mode to_mode
,
324 machine_mode from_mode
)
326 unsigned scode
= (op
<< 16) | to_mode
;
327 if (to_mode
!= from_mode
&& from_mode
!= VOIDmode
)
329 /* ??? Why does find_widening_optab_handler_and_mode attempt to
330 widen things that can't be widened? E.g. add_optab... */
331 if (op
> LAST_CONV_OPTAB
)
332 return CODE_FOR_nothing
;
333 scode
|= from_mode
<< 8;
335 return raw_optab_handler (scode
);
338 /* Find a widening optab even if it doesn't widen as much as we want.
339 E.g. if from_mode is HImode, and to_mode is DImode, and there is no
340 direct HI->SI insn, then return SI->DI, if that exists.
341 If PERMIT_NON_WIDENING is non-zero then this can be used with
342 non-widening optabs also. */
345 find_widening_optab_handler_and_mode (optab op
, machine_mode to_mode
,
346 machine_mode from_mode
,
347 int permit_non_widening
,
348 machine_mode
*found_mode
)
350 for (; (permit_non_widening
|| from_mode
!= to_mode
)
351 && GET_MODE_SIZE (from_mode
) <= GET_MODE_SIZE (to_mode
)
352 && from_mode
!= VOIDmode
;
353 from_mode
= GET_MODE_WIDER_MODE (from_mode
))
355 enum insn_code handler
= widening_optab_handler (op
, to_mode
,
358 if (handler
!= CODE_FOR_nothing
)
361 *found_mode
= from_mode
;
366 return CODE_FOR_nothing
;
369 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
370 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
371 not actually do a sign-extend or zero-extend, but can leave the
372 higher-order bits of the result rtx undefined, for example, in the case
373 of logical operations, but not right shifts. */
376 widen_operand (rtx op
, machine_mode mode
, machine_mode oldmode
,
377 int unsignedp
, int no_extend
)
381 /* If we don't have to extend and this is a constant, return it. */
382 if (no_extend
&& GET_MODE (op
) == VOIDmode
)
385 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
386 extend since it will be more efficient to do so unless the signedness of
387 a promoted object differs from our extension. */
389 || (GET_CODE (op
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (op
)
390 && SUBREG_CHECK_PROMOTED_SIGN (op
, unsignedp
)))
391 return convert_modes (mode
, oldmode
, op
, unsignedp
);
393 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
395 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
396 return gen_lowpart (mode
, force_reg (GET_MODE (op
), op
));
398 /* Otherwise, get an object of MODE, clobber it, and set the low-order
401 result
= gen_reg_rtx (mode
);
402 emit_clobber (result
);
403 emit_move_insn (gen_lowpart (GET_MODE (op
), result
), op
);
407 /* Return the optab used for computing the operation given by the tree code,
408 CODE and the tree EXP. This function is not always usable (for example, it
409 cannot give complete results for multiplication or division) but probably
410 ought to be relied on more widely throughout the expander. */
412 optab_for_tree_code (enum tree_code code
, const_tree type
,
413 enum optab_subtype subtype
)
425 return one_cmpl_optab
;
430 case MULT_HIGHPART_EXPR
:
431 return TYPE_UNSIGNED (type
) ? umul_highpart_optab
: smul_highpart_optab
;
437 return TYPE_UNSIGNED (type
) ? umod_optab
: smod_optab
;
445 if (TYPE_SATURATING (type
))
446 return TYPE_UNSIGNED (type
) ? usdiv_optab
: ssdiv_optab
;
447 return TYPE_UNSIGNED (type
) ? udiv_optab
: sdiv_optab
;
450 if (TREE_CODE (type
) == VECTOR_TYPE
)
452 if (subtype
== optab_vector
)
453 return TYPE_SATURATING (type
) ? unknown_optab
: vashl_optab
;
455 gcc_assert (subtype
== optab_scalar
);
457 if (TYPE_SATURATING (type
))
458 return TYPE_UNSIGNED (type
) ? usashl_optab
: ssashl_optab
;
462 if (TREE_CODE (type
) == VECTOR_TYPE
)
464 if (subtype
== optab_vector
)
465 return TYPE_UNSIGNED (type
) ? vlshr_optab
: vashr_optab
;
467 gcc_assert (subtype
== optab_scalar
);
469 return TYPE_UNSIGNED (type
) ? lshr_optab
: ashr_optab
;
472 if (TREE_CODE (type
) == VECTOR_TYPE
)
474 if (subtype
== optab_vector
)
477 gcc_assert (subtype
== optab_scalar
);
482 if (TREE_CODE (type
) == VECTOR_TYPE
)
484 if (subtype
== optab_vector
)
487 gcc_assert (subtype
== optab_scalar
);
492 return TYPE_UNSIGNED (type
) ? umax_optab
: smax_optab
;
495 return TYPE_UNSIGNED (type
) ? umin_optab
: smin_optab
;
497 case REALIGN_LOAD_EXPR
:
498 return vec_realign_load_optab
;
501 return TYPE_UNSIGNED (type
) ? usum_widen_optab
: ssum_widen_optab
;
504 return TYPE_UNSIGNED (type
) ? udot_prod_optab
: sdot_prod_optab
;
507 return TYPE_UNSIGNED (type
) ? usad_optab
: ssad_optab
;
509 case WIDEN_MULT_PLUS_EXPR
:
510 return (TYPE_UNSIGNED (type
)
511 ? (TYPE_SATURATING (type
)
512 ? usmadd_widen_optab
: umadd_widen_optab
)
513 : (TYPE_SATURATING (type
)
514 ? ssmadd_widen_optab
: smadd_widen_optab
));
516 case WIDEN_MULT_MINUS_EXPR
:
517 return (TYPE_UNSIGNED (type
)
518 ? (TYPE_SATURATING (type
)
519 ? usmsub_widen_optab
: umsub_widen_optab
)
520 : (TYPE_SATURATING (type
)
521 ? ssmsub_widen_optab
: smsub_widen_optab
));
527 return TYPE_UNSIGNED (type
)
528 ? reduc_umax_scal_optab
: reduc_smax_scal_optab
;
531 return TYPE_UNSIGNED (type
)
532 ? reduc_umin_scal_optab
: reduc_smin_scal_optab
;
534 case REDUC_PLUS_EXPR
:
535 return reduc_plus_scal_optab
;
537 case VEC_WIDEN_MULT_HI_EXPR
:
538 return TYPE_UNSIGNED (type
) ?
539 vec_widen_umult_hi_optab
: vec_widen_smult_hi_optab
;
541 case VEC_WIDEN_MULT_LO_EXPR
:
542 return TYPE_UNSIGNED (type
) ?
543 vec_widen_umult_lo_optab
: vec_widen_smult_lo_optab
;
545 case VEC_WIDEN_MULT_EVEN_EXPR
:
546 return TYPE_UNSIGNED (type
) ?
547 vec_widen_umult_even_optab
: vec_widen_smult_even_optab
;
549 case VEC_WIDEN_MULT_ODD_EXPR
:
550 return TYPE_UNSIGNED (type
) ?
551 vec_widen_umult_odd_optab
: vec_widen_smult_odd_optab
;
553 case VEC_WIDEN_LSHIFT_HI_EXPR
:
554 return TYPE_UNSIGNED (type
) ?
555 vec_widen_ushiftl_hi_optab
: vec_widen_sshiftl_hi_optab
;
557 case VEC_WIDEN_LSHIFT_LO_EXPR
:
558 return TYPE_UNSIGNED (type
) ?
559 vec_widen_ushiftl_lo_optab
: vec_widen_sshiftl_lo_optab
;
561 case VEC_UNPACK_HI_EXPR
:
562 return TYPE_UNSIGNED (type
) ?
563 vec_unpacku_hi_optab
: vec_unpacks_hi_optab
;
565 case VEC_UNPACK_LO_EXPR
:
566 return TYPE_UNSIGNED (type
) ?
567 vec_unpacku_lo_optab
: vec_unpacks_lo_optab
;
569 case VEC_UNPACK_FLOAT_HI_EXPR
:
570 /* The signedness is determined from input operand. */
571 return TYPE_UNSIGNED (type
) ?
572 vec_unpacku_float_hi_optab
: vec_unpacks_float_hi_optab
;
574 case VEC_UNPACK_FLOAT_LO_EXPR
:
575 /* The signedness is determined from input operand. */
576 return TYPE_UNSIGNED (type
) ?
577 vec_unpacku_float_lo_optab
: vec_unpacks_float_lo_optab
;
579 case VEC_PACK_TRUNC_EXPR
:
580 return vec_pack_trunc_optab
;
582 case VEC_PACK_SAT_EXPR
:
583 return TYPE_UNSIGNED (type
) ? vec_pack_usat_optab
: vec_pack_ssat_optab
;
585 case VEC_PACK_FIX_TRUNC_EXPR
:
586 /* The signedness is determined from output operand. */
587 return TYPE_UNSIGNED (type
) ?
588 vec_pack_ufix_trunc_optab
: vec_pack_sfix_trunc_optab
;
594 trapv
= INTEGRAL_TYPE_P (type
) && TYPE_OVERFLOW_TRAPS (type
);
597 case POINTER_PLUS_EXPR
:
599 if (TYPE_SATURATING (type
))
600 return TYPE_UNSIGNED (type
) ? usadd_optab
: ssadd_optab
;
601 return trapv
? addv_optab
: add_optab
;
604 if (TYPE_SATURATING (type
))
605 return TYPE_UNSIGNED (type
) ? ussub_optab
: sssub_optab
;
606 return trapv
? subv_optab
: sub_optab
;
609 if (TYPE_SATURATING (type
))
610 return TYPE_UNSIGNED (type
) ? usmul_optab
: ssmul_optab
;
611 return trapv
? smulv_optab
: smul_optab
;
614 if (TYPE_SATURATING (type
))
615 return TYPE_UNSIGNED (type
) ? usneg_optab
: ssneg_optab
;
616 return trapv
? negv_optab
: neg_optab
;
619 return trapv
? absv_optab
: abs_optab
;
622 return unknown_optab
;
626 /* Given optab UNOPTAB that reduces a vector to a scalar, find instead the old
627 optab that produces a vector with the reduction result in one element,
628 for a tree with type TYPE. */
631 scalar_reduc_to_vector (optab unoptab
, const_tree type
)
635 case reduc_plus_scal_optab
:
636 return TYPE_UNSIGNED (type
) ? reduc_uplus_optab
: reduc_splus_optab
;
638 case reduc_smin_scal_optab
: return reduc_smin_optab
;
639 case reduc_umin_scal_optab
: return reduc_umin_optab
;
640 case reduc_smax_scal_optab
: return reduc_smax_optab
;
641 case reduc_umax_scal_optab
: return reduc_umax_optab
;
642 default: return unknown_optab
;
646 /* Expand vector widening operations.
648 There are two different classes of operations handled here:
649 1) Operations whose result is wider than all the arguments to the operation.
650 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
651 In this case OP0 and optionally OP1 would be initialized,
652 but WIDE_OP wouldn't (not relevant for this case).
653 2) Operations whose result is of the same size as the last argument to the
654 operation, but wider than all the other arguments to the operation.
655 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
656 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
658 E.g, when called to expand the following operations, this is how
659 the arguments will be initialized:
661 widening-sum 2 oprnd0 - oprnd1
662 widening-dot-product 3 oprnd0 oprnd1 oprnd2
663 widening-mult 2 oprnd0 oprnd1 -
664 type-promotion (vec-unpack) 1 oprnd0 - - */
667 expand_widen_pattern_expr (sepops ops
, rtx op0
, rtx op1
, rtx wide_op
,
668 rtx target
, int unsignedp
)
670 struct expand_operand eops
[4];
671 tree oprnd0
, oprnd1
, oprnd2
;
672 machine_mode wmode
= VOIDmode
, tmode0
, tmode1
= VOIDmode
;
673 optab widen_pattern_optab
;
674 enum insn_code icode
;
675 int nops
= TREE_CODE_LENGTH (ops
->code
);
679 tmode0
= TYPE_MODE (TREE_TYPE (oprnd0
));
680 widen_pattern_optab
=
681 optab_for_tree_code (ops
->code
, TREE_TYPE (oprnd0
), optab_default
);
682 if (ops
->code
== WIDEN_MULT_PLUS_EXPR
683 || ops
->code
== WIDEN_MULT_MINUS_EXPR
)
684 icode
= find_widening_optab_handler (widen_pattern_optab
,
685 TYPE_MODE (TREE_TYPE (ops
->op2
)),
688 icode
= optab_handler (widen_pattern_optab
, tmode0
);
689 gcc_assert (icode
!= CODE_FOR_nothing
);
694 tmode1
= TYPE_MODE (TREE_TYPE (oprnd1
));
697 /* The last operand is of a wider mode than the rest of the operands. */
702 gcc_assert (tmode1
== tmode0
);
705 wmode
= TYPE_MODE (TREE_TYPE (oprnd2
));
709 create_output_operand (&eops
[op
++], target
, TYPE_MODE (ops
->type
));
710 create_convert_operand_from (&eops
[op
++], op0
, tmode0
, unsignedp
);
712 create_convert_operand_from (&eops
[op
++], op1
, tmode1
, unsignedp
);
714 create_convert_operand_from (&eops
[op
++], wide_op
, wmode
, unsignedp
);
715 expand_insn (icode
, op
, eops
);
716 return eops
[0].value
;
719 /* Generate code to perform an operation specified by TERNARY_OPTAB
720 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
722 UNSIGNEDP is for the case where we have to widen the operands
723 to perform the operation. It says to use zero-extension.
725 If TARGET is nonzero, the value
726 is generated there, if it is convenient to do so.
727 In all cases an rtx is returned for the locus of the value;
728 this may or may not be TARGET. */
731 expand_ternary_op (machine_mode mode
, optab ternary_optab
, rtx op0
,
732 rtx op1
, rtx op2
, rtx target
, int unsignedp
)
734 struct expand_operand ops
[4];
735 enum insn_code icode
= optab_handler (ternary_optab
, mode
);
737 gcc_assert (optab_handler (ternary_optab
, mode
) != CODE_FOR_nothing
);
739 create_output_operand (&ops
[0], target
, mode
);
740 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
741 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
742 create_convert_operand_from (&ops
[3], op2
, mode
, unsignedp
);
743 expand_insn (icode
, 4, ops
);
748 /* Like expand_binop, but return a constant rtx if the result can be
749 calculated at compile time. The arguments and return value are
750 otherwise the same as for expand_binop. */
753 simplify_expand_binop (machine_mode mode
, optab binoptab
,
754 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
755 enum optab_methods methods
)
757 if (CONSTANT_P (op0
) && CONSTANT_P (op1
))
759 rtx x
= simplify_binary_operation (optab_to_code (binoptab
),
765 return expand_binop (mode
, binoptab
, op0
, op1
, target
, unsignedp
, methods
);
768 /* Like simplify_expand_binop, but always put the result in TARGET.
769 Return true if the expansion succeeded. */
772 force_expand_binop (machine_mode mode
, optab binoptab
,
773 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
774 enum optab_methods methods
)
776 rtx x
= simplify_expand_binop (mode
, binoptab
, op0
, op1
,
777 target
, unsignedp
, methods
);
781 emit_move_insn (target
, x
);
785 /* Create a new vector value in VMODE with all elements set to OP. The
786 mode of OP must be the element mode of VMODE. If OP is a constant,
787 then the return value will be a constant. */
790 expand_vector_broadcast (machine_mode vmode
, rtx op
)
792 enum insn_code icode
;
797 gcc_checking_assert (VECTOR_MODE_P (vmode
));
799 n
= GET_MODE_NUNITS (vmode
);
800 vec
= rtvec_alloc (n
);
801 for (i
= 0; i
< n
; ++i
)
802 RTVEC_ELT (vec
, i
) = op
;
805 return gen_rtx_CONST_VECTOR (vmode
, vec
);
807 /* ??? If the target doesn't have a vec_init, then we have no easy way
808 of performing this operation. Most of this sort of generic support
809 is hidden away in the vector lowering support in gimple. */
810 icode
= optab_handler (vec_init_optab
, vmode
);
811 if (icode
== CODE_FOR_nothing
)
814 ret
= gen_reg_rtx (vmode
);
815 emit_insn (GEN_FCN (icode
) (ret
, gen_rtx_PARALLEL (vmode
, vec
)));
820 /* This subroutine of expand_doubleword_shift handles the cases in which
821 the effective shift value is >= BITS_PER_WORD. The arguments and return
822 value are the same as for the parent routine, except that SUPERWORD_OP1
823 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
824 INTO_TARGET may be null if the caller has decided to calculate it. */
827 expand_superword_shift (optab binoptab
, rtx outof_input
, rtx superword_op1
,
828 rtx outof_target
, rtx into_target
,
829 int unsignedp
, enum optab_methods methods
)
831 if (into_target
!= 0)
832 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, superword_op1
,
833 into_target
, unsignedp
, methods
))
836 if (outof_target
!= 0)
838 /* For a signed right shift, we must fill OUTOF_TARGET with copies
839 of the sign bit, otherwise we must fill it with zeros. */
840 if (binoptab
!= ashr_optab
)
841 emit_move_insn (outof_target
, CONST0_RTX (word_mode
));
843 if (!force_expand_binop (word_mode
, binoptab
,
844 outof_input
, GEN_INT (BITS_PER_WORD
- 1),
845 outof_target
, unsignedp
, methods
))
851 /* This subroutine of expand_doubleword_shift handles the cases in which
852 the effective shift value is < BITS_PER_WORD. The arguments and return
853 value are the same as for the parent routine. */
856 expand_subword_shift (machine_mode op1_mode
, optab binoptab
,
857 rtx outof_input
, rtx into_input
, rtx op1
,
858 rtx outof_target
, rtx into_target
,
859 int unsignedp
, enum optab_methods methods
,
860 unsigned HOST_WIDE_INT shift_mask
)
862 optab reverse_unsigned_shift
, unsigned_shift
;
865 reverse_unsigned_shift
= (binoptab
== ashl_optab
? lshr_optab
: ashl_optab
);
866 unsigned_shift
= (binoptab
== ashl_optab
? ashl_optab
: lshr_optab
);
868 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
869 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
870 the opposite direction to BINOPTAB. */
871 if (CONSTANT_P (op1
) || shift_mask
>= BITS_PER_WORD
)
873 carries
= outof_input
;
874 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
,
875 op1_mode
), op1_mode
);
876 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
881 /* We must avoid shifting by BITS_PER_WORD bits since that is either
882 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
883 has unknown behavior. Do a single shift first, then shift by the
884 remainder. It's OK to use ~OP1 as the remainder if shift counts
885 are truncated to the mode size. */
886 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
887 outof_input
, const1_rtx
, 0, unsignedp
, methods
);
888 if (shift_mask
== BITS_PER_WORD
- 1)
890 tmp
= immed_wide_int_const
891 (wi::minus_one (GET_MODE_PRECISION (op1_mode
)), op1_mode
);
892 tmp
= simplify_expand_binop (op1_mode
, xor_optab
, op1
, tmp
,
897 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
- 1,
898 op1_mode
), op1_mode
);
899 tmp
= simplify_expand_binop (op1_mode
, sub_optab
, tmp
, op1
,
903 if (tmp
== 0 || carries
== 0)
905 carries
= expand_binop (word_mode
, reverse_unsigned_shift
,
906 carries
, tmp
, 0, unsignedp
, methods
);
910 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
911 so the result can go directly into INTO_TARGET if convenient. */
912 tmp
= expand_binop (word_mode
, unsigned_shift
, into_input
, op1
,
913 into_target
, unsignedp
, methods
);
917 /* Now OR in the bits carried over from OUTOF_INPUT. */
918 if (!force_expand_binop (word_mode
, ior_optab
, tmp
, carries
,
919 into_target
, unsignedp
, methods
))
922 /* Use a standard word_mode shift for the out-of half. */
923 if (outof_target
!= 0)
924 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
925 outof_target
, unsignedp
, methods
))
932 #ifdef HAVE_conditional_move
933 /* Try implementing expand_doubleword_shift using conditional moves.
934 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
935 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
936 are the shift counts to use in the former and latter case. All other
937 arguments are the same as the parent routine. */
940 expand_doubleword_shift_condmove (machine_mode op1_mode
, optab binoptab
,
941 enum rtx_code cmp_code
, rtx cmp1
, rtx cmp2
,
942 rtx outof_input
, rtx into_input
,
943 rtx subword_op1
, rtx superword_op1
,
944 rtx outof_target
, rtx into_target
,
945 int unsignedp
, enum optab_methods methods
,
946 unsigned HOST_WIDE_INT shift_mask
)
948 rtx outof_superword
, into_superword
;
950 /* Put the superword version of the output into OUTOF_SUPERWORD and
952 outof_superword
= outof_target
!= 0 ? gen_reg_rtx (word_mode
) : 0;
953 if (outof_target
!= 0 && subword_op1
== superword_op1
)
955 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
956 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
957 into_superword
= outof_target
;
958 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
959 outof_superword
, 0, unsignedp
, methods
))
964 into_superword
= gen_reg_rtx (word_mode
);
965 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
966 outof_superword
, into_superword
,
971 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
972 if (!expand_subword_shift (op1_mode
, binoptab
,
973 outof_input
, into_input
, subword_op1
,
974 outof_target
, into_target
,
975 unsignedp
, methods
, shift_mask
))
978 /* Select between them. Do the INTO half first because INTO_SUPERWORD
979 might be the current value of OUTOF_TARGET. */
980 if (!emit_conditional_move (into_target
, cmp_code
, cmp1
, cmp2
, op1_mode
,
981 into_target
, into_superword
, word_mode
, false))
984 if (outof_target
!= 0)
985 if (!emit_conditional_move (outof_target
, cmp_code
, cmp1
, cmp2
, op1_mode
,
986 outof_target
, outof_superword
,
994 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
995 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
996 input operand; the shift moves bits in the direction OUTOF_INPUT->
997 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
998 of the target. OP1 is the shift count and OP1_MODE is its mode.
999 If OP1 is constant, it will have been truncated as appropriate
1000 and is known to be nonzero.
1002 If SHIFT_MASK is zero, the result of word shifts is undefined when the
1003 shift count is outside the range [0, BITS_PER_WORD). This routine must
1004 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
1006 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
1007 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
1008 fill with zeros or sign bits as appropriate.
1010 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
1011 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
1012 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
1013 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
1016 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
1017 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
1018 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
1019 function wants to calculate it itself.
1021 Return true if the shift could be successfully synthesized. */
1024 expand_doubleword_shift (machine_mode op1_mode
, optab binoptab
,
1025 rtx outof_input
, rtx into_input
, rtx op1
,
1026 rtx outof_target
, rtx into_target
,
1027 int unsignedp
, enum optab_methods methods
,
1028 unsigned HOST_WIDE_INT shift_mask
)
1030 rtx superword_op1
, tmp
, cmp1
, cmp2
;
1031 enum rtx_code cmp_code
;
1033 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
1034 fill the result with sign or zero bits as appropriate. If so, the value
1035 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
1036 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
1037 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
1039 This isn't worthwhile for constant shifts since the optimizers will
1040 cope better with in-range shift counts. */
1041 if (shift_mask
>= BITS_PER_WORD
1042 && outof_target
!= 0
1043 && !CONSTANT_P (op1
))
1045 if (!expand_doubleword_shift (op1_mode
, binoptab
,
1046 outof_input
, into_input
, op1
,
1048 unsignedp
, methods
, shift_mask
))
1050 if (!force_expand_binop (word_mode
, binoptab
, outof_input
, op1
,
1051 outof_target
, unsignedp
, methods
))
1056 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
1057 is true when the effective shift value is less than BITS_PER_WORD.
1058 Set SUPERWORD_OP1 to the shift count that should be used to shift
1059 OUTOF_INPUT into INTO_TARGET when the condition is false. */
1060 tmp
= immed_wide_int_const (wi::shwi (BITS_PER_WORD
, op1_mode
), op1_mode
);
1061 if (!CONSTANT_P (op1
) && shift_mask
== BITS_PER_WORD
- 1)
1063 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
1064 is a subword shift count. */
1065 cmp1
= simplify_expand_binop (op1_mode
, and_optab
, op1
, tmp
,
1067 cmp2
= CONST0_RTX (op1_mode
);
1069 superword_op1
= op1
;
1073 /* Set CMP1 to OP1 - BITS_PER_WORD. */
1074 cmp1
= simplify_expand_binop (op1_mode
, sub_optab
, op1
, tmp
,
1076 cmp2
= CONST0_RTX (op1_mode
);
1078 superword_op1
= cmp1
;
1083 /* If we can compute the condition at compile time, pick the
1084 appropriate subroutine. */
1085 tmp
= simplify_relational_operation (cmp_code
, SImode
, op1_mode
, cmp1
, cmp2
);
1086 if (tmp
!= 0 && CONST_INT_P (tmp
))
1088 if (tmp
== const0_rtx
)
1089 return expand_superword_shift (binoptab
, outof_input
, superword_op1
,
1090 outof_target
, into_target
,
1091 unsignedp
, methods
);
1093 return expand_subword_shift (op1_mode
, binoptab
,
1094 outof_input
, into_input
, op1
,
1095 outof_target
, into_target
,
1096 unsignedp
, methods
, shift_mask
);
1099 #ifdef HAVE_conditional_move
1100 /* Try using conditional moves to generate straight-line code. */
1102 rtx_insn
*start
= get_last_insn ();
1103 if (expand_doubleword_shift_condmove (op1_mode
, binoptab
,
1104 cmp_code
, cmp1
, cmp2
,
1105 outof_input
, into_input
,
1107 outof_target
, into_target
,
1108 unsignedp
, methods
, shift_mask
))
1110 delete_insns_since (start
);
1114 /* As a last resort, use branches to select the correct alternative. */
1115 rtx_code_label
*subword_label
= gen_label_rtx ();
1116 rtx_code_label
*done_label
= gen_label_rtx ();
1119 do_compare_rtx_and_jump (cmp1
, cmp2
, cmp_code
, false, op1_mode
,
1120 0, 0, subword_label
, -1);
1123 if (!expand_superword_shift (binoptab
, outof_input
, superword_op1
,
1124 outof_target
, into_target
,
1125 unsignedp
, methods
))
1128 emit_jump_insn (gen_jump (done_label
));
1130 emit_label (subword_label
);
1132 if (!expand_subword_shift (op1_mode
, binoptab
,
1133 outof_input
, into_input
, op1
,
1134 outof_target
, into_target
,
1135 unsignedp
, methods
, shift_mask
))
1138 emit_label (done_label
);
1142 /* Subroutine of expand_binop. Perform a double word multiplication of
1143 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
1144 as the target's word_mode. This function return NULL_RTX if anything
1145 goes wrong, in which case it may have already emitted instructions
1146 which need to be deleted.
1148 If we want to multiply two two-word values and have normal and widening
1149 multiplies of single-word values, we can do this with three smaller
1152 The multiplication proceeds as follows:
1153 _______________________
1154 [__op0_high_|__op0_low__]
1155 _______________________
1156 * [__op1_high_|__op1_low__]
1157 _______________________________________________
1158 _______________________
1159 (1) [__op0_low__*__op1_low__]
1160 _______________________
1161 (2a) [__op0_low__*__op1_high_]
1162 _______________________
1163 (2b) [__op0_high_*__op1_low__]
1164 _______________________
1165 (3) [__op0_high_*__op1_high_]
1168 This gives a 4-word result. Since we are only interested in the
1169 lower 2 words, partial result (3) and the upper words of (2a) and
1170 (2b) don't need to be calculated. Hence (2a) and (2b) can be
1171 calculated using non-widening multiplication.
1173 (1), however, needs to be calculated with an unsigned widening
1174 multiplication. If this operation is not directly supported we
1175 try using a signed widening multiplication and adjust the result.
1176 This adjustment works as follows:
1178 If both operands are positive then no adjustment is needed.
1180 If the operands have different signs, for example op0_low < 0 and
1181 op1_low >= 0, the instruction treats the most significant bit of
1182 op0_low as a sign bit instead of a bit with significance
1183 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
1184 with 2**BITS_PER_WORD - op0_low, and two's complements the
1185 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
1188 Similarly, if both operands are negative, we need to add
1189 (op0_low + op1_low) * 2**BITS_PER_WORD.
1191 We use a trick to adjust quickly. We logically shift op0_low right
1192 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
1193 op0_high (op1_high) before it is used to calculate 2b (2a). If no
1194 logical shift exists, we do an arithmetic right shift and subtract
1198 expand_doubleword_mult (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
1199 bool umulp
, enum optab_methods methods
)
1201 int low
= (WORDS_BIG_ENDIAN
? 1 : 0);
1202 int high
= (WORDS_BIG_ENDIAN
? 0 : 1);
1203 rtx wordm1
= umulp
? NULL_RTX
: GEN_INT (BITS_PER_WORD
- 1);
1204 rtx product
, adjust
, product_high
, temp
;
1206 rtx op0_high
= operand_subword_force (op0
, high
, mode
);
1207 rtx op0_low
= operand_subword_force (op0
, low
, mode
);
1208 rtx op1_high
= operand_subword_force (op1
, high
, mode
);
1209 rtx op1_low
= operand_subword_force (op1
, low
, mode
);
1211 /* If we're using an unsigned multiply to directly compute the product
1212 of the low-order words of the operands and perform any required
1213 adjustments of the operands, we begin by trying two more multiplications
1214 and then computing the appropriate sum.
1216 We have checked above that the required addition is provided.
1217 Full-word addition will normally always succeed, especially if
1218 it is provided at all, so we don't worry about its failure. The
1219 multiplication may well fail, however, so we do handle that. */
1223 /* ??? This could be done with emit_store_flag where available. */
1224 temp
= expand_binop (word_mode
, lshr_optab
, op0_low
, wordm1
,
1225 NULL_RTX
, 1, methods
);
1227 op0_high
= expand_binop (word_mode
, add_optab
, op0_high
, temp
,
1228 NULL_RTX
, 0, OPTAB_DIRECT
);
1231 temp
= expand_binop (word_mode
, ashr_optab
, op0_low
, wordm1
,
1232 NULL_RTX
, 0, methods
);
1235 op0_high
= expand_binop (word_mode
, sub_optab
, op0_high
, temp
,
1236 NULL_RTX
, 0, OPTAB_DIRECT
);
1243 adjust
= expand_binop (word_mode
, smul_optab
, op0_high
, op1_low
,
1244 NULL_RTX
, 0, OPTAB_DIRECT
);
1248 /* OP0_HIGH should now be dead. */
1252 /* ??? This could be done with emit_store_flag where available. */
1253 temp
= expand_binop (word_mode
, lshr_optab
, op1_low
, wordm1
,
1254 NULL_RTX
, 1, methods
);
1256 op1_high
= expand_binop (word_mode
, add_optab
, op1_high
, temp
,
1257 NULL_RTX
, 0, OPTAB_DIRECT
);
1260 temp
= expand_binop (word_mode
, ashr_optab
, op1_low
, wordm1
,
1261 NULL_RTX
, 0, methods
);
1264 op1_high
= expand_binop (word_mode
, sub_optab
, op1_high
, temp
,
1265 NULL_RTX
, 0, OPTAB_DIRECT
);
1272 temp
= expand_binop (word_mode
, smul_optab
, op1_high
, op0_low
,
1273 NULL_RTX
, 0, OPTAB_DIRECT
);
1277 /* OP1_HIGH should now be dead. */
1279 adjust
= expand_binop (word_mode
, add_optab
, adjust
, temp
,
1280 NULL_RTX
, 0, OPTAB_DIRECT
);
1282 if (target
&& !REG_P (target
))
1286 product
= expand_binop (mode
, umul_widen_optab
, op0_low
, op1_low
,
1287 target
, 1, OPTAB_DIRECT
);
1289 product
= expand_binop (mode
, smul_widen_optab
, op0_low
, op1_low
,
1290 target
, 1, OPTAB_DIRECT
);
1295 product_high
= operand_subword (product
, high
, 1, mode
);
1296 adjust
= expand_binop (word_mode
, add_optab
, product_high
, adjust
,
1297 NULL_RTX
, 0, OPTAB_DIRECT
);
1298 emit_move_insn (product_high
, adjust
);
1302 /* Wrapper around expand_binop which takes an rtx code to specify
1303 the operation to perform, not an optab pointer. All other
1304 arguments are the same. */
1306 expand_simple_binop (machine_mode mode
, enum rtx_code code
, rtx op0
,
1307 rtx op1
, rtx target
, int unsignedp
,
1308 enum optab_methods methods
)
1310 optab binop
= code_to_optab (code
);
1313 return expand_binop (mode
, binop
, op0
, op1
, target
, unsignedp
, methods
);
1316 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
1317 binop. Order them according to commutative_operand_precedence and, if
1318 possible, try to put TARGET or a pseudo first. */
1320 swap_commutative_operands_with_target (rtx target
, rtx op0
, rtx op1
)
1322 int op0_prec
= commutative_operand_precedence (op0
);
1323 int op1_prec
= commutative_operand_precedence (op1
);
1325 if (op0_prec
< op1_prec
)
1328 if (op0_prec
> op1_prec
)
1331 /* With equal precedence, both orders are ok, but it is better if the
1332 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1333 if (target
== 0 || REG_P (target
))
1334 return (REG_P (op1
) && !REG_P (op0
)) || target
== op1
;
1336 return rtx_equal_p (op1
, target
);
1339 /* Return true if BINOPTAB implements a shift operation. */
1342 shift_optab_p (optab binoptab
)
1344 switch (optab_to_code (binoptab
))
1360 /* Return true if BINOPTAB implements a commutative binary operation. */
1363 commutative_optab_p (optab binoptab
)
1365 return (GET_RTX_CLASS (optab_to_code (binoptab
)) == RTX_COMM_ARITH
1366 || binoptab
== smul_widen_optab
1367 || binoptab
== umul_widen_optab
1368 || binoptab
== smul_highpart_optab
1369 || binoptab
== umul_highpart_optab
);
1372 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1373 optimizing, and if the operand is a constant that costs more than
1374 1 instruction, force the constant into a register and return that
1375 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1378 avoid_expensive_constant (machine_mode mode
, optab binoptab
,
1379 int opn
, rtx x
, bool unsignedp
)
1381 bool speed
= optimize_insn_for_speed_p ();
1383 if (mode
!= VOIDmode
1386 && (rtx_cost (x
, optab_to_code (binoptab
), opn
, speed
)
1387 > set_src_cost (x
, speed
)))
1389 if (CONST_INT_P (x
))
1391 HOST_WIDE_INT intval
= trunc_int_for_mode (INTVAL (x
), mode
);
1392 if (intval
!= INTVAL (x
))
1393 x
= GEN_INT (intval
);
1396 x
= convert_modes (mode
, VOIDmode
, x
, unsignedp
);
1397 x
= force_reg (mode
, x
);
1402 /* Helper function for expand_binop: handle the case where there
1403 is an insn that directly implements the indicated operation.
1404 Returns null if this is not possible. */
1406 expand_binop_directly (machine_mode mode
, optab binoptab
,
1408 rtx target
, int unsignedp
, enum optab_methods methods
,
1411 machine_mode from_mode
= widened_mode (mode
, op0
, op1
);
1412 enum insn_code icode
= find_widening_optab_handler (binoptab
, mode
,
1414 machine_mode xmode0
= insn_data
[(int) icode
].operand
[1].mode
;
1415 machine_mode xmode1
= insn_data
[(int) icode
].operand
[2].mode
;
1416 machine_mode mode0
, mode1
, tmp_mode
;
1417 struct expand_operand ops
[3];
1420 rtx xop0
= op0
, xop1
= op1
;
1422 /* If it is a commutative operator and the modes would match
1423 if we would swap the operands, we can save the conversions. */
1424 commutative_p
= commutative_optab_p (binoptab
);
1426 && GET_MODE (xop0
) != xmode0
&& GET_MODE (xop1
) != xmode1
1427 && GET_MODE (xop0
) == xmode1
&& GET_MODE (xop1
) == xmode1
)
1428 std::swap (xop0
, xop1
);
1430 /* If we are optimizing, force expensive constants into a register. */
1431 xop0
= avoid_expensive_constant (xmode0
, binoptab
, 0, xop0
, unsignedp
);
1432 if (!shift_optab_p (binoptab
))
1433 xop1
= avoid_expensive_constant (xmode1
, binoptab
, 1, xop1
, unsignedp
);
1435 /* In case the insn wants input operands in modes different from
1436 those of the actual operands, convert the operands. It would
1437 seem that we don't need to convert CONST_INTs, but we do, so
1438 that they're properly zero-extended, sign-extended or truncated
1441 mode0
= GET_MODE (xop0
) != VOIDmode
? GET_MODE (xop0
) : mode
;
1442 if (xmode0
!= VOIDmode
&& xmode0
!= mode0
)
1444 xop0
= convert_modes (xmode0
, mode0
, xop0
, unsignedp
);
1448 mode1
= GET_MODE (xop1
) != VOIDmode
? GET_MODE (xop1
) : mode
;
1449 if (xmode1
!= VOIDmode
&& xmode1
!= mode1
)
1451 xop1
= convert_modes (xmode1
, mode1
, xop1
, unsignedp
);
1455 /* If operation is commutative,
1456 try to make the first operand a register.
1457 Even better, try to make it the same as the target.
1458 Also try to make the last operand a constant. */
1460 && swap_commutative_operands_with_target (target
, xop0
, xop1
))
1461 std::swap (xop0
, xop1
);
1463 /* Now, if insn's predicates don't allow our operands, put them into
1466 if (binoptab
== vec_pack_trunc_optab
1467 || binoptab
== vec_pack_usat_optab
1468 || binoptab
== vec_pack_ssat_optab
1469 || binoptab
== vec_pack_ufix_trunc_optab
1470 || binoptab
== vec_pack_sfix_trunc_optab
)
1472 /* The mode of the result is different then the mode of the
1474 tmp_mode
= insn_data
[(int) icode
].operand
[0].mode
;
1475 if (GET_MODE_NUNITS (tmp_mode
) != 2 * GET_MODE_NUNITS (mode
))
1477 delete_insns_since (last
);
1484 create_output_operand (&ops
[0], target
, tmp_mode
);
1485 create_input_operand (&ops
[1], xop0
, mode0
);
1486 create_input_operand (&ops
[2], xop1
, mode1
);
1487 pat
= maybe_gen_insn (icode
, 3, ops
);
1490 /* If PAT is composed of more than one insn, try to add an appropriate
1491 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1492 operand, call expand_binop again, this time without a target. */
1493 if (INSN_P (pat
) && NEXT_INSN (as_a
<rtx_insn
*> (pat
)) != NULL_RTX
1494 && ! add_equal_note (as_a
<rtx_insn
*> (pat
), ops
[0].value
,
1495 optab_to_code (binoptab
),
1496 ops
[1].value
, ops
[2].value
))
1498 delete_insns_since (last
);
1499 return expand_binop (mode
, binoptab
, op0
, op1
, NULL_RTX
,
1500 unsignedp
, methods
);
1504 return ops
[0].value
;
1506 delete_insns_since (last
);
1510 /* Generate code to perform an operation specified by BINOPTAB
1511 on operands OP0 and OP1, with result having machine-mode MODE.
1513 UNSIGNEDP is for the case where we have to widen the operands
1514 to perform the operation. It says to use zero-extension.
1516 If TARGET is nonzero, the value
1517 is generated there, if it is convenient to do so.
1518 In all cases an rtx is returned for the locus of the value;
1519 this may or may not be TARGET. */
1522 expand_binop (machine_mode mode
, optab binoptab
, rtx op0
, rtx op1
,
1523 rtx target
, int unsignedp
, enum optab_methods methods
)
1525 enum optab_methods next_methods
1526 = (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
1527 ? OPTAB_WIDEN
: methods
);
1528 enum mode_class mclass
;
1529 machine_mode wider_mode
;
1532 rtx_insn
*entry_last
= get_last_insn ();
1535 mclass
= GET_MODE_CLASS (mode
);
1537 /* If subtracting an integer constant, convert this into an addition of
1538 the negated constant. */
1540 if (binoptab
== sub_optab
&& CONST_INT_P (op1
))
1542 op1
= negate_rtx (mode
, op1
);
1543 binoptab
= add_optab
;
1546 /* Record where to delete back to if we backtrack. */
1547 last
= get_last_insn ();
1549 /* If we can do it with a three-operand insn, do so. */
1551 if (methods
!= OPTAB_MUST_WIDEN
1552 && find_widening_optab_handler (binoptab
, mode
,
1553 widened_mode (mode
, op0
, op1
), 1)
1554 != CODE_FOR_nothing
)
1556 temp
= expand_binop_directly (mode
, binoptab
, op0
, op1
, target
,
1557 unsignedp
, methods
, last
);
1562 /* If we were trying to rotate, and that didn't work, try rotating
1563 the other direction before falling back to shifts and bitwise-or. */
1564 if (((binoptab
== rotl_optab
1565 && optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
1566 || (binoptab
== rotr_optab
1567 && optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
))
1568 && mclass
== MODE_INT
)
1570 optab otheroptab
= (binoptab
== rotl_optab
? rotr_optab
: rotl_optab
);
1572 unsigned int bits
= GET_MODE_PRECISION (mode
);
1574 if (CONST_INT_P (op1
))
1575 newop1
= GEN_INT (bits
- INTVAL (op1
));
1576 else if (targetm
.shift_truncation_mask (mode
) == bits
- 1)
1577 newop1
= negate_rtx (GET_MODE (op1
), op1
);
1579 newop1
= expand_binop (GET_MODE (op1
), sub_optab
,
1580 gen_int_mode (bits
, GET_MODE (op1
)), op1
,
1581 NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1583 temp
= expand_binop_directly (mode
, otheroptab
, op0
, newop1
,
1584 target
, unsignedp
, methods
, last
);
1589 /* If this is a multiply, see if we can do a widening operation that
1590 takes operands of this mode and makes a wider mode. */
1592 if (binoptab
== smul_optab
1593 && GET_MODE_2XWIDER_MODE (mode
) != VOIDmode
1594 && (widening_optab_handler ((unsignedp
? umul_widen_optab
1595 : smul_widen_optab
),
1596 GET_MODE_2XWIDER_MODE (mode
), mode
)
1597 != CODE_FOR_nothing
))
1599 temp
= expand_binop (GET_MODE_2XWIDER_MODE (mode
),
1600 unsignedp
? umul_widen_optab
: smul_widen_optab
,
1601 op0
, op1
, NULL_RTX
, unsignedp
, OPTAB_DIRECT
);
1605 if (GET_MODE_CLASS (mode
) == MODE_INT
1606 && TRULY_NOOP_TRUNCATION_MODES_P (mode
, GET_MODE (temp
)))
1607 return gen_lowpart (mode
, temp
);
1609 return convert_to_mode (mode
, temp
, unsignedp
);
1613 /* If this is a vector shift by a scalar, see if we can do a vector
1614 shift by a vector. If so, broadcast the scalar into a vector. */
1615 if (mclass
== MODE_VECTOR_INT
)
1617 optab otheroptab
= unknown_optab
;
1619 if (binoptab
== ashl_optab
)
1620 otheroptab
= vashl_optab
;
1621 else if (binoptab
== ashr_optab
)
1622 otheroptab
= vashr_optab
;
1623 else if (binoptab
== lshr_optab
)
1624 otheroptab
= vlshr_optab
;
1625 else if (binoptab
== rotl_optab
)
1626 otheroptab
= vrotl_optab
;
1627 else if (binoptab
== rotr_optab
)
1628 otheroptab
= vrotr_optab
;
1630 if (otheroptab
&& optab_handler (otheroptab
, mode
) != CODE_FOR_nothing
)
1632 rtx vop1
= expand_vector_broadcast (mode
, op1
);
1635 temp
= expand_binop_directly (mode
, otheroptab
, op0
, vop1
,
1636 target
, unsignedp
, methods
, last
);
1643 /* Look for a wider mode of the same class for which we think we
1644 can open-code the operation. Check for a widening multiply at the
1645 wider mode as well. */
1647 if (CLASS_HAS_WIDER_MODES_P (mclass
)
1648 && methods
!= OPTAB_DIRECT
&& methods
!= OPTAB_LIB
)
1649 for (wider_mode
= GET_MODE_WIDER_MODE (mode
);
1650 wider_mode
!= VOIDmode
;
1651 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
1653 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
1654 || (binoptab
== smul_optab
1655 && GET_MODE_WIDER_MODE (wider_mode
) != VOIDmode
1656 && (find_widening_optab_handler ((unsignedp
1658 : smul_widen_optab
),
1659 GET_MODE_WIDER_MODE (wider_mode
),
1661 != CODE_FOR_nothing
)))
1663 rtx xop0
= op0
, xop1
= op1
;
1666 /* For certain integer operations, we need not actually extend
1667 the narrow operands, as long as we will truncate
1668 the results to the same narrowness. */
1670 if ((binoptab
== ior_optab
|| binoptab
== and_optab
1671 || binoptab
== xor_optab
1672 || binoptab
== add_optab
|| binoptab
== sub_optab
1673 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
1674 && mclass
== MODE_INT
)
1677 xop0
= avoid_expensive_constant (mode
, binoptab
, 0,
1679 if (binoptab
!= ashl_optab
)
1680 xop1
= avoid_expensive_constant (mode
, binoptab
, 1,
1684 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
, no_extend
);
1686 /* The second operand of a shift must always be extended. */
1687 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
1688 no_extend
&& binoptab
!= ashl_optab
);
1690 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
1691 unsignedp
, OPTAB_DIRECT
);
1694 if (mclass
!= MODE_INT
1695 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
1698 target
= gen_reg_rtx (mode
);
1699 convert_move (target
, temp
, 0);
1703 return gen_lowpart (mode
, temp
);
1706 delete_insns_since (last
);
1710 /* If operation is commutative,
1711 try to make the first operand a register.
1712 Even better, try to make it the same as the target.
1713 Also try to make the last operand a constant. */
1714 if (commutative_optab_p (binoptab
)
1715 && swap_commutative_operands_with_target (target
, op0
, op1
))
1716 std::swap (op0
, op1
);
1718 /* These can be done a word at a time. */
1719 if ((binoptab
== and_optab
|| binoptab
== ior_optab
|| binoptab
== xor_optab
)
1720 && mclass
== MODE_INT
1721 && GET_MODE_SIZE (mode
) > UNITS_PER_WORD
1722 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1727 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1728 won't be accurate, so use a new target. */
1732 || !valid_multiword_target_p (target
))
1733 target
= gen_reg_rtx (mode
);
1737 /* Do the actual arithmetic. */
1738 for (i
= 0; i
< GET_MODE_BITSIZE (mode
) / BITS_PER_WORD
; i
++)
1740 rtx target_piece
= operand_subword (target
, i
, 1, mode
);
1741 rtx x
= expand_binop (word_mode
, binoptab
,
1742 operand_subword_force (op0
, i
, mode
),
1743 operand_subword_force (op1
, i
, mode
),
1744 target_piece
, unsignedp
, next_methods
);
1749 if (target_piece
!= x
)
1750 emit_move_insn (target_piece
, x
);
1753 insns
= get_insns ();
1756 if (i
== GET_MODE_BITSIZE (mode
) / BITS_PER_WORD
)
1763 /* Synthesize double word shifts from single word shifts. */
1764 if ((binoptab
== lshr_optab
|| binoptab
== ashl_optab
1765 || binoptab
== ashr_optab
)
1766 && mclass
== MODE_INT
1767 && (CONST_INT_P (op1
) || optimize_insn_for_speed_p ())
1768 && GET_MODE_SIZE (mode
) == 2 * UNITS_PER_WORD
1769 && GET_MODE_PRECISION (mode
) == GET_MODE_BITSIZE (mode
)
1770 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
1771 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1772 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1774 unsigned HOST_WIDE_INT shift_mask
, double_shift_mask
;
1775 machine_mode op1_mode
;
1777 double_shift_mask
= targetm
.shift_truncation_mask (mode
);
1778 shift_mask
= targetm
.shift_truncation_mask (word_mode
);
1779 op1_mode
= GET_MODE (op1
) != VOIDmode
? GET_MODE (op1
) : word_mode
;
1781 /* Apply the truncation to constant shifts. */
1782 if (double_shift_mask
> 0 && CONST_INT_P (op1
))
1783 op1
= GEN_INT (INTVAL (op1
) & double_shift_mask
);
1785 if (op1
== CONST0_RTX (op1_mode
))
1788 /* Make sure that this is a combination that expand_doubleword_shift
1789 can handle. See the comments there for details. */
1790 if (double_shift_mask
== 0
1791 || (shift_mask
== BITS_PER_WORD
- 1
1792 && double_shift_mask
== BITS_PER_WORD
* 2 - 1))
1795 rtx into_target
, outof_target
;
1796 rtx into_input
, outof_input
;
1797 int left_shift
, outof_word
;
1799 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1800 won't be accurate, so use a new target. */
1804 || !valid_multiword_target_p (target
))
1805 target
= gen_reg_rtx (mode
);
1809 /* OUTOF_* is the word we are shifting bits away from, and
1810 INTO_* is the word that we are shifting bits towards, thus
1811 they differ depending on the direction of the shift and
1812 WORDS_BIG_ENDIAN. */
1814 left_shift
= binoptab
== ashl_optab
;
1815 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1817 outof_target
= operand_subword (target
, outof_word
, 1, mode
);
1818 into_target
= operand_subword (target
, 1 - outof_word
, 1, mode
);
1820 outof_input
= operand_subword_force (op0
, outof_word
, mode
);
1821 into_input
= operand_subword_force (op0
, 1 - outof_word
, mode
);
1823 if (expand_doubleword_shift (op1_mode
, binoptab
,
1824 outof_input
, into_input
, op1
,
1825 outof_target
, into_target
,
1826 unsignedp
, next_methods
, shift_mask
))
1828 insns
= get_insns ();
1838 /* Synthesize double word rotates from single word shifts. */
1839 if ((binoptab
== rotl_optab
|| binoptab
== rotr_optab
)
1840 && mclass
== MODE_INT
1841 && CONST_INT_P (op1
)
1842 && GET_MODE_PRECISION (mode
) == 2 * BITS_PER_WORD
1843 && optab_handler (ashl_optab
, word_mode
) != CODE_FOR_nothing
1844 && optab_handler (lshr_optab
, word_mode
) != CODE_FOR_nothing
)
1847 rtx into_target
, outof_target
;
1848 rtx into_input
, outof_input
;
1850 int shift_count
, left_shift
, outof_word
;
1852 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1853 won't be accurate, so use a new target. Do this also if target is not
1854 a REG, first because having a register instead may open optimization
1855 opportunities, and second because if target and op0 happen to be MEMs
1856 designating the same location, we would risk clobbering it too early
1857 in the code sequence we generate below. */
1862 || !valid_multiword_target_p (target
))
1863 target
= gen_reg_rtx (mode
);
1867 shift_count
= INTVAL (op1
);
1869 /* OUTOF_* is the word we are shifting bits away from, and
1870 INTO_* is the word that we are shifting bits towards, thus
1871 they differ depending on the direction of the shift and
1872 WORDS_BIG_ENDIAN. */
1874 left_shift
= (binoptab
== rotl_optab
);
1875 outof_word
= left_shift
^ ! WORDS_BIG_ENDIAN
;
1877 outof_target
= operand_subword (target
, outof_word
, 1, mode
);
1878 into_target
= operand_subword (target
, 1 - outof_word
, 1, mode
);
1880 outof_input
= operand_subword_force (op0
, outof_word
, mode
);
1881 into_input
= operand_subword_force (op0
, 1 - outof_word
, mode
);
1883 if (shift_count
== BITS_PER_WORD
)
1885 /* This is just a word swap. */
1886 emit_move_insn (outof_target
, into_input
);
1887 emit_move_insn (into_target
, outof_input
);
1892 rtx into_temp1
, into_temp2
, outof_temp1
, outof_temp2
;
1893 rtx first_shift_count
, second_shift_count
;
1894 optab reverse_unsigned_shift
, unsigned_shift
;
1896 reverse_unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1897 ? lshr_optab
: ashl_optab
);
1899 unsigned_shift
= (left_shift
^ (shift_count
< BITS_PER_WORD
)
1900 ? ashl_optab
: lshr_optab
);
1902 if (shift_count
> BITS_PER_WORD
)
1904 first_shift_count
= GEN_INT (shift_count
- BITS_PER_WORD
);
1905 second_shift_count
= GEN_INT (2 * BITS_PER_WORD
- shift_count
);
1909 first_shift_count
= GEN_INT (BITS_PER_WORD
- shift_count
);
1910 second_shift_count
= GEN_INT (shift_count
);
1913 into_temp1
= expand_binop (word_mode
, unsigned_shift
,
1914 outof_input
, first_shift_count
,
1915 NULL_RTX
, unsignedp
, next_methods
);
1916 into_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1917 into_input
, second_shift_count
,
1918 NULL_RTX
, unsignedp
, next_methods
);
1920 if (into_temp1
!= 0 && into_temp2
!= 0)
1921 inter
= expand_binop (word_mode
, ior_optab
, into_temp1
, into_temp2
,
1922 into_target
, unsignedp
, next_methods
);
1926 if (inter
!= 0 && inter
!= into_target
)
1927 emit_move_insn (into_target
, inter
);
1929 outof_temp1
= expand_binop (word_mode
, unsigned_shift
,
1930 into_input
, first_shift_count
,
1931 NULL_RTX
, unsignedp
, next_methods
);
1932 outof_temp2
= expand_binop (word_mode
, reverse_unsigned_shift
,
1933 outof_input
, second_shift_count
,
1934 NULL_RTX
, unsignedp
, next_methods
);
1936 if (inter
!= 0 && outof_temp1
!= 0 && outof_temp2
!= 0)
1937 inter
= expand_binop (word_mode
, ior_optab
,
1938 outof_temp1
, outof_temp2
,
1939 outof_target
, unsignedp
, next_methods
);
1941 if (inter
!= 0 && inter
!= outof_target
)
1942 emit_move_insn (outof_target
, inter
);
1945 insns
= get_insns ();
1955 /* These can be done a word at a time by propagating carries. */
1956 if ((binoptab
== add_optab
|| binoptab
== sub_optab
)
1957 && mclass
== MODE_INT
1958 && GET_MODE_SIZE (mode
) >= 2 * UNITS_PER_WORD
1959 && optab_handler (binoptab
, word_mode
) != CODE_FOR_nothing
)
1962 optab otheroptab
= binoptab
== add_optab
? sub_optab
: add_optab
;
1963 const unsigned int nwords
= GET_MODE_BITSIZE (mode
) / BITS_PER_WORD
;
1964 rtx carry_in
= NULL_RTX
, carry_out
= NULL_RTX
;
1965 rtx xop0
, xop1
, xtarget
;
1967 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1968 value is one of those, use it. Otherwise, use 1 since it is the
1969 one easiest to get. */
1970 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1971 int normalizep
= STORE_FLAG_VALUE
;
1976 /* Prepare the operands. */
1977 xop0
= force_reg (mode
, op0
);
1978 xop1
= force_reg (mode
, op1
);
1980 xtarget
= gen_reg_rtx (mode
);
1982 if (target
== 0 || !REG_P (target
) || !valid_multiword_target_p (target
))
1985 /* Indicate for flow that the entire target reg is being set. */
1987 emit_clobber (xtarget
);
1989 /* Do the actual arithmetic. */
1990 for (i
= 0; i
< nwords
; i
++)
1992 int index
= (WORDS_BIG_ENDIAN
? nwords
- i
- 1 : i
);
1993 rtx target_piece
= operand_subword (xtarget
, index
, 1, mode
);
1994 rtx op0_piece
= operand_subword_force (xop0
, index
, mode
);
1995 rtx op1_piece
= operand_subword_force (xop1
, index
, mode
);
1998 /* Main add/subtract of the input operands. */
1999 x
= expand_binop (word_mode
, binoptab
,
2000 op0_piece
, op1_piece
,
2001 target_piece
, unsignedp
, next_methods
);
2007 /* Store carry from main add/subtract. */
2008 carry_out
= gen_reg_rtx (word_mode
);
2009 carry_out
= emit_store_flag_force (carry_out
,
2010 (binoptab
== add_optab
2013 word_mode
, 1, normalizep
);
2020 /* Add/subtract previous carry to main result. */
2021 newx
= expand_binop (word_mode
,
2022 normalizep
== 1 ? binoptab
: otheroptab
,
2024 NULL_RTX
, 1, next_methods
);
2028 /* Get out carry from adding/subtracting carry in. */
2029 rtx carry_tmp
= gen_reg_rtx (word_mode
);
2030 carry_tmp
= emit_store_flag_force (carry_tmp
,
2031 (binoptab
== add_optab
2034 word_mode
, 1, normalizep
);
2036 /* Logical-ior the two poss. carry together. */
2037 carry_out
= expand_binop (word_mode
, ior_optab
,
2038 carry_out
, carry_tmp
,
2039 carry_out
, 0, next_methods
);
2043 emit_move_insn (target_piece
, newx
);
2047 if (x
!= target_piece
)
2048 emit_move_insn (target_piece
, x
);
2051 carry_in
= carry_out
;
2054 if (i
== GET_MODE_BITSIZE (mode
) / (unsigned) BITS_PER_WORD
)
2056 if (optab_handler (mov_optab
, mode
) != CODE_FOR_nothing
2057 || ! rtx_equal_p (target
, xtarget
))
2059 rtx temp
= emit_move_insn (target
, xtarget
);
2061 set_dst_reg_note (temp
, REG_EQUAL
,
2062 gen_rtx_fmt_ee (optab_to_code (binoptab
),
2063 mode
, copy_rtx (xop0
),
2074 delete_insns_since (last
);
2077 /* Attempt to synthesize double word multiplies using a sequence of word
2078 mode multiplications. We first attempt to generate a sequence using a
2079 more efficient unsigned widening multiply, and if that fails we then
2080 try using a signed widening multiply. */
2082 if (binoptab
== smul_optab
2083 && mclass
== MODE_INT
2084 && GET_MODE_SIZE (mode
) == 2 * UNITS_PER_WORD
2085 && optab_handler (smul_optab
, word_mode
) != CODE_FOR_nothing
2086 && optab_handler (add_optab
, word_mode
) != CODE_FOR_nothing
)
2088 rtx product
= NULL_RTX
;
2089 if (widening_optab_handler (umul_widen_optab
, mode
, word_mode
)
2090 != CODE_FOR_nothing
)
2092 product
= expand_doubleword_mult (mode
, op0
, op1
, target
,
2095 delete_insns_since (last
);
2098 if (product
== NULL_RTX
2099 && widening_optab_handler (smul_widen_optab
, mode
, word_mode
)
2100 != CODE_FOR_nothing
)
2102 product
= expand_doubleword_mult (mode
, op0
, op1
, target
,
2105 delete_insns_since (last
);
2108 if (product
!= NULL_RTX
)
2110 if (optab_handler (mov_optab
, mode
) != CODE_FOR_nothing
)
2112 temp
= emit_move_insn (target
? target
: product
, product
);
2113 set_dst_reg_note (temp
,
2115 gen_rtx_fmt_ee (MULT
, mode
,
2118 target
? target
: product
);
2124 /* It can't be open-coded in this mode.
2125 Use a library call if one is available and caller says that's ok. */
2127 libfunc
= optab_libfunc (binoptab
, mode
);
2129 && (methods
== OPTAB_LIB
|| methods
== OPTAB_LIB_WIDEN
))
2133 machine_mode op1_mode
= mode
;
2138 if (shift_optab_p (binoptab
))
2140 op1_mode
= targetm
.libgcc_shift_count_mode ();
2141 /* Specify unsigned here,
2142 since negative shift counts are meaningless. */
2143 op1x
= convert_to_mode (op1_mode
, op1
, 1);
2146 if (GET_MODE (op0
) != VOIDmode
2147 && GET_MODE (op0
) != mode
)
2148 op0
= convert_to_mode (mode
, op0
, unsignedp
);
2150 /* Pass 1 for NO_QUEUE so we don't lose any increments
2151 if the libcall is cse'd or moved. */
2152 value
= emit_library_call_value (libfunc
,
2153 NULL_RTX
, LCT_CONST
, mode
, 2,
2154 op0
, mode
, op1x
, op1_mode
);
2156 insns
= get_insns ();
2159 target
= gen_reg_rtx (mode
);
2160 emit_libcall_block_1 (insns
, target
, value
,
2161 gen_rtx_fmt_ee (optab_to_code (binoptab
),
2163 trapv_binoptab_p (binoptab
));
2168 delete_insns_since (last
);
2170 /* It can't be done in this mode. Can we do it in a wider mode? */
2172 if (! (methods
== OPTAB_WIDEN
|| methods
== OPTAB_LIB_WIDEN
2173 || methods
== OPTAB_MUST_WIDEN
))
2175 /* Caller says, don't even try. */
2176 delete_insns_since (entry_last
);
2180 /* Compute the value of METHODS to pass to recursive calls.
2181 Don't allow widening to be tried recursively. */
2183 methods
= (methods
== OPTAB_LIB_WIDEN
? OPTAB_LIB
: OPTAB_DIRECT
);
2185 /* Look for a wider mode of the same class for which it appears we can do
2188 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2190 for (wider_mode
= GET_MODE_WIDER_MODE (mode
);
2191 wider_mode
!= VOIDmode
;
2192 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
2194 if (find_widening_optab_handler (binoptab
, wider_mode
, mode
, 1)
2196 || (methods
== OPTAB_LIB
2197 && optab_libfunc (binoptab
, wider_mode
)))
2199 rtx xop0
= op0
, xop1
= op1
;
2202 /* For certain integer operations, we need not actually extend
2203 the narrow operands, as long as we will truncate
2204 the results to the same narrowness. */
2206 if ((binoptab
== ior_optab
|| binoptab
== and_optab
2207 || binoptab
== xor_optab
2208 || binoptab
== add_optab
|| binoptab
== sub_optab
2209 || binoptab
== smul_optab
|| binoptab
== ashl_optab
)
2210 && mclass
== MODE_INT
)
2213 xop0
= widen_operand (xop0
, wider_mode
, mode
,
2214 unsignedp
, no_extend
);
2216 /* The second operand of a shift must always be extended. */
2217 xop1
= widen_operand (xop1
, wider_mode
, mode
, unsignedp
,
2218 no_extend
&& binoptab
!= ashl_optab
);
2220 temp
= expand_binop (wider_mode
, binoptab
, xop0
, xop1
, NULL_RTX
,
2221 unsignedp
, methods
);
2224 if (mclass
!= MODE_INT
2225 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
2228 target
= gen_reg_rtx (mode
);
2229 convert_move (target
, temp
, 0);
2233 return gen_lowpart (mode
, temp
);
2236 delete_insns_since (last
);
2241 delete_insns_since (entry_last
);
2245 /* Expand a binary operator which has both signed and unsigned forms.
2246 UOPTAB is the optab for unsigned operations, and SOPTAB is for
2249 If we widen unsigned operands, we may use a signed wider operation instead
2250 of an unsigned wider operation, since the result would be the same. */
2253 sign_expand_binop (machine_mode mode
, optab uoptab
, optab soptab
,
2254 rtx op0
, rtx op1
, rtx target
, int unsignedp
,
2255 enum optab_methods methods
)
2258 optab direct_optab
= unsignedp
? uoptab
: soptab
;
2261 /* Do it without widening, if possible. */
2262 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
2263 unsignedp
, OPTAB_DIRECT
);
2264 if (temp
|| methods
== OPTAB_DIRECT
)
2267 /* Try widening to a signed int. Disable any direct use of any
2268 signed insn in the current mode. */
2269 save_enable
= swap_optab_enable (soptab
, mode
, false);
2271 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
2272 unsignedp
, OPTAB_WIDEN
);
2274 /* For unsigned operands, try widening to an unsigned int. */
2275 if (!temp
&& unsignedp
)
2276 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
2277 unsignedp
, OPTAB_WIDEN
);
2278 if (temp
|| methods
== OPTAB_WIDEN
)
2281 /* Use the right width libcall if that exists. */
2282 temp
= expand_binop (mode
, direct_optab
, op0
, op1
, target
,
2283 unsignedp
, OPTAB_LIB
);
2284 if (temp
|| methods
== OPTAB_LIB
)
2287 /* Must widen and use a libcall, use either signed or unsigned. */
2288 temp
= expand_binop (mode
, soptab
, op0
, op1
, target
,
2289 unsignedp
, methods
);
2290 if (!temp
&& unsignedp
)
2291 temp
= expand_binop (mode
, uoptab
, op0
, op1
, target
,
2292 unsignedp
, methods
);
2295 /* Undo the fiddling above. */
2297 swap_optab_enable (soptab
, mode
, true);
2301 /* Generate code to perform an operation specified by UNOPPTAB
2302 on operand OP0, with two results to TARG0 and TARG1.
2303 We assume that the order of the operands for the instruction
2304 is TARG0, TARG1, OP0.
2306 Either TARG0 or TARG1 may be zero, but what that means is that
2307 the result is not actually wanted. We will generate it into
2308 a dummy pseudo-reg and discard it. They may not both be zero.
2310 Returns 1 if this operation can be performed; 0 if not. */
2313 expand_twoval_unop (optab unoptab
, rtx op0
, rtx targ0
, rtx targ1
,
2316 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
2317 enum mode_class mclass
;
2318 machine_mode wider_mode
;
2319 rtx_insn
*entry_last
= get_last_insn ();
2322 mclass
= GET_MODE_CLASS (mode
);
2325 targ0
= gen_reg_rtx (mode
);
2327 targ1
= gen_reg_rtx (mode
);
2329 /* Record where to go back to if we fail. */
2330 last
= get_last_insn ();
2332 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
2334 struct expand_operand ops
[3];
2335 enum insn_code icode
= optab_handler (unoptab
, mode
);
2337 create_fixed_operand (&ops
[0], targ0
);
2338 create_fixed_operand (&ops
[1], targ1
);
2339 create_convert_operand_from (&ops
[2], op0
, mode
, unsignedp
);
2340 if (maybe_expand_insn (icode
, 3, ops
))
2344 /* It can't be done in this mode. Can we do it in a wider mode? */
2346 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2348 for (wider_mode
= GET_MODE_WIDER_MODE (mode
);
2349 wider_mode
!= VOIDmode
;
2350 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
2352 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2354 rtx t0
= gen_reg_rtx (wider_mode
);
2355 rtx t1
= gen_reg_rtx (wider_mode
);
2356 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2358 if (expand_twoval_unop (unoptab
, cop0
, t0
, t1
, unsignedp
))
2360 convert_move (targ0
, t0
, unsignedp
);
2361 convert_move (targ1
, t1
, unsignedp
);
2365 delete_insns_since (last
);
2370 delete_insns_since (entry_last
);
2374 /* Generate code to perform an operation specified by BINOPTAB
2375 on operands OP0 and OP1, with two results to TARG1 and TARG2.
2376 We assume that the order of the operands for the instruction
2377 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2378 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2380 Either TARG0 or TARG1 may be zero, but what that means is that
2381 the result is not actually wanted. We will generate it into
2382 a dummy pseudo-reg and discard it. They may not both be zero.
2384 Returns 1 if this operation can be performed; 0 if not. */
2387 expand_twoval_binop (optab binoptab
, rtx op0
, rtx op1
, rtx targ0
, rtx targ1
,
2390 machine_mode mode
= GET_MODE (targ0
? targ0
: targ1
);
2391 enum mode_class mclass
;
2392 machine_mode wider_mode
;
2393 rtx_insn
*entry_last
= get_last_insn ();
2396 mclass
= GET_MODE_CLASS (mode
);
2399 targ0
= gen_reg_rtx (mode
);
2401 targ1
= gen_reg_rtx (mode
);
2403 /* Record where to go back to if we fail. */
2404 last
= get_last_insn ();
2406 if (optab_handler (binoptab
, mode
) != CODE_FOR_nothing
)
2408 struct expand_operand ops
[4];
2409 enum insn_code icode
= optab_handler (binoptab
, mode
);
2410 machine_mode mode0
= insn_data
[icode
].operand
[1].mode
;
2411 machine_mode mode1
= insn_data
[icode
].operand
[2].mode
;
2412 rtx xop0
= op0
, xop1
= op1
;
2414 /* If we are optimizing, force expensive constants into a register. */
2415 xop0
= avoid_expensive_constant (mode0
, binoptab
, 0, xop0
, unsignedp
);
2416 xop1
= avoid_expensive_constant (mode1
, binoptab
, 1, xop1
, unsignedp
);
2418 create_fixed_operand (&ops
[0], targ0
);
2419 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
2420 create_convert_operand_from (&ops
[2], op1
, mode
, unsignedp
);
2421 create_fixed_operand (&ops
[3], targ1
);
2422 if (maybe_expand_insn (icode
, 4, ops
))
2424 delete_insns_since (last
);
2427 /* It can't be done in this mode. Can we do it in a wider mode? */
2429 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2431 for (wider_mode
= GET_MODE_WIDER_MODE (mode
);
2432 wider_mode
!= VOIDmode
;
2433 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
2435 if (optab_handler (binoptab
, wider_mode
) != CODE_FOR_nothing
)
2437 rtx t0
= gen_reg_rtx (wider_mode
);
2438 rtx t1
= gen_reg_rtx (wider_mode
);
2439 rtx cop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
2440 rtx cop1
= convert_modes (wider_mode
, mode
, op1
, unsignedp
);
2442 if (expand_twoval_binop (binoptab
, cop0
, cop1
,
2445 convert_move (targ0
, t0
, unsignedp
);
2446 convert_move (targ1
, t1
, unsignedp
);
2450 delete_insns_since (last
);
2455 delete_insns_since (entry_last
);
2459 /* Expand the two-valued library call indicated by BINOPTAB, but
2460 preserve only one of the values. If TARG0 is non-NULL, the first
2461 value is placed into TARG0; otherwise the second value is placed
2462 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2463 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2464 This routine assumes that the value returned by the library call is
2465 as if the return value was of an integral mode twice as wide as the
2466 mode of OP0. Returns 1 if the call was successful. */
2469 expand_twoval_binop_libfunc (optab binoptab
, rtx op0
, rtx op1
,
2470 rtx targ0
, rtx targ1
, enum rtx_code code
)
2473 machine_mode libval_mode
;
2478 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2479 gcc_assert (!targ0
!= !targ1
);
2481 mode
= GET_MODE (op0
);
2482 libfunc
= optab_libfunc (binoptab
, mode
);
2486 /* The value returned by the library function will have twice as
2487 many bits as the nominal MODE. */
2488 libval_mode
= smallest_mode_for_size (2 * GET_MODE_BITSIZE (mode
),
2491 libval
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
2495 /* Get the part of VAL containing the value that we want. */
2496 libval
= simplify_gen_subreg (mode
, libval
, libval_mode
,
2497 targ0
? 0 : GET_MODE_SIZE (mode
));
2498 insns
= get_insns ();
2500 /* Move the into the desired location. */
2501 emit_libcall_block (insns
, targ0
? targ0
: targ1
, libval
,
2502 gen_rtx_fmt_ee (code
, mode
, op0
, op1
));
2508 /* Wrapper around expand_unop which takes an rtx code to specify
2509 the operation to perform, not an optab pointer. All other
2510 arguments are the same. */
2512 expand_simple_unop (machine_mode mode
, enum rtx_code code
, rtx op0
,
2513 rtx target
, int unsignedp
)
2515 optab unop
= code_to_optab (code
);
2518 return expand_unop (mode
, unop
, op0
, target
, unsignedp
);
2524 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2526 A similar operation can be used for clrsb. UNOPTAB says which operation
2527 we are trying to expand. */
2529 widen_leading (machine_mode mode
, rtx op0
, rtx target
, optab unoptab
)
2531 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2532 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2534 machine_mode wider_mode
;
2535 for (wider_mode
= GET_MODE_WIDER_MODE (mode
);
2536 wider_mode
!= VOIDmode
;
2537 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
2539 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
2544 last
= get_last_insn ();
2547 target
= gen_reg_rtx (mode
);
2548 xop0
= widen_operand (op0
, wider_mode
, mode
,
2549 unoptab
!= clrsb_optab
, false);
2550 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
2551 unoptab
!= clrsb_optab
);
2554 (wider_mode
, sub_optab
, temp
,
2555 gen_int_mode (GET_MODE_PRECISION (wider_mode
)
2556 - GET_MODE_PRECISION (mode
),
2558 target
, true, OPTAB_DIRECT
);
2560 delete_insns_since (last
);
2569 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2570 quantities, choosing which based on whether the high word is nonzero. */
2572 expand_doubleword_clz (machine_mode mode
, rtx op0
, rtx target
)
2574 rtx xop0
= force_reg (mode
, op0
);
2575 rtx subhi
= gen_highpart (word_mode
, xop0
);
2576 rtx sublo
= gen_lowpart (word_mode
, xop0
);
2577 rtx_code_label
*hi0_label
= gen_label_rtx ();
2578 rtx_code_label
*after_label
= gen_label_rtx ();
2582 /* If we were not given a target, use a word_mode register, not a
2583 'mode' register. The result will fit, and nobody is expecting
2584 anything bigger (the return type of __builtin_clz* is int). */
2586 target
= gen_reg_rtx (word_mode
);
2588 /* In any case, write to a word_mode scratch in both branches of the
2589 conditional, so we can ensure there is a single move insn setting
2590 'target' to tag a REG_EQUAL note on. */
2591 result
= gen_reg_rtx (word_mode
);
2595 /* If the high word is not equal to zero,
2596 then clz of the full value is clz of the high word. */
2597 emit_cmp_and_jump_insns (subhi
, CONST0_RTX (word_mode
), EQ
, 0,
2598 word_mode
, true, hi0_label
);
2600 temp
= expand_unop_direct (word_mode
, clz_optab
, subhi
, result
, true);
2605 convert_move (result
, temp
, true);
2607 emit_jump_insn (gen_jump (after_label
));
2610 /* Else clz of the full value is clz of the low word plus the number
2611 of bits in the high word. */
2612 emit_label (hi0_label
);
2614 temp
= expand_unop_direct (word_mode
, clz_optab
, sublo
, 0, true);
2617 temp
= expand_binop (word_mode
, add_optab
, temp
,
2618 gen_int_mode (GET_MODE_BITSIZE (word_mode
), word_mode
),
2619 result
, true, OPTAB_DIRECT
);
2623 convert_move (result
, temp
, true);
2625 emit_label (after_label
);
2626 convert_move (target
, result
, true);
2631 add_equal_note (seq
, target
, CLZ
, xop0
, 0);
2643 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2645 widen_bswap (machine_mode mode
, rtx op0
, rtx target
)
2647 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2648 machine_mode wider_mode
;
2652 if (!CLASS_HAS_WIDER_MODES_P (mclass
))
2655 for (wider_mode
= GET_MODE_WIDER_MODE (mode
);
2656 wider_mode
!= VOIDmode
;
2657 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
2658 if (optab_handler (bswap_optab
, wider_mode
) != CODE_FOR_nothing
)
2663 last
= get_last_insn ();
2665 x
= widen_operand (op0
, wider_mode
, mode
, true, true);
2666 x
= expand_unop (wider_mode
, bswap_optab
, x
, NULL_RTX
, true);
2668 gcc_assert (GET_MODE_PRECISION (wider_mode
) == GET_MODE_BITSIZE (wider_mode
)
2669 && GET_MODE_PRECISION (mode
) == GET_MODE_BITSIZE (mode
));
2671 x
= expand_shift (RSHIFT_EXPR
, wider_mode
, x
,
2672 GET_MODE_BITSIZE (wider_mode
)
2673 - GET_MODE_BITSIZE (mode
),
2679 target
= gen_reg_rtx (mode
);
2680 emit_move_insn (target
, gen_lowpart (mode
, x
));
2683 delete_insns_since (last
);
2688 /* Try calculating bswap as two bswaps of two word-sized operands. */
2691 expand_doubleword_bswap (machine_mode mode
, rtx op
, rtx target
)
2695 t1
= expand_unop (word_mode
, bswap_optab
,
2696 operand_subword_force (op
, 0, mode
), NULL_RTX
, true);
2697 t0
= expand_unop (word_mode
, bswap_optab
,
2698 operand_subword_force (op
, 1, mode
), NULL_RTX
, true);
2700 if (target
== 0 || !valid_multiword_target_p (target
))
2701 target
= gen_reg_rtx (mode
);
2703 emit_clobber (target
);
2704 emit_move_insn (operand_subword (target
, 0, 1, mode
), t0
);
2705 emit_move_insn (operand_subword (target
, 1, 1, mode
), t1
);
2710 /* Try calculating (parity x) as (and (popcount x) 1), where
2711 popcount can also be done in a wider mode. */
2713 expand_parity (machine_mode mode
, rtx op0
, rtx target
)
2715 enum mode_class mclass
= GET_MODE_CLASS (mode
);
2716 if (CLASS_HAS_WIDER_MODES_P (mclass
))
2718 machine_mode wider_mode
;
2719 for (wider_mode
= mode
; wider_mode
!= VOIDmode
;
2720 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
2722 if (optab_handler (popcount_optab
, wider_mode
) != CODE_FOR_nothing
)
2727 last
= get_last_insn ();
2730 target
= gen_reg_rtx (mode
);
2731 xop0
= widen_operand (op0
, wider_mode
, mode
, true, false);
2732 temp
= expand_unop (wider_mode
, popcount_optab
, xop0
, NULL_RTX
,
2735 temp
= expand_binop (wider_mode
, and_optab
, temp
, const1_rtx
,
2736 target
, true, OPTAB_DIRECT
);
2738 delete_insns_since (last
);
2747 /* Try calculating ctz(x) as K - clz(x & -x) ,
2748 where K is GET_MODE_PRECISION(mode) - 1.
2750 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2751 don't have to worry about what the hardware does in that case. (If
2752 the clz instruction produces the usual value at 0, which is K, the
2753 result of this code sequence will be -1; expand_ffs, below, relies
2754 on this. It might be nice to have it be K instead, for consistency
2755 with the (very few) processors that provide a ctz with a defined
2756 value, but that would take one more instruction, and it would be
2757 less convenient for expand_ffs anyway. */
2760 expand_ctz (machine_mode mode
, rtx op0
, rtx target
)
2765 if (optab_handler (clz_optab
, mode
) == CODE_FOR_nothing
)
2770 temp
= expand_unop_direct (mode
, neg_optab
, op0
, NULL_RTX
, true);
2772 temp
= expand_binop (mode
, and_optab
, op0
, temp
, NULL_RTX
,
2773 true, OPTAB_DIRECT
);
2775 temp
= expand_unop_direct (mode
, clz_optab
, temp
, NULL_RTX
, true);
2777 temp
= expand_binop (mode
, sub_optab
,
2778 gen_int_mode (GET_MODE_PRECISION (mode
) - 1, mode
),
2780 true, OPTAB_DIRECT
);
2790 add_equal_note (seq
, temp
, CTZ
, op0
, 0);
2796 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2797 else with the sequence used by expand_clz.
2799 The ffs builtin promises to return zero for a zero value and ctz/clz
2800 may have an undefined value in that case. If they do not give us a
2801 convenient value, we have to generate a test and branch. */
2803 expand_ffs (machine_mode mode
, rtx op0
, rtx target
)
2805 HOST_WIDE_INT val
= 0;
2806 bool defined_at_zero
= false;
2810 if (optab_handler (ctz_optab
, mode
) != CODE_FOR_nothing
)
2814 temp
= expand_unop_direct (mode
, ctz_optab
, op0
, 0, true);
2818 defined_at_zero
= (CTZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2);
2820 else if (optab_handler (clz_optab
, mode
) != CODE_FOR_nothing
)
2823 temp
= expand_ctz (mode
, op0
, 0);
2827 if (CLZ_DEFINED_VALUE_AT_ZERO (mode
, val
) == 2)
2829 defined_at_zero
= true;
2830 val
= (GET_MODE_PRECISION (mode
) - 1) - val
;
2836 if (defined_at_zero
&& val
== -1)
2837 /* No correction needed at zero. */;
2840 /* We don't try to do anything clever with the situation found
2841 on some processors (eg Alpha) where ctz(0:mode) ==
2842 bitsize(mode). If someone can think of a way to send N to -1
2843 and leave alone all values in the range 0..N-1 (where N is a
2844 power of two), cheaper than this test-and-branch, please add it.
2846 The test-and-branch is done after the operation itself, in case
2847 the operation sets condition codes that can be recycled for this.
2848 (This is true on i386, for instance.) */
2850 rtx_code_label
*nonzero_label
= gen_label_rtx ();
2851 emit_cmp_and_jump_insns (op0
, CONST0_RTX (mode
), NE
, 0,
2852 mode
, true, nonzero_label
);
2854 convert_move (temp
, GEN_INT (-1), false);
2855 emit_label (nonzero_label
);
2858 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2859 to produce a value in the range 0..bitsize. */
2860 temp
= expand_binop (mode
, add_optab
, temp
, gen_int_mode (1, mode
),
2861 target
, false, OPTAB_DIRECT
);
2868 add_equal_note (seq
, temp
, FFS
, op0
, 0);
2877 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2878 conditions, VAL may already be a SUBREG against which we cannot generate
2879 a further SUBREG. In this case, we expect forcing the value into a
2880 register will work around the situation. */
2883 lowpart_subreg_maybe_copy (machine_mode omode
, rtx val
,
2887 ret
= lowpart_subreg (omode
, val
, imode
);
2890 val
= force_reg (imode
, val
);
2891 ret
= lowpart_subreg (omode
, val
, imode
);
2892 gcc_assert (ret
!= NULL
);
2897 /* Expand a floating point absolute value or negation operation via a
2898 logical operation on the sign bit. */
2901 expand_absneg_bit (enum rtx_code code
, machine_mode mode
,
2902 rtx op0
, rtx target
)
2904 const struct real_format
*fmt
;
2905 int bitpos
, word
, nwords
, i
;
2910 /* The format has to have a simple sign bit. */
2911 fmt
= REAL_MODE_FORMAT (mode
);
2915 bitpos
= fmt
->signbit_rw
;
2919 /* Don't create negative zeros if the format doesn't support them. */
2920 if (code
== NEG
&& !fmt
->has_signed_zero
)
2923 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
2925 imode
= int_mode_for_mode (mode
);
2926 if (imode
== BLKmode
)
2935 if (FLOAT_WORDS_BIG_ENDIAN
)
2936 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
2938 word
= bitpos
/ BITS_PER_WORD
;
2939 bitpos
= bitpos
% BITS_PER_WORD
;
2940 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
2943 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
2949 || (nwords
> 1 && !valid_multiword_target_p (target
)))
2950 target
= gen_reg_rtx (mode
);
2956 for (i
= 0; i
< nwords
; ++i
)
2958 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
2959 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
2963 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
2965 immed_wide_int_const (mask
, imode
),
2966 targ_piece
, 1, OPTAB_LIB_WIDEN
);
2967 if (temp
!= targ_piece
)
2968 emit_move_insn (targ_piece
, temp
);
2971 emit_move_insn (targ_piece
, op0_piece
);
2974 insns
= get_insns ();
2981 temp
= expand_binop (imode
, code
== ABS
? and_optab
: xor_optab
,
2982 gen_lowpart (imode
, op0
),
2983 immed_wide_int_const (mask
, imode
),
2984 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
2985 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
2987 set_dst_reg_note (get_last_insn (), REG_EQUAL
,
2988 gen_rtx_fmt_e (code
, mode
, copy_rtx (op0
)),
2995 /* As expand_unop, but will fail rather than attempt the operation in a
2996 different mode or with a libcall. */
2998 expand_unop_direct (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
3001 if (optab_handler (unoptab
, mode
) != CODE_FOR_nothing
)
3003 struct expand_operand ops
[2];
3004 enum insn_code icode
= optab_handler (unoptab
, mode
);
3005 rtx_insn
*last
= get_last_insn ();
3008 create_output_operand (&ops
[0], target
, mode
);
3009 create_convert_operand_from (&ops
[1], op0
, mode
, unsignedp
);
3010 pat
= maybe_gen_insn (icode
, 2, ops
);
3013 if (INSN_P (pat
) && NEXT_INSN (as_a
<rtx_insn
*> (pat
)) != NULL_RTX
3014 && ! add_equal_note (as_a
<rtx_insn
*> (pat
), ops
[0].value
,
3015 optab_to_code (unoptab
),
3016 ops
[1].value
, NULL_RTX
))
3018 delete_insns_since (last
);
3019 return expand_unop (mode
, unoptab
, op0
, NULL_RTX
, unsignedp
);
3024 return ops
[0].value
;
3030 /* Generate code to perform an operation specified by UNOPTAB
3031 on operand OP0, with result having machine-mode MODE.
3033 UNSIGNEDP is for the case where we have to widen the operands
3034 to perform the operation. It says to use zero-extension.
3036 If TARGET is nonzero, the value
3037 is generated there, if it is convenient to do so.
3038 In all cases an rtx is returned for the locus of the value;
3039 this may or may not be TARGET. */
3042 expand_unop (machine_mode mode
, optab unoptab
, rtx op0
, rtx target
,
3045 enum mode_class mclass
= GET_MODE_CLASS (mode
);
3046 machine_mode wider_mode
;
3050 temp
= expand_unop_direct (mode
, unoptab
, op0
, target
, unsignedp
);
3054 /* It can't be done in this mode. Can we open-code it in a wider mode? */
3056 /* Widening (or narrowing) clz needs special treatment. */
3057 if (unoptab
== clz_optab
)
3059 temp
= widen_leading (mode
, op0
, target
, unoptab
);
3063 if (GET_MODE_SIZE (mode
) == 2 * UNITS_PER_WORD
3064 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
3066 temp
= expand_doubleword_clz (mode
, op0
, target
);
3074 if (unoptab
== clrsb_optab
)
3076 temp
= widen_leading (mode
, op0
, target
, unoptab
);
3082 /* Widening (or narrowing) bswap needs special treatment. */
3083 if (unoptab
== bswap_optab
)
3085 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
3086 or ROTATERT. First try these directly; if this fails, then try the
3087 obvious pair of shifts with allowed widening, as this will probably
3088 be always more efficient than the other fallback methods. */
3094 if (optab_handler (rotl_optab
, mode
) != CODE_FOR_nothing
)
3096 temp
= expand_binop (mode
, rotl_optab
, op0
, GEN_INT (8), target
,
3097 unsignedp
, OPTAB_DIRECT
);
3102 if (optab_handler (rotr_optab
, mode
) != CODE_FOR_nothing
)
3104 temp
= expand_binop (mode
, rotr_optab
, op0
, GEN_INT (8), target
,
3105 unsignedp
, OPTAB_DIRECT
);
3110 last
= get_last_insn ();
3112 temp1
= expand_binop (mode
, ashl_optab
, op0
, GEN_INT (8), NULL_RTX
,
3113 unsignedp
, OPTAB_WIDEN
);
3114 temp2
= expand_binop (mode
, lshr_optab
, op0
, GEN_INT (8), NULL_RTX
,
3115 unsignedp
, OPTAB_WIDEN
);
3118 temp
= expand_binop (mode
, ior_optab
, temp1
, temp2
, target
,
3119 unsignedp
, OPTAB_WIDEN
);
3124 delete_insns_since (last
);
3127 temp
= widen_bswap (mode
, op0
, target
);
3131 if (GET_MODE_SIZE (mode
) == 2 * UNITS_PER_WORD
3132 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
3134 temp
= expand_doubleword_bswap (mode
, op0
, target
);
3142 if (CLASS_HAS_WIDER_MODES_P (mclass
))
3143 for (wider_mode
= GET_MODE_WIDER_MODE (mode
);
3144 wider_mode
!= VOIDmode
;
3145 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
3147 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
)
3150 rtx_insn
*last
= get_last_insn ();
3152 /* For certain operations, we need not actually extend
3153 the narrow operand, as long as we will truncate the
3154 results to the same narrowness. */
3156 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3157 (unoptab
== neg_optab
3158 || unoptab
== one_cmpl_optab
)
3159 && mclass
== MODE_INT
);
3161 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3166 if (mclass
!= MODE_INT
3167 || !TRULY_NOOP_TRUNCATION_MODES_P (mode
, wider_mode
))
3170 target
= gen_reg_rtx (mode
);
3171 convert_move (target
, temp
, 0);
3175 return gen_lowpart (mode
, temp
);
3178 delete_insns_since (last
);
3182 /* These can be done a word at a time. */
3183 if (unoptab
== one_cmpl_optab
3184 && mclass
== MODE_INT
3185 && GET_MODE_SIZE (mode
) > UNITS_PER_WORD
3186 && optab_handler (unoptab
, word_mode
) != CODE_FOR_nothing
)
3191 if (target
== 0 || target
== op0
|| !valid_multiword_target_p (target
))
3192 target
= gen_reg_rtx (mode
);
3196 /* Do the actual arithmetic. */
3197 for (i
= 0; i
< GET_MODE_BITSIZE (mode
) / BITS_PER_WORD
; i
++)
3199 rtx target_piece
= operand_subword (target
, i
, 1, mode
);
3200 rtx x
= expand_unop (word_mode
, unoptab
,
3201 operand_subword_force (op0
, i
, mode
),
3202 target_piece
, unsignedp
);
3204 if (target_piece
!= x
)
3205 emit_move_insn (target_piece
, x
);
3208 insns
= get_insns ();
3215 if (optab_to_code (unoptab
) == NEG
)
3217 /* Try negating floating point values by flipping the sign bit. */
3218 if (SCALAR_FLOAT_MODE_P (mode
))
3220 temp
= expand_absneg_bit (NEG
, mode
, op0
, target
);
3225 /* If there is no negation pattern, and we have no negative zero,
3226 try subtracting from zero. */
3227 if (!HONOR_SIGNED_ZEROS (mode
))
3229 temp
= expand_binop (mode
, (unoptab
== negv_optab
3230 ? subv_optab
: sub_optab
),
3231 CONST0_RTX (mode
), op0
, target
,
3232 unsignedp
, OPTAB_DIRECT
);
3238 /* Try calculating parity (x) as popcount (x) % 2. */
3239 if (unoptab
== parity_optab
)
3241 temp
= expand_parity (mode
, op0
, target
);
3246 /* Try implementing ffs (x) in terms of clz (x). */
3247 if (unoptab
== ffs_optab
)
3249 temp
= expand_ffs (mode
, op0
, target
);
3254 /* Try implementing ctz (x) in terms of clz (x). */
3255 if (unoptab
== ctz_optab
)
3257 temp
= expand_ctz (mode
, op0
, target
);
3263 /* Now try a library call in this mode. */
3264 libfunc
= optab_libfunc (unoptab
, mode
);
3270 machine_mode outmode
= mode
;
3272 /* All of these functions return small values. Thus we choose to
3273 have them return something that isn't a double-word. */
3274 if (unoptab
== ffs_optab
|| unoptab
== clz_optab
|| unoptab
== ctz_optab
3275 || unoptab
== clrsb_optab
|| unoptab
== popcount_optab
3276 || unoptab
== parity_optab
)
3278 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node
),
3279 optab_libfunc (unoptab
, mode
)));
3283 /* Pass 1 for NO_QUEUE so we don't lose any increments
3284 if the libcall is cse'd or moved. */
3285 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, outmode
,
3287 insns
= get_insns ();
3290 target
= gen_reg_rtx (outmode
);
3291 eq_value
= gen_rtx_fmt_e (optab_to_code (unoptab
), mode
, op0
);
3292 if (GET_MODE_SIZE (outmode
) < GET_MODE_SIZE (mode
))
3293 eq_value
= simplify_gen_unary (TRUNCATE
, outmode
, eq_value
, mode
);
3294 else if (GET_MODE_SIZE (outmode
) > GET_MODE_SIZE (mode
))
3295 eq_value
= simplify_gen_unary (ZERO_EXTEND
, outmode
, eq_value
, mode
);
3296 emit_libcall_block_1 (insns
, target
, value
, eq_value
,
3297 trapv_unoptab_p (unoptab
));
3302 /* It can't be done in this mode. Can we do it in a wider mode? */
3304 if (CLASS_HAS_WIDER_MODES_P (mclass
))
3306 for (wider_mode
= GET_MODE_WIDER_MODE (mode
);
3307 wider_mode
!= VOIDmode
;
3308 wider_mode
= GET_MODE_WIDER_MODE (wider_mode
))
3310 if (optab_handler (unoptab
, wider_mode
) != CODE_FOR_nothing
3311 || optab_libfunc (unoptab
, wider_mode
))
3314 rtx_insn
*last
= get_last_insn ();
3316 /* For certain operations, we need not actually extend
3317 the narrow operand, as long as we will truncate the
3318 results to the same narrowness. */
3319 xop0
= widen_operand (xop0
, wider_mode
, mode
, unsignedp
,
3320 (unoptab
== neg_optab
3321 || unoptab
== one_cmpl_optab
3322 || unoptab
== bswap_optab
)
3323 && mclass
== MODE_INT
);
3325 temp
= expand_unop (wider_mode
, unoptab
, xop0
, NULL_RTX
,
3328 /* If we are generating clz using wider mode, adjust the
3329 result. Similarly for clrsb. */
3330 if ((unoptab
== clz_optab
|| unoptab
== clrsb_optab
)
3333 (wider_mode
, sub_optab
, temp
,
3334 gen_int_mode (GET_MODE_PRECISION (wider_mode
)
3335 - GET_MODE_PRECISION (mode
),
3337 target
, true, OPTAB_DIRECT
);
3339 /* Likewise for bswap. */
3340 if (unoptab
== bswap_optab
&& temp
!= 0)
3342 gcc_assert (GET_MODE_PRECISION (wider_mode
)
3343 == GET_MODE_BITSIZE (wider_mode
)
3344 && GET_MODE_PRECISION (mode
)
3345 == GET_MODE_BITSIZE (mode
));
3347 temp
= expand_shift (RSHIFT_EXPR
, wider_mode
, temp
,
3348 GET_MODE_BITSIZE (wider_mode
)
3349 - GET_MODE_BITSIZE (mode
),
3355 if (mclass
!= MODE_INT
)
3358 target
= gen_reg_rtx (mode
);
3359 convert_move (target
, temp
, 0);
3363 return gen_lowpart (mode
, temp
);
3366 delete_insns_since (last
);
3371 /* One final attempt at implementing negation via subtraction,
3372 this time allowing widening of the operand. */
3373 if (optab_to_code (unoptab
) == NEG
&& !HONOR_SIGNED_ZEROS (mode
))
3376 temp
= expand_binop (mode
,
3377 unoptab
== negv_optab
? subv_optab
: sub_optab
,
3378 CONST0_RTX (mode
), op0
,
3379 target
, unsignedp
, OPTAB_LIB_WIDEN
);
3387 /* Emit code to compute the absolute value of OP0, with result to
3388 TARGET if convenient. (TARGET may be 0.) The return value says
3389 where the result actually is to be found.
3391 MODE is the mode of the operand; the mode of the result is
3392 different but can be deduced from MODE.
3397 expand_abs_nojump (machine_mode mode
, rtx op0
, rtx target
,
3398 int result_unsignedp
)
3402 if (GET_MODE_CLASS (mode
) != MODE_INT
3404 result_unsignedp
= 1;
3406 /* First try to do it with a special abs instruction. */
3407 temp
= expand_unop (mode
, result_unsignedp
? abs_optab
: absv_optab
,
3412 /* For floating point modes, try clearing the sign bit. */
3413 if (SCALAR_FLOAT_MODE_P (mode
))
3415 temp
= expand_absneg_bit (ABS
, mode
, op0
, target
);
3420 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3421 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
3422 && !HONOR_SIGNED_ZEROS (mode
))
3424 rtx_insn
*last
= get_last_insn ();
3426 temp
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3429 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3435 delete_insns_since (last
);
3438 /* If this machine has expensive jumps, we can do integer absolute
3439 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3440 where W is the width of MODE. */
3442 if (GET_MODE_CLASS (mode
) == MODE_INT
3443 && BRANCH_COST (optimize_insn_for_speed_p (),
3446 rtx extended
= expand_shift (RSHIFT_EXPR
, mode
, op0
,
3447 GET_MODE_PRECISION (mode
) - 1,
3450 temp
= expand_binop (mode
, xor_optab
, extended
, op0
, target
, 0,
3453 temp
= expand_binop (mode
, result_unsignedp
? sub_optab
: subv_optab
,
3454 temp
, extended
, target
, 0, OPTAB_LIB_WIDEN
);
3464 expand_abs (machine_mode mode
, rtx op0
, rtx target
,
3465 int result_unsignedp
, int safe
)
3468 rtx_code_label
*op1
;
3470 if (GET_MODE_CLASS (mode
) != MODE_INT
3472 result_unsignedp
= 1;
3474 temp
= expand_abs_nojump (mode
, op0
, target
, result_unsignedp
);
3478 /* If that does not win, use conditional jump and negate. */
3480 /* It is safe to use the target if it is the same
3481 as the source if this is also a pseudo register */
3482 if (op0
== target
&& REG_P (op0
)
3483 && REGNO (op0
) >= FIRST_PSEUDO_REGISTER
)
3486 op1
= gen_label_rtx ();
3487 if (target
== 0 || ! safe
3488 || GET_MODE (target
) != mode
3489 || (MEM_P (target
) && MEM_VOLATILE_P (target
))
3491 && REGNO (target
) < FIRST_PSEUDO_REGISTER
))
3492 target
= gen_reg_rtx (mode
);
3494 emit_move_insn (target
, op0
);
3497 do_compare_rtx_and_jump (target
, CONST0_RTX (mode
), GE
, 0, mode
,
3498 NULL_RTX
, NULL_RTX
, op1
, -1);
3500 op0
= expand_unop (mode
, result_unsignedp
? neg_optab
: negv_optab
,
3503 emit_move_insn (target
, op0
);
3509 /* Emit code to compute the one's complement absolute value of OP0
3510 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3511 (TARGET may be NULL_RTX.) The return value says where the result
3512 actually is to be found.
3514 MODE is the mode of the operand; the mode of the result is
3515 different but can be deduced from MODE. */
3518 expand_one_cmpl_abs_nojump (machine_mode mode
, rtx op0
, rtx target
)
3522 /* Not applicable for floating point modes. */
3523 if (FLOAT_MODE_P (mode
))
3526 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3527 if (optab_handler (smax_optab
, mode
) != CODE_FOR_nothing
)
3529 rtx_insn
*last
= get_last_insn ();
3531 temp
= expand_unop (mode
, one_cmpl_optab
, op0
, NULL_RTX
, 0);
3533 temp
= expand_binop (mode
, smax_optab
, op0
, temp
, target
, 0,
3539 delete_insns_since (last
);
3542 /* If this machine has expensive jumps, we can do one's complement
3543 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3545 if (GET_MODE_CLASS (mode
) == MODE_INT
3546 && BRANCH_COST (optimize_insn_for_speed_p (),
3549 rtx extended
= expand_shift (RSHIFT_EXPR
, mode
, op0
,
3550 GET_MODE_PRECISION (mode
) - 1,
3553 temp
= expand_binop (mode
, xor_optab
, extended
, op0
, target
, 0,
3563 /* A subroutine of expand_copysign, perform the copysign operation using the
3564 abs and neg primitives advertised to exist on the target. The assumption
3565 is that we have a split register file, and leaving op0 in fp registers,
3566 and not playing with subregs so much, will help the register allocator. */
3569 expand_copysign_absneg (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
3570 int bitpos
, bool op0_is_abs
)
3573 enum insn_code icode
;
3575 rtx_code_label
*label
;
3580 /* Check if the back end provides an insn that handles signbit for the
3582 icode
= optab_handler (signbit_optab
, mode
);
3583 if (icode
!= CODE_FOR_nothing
)
3585 imode
= insn_data
[(int) icode
].operand
[0].mode
;
3586 sign
= gen_reg_rtx (imode
);
3587 emit_unop_insn (icode
, sign
, op1
, UNKNOWN
);
3591 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3593 imode
= int_mode_for_mode (mode
);
3594 if (imode
== BLKmode
)
3596 op1
= gen_lowpart (imode
, op1
);
3603 if (FLOAT_WORDS_BIG_ENDIAN
)
3604 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3606 word
= bitpos
/ BITS_PER_WORD
;
3607 bitpos
= bitpos
% BITS_PER_WORD
;
3608 op1
= operand_subword_force (op1
, word
, mode
);
3611 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3612 sign
= expand_binop (imode
, and_optab
, op1
,
3613 immed_wide_int_const (mask
, imode
),
3614 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3619 op0
= expand_unop (mode
, abs_optab
, op0
, target
, 0);
3626 if (target
== NULL_RTX
)
3627 target
= copy_to_reg (op0
);
3629 emit_move_insn (target
, op0
);
3632 label
= gen_label_rtx ();
3633 emit_cmp_and_jump_insns (sign
, const0_rtx
, EQ
, NULL_RTX
, imode
, 1, label
);
3635 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3636 op0
= simplify_unary_operation (NEG
, mode
, op0
, mode
);
3638 op0
= expand_unop (mode
, neg_optab
, op0
, target
, 0);
3640 emit_move_insn (target
, op0
);
3648 /* A subroutine of expand_copysign, perform the entire copysign operation
3649 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3650 is true if op0 is known to have its sign bit clear. */
3653 expand_copysign_bit (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
3654 int bitpos
, bool op0_is_abs
)
3657 int word
, nwords
, i
;
3661 if (GET_MODE_SIZE (mode
) <= UNITS_PER_WORD
)
3663 imode
= int_mode_for_mode (mode
);
3664 if (imode
== BLKmode
)
3673 if (FLOAT_WORDS_BIG_ENDIAN
)
3674 word
= (GET_MODE_BITSIZE (mode
) - bitpos
) / BITS_PER_WORD
;
3676 word
= bitpos
/ BITS_PER_WORD
;
3677 bitpos
= bitpos
% BITS_PER_WORD
;
3678 nwords
= (GET_MODE_BITSIZE (mode
) + BITS_PER_WORD
- 1) / BITS_PER_WORD
;
3681 wide_int mask
= wi::set_bit_in_zero (bitpos
, GET_MODE_PRECISION (imode
));
3686 || (nwords
> 1 && !valid_multiword_target_p (target
)))
3687 target
= gen_reg_rtx (mode
);
3693 for (i
= 0; i
< nwords
; ++i
)
3695 rtx targ_piece
= operand_subword (target
, i
, 1, mode
);
3696 rtx op0_piece
= operand_subword_force (op0
, i
, mode
);
3702 = expand_binop (imode
, and_optab
, op0_piece
,
3703 immed_wide_int_const (~mask
, imode
),
3704 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3705 op1
= expand_binop (imode
, and_optab
,
3706 operand_subword_force (op1
, i
, mode
),
3707 immed_wide_int_const (mask
, imode
),
3708 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3710 temp
= expand_binop (imode
, ior_optab
, op0_piece
, op1
,
3711 targ_piece
, 1, OPTAB_LIB_WIDEN
);
3712 if (temp
!= targ_piece
)
3713 emit_move_insn (targ_piece
, temp
);
3716 emit_move_insn (targ_piece
, op0_piece
);
3719 insns
= get_insns ();
3726 op1
= expand_binop (imode
, and_optab
, gen_lowpart (imode
, op1
),
3727 immed_wide_int_const (mask
, imode
),
3728 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3730 op0
= gen_lowpart (imode
, op0
);
3732 op0
= expand_binop (imode
, and_optab
, op0
,
3733 immed_wide_int_const (~mask
, imode
),
3734 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3736 temp
= expand_binop (imode
, ior_optab
, op0
, op1
,
3737 gen_lowpart (imode
, target
), 1, OPTAB_LIB_WIDEN
);
3738 target
= lowpart_subreg_maybe_copy (mode
, temp
, imode
);
3744 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3745 scalar floating point mode. Return NULL if we do not know how to
3746 expand the operation inline. */
3749 expand_copysign (rtx op0
, rtx op1
, rtx target
)
3751 machine_mode mode
= GET_MODE (op0
);
3752 const struct real_format
*fmt
;
3756 gcc_assert (SCALAR_FLOAT_MODE_P (mode
));
3757 gcc_assert (GET_MODE (op1
) == mode
);
3759 /* First try to do it with a special instruction. */
3760 temp
= expand_binop (mode
, copysign_optab
, op0
, op1
,
3761 target
, 0, OPTAB_DIRECT
);
3765 fmt
= REAL_MODE_FORMAT (mode
);
3766 if (fmt
== NULL
|| !fmt
->has_signed_zero
)
3770 if (CONST_DOUBLE_AS_FLOAT_P (op0
))
3772 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0
)))
3773 op0
= simplify_unary_operation (ABS
, mode
, op0
, mode
);
3777 if (fmt
->signbit_ro
>= 0
3778 && (CONST_DOUBLE_AS_FLOAT_P (op0
)
3779 || (optab_handler (neg_optab
, mode
) != CODE_FOR_nothing
3780 && optab_handler (abs_optab
, mode
) != CODE_FOR_nothing
)))
3782 temp
= expand_copysign_absneg (mode
, op0
, op1
, target
,
3783 fmt
->signbit_ro
, op0_is_abs
);
3788 if (fmt
->signbit_rw
< 0)
3790 return expand_copysign_bit (mode
, op0
, op1
, target
,
3791 fmt
->signbit_rw
, op0_is_abs
);
3794 /* Generate an instruction whose insn-code is INSN_CODE,
3795 with two operands: an output TARGET and an input OP0.
3796 TARGET *must* be nonzero, and the output is always stored there.
3797 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3798 the value that is stored into TARGET.
3800 Return false if expansion failed. */
3803 maybe_emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
,
3806 struct expand_operand ops
[2];
3809 create_output_operand (&ops
[0], target
, GET_MODE (target
));
3810 create_input_operand (&ops
[1], op0
, GET_MODE (op0
));
3811 pat
= maybe_gen_insn (icode
, 2, ops
);
3815 if (INSN_P (pat
) && NEXT_INSN (as_a
<rtx_insn
*> (pat
)) != NULL_RTX
3817 add_equal_note (as_a
<rtx_insn
*> (pat
), ops
[0].value
, code
, ops
[1].value
,
3822 if (ops
[0].value
!= target
)
3823 emit_move_insn (target
, ops
[0].value
);
3826 /* Generate an instruction whose insn-code is INSN_CODE,
3827 with two operands: an output TARGET and an input OP0.
3828 TARGET *must* be nonzero, and the output is always stored there.
3829 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3830 the value that is stored into TARGET. */
3833 emit_unop_insn (enum insn_code icode
, rtx target
, rtx op0
, enum rtx_code code
)
3835 bool ok
= maybe_emit_unop_insn (icode
, target
, op0
, code
);
3839 struct no_conflict_data
3842 rtx_insn
*first
, *insn
;
3846 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
3847 the currently examined clobber / store has to stay in the list of
3848 insns that constitute the actual libcall block. */
3850 no_conflict_move_test (rtx dest
, const_rtx set
, void *p0
)
3852 struct no_conflict_data
*p
= (struct no_conflict_data
*) p0
;
3854 /* If this inns directly contributes to setting the target, it must stay. */
3855 if (reg_overlap_mentioned_p (p
->target
, dest
))
3856 p
->must_stay
= true;
3857 /* If we haven't committed to keeping any other insns in the list yet,
3858 there is nothing more to check. */
3859 else if (p
->insn
== p
->first
)
3861 /* If this insn sets / clobbers a register that feeds one of the insns
3862 already in the list, this insn has to stay too. */
3863 else if (reg_overlap_mentioned_p (dest
, PATTERN (p
->first
))
3864 || (CALL_P (p
->first
) && (find_reg_fusage (p
->first
, USE
, dest
)))
3865 || reg_used_between_p (dest
, p
->first
, p
->insn
)
3866 /* Likewise if this insn depends on a register set by a previous
3867 insn in the list, or if it sets a result (presumably a hard
3868 register) that is set or clobbered by a previous insn.
3869 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3870 SET_DEST perform the former check on the address, and the latter
3871 check on the MEM. */
3872 || (GET_CODE (set
) == SET
3873 && (modified_in_p (SET_SRC (set
), p
->first
)
3874 || modified_in_p (SET_DEST (set
), p
->first
)
3875 || modified_between_p (SET_SRC (set
), p
->first
, p
->insn
)
3876 || modified_between_p (SET_DEST (set
), p
->first
, p
->insn
))))
3877 p
->must_stay
= true;
3881 /* Emit code to make a call to a constant function or a library call.
3883 INSNS is a list containing all insns emitted in the call.
3884 These insns leave the result in RESULT. Our block is to copy RESULT
3885 to TARGET, which is logically equivalent to EQUIV.
3887 We first emit any insns that set a pseudo on the assumption that these are
3888 loading constants into registers; doing so allows them to be safely cse'ed
3889 between blocks. Then we emit all the other insns in the block, followed by
3890 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3891 note with an operand of EQUIV. */
3894 emit_libcall_block_1 (rtx_insn
*insns
, rtx target
, rtx result
, rtx equiv
,
3895 bool equiv_may_trap
)
3897 rtx final_dest
= target
;
3898 rtx_insn
*next
, *last
, *insn
;
3900 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3901 into a MEM later. Protect the libcall block from this change. */
3902 if (! REG_P (target
) || REG_USERVAR_P (target
))
3903 target
= gen_reg_rtx (GET_MODE (target
));
3905 /* If we're using non-call exceptions, a libcall corresponding to an
3906 operation that may trap may also trap. */
3907 /* ??? See the comment in front of make_reg_eh_region_note. */
3908 if (cfun
->can_throw_non_call_exceptions
3909 && (equiv_may_trap
|| may_trap_p (equiv
)))
3911 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
3914 rtx note
= find_reg_note (insn
, REG_EH_REGION
, NULL_RTX
);
3917 int lp_nr
= INTVAL (XEXP (note
, 0));
3918 if (lp_nr
== 0 || lp_nr
== INT_MIN
)
3919 remove_note (insn
, note
);
3925 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3926 reg note to indicate that this call cannot throw or execute a nonlocal
3927 goto (unless there is already a REG_EH_REGION note, in which case
3929 for (insn
= insns
; insn
; insn
= NEXT_INSN (insn
))
3931 make_reg_eh_region_note_nothrow_nononlocal (insn
);
3934 /* First emit all insns that set pseudos. Remove them from the list as
3935 we go. Avoid insns that set pseudos which were referenced in previous
3936 insns. These can be generated by move_by_pieces, for example,
3937 to update an address. Similarly, avoid insns that reference things
3938 set in previous insns. */
3940 for (insn
= insns
; insn
; insn
= next
)
3942 rtx set
= single_set (insn
);
3944 next
= NEXT_INSN (insn
);
3946 if (set
!= 0 && REG_P (SET_DEST (set
))
3947 && REGNO (SET_DEST (set
)) >= FIRST_PSEUDO_REGISTER
)
3949 struct no_conflict_data data
;
3951 data
.target
= const0_rtx
;
3955 note_stores (PATTERN (insn
), no_conflict_move_test
, &data
);
3956 if (! data
.must_stay
)
3958 if (PREV_INSN (insn
))
3959 SET_NEXT_INSN (PREV_INSN (insn
)) = next
;
3964 SET_PREV_INSN (next
) = PREV_INSN (insn
);
3970 /* Some ports use a loop to copy large arguments onto the stack.
3971 Don't move anything outside such a loop. */
3976 /* Write the remaining insns followed by the final copy. */
3977 for (insn
= insns
; insn
; insn
= next
)
3979 next
= NEXT_INSN (insn
);
3984 last
= emit_move_insn (target
, result
);
3985 set_dst_reg_note (last
, REG_EQUAL
, copy_rtx (equiv
), target
);
3987 if (final_dest
!= target
)
3988 emit_move_insn (final_dest
, target
);
3992 emit_libcall_block (rtx insns
, rtx target
, rtx result
, rtx equiv
)
3994 emit_libcall_block_1 (safe_as_a
<rtx_insn
*> (insns
),
3995 target
, result
, equiv
, false);
3998 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
3999 PURPOSE describes how this comparison will be used. CODE is the rtx
4000 comparison code we will be using.
4002 ??? Actually, CODE is slightly weaker than that. A target is still
4003 required to implement all of the normal bcc operations, but not
4004 required to implement all (or any) of the unordered bcc operations. */
4007 can_compare_p (enum rtx_code code
, machine_mode mode
,
4008 enum can_compare_purpose purpose
)
4011 test
= gen_rtx_fmt_ee (code
, mode
, const0_rtx
, const0_rtx
);
4014 enum insn_code icode
;
4016 if (purpose
== ccp_jump
4017 && (icode
= optab_handler (cbranch_optab
, mode
)) != CODE_FOR_nothing
4018 && insn_operand_matches (icode
, 0, test
))
4020 if (purpose
== ccp_store_flag
4021 && (icode
= optab_handler (cstore_optab
, mode
)) != CODE_FOR_nothing
4022 && insn_operand_matches (icode
, 1, test
))
4024 if (purpose
== ccp_cmov
4025 && optab_handler (cmov_optab
, mode
) != CODE_FOR_nothing
)
4028 mode
= GET_MODE_WIDER_MODE (mode
);
4029 PUT_MODE (test
, mode
);
4031 while (mode
!= VOIDmode
);
4036 /* This function is called when we are going to emit a compare instruction that
4037 compares the values found in *PX and *PY, using the rtl operator COMPARISON.
4039 *PMODE is the mode of the inputs (in case they are const_int).
4040 *PUNSIGNEDP nonzero says that the operands are unsigned;
4041 this matters if they need to be widened (as given by METHODS).
4043 If they have mode BLKmode, then SIZE specifies the size of both operands.
4045 This function performs all the setup necessary so that the caller only has
4046 to emit a single comparison insn. This setup can involve doing a BLKmode
4047 comparison or emitting a library call to perform the comparison if no insn
4048 is available to handle it.
4049 The values which are passed in through pointers can be modified; the caller
4050 should perform the comparison on the modified values. Constant
4051 comparisons must have already been folded. */
4054 prepare_cmp_insn (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4055 int unsignedp
, enum optab_methods methods
,
4056 rtx
*ptest
, machine_mode
*pmode
)
4058 machine_mode mode
= *pmode
;
4060 machine_mode cmp_mode
;
4061 enum mode_class mclass
;
4063 /* The other methods are not needed. */
4064 gcc_assert (methods
== OPTAB_DIRECT
|| methods
== OPTAB_WIDEN
4065 || methods
== OPTAB_LIB_WIDEN
);
4067 /* If we are optimizing, force expensive constants into a register. */
4068 if (CONSTANT_P (x
) && optimize
4069 && (rtx_cost (x
, COMPARE
, 0, optimize_insn_for_speed_p ())
4070 > COSTS_N_INSNS (1)))
4071 x
= force_reg (mode
, x
);
4073 if (CONSTANT_P (y
) && optimize
4074 && (rtx_cost (y
, COMPARE
, 1, optimize_insn_for_speed_p ())
4075 > COSTS_N_INSNS (1)))
4076 y
= force_reg (mode
, y
);
4079 /* Make sure if we have a canonical comparison. The RTL
4080 documentation states that canonical comparisons are required only
4081 for targets which have cc0. */
4082 gcc_assert (!CONSTANT_P (x
) || CONSTANT_P (y
));
4085 /* Don't let both operands fail to indicate the mode. */
4086 if (GET_MODE (x
) == VOIDmode
&& GET_MODE (y
) == VOIDmode
)
4087 x
= force_reg (mode
, x
);
4088 if (mode
== VOIDmode
)
4089 mode
= GET_MODE (x
) != VOIDmode
? GET_MODE (x
) : GET_MODE (y
);
4091 /* Handle all BLKmode compares. */
4093 if (mode
== BLKmode
)
4095 machine_mode result_mode
;
4096 enum insn_code cmp_code
;
4101 = GEN_INT (MIN (MEM_ALIGN (x
), MEM_ALIGN (y
)) / BITS_PER_UNIT
);
4105 /* Try to use a memory block compare insn - either cmpstr
4106 or cmpmem will do. */
4107 for (cmp_mode
= GET_CLASS_NARROWEST_MODE (MODE_INT
);
4108 cmp_mode
!= VOIDmode
;
4109 cmp_mode
= GET_MODE_WIDER_MODE (cmp_mode
))
4111 cmp_code
= direct_optab_handler (cmpmem_optab
, cmp_mode
);
4112 if (cmp_code
== CODE_FOR_nothing
)
4113 cmp_code
= direct_optab_handler (cmpstr_optab
, cmp_mode
);
4114 if (cmp_code
== CODE_FOR_nothing
)
4115 cmp_code
= direct_optab_handler (cmpstrn_optab
, cmp_mode
);
4116 if (cmp_code
== CODE_FOR_nothing
)
4119 /* Must make sure the size fits the insn's mode. */
4120 if ((CONST_INT_P (size
)
4121 && INTVAL (size
) >= (1 << GET_MODE_BITSIZE (cmp_mode
)))
4122 || (GET_MODE_BITSIZE (GET_MODE (size
))
4123 > GET_MODE_BITSIZE (cmp_mode
)))
4126 result_mode
= insn_data
[cmp_code
].operand
[0].mode
;
4127 result
= gen_reg_rtx (result_mode
);
4128 size
= convert_to_mode (cmp_mode
, size
, 1);
4129 emit_insn (GEN_FCN (cmp_code
) (result
, x
, y
, size
, opalign
));
4131 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, result
, const0_rtx
);
4132 *pmode
= result_mode
;
4136 if (methods
!= OPTAB_LIB
&& methods
!= OPTAB_LIB_WIDEN
)
4139 /* Otherwise call a library function, memcmp. */
4140 libfunc
= memcmp_libfunc
;
4141 length_type
= sizetype
;
4142 result_mode
= TYPE_MODE (integer_type_node
);
4143 cmp_mode
= TYPE_MODE (length_type
);
4144 size
= convert_to_mode (TYPE_MODE (length_type
), size
,
4145 TYPE_UNSIGNED (length_type
));
4147 result
= emit_library_call_value (libfunc
, 0, LCT_PURE
,
4155 methods
= OPTAB_LIB_WIDEN
;
4159 /* Don't allow operands to the compare to trap, as that can put the
4160 compare and branch in different basic blocks. */
4161 if (cfun
->can_throw_non_call_exceptions
)
4164 x
= force_reg (mode
, x
);
4166 y
= force_reg (mode
, y
);
4169 if (GET_MODE_CLASS (mode
) == MODE_CC
)
4171 enum insn_code icode
= optab_handler (cbranch_optab
, CCmode
);
4172 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
4173 gcc_assert (icode
!= CODE_FOR_nothing
4174 && insn_operand_matches (icode
, 0, test
));
4179 mclass
= GET_MODE_CLASS (mode
);
4180 test
= gen_rtx_fmt_ee (comparison
, VOIDmode
, x
, y
);
4184 enum insn_code icode
;
4185 icode
= optab_handler (cbranch_optab
, cmp_mode
);
4186 if (icode
!= CODE_FOR_nothing
4187 && insn_operand_matches (icode
, 0, test
))
4189 rtx_insn
*last
= get_last_insn ();
4190 rtx op0
= prepare_operand (icode
, x
, 1, mode
, cmp_mode
, unsignedp
);
4191 rtx op1
= prepare_operand (icode
, y
, 2, mode
, cmp_mode
, unsignedp
);
4193 && insn_operand_matches (icode
, 1, op0
)
4194 && insn_operand_matches (icode
, 2, op1
))
4196 XEXP (test
, 0) = op0
;
4197 XEXP (test
, 1) = op1
;
4202 delete_insns_since (last
);
4205 if (methods
== OPTAB_DIRECT
|| !CLASS_HAS_WIDER_MODES_P (mclass
))
4207 cmp_mode
= GET_MODE_WIDER_MODE (cmp_mode
);
4209 while (cmp_mode
!= VOIDmode
);
4211 if (methods
!= OPTAB_LIB_WIDEN
)
4214 if (!SCALAR_FLOAT_MODE_P (mode
))
4217 machine_mode ret_mode
;
4219 /* Handle a libcall just for the mode we are using. */
4220 libfunc
= optab_libfunc (cmp_optab
, mode
);
4221 gcc_assert (libfunc
);
4223 /* If we want unsigned, and this mode has a distinct unsigned
4224 comparison routine, use that. */
4227 rtx ulibfunc
= optab_libfunc (ucmp_optab
, mode
);
4232 ret_mode
= targetm
.libgcc_cmp_return_mode ();
4233 result
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4234 ret_mode
, 2, x
, mode
, y
, mode
);
4236 /* There are two kinds of comparison routines. Biased routines
4237 return 0/1/2, and unbiased routines return -1/0/1. Other parts
4238 of gcc expect that the comparison operation is equivalent
4239 to the modified comparison. For signed comparisons compare the
4240 result against 1 in the biased case, and zero in the unbiased
4241 case. For unsigned comparisons always compare against 1 after
4242 biasing the unbiased result by adding 1. This gives us a way to
4244 The comparisons in the fixed-point helper library are always
4249 if (!TARGET_LIB_INT_CMP_BIASED
&& !ALL_FIXED_POINT_MODE_P (mode
))
4252 x
= plus_constant (ret_mode
, result
, 1);
4258 prepare_cmp_insn (x
, y
, comparison
, NULL_RTX
, unsignedp
, methods
,
4262 prepare_float_lib_cmp (x
, y
, comparison
, ptest
, pmode
);
4270 /* Before emitting an insn with code ICODE, make sure that X, which is going
4271 to be used for operand OPNUM of the insn, is converted from mode MODE to
4272 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4273 that it is accepted by the operand predicate. Return the new value. */
4276 prepare_operand (enum insn_code icode
, rtx x
, int opnum
, machine_mode mode
,
4277 machine_mode wider_mode
, int unsignedp
)
4279 if (mode
!= wider_mode
)
4280 x
= convert_modes (wider_mode
, mode
, x
, unsignedp
);
4282 if (!insn_operand_matches (icode
, opnum
, x
))
4284 machine_mode op_mode
= insn_data
[(int) icode
].operand
[opnum
].mode
;
4285 if (reload_completed
)
4287 if (GET_MODE (x
) != op_mode
&& GET_MODE (x
) != VOIDmode
)
4289 x
= copy_to_mode_reg (op_mode
, x
);
4295 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4296 we can do the branch. */
4299 emit_cmp_and_jump_insn_1 (rtx test
, machine_mode mode
, rtx label
, int prob
)
4301 machine_mode optab_mode
;
4302 enum mode_class mclass
;
4303 enum insn_code icode
;
4306 mclass
= GET_MODE_CLASS (mode
);
4307 optab_mode
= (mclass
== MODE_CC
) ? CCmode
: mode
;
4308 icode
= optab_handler (cbranch_optab
, optab_mode
);
4310 gcc_assert (icode
!= CODE_FOR_nothing
);
4311 gcc_assert (insn_operand_matches (icode
, 0, test
));
4312 insn
= emit_jump_insn (GEN_FCN (icode
) (test
, XEXP (test
, 0),
4313 XEXP (test
, 1), label
));
4315 && profile_status_for_fn (cfun
) != PROFILE_ABSENT
4318 && any_condjump_p (insn
)
4319 && !find_reg_note (insn
, REG_BR_PROB
, 0))
4320 add_int_reg_note (insn
, REG_BR_PROB
, prob
);
4323 /* Generate code to compare X with Y so that the condition codes are
4324 set and to jump to LABEL if the condition is true. If X is a
4325 constant and Y is not a constant, then the comparison is swapped to
4326 ensure that the comparison RTL has the canonical form.
4328 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4329 need to be widened. UNSIGNEDP is also used to select the proper
4330 branch condition code.
4332 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4334 MODE is the mode of the inputs (in case they are const_int).
4336 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4337 It will be potentially converted into an unsigned variant based on
4338 UNSIGNEDP to select a proper jump instruction.
4340 PROB is the probability of jumping to LABEL. */
4343 emit_cmp_and_jump_insns (rtx x
, rtx y
, enum rtx_code comparison
, rtx size
,
4344 machine_mode mode
, int unsignedp
, rtx label
,
4347 rtx op0
= x
, op1
= y
;
4350 /* Swap operands and condition to ensure canonical RTL. */
4351 if (swap_commutative_operands_p (x
, y
)
4352 && can_compare_p (swap_condition (comparison
), mode
, ccp_jump
))
4355 comparison
= swap_condition (comparison
);
4358 /* If OP0 is still a constant, then both X and Y must be constants
4359 or the opposite comparison is not supported. Force X into a register
4360 to create canonical RTL. */
4361 if (CONSTANT_P (op0
))
4362 op0
= force_reg (mode
, op0
);
4365 comparison
= unsigned_condition (comparison
);
4367 prepare_cmp_insn (op0
, op1
, comparison
, size
, unsignedp
, OPTAB_LIB_WIDEN
,
4369 emit_cmp_and_jump_insn_1 (test
, mode
, label
, prob
);
4373 /* Emit a library call comparison between floating point X and Y.
4374 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4377 prepare_float_lib_cmp (rtx x
, rtx y
, enum rtx_code comparison
,
4378 rtx
*ptest
, machine_mode
*pmode
)
4380 enum rtx_code swapped
= swap_condition (comparison
);
4381 enum rtx_code reversed
= reverse_condition_maybe_unordered (comparison
);
4382 machine_mode orig_mode
= GET_MODE (x
);
4383 machine_mode mode
, cmp_mode
;
4384 rtx true_rtx
, false_rtx
;
4385 rtx value
, target
, equiv
;
4388 bool reversed_p
= false;
4389 cmp_mode
= targetm
.libgcc_cmp_return_mode ();
4391 for (mode
= orig_mode
;
4393 mode
= GET_MODE_WIDER_MODE (mode
))
4395 if (code_to_optab (comparison
)
4396 && (libfunc
= optab_libfunc (code_to_optab (comparison
), mode
)))
4399 if (code_to_optab (swapped
)
4400 && (libfunc
= optab_libfunc (code_to_optab (swapped
), mode
)))
4403 tmp
= x
; x
= y
; y
= tmp
;
4404 comparison
= swapped
;
4408 if (code_to_optab (reversed
)
4409 && (libfunc
= optab_libfunc (code_to_optab (reversed
), mode
)))
4411 comparison
= reversed
;
4417 gcc_assert (mode
!= VOIDmode
);
4419 if (mode
!= orig_mode
)
4421 x
= convert_to_mode (mode
, x
, 0);
4422 y
= convert_to_mode (mode
, y
, 0);
4425 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4426 the RTL. The allows the RTL optimizers to delete the libcall if the
4427 condition can be determined at compile-time. */
4428 if (comparison
== UNORDERED
4429 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4431 true_rtx
= const_true_rtx
;
4432 false_rtx
= const0_rtx
;
4439 true_rtx
= const0_rtx
;
4440 false_rtx
= const_true_rtx
;
4444 true_rtx
= const_true_rtx
;
4445 false_rtx
= const0_rtx
;
4449 true_rtx
= const1_rtx
;
4450 false_rtx
= const0_rtx
;
4454 true_rtx
= const0_rtx
;
4455 false_rtx
= constm1_rtx
;
4459 true_rtx
= constm1_rtx
;
4460 false_rtx
= const0_rtx
;
4464 true_rtx
= const0_rtx
;
4465 false_rtx
= const1_rtx
;
4473 if (comparison
== UNORDERED
)
4475 rtx temp
= simplify_gen_relational (NE
, cmp_mode
, mode
, x
, x
);
4476 equiv
= simplify_gen_relational (NE
, cmp_mode
, mode
, y
, y
);
4477 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4478 temp
, const_true_rtx
, equiv
);
4482 equiv
= simplify_gen_relational (comparison
, cmp_mode
, mode
, x
, y
);
4483 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
))
4484 equiv
= simplify_gen_ternary (IF_THEN_ELSE
, cmp_mode
, cmp_mode
,
4485 equiv
, true_rtx
, false_rtx
);
4489 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
4490 cmp_mode
, 2, x
, mode
, y
, mode
);
4491 insns
= get_insns ();
4494 target
= gen_reg_rtx (cmp_mode
);
4495 emit_libcall_block (insns
, target
, value
, equiv
);
4497 if (comparison
== UNORDERED
4498 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode
, comparison
)
4500 *ptest
= gen_rtx_fmt_ee (reversed_p
? EQ
: NE
, VOIDmode
, target
, false_rtx
);
4502 *ptest
= gen_rtx_fmt_ee (comparison
, VOIDmode
, target
, const0_rtx
);
4507 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4510 emit_indirect_jump (rtx loc ATTRIBUTE_UNUSED
)
4512 #ifndef HAVE_indirect_jump
4513 sorry ("indirect jumps are not available on this target");
4515 struct expand_operand ops
[1];
4516 create_address_operand (&ops
[0], loc
);
4517 expand_jump_insn (CODE_FOR_indirect_jump
, 1, ops
);
4522 #ifdef HAVE_conditional_move
4524 /* Emit a conditional move instruction if the machine supports one for that
4525 condition and machine mode.
4527 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4528 the mode to use should they be constants. If it is VOIDmode, they cannot
4531 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4532 should be stored there. MODE is the mode to use should they be constants.
4533 If it is VOIDmode, they cannot both be constants.
4535 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4536 is not supported. */
4539 emit_conditional_move (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
4540 machine_mode cmode
, rtx op2
, rtx op3
,
4541 machine_mode mode
, int unsignedp
)
4545 enum insn_code icode
;
4546 enum rtx_code reversed
;
4548 /* If one operand is constant, make it the second one. Only do this
4549 if the other operand is not constant as well. */
4551 if (swap_commutative_operands_p (op0
, op1
))
4553 std::swap (op0
, op1
);
4554 code
= swap_condition (code
);
4557 /* get_condition will prefer to generate LT and GT even if the old
4558 comparison was against zero, so undo that canonicalization here since
4559 comparisons against zero are cheaper. */
4560 if (code
== LT
&& op1
== const1_rtx
)
4561 code
= LE
, op1
= const0_rtx
;
4562 else if (code
== GT
&& op1
== constm1_rtx
)
4563 code
= GE
, op1
= const0_rtx
;
4565 if (cmode
== VOIDmode
)
4566 cmode
= GET_MODE (op0
);
4568 if (swap_commutative_operands_p (op2
, op3
)
4569 && ((reversed
= reversed_comparison_code_parts (code
, op0
, op1
, NULL
))
4572 std::swap (op2
, op3
);
4576 if (mode
== VOIDmode
)
4577 mode
= GET_MODE (op2
);
4579 icode
= direct_optab_handler (movcc_optab
, mode
);
4581 if (icode
== CODE_FOR_nothing
)
4585 target
= gen_reg_rtx (mode
);
4587 code
= unsignedp
? unsigned_condition (code
) : code
;
4588 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
4590 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4591 return NULL and let the caller figure out how best to deal with this
4593 if (!COMPARISON_P (comparison
))
4596 saved_pending_stack_adjust save
;
4597 save_pending_stack_adjust (&save
);
4598 last
= get_last_insn ();
4599 do_pending_stack_adjust ();
4600 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4601 GET_CODE (comparison
), NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
4602 &comparison
, &cmode
);
4605 struct expand_operand ops
[4];
4607 create_output_operand (&ops
[0], target
, mode
);
4608 create_fixed_operand (&ops
[1], comparison
);
4609 create_input_operand (&ops
[2], op2
, mode
);
4610 create_input_operand (&ops
[3], op3
, mode
);
4611 if (maybe_expand_insn (icode
, 4, ops
))
4613 if (ops
[0].value
!= target
)
4614 convert_move (target
, ops
[0].value
, false);
4618 delete_insns_since (last
);
4619 restore_pending_stack_adjust (&save
);
4623 /* Return nonzero if a conditional move of mode MODE is supported.
4625 This function is for combine so it can tell whether an insn that looks
4626 like a conditional move is actually supported by the hardware. If we
4627 guess wrong we lose a bit on optimization, but that's it. */
4628 /* ??? sparc64 supports conditionally moving integers values based on fp
4629 comparisons, and vice versa. How do we handle them? */
4632 can_conditionally_move_p (machine_mode mode
)
4634 if (direct_optab_handler (movcc_optab
, mode
) != CODE_FOR_nothing
)
4640 #endif /* HAVE_conditional_move */
4642 /* Emit a conditional addition instruction if the machine supports one for that
4643 condition and machine mode.
4645 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4646 the mode to use should they be constants. If it is VOIDmode, they cannot
4649 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
4650 should be stored there. MODE is the mode to use should they be constants.
4651 If it is VOIDmode, they cannot both be constants.
4653 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4654 is not supported. */
4657 emit_conditional_add (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
4658 machine_mode cmode
, rtx op2
, rtx op3
,
4659 machine_mode mode
, int unsignedp
)
4663 enum insn_code icode
;
4665 /* If one operand is constant, make it the second one. Only do this
4666 if the other operand is not constant as well. */
4668 if (swap_commutative_operands_p (op0
, op1
))
4670 std::swap (op0
, op1
);
4671 code
= swap_condition (code
);
4674 /* get_condition will prefer to generate LT and GT even if the old
4675 comparison was against zero, so undo that canonicalization here since
4676 comparisons against zero are cheaper. */
4677 if (code
== LT
&& op1
== const1_rtx
)
4678 code
= LE
, op1
= const0_rtx
;
4679 else if (code
== GT
&& op1
== constm1_rtx
)
4680 code
= GE
, op1
= const0_rtx
;
4682 if (cmode
== VOIDmode
)
4683 cmode
= GET_MODE (op0
);
4685 if (mode
== VOIDmode
)
4686 mode
= GET_MODE (op2
);
4688 icode
= optab_handler (addcc_optab
, mode
);
4690 if (icode
== CODE_FOR_nothing
)
4694 target
= gen_reg_rtx (mode
);
4696 code
= unsignedp
? unsigned_condition (code
) : code
;
4697 comparison
= simplify_gen_relational (code
, VOIDmode
, cmode
, op0
, op1
);
4699 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4700 return NULL and let the caller figure out how best to deal with this
4702 if (!COMPARISON_P (comparison
))
4705 do_pending_stack_adjust ();
4706 last
= get_last_insn ();
4707 prepare_cmp_insn (XEXP (comparison
, 0), XEXP (comparison
, 1),
4708 GET_CODE (comparison
), NULL_RTX
, unsignedp
, OPTAB_WIDEN
,
4709 &comparison
, &cmode
);
4712 struct expand_operand ops
[4];
4714 create_output_operand (&ops
[0], target
, mode
);
4715 create_fixed_operand (&ops
[1], comparison
);
4716 create_input_operand (&ops
[2], op2
, mode
);
4717 create_input_operand (&ops
[3], op3
, mode
);
4718 if (maybe_expand_insn (icode
, 4, ops
))
4720 if (ops
[0].value
!= target
)
4721 convert_move (target
, ops
[0].value
, false);
4725 delete_insns_since (last
);
4729 /* These functions attempt to generate an insn body, rather than
4730 emitting the insn, but if the gen function already emits them, we
4731 make no attempt to turn them back into naked patterns. */
4733 /* Generate and return an insn body to add Y to X. */
4736 gen_add2_insn (rtx x
, rtx y
)
4738 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (x
));
4740 gcc_assert (insn_operand_matches (icode
, 0, x
));
4741 gcc_assert (insn_operand_matches (icode
, 1, x
));
4742 gcc_assert (insn_operand_matches (icode
, 2, y
));
4744 return GEN_FCN (icode
) (x
, x
, y
);
4747 /* Generate and return an insn body to add r1 and c,
4748 storing the result in r0. */
4751 gen_add3_insn (rtx r0
, rtx r1
, rtx c
)
4753 enum insn_code icode
= optab_handler (add_optab
, GET_MODE (r0
));
4755 if (icode
== CODE_FOR_nothing
4756 || !insn_operand_matches (icode
, 0, r0
)
4757 || !insn_operand_matches (icode
, 1, r1
)
4758 || !insn_operand_matches (icode
, 2, c
))
4761 return GEN_FCN (icode
) (r0
, r1
, c
);
4765 have_add2_insn (rtx x
, rtx y
)
4767 enum insn_code icode
;
4769 gcc_assert (GET_MODE (x
) != VOIDmode
);
4771 icode
= optab_handler (add_optab
, GET_MODE (x
));
4773 if (icode
== CODE_FOR_nothing
)
4776 if (!insn_operand_matches (icode
, 0, x
)
4777 || !insn_operand_matches (icode
, 1, x
)
4778 || !insn_operand_matches (icode
, 2, y
))
4784 /* Generate and return an insn body to add Y to X. */
4787 gen_addptr3_insn (rtx x
, rtx y
, rtx z
)
4789 enum insn_code icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
4791 gcc_assert (insn_operand_matches (icode
, 0, x
));
4792 gcc_assert (insn_operand_matches (icode
, 1, y
));
4793 gcc_assert (insn_operand_matches (icode
, 2, z
));
4795 return GEN_FCN (icode
) (x
, y
, z
);
4798 /* Return true if the target implements an addptr pattern and X, Y,
4799 and Z are valid for the pattern predicates. */
4802 have_addptr3_insn (rtx x
, rtx y
, rtx z
)
4804 enum insn_code icode
;
4806 gcc_assert (GET_MODE (x
) != VOIDmode
);
4808 icode
= optab_handler (addptr3_optab
, GET_MODE (x
));
4810 if (icode
== CODE_FOR_nothing
)
4813 if (!insn_operand_matches (icode
, 0, x
)
4814 || !insn_operand_matches (icode
, 1, y
)
4815 || !insn_operand_matches (icode
, 2, z
))
4821 /* Generate and return an insn body to subtract Y from X. */
4824 gen_sub2_insn (rtx x
, rtx y
)
4826 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (x
));
4828 gcc_assert (insn_operand_matches (icode
, 0, x
));
4829 gcc_assert (insn_operand_matches (icode
, 1, x
));
4830 gcc_assert (insn_operand_matches (icode
, 2, y
));
4832 return GEN_FCN (icode
) (x
, x
, y
);
4835 /* Generate and return an insn body to subtract r1 and c,
4836 storing the result in r0. */
4839 gen_sub3_insn (rtx r0
, rtx r1
, rtx c
)
4841 enum insn_code icode
= optab_handler (sub_optab
, GET_MODE (r0
));
4843 if (icode
== CODE_FOR_nothing
4844 || !insn_operand_matches (icode
, 0, r0
)
4845 || !insn_operand_matches (icode
, 1, r1
)
4846 || !insn_operand_matches (icode
, 2, c
))
4849 return GEN_FCN (icode
) (r0
, r1
, c
);
4853 have_sub2_insn (rtx x
, rtx y
)
4855 enum insn_code icode
;
4857 gcc_assert (GET_MODE (x
) != VOIDmode
);
4859 icode
= optab_handler (sub_optab
, GET_MODE (x
));
4861 if (icode
== CODE_FOR_nothing
)
4864 if (!insn_operand_matches (icode
, 0, x
)
4865 || !insn_operand_matches (icode
, 1, x
)
4866 || !insn_operand_matches (icode
, 2, y
))
4872 /* Return the insn code used to extend FROM_MODE to TO_MODE.
4873 UNSIGNEDP specifies zero-extension instead of sign-extension. If
4874 no such operation exists, CODE_FOR_nothing will be returned. */
4877 can_extend_p (machine_mode to_mode
, machine_mode from_mode
,
4881 #ifdef HAVE_ptr_extend
4883 return CODE_FOR_ptr_extend
;
4886 tab
= unsignedp
? zext_optab
: sext_optab
;
4887 return convert_optab_handler (tab
, to_mode
, from_mode
);
4890 /* Generate the body of an insn to extend Y (with mode MFROM)
4891 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
4894 gen_extend_insn (rtx x
, rtx y
, machine_mode mto
,
4895 machine_mode mfrom
, int unsignedp
)
4897 enum insn_code icode
= can_extend_p (mto
, mfrom
, unsignedp
);
4898 return GEN_FCN (icode
) (x
, y
);
4901 /* can_fix_p and can_float_p say whether the target machine
4902 can directly convert a given fixed point type to
4903 a given floating point type, or vice versa.
4904 The returned value is the CODE_FOR_... value to use,
4905 or CODE_FOR_nothing if these modes cannot be directly converted.
4907 *TRUNCP_PTR is set to 1 if it is necessary to output
4908 an explicit FTRUNC insn before the fix insn; otherwise 0. */
4910 static enum insn_code
4911 can_fix_p (machine_mode fixmode
, machine_mode fltmode
,
4912 int unsignedp
, int *truncp_ptr
)
4915 enum insn_code icode
;
4917 tab
= unsignedp
? ufixtrunc_optab
: sfixtrunc_optab
;
4918 icode
= convert_optab_handler (tab
, fixmode
, fltmode
);
4919 if (icode
!= CODE_FOR_nothing
)
4925 /* FIXME: This requires a port to define both FIX and FTRUNC pattern
4926 for this to work. We need to rework the fix* and ftrunc* patterns
4927 and documentation. */
4928 tab
= unsignedp
? ufix_optab
: sfix_optab
;
4929 icode
= convert_optab_handler (tab
, fixmode
, fltmode
);
4930 if (icode
!= CODE_FOR_nothing
4931 && optab_handler (ftrunc_optab
, fltmode
) != CODE_FOR_nothing
)
4938 return CODE_FOR_nothing
;
4942 can_float_p (machine_mode fltmode
, machine_mode fixmode
,
4947 tab
= unsignedp
? ufloat_optab
: sfloat_optab
;
4948 return convert_optab_handler (tab
, fltmode
, fixmode
);
4951 /* Function supportable_convert_operation
4953 Check whether an operation represented by the code CODE is a
4954 convert operation that is supported by the target platform in
4955 vector form (i.e., when operating on arguments of type VECTYPE_IN
4956 producing a result of type VECTYPE_OUT).
4958 Convert operations we currently support directly are FIX_TRUNC and FLOAT.
4959 This function checks if these operations are supported
4960 by the target platform either directly (via vector tree-codes), or via
4964 - CODE1 is code of vector operation to be used when
4965 vectorizing the operation, if available.
4966 - DECL is decl of target builtin functions to be used
4967 when vectorizing the operation, if available. In this case,
4968 CODE1 is CALL_EXPR. */
4971 supportable_convert_operation (enum tree_code code
,
4972 tree vectype_out
, tree vectype_in
,
4973 tree
*decl
, enum tree_code
*code1
)
4978 m1
= TYPE_MODE (vectype_out
);
4979 m2
= TYPE_MODE (vectype_in
);
4981 /* First check if we can done conversion directly. */
4982 if ((code
== FIX_TRUNC_EXPR
4983 && can_fix_p (m1
,m2
,TYPE_UNSIGNED (vectype_out
), &truncp
)
4984 != CODE_FOR_nothing
)
4985 || (code
== FLOAT_EXPR
4986 && can_float_p (m1
,m2
,TYPE_UNSIGNED (vectype_in
))
4987 != CODE_FOR_nothing
))
4993 /* Now check for builtin. */
4994 if (targetm
.vectorize
.builtin_conversion
4995 && targetm
.vectorize
.builtin_conversion (code
, vectype_out
, vectype_in
))
4998 *decl
= targetm
.vectorize
.builtin_conversion (code
, vectype_out
, vectype_in
);
5005 /* Generate code to convert FROM to floating point
5006 and store in TO. FROM must be fixed point and not VOIDmode.
5007 UNSIGNEDP nonzero means regard FROM as unsigned.
5008 Normally this is done by correcting the final value
5009 if it is negative. */
5012 expand_float (rtx to
, rtx from
, int unsignedp
)
5014 enum insn_code icode
;
5016 machine_mode fmode
, imode
;
5017 bool can_do_signed
= false;
5019 /* Crash now, because we won't be able to decide which mode to use. */
5020 gcc_assert (GET_MODE (from
) != VOIDmode
);
5022 /* Look for an insn to do the conversion. Do it in the specified
5023 modes if possible; otherwise convert either input, output or both to
5024 wider mode. If the integer mode is wider than the mode of FROM,
5025 we can do the conversion signed even if the input is unsigned. */
5027 for (fmode
= GET_MODE (to
); fmode
!= VOIDmode
;
5028 fmode
= GET_MODE_WIDER_MODE (fmode
))
5029 for (imode
= GET_MODE (from
); imode
!= VOIDmode
;
5030 imode
= GET_MODE_WIDER_MODE (imode
))
5032 int doing_unsigned
= unsignedp
;
5034 if (fmode
!= GET_MODE (to
)
5035 && significand_size (fmode
) < GET_MODE_PRECISION (GET_MODE (from
)))
5038 icode
= can_float_p (fmode
, imode
, unsignedp
);
5039 if (icode
== CODE_FOR_nothing
&& unsignedp
)
5041 enum insn_code scode
= can_float_p (fmode
, imode
, 0);
5042 if (scode
!= CODE_FOR_nothing
)
5043 can_do_signed
= true;
5044 if (imode
!= GET_MODE (from
))
5045 icode
= scode
, doing_unsigned
= 0;
5048 if (icode
!= CODE_FOR_nothing
)
5050 if (imode
!= GET_MODE (from
))
5051 from
= convert_to_mode (imode
, from
, unsignedp
);
5053 if (fmode
!= GET_MODE (to
))
5054 target
= gen_reg_rtx (fmode
);
5056 emit_unop_insn (icode
, target
, from
,
5057 doing_unsigned
? UNSIGNED_FLOAT
: FLOAT
);
5060 convert_move (to
, target
, 0);
5065 /* Unsigned integer, and no way to convert directly. Convert as signed,
5066 then unconditionally adjust the result. */
5067 if (unsignedp
&& can_do_signed
)
5069 rtx_code_label
*label
= gen_label_rtx ();
5071 REAL_VALUE_TYPE offset
;
5073 /* Look for a usable floating mode FMODE wider than the source and at
5074 least as wide as the target. Using FMODE will avoid rounding woes
5075 with unsigned values greater than the signed maximum value. */
5077 for (fmode
= GET_MODE (to
); fmode
!= VOIDmode
;
5078 fmode
= GET_MODE_WIDER_MODE (fmode
))
5079 if (GET_MODE_PRECISION (GET_MODE (from
)) < GET_MODE_BITSIZE (fmode
)
5080 && can_float_p (fmode
, GET_MODE (from
), 0) != CODE_FOR_nothing
)
5083 if (fmode
== VOIDmode
)
5085 /* There is no such mode. Pretend the target is wide enough. */
5086 fmode
= GET_MODE (to
);
5088 /* Avoid double-rounding when TO is narrower than FROM. */
5089 if ((significand_size (fmode
) + 1)
5090 < GET_MODE_PRECISION (GET_MODE (from
)))
5093 rtx_code_label
*neglabel
= gen_label_rtx ();
5095 /* Don't use TARGET if it isn't a register, is a hard register,
5096 or is the wrong mode. */
5098 || REGNO (target
) < FIRST_PSEUDO_REGISTER
5099 || GET_MODE (target
) != fmode
)
5100 target
= gen_reg_rtx (fmode
);
5102 imode
= GET_MODE (from
);
5103 do_pending_stack_adjust ();
5105 /* Test whether the sign bit is set. */
5106 emit_cmp_and_jump_insns (from
, const0_rtx
, LT
, NULL_RTX
, imode
,
5109 /* The sign bit is not set. Convert as signed. */
5110 expand_float (target
, from
, 0);
5111 emit_jump_insn (gen_jump (label
));
5114 /* The sign bit is set.
5115 Convert to a usable (positive signed) value by shifting right
5116 one bit, while remembering if a nonzero bit was shifted
5117 out; i.e., compute (from & 1) | (from >> 1). */
5119 emit_label (neglabel
);
5120 temp
= expand_binop (imode
, and_optab
, from
, const1_rtx
,
5121 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
5122 temp1
= expand_shift (RSHIFT_EXPR
, imode
, from
, 1, NULL_RTX
, 1);
5123 temp
= expand_binop (imode
, ior_optab
, temp
, temp1
, temp
, 1,
5125 expand_float (target
, temp
, 0);
5127 /* Multiply by 2 to undo the shift above. */
5128 temp
= expand_binop (fmode
, add_optab
, target
, target
,
5129 target
, 0, OPTAB_LIB_WIDEN
);
5131 emit_move_insn (target
, temp
);
5133 do_pending_stack_adjust ();
5139 /* If we are about to do some arithmetic to correct for an
5140 unsigned operand, do it in a pseudo-register. */
5142 if (GET_MODE (to
) != fmode
5143 || !REG_P (to
) || REGNO (to
) < FIRST_PSEUDO_REGISTER
)
5144 target
= gen_reg_rtx (fmode
);
5146 /* Convert as signed integer to floating. */
5147 expand_float (target
, from
, 0);
5149 /* If FROM is negative (and therefore TO is negative),
5150 correct its value by 2**bitwidth. */
5152 do_pending_stack_adjust ();
5153 emit_cmp_and_jump_insns (from
, const0_rtx
, GE
, NULL_RTX
, GET_MODE (from
),
5157 real_2expN (&offset
, GET_MODE_PRECISION (GET_MODE (from
)), fmode
);
5158 temp
= expand_binop (fmode
, add_optab
, target
,
5159 CONST_DOUBLE_FROM_REAL_VALUE (offset
, fmode
),
5160 target
, 0, OPTAB_LIB_WIDEN
);
5162 emit_move_insn (target
, temp
);
5164 do_pending_stack_adjust ();
5169 /* No hardware instruction available; call a library routine. */
5174 convert_optab tab
= unsignedp
? ufloat_optab
: sfloat_optab
;
5176 if (GET_MODE_PRECISION (GET_MODE (from
)) < GET_MODE_PRECISION (SImode
))
5177 from
= convert_to_mode (SImode
, from
, unsignedp
);
5179 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
5180 gcc_assert (libfunc
);
5184 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
5185 GET_MODE (to
), 1, from
,
5187 insns
= get_insns ();
5190 emit_libcall_block (insns
, target
, value
,
5191 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FLOAT
: FLOAT
,
5192 GET_MODE (to
), from
));
5197 /* Copy result to requested destination
5198 if we have been computing in a temp location. */
5202 if (GET_MODE (target
) == GET_MODE (to
))
5203 emit_move_insn (to
, target
);
5205 convert_move (to
, target
, 0);
5209 /* Generate code to convert FROM to fixed point and store in TO. FROM
5210 must be floating point. */
5213 expand_fix (rtx to
, rtx from
, int unsignedp
)
5215 enum insn_code icode
;
5217 machine_mode fmode
, imode
;
5220 /* We first try to find a pair of modes, one real and one integer, at
5221 least as wide as FROM and TO, respectively, in which we can open-code
5222 this conversion. If the integer mode is wider than the mode of TO,
5223 we can do the conversion either signed or unsigned. */
5225 for (fmode
= GET_MODE (from
); fmode
!= VOIDmode
;
5226 fmode
= GET_MODE_WIDER_MODE (fmode
))
5227 for (imode
= GET_MODE (to
); imode
!= VOIDmode
;
5228 imode
= GET_MODE_WIDER_MODE (imode
))
5230 int doing_unsigned
= unsignedp
;
5232 icode
= can_fix_p (imode
, fmode
, unsignedp
, &must_trunc
);
5233 if (icode
== CODE_FOR_nothing
&& imode
!= GET_MODE (to
) && unsignedp
)
5234 icode
= can_fix_p (imode
, fmode
, 0, &must_trunc
), doing_unsigned
= 0;
5236 if (icode
!= CODE_FOR_nothing
)
5238 rtx_insn
*last
= get_last_insn ();
5239 if (fmode
!= GET_MODE (from
))
5240 from
= convert_to_mode (fmode
, from
, 0);
5244 rtx temp
= gen_reg_rtx (GET_MODE (from
));
5245 from
= expand_unop (GET_MODE (from
), ftrunc_optab
, from
,
5249 if (imode
!= GET_MODE (to
))
5250 target
= gen_reg_rtx (imode
);
5252 if (maybe_emit_unop_insn (icode
, target
, from
,
5253 doing_unsigned
? UNSIGNED_FIX
: FIX
))
5256 convert_move (to
, target
, unsignedp
);
5259 delete_insns_since (last
);
5263 /* For an unsigned conversion, there is one more way to do it.
5264 If we have a signed conversion, we generate code that compares
5265 the real value to the largest representable positive number. If if
5266 is smaller, the conversion is done normally. Otherwise, subtract
5267 one plus the highest signed number, convert, and add it back.
5269 We only need to check all real modes, since we know we didn't find
5270 anything with a wider integer mode.
5272 This code used to extend FP value into mode wider than the destination.
5273 This is needed for decimal float modes which cannot accurately
5274 represent one plus the highest signed number of the same size, but
5275 not for binary modes. Consider, for instance conversion from SFmode
5278 The hot path through the code is dealing with inputs smaller than 2^63
5279 and doing just the conversion, so there is no bits to lose.
5281 In the other path we know the value is positive in the range 2^63..2^64-1
5282 inclusive. (as for other input overflow happens and result is undefined)
5283 So we know that the most important bit set in mantissa corresponds to
5284 2^63. The subtraction of 2^63 should not generate any rounding as it
5285 simply clears out that bit. The rest is trivial. */
5287 if (unsignedp
&& GET_MODE_PRECISION (GET_MODE (to
)) <= HOST_BITS_PER_WIDE_INT
)
5288 for (fmode
= GET_MODE (from
); fmode
!= VOIDmode
;
5289 fmode
= GET_MODE_WIDER_MODE (fmode
))
5290 if (CODE_FOR_nothing
!= can_fix_p (GET_MODE (to
), fmode
, 0, &must_trunc
)
5291 && (!DECIMAL_FLOAT_MODE_P (fmode
)
5292 || GET_MODE_BITSIZE (fmode
) > GET_MODE_PRECISION (GET_MODE (to
))))
5295 REAL_VALUE_TYPE offset
;
5297 rtx_code_label
*lab1
, *lab2
;
5300 bitsize
= GET_MODE_PRECISION (GET_MODE (to
));
5301 real_2expN (&offset
, bitsize
- 1, fmode
);
5302 limit
= CONST_DOUBLE_FROM_REAL_VALUE (offset
, fmode
);
5303 lab1
= gen_label_rtx ();
5304 lab2
= gen_label_rtx ();
5306 if (fmode
!= GET_MODE (from
))
5307 from
= convert_to_mode (fmode
, from
, 0);
5309 /* See if we need to do the subtraction. */
5310 do_pending_stack_adjust ();
5311 emit_cmp_and_jump_insns (from
, limit
, GE
, NULL_RTX
, GET_MODE (from
),
5314 /* If not, do the signed "fix" and branch around fixup code. */
5315 expand_fix (to
, from
, 0);
5316 emit_jump_insn (gen_jump (lab2
));
5319 /* Otherwise, subtract 2**(N-1), convert to signed number,
5320 then add 2**(N-1). Do the addition using XOR since this
5321 will often generate better code. */
5323 target
= expand_binop (GET_MODE (from
), sub_optab
, from
, limit
,
5324 NULL_RTX
, 0, OPTAB_LIB_WIDEN
);
5325 expand_fix (to
, target
, 0);
5326 target
= expand_binop (GET_MODE (to
), xor_optab
, to
,
5328 ((HOST_WIDE_INT
) 1 << (bitsize
- 1),
5330 to
, 1, OPTAB_LIB_WIDEN
);
5333 emit_move_insn (to
, target
);
5337 if (optab_handler (mov_optab
, GET_MODE (to
)) != CODE_FOR_nothing
)
5339 /* Make a place for a REG_NOTE and add it. */
5340 insn
= emit_move_insn (to
, to
);
5341 set_dst_reg_note (insn
, REG_EQUAL
,
5342 gen_rtx_fmt_e (UNSIGNED_FIX
, GET_MODE (to
),
5350 /* We can't do it with an insn, so use a library call. But first ensure
5351 that the mode of TO is at least as wide as SImode, since those are the
5352 only library calls we know about. */
5354 if (GET_MODE_PRECISION (GET_MODE (to
)) < GET_MODE_PRECISION (SImode
))
5356 target
= gen_reg_rtx (SImode
);
5358 expand_fix (target
, from
, unsignedp
);
5366 convert_optab tab
= unsignedp
? ufix_optab
: sfix_optab
;
5367 libfunc
= convert_optab_libfunc (tab
, GET_MODE (to
), GET_MODE (from
));
5368 gcc_assert (libfunc
);
5372 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
,
5373 GET_MODE (to
), 1, from
,
5375 insns
= get_insns ();
5378 emit_libcall_block (insns
, target
, value
,
5379 gen_rtx_fmt_e (unsignedp
? UNSIGNED_FIX
: FIX
,
5380 GET_MODE (to
), from
));
5385 if (GET_MODE (to
) == GET_MODE (target
))
5386 emit_move_insn (to
, target
);
5388 convert_move (to
, target
, 0);
5392 /* Generate code to convert FROM or TO a fixed-point.
5393 If UINTP is true, either TO or FROM is an unsigned integer.
5394 If SATP is true, we need to saturate the result. */
5397 expand_fixed_convert (rtx to
, rtx from
, int uintp
, int satp
)
5399 machine_mode to_mode
= GET_MODE (to
);
5400 machine_mode from_mode
= GET_MODE (from
);
5402 enum rtx_code this_code
;
5403 enum insn_code code
;
5408 if (to_mode
== from_mode
)
5410 emit_move_insn (to
, from
);
5416 tab
= satp
? satfractuns_optab
: fractuns_optab
;
5417 this_code
= satp
? UNSIGNED_SAT_FRACT
: UNSIGNED_FRACT_CONVERT
;
5421 tab
= satp
? satfract_optab
: fract_optab
;
5422 this_code
= satp
? SAT_FRACT
: FRACT_CONVERT
;
5424 code
= convert_optab_handler (tab
, to_mode
, from_mode
);
5425 if (code
!= CODE_FOR_nothing
)
5427 emit_unop_insn (code
, to
, from
, this_code
);
5431 libfunc
= convert_optab_libfunc (tab
, to_mode
, from_mode
);
5432 gcc_assert (libfunc
);
5435 value
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_CONST
, to_mode
,
5436 1, from
, from_mode
);
5437 insns
= get_insns ();
5440 emit_libcall_block (insns
, to
, value
,
5441 gen_rtx_fmt_e (optab_to_code (tab
), to_mode
, from
));
5444 /* Generate code to convert FROM to fixed point and store in TO. FROM
5445 must be floating point, TO must be signed. Use the conversion optab
5446 TAB to do the conversion. */
5449 expand_sfix_optab (rtx to
, rtx from
, convert_optab tab
)
5451 enum insn_code icode
;
5453 machine_mode fmode
, imode
;
5455 /* We first try to find a pair of modes, one real and one integer, at
5456 least as wide as FROM and TO, respectively, in which we can open-code
5457 this conversion. If the integer mode is wider than the mode of TO,
5458 we can do the conversion either signed or unsigned. */
5460 for (fmode
= GET_MODE (from
); fmode
!= VOIDmode
;
5461 fmode
= GET_MODE_WIDER_MODE (fmode
))
5462 for (imode
= GET_MODE (to
); imode
!= VOIDmode
;
5463 imode
= GET_MODE_WIDER_MODE (imode
))
5465 icode
= convert_optab_handler (tab
, imode
, fmode
);
5466 if (icode
!= CODE_FOR_nothing
)
5468 rtx_insn
*last
= get_last_insn ();
5469 if (fmode
!= GET_MODE (from
))
5470 from
= convert_to_mode (fmode
, from
, 0);
5472 if (imode
!= GET_MODE (to
))
5473 target
= gen_reg_rtx (imode
);
5475 if (!maybe_emit_unop_insn (icode
, target
, from
, UNKNOWN
))
5477 delete_insns_since (last
);
5481 convert_move (to
, target
, 0);
5489 /* Report whether we have an instruction to perform the operation
5490 specified by CODE on operands of mode MODE. */
5492 have_insn_for (enum rtx_code code
, machine_mode mode
)
5494 return (code_to_optab (code
)
5495 && (optab_handler (code_to_optab (code
), mode
)
5496 != CODE_FOR_nothing
));
5499 /* Initialize the libfunc fields of an entire group of entries in some
5500 optab. Each entry is set equal to a string consisting of a leading
5501 pair of underscores followed by a generic operation name followed by
5502 a mode name (downshifted to lowercase) followed by a single character
5503 representing the number of operands for the given operation (which is
5504 usually one of the characters '2', '3', or '4').
5506 OPTABLE is the table in which libfunc fields are to be initialized.
5507 OPNAME is the generic (string) name of the operation.
5508 SUFFIX is the character which specifies the number of operands for
5509 the given generic operation.
5510 MODE is the mode to generate for.
5514 gen_libfunc (optab optable
, const char *opname
, int suffix
,
5517 unsigned opname_len
= strlen (opname
);
5518 const char *mname
= GET_MODE_NAME (mode
);
5519 unsigned mname_len
= strlen (mname
);
5520 int prefix_len
= targetm
.libfunc_gnu_prefix
? 6 : 2;
5521 int len
= prefix_len
+ opname_len
+ mname_len
+ 1 + 1;
5522 char *libfunc_name
= XALLOCAVEC (char, len
);
5529 if (targetm
.libfunc_gnu_prefix
)
5536 for (q
= opname
; *q
; )
5538 for (q
= mname
; *q
; q
++)
5539 *p
++ = TOLOWER (*q
);
5543 set_optab_libfunc (optable
, mode
,
5544 ggc_alloc_string (libfunc_name
, p
- libfunc_name
));
5547 /* Like gen_libfunc, but verify that integer operation is involved. */
5550 gen_int_libfunc (optab optable
, const char *opname
, char suffix
,
5553 int maxsize
= 2 * BITS_PER_WORD
;
5554 int minsize
= BITS_PER_WORD
;
5556 if (GET_MODE_CLASS (mode
) != MODE_INT
)
5558 if (maxsize
< LONG_LONG_TYPE_SIZE
)
5559 maxsize
= LONG_LONG_TYPE_SIZE
;
5560 if (minsize
> INT_TYPE_SIZE
5561 && (trapv_binoptab_p (optable
)
5562 || trapv_unoptab_p (optable
)))
5563 minsize
= INT_TYPE_SIZE
;
5564 if (GET_MODE_BITSIZE (mode
) < minsize
5565 || GET_MODE_BITSIZE (mode
) > maxsize
)
5567 gen_libfunc (optable
, opname
, suffix
, mode
);
5570 /* Like gen_libfunc, but verify that FP and set decimal prefix if needed. */
5573 gen_fp_libfunc (optab optable
, const char *opname
, char suffix
,
5578 if (GET_MODE_CLASS (mode
) == MODE_FLOAT
)
5579 gen_libfunc (optable
, opname
, suffix
, mode
);
5580 if (DECIMAL_FLOAT_MODE_P (mode
))
5582 dec_opname
= XALLOCAVEC (char, sizeof (DECIMAL_PREFIX
) + strlen (opname
));
5583 /* For BID support, change the name to have either a bid_ or dpd_ prefix
5584 depending on the low level floating format used. */
5585 memcpy (dec_opname
, DECIMAL_PREFIX
, sizeof (DECIMAL_PREFIX
) - 1);
5586 strcpy (dec_opname
+ sizeof (DECIMAL_PREFIX
) - 1, opname
);
5587 gen_libfunc (optable
, dec_opname
, suffix
, mode
);
5591 /* Like gen_libfunc, but verify that fixed-point operation is involved. */
5594 gen_fixed_libfunc (optab optable
, const char *opname
, char suffix
,
5597 if (!ALL_FIXED_POINT_MODE_P (mode
))
5599 gen_libfunc (optable
, opname
, suffix
, mode
);
5602 /* Like gen_libfunc, but verify that signed fixed-point operation is
5606 gen_signed_fixed_libfunc (optab optable
, const char *opname
, char suffix
,
5609 if (!SIGNED_FIXED_POINT_MODE_P (mode
))
5611 gen_libfunc (optable
, opname
, suffix
, mode
);
5614 /* Like gen_libfunc, but verify that unsigned fixed-point operation is
5618 gen_unsigned_fixed_libfunc (optab optable
, const char *opname
, char suffix
,
5621 if (!UNSIGNED_FIXED_POINT_MODE_P (mode
))
5623 gen_libfunc (optable
, opname
, suffix
, mode
);
5626 /* Like gen_libfunc, but verify that FP or INT operation is involved. */
5629 gen_int_fp_libfunc (optab optable
, const char *name
, char suffix
,
5632 if (DECIMAL_FLOAT_MODE_P (mode
) || GET_MODE_CLASS (mode
) == MODE_FLOAT
)
5633 gen_fp_libfunc (optable
, name
, suffix
, mode
);
5634 if (INTEGRAL_MODE_P (mode
))
5635 gen_int_libfunc (optable
, name
, suffix
, mode
);
5638 /* Like gen_libfunc, but verify that FP or INT operation is involved
5639 and add 'v' suffix for integer operation. */
5642 gen_intv_fp_libfunc (optab optable
, const char *name
, char suffix
,
5645 if (DECIMAL_FLOAT_MODE_P (mode
) || GET_MODE_CLASS (mode
) == MODE_FLOAT
)
5646 gen_fp_libfunc (optable
, name
, suffix
, mode
);
5647 if (GET_MODE_CLASS (mode
) == MODE_INT
)
5649 int len
= strlen (name
);
5650 char *v_name
= XALLOCAVEC (char, len
+ 2);
5651 strcpy (v_name
, name
);
5653 v_name
[len
+ 1] = 0;
5654 gen_int_libfunc (optable
, v_name
, suffix
, mode
);
5658 /* Like gen_libfunc, but verify that FP or INT or FIXED operation is
5662 gen_int_fp_fixed_libfunc (optab optable
, const char *name
, char suffix
,
5665 if (DECIMAL_FLOAT_MODE_P (mode
) || GET_MODE_CLASS (mode
) == MODE_FLOAT
)
5666 gen_fp_libfunc (optable
, name
, suffix
, mode
);
5667 if (INTEGRAL_MODE_P (mode
))
5668 gen_int_libfunc (optable
, name
, suffix
, mode
);
5669 if (ALL_FIXED_POINT_MODE_P (mode
))
5670 gen_fixed_libfunc (optable
, name
, suffix
, mode
);
5673 /* Like gen_libfunc, but verify that FP or INT or signed FIXED operation is
5677 gen_int_fp_signed_fixed_libfunc (optab optable
, const char *name
, char suffix
,
5680 if (DECIMAL_FLOAT_MODE_P (mode
) || GET_MODE_CLASS (mode
) == MODE_FLOAT
)
5681 gen_fp_libfunc (optable
, name
, suffix
, mode
);
5682 if (INTEGRAL_MODE_P (mode
))
5683 gen_int_libfunc (optable
, name
, suffix
, mode
);
5684 if (SIGNED_FIXED_POINT_MODE_P (mode
))
5685 gen_signed_fixed_libfunc (optable
, name
, suffix
, mode
);
5688 /* Like gen_libfunc, but verify that INT or FIXED operation is
5692 gen_int_fixed_libfunc (optab optable
, const char *name
, char suffix
,
5695 if (INTEGRAL_MODE_P (mode
))
5696 gen_int_libfunc (optable
, name
, suffix
, mode
);
5697 if (ALL_FIXED_POINT_MODE_P (mode
))
5698 gen_fixed_libfunc (optable
, name
, suffix
, mode
);
5701 /* Like gen_libfunc, but verify that INT or signed FIXED operation is
5705 gen_int_signed_fixed_libfunc (optab optable
, const char *name
, char suffix
,
5708 if (INTEGRAL_MODE_P (mode
))
5709 gen_int_libfunc (optable
, name
, suffix
, mode
);
5710 if (SIGNED_FIXED_POINT_MODE_P (mode
))
5711 gen_signed_fixed_libfunc (optable
, name
, suffix
, mode
);
5714 /* Like gen_libfunc, but verify that INT or unsigned FIXED operation is
5718 gen_int_unsigned_fixed_libfunc (optab optable
, const char *name
, char suffix
,
5721 if (INTEGRAL_MODE_P (mode
))
5722 gen_int_libfunc (optable
, name
, suffix
, mode
);
5723 if (UNSIGNED_FIXED_POINT_MODE_P (mode
))
5724 gen_unsigned_fixed_libfunc (optable
, name
, suffix
, mode
);
5727 /* Initialize the libfunc fields of an entire group of entries of an
5728 inter-mode-class conversion optab. The string formation rules are
5729 similar to the ones for init_libfuncs, above, but instead of having
5730 a mode name and an operand count these functions have two mode names
5731 and no operand count. */
5734 gen_interclass_conv_libfunc (convert_optab tab
,
5739 size_t opname_len
= strlen (opname
);
5740 size_t mname_len
= 0;
5742 const char *fname
, *tname
;
5744 int prefix_len
= targetm
.libfunc_gnu_prefix
? 6 : 2;
5745 char *libfunc_name
, *suffix
;
5746 char *nondec_name
, *dec_name
, *nondec_suffix
, *dec_suffix
;
5749 /* If this is a decimal conversion, add the current BID vs. DPD prefix that
5750 depends on which underlying decimal floating point format is used. */
5751 const size_t dec_len
= sizeof (DECIMAL_PREFIX
) - 1;
5753 mname_len
= strlen (GET_MODE_NAME (tmode
)) + strlen (GET_MODE_NAME (fmode
));
5755 nondec_name
= XALLOCAVEC (char, prefix_len
+ opname_len
+ mname_len
+ 1 + 1);
5756 nondec_name
[0] = '_';
5757 nondec_name
[1] = '_';
5758 if (targetm
.libfunc_gnu_prefix
)
5760 nondec_name
[2] = 'g';
5761 nondec_name
[3] = 'n';
5762 nondec_name
[4] = 'u';
5763 nondec_name
[5] = '_';
5766 memcpy (&nondec_name
[prefix_len
], opname
, opname_len
);
5767 nondec_suffix
= nondec_name
+ opname_len
+ prefix_len
;
5769 dec_name
= XALLOCAVEC (char, 2 + dec_len
+ opname_len
+ mname_len
+ 1 + 1);
5772 memcpy (&dec_name
[2], DECIMAL_PREFIX
, dec_len
);
5773 memcpy (&dec_name
[2+dec_len
], opname
, opname_len
);
5774 dec_suffix
= dec_name
+ dec_len
+ opname_len
+ 2;
5776 fname
= GET_MODE_NAME (fmode
);
5777 tname
= GET_MODE_NAME (tmode
);
5779 if (DECIMAL_FLOAT_MODE_P (fmode
) || DECIMAL_FLOAT_MODE_P (tmode
))
5781 libfunc_name
= dec_name
;
5782 suffix
= dec_suffix
;
5786 libfunc_name
= nondec_name
;
5787 suffix
= nondec_suffix
;
5791 for (q
= fname
; *q
; p
++, q
++)
5793 for (q
= tname
; *q
; p
++, q
++)
5798 set_conv_libfunc (tab
, tmode
, fmode
,
5799 ggc_alloc_string (libfunc_name
, p
- libfunc_name
));
5802 /* Same as gen_interclass_conv_libfunc but verify that we are producing
5803 int->fp conversion. */
5806 gen_int_to_fp_conv_libfunc (convert_optab tab
,
5811 if (GET_MODE_CLASS (fmode
) != MODE_INT
)
5813 if (GET_MODE_CLASS (tmode
) != MODE_FLOAT
&& !DECIMAL_FLOAT_MODE_P (tmode
))
5815 gen_interclass_conv_libfunc (tab
, opname
, tmode
, fmode
);
5818 /* ufloat_optab is special by using floatun for FP and floatuns decimal fp
5822 gen_ufloat_conv_libfunc (convert_optab tab
,
5823 const char *opname ATTRIBUTE_UNUSED
,
5827 if (DECIMAL_FLOAT_MODE_P (tmode
))
5828 gen_int_to_fp_conv_libfunc (tab
, "floatuns", tmode
, fmode
);
5830 gen_int_to_fp_conv_libfunc (tab
, "floatun", tmode
, fmode
);
5833 /* Same as gen_interclass_conv_libfunc but verify that we are producing
5834 fp->int conversion. */
5837 gen_int_to_fp_nondecimal_conv_libfunc (convert_optab tab
,
5842 if (GET_MODE_CLASS (fmode
) != MODE_INT
)
5844 if (GET_MODE_CLASS (tmode
) != MODE_FLOAT
)
5846 gen_interclass_conv_libfunc (tab
, opname
, tmode
, fmode
);
5849 /* Same as gen_interclass_conv_libfunc but verify that we are producing
5850 fp->int conversion with no decimal floating point involved. */
5853 gen_fp_to_int_conv_libfunc (convert_optab tab
,
5858 if (GET_MODE_CLASS (fmode
) != MODE_FLOAT
&& !DECIMAL_FLOAT_MODE_P (fmode
))
5860 if (GET_MODE_CLASS (tmode
) != MODE_INT
)
5862 gen_interclass_conv_libfunc (tab
, opname
, tmode
, fmode
);
5865 /* Initialize the libfunc fields of an of an intra-mode-class conversion optab.
5866 The string formation rules are
5867 similar to the ones for init_libfunc, above. */
5870 gen_intraclass_conv_libfunc (convert_optab tab
, const char *opname
,
5871 machine_mode tmode
, machine_mode fmode
)
5873 size_t opname_len
= strlen (opname
);
5874 size_t mname_len
= 0;
5876 const char *fname
, *tname
;
5878 int prefix_len
= targetm
.libfunc_gnu_prefix
? 6 : 2;
5879 char *nondec_name
, *dec_name
, *nondec_suffix
, *dec_suffix
;
5880 char *libfunc_name
, *suffix
;
5883 /* If this is a decimal conversion, add the current BID vs. DPD prefix that
5884 depends on which underlying decimal floating point format is used. */
5885 const size_t dec_len
= sizeof (DECIMAL_PREFIX
) - 1;
5887 mname_len
= strlen (GET_MODE_NAME (tmode
)) + strlen (GET_MODE_NAME (fmode
));
5889 nondec_name
= XALLOCAVEC (char, 2 + opname_len
+ mname_len
+ 1 + 1);
5890 nondec_name
[0] = '_';
5891 nondec_name
[1] = '_';
5892 if (targetm
.libfunc_gnu_prefix
)
5894 nondec_name
[2] = 'g';
5895 nondec_name
[3] = 'n';
5896 nondec_name
[4] = 'u';
5897 nondec_name
[5] = '_';
5899 memcpy (&nondec_name
[prefix_len
], opname
, opname_len
);
5900 nondec_suffix
= nondec_name
+ opname_len
+ prefix_len
;
5902 dec_name
= XALLOCAVEC (char, 2 + dec_len
+ opname_len
+ mname_len
+ 1 + 1);
5905 memcpy (&dec_name
[2], DECIMAL_PREFIX
, dec_len
);
5906 memcpy (&dec_name
[2 + dec_len
], opname
, opname_len
);
5907 dec_suffix
= dec_name
+ dec_len
+ opname_len
+ 2;
5909 fname
= GET_MODE_NAME (fmode
);
5910 tname
= GET_MODE_NAME (tmode
);
5912 if (DECIMAL_FLOAT_MODE_P (fmode
) || DECIMAL_FLOAT_MODE_P (tmode
))
5914 libfunc_name
= dec_name
;
5915 suffix
= dec_suffix
;
5919 libfunc_name
= nondec_name
;
5920 suffix
= nondec_suffix
;
5924 for (q
= fname
; *q
; p
++, q
++)
5926 for (q
= tname
; *q
; p
++, q
++)
5932 set_conv_libfunc (tab
, tmode
, fmode
,
5933 ggc_alloc_string (libfunc_name
, p
- libfunc_name
));
5936 /* Pick proper libcall for trunc_optab. We need to chose if we do
5937 truncation or extension and interclass or intraclass. */
5940 gen_trunc_conv_libfunc (convert_optab tab
,
5945 if (GET_MODE_CLASS (tmode
) != MODE_FLOAT
&& !DECIMAL_FLOAT_MODE_P (tmode
))
5947 if (GET_MODE_CLASS (fmode
) != MODE_FLOAT
&& !DECIMAL_FLOAT_MODE_P (fmode
))
5952 if ((GET_MODE_CLASS (tmode
) == MODE_FLOAT
&& DECIMAL_FLOAT_MODE_P (fmode
))
5953 || (GET_MODE_CLASS (fmode
) == MODE_FLOAT
&& DECIMAL_FLOAT_MODE_P (tmode
)))
5954 gen_interclass_conv_libfunc (tab
, opname
, tmode
, fmode
);
5956 if (GET_MODE_PRECISION (fmode
) <= GET_MODE_PRECISION (tmode
))
5959 if ((GET_MODE_CLASS (tmode
) == MODE_FLOAT
5960 && GET_MODE_CLASS (fmode
) == MODE_FLOAT
)
5961 || (DECIMAL_FLOAT_MODE_P (fmode
) && DECIMAL_FLOAT_MODE_P (tmode
)))
5962 gen_intraclass_conv_libfunc (tab
, opname
, tmode
, fmode
);
5965 /* Pick proper libcall for extend_optab. We need to chose if we do
5966 truncation or extension and interclass or intraclass. */
5969 gen_extend_conv_libfunc (convert_optab tab
,
5970 const char *opname ATTRIBUTE_UNUSED
,
5974 if (GET_MODE_CLASS (tmode
) != MODE_FLOAT
&& !DECIMAL_FLOAT_MODE_P (tmode
))
5976 if (GET_MODE_CLASS (fmode
) != MODE_FLOAT
&& !DECIMAL_FLOAT_MODE_P (fmode
))
5981 if ((GET_MODE_CLASS (tmode
) == MODE_FLOAT
&& DECIMAL_FLOAT_MODE_P (fmode
))
5982 || (GET_MODE_CLASS (fmode
) == MODE_FLOAT
&& DECIMAL_FLOAT_MODE_P (tmode
)))
5983 gen_interclass_conv_libfunc (tab
, opname
, tmode
, fmode
);
5985 if (GET_MODE_PRECISION (fmode
) > GET_MODE_PRECISION (tmode
))
5988 if ((GET_MODE_CLASS (tmode
) == MODE_FLOAT
5989 && GET_MODE_CLASS (fmode
) == MODE_FLOAT
)
5990 || (DECIMAL_FLOAT_MODE_P (fmode
) && DECIMAL_FLOAT_MODE_P (tmode
)))
5991 gen_intraclass_conv_libfunc (tab
, opname
, tmode
, fmode
);
5994 /* Pick proper libcall for fract_optab. We need to chose if we do
5995 interclass or intraclass. */
5998 gen_fract_conv_libfunc (convert_optab tab
,
6005 if (!(ALL_FIXED_POINT_MODE_P (tmode
) || ALL_FIXED_POINT_MODE_P (fmode
)))
6008 if (GET_MODE_CLASS (tmode
) == GET_MODE_CLASS (fmode
))
6009 gen_intraclass_conv_libfunc (tab
, opname
, tmode
, fmode
);
6011 gen_interclass_conv_libfunc (tab
, opname
, tmode
, fmode
);
6014 /* Pick proper libcall for fractuns_optab. */
6017 gen_fractuns_conv_libfunc (convert_optab tab
,
6024 /* One mode must be a fixed-point mode, and the other must be an integer
6026 if (!((ALL_FIXED_POINT_MODE_P (tmode
) && GET_MODE_CLASS (fmode
) == MODE_INT
)
6027 || (ALL_FIXED_POINT_MODE_P (fmode
)
6028 && GET_MODE_CLASS (tmode
) == MODE_INT
)))
6031 gen_interclass_conv_libfunc (tab
, opname
, tmode
, fmode
);
6034 /* Pick proper libcall for satfract_optab. We need to chose if we do
6035 interclass or intraclass. */
6038 gen_satfract_conv_libfunc (convert_optab tab
,
6045 /* TMODE must be a fixed-point mode. */
6046 if (!ALL_FIXED_POINT_MODE_P (tmode
))
6049 if (GET_MODE_CLASS (tmode
) == GET_MODE_CLASS (fmode
))
6050 gen_intraclass_conv_libfunc (tab
, opname
, tmode
, fmode
);
6052 gen_interclass_conv_libfunc (tab
, opname
, tmode
, fmode
);
6055 /* Pick proper libcall for satfractuns_optab. */
6058 gen_satfractuns_conv_libfunc (convert_optab tab
,
6065 /* TMODE must be a fixed-point mode, and FMODE must be an integer mode. */
6066 if (!(ALL_FIXED_POINT_MODE_P (tmode
) && GET_MODE_CLASS (fmode
) == MODE_INT
))
6069 gen_interclass_conv_libfunc (tab
, opname
, tmode
, fmode
);
6072 /* Hashtable callbacks for libfunc_decls. */
6074 struct libfunc_decl_hasher
: ggc_hasher
<tree
>
6079 return IDENTIFIER_HASH_VALUE (DECL_NAME (entry
));
6083 equal (tree decl
, tree name
)
6085 return DECL_NAME (decl
) == name
;
6089 /* A table of previously-created libfuncs, hashed by name. */
6090 static GTY (()) hash_table
<libfunc_decl_hasher
> *libfunc_decls
;
6092 /* Build a decl for a libfunc named NAME. */
6095 build_libfunc_function (const char *name
)
6097 tree decl
= build_decl (UNKNOWN_LOCATION
, FUNCTION_DECL
,
6098 get_identifier (name
),
6099 build_function_type (integer_type_node
, NULL_TREE
));
6100 /* ??? We don't have any type information except for this is
6101 a function. Pretend this is "int foo()". */
6102 DECL_ARTIFICIAL (decl
) = 1;
6103 DECL_EXTERNAL (decl
) = 1;
6104 TREE_PUBLIC (decl
) = 1;
6105 gcc_assert (DECL_ASSEMBLER_NAME (decl
));
6107 /* Zap the nonsensical SYMBOL_REF_DECL for this. What we're left with
6108 are the flags assigned by targetm.encode_section_info. */
6109 SET_SYMBOL_REF_DECL (XEXP (DECL_RTL (decl
), 0), NULL
);
6115 init_one_libfunc (const char *name
)
6120 if (libfunc_decls
== NULL
)
6121 libfunc_decls
= hash_table
<libfunc_decl_hasher
>::create_ggc (37);
6123 /* See if we have already created a libfunc decl for this function. */
6124 id
= get_identifier (name
);
6125 hash
= IDENTIFIER_HASH_VALUE (id
);
6126 tree
*slot
= libfunc_decls
->find_slot_with_hash (id
, hash
, INSERT
);
6130 /* Create a new decl, so that it can be passed to
6131 targetm.encode_section_info. */
6132 decl
= build_libfunc_function (name
);
6135 return XEXP (DECL_RTL (decl
), 0);
6138 /* Adjust the assembler name of libfunc NAME to ASMSPEC. */
6141 set_user_assembler_libfunc (const char *name
, const char *asmspec
)
6146 id
= get_identifier (name
);
6147 hash
= IDENTIFIER_HASH_VALUE (id
);
6148 tree
*slot
= libfunc_decls
->find_slot_with_hash (id
, hash
, NO_INSERT
);
6150 decl
= (tree
) *slot
;
6151 set_user_assembler_name (decl
, asmspec
);
6152 return XEXP (DECL_RTL (decl
), 0);
6155 /* Call this to reset the function entry for one optab (OPTABLE) in mode
6156 MODE to NAME, which should be either 0 or a string constant. */
6158 set_optab_libfunc (optab op
, machine_mode mode
, const char *name
)
6161 struct libfunc_entry e
;
6162 struct libfunc_entry
**slot
;
6169 val
= init_one_libfunc (name
);
6172 slot
= libfunc_hash
->find_slot (&e
, INSERT
);
6174 *slot
= ggc_alloc
<libfunc_entry
> ();
6176 (*slot
)->mode1
= mode
;
6177 (*slot
)->mode2
= VOIDmode
;
6178 (*slot
)->libfunc
= val
;
6181 /* Call this to reset the function entry for one conversion optab
6182 (OPTABLE) from mode FMODE to mode TMODE to NAME, which should be
6183 either 0 or a string constant. */
6185 set_conv_libfunc (convert_optab optab
, machine_mode tmode
,
6186 machine_mode fmode
, const char *name
)
6189 struct libfunc_entry e
;
6190 struct libfunc_entry
**slot
;
6197 val
= init_one_libfunc (name
);
6200 slot
= libfunc_hash
->find_slot (&e
, INSERT
);
6202 *slot
= ggc_alloc
<libfunc_entry
> ();
6203 (*slot
)->op
= optab
;
6204 (*slot
)->mode1
= tmode
;
6205 (*slot
)->mode2
= fmode
;
6206 (*slot
)->libfunc
= val
;
6209 /* Call this to initialize the contents of the optabs
6210 appropriately for the current target machine. */
6216 libfunc_hash
->empty ();
6218 libfunc_hash
= hash_table
<libfunc_hasher
>::create_ggc (10);
6220 /* Fill in the optabs with the insns we support. */
6221 init_all_optabs (this_fn_optabs
);
6223 /* The ffs function operates on `int'. Fall back on it if we do not
6224 have a libgcc2 function for that width. */
6225 if (INT_TYPE_SIZE
< BITS_PER_WORD
)
6226 set_optab_libfunc (ffs_optab
, mode_for_size (INT_TYPE_SIZE
, MODE_INT
, 0),
6229 /* Explicitly initialize the bswap libfuncs since we need them to be
6230 valid for things other than word_mode. */
6231 if (targetm
.libfunc_gnu_prefix
)
6233 set_optab_libfunc (bswap_optab
, SImode
, "__gnu_bswapsi2");
6234 set_optab_libfunc (bswap_optab
, DImode
, "__gnu_bswapdi2");
6238 set_optab_libfunc (bswap_optab
, SImode
, "__bswapsi2");
6239 set_optab_libfunc (bswap_optab
, DImode
, "__bswapdi2");
6242 /* Use cabs for double complex abs, since systems generally have cabs.
6243 Don't define any libcall for float complex, so that cabs will be used. */
6244 if (complex_double_type_node
)
6245 set_optab_libfunc (abs_optab
, TYPE_MODE (complex_double_type_node
),
6248 abort_libfunc
= init_one_libfunc ("abort");
6249 memcpy_libfunc
= init_one_libfunc ("memcpy");
6250 memmove_libfunc
= init_one_libfunc ("memmove");
6251 memcmp_libfunc
= init_one_libfunc ("memcmp");
6252 memset_libfunc
= init_one_libfunc ("memset");
6253 setbits_libfunc
= init_one_libfunc ("__setbits");
6255 #ifndef DONT_USE_BUILTIN_SETJMP
6256 setjmp_libfunc
= init_one_libfunc ("__builtin_setjmp");
6257 longjmp_libfunc
= init_one_libfunc ("__builtin_longjmp");
6259 setjmp_libfunc
= init_one_libfunc ("setjmp");
6260 longjmp_libfunc
= init_one_libfunc ("longjmp");
6262 unwind_sjlj_register_libfunc
= init_one_libfunc ("_Unwind_SjLj_Register");
6263 unwind_sjlj_unregister_libfunc
6264 = init_one_libfunc ("_Unwind_SjLj_Unregister");
6266 /* For function entry/exit instrumentation. */
6267 profile_function_entry_libfunc
6268 = init_one_libfunc ("__cyg_profile_func_enter");
6269 profile_function_exit_libfunc
6270 = init_one_libfunc ("__cyg_profile_func_exit");
6272 gcov_flush_libfunc
= init_one_libfunc ("__gcov_flush");
6274 /* Allow the target to add more libcalls or rename some, etc. */
6275 targetm
.init_libfuncs ();
6278 /* Use the current target and options to initialize
6279 TREE_OPTIMIZATION_OPTABS (OPTNODE). */
6282 init_tree_optimization_optabs (tree optnode
)
6284 /* Quick exit if we have already computed optabs for this target. */
6285 if (TREE_OPTIMIZATION_BASE_OPTABS (optnode
) == this_target_optabs
)
6288 /* Forget any previous information and set up for the current target. */
6289 TREE_OPTIMIZATION_BASE_OPTABS (optnode
) = this_target_optabs
;
6290 struct target_optabs
*tmp_optabs
= (struct target_optabs
*)
6291 TREE_OPTIMIZATION_OPTABS (optnode
);
6293 memset (tmp_optabs
, 0, sizeof (struct target_optabs
));
6295 tmp_optabs
= ggc_alloc
<target_optabs
> ();
6297 /* Generate a new set of optabs into tmp_optabs. */
6298 init_all_optabs (tmp_optabs
);
6300 /* If the optabs changed, record it. */
6301 if (memcmp (tmp_optabs
, this_target_optabs
, sizeof (struct target_optabs
)))
6302 TREE_OPTIMIZATION_OPTABS (optnode
) = tmp_optabs
;
6305 TREE_OPTIMIZATION_OPTABS (optnode
) = NULL
;
6306 ggc_free (tmp_optabs
);
6310 /* A helper function for init_sync_libfuncs. Using the basename BASE,
6311 install libfuncs into TAB for BASE_N for 1 <= N <= MAX. */
6314 init_sync_libfuncs_1 (optab tab
, const char *base
, int max
)
6318 size_t len
= strlen (base
);
6321 gcc_assert (max
<= 8);
6322 gcc_assert (len
+ 3 < sizeof (buf
));
6324 memcpy (buf
, base
, len
);
6327 buf
[len
+ 2] = '\0';
6330 for (i
= 1; i
<= max
; i
*= 2)
6332 buf
[len
+ 1] = '0' + i
;
6333 set_optab_libfunc (tab
, mode
, buf
);
6334 mode
= GET_MODE_2XWIDER_MODE (mode
);
6339 init_sync_libfuncs (int max
)
6341 if (!flag_sync_libcalls
)
6344 init_sync_libfuncs_1 (sync_compare_and_swap_optab
,
6345 "__sync_val_compare_and_swap", max
);
6346 init_sync_libfuncs_1 (sync_lock_test_and_set_optab
,
6347 "__sync_lock_test_and_set", max
);
6349 init_sync_libfuncs_1 (sync_old_add_optab
, "__sync_fetch_and_add", max
);
6350 init_sync_libfuncs_1 (sync_old_sub_optab
, "__sync_fetch_and_sub", max
);
6351 init_sync_libfuncs_1 (sync_old_ior_optab
, "__sync_fetch_and_or", max
);
6352 init_sync_libfuncs_1 (sync_old_and_optab
, "__sync_fetch_and_and", max
);
6353 init_sync_libfuncs_1 (sync_old_xor_optab
, "__sync_fetch_and_xor", max
);
6354 init_sync_libfuncs_1 (sync_old_nand_optab
, "__sync_fetch_and_nand", max
);
6356 init_sync_libfuncs_1 (sync_new_add_optab
, "__sync_add_and_fetch", max
);
6357 init_sync_libfuncs_1 (sync_new_sub_optab
, "__sync_sub_and_fetch", max
);
6358 init_sync_libfuncs_1 (sync_new_ior_optab
, "__sync_or_and_fetch", max
);
6359 init_sync_libfuncs_1 (sync_new_and_optab
, "__sync_and_and_fetch", max
);
6360 init_sync_libfuncs_1 (sync_new_xor_optab
, "__sync_xor_and_fetch", max
);
6361 init_sync_libfuncs_1 (sync_new_nand_optab
, "__sync_nand_and_fetch", max
);
6364 /* Print information about the current contents of the optabs on
6368 debug_optab_libfuncs (void)
6372 /* Dump the arithmetic optabs. */
6373 for (i
= FIRST_NORM_OPTAB
; i
<= LAST_NORMLIB_OPTAB
; ++i
)
6374 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
6376 rtx l
= optab_libfunc ((optab
) i
, (machine_mode
) j
);
6379 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
6380 fprintf (stderr
, "%s\t%s:\t%s\n",
6381 GET_RTX_NAME (optab_to_code ((optab
) i
)),
6387 /* Dump the conversion optabs. */
6388 for (i
= FIRST_CONV_OPTAB
; i
<= LAST_CONVLIB_OPTAB
; ++i
)
6389 for (j
= 0; j
< NUM_MACHINE_MODES
; ++j
)
6390 for (k
= 0; k
< NUM_MACHINE_MODES
; ++k
)
6392 rtx l
= convert_optab_libfunc ((optab
) i
, (machine_mode
) j
,
6396 gcc_assert (GET_CODE (l
) == SYMBOL_REF
);
6397 fprintf (stderr
, "%s\t%s\t%s:\t%s\n",
6398 GET_RTX_NAME (optab_to_code ((optab
) i
)),
6407 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
6408 CODE. Return 0 on failure. */
6411 gen_cond_trap (enum rtx_code code
, rtx op1
, rtx op2
, rtx tcode
)
6413 machine_mode mode
= GET_MODE (op1
);
6414 enum insn_code icode
;
6418 if (mode
== VOIDmode
)
6421 icode
= optab_handler (ctrap_optab
, mode
);
6422 if (icode
== CODE_FOR_nothing
)
6425 /* Some targets only accept a zero trap code. */
6426 if (!insn_operand_matches (icode
, 3, tcode
))
6429 do_pending_stack_adjust ();
6431 prepare_cmp_insn (op1
, op2
, code
, NULL_RTX
, false, OPTAB_DIRECT
,
6436 insn
= GEN_FCN (icode
) (trap_rtx
, XEXP (trap_rtx
, 0), XEXP (trap_rtx
, 1),
6439 /* If that failed, then give up. */
6447 insn
= get_insns ();
6452 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
6453 or unsigned operation code. */
6456 get_rtx_code (enum tree_code tcode
, bool unsignedp
)
6468 code
= unsignedp
? LTU
: LT
;
6471 code
= unsignedp
? LEU
: LE
;
6474 code
= unsignedp
? GTU
: GT
;
6477 code
= unsignedp
? GEU
: GE
;
6480 case UNORDERED_EXPR
:
6519 /* Return comparison rtx for COND. Use UNSIGNEDP to select signed or
6520 unsigned operators. Do not generate compare instruction. */
6523 vector_compare_rtx (enum tree_code tcode
, tree t_op0
, tree t_op1
,
6524 bool unsignedp
, enum insn_code icode
)
6526 struct expand_operand ops
[2];
6527 rtx rtx_op0
, rtx_op1
;
6528 machine_mode m0
, m1
;
6529 enum rtx_code rcode
= get_rtx_code (tcode
, unsignedp
);
6531 gcc_assert (TREE_CODE_CLASS (tcode
) == tcc_comparison
);
6533 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
6534 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
6535 cases, use the original mode. */
6536 rtx_op0
= expand_expr (t_op0
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op0
)),
6538 m0
= GET_MODE (rtx_op0
);
6540 m0
= TYPE_MODE (TREE_TYPE (t_op0
));
6542 rtx_op1
= expand_expr (t_op1
, NULL_RTX
, TYPE_MODE (TREE_TYPE (t_op1
)),
6544 m1
= GET_MODE (rtx_op1
);
6546 m1
= TYPE_MODE (TREE_TYPE (t_op1
));
6548 create_input_operand (&ops
[0], rtx_op0
, m0
);
6549 create_input_operand (&ops
[1], rtx_op1
, m1
);
6550 if (!maybe_legitimize_operands (icode
, 4, 2, ops
))
6552 return gen_rtx_fmt_ee (rcode
, VOIDmode
, ops
[0].value
, ops
[1].value
);
6555 /* Return true if VEC_PERM_EXPR of arbitrary input vectors can be expanded using
6556 SIMD extensions of the CPU. SEL may be NULL, which stands for an unknown
6557 constant. Note that additional permutations representing whole-vector shifts
6558 may also be handled via the vec_shr optab, but only where the second input
6559 vector is entirely constant zeroes; this case is not dealt with here. */
6562 can_vec_perm_p (machine_mode mode
, bool variable
,
6563 const unsigned char *sel
)
6565 machine_mode qimode
;
6567 /* If the target doesn't implement a vector mode for the vector type,
6568 then no operations are supported. */
6569 if (!VECTOR_MODE_P (mode
))
6574 if (direct_optab_handler (vec_perm_const_optab
, mode
) != CODE_FOR_nothing
6576 || targetm
.vectorize
.vec_perm_const_ok
== NULL
6577 || targetm
.vectorize
.vec_perm_const_ok (mode
, sel
)))
6581 if (direct_optab_handler (vec_perm_optab
, mode
) != CODE_FOR_nothing
)
6584 /* We allow fallback to a QI vector mode, and adjust the mask. */
6585 if (GET_MODE_INNER (mode
) == QImode
)
6587 qimode
= mode_for_vector (QImode
, GET_MODE_SIZE (mode
));
6588 if (!VECTOR_MODE_P (qimode
))
6591 /* ??? For completeness, we ought to check the QImode version of
6592 vec_perm_const_optab. But all users of this implicit lowering
6593 feature implement the variable vec_perm_optab. */
6594 if (direct_optab_handler (vec_perm_optab
, qimode
) == CODE_FOR_nothing
)
6597 /* In order to support the lowering of variable permutations,
6598 we need to support shifts and adds. */
6601 if (GET_MODE_UNIT_SIZE (mode
) > 2
6602 && optab_handler (ashl_optab
, mode
) == CODE_FOR_nothing
6603 && optab_handler (vashl_optab
, mode
) == CODE_FOR_nothing
)
6605 if (optab_handler (add_optab
, qimode
) == CODE_FOR_nothing
)
6612 /* Checks if vec_perm mask SEL is a constant equivalent to a shift of the first
6613 vec_perm operand, assuming the second operand is a constant vector of zeroes.
6614 Return the shift distance in bits if so, or NULL_RTX if the vec_perm is not a
6617 shift_amt_for_vec_perm_mask (rtx sel
)
6619 unsigned int i
, first
, nelt
= GET_MODE_NUNITS (GET_MODE (sel
));
6620 unsigned int bitsize
= GET_MODE_BITSIZE (GET_MODE_INNER (GET_MODE (sel
)));
6622 if (GET_CODE (sel
) != CONST_VECTOR
)
6625 first
= INTVAL (CONST_VECTOR_ELT (sel
, 0));
6626 if (first
>= 2*nelt
)
6628 for (i
= 1; i
< nelt
; i
++)
6630 int idx
= INTVAL (CONST_VECTOR_ELT (sel
, i
));
6631 unsigned int expected
= (i
+ first
) & (2 * nelt
- 1);
6632 /* Indices into the second vector are all equivalent. */
6633 if (idx
< 0 || (MIN (nelt
, (unsigned) idx
) != MIN (nelt
, expected
)))
6637 return GEN_INT (first
* bitsize
);
6640 /* A subroutine of expand_vec_perm for expanding one vec_perm insn. */
6643 expand_vec_perm_1 (enum insn_code icode
, rtx target
,
6644 rtx v0
, rtx v1
, rtx sel
)
6646 machine_mode tmode
= GET_MODE (target
);
6647 machine_mode smode
= GET_MODE (sel
);
6648 struct expand_operand ops
[4];
6650 create_output_operand (&ops
[0], target
, tmode
);
6651 create_input_operand (&ops
[3], sel
, smode
);
6653 /* Make an effort to preserve v0 == v1. The target expander is able to
6654 rely on this to determine if we're permuting a single input operand. */
6655 if (rtx_equal_p (v0
, v1
))
6657 if (!insn_operand_matches (icode
, 1, v0
))
6658 v0
= force_reg (tmode
, v0
);
6659 gcc_checking_assert (insn_operand_matches (icode
, 1, v0
));
6660 gcc_checking_assert (insn_operand_matches (icode
, 2, v0
));
6662 create_fixed_operand (&ops
[1], v0
);
6663 create_fixed_operand (&ops
[2], v0
);
6667 create_input_operand (&ops
[1], v0
, tmode
);
6668 /* See if this can be handled with a vec_shr. We only do this if the
6669 second vector is all zeroes. */
6670 enum insn_code shift_code
= optab_handler (vec_shr_optab
, GET_MODE (v0
));
6671 if (v1
== CONST0_RTX (GET_MODE (v1
)) && shift_code
)
6672 if (rtx shift_amt
= shift_amt_for_vec_perm_mask (sel
))
6674 create_convert_operand_from_type (&ops
[2], shift_amt
,
6675 sizetype_tab
[(int) stk_sizetype
]);
6676 if (maybe_expand_insn (shift_code
, 3, ops
))
6677 return ops
[0].value
;
6679 create_input_operand (&ops
[2], v1
, tmode
);
6682 if (maybe_expand_insn (icode
, 4, ops
))
6683 return ops
[0].value
;
6687 /* Generate instructions for vec_perm optab given its mode
6688 and three operands. */
6691 expand_vec_perm (machine_mode mode
, rtx v0
, rtx v1
, rtx sel
, rtx target
)
6693 enum insn_code icode
;
6694 machine_mode qimode
;
6695 unsigned int i
, w
, e
, u
;
6696 rtx tmp
, sel_qi
= NULL
;
6699 if (!target
|| GET_MODE (target
) != mode
)
6700 target
= gen_reg_rtx (mode
);
6702 w
= GET_MODE_SIZE (mode
);
6703 e
= GET_MODE_NUNITS (mode
);
6704 u
= GET_MODE_UNIT_SIZE (mode
);
6706 /* Set QIMODE to a different vector mode with byte elements.
6707 If no such mode, or if MODE already has byte elements, use VOIDmode. */
6709 if (GET_MODE_INNER (mode
) != QImode
)
6711 qimode
= mode_for_vector (QImode
, w
);
6712 if (!VECTOR_MODE_P (qimode
))
6716 /* If the input is a constant, expand it specially. */
6717 gcc_assert (GET_MODE_CLASS (GET_MODE (sel
)) == MODE_VECTOR_INT
);
6718 if (GET_CODE (sel
) == CONST_VECTOR
)
6720 icode
= direct_optab_handler (vec_perm_const_optab
, mode
);
6721 if (icode
!= CODE_FOR_nothing
)
6723 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
6728 /* Fall back to a constant byte-based permutation. */
6729 if (qimode
!= VOIDmode
)
6731 vec
= rtvec_alloc (w
);
6732 for (i
= 0; i
< e
; ++i
)
6734 unsigned int j
, this_e
;
6736 this_e
= INTVAL (CONST_VECTOR_ELT (sel
, i
));
6737 this_e
&= 2 * e
- 1;
6740 for (j
= 0; j
< u
; ++j
)
6741 RTVEC_ELT (vec
, i
* u
+ j
) = GEN_INT (this_e
+ j
);
6743 sel_qi
= gen_rtx_CONST_VECTOR (qimode
, vec
);
6745 icode
= direct_optab_handler (vec_perm_const_optab
, qimode
);
6746 if (icode
!= CODE_FOR_nothing
)
6748 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
6749 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
6750 gen_lowpart (qimode
, v1
), sel_qi
);
6752 return gen_lowpart (mode
, tmp
);
6757 /* Otherwise expand as a fully variable permuation. */
6758 icode
= direct_optab_handler (vec_perm_optab
, mode
);
6759 if (icode
!= CODE_FOR_nothing
)
6761 tmp
= expand_vec_perm_1 (icode
, target
, v0
, v1
, sel
);
6766 /* As a special case to aid several targets, lower the element-based
6767 permutation to a byte-based permutation and try again. */
6768 if (qimode
== VOIDmode
)
6770 icode
= direct_optab_handler (vec_perm_optab
, qimode
);
6771 if (icode
== CODE_FOR_nothing
)
6776 /* Multiply each element by its byte size. */
6777 machine_mode selmode
= GET_MODE (sel
);
6779 sel
= expand_simple_binop (selmode
, PLUS
, sel
, sel
,
6780 sel
, 0, OPTAB_DIRECT
);
6782 sel
= expand_simple_binop (selmode
, ASHIFT
, sel
,
6783 GEN_INT (exact_log2 (u
)),
6784 sel
, 0, OPTAB_DIRECT
);
6785 gcc_assert (sel
!= NULL
);
6787 /* Broadcast the low byte each element into each of its bytes. */
6788 vec
= rtvec_alloc (w
);
6789 for (i
= 0; i
< w
; ++i
)
6791 int this_e
= i
/ u
* u
;
6792 if (BYTES_BIG_ENDIAN
)
6794 RTVEC_ELT (vec
, i
) = GEN_INT (this_e
);
6796 tmp
= gen_rtx_CONST_VECTOR (qimode
, vec
);
6797 sel
= gen_lowpart (qimode
, sel
);
6798 sel
= expand_vec_perm (qimode
, sel
, sel
, tmp
, NULL
);
6799 gcc_assert (sel
!= NULL
);
6801 /* Add the byte offset to each byte element. */
6802 /* Note that the definition of the indicies here is memory ordering,
6803 so there should be no difference between big and little endian. */
6804 vec
= rtvec_alloc (w
);
6805 for (i
= 0; i
< w
; ++i
)
6806 RTVEC_ELT (vec
, i
) = GEN_INT (i
% u
);
6807 tmp
= gen_rtx_CONST_VECTOR (qimode
, vec
);
6808 sel_qi
= expand_simple_binop (qimode
, PLUS
, sel
, tmp
,
6809 sel
, 0, OPTAB_DIRECT
);
6810 gcc_assert (sel_qi
!= NULL
);
6813 tmp
= mode
!= qimode
? gen_reg_rtx (qimode
) : target
;
6814 tmp
= expand_vec_perm_1 (icode
, tmp
, gen_lowpart (qimode
, v0
),
6815 gen_lowpart (qimode
, v1
), sel_qi
);
6817 tmp
= gen_lowpart (mode
, tmp
);
6821 /* Return insn code for a conditional operator with a comparison in
6822 mode CMODE, unsigned if UNS is true, resulting in a value of mode VMODE. */
6824 static inline enum insn_code
6825 get_vcond_icode (machine_mode vmode
, machine_mode cmode
, bool uns
)
6827 enum insn_code icode
= CODE_FOR_nothing
;
6829 icode
= convert_optab_handler (vcondu_optab
, vmode
, cmode
);
6831 icode
= convert_optab_handler (vcond_optab
, vmode
, cmode
);
6835 /* Return TRUE iff, appropriate vector insns are available
6836 for vector cond expr with vector type VALUE_TYPE and a comparison
6837 with operand vector types in CMP_OP_TYPE. */
6840 expand_vec_cond_expr_p (tree value_type
, tree cmp_op_type
)
6842 machine_mode value_mode
= TYPE_MODE (value_type
);
6843 machine_mode cmp_op_mode
= TYPE_MODE (cmp_op_type
);
6844 if (GET_MODE_SIZE (value_mode
) != GET_MODE_SIZE (cmp_op_mode
)
6845 || GET_MODE_NUNITS (value_mode
) != GET_MODE_NUNITS (cmp_op_mode
)
6846 || get_vcond_icode (TYPE_MODE (value_type
), TYPE_MODE (cmp_op_type
),
6847 TYPE_UNSIGNED (cmp_op_type
)) == CODE_FOR_nothing
)
6852 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
6856 expand_vec_cond_expr (tree vec_cond_type
, tree op0
, tree op1
, tree op2
,
6859 struct expand_operand ops
[6];
6860 enum insn_code icode
;
6861 rtx comparison
, rtx_op1
, rtx_op2
;
6862 machine_mode mode
= TYPE_MODE (vec_cond_type
);
6863 machine_mode cmp_op_mode
;
6866 enum tree_code tcode
;
6868 if (COMPARISON_CLASS_P (op0
))
6870 op0a
= TREE_OPERAND (op0
, 0);
6871 op0b
= TREE_OPERAND (op0
, 1);
6872 tcode
= TREE_CODE (op0
);
6877 gcc_assert (!TYPE_UNSIGNED (TREE_TYPE (op0
)));
6879 op0b
= build_zero_cst (TREE_TYPE (op0
));
6882 unsignedp
= TYPE_UNSIGNED (TREE_TYPE (op0a
));
6883 cmp_op_mode
= TYPE_MODE (TREE_TYPE (op0a
));
6886 gcc_assert (GET_MODE_SIZE (mode
) == GET_MODE_SIZE (cmp_op_mode
)
6887 && GET_MODE_NUNITS (mode
) == GET_MODE_NUNITS (cmp_op_mode
));
6889 icode
= get_vcond_icode (mode
, cmp_op_mode
, unsignedp
);
6890 if (icode
== CODE_FOR_nothing
)
6893 comparison
= vector_compare_rtx (tcode
, op0a
, op0b
, unsignedp
, icode
);
6894 rtx_op1
= expand_normal (op1
);
6895 rtx_op2
= expand_normal (op2
);
6897 create_output_operand (&ops
[0], target
, mode
);
6898 create_input_operand (&ops
[1], rtx_op1
, mode
);
6899 create_input_operand (&ops
[2], rtx_op2
, mode
);
6900 create_fixed_operand (&ops
[3], comparison
);
6901 create_fixed_operand (&ops
[4], XEXP (comparison
, 0));
6902 create_fixed_operand (&ops
[5], XEXP (comparison
, 1));
6903 expand_insn (icode
, 6, ops
);
6904 return ops
[0].value
;
6907 /* Return non-zero if a highpart multiply is supported of can be synthisized.
6908 For the benefit of expand_mult_highpart, the return value is 1 for direct,
6909 2 for even/odd widening, and 3 for hi/lo widening. */
6912 can_mult_highpart_p (machine_mode mode
, bool uns_p
)
6918 op
= uns_p
? umul_highpart_optab
: smul_highpart_optab
;
6919 if (optab_handler (op
, mode
) != CODE_FOR_nothing
)
6922 /* If the mode is an integral vector, synth from widening operations. */
6923 if (GET_MODE_CLASS (mode
) != MODE_VECTOR_INT
)
6926 nunits
= GET_MODE_NUNITS (mode
);
6927 sel
= XALLOCAVEC (unsigned char, nunits
);
6929 op
= uns_p
? vec_widen_umult_even_optab
: vec_widen_smult_even_optab
;
6930 if (optab_handler (op
, mode
) != CODE_FOR_nothing
)
6932 op
= uns_p
? vec_widen_umult_odd_optab
: vec_widen_smult_odd_optab
;
6933 if (optab_handler (op
, mode
) != CODE_FOR_nothing
)
6935 for (i
= 0; i
< nunits
; ++i
)
6936 sel
[i
] = !BYTES_BIG_ENDIAN
+ (i
& ~1) + ((i
& 1) ? nunits
: 0);
6937 if (can_vec_perm_p (mode
, false, sel
))
6942 op
= uns_p
? vec_widen_umult_hi_optab
: vec_widen_smult_hi_optab
;
6943 if (optab_handler (op
, mode
) != CODE_FOR_nothing
)
6945 op
= uns_p
? vec_widen_umult_lo_optab
: vec_widen_smult_lo_optab
;
6946 if (optab_handler (op
, mode
) != CODE_FOR_nothing
)
6948 for (i
= 0; i
< nunits
; ++i
)
6949 sel
[i
] = 2 * i
+ (BYTES_BIG_ENDIAN
? 0 : 1);
6950 if (can_vec_perm_p (mode
, false, sel
))
6958 /* Expand a highpart multiply. */
6961 expand_mult_highpart (machine_mode mode
, rtx op0
, rtx op1
,
6962 rtx target
, bool uns_p
)
6964 struct expand_operand eops
[3];
6965 enum insn_code icode
;
6966 int method
, i
, nunits
;
6972 method
= can_mult_highpart_p (mode
, uns_p
);
6978 tab1
= uns_p
? umul_highpart_optab
: smul_highpart_optab
;
6979 return expand_binop (mode
, tab1
, op0
, op1
, target
, uns_p
,
6982 tab1
= uns_p
? vec_widen_umult_even_optab
: vec_widen_smult_even_optab
;
6983 tab2
= uns_p
? vec_widen_umult_odd_optab
: vec_widen_smult_odd_optab
;
6986 tab1
= uns_p
? vec_widen_umult_lo_optab
: vec_widen_smult_lo_optab
;
6987 tab2
= uns_p
? vec_widen_umult_hi_optab
: vec_widen_smult_hi_optab
;
6988 if (BYTES_BIG_ENDIAN
)
6999 icode
= optab_handler (tab1
, mode
);
7000 nunits
= GET_MODE_NUNITS (mode
);
7001 wmode
= insn_data
[icode
].operand
[0].mode
;
7002 gcc_checking_assert (2 * GET_MODE_NUNITS (wmode
) == nunits
);
7003 gcc_checking_assert (GET_MODE_SIZE (wmode
) == GET_MODE_SIZE (mode
));
7005 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
7006 create_input_operand (&eops
[1], op0
, mode
);
7007 create_input_operand (&eops
[2], op1
, mode
);
7008 expand_insn (icode
, 3, eops
);
7009 m1
= gen_lowpart (mode
, eops
[0].value
);
7011 create_output_operand (&eops
[0], gen_reg_rtx (wmode
), wmode
);
7012 create_input_operand (&eops
[1], op0
, mode
);
7013 create_input_operand (&eops
[2], op1
, mode
);
7014 expand_insn (optab_handler (tab2
, mode
), 3, eops
);
7015 m2
= gen_lowpart (mode
, eops
[0].value
);
7017 v
= rtvec_alloc (nunits
);
7020 for (i
= 0; i
< nunits
; ++i
)
7021 RTVEC_ELT (v
, i
) = GEN_INT (!BYTES_BIG_ENDIAN
+ (i
& ~1)
7022 + ((i
& 1) ? nunits
: 0));
7026 for (i
= 0; i
< nunits
; ++i
)
7027 RTVEC_ELT (v
, i
) = GEN_INT (2 * i
+ (BYTES_BIG_ENDIAN
? 0 : 1));
7029 perm
= gen_rtx_CONST_VECTOR (mode
, v
);
7031 return expand_vec_perm (mode
, m1
, m2
, perm
, target
);
7034 /* Return true if target supports vector masked load/store for mode. */
7036 can_vec_mask_load_store_p (machine_mode mode
, bool is_load
)
7038 optab op
= is_load
? maskload_optab
: maskstore_optab
;
7040 unsigned int vector_sizes
;
7042 /* If mode is vector mode, check it directly. */
7043 if (VECTOR_MODE_P (mode
))
7044 return optab_handler (op
, mode
) != CODE_FOR_nothing
;
7046 /* Otherwise, return true if there is some vector mode with
7047 the mask load/store supported. */
7049 /* See if there is any chance the mask load or store might be
7050 vectorized. If not, punt. */
7051 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
7052 if (!VECTOR_MODE_P (vmode
))
7055 if (optab_handler (op
, vmode
) != CODE_FOR_nothing
)
7058 vector_sizes
= targetm
.vectorize
.autovectorize_vector_sizes ();
7059 while (vector_sizes
!= 0)
7061 unsigned int cur
= 1 << floor_log2 (vector_sizes
);
7062 vector_sizes
&= ~cur
;
7063 if (cur
<= GET_MODE_SIZE (mode
))
7065 vmode
= mode_for_vector (mode
, cur
/ GET_MODE_SIZE (mode
));
7066 if (VECTOR_MODE_P (vmode
)
7067 && optab_handler (op
, vmode
) != CODE_FOR_nothing
)
7073 /* Return true if there is a compare_and_swap pattern. */
7076 can_compare_and_swap_p (machine_mode mode
, bool allow_libcall
)
7078 enum insn_code icode
;
7080 /* Check for __atomic_compare_and_swap. */
7081 icode
= direct_optab_handler (atomic_compare_and_swap_optab
, mode
);
7082 if (icode
!= CODE_FOR_nothing
)
7085 /* Check for __sync_compare_and_swap. */
7086 icode
= optab_handler (sync_compare_and_swap_optab
, mode
);
7087 if (icode
!= CODE_FOR_nothing
)
7089 if (allow_libcall
&& optab_libfunc (sync_compare_and_swap_optab
, mode
))
7092 /* No inline compare and swap. */
7096 /* Return true if an atomic exchange can be performed. */
7099 can_atomic_exchange_p (machine_mode mode
, bool allow_libcall
)
7101 enum insn_code icode
;
7103 /* Check for __atomic_exchange. */
7104 icode
= direct_optab_handler (atomic_exchange_optab
, mode
);
7105 if (icode
!= CODE_FOR_nothing
)
7108 /* Don't check __sync_test_and_set, as on some platforms that
7109 has reduced functionality. Targets that really do support
7110 a proper exchange should simply be updated to the __atomics. */
7112 return can_compare_and_swap_p (mode
, allow_libcall
);
7116 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
7120 find_cc_set (rtx x
, const_rtx pat
, void *data
)
7122 if (REG_P (x
) && GET_MODE_CLASS (GET_MODE (x
)) == MODE_CC
7123 && GET_CODE (pat
) == SET
)
7125 rtx
*p_cc_reg
= (rtx
*) data
;
7126 gcc_assert (!*p_cc_reg
);
7131 /* This is a helper function for the other atomic operations. This function
7132 emits a loop that contains SEQ that iterates until a compare-and-swap
7133 operation at the end succeeds. MEM is the memory to be modified. SEQ is
7134 a set of instructions that takes a value from OLD_REG as an input and
7135 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
7136 set to the current contents of MEM. After SEQ, a compare-and-swap will
7137 attempt to update MEM with NEW_REG. The function returns true when the
7138 loop was generated successfully. */
7141 expand_compare_and_swap_loop (rtx mem
, rtx old_reg
, rtx new_reg
, rtx seq
)
7143 machine_mode mode
= GET_MODE (mem
);
7144 rtx_code_label
*label
;
7145 rtx cmp_reg
, success
, oldval
;
7147 /* The loop we want to generate looks like
7153 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
7157 Note that we only do the plain load from memory once. Subsequent
7158 iterations use the value loaded by the compare-and-swap pattern. */
7160 label
= gen_label_rtx ();
7161 cmp_reg
= gen_reg_rtx (mode
);
7163 emit_move_insn (cmp_reg
, mem
);
7165 emit_move_insn (old_reg
, cmp_reg
);
7171 if (!expand_atomic_compare_and_swap (&success
, &oldval
, mem
, old_reg
,
7172 new_reg
, false, MEMMODEL_SYNC_SEQ_CST
,
7176 if (oldval
!= cmp_reg
)
7177 emit_move_insn (cmp_reg
, oldval
);
7179 /* Mark this jump predicted not taken. */
7180 emit_cmp_and_jump_insns (success
, const0_rtx
, EQ
, const0_rtx
,
7181 GET_MODE (success
), 1, label
, 0);
7186 /* This function tries to emit an atomic_exchange intruction. VAL is written
7187 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
7188 using TARGET if possible. */
7191 maybe_emit_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
7193 machine_mode mode
= GET_MODE (mem
);
7194 enum insn_code icode
;
7196 /* If the target supports the exchange directly, great. */
7197 icode
= direct_optab_handler (atomic_exchange_optab
, mode
);
7198 if (icode
!= CODE_FOR_nothing
)
7200 struct expand_operand ops
[4];
7202 create_output_operand (&ops
[0], target
, mode
);
7203 create_fixed_operand (&ops
[1], mem
);
7204 create_input_operand (&ops
[2], val
, mode
);
7205 create_integer_operand (&ops
[3], model
);
7206 if (maybe_expand_insn (icode
, 4, ops
))
7207 return ops
[0].value
;
7213 /* This function tries to implement an atomic exchange operation using
7214 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
7215 The previous contents of *MEM are returned, using TARGET if possible.
7216 Since this instructionn is an acquire barrier only, stronger memory
7217 models may require additional barriers to be emitted. */
7220 maybe_emit_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
,
7221 enum memmodel model
)
7223 machine_mode mode
= GET_MODE (mem
);
7224 enum insn_code icode
;
7225 rtx_insn
*last_insn
= get_last_insn ();
7227 icode
= optab_handler (sync_lock_test_and_set_optab
, mode
);
7229 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
7230 exists, and the memory model is stronger than acquire, add a release
7231 barrier before the instruction. */
7233 if (is_mm_seq_cst (model
) || is_mm_release (model
) || is_mm_acq_rel (model
))
7234 expand_mem_thread_fence (model
);
7236 if (icode
!= CODE_FOR_nothing
)
7238 struct expand_operand ops
[3];
7239 create_output_operand (&ops
[0], target
, mode
);
7240 create_fixed_operand (&ops
[1], mem
);
7241 create_input_operand (&ops
[2], val
, mode
);
7242 if (maybe_expand_insn (icode
, 3, ops
))
7243 return ops
[0].value
;
7246 /* If an external test-and-set libcall is provided, use that instead of
7247 any external compare-and-swap that we might get from the compare-and-
7248 swap-loop expansion later. */
7249 if (!can_compare_and_swap_p (mode
, false))
7251 rtx libfunc
= optab_libfunc (sync_lock_test_and_set_optab
, mode
);
7252 if (libfunc
!= NULL
)
7256 addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
7257 return emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
7258 mode
, 2, addr
, ptr_mode
,
7263 /* If the test_and_set can't be emitted, eliminate any barrier that might
7264 have been emitted. */
7265 delete_insns_since (last_insn
);
7269 /* This function tries to implement an atomic exchange operation using a
7270 compare_and_swap loop. VAL is written to *MEM. The previous contents of
7271 *MEM are returned, using TARGET if possible. No memory model is required
7272 since a compare_and_swap loop is seq-cst. */
7275 maybe_emit_compare_and_swap_exchange_loop (rtx target
, rtx mem
, rtx val
)
7277 machine_mode mode
= GET_MODE (mem
);
7279 if (can_compare_and_swap_p (mode
, true))
7281 if (!target
|| !register_operand (target
, mode
))
7282 target
= gen_reg_rtx (mode
);
7283 if (expand_compare_and_swap_loop (mem
, target
, val
, NULL_RTX
))
7290 /* This function tries to implement an atomic test-and-set operation
7291 using the atomic_test_and_set instruction pattern. A boolean value
7292 is returned from the operation, using TARGET if possible. */
7294 #ifndef HAVE_atomic_test_and_set
7295 #define HAVE_atomic_test_and_set 0
7296 #define CODE_FOR_atomic_test_and_set CODE_FOR_nothing
7300 maybe_emit_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
7302 machine_mode pat_bool_mode
;
7303 struct expand_operand ops
[3];
7305 if (!HAVE_atomic_test_and_set
)
7308 /* While we always get QImode from __atomic_test_and_set, we get
7309 other memory modes from __sync_lock_test_and_set. Note that we
7310 use no endian adjustment here. This matches the 4.6 behavior
7311 in the Sparc backend. */
7313 (insn_data
[CODE_FOR_atomic_test_and_set
].operand
[1].mode
== QImode
);
7314 if (GET_MODE (mem
) != QImode
)
7315 mem
= adjust_address_nv (mem
, QImode
, 0);
7317 pat_bool_mode
= insn_data
[CODE_FOR_atomic_test_and_set
].operand
[0].mode
;
7318 create_output_operand (&ops
[0], target
, pat_bool_mode
);
7319 create_fixed_operand (&ops
[1], mem
);
7320 create_integer_operand (&ops
[2], model
);
7322 if (maybe_expand_insn (CODE_FOR_atomic_test_and_set
, 3, ops
))
7323 return ops
[0].value
;
7327 /* This function expands the legacy _sync_lock test_and_set operation which is
7328 generally an atomic exchange. Some limited targets only allow the
7329 constant 1 to be stored. This is an ACQUIRE operation.
7331 TARGET is an optional place to stick the return value.
7332 MEM is where VAL is stored. */
7335 expand_sync_lock_test_and_set (rtx target
, rtx mem
, rtx val
)
7339 /* Try an atomic_exchange first. */
7340 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, MEMMODEL_SYNC_ACQUIRE
);
7344 ret
= maybe_emit_sync_lock_test_and_set (target
, mem
, val
,
7345 MEMMODEL_SYNC_ACQUIRE
);
7349 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
7353 /* If there are no other options, try atomic_test_and_set if the value
7354 being stored is 1. */
7355 if (val
== const1_rtx
)
7356 ret
= maybe_emit_atomic_test_and_set (target
, mem
, MEMMODEL_SYNC_ACQUIRE
);
7361 /* This function expands the atomic test_and_set operation:
7362 atomically store a boolean TRUE into MEM and return the previous value.
7364 MEMMODEL is the memory model variant to use.
7365 TARGET is an optional place to stick the return value. */
7368 expand_atomic_test_and_set (rtx target
, rtx mem
, enum memmodel model
)
7370 machine_mode mode
= GET_MODE (mem
);
7371 rtx ret
, trueval
, subtarget
;
7373 ret
= maybe_emit_atomic_test_and_set (target
, mem
, model
);
7377 /* Be binary compatible with non-default settings of trueval, and different
7378 cpu revisions. E.g. one revision may have atomic-test-and-set, but
7379 another only has atomic-exchange. */
7380 if (targetm
.atomic_test_and_set_trueval
== 1)
7382 trueval
= const1_rtx
;
7383 subtarget
= target
? target
: gen_reg_rtx (mode
);
7387 trueval
= gen_int_mode (targetm
.atomic_test_and_set_trueval
, mode
);
7388 subtarget
= gen_reg_rtx (mode
);
7391 /* Try the atomic-exchange optab... */
7392 ret
= maybe_emit_atomic_exchange (subtarget
, mem
, trueval
, model
);
7394 /* ... then an atomic-compare-and-swap loop ... */
7396 ret
= maybe_emit_compare_and_swap_exchange_loop (subtarget
, mem
, trueval
);
7398 /* ... before trying the vaguely defined legacy lock_test_and_set. */
7400 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, trueval
, model
);
7402 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
7403 things with the value 1. Thus we try again without trueval. */
7404 if (!ret
&& targetm
.atomic_test_and_set_trueval
!= 1)
7405 ret
= maybe_emit_sync_lock_test_and_set (subtarget
, mem
, const1_rtx
, model
);
7407 /* Failing all else, assume a single threaded environment and simply
7408 perform the operation. */
7411 /* If the result is ignored skip the move to target. */
7412 if (subtarget
!= const0_rtx
)
7413 emit_move_insn (subtarget
, mem
);
7415 emit_move_insn (mem
, trueval
);
7419 /* Recall that have to return a boolean value; rectify if trueval
7420 is not exactly one. */
7421 if (targetm
.atomic_test_and_set_trueval
!= 1)
7422 ret
= emit_store_flag_force (target
, NE
, ret
, const0_rtx
, mode
, 0, 1);
7427 /* This function expands the atomic exchange operation:
7428 atomically store VAL in MEM and return the previous value in MEM.
7430 MEMMODEL is the memory model variant to use.
7431 TARGET is an optional place to stick the return value. */
7434 expand_atomic_exchange (rtx target
, rtx mem
, rtx val
, enum memmodel model
)
7438 ret
= maybe_emit_atomic_exchange (target
, mem
, val
, model
);
7440 /* Next try a compare-and-swap loop for the exchange. */
7442 ret
= maybe_emit_compare_and_swap_exchange_loop (target
, mem
, val
);
7447 /* This function expands the atomic compare exchange operation:
7449 *PTARGET_BOOL is an optional place to store the boolean success/failure.
7450 *PTARGET_OVAL is an optional place to store the old value from memory.
7451 Both target parameters may be NULL to indicate that we do not care about
7452 that return value. Both target parameters are updated on success to
7453 the actual location of the corresponding result.
7455 MEMMODEL is the memory model variant to use.
7457 The return value of the function is true for success. */
7460 expand_atomic_compare_and_swap (rtx
*ptarget_bool
, rtx
*ptarget_oval
,
7461 rtx mem
, rtx expected
, rtx desired
,
7462 bool is_weak
, enum memmodel succ_model
,
7463 enum memmodel fail_model
)
7465 machine_mode mode
= GET_MODE (mem
);
7466 struct expand_operand ops
[8];
7467 enum insn_code icode
;
7468 rtx target_oval
, target_bool
= NULL_RTX
;
7471 /* Load expected into a register for the compare and swap. */
7472 if (MEM_P (expected
))
7473 expected
= copy_to_reg (expected
);
7475 /* Make sure we always have some place to put the return oldval.
7476 Further, make sure that place is distinct from the input expected,
7477 just in case we need that path down below. */
7478 if (ptarget_oval
== NULL
7479 || (target_oval
= *ptarget_oval
) == NULL
7480 || reg_overlap_mentioned_p (expected
, target_oval
))
7481 target_oval
= gen_reg_rtx (mode
);
7483 icode
= direct_optab_handler (atomic_compare_and_swap_optab
, mode
);
7484 if (icode
!= CODE_FOR_nothing
)
7486 machine_mode bool_mode
= insn_data
[icode
].operand
[0].mode
;
7488 /* Make sure we always have a place for the bool operand. */
7489 if (ptarget_bool
== NULL
7490 || (target_bool
= *ptarget_bool
) == NULL
7491 || GET_MODE (target_bool
) != bool_mode
)
7492 target_bool
= gen_reg_rtx (bool_mode
);
7494 /* Emit the compare_and_swap. */
7495 create_output_operand (&ops
[0], target_bool
, bool_mode
);
7496 create_output_operand (&ops
[1], target_oval
, mode
);
7497 create_fixed_operand (&ops
[2], mem
);
7498 create_input_operand (&ops
[3], expected
, mode
);
7499 create_input_operand (&ops
[4], desired
, mode
);
7500 create_integer_operand (&ops
[5], is_weak
);
7501 create_integer_operand (&ops
[6], succ_model
);
7502 create_integer_operand (&ops
[7], fail_model
);
7503 if (maybe_expand_insn (icode
, 8, ops
))
7505 /* Return success/failure. */
7506 target_bool
= ops
[0].value
;
7507 target_oval
= ops
[1].value
;
7512 /* Otherwise fall back to the original __sync_val_compare_and_swap
7513 which is always seq-cst. */
7514 icode
= optab_handler (sync_compare_and_swap_optab
, mode
);
7515 if (icode
!= CODE_FOR_nothing
)
7519 create_output_operand (&ops
[0], target_oval
, mode
);
7520 create_fixed_operand (&ops
[1], mem
);
7521 create_input_operand (&ops
[2], expected
, mode
);
7522 create_input_operand (&ops
[3], desired
, mode
);
7523 if (!maybe_expand_insn (icode
, 4, ops
))
7526 target_oval
= ops
[0].value
;
7528 /* If the caller isn't interested in the boolean return value,
7529 skip the computation of it. */
7530 if (ptarget_bool
== NULL
)
7533 /* Otherwise, work out if the compare-and-swap succeeded. */
7535 if (have_insn_for (COMPARE
, CCmode
))
7536 note_stores (PATTERN (get_last_insn ()), find_cc_set
, &cc_reg
);
7539 target_bool
= emit_store_flag_force (target_bool
, EQ
, cc_reg
,
7540 const0_rtx
, VOIDmode
, 0, 1);
7543 goto success_bool_from_val
;
7546 /* Also check for library support for __sync_val_compare_and_swap. */
7547 libfunc
= optab_libfunc (sync_compare_and_swap_optab
, mode
);
7548 if (libfunc
!= NULL
)
7550 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
7551 target_oval
= emit_library_call_value (libfunc
, NULL_RTX
, LCT_NORMAL
,
7552 mode
, 3, addr
, ptr_mode
,
7553 expected
, mode
, desired
, mode
);
7555 /* Compute the boolean return value only if requested. */
7557 goto success_bool_from_val
;
7565 success_bool_from_val
:
7566 target_bool
= emit_store_flag_force (target_bool
, EQ
, target_oval
,
7567 expected
, VOIDmode
, 1, 1);
7569 /* Make sure that the oval output winds up where the caller asked. */
7571 *ptarget_oval
= target_oval
;
7573 *ptarget_bool
= target_bool
;
7577 /* Generate asm volatile("" : : : "memory") as the memory barrier. */
7580 expand_asm_memory_barrier (void)
7584 asm_op
= gen_rtx_ASM_OPERANDS (VOIDmode
, empty_string
, empty_string
, 0,
7585 rtvec_alloc (0), rtvec_alloc (0),
7586 rtvec_alloc (0), UNKNOWN_LOCATION
);
7587 MEM_VOLATILE_P (asm_op
) = 1;
7589 clob
= gen_rtx_SCRATCH (VOIDmode
);
7590 clob
= gen_rtx_MEM (BLKmode
, clob
);
7591 clob
= gen_rtx_CLOBBER (VOIDmode
, clob
);
7593 emit_insn (gen_rtx_PARALLEL (VOIDmode
, gen_rtvec (2, asm_op
, clob
)));
7596 /* This routine will either emit the mem_thread_fence pattern or issue a
7597 sync_synchronize to generate a fence for memory model MEMMODEL. */
7599 #ifndef HAVE_mem_thread_fence
7600 # define HAVE_mem_thread_fence 0
7601 # define gen_mem_thread_fence(x) (gcc_unreachable (), NULL_RTX)
7603 #ifndef HAVE_memory_barrier
7604 # define HAVE_memory_barrier 0
7605 # define gen_memory_barrier() (gcc_unreachable (), NULL_RTX)
7609 expand_mem_thread_fence (enum memmodel model
)
7611 if (HAVE_mem_thread_fence
)
7612 emit_insn (gen_mem_thread_fence (GEN_INT (model
)));
7613 else if (!is_mm_relaxed (model
))
7615 if (HAVE_memory_barrier
)
7616 emit_insn (gen_memory_barrier ());
7617 else if (synchronize_libfunc
!= NULL_RTX
)
7618 emit_library_call (synchronize_libfunc
, LCT_NORMAL
, VOIDmode
, 0);
7620 expand_asm_memory_barrier ();
7624 /* This routine will either emit the mem_signal_fence pattern or issue a
7625 sync_synchronize to generate a fence for memory model MEMMODEL. */
7627 #ifndef HAVE_mem_signal_fence
7628 # define HAVE_mem_signal_fence 0
7629 # define gen_mem_signal_fence(x) (gcc_unreachable (), NULL_RTX)
7633 expand_mem_signal_fence (enum memmodel model
)
7635 if (HAVE_mem_signal_fence
)
7636 emit_insn (gen_mem_signal_fence (GEN_INT (model
)));
7637 else if (!is_mm_relaxed (model
))
7639 /* By default targets are coherent between a thread and the signal
7640 handler running on the same thread. Thus this really becomes a
7641 compiler barrier, in that stores must not be sunk past
7642 (or raised above) a given point. */
7643 expand_asm_memory_barrier ();
7647 /* This function expands the atomic load operation:
7648 return the atomically loaded value in MEM.
7650 MEMMODEL is the memory model variant to use.
7651 TARGET is an option place to stick the return value. */
7654 expand_atomic_load (rtx target
, rtx mem
, enum memmodel model
)
7656 machine_mode mode
= GET_MODE (mem
);
7657 enum insn_code icode
;
7659 /* If the target supports the load directly, great. */
7660 icode
= direct_optab_handler (atomic_load_optab
, mode
);
7661 if (icode
!= CODE_FOR_nothing
)
7663 struct expand_operand ops
[3];
7665 create_output_operand (&ops
[0], target
, mode
);
7666 create_fixed_operand (&ops
[1], mem
);
7667 create_integer_operand (&ops
[2], model
);
7668 if (maybe_expand_insn (icode
, 3, ops
))
7669 return ops
[0].value
;
7672 /* If the size of the object is greater than word size on this target,
7673 then we assume that a load will not be atomic. */
7674 if (GET_MODE_PRECISION (mode
) > BITS_PER_WORD
)
7676 /* Issue val = compare_and_swap (mem, 0, 0).
7677 This may cause the occasional harmless store of 0 when the value is
7678 already 0, but it seems to be OK according to the standards guys. */
7679 if (expand_atomic_compare_and_swap (NULL
, &target
, mem
, const0_rtx
,
7680 const0_rtx
, false, model
, model
))
7683 /* Otherwise there is no atomic load, leave the library call. */
7687 /* Otherwise assume loads are atomic, and emit the proper barriers. */
7688 if (!target
|| target
== const0_rtx
)
7689 target
= gen_reg_rtx (mode
);
7691 /* For SEQ_CST, emit a barrier before the load. */
7692 if (is_mm_seq_cst (model
))
7693 expand_mem_thread_fence (model
);
7695 emit_move_insn (target
, mem
);
7697 /* Emit the appropriate barrier after the load. */
7698 expand_mem_thread_fence (model
);
7703 /* This function expands the atomic store operation:
7704 Atomically store VAL in MEM.
7705 MEMMODEL is the memory model variant to use.
7706 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
7707 function returns const0_rtx if a pattern was emitted. */
7710 expand_atomic_store (rtx mem
, rtx val
, enum memmodel model
, bool use_release
)
7712 machine_mode mode
= GET_MODE (mem
);
7713 enum insn_code icode
;
7714 struct expand_operand ops
[3];
7716 /* If the target supports the store directly, great. */
7717 icode
= direct_optab_handler (atomic_store_optab
, mode
);
7718 if (icode
!= CODE_FOR_nothing
)
7720 create_fixed_operand (&ops
[0], mem
);
7721 create_input_operand (&ops
[1], val
, mode
);
7722 create_integer_operand (&ops
[2], model
);
7723 if (maybe_expand_insn (icode
, 3, ops
))
7727 /* If using __sync_lock_release is a viable alternative, try it. */
7730 icode
= direct_optab_handler (sync_lock_release_optab
, mode
);
7731 if (icode
!= CODE_FOR_nothing
)
7733 create_fixed_operand (&ops
[0], mem
);
7734 create_input_operand (&ops
[1], const0_rtx
, mode
);
7735 if (maybe_expand_insn (icode
, 2, ops
))
7737 /* lock_release is only a release barrier. */
7738 if (is_mm_seq_cst (model
))
7739 expand_mem_thread_fence (model
);
7745 /* If the size of the object is greater than word size on this target,
7746 a default store will not be atomic, Try a mem_exchange and throw away
7747 the result. If that doesn't work, don't do anything. */
7748 if (GET_MODE_PRECISION (mode
) > BITS_PER_WORD
)
7750 rtx target
= maybe_emit_atomic_exchange (NULL_RTX
, mem
, val
, model
);
7752 target
= maybe_emit_compare_and_swap_exchange_loop (NULL_RTX
, mem
, val
);
7759 /* Otherwise assume stores are atomic, and emit the proper barriers. */
7760 expand_mem_thread_fence (model
);
7762 emit_move_insn (mem
, val
);
7764 /* For SEQ_CST, also emit a barrier after the store. */
7765 if (is_mm_seq_cst (model
))
7766 expand_mem_thread_fence (model
);
7772 /* Structure containing the pointers and values required to process the
7773 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
7775 struct atomic_op_functions
7777 direct_optab mem_fetch_before
;
7778 direct_optab mem_fetch_after
;
7779 direct_optab mem_no_result
;
7782 direct_optab no_result
;
7783 enum rtx_code reverse_code
;
7787 /* Fill in structure pointed to by OP with the various optab entries for an
7788 operation of type CODE. */
7791 get_atomic_op_for_code (struct atomic_op_functions
*op
, enum rtx_code code
)
7793 gcc_assert (op
!= NULL
);
7795 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
7796 in the source code during compilation, and the optab entries are not
7797 computable until runtime. Fill in the values at runtime. */
7801 op
->mem_fetch_before
= atomic_fetch_add_optab
;
7802 op
->mem_fetch_after
= atomic_add_fetch_optab
;
7803 op
->mem_no_result
= atomic_add_optab
;
7804 op
->fetch_before
= sync_old_add_optab
;
7805 op
->fetch_after
= sync_new_add_optab
;
7806 op
->no_result
= sync_add_optab
;
7807 op
->reverse_code
= MINUS
;
7810 op
->mem_fetch_before
= atomic_fetch_sub_optab
;
7811 op
->mem_fetch_after
= atomic_sub_fetch_optab
;
7812 op
->mem_no_result
= atomic_sub_optab
;
7813 op
->fetch_before
= sync_old_sub_optab
;
7814 op
->fetch_after
= sync_new_sub_optab
;
7815 op
->no_result
= sync_sub_optab
;
7816 op
->reverse_code
= PLUS
;
7819 op
->mem_fetch_before
= atomic_fetch_xor_optab
;
7820 op
->mem_fetch_after
= atomic_xor_fetch_optab
;
7821 op
->mem_no_result
= atomic_xor_optab
;
7822 op
->fetch_before
= sync_old_xor_optab
;
7823 op
->fetch_after
= sync_new_xor_optab
;
7824 op
->no_result
= sync_xor_optab
;
7825 op
->reverse_code
= XOR
;
7828 op
->mem_fetch_before
= atomic_fetch_and_optab
;
7829 op
->mem_fetch_after
= atomic_and_fetch_optab
;
7830 op
->mem_no_result
= atomic_and_optab
;
7831 op
->fetch_before
= sync_old_and_optab
;
7832 op
->fetch_after
= sync_new_and_optab
;
7833 op
->no_result
= sync_and_optab
;
7834 op
->reverse_code
= UNKNOWN
;
7837 op
->mem_fetch_before
= atomic_fetch_or_optab
;
7838 op
->mem_fetch_after
= atomic_or_fetch_optab
;
7839 op
->mem_no_result
= atomic_or_optab
;
7840 op
->fetch_before
= sync_old_ior_optab
;
7841 op
->fetch_after
= sync_new_ior_optab
;
7842 op
->no_result
= sync_ior_optab
;
7843 op
->reverse_code
= UNKNOWN
;
7846 op
->mem_fetch_before
= atomic_fetch_nand_optab
;
7847 op
->mem_fetch_after
= atomic_nand_fetch_optab
;
7848 op
->mem_no_result
= atomic_nand_optab
;
7849 op
->fetch_before
= sync_old_nand_optab
;
7850 op
->fetch_after
= sync_new_nand_optab
;
7851 op
->no_result
= sync_nand_optab
;
7852 op
->reverse_code
= UNKNOWN
;
7859 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
7860 using memory order MODEL. If AFTER is true the operation needs to return
7861 the value of *MEM after the operation, otherwise the previous value.
7862 TARGET is an optional place to place the result. The result is unused if
7864 Return the result if there is a better sequence, otherwise NULL_RTX. */
7867 maybe_optimize_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
7868 enum memmodel model
, bool after
)
7870 /* If the value is prefetched, or not used, it may be possible to replace
7871 the sequence with a native exchange operation. */
7872 if (!after
|| target
== const0_rtx
)
7874 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
7875 if (code
== AND
&& val
== const0_rtx
)
7877 if (target
== const0_rtx
)
7878 target
= gen_reg_rtx (GET_MODE (mem
));
7879 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
7882 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
7883 if (code
== IOR
&& val
== constm1_rtx
)
7885 if (target
== const0_rtx
)
7886 target
= gen_reg_rtx (GET_MODE (mem
));
7887 return maybe_emit_atomic_exchange (target
, mem
, val
, model
);
7894 /* Try to emit an instruction for a specific operation varaition.
7895 OPTAB contains the OP functions.
7896 TARGET is an optional place to return the result. const0_rtx means unused.
7897 MEM is the memory location to operate on.
7898 VAL is the value to use in the operation.
7899 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
7900 MODEL is the memory model, if used.
7901 AFTER is true if the returned result is the value after the operation. */
7904 maybe_emit_op (const struct atomic_op_functions
*optab
, rtx target
, rtx mem
,
7905 rtx val
, bool use_memmodel
, enum memmodel model
, bool after
)
7907 machine_mode mode
= GET_MODE (mem
);
7908 struct expand_operand ops
[4];
7909 enum insn_code icode
;
7913 /* Check to see if there is a result returned. */
7914 if (target
== const0_rtx
)
7918 icode
= direct_optab_handler (optab
->mem_no_result
, mode
);
7919 create_integer_operand (&ops
[2], model
);
7924 icode
= direct_optab_handler (optab
->no_result
, mode
);
7928 /* Otherwise, we need to generate a result. */
7933 icode
= direct_optab_handler (after
? optab
->mem_fetch_after
7934 : optab
->mem_fetch_before
, mode
);
7935 create_integer_operand (&ops
[3], model
);
7940 icode
= optab_handler (after
? optab
->fetch_after
7941 : optab
->fetch_before
, mode
);
7944 create_output_operand (&ops
[op_counter
++], target
, mode
);
7946 if (icode
== CODE_FOR_nothing
)
7949 create_fixed_operand (&ops
[op_counter
++], mem
);
7950 /* VAL may have been promoted to a wider mode. Shrink it if so. */
7951 create_convert_operand_to (&ops
[op_counter
++], val
, mode
, true);
7953 if (maybe_expand_insn (icode
, num_ops
, ops
))
7954 return (target
== const0_rtx
? const0_rtx
: ops
[0].value
);
7960 /* This function expands an atomic fetch_OP or OP_fetch operation:
7961 TARGET is an option place to stick the return value. const0_rtx indicates
7962 the result is unused.
7963 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7964 CODE is the operation being performed (OP)
7965 MEMMODEL is the memory model variant to use.
7966 AFTER is true to return the result of the operation (OP_fetch).
7967 AFTER is false to return the value before the operation (fetch_OP).
7969 This function will *only* generate instructions if there is a direct
7970 optab. No compare and swap loops or libcalls will be generated. */
7973 expand_atomic_fetch_op_no_fallback (rtx target
, rtx mem
, rtx val
,
7974 enum rtx_code code
, enum memmodel model
,
7977 machine_mode mode
= GET_MODE (mem
);
7978 struct atomic_op_functions optab
;
7980 bool unused_result
= (target
== const0_rtx
);
7982 get_atomic_op_for_code (&optab
, code
);
7984 /* Check to see if there are any better instructions. */
7985 result
= maybe_optimize_fetch_op (target
, mem
, val
, code
, model
, after
);
7989 /* Check for the case where the result isn't used and try those patterns. */
7992 /* Try the memory model variant first. */
7993 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, true);
7997 /* Next try the old style withuot a memory model. */
7998 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, true);
8002 /* There is no no-result pattern, so try patterns with a result. */
8006 /* Try the __atomic version. */
8007 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, after
);
8011 /* Try the older __sync version. */
8012 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, after
);
8016 /* If the fetch value can be calculated from the other variation of fetch,
8017 try that operation. */
8018 if (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
)
8020 /* Try the __atomic version, then the older __sync version. */
8021 result
= maybe_emit_op (&optab
, target
, mem
, val
, true, model
, !after
);
8023 result
= maybe_emit_op (&optab
, target
, mem
, val
, false, model
, !after
);
8027 /* If the result isn't used, no need to do compensation code. */
8031 /* Issue compensation code. Fetch_after == fetch_before OP val.
8032 Fetch_before == after REVERSE_OP val. */
8034 code
= optab
.reverse_code
;
8037 result
= expand_simple_binop (mode
, AND
, result
, val
, NULL_RTX
,
8038 true, OPTAB_LIB_WIDEN
);
8039 result
= expand_simple_unop (mode
, NOT
, result
, target
, true);
8042 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
8043 true, OPTAB_LIB_WIDEN
);
8048 /* No direct opcode can be generated. */
8054 /* This function expands an atomic fetch_OP or OP_fetch operation:
8055 TARGET is an option place to stick the return value. const0_rtx indicates
8056 the result is unused.
8057 atomically fetch MEM, perform the operation with VAL and return it to MEM.
8058 CODE is the operation being performed (OP)
8059 MEMMODEL is the memory model variant to use.
8060 AFTER is true to return the result of the operation (OP_fetch).
8061 AFTER is false to return the value before the operation (fetch_OP). */
8063 expand_atomic_fetch_op (rtx target
, rtx mem
, rtx val
, enum rtx_code code
,
8064 enum memmodel model
, bool after
)
8066 machine_mode mode
= GET_MODE (mem
);
8068 bool unused_result
= (target
== const0_rtx
);
8070 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, val
, code
, model
,
8076 /* Add/sub can be implemented by doing the reverse operation with -(val). */
8077 if (code
== PLUS
|| code
== MINUS
)
8080 enum rtx_code reverse
= (code
== PLUS
? MINUS
: PLUS
);
8083 tmp
= expand_simple_unop (mode
, NEG
, val
, NULL_RTX
, true);
8084 result
= expand_atomic_fetch_op_no_fallback (target
, mem
, tmp
, reverse
,
8088 /* PLUS worked so emit the insns and return. */
8095 /* PLUS did not work, so throw away the negation code and continue. */
8099 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
8100 if (!can_compare_and_swap_p (mode
, false))
8104 enum rtx_code orig_code
= code
;
8105 struct atomic_op_functions optab
;
8107 get_atomic_op_for_code (&optab
, code
);
8108 libfunc
= optab_libfunc (after
? optab
.fetch_after
8109 : optab
.fetch_before
, mode
);
8111 && (after
|| unused_result
|| optab
.reverse_code
!= UNKNOWN
))
8115 code
= optab
.reverse_code
;
8116 libfunc
= optab_libfunc (after
? optab
.fetch_before
8117 : optab
.fetch_after
, mode
);
8119 if (libfunc
!= NULL
)
8121 rtx addr
= convert_memory_address (ptr_mode
, XEXP (mem
, 0));
8122 result
= emit_library_call_value (libfunc
, NULL
, LCT_NORMAL
, mode
,
8123 2, addr
, ptr_mode
, val
, mode
);
8125 if (!unused_result
&& fixup
)
8126 result
= expand_simple_binop (mode
, code
, result
, val
, target
,
8127 true, OPTAB_LIB_WIDEN
);
8131 /* We need the original code for any further attempts. */
8135 /* If nothing else has succeeded, default to a compare and swap loop. */
8136 if (can_compare_and_swap_p (mode
, true))
8139 rtx t0
= gen_reg_rtx (mode
), t1
;
8143 /* If the result is used, get a register for it. */
8146 if (!target
|| !register_operand (target
, mode
))
8147 target
= gen_reg_rtx (mode
);
8148 /* If fetch_before, copy the value now. */
8150 emit_move_insn (target
, t0
);
8153 target
= const0_rtx
;
8158 t1
= expand_simple_binop (mode
, AND
, t1
, val
, NULL_RTX
,
8159 true, OPTAB_LIB_WIDEN
);
8160 t1
= expand_simple_unop (mode
, code
, t1
, NULL_RTX
, true);
8163 t1
= expand_simple_binop (mode
, code
, t1
, val
, NULL_RTX
, true,
8166 /* For after, copy the value now. */
8167 if (!unused_result
&& after
)
8168 emit_move_insn (target
, t1
);
8169 insn
= get_insns ();
8172 if (t1
!= NULL
&& expand_compare_and_swap_loop (mem
, t0
, t1
, insn
))
8179 /* Return true if OPERAND is suitable for operand number OPNO of
8180 instruction ICODE. */
8183 insn_operand_matches (enum insn_code icode
, unsigned int opno
, rtx operand
)
8185 return (!insn_data
[(int) icode
].operand
[opno
].predicate
8186 || (insn_data
[(int) icode
].operand
[opno
].predicate
8187 (operand
, insn_data
[(int) icode
].operand
[opno
].mode
)));
8190 /* TARGET is a target of a multiword operation that we are going to
8191 implement as a series of word-mode operations. Return true if
8192 TARGET is suitable for this purpose. */
8195 valid_multiword_target_p (rtx target
)
8200 mode
= GET_MODE (target
);
8201 for (i
= 0; i
< GET_MODE_SIZE (mode
); i
+= UNITS_PER_WORD
)
8202 if (!validate_subreg (word_mode
, mode
, target
, i
))
8207 /* Like maybe_legitimize_operand, but do not change the code of the
8208 current rtx value. */
8211 maybe_legitimize_operand_same_code (enum insn_code icode
, unsigned int opno
,
8212 struct expand_operand
*op
)
8214 /* See if the operand matches in its current form. */
8215 if (insn_operand_matches (icode
, opno
, op
->value
))
8218 /* If the operand is a memory whose address has no side effects,
8219 try forcing the address into a non-virtual pseudo register.
8220 The check for side effects is important because copy_to_mode_reg
8221 cannot handle things like auto-modified addresses. */
8222 if (insn_data
[(int) icode
].operand
[opno
].allows_mem
&& MEM_P (op
->value
))
8227 addr
= XEXP (mem
, 0);
8228 if (!(REG_P (addr
) && REGNO (addr
) > LAST_VIRTUAL_REGISTER
)
8229 && !side_effects_p (addr
))
8234 last
= get_last_insn ();
8235 mode
= get_address_mode (mem
);
8236 mem
= replace_equiv_address (mem
, copy_to_mode_reg (mode
, addr
));
8237 if (insn_operand_matches (icode
, opno
, mem
))
8242 delete_insns_since (last
);
8249 /* Try to make OP match operand OPNO of instruction ICODE. Return true
8250 on success, storing the new operand value back in OP. */
8253 maybe_legitimize_operand (enum insn_code icode
, unsigned int opno
,
8254 struct expand_operand
*op
)
8256 machine_mode mode
, imode
;
8257 bool old_volatile_ok
, result
;
8263 old_volatile_ok
= volatile_ok
;
8265 result
= maybe_legitimize_operand_same_code (icode
, opno
, op
);
8266 volatile_ok
= old_volatile_ok
;
8270 gcc_assert (mode
!= VOIDmode
);
8272 && op
->value
!= const0_rtx
8273 && GET_MODE (op
->value
) == mode
8274 && maybe_legitimize_operand_same_code (icode
, opno
, op
))
8277 op
->value
= gen_reg_rtx (mode
);
8282 gcc_assert (mode
!= VOIDmode
);
8283 gcc_assert (GET_MODE (op
->value
) == VOIDmode
8284 || GET_MODE (op
->value
) == mode
);
8285 if (maybe_legitimize_operand_same_code (icode
, opno
, op
))
8288 op
->value
= copy_to_mode_reg (mode
, op
->value
);
8291 case EXPAND_CONVERT_TO
:
8292 gcc_assert (mode
!= VOIDmode
);
8293 op
->value
= convert_to_mode (mode
, op
->value
, op
->unsigned_p
);
8296 case EXPAND_CONVERT_FROM
:
8297 if (GET_MODE (op
->value
) != VOIDmode
)
8298 mode
= GET_MODE (op
->value
);
8300 /* The caller must tell us what mode this value has. */
8301 gcc_assert (mode
!= VOIDmode
);
8303 imode
= insn_data
[(int) icode
].operand
[opno
].mode
;
8304 if (imode
!= VOIDmode
&& imode
!= mode
)
8306 op
->value
= convert_modes (imode
, mode
, op
->value
, op
->unsigned_p
);
8311 case EXPAND_ADDRESS
:
8312 gcc_assert (mode
!= VOIDmode
);
8313 op
->value
= convert_memory_address (mode
, op
->value
);
8316 case EXPAND_INTEGER
:
8317 mode
= insn_data
[(int) icode
].operand
[opno
].mode
;
8318 if (mode
!= VOIDmode
&& const_int_operand (op
->value
, mode
))
8322 return insn_operand_matches (icode
, opno
, op
->value
);
8325 /* Make OP describe an input operand that should have the same value
8326 as VALUE, after any mode conversion that the target might request.
8327 TYPE is the type of VALUE. */
8330 create_convert_operand_from_type (struct expand_operand
*op
,
8331 rtx value
, tree type
)
8333 create_convert_operand_from (op
, value
, TYPE_MODE (type
),
8334 TYPE_UNSIGNED (type
));
8337 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
8338 of instruction ICODE. Return true on success, leaving the new operand
8339 values in the OPS themselves. Emit no code on failure. */
8342 maybe_legitimize_operands (enum insn_code icode
, unsigned int opno
,
8343 unsigned int nops
, struct expand_operand
*ops
)
8348 last
= get_last_insn ();
8349 for (i
= 0; i
< nops
; i
++)
8350 if (!maybe_legitimize_operand (icode
, opno
+ i
, &ops
[i
]))
8352 delete_insns_since (last
);
8358 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
8359 as its operands. Return the instruction pattern on success,
8360 and emit any necessary set-up code. Return null and emit no
8364 maybe_gen_insn (enum insn_code icode
, unsigned int nops
,
8365 struct expand_operand
*ops
)
8367 gcc_assert (nops
== (unsigned int) insn_data
[(int) icode
].n_generator_args
);
8368 if (!maybe_legitimize_operands (icode
, 0, nops
, ops
))
8374 return GEN_FCN (icode
) (ops
[0].value
);
8376 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
);
8378 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
);
8380 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8383 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8384 ops
[3].value
, ops
[4].value
);
8386 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8387 ops
[3].value
, ops
[4].value
, ops
[5].value
);
8389 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8390 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8393 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8394 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8395 ops
[6].value
, ops
[7].value
);
8397 return GEN_FCN (icode
) (ops
[0].value
, ops
[1].value
, ops
[2].value
,
8398 ops
[3].value
, ops
[4].value
, ops
[5].value
,
8399 ops
[6].value
, ops
[7].value
, ops
[8].value
);
8404 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
8405 as its operands. Return true on success and emit no code on failure. */
8408 maybe_expand_insn (enum insn_code icode
, unsigned int nops
,
8409 struct expand_operand
*ops
)
8411 rtx pat
= maybe_gen_insn (icode
, nops
, ops
);
8420 /* Like maybe_expand_insn, but for jumps. */
8423 maybe_expand_jump_insn (enum insn_code icode
, unsigned int nops
,
8424 struct expand_operand
*ops
)
8426 rtx pat
= maybe_gen_insn (icode
, nops
, ops
);
8429 emit_jump_insn (pat
);
8435 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
8439 expand_insn (enum insn_code icode
, unsigned int nops
,
8440 struct expand_operand
*ops
)
8442 if (!maybe_expand_insn (icode
, nops
, ops
))
8446 /* Like expand_insn, but for jumps. */
8449 expand_jump_insn (enum insn_code icode
, unsigned int nops
,
8450 struct expand_operand
*ops
)
8452 if (!maybe_expand_jump_insn (icode
, nops
, ops
))
8456 /* Reduce conditional compilation elsewhere. */
8459 #define CODE_FOR_insv CODE_FOR_nothing
8463 #define CODE_FOR_extv CODE_FOR_nothing
8466 #define HAVE_extzv 0
8467 #define CODE_FOR_extzv CODE_FOR_nothing
8470 /* Enumerates the possible types of structure operand to an
8472 enum extraction_type
{ ET_unaligned_mem
, ET_reg
};
8474 /* Check whether insv, extv or extzv pattern ICODE can be used for an
8475 insertion or extraction of type TYPE on a structure of mode MODE.
8476 Return true if so and fill in *INSN accordingly. STRUCT_OP is the
8477 operand number of the structure (the first sign_extract or zero_extract
8478 operand) and FIELD_OP is the operand number of the field (the other
8479 side of the set from the sign_extract or zero_extract). */
8482 get_traditional_extraction_insn (extraction_insn
*insn
,
8483 enum extraction_type type
,
8485 enum insn_code icode
,
8486 int struct_op
, int field_op
)
8488 const struct insn_data_d
*data
= &insn_data
[icode
];
8490 machine_mode struct_mode
= data
->operand
[struct_op
].mode
;
8491 if (struct_mode
== VOIDmode
)
8492 struct_mode
= word_mode
;
8493 if (mode
!= struct_mode
)
8496 machine_mode field_mode
= data
->operand
[field_op
].mode
;
8497 if (field_mode
== VOIDmode
)
8498 field_mode
= word_mode
;
8500 machine_mode pos_mode
= data
->operand
[struct_op
+ 2].mode
;
8501 if (pos_mode
== VOIDmode
)
8502 pos_mode
= word_mode
;
8504 insn
->icode
= icode
;
8505 insn
->field_mode
= field_mode
;
8506 insn
->struct_mode
= (type
== ET_unaligned_mem
? byte_mode
: struct_mode
);
8507 insn
->pos_mode
= pos_mode
;
8511 /* Return true if an optab exists to perform an insertion or extraction
8512 of type TYPE in mode MODE. Describe the instruction in *INSN if so.
8514 REG_OPTAB is the optab to use for register structures and
8515 MISALIGN_OPTAB is the optab to use for misaligned memory structures.
8516 POS_OP is the operand number of the bit position. */
8519 get_optab_extraction_insn (struct extraction_insn
*insn
,
8520 enum extraction_type type
,
8521 machine_mode mode
, direct_optab reg_optab
,
8522 direct_optab misalign_optab
, int pos_op
)
8524 direct_optab optab
= (type
== ET_unaligned_mem
? misalign_optab
: reg_optab
);
8525 enum insn_code icode
= direct_optab_handler (optab
, mode
);
8526 if (icode
== CODE_FOR_nothing
)
8529 const struct insn_data_d
*data
= &insn_data
[icode
];
8531 insn
->icode
= icode
;
8532 insn
->field_mode
= mode
;
8533 insn
->struct_mode
= (type
== ET_unaligned_mem
? BLKmode
: mode
);
8534 insn
->pos_mode
= data
->operand
[pos_op
].mode
;
8535 if (insn
->pos_mode
== VOIDmode
)
8536 insn
->pos_mode
= word_mode
;
8540 /* Return true if an instruction exists to perform an insertion or
8541 extraction (PATTERN says which) of type TYPE in mode MODE.
8542 Describe the instruction in *INSN if so. */
8545 get_extraction_insn (extraction_insn
*insn
,
8546 enum extraction_pattern pattern
,
8547 enum extraction_type type
,
8554 && get_traditional_extraction_insn (insn
, type
, mode
,
8555 CODE_FOR_insv
, 0, 3))
8557 return get_optab_extraction_insn (insn
, type
, mode
, insv_optab
,
8558 insvmisalign_optab
, 2);
8562 && get_traditional_extraction_insn (insn
, type
, mode
,
8563 CODE_FOR_extv
, 1, 0))
8565 return get_optab_extraction_insn (insn
, type
, mode
, extv_optab
,
8566 extvmisalign_optab
, 3);
8570 && get_traditional_extraction_insn (insn
, type
, mode
,
8571 CODE_FOR_extzv
, 1, 0))
8573 return get_optab_extraction_insn (insn
, type
, mode
, extzv_optab
,
8574 extzvmisalign_optab
, 3);
8581 /* Return true if an instruction exists to access a field of mode
8582 FIELDMODE in a structure that has STRUCT_BITS significant bits.
8583 Describe the "best" such instruction in *INSN if so. PATTERN and
8584 TYPE describe the type of insertion or extraction we want to perform.
8586 For an insertion, the number of significant structure bits includes
8587 all bits of the target. For an extraction, it need only include the
8588 most significant bit of the field. Larger widths are acceptable
8592 get_best_extraction_insn (extraction_insn
*insn
,
8593 enum extraction_pattern pattern
,
8594 enum extraction_type type
,
8595 unsigned HOST_WIDE_INT struct_bits
,
8596 machine_mode field_mode
)
8598 machine_mode mode
= smallest_mode_for_size (struct_bits
, MODE_INT
);
8599 while (mode
!= VOIDmode
)
8601 if (get_extraction_insn (insn
, pattern
, type
, mode
))
8603 while (mode
!= VOIDmode
8604 && GET_MODE_SIZE (mode
) <= GET_MODE_SIZE (field_mode
)
8605 && !TRULY_NOOP_TRUNCATION_MODES_P (insn
->field_mode
,
8608 get_extraction_insn (insn
, pattern
, type
, mode
);
8609 mode
= GET_MODE_WIDER_MODE (mode
);
8613 mode
= GET_MODE_WIDER_MODE (mode
);
8618 /* Return true if an instruction exists to access a field of mode
8619 FIELDMODE in a register structure that has STRUCT_BITS significant bits.
8620 Describe the "best" such instruction in *INSN if so. PATTERN describes
8621 the type of insertion or extraction we want to perform.
8623 For an insertion, the number of significant structure bits includes
8624 all bits of the target. For an extraction, it need only include the
8625 most significant bit of the field. Larger widths are acceptable
8629 get_best_reg_extraction_insn (extraction_insn
*insn
,
8630 enum extraction_pattern pattern
,
8631 unsigned HOST_WIDE_INT struct_bits
,
8632 machine_mode field_mode
)
8634 return get_best_extraction_insn (insn
, pattern
, ET_reg
, struct_bits
,
8638 /* Return true if an instruction exists to access a field of BITSIZE
8639 bits starting BITNUM bits into a memory structure. Describe the
8640 "best" such instruction in *INSN if so. PATTERN describes the type
8641 of insertion or extraction we want to perform and FIELDMODE is the
8642 natural mode of the extracted field.
8644 The instructions considered here only access bytes that overlap
8645 the bitfield; they do not touch any surrounding bytes. */
8648 get_best_mem_extraction_insn (extraction_insn
*insn
,
8649 enum extraction_pattern pattern
,
8650 HOST_WIDE_INT bitsize
, HOST_WIDE_INT bitnum
,
8651 machine_mode field_mode
)
8653 unsigned HOST_WIDE_INT struct_bits
= (bitnum
% BITS_PER_UNIT
8655 + BITS_PER_UNIT
- 1);
8656 struct_bits
-= struct_bits
% BITS_PER_UNIT
;
8657 return get_best_extraction_insn (insn
, pattern
, ET_unaligned_mem
,
8658 struct_bits
, field_mode
);
8661 /* Determine whether "1 << x" is relatively cheap in word_mode. */
8664 lshift_cheap_p (bool speed_p
)
8666 /* FIXME: This should be made target dependent via this "this_target"
8667 mechanism, similar to e.g. can_copy_init_p in gcse.c. */
8668 static bool init
[2] = { false, false };
8669 static bool cheap
[2] = { true, true };
8671 /* If the targer has no lshift in word_mode, the operation will most
8672 probably not be cheap. ??? Does GCC even work for such targets? */
8673 if (optab_handler (ashl_optab
, word_mode
) == CODE_FOR_nothing
)
8678 rtx reg
= gen_raw_REG (word_mode
, 10000);
8679 int cost
= set_src_cost (gen_rtx_ASHIFT (word_mode
, const1_rtx
, reg
),
8681 cheap
[speed_p
] = cost
< COSTS_N_INSNS (3);
8682 init
[speed_p
] = true;
8685 return cheap
[speed_p
];
8688 #include "gt-optabs.h"