bind_c_array_params_2.f90: Add "-mno-explicit-relocs" for alpha*-*-* targets.
[gcc.git] / gcc / optabs.c
1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987, 1988, 1992, 1993, 1994, 1995, 1996, 1997, 1998,
3 1999, 2000, 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010,
4 2011, 2012 Free Software Foundation, Inc.
5
6 This file is part of GCC.
7
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
11 version.
12
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
21
22
23 #include "config.h"
24 #include "system.h"
25 #include "coretypes.h"
26 #include "tm.h"
27 #include "diagnostic-core.h"
28
29 /* Include insn-config.h before expr.h so that HAVE_conditional_move
30 is properly defined. */
31 #include "insn-config.h"
32 #include "rtl.h"
33 #include "tree.h"
34 #include "tm_p.h"
35 #include "flags.h"
36 #include "function.h"
37 #include "except.h"
38 #include "expr.h"
39 #include "optabs.h"
40 #include "libfuncs.h"
41 #include "recog.h"
42 #include "reload.h"
43 #include "ggc.h"
44 #include "basic-block.h"
45 #include "target.h"
46
47 struct target_optabs default_target_optabs;
48 struct target_libfuncs default_target_libfuncs;
49 #if SWITCHABLE_TARGET
50 struct target_optabs *this_target_optabs = &default_target_optabs;
51 struct target_libfuncs *this_target_libfuncs = &default_target_libfuncs;
52 #endif
53
54 #define libfunc_hash \
55 (this_target_libfuncs->x_libfunc_hash)
56
57 static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
58 enum machine_mode *);
59 static rtx expand_unop_direct (enum machine_mode, optab, rtx, rtx, int);
60 static void emit_libcall_block_1 (rtx, rtx, rtx, rtx, bool);
61
62 /* Debug facility for use in GDB. */
63 void debug_optab_libfuncs (void);
64
65 /* Prefixes for the current version of decimal floating point (BID vs. DPD) */
66 #if ENABLE_DECIMAL_BID_FORMAT
67 #define DECIMAL_PREFIX "bid_"
68 #else
69 #define DECIMAL_PREFIX "dpd_"
70 #endif
71 \f
72 /* Used for libfunc_hash. */
73
74 static hashval_t
75 hash_libfunc (const void *p)
76 {
77 const struct libfunc_entry *const e = (const struct libfunc_entry *) p;
78 return ((e->mode1 + e->mode2 * NUM_MACHINE_MODES) ^ e->op);
79 }
80
81 /* Used for libfunc_hash. */
82
83 static int
84 eq_libfunc (const void *p, const void *q)
85 {
86 const struct libfunc_entry *const e1 = (const struct libfunc_entry *) p;
87 const struct libfunc_entry *const e2 = (const struct libfunc_entry *) q;
88 return e1->op == e2->op && e1->mode1 == e2->mode1 && e1->mode2 == e2->mode2;
89 }
90
91 /* Return libfunc corresponding operation defined by OPTAB converting
92 from MODE2 to MODE1. Trigger lazy initialization if needed, return NULL
93 if no libfunc is available. */
94 rtx
95 convert_optab_libfunc (convert_optab optab, enum machine_mode mode1,
96 enum machine_mode mode2)
97 {
98 struct libfunc_entry e;
99 struct libfunc_entry **slot;
100
101 /* ??? This ought to be an assert, but not all of the places
102 that we expand optabs know about the optabs that got moved
103 to being direct. */
104 if (!(optab >= FIRST_CONV_OPTAB && optab <= LAST_CONVLIB_OPTAB))
105 return NULL_RTX;
106
107 e.op = optab;
108 e.mode1 = mode1;
109 e.mode2 = mode2;
110 slot = (struct libfunc_entry **)
111 htab_find_slot (libfunc_hash, &e, NO_INSERT);
112 if (!slot)
113 {
114 const struct convert_optab_libcall_d *d
115 = &convlib_def[optab - FIRST_CONV_OPTAB];
116
117 if (d->libcall_gen == NULL)
118 return NULL;
119
120 d->libcall_gen (optab, d->libcall_basename, mode1, mode2);
121 slot = (struct libfunc_entry **)
122 htab_find_slot (libfunc_hash, &e, NO_INSERT);
123 if (!slot)
124 return NULL;
125 }
126 return (*slot)->libfunc;
127 }
128
129 /* Return libfunc corresponding operation defined by OPTAB in MODE.
130 Trigger lazy initialization if needed, return NULL if no libfunc is
131 available. */
132 rtx
133 optab_libfunc (optab optab, enum machine_mode mode)
134 {
135 struct libfunc_entry e;
136 struct libfunc_entry **slot;
137
138 /* ??? This ought to be an assert, but not all of the places
139 that we expand optabs know about the optabs that got moved
140 to being direct. */
141 if (!(optab >= FIRST_NORM_OPTAB && optab <= LAST_NORMLIB_OPTAB))
142 return NULL_RTX;
143
144 e.op = optab;
145 e.mode1 = mode;
146 e.mode2 = VOIDmode;
147 slot = (struct libfunc_entry **)
148 htab_find_slot (libfunc_hash, &e, NO_INSERT);
149 if (!slot)
150 {
151 const struct optab_libcall_d *d
152 = &normlib_def[optab - FIRST_NORM_OPTAB];
153
154 if (d->libcall_gen == NULL)
155 return NULL;
156
157 d->libcall_gen (optab, d->libcall_basename, d->libcall_suffix, mode);
158 slot = (struct libfunc_entry **)
159 htab_find_slot (libfunc_hash, &e, NO_INSERT);
160 if (!slot)
161 return NULL;
162 }
163 return (*slot)->libfunc;
164 }
165
166 \f
167 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
168 the result of operation CODE applied to OP0 (and OP1 if it is a binary
169 operation).
170
171 If the last insn does not set TARGET, don't do anything, but return 1.
172
173 If a previous insn sets TARGET and TARGET is one of OP0 or OP1,
174 don't add the REG_EQUAL note but return 0. Our caller can then try
175 again, ensuring that TARGET is not one of the operands. */
176
177 static int
178 add_equal_note (rtx insns, rtx target, enum rtx_code code, rtx op0, rtx op1)
179 {
180 rtx last_insn, insn, set;
181 rtx note;
182
183 gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
184
185 if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
186 && GET_RTX_CLASS (code) != RTX_BIN_ARITH
187 && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
188 && GET_RTX_CLASS (code) != RTX_COMPARE
189 && GET_RTX_CLASS (code) != RTX_UNARY)
190 return 1;
191
192 if (GET_CODE (target) == ZERO_EXTRACT)
193 return 1;
194
195 for (last_insn = insns;
196 NEXT_INSN (last_insn) != NULL_RTX;
197 last_insn = NEXT_INSN (last_insn))
198 ;
199
200 set = single_set (last_insn);
201 if (set == NULL_RTX)
202 return 1;
203
204 if (! rtx_equal_p (SET_DEST (set), target)
205 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
206 && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
207 || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
208 return 1;
209
210 /* If TARGET is in OP0 or OP1, check if anything in SEQ sets TARGET
211 besides the last insn. */
212 if (reg_overlap_mentioned_p (target, op0)
213 || (op1 && reg_overlap_mentioned_p (target, op1)))
214 {
215 insn = PREV_INSN (last_insn);
216 while (insn != NULL_RTX)
217 {
218 if (reg_set_p (target, insn))
219 return 0;
220
221 insn = PREV_INSN (insn);
222 }
223 }
224
225 if (GET_RTX_CLASS (code) == RTX_UNARY)
226 switch (code)
227 {
228 case FFS:
229 case CLZ:
230 case CTZ:
231 case CLRSB:
232 case POPCOUNT:
233 case PARITY:
234 case BSWAP:
235 if (GET_MODE (op0) != VOIDmode && GET_MODE (target) != GET_MODE (op0))
236 {
237 note = gen_rtx_fmt_e (code, GET_MODE (op0), copy_rtx (op0));
238 if (GET_MODE_SIZE (GET_MODE (op0))
239 > GET_MODE_SIZE (GET_MODE (target)))
240 note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
241 note, GET_MODE (op0));
242 else
243 note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
244 note, GET_MODE (op0));
245 break;
246 }
247 /* FALLTHRU */
248 default:
249 note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
250 break;
251 }
252 else
253 note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
254
255 set_unique_reg_note (last_insn, REG_EQUAL, note);
256
257 return 1;
258 }
259 \f
260 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
261 for a widening operation would be. In most cases this would be OP0, but if
262 that's a constant it'll be VOIDmode, which isn't useful. */
263
264 static enum machine_mode
265 widened_mode (enum machine_mode to_mode, rtx op0, rtx op1)
266 {
267 enum machine_mode m0 = GET_MODE (op0);
268 enum machine_mode m1 = GET_MODE (op1);
269 enum machine_mode result;
270
271 if (m0 == VOIDmode && m1 == VOIDmode)
272 return to_mode;
273 else if (m0 == VOIDmode || GET_MODE_SIZE (m0) < GET_MODE_SIZE (m1))
274 result = m1;
275 else
276 result = m0;
277
278 if (GET_MODE_SIZE (result) > GET_MODE_SIZE (to_mode))
279 return to_mode;
280
281 return result;
282 }
283 \f
284 /* Find a widening optab even if it doesn't widen as much as we want.
285 E.g. if from_mode is HImode, and to_mode is DImode, and there is no
286 direct HI->SI insn, then return SI->DI, if that exists.
287 If PERMIT_NON_WIDENING is non-zero then this can be used with
288 non-widening optabs also. */
289
290 enum insn_code
291 find_widening_optab_handler_and_mode (optab op, enum machine_mode to_mode,
292 enum machine_mode from_mode,
293 int permit_non_widening,
294 enum machine_mode *found_mode)
295 {
296 for (; (permit_non_widening || from_mode != to_mode)
297 && GET_MODE_SIZE (from_mode) <= GET_MODE_SIZE (to_mode)
298 && from_mode != VOIDmode;
299 from_mode = GET_MODE_WIDER_MODE (from_mode))
300 {
301 enum insn_code handler = widening_optab_handler (op, to_mode,
302 from_mode);
303
304 if (handler != CODE_FOR_nothing)
305 {
306 if (found_mode)
307 *found_mode = from_mode;
308 return handler;
309 }
310 }
311
312 return CODE_FOR_nothing;
313 }
314 \f
315 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
316 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
317 not actually do a sign-extend or zero-extend, but can leave the
318 higher-order bits of the result rtx undefined, for example, in the case
319 of logical operations, but not right shifts. */
320
321 static rtx
322 widen_operand (rtx op, enum machine_mode mode, enum machine_mode oldmode,
323 int unsignedp, int no_extend)
324 {
325 rtx result;
326
327 /* If we don't have to extend and this is a constant, return it. */
328 if (no_extend && GET_MODE (op) == VOIDmode)
329 return op;
330
331 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
332 extend since it will be more efficient to do so unless the signedness of
333 a promoted object differs from our extension. */
334 if (! no_extend
335 || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
336 && SUBREG_PROMOTED_UNSIGNED_P (op) == unsignedp))
337 return convert_modes (mode, oldmode, op, unsignedp);
338
339 /* If MODE is no wider than a single word, we return a paradoxical
340 SUBREG. */
341 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
342 return gen_rtx_SUBREG (mode, force_reg (GET_MODE (op), op), 0);
343
344 /* Otherwise, get an object of MODE, clobber it, and set the low-order
345 part to OP. */
346
347 result = gen_reg_rtx (mode);
348 emit_clobber (result);
349 emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
350 return result;
351 }
352 \f
353 /* Return the optab used for computing the operation given by the tree code,
354 CODE and the tree EXP. This function is not always usable (for example, it
355 cannot give complete results for multiplication or division) but probably
356 ought to be relied on more widely throughout the expander. */
357 optab
358 optab_for_tree_code (enum tree_code code, const_tree type,
359 enum optab_subtype subtype)
360 {
361 bool trapv;
362 switch (code)
363 {
364 case BIT_AND_EXPR:
365 return and_optab;
366
367 case BIT_IOR_EXPR:
368 return ior_optab;
369
370 case BIT_NOT_EXPR:
371 return one_cmpl_optab;
372
373 case BIT_XOR_EXPR:
374 return xor_optab;
375
376 case MULT_HIGHPART_EXPR:
377 return TYPE_UNSIGNED (type) ? umul_highpart_optab : smul_highpart_optab;
378
379 case TRUNC_MOD_EXPR:
380 case CEIL_MOD_EXPR:
381 case FLOOR_MOD_EXPR:
382 case ROUND_MOD_EXPR:
383 return TYPE_UNSIGNED (type) ? umod_optab : smod_optab;
384
385 case RDIV_EXPR:
386 case TRUNC_DIV_EXPR:
387 case CEIL_DIV_EXPR:
388 case FLOOR_DIV_EXPR:
389 case ROUND_DIV_EXPR:
390 case EXACT_DIV_EXPR:
391 if (TYPE_SATURATING(type))
392 return TYPE_UNSIGNED(type) ? usdiv_optab : ssdiv_optab;
393 return TYPE_UNSIGNED (type) ? udiv_optab : sdiv_optab;
394
395 case LSHIFT_EXPR:
396 if (TREE_CODE (type) == VECTOR_TYPE)
397 {
398 if (subtype == optab_vector)
399 return TYPE_SATURATING (type) ? unknown_optab : vashl_optab;
400
401 gcc_assert (subtype == optab_scalar);
402 }
403 if (TYPE_SATURATING(type))
404 return TYPE_UNSIGNED(type) ? usashl_optab : ssashl_optab;
405 return ashl_optab;
406
407 case RSHIFT_EXPR:
408 if (TREE_CODE (type) == VECTOR_TYPE)
409 {
410 if (subtype == optab_vector)
411 return TYPE_UNSIGNED (type) ? vlshr_optab : vashr_optab;
412
413 gcc_assert (subtype == optab_scalar);
414 }
415 return TYPE_UNSIGNED (type) ? lshr_optab : ashr_optab;
416
417 case LROTATE_EXPR:
418 if (TREE_CODE (type) == VECTOR_TYPE)
419 {
420 if (subtype == optab_vector)
421 return vrotl_optab;
422
423 gcc_assert (subtype == optab_scalar);
424 }
425 return rotl_optab;
426
427 case RROTATE_EXPR:
428 if (TREE_CODE (type) == VECTOR_TYPE)
429 {
430 if (subtype == optab_vector)
431 return vrotr_optab;
432
433 gcc_assert (subtype == optab_scalar);
434 }
435 return rotr_optab;
436
437 case MAX_EXPR:
438 return TYPE_UNSIGNED (type) ? umax_optab : smax_optab;
439
440 case MIN_EXPR:
441 return TYPE_UNSIGNED (type) ? umin_optab : smin_optab;
442
443 case REALIGN_LOAD_EXPR:
444 return vec_realign_load_optab;
445
446 case WIDEN_SUM_EXPR:
447 return TYPE_UNSIGNED (type) ? usum_widen_optab : ssum_widen_optab;
448
449 case DOT_PROD_EXPR:
450 return TYPE_UNSIGNED (type) ? udot_prod_optab : sdot_prod_optab;
451
452 case WIDEN_MULT_PLUS_EXPR:
453 return (TYPE_UNSIGNED (type)
454 ? (TYPE_SATURATING (type)
455 ? usmadd_widen_optab : umadd_widen_optab)
456 : (TYPE_SATURATING (type)
457 ? ssmadd_widen_optab : smadd_widen_optab));
458
459 case WIDEN_MULT_MINUS_EXPR:
460 return (TYPE_UNSIGNED (type)
461 ? (TYPE_SATURATING (type)
462 ? usmsub_widen_optab : umsub_widen_optab)
463 : (TYPE_SATURATING (type)
464 ? ssmsub_widen_optab : smsub_widen_optab));
465
466 case FMA_EXPR:
467 return fma_optab;
468
469 case REDUC_MAX_EXPR:
470 return TYPE_UNSIGNED (type) ? reduc_umax_optab : reduc_smax_optab;
471
472 case REDUC_MIN_EXPR:
473 return TYPE_UNSIGNED (type) ? reduc_umin_optab : reduc_smin_optab;
474
475 case REDUC_PLUS_EXPR:
476 return TYPE_UNSIGNED (type) ? reduc_uplus_optab : reduc_splus_optab;
477
478 case VEC_LSHIFT_EXPR:
479 return vec_shl_optab;
480
481 case VEC_RSHIFT_EXPR:
482 return vec_shr_optab;
483
484 case VEC_WIDEN_MULT_HI_EXPR:
485 return TYPE_UNSIGNED (type) ?
486 vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
487
488 case VEC_WIDEN_MULT_LO_EXPR:
489 return TYPE_UNSIGNED (type) ?
490 vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
491
492 case VEC_WIDEN_MULT_EVEN_EXPR:
493 return TYPE_UNSIGNED (type) ?
494 vec_widen_umult_even_optab : vec_widen_smult_even_optab;
495
496 case VEC_WIDEN_MULT_ODD_EXPR:
497 return TYPE_UNSIGNED (type) ?
498 vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
499
500 case VEC_WIDEN_LSHIFT_HI_EXPR:
501 return TYPE_UNSIGNED (type) ?
502 vec_widen_ushiftl_hi_optab : vec_widen_sshiftl_hi_optab;
503
504 case VEC_WIDEN_LSHIFT_LO_EXPR:
505 return TYPE_UNSIGNED (type) ?
506 vec_widen_ushiftl_lo_optab : vec_widen_sshiftl_lo_optab;
507
508 case VEC_UNPACK_HI_EXPR:
509 return TYPE_UNSIGNED (type) ?
510 vec_unpacku_hi_optab : vec_unpacks_hi_optab;
511
512 case VEC_UNPACK_LO_EXPR:
513 return TYPE_UNSIGNED (type) ?
514 vec_unpacku_lo_optab : vec_unpacks_lo_optab;
515
516 case VEC_UNPACK_FLOAT_HI_EXPR:
517 /* The signedness is determined from input operand. */
518 return TYPE_UNSIGNED (type) ?
519 vec_unpacku_float_hi_optab : vec_unpacks_float_hi_optab;
520
521 case VEC_UNPACK_FLOAT_LO_EXPR:
522 /* The signedness is determined from input operand. */
523 return TYPE_UNSIGNED (type) ?
524 vec_unpacku_float_lo_optab : vec_unpacks_float_lo_optab;
525
526 case VEC_PACK_TRUNC_EXPR:
527 return vec_pack_trunc_optab;
528
529 case VEC_PACK_SAT_EXPR:
530 return TYPE_UNSIGNED (type) ? vec_pack_usat_optab : vec_pack_ssat_optab;
531
532 case VEC_PACK_FIX_TRUNC_EXPR:
533 /* The signedness is determined from output operand. */
534 return TYPE_UNSIGNED (type) ?
535 vec_pack_ufix_trunc_optab : vec_pack_sfix_trunc_optab;
536
537 default:
538 break;
539 }
540
541 trapv = INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type);
542 switch (code)
543 {
544 case POINTER_PLUS_EXPR:
545 case PLUS_EXPR:
546 if (TYPE_SATURATING(type))
547 return TYPE_UNSIGNED(type) ? usadd_optab : ssadd_optab;
548 return trapv ? addv_optab : add_optab;
549
550 case MINUS_EXPR:
551 if (TYPE_SATURATING(type))
552 return TYPE_UNSIGNED(type) ? ussub_optab : sssub_optab;
553 return trapv ? subv_optab : sub_optab;
554
555 case MULT_EXPR:
556 if (TYPE_SATURATING(type))
557 return TYPE_UNSIGNED(type) ? usmul_optab : ssmul_optab;
558 return trapv ? smulv_optab : smul_optab;
559
560 case NEGATE_EXPR:
561 if (TYPE_SATURATING(type))
562 return TYPE_UNSIGNED(type) ? usneg_optab : ssneg_optab;
563 return trapv ? negv_optab : neg_optab;
564
565 case ABS_EXPR:
566 return trapv ? absv_optab : abs_optab;
567
568 default:
569 return unknown_optab;
570 }
571 }
572 \f
573
574 /* Expand vector widening operations.
575
576 There are two different classes of operations handled here:
577 1) Operations whose result is wider than all the arguments to the operation.
578 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
579 In this case OP0 and optionally OP1 would be initialized,
580 but WIDE_OP wouldn't (not relevant for this case).
581 2) Operations whose result is of the same size as the last argument to the
582 operation, but wider than all the other arguments to the operation.
583 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
584 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
585
586 E.g, when called to expand the following operations, this is how
587 the arguments will be initialized:
588 nops OP0 OP1 WIDE_OP
589 widening-sum 2 oprnd0 - oprnd1
590 widening-dot-product 3 oprnd0 oprnd1 oprnd2
591 widening-mult 2 oprnd0 oprnd1 -
592 type-promotion (vec-unpack) 1 oprnd0 - - */
593
594 rtx
595 expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op,
596 rtx target, int unsignedp)
597 {
598 struct expand_operand eops[4];
599 tree oprnd0, oprnd1, oprnd2;
600 enum machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
601 optab widen_pattern_optab;
602 enum insn_code icode;
603 int nops = TREE_CODE_LENGTH (ops->code);
604 int op;
605
606 oprnd0 = ops->op0;
607 tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
608 widen_pattern_optab =
609 optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
610 if (ops->code == WIDEN_MULT_PLUS_EXPR
611 || ops->code == WIDEN_MULT_MINUS_EXPR)
612 icode = find_widening_optab_handler (widen_pattern_optab,
613 TYPE_MODE (TREE_TYPE (ops->op2)),
614 tmode0, 0);
615 else
616 icode = optab_handler (widen_pattern_optab, tmode0);
617 gcc_assert (icode != CODE_FOR_nothing);
618
619 if (nops >= 2)
620 {
621 oprnd1 = ops->op1;
622 tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
623 }
624
625 /* The last operand is of a wider mode than the rest of the operands. */
626 if (nops == 2)
627 wmode = tmode1;
628 else if (nops == 3)
629 {
630 gcc_assert (tmode1 == tmode0);
631 gcc_assert (op1);
632 oprnd2 = ops->op2;
633 wmode = TYPE_MODE (TREE_TYPE (oprnd2));
634 }
635
636 op = 0;
637 create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
638 create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
639 if (op1)
640 create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
641 if (wide_op)
642 create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
643 expand_insn (icode, op, eops);
644 return eops[0].value;
645 }
646
647 /* Generate code to perform an operation specified by TERNARY_OPTAB
648 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
649
650 UNSIGNEDP is for the case where we have to widen the operands
651 to perform the operation. It says to use zero-extension.
652
653 If TARGET is nonzero, the value
654 is generated there, if it is convenient to do so.
655 In all cases an rtx is returned for the locus of the value;
656 this may or may not be TARGET. */
657
658 rtx
659 expand_ternary_op (enum machine_mode mode, optab ternary_optab, rtx op0,
660 rtx op1, rtx op2, rtx target, int unsignedp)
661 {
662 struct expand_operand ops[4];
663 enum insn_code icode = optab_handler (ternary_optab, mode);
664
665 gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
666
667 create_output_operand (&ops[0], target, mode);
668 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
669 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
670 create_convert_operand_from (&ops[3], op2, mode, unsignedp);
671 expand_insn (icode, 4, ops);
672 return ops[0].value;
673 }
674
675
676 /* Like expand_binop, but return a constant rtx if the result can be
677 calculated at compile time. The arguments and return value are
678 otherwise the same as for expand_binop. */
679
680 rtx
681 simplify_expand_binop (enum machine_mode mode, optab binoptab,
682 rtx op0, rtx op1, rtx target, int unsignedp,
683 enum optab_methods methods)
684 {
685 if (CONSTANT_P (op0) && CONSTANT_P (op1))
686 {
687 rtx x = simplify_binary_operation (optab_to_code (binoptab),
688 mode, op0, op1);
689 if (x)
690 return x;
691 }
692
693 return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
694 }
695
696 /* Like simplify_expand_binop, but always put the result in TARGET.
697 Return true if the expansion succeeded. */
698
699 bool
700 force_expand_binop (enum machine_mode mode, optab binoptab,
701 rtx op0, rtx op1, rtx target, int unsignedp,
702 enum optab_methods methods)
703 {
704 rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
705 target, unsignedp, methods);
706 if (x == 0)
707 return false;
708 if (x != target)
709 emit_move_insn (target, x);
710 return true;
711 }
712
713 /* Generate insns for VEC_LSHIFT_EXPR, VEC_RSHIFT_EXPR. */
714
715 rtx
716 expand_vec_shift_expr (sepops ops, rtx target)
717 {
718 struct expand_operand eops[3];
719 enum insn_code icode;
720 rtx rtx_op1, rtx_op2;
721 enum machine_mode mode = TYPE_MODE (ops->type);
722 tree vec_oprnd = ops->op0;
723 tree shift_oprnd = ops->op1;
724 optab shift_optab;
725
726 switch (ops->code)
727 {
728 case VEC_RSHIFT_EXPR:
729 shift_optab = vec_shr_optab;
730 break;
731 case VEC_LSHIFT_EXPR:
732 shift_optab = vec_shl_optab;
733 break;
734 default:
735 gcc_unreachable ();
736 }
737
738 icode = optab_handler (shift_optab, mode);
739 gcc_assert (icode != CODE_FOR_nothing);
740
741 rtx_op1 = expand_normal (vec_oprnd);
742 rtx_op2 = expand_normal (shift_oprnd);
743
744 create_output_operand (&eops[0], target, mode);
745 create_input_operand (&eops[1], rtx_op1, GET_MODE (rtx_op1));
746 create_convert_operand_from_type (&eops[2], rtx_op2, TREE_TYPE (shift_oprnd));
747 expand_insn (icode, 3, eops);
748
749 return eops[0].value;
750 }
751
752 /* Create a new vector value in VMODE with all elements set to OP. The
753 mode of OP must be the element mode of VMODE. If OP is a constant,
754 then the return value will be a constant. */
755
756 static rtx
757 expand_vector_broadcast (enum machine_mode vmode, rtx op)
758 {
759 enum insn_code icode;
760 rtvec vec;
761 rtx ret;
762 int i, n;
763
764 gcc_checking_assert (VECTOR_MODE_P (vmode));
765
766 n = GET_MODE_NUNITS (vmode);
767 vec = rtvec_alloc (n);
768 for (i = 0; i < n; ++i)
769 RTVEC_ELT (vec, i) = op;
770
771 if (CONSTANT_P (op))
772 return gen_rtx_CONST_VECTOR (vmode, vec);
773
774 /* ??? If the target doesn't have a vec_init, then we have no easy way
775 of performing this operation. Most of this sort of generic support
776 is hidden away in the vector lowering support in gimple. */
777 icode = optab_handler (vec_init_optab, vmode);
778 if (icode == CODE_FOR_nothing)
779 return NULL;
780
781 ret = gen_reg_rtx (vmode);
782 emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
783
784 return ret;
785 }
786
787 /* This subroutine of expand_doubleword_shift handles the cases in which
788 the effective shift value is >= BITS_PER_WORD. The arguments and return
789 value are the same as for the parent routine, except that SUPERWORD_OP1
790 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
791 INTO_TARGET may be null if the caller has decided to calculate it. */
792
793 static bool
794 expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
795 rtx outof_target, rtx into_target,
796 int unsignedp, enum optab_methods methods)
797 {
798 if (into_target != 0)
799 if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
800 into_target, unsignedp, methods))
801 return false;
802
803 if (outof_target != 0)
804 {
805 /* For a signed right shift, we must fill OUTOF_TARGET with copies
806 of the sign bit, otherwise we must fill it with zeros. */
807 if (binoptab != ashr_optab)
808 emit_move_insn (outof_target, CONST0_RTX (word_mode));
809 else
810 if (!force_expand_binop (word_mode, binoptab,
811 outof_input, GEN_INT (BITS_PER_WORD - 1),
812 outof_target, unsignedp, methods))
813 return false;
814 }
815 return true;
816 }
817
818 /* This subroutine of expand_doubleword_shift handles the cases in which
819 the effective shift value is < BITS_PER_WORD. The arguments and return
820 value are the same as for the parent routine. */
821
822 static bool
823 expand_subword_shift (enum machine_mode op1_mode, optab binoptab,
824 rtx outof_input, rtx into_input, rtx op1,
825 rtx outof_target, rtx into_target,
826 int unsignedp, enum optab_methods methods,
827 unsigned HOST_WIDE_INT shift_mask)
828 {
829 optab reverse_unsigned_shift, unsigned_shift;
830 rtx tmp, carries;
831
832 reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
833 unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
834
835 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
836 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
837 the opposite direction to BINOPTAB. */
838 if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
839 {
840 carries = outof_input;
841 tmp = immed_double_const (BITS_PER_WORD, 0, op1_mode);
842 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
843 0, true, methods);
844 }
845 else
846 {
847 /* We must avoid shifting by BITS_PER_WORD bits since that is either
848 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
849 has unknown behavior. Do a single shift first, then shift by the
850 remainder. It's OK to use ~OP1 as the remainder if shift counts
851 are truncated to the mode size. */
852 carries = expand_binop (word_mode, reverse_unsigned_shift,
853 outof_input, const1_rtx, 0, unsignedp, methods);
854 if (shift_mask == BITS_PER_WORD - 1)
855 {
856 tmp = immed_double_const (-1, -1, op1_mode);
857 tmp = simplify_expand_binop (op1_mode, xor_optab, op1, tmp,
858 0, true, methods);
859 }
860 else
861 {
862 tmp = immed_double_const (BITS_PER_WORD - 1, 0, op1_mode);
863 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
864 0, true, methods);
865 }
866 }
867 if (tmp == 0 || carries == 0)
868 return false;
869 carries = expand_binop (word_mode, reverse_unsigned_shift,
870 carries, tmp, 0, unsignedp, methods);
871 if (carries == 0)
872 return false;
873
874 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
875 so the result can go directly into INTO_TARGET if convenient. */
876 tmp = expand_binop (word_mode, unsigned_shift, into_input, op1,
877 into_target, unsignedp, methods);
878 if (tmp == 0)
879 return false;
880
881 /* Now OR in the bits carried over from OUTOF_INPUT. */
882 if (!force_expand_binop (word_mode, ior_optab, tmp, carries,
883 into_target, unsignedp, methods))
884 return false;
885
886 /* Use a standard word_mode shift for the out-of half. */
887 if (outof_target != 0)
888 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
889 outof_target, unsignedp, methods))
890 return false;
891
892 return true;
893 }
894
895
896 #ifdef HAVE_conditional_move
897 /* Try implementing expand_doubleword_shift using conditional moves.
898 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
899 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
900 are the shift counts to use in the former and latter case. All other
901 arguments are the same as the parent routine. */
902
903 static bool
904 expand_doubleword_shift_condmove (enum machine_mode op1_mode, optab binoptab,
905 enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
906 rtx outof_input, rtx into_input,
907 rtx subword_op1, rtx superword_op1,
908 rtx outof_target, rtx into_target,
909 int unsignedp, enum optab_methods methods,
910 unsigned HOST_WIDE_INT shift_mask)
911 {
912 rtx outof_superword, into_superword;
913
914 /* Put the superword version of the output into OUTOF_SUPERWORD and
915 INTO_SUPERWORD. */
916 outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
917 if (outof_target != 0 && subword_op1 == superword_op1)
918 {
919 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
920 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
921 into_superword = outof_target;
922 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
923 outof_superword, 0, unsignedp, methods))
924 return false;
925 }
926 else
927 {
928 into_superword = gen_reg_rtx (word_mode);
929 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
930 outof_superword, into_superword,
931 unsignedp, methods))
932 return false;
933 }
934
935 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
936 if (!expand_subword_shift (op1_mode, binoptab,
937 outof_input, into_input, subword_op1,
938 outof_target, into_target,
939 unsignedp, methods, shift_mask))
940 return false;
941
942 /* Select between them. Do the INTO half first because INTO_SUPERWORD
943 might be the current value of OUTOF_TARGET. */
944 if (!emit_conditional_move (into_target, cmp_code, cmp1, cmp2, op1_mode,
945 into_target, into_superword, word_mode, false))
946 return false;
947
948 if (outof_target != 0)
949 if (!emit_conditional_move (outof_target, cmp_code, cmp1, cmp2, op1_mode,
950 outof_target, outof_superword,
951 word_mode, false))
952 return false;
953
954 return true;
955 }
956 #endif
957
958 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
959 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
960 input operand; the shift moves bits in the direction OUTOF_INPUT->
961 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
962 of the target. OP1 is the shift count and OP1_MODE is its mode.
963 If OP1 is constant, it will have been truncated as appropriate
964 and is known to be nonzero.
965
966 If SHIFT_MASK is zero, the result of word shifts is undefined when the
967 shift count is outside the range [0, BITS_PER_WORD). This routine must
968 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
969
970 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
971 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
972 fill with zeros or sign bits as appropriate.
973
974 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
975 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
976 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
977 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
978 are undefined.
979
980 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
981 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
982 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
983 function wants to calculate it itself.
984
985 Return true if the shift could be successfully synthesized. */
986
987 static bool
988 expand_doubleword_shift (enum machine_mode op1_mode, optab binoptab,
989 rtx outof_input, rtx into_input, rtx op1,
990 rtx outof_target, rtx into_target,
991 int unsignedp, enum optab_methods methods,
992 unsigned HOST_WIDE_INT shift_mask)
993 {
994 rtx superword_op1, tmp, cmp1, cmp2;
995 rtx subword_label, done_label;
996 enum rtx_code cmp_code;
997
998 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
999 fill the result with sign or zero bits as appropriate. If so, the value
1000 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
1001 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
1002 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
1003
1004 This isn't worthwhile for constant shifts since the optimizers will
1005 cope better with in-range shift counts. */
1006 if (shift_mask >= BITS_PER_WORD
1007 && outof_target != 0
1008 && !CONSTANT_P (op1))
1009 {
1010 if (!expand_doubleword_shift (op1_mode, binoptab,
1011 outof_input, into_input, op1,
1012 0, into_target,
1013 unsignedp, methods, shift_mask))
1014 return false;
1015 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
1016 outof_target, unsignedp, methods))
1017 return false;
1018 return true;
1019 }
1020
1021 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
1022 is true when the effective shift value is less than BITS_PER_WORD.
1023 Set SUPERWORD_OP1 to the shift count that should be used to shift
1024 OUTOF_INPUT into INTO_TARGET when the condition is false. */
1025 tmp = immed_double_const (BITS_PER_WORD, 0, op1_mode);
1026 if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
1027 {
1028 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
1029 is a subword shift count. */
1030 cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
1031 0, true, methods);
1032 cmp2 = CONST0_RTX (op1_mode);
1033 cmp_code = EQ;
1034 superword_op1 = op1;
1035 }
1036 else
1037 {
1038 /* Set CMP1 to OP1 - BITS_PER_WORD. */
1039 cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
1040 0, true, methods);
1041 cmp2 = CONST0_RTX (op1_mode);
1042 cmp_code = LT;
1043 superword_op1 = cmp1;
1044 }
1045 if (cmp1 == 0)
1046 return false;
1047
1048 /* If we can compute the condition at compile time, pick the
1049 appropriate subroutine. */
1050 tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
1051 if (tmp != 0 && CONST_INT_P (tmp))
1052 {
1053 if (tmp == const0_rtx)
1054 return expand_superword_shift (binoptab, outof_input, superword_op1,
1055 outof_target, into_target,
1056 unsignedp, methods);
1057 else
1058 return expand_subword_shift (op1_mode, binoptab,
1059 outof_input, into_input, op1,
1060 outof_target, into_target,
1061 unsignedp, methods, shift_mask);
1062 }
1063
1064 #ifdef HAVE_conditional_move
1065 /* Try using conditional moves to generate straight-line code. */
1066 {
1067 rtx start = get_last_insn ();
1068 if (expand_doubleword_shift_condmove (op1_mode, binoptab,
1069 cmp_code, cmp1, cmp2,
1070 outof_input, into_input,
1071 op1, superword_op1,
1072 outof_target, into_target,
1073 unsignedp, methods, shift_mask))
1074 return true;
1075 delete_insns_since (start);
1076 }
1077 #endif
1078
1079 /* As a last resort, use branches to select the correct alternative. */
1080 subword_label = gen_label_rtx ();
1081 done_label = gen_label_rtx ();
1082
1083 NO_DEFER_POP;
1084 do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
1085 0, 0, subword_label, -1);
1086 OK_DEFER_POP;
1087
1088 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
1089 outof_target, into_target,
1090 unsignedp, methods))
1091 return false;
1092
1093 emit_jump_insn (gen_jump (done_label));
1094 emit_barrier ();
1095 emit_label (subword_label);
1096
1097 if (!expand_subword_shift (op1_mode, binoptab,
1098 outof_input, into_input, op1,
1099 outof_target, into_target,
1100 unsignedp, methods, shift_mask))
1101 return false;
1102
1103 emit_label (done_label);
1104 return true;
1105 }
1106 \f
1107 /* Subroutine of expand_binop. Perform a double word multiplication of
1108 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
1109 as the target's word_mode. This function return NULL_RTX if anything
1110 goes wrong, in which case it may have already emitted instructions
1111 which need to be deleted.
1112
1113 If we want to multiply two two-word values and have normal and widening
1114 multiplies of single-word values, we can do this with three smaller
1115 multiplications.
1116
1117 The multiplication proceeds as follows:
1118 _______________________
1119 [__op0_high_|__op0_low__]
1120 _______________________
1121 * [__op1_high_|__op1_low__]
1122 _______________________________________________
1123 _______________________
1124 (1) [__op0_low__*__op1_low__]
1125 _______________________
1126 (2a) [__op0_low__*__op1_high_]
1127 _______________________
1128 (2b) [__op0_high_*__op1_low__]
1129 _______________________
1130 (3) [__op0_high_*__op1_high_]
1131
1132
1133 This gives a 4-word result. Since we are only interested in the
1134 lower 2 words, partial result (3) and the upper words of (2a) and
1135 (2b) don't need to be calculated. Hence (2a) and (2b) can be
1136 calculated using non-widening multiplication.
1137
1138 (1), however, needs to be calculated with an unsigned widening
1139 multiplication. If this operation is not directly supported we
1140 try using a signed widening multiplication and adjust the result.
1141 This adjustment works as follows:
1142
1143 If both operands are positive then no adjustment is needed.
1144
1145 If the operands have different signs, for example op0_low < 0 and
1146 op1_low >= 0, the instruction treats the most significant bit of
1147 op0_low as a sign bit instead of a bit with significance
1148 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
1149 with 2**BITS_PER_WORD - op0_low, and two's complements the
1150 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
1151 the result.
1152
1153 Similarly, if both operands are negative, we need to add
1154 (op0_low + op1_low) * 2**BITS_PER_WORD.
1155
1156 We use a trick to adjust quickly. We logically shift op0_low right
1157 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
1158 op0_high (op1_high) before it is used to calculate 2b (2a). If no
1159 logical shift exists, we do an arithmetic right shift and subtract
1160 the 0 or -1. */
1161
1162 static rtx
1163 expand_doubleword_mult (enum machine_mode mode, rtx op0, rtx op1, rtx target,
1164 bool umulp, enum optab_methods methods)
1165 {
1166 int low = (WORDS_BIG_ENDIAN ? 1 : 0);
1167 int high = (WORDS_BIG_ENDIAN ? 0 : 1);
1168 rtx wordm1 = umulp ? NULL_RTX : GEN_INT (BITS_PER_WORD - 1);
1169 rtx product, adjust, product_high, temp;
1170
1171 rtx op0_high = operand_subword_force (op0, high, mode);
1172 rtx op0_low = operand_subword_force (op0, low, mode);
1173 rtx op1_high = operand_subword_force (op1, high, mode);
1174 rtx op1_low = operand_subword_force (op1, low, mode);
1175
1176 /* If we're using an unsigned multiply to directly compute the product
1177 of the low-order words of the operands and perform any required
1178 adjustments of the operands, we begin by trying two more multiplications
1179 and then computing the appropriate sum.
1180
1181 We have checked above that the required addition is provided.
1182 Full-word addition will normally always succeed, especially if
1183 it is provided at all, so we don't worry about its failure. The
1184 multiplication may well fail, however, so we do handle that. */
1185
1186 if (!umulp)
1187 {
1188 /* ??? This could be done with emit_store_flag where available. */
1189 temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
1190 NULL_RTX, 1, methods);
1191 if (temp)
1192 op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
1193 NULL_RTX, 0, OPTAB_DIRECT);
1194 else
1195 {
1196 temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
1197 NULL_RTX, 0, methods);
1198 if (!temp)
1199 return NULL_RTX;
1200 op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
1201 NULL_RTX, 0, OPTAB_DIRECT);
1202 }
1203
1204 if (!op0_high)
1205 return NULL_RTX;
1206 }
1207
1208 adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
1209 NULL_RTX, 0, OPTAB_DIRECT);
1210 if (!adjust)
1211 return NULL_RTX;
1212
1213 /* OP0_HIGH should now be dead. */
1214
1215 if (!umulp)
1216 {
1217 /* ??? This could be done with emit_store_flag where available. */
1218 temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
1219 NULL_RTX, 1, methods);
1220 if (temp)
1221 op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
1222 NULL_RTX, 0, OPTAB_DIRECT);
1223 else
1224 {
1225 temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
1226 NULL_RTX, 0, methods);
1227 if (!temp)
1228 return NULL_RTX;
1229 op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
1230 NULL_RTX, 0, OPTAB_DIRECT);
1231 }
1232
1233 if (!op1_high)
1234 return NULL_RTX;
1235 }
1236
1237 temp = expand_binop (word_mode, smul_optab, op1_high, op0_low,
1238 NULL_RTX, 0, OPTAB_DIRECT);
1239 if (!temp)
1240 return NULL_RTX;
1241
1242 /* OP1_HIGH should now be dead. */
1243
1244 adjust = expand_binop (word_mode, add_optab, adjust, temp,
1245 NULL_RTX, 0, OPTAB_DIRECT);
1246
1247 if (target && !REG_P (target))
1248 target = NULL_RTX;
1249
1250 if (umulp)
1251 product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
1252 target, 1, OPTAB_DIRECT);
1253 else
1254 product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
1255 target, 1, OPTAB_DIRECT);
1256
1257 if (!product)
1258 return NULL_RTX;
1259
1260 product_high = operand_subword (product, high, 1, mode);
1261 adjust = expand_binop (word_mode, add_optab, product_high, adjust,
1262 NULL_RTX, 0, OPTAB_DIRECT);
1263 emit_move_insn (product_high, adjust);
1264 return product;
1265 }
1266 \f
1267 /* Wrapper around expand_binop which takes an rtx code to specify
1268 the operation to perform, not an optab pointer. All other
1269 arguments are the same. */
1270 rtx
1271 expand_simple_binop (enum machine_mode mode, enum rtx_code code, rtx op0,
1272 rtx op1, rtx target, int unsignedp,
1273 enum optab_methods methods)
1274 {
1275 optab binop = code_to_optab (code);
1276 gcc_assert (binop);
1277
1278 return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
1279 }
1280
1281 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
1282 binop. Order them according to commutative_operand_precedence and, if
1283 possible, try to put TARGET or a pseudo first. */
1284 static bool
1285 swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
1286 {
1287 int op0_prec = commutative_operand_precedence (op0);
1288 int op1_prec = commutative_operand_precedence (op1);
1289
1290 if (op0_prec < op1_prec)
1291 return true;
1292
1293 if (op0_prec > op1_prec)
1294 return false;
1295
1296 /* With equal precedence, both orders are ok, but it is better if the
1297 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1298 if (target == 0 || REG_P (target))
1299 return (REG_P (op1) && !REG_P (op0)) || target == op1;
1300 else
1301 return rtx_equal_p (op1, target);
1302 }
1303
1304 /* Return true if BINOPTAB implements a shift operation. */
1305
1306 static bool
1307 shift_optab_p (optab binoptab)
1308 {
1309 switch (optab_to_code (binoptab))
1310 {
1311 case ASHIFT:
1312 case SS_ASHIFT:
1313 case US_ASHIFT:
1314 case ASHIFTRT:
1315 case LSHIFTRT:
1316 case ROTATE:
1317 case ROTATERT:
1318 return true;
1319
1320 default:
1321 return false;
1322 }
1323 }
1324
1325 /* Return true if BINOPTAB implements a commutative binary operation. */
1326
1327 static bool
1328 commutative_optab_p (optab binoptab)
1329 {
1330 return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
1331 || binoptab == smul_widen_optab
1332 || binoptab == umul_widen_optab
1333 || binoptab == smul_highpart_optab
1334 || binoptab == umul_highpart_optab);
1335 }
1336
1337 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1338 optimizing, and if the operand is a constant that costs more than
1339 1 instruction, force the constant into a register and return that
1340 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1341
1342 static rtx
1343 avoid_expensive_constant (enum machine_mode mode, optab binoptab,
1344 int opn, rtx x, bool unsignedp)
1345 {
1346 bool speed = optimize_insn_for_speed_p ();
1347
1348 if (mode != VOIDmode
1349 && optimize
1350 && CONSTANT_P (x)
1351 && (rtx_cost (x, optab_to_code (binoptab), opn, speed)
1352 > set_src_cost (x, speed)))
1353 {
1354 if (CONST_INT_P (x))
1355 {
1356 HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
1357 if (intval != INTVAL (x))
1358 x = GEN_INT (intval);
1359 }
1360 else
1361 x = convert_modes (mode, VOIDmode, x, unsignedp);
1362 x = force_reg (mode, x);
1363 }
1364 return x;
1365 }
1366
1367 /* Helper function for expand_binop: handle the case where there
1368 is an insn that directly implements the indicated operation.
1369 Returns null if this is not possible. */
1370 static rtx
1371 expand_binop_directly (enum machine_mode mode, optab binoptab,
1372 rtx op0, rtx op1,
1373 rtx target, int unsignedp, enum optab_methods methods,
1374 rtx last)
1375 {
1376 enum machine_mode from_mode = widened_mode (mode, op0, op1);
1377 enum insn_code icode = find_widening_optab_handler (binoptab, mode,
1378 from_mode, 1);
1379 enum machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
1380 enum machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
1381 enum machine_mode mode0, mode1, tmp_mode;
1382 struct expand_operand ops[3];
1383 bool commutative_p;
1384 rtx pat;
1385 rtx xop0 = op0, xop1 = op1;
1386 rtx swap;
1387
1388 /* If it is a commutative operator and the modes would match
1389 if we would swap the operands, we can save the conversions. */
1390 commutative_p = commutative_optab_p (binoptab);
1391 if (commutative_p
1392 && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1393 && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode1)
1394 {
1395 swap = xop0;
1396 xop0 = xop1;
1397 xop1 = swap;
1398 }
1399
1400 /* If we are optimizing, force expensive constants into a register. */
1401 xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
1402 if (!shift_optab_p (binoptab))
1403 xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
1404
1405 /* In case the insn wants input operands in modes different from
1406 those of the actual operands, convert the operands. It would
1407 seem that we don't need to convert CONST_INTs, but we do, so
1408 that they're properly zero-extended, sign-extended or truncated
1409 for their mode. */
1410
1411 mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1412 if (xmode0 != VOIDmode && xmode0 != mode0)
1413 {
1414 xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
1415 mode0 = xmode0;
1416 }
1417
1418 mode1 = GET_MODE (xop1) != VOIDmode ? GET_MODE (xop1) : mode;
1419 if (xmode1 != VOIDmode && xmode1 != mode1)
1420 {
1421 xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
1422 mode1 = xmode1;
1423 }
1424
1425 /* If operation is commutative,
1426 try to make the first operand a register.
1427 Even better, try to make it the same as the target.
1428 Also try to make the last operand a constant. */
1429 if (commutative_p
1430 && swap_commutative_operands_with_target (target, xop0, xop1))
1431 {
1432 swap = xop1;
1433 xop1 = xop0;
1434 xop0 = swap;
1435 }
1436
1437 /* Now, if insn's predicates don't allow our operands, put them into
1438 pseudo regs. */
1439
1440 if (binoptab == vec_pack_trunc_optab
1441 || binoptab == vec_pack_usat_optab
1442 || binoptab == vec_pack_ssat_optab
1443 || binoptab == vec_pack_ufix_trunc_optab
1444 || binoptab == vec_pack_sfix_trunc_optab)
1445 {
1446 /* The mode of the result is different then the mode of the
1447 arguments. */
1448 tmp_mode = insn_data[(int) icode].operand[0].mode;
1449 if (GET_MODE_NUNITS (tmp_mode) != 2 * GET_MODE_NUNITS (mode))
1450 {
1451 delete_insns_since (last);
1452 return NULL_RTX;
1453 }
1454 }
1455 else
1456 tmp_mode = mode;
1457
1458 create_output_operand (&ops[0], target, tmp_mode);
1459 create_input_operand (&ops[1], xop0, mode0);
1460 create_input_operand (&ops[2], xop1, mode1);
1461 pat = maybe_gen_insn (icode, 3, ops);
1462 if (pat)
1463 {
1464 /* If PAT is composed of more than one insn, try to add an appropriate
1465 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1466 operand, call expand_binop again, this time without a target. */
1467 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
1468 && ! add_equal_note (pat, ops[0].value, optab_to_code (binoptab),
1469 ops[1].value, ops[2].value))
1470 {
1471 delete_insns_since (last);
1472 return expand_binop (mode, binoptab, op0, op1, NULL_RTX,
1473 unsignedp, methods);
1474 }
1475
1476 emit_insn (pat);
1477 return ops[0].value;
1478 }
1479 delete_insns_since (last);
1480 return NULL_RTX;
1481 }
1482
1483 /* Generate code to perform an operation specified by BINOPTAB
1484 on operands OP0 and OP1, with result having machine-mode MODE.
1485
1486 UNSIGNEDP is for the case where we have to widen the operands
1487 to perform the operation. It says to use zero-extension.
1488
1489 If TARGET is nonzero, the value
1490 is generated there, if it is convenient to do so.
1491 In all cases an rtx is returned for the locus of the value;
1492 this may or may not be TARGET. */
1493
1494 rtx
1495 expand_binop (enum machine_mode mode, optab binoptab, rtx op0, rtx op1,
1496 rtx target, int unsignedp, enum optab_methods methods)
1497 {
1498 enum optab_methods next_methods
1499 = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1500 ? OPTAB_WIDEN : methods);
1501 enum mode_class mclass;
1502 enum machine_mode wider_mode;
1503 rtx libfunc;
1504 rtx temp;
1505 rtx entry_last = get_last_insn ();
1506 rtx last;
1507
1508 mclass = GET_MODE_CLASS (mode);
1509
1510 /* If subtracting an integer constant, convert this into an addition of
1511 the negated constant. */
1512
1513 if (binoptab == sub_optab && CONST_INT_P (op1))
1514 {
1515 op1 = negate_rtx (mode, op1);
1516 binoptab = add_optab;
1517 }
1518
1519 /* Record where to delete back to if we backtrack. */
1520 last = get_last_insn ();
1521
1522 /* If we can do it with a three-operand insn, do so. */
1523
1524 if (methods != OPTAB_MUST_WIDEN
1525 && find_widening_optab_handler (binoptab, mode,
1526 widened_mode (mode, op0, op1), 1)
1527 != CODE_FOR_nothing)
1528 {
1529 temp = expand_binop_directly (mode, binoptab, op0, op1, target,
1530 unsignedp, methods, last);
1531 if (temp)
1532 return temp;
1533 }
1534
1535 /* If we were trying to rotate, and that didn't work, try rotating
1536 the other direction before falling back to shifts and bitwise-or. */
1537 if (((binoptab == rotl_optab
1538 && optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
1539 || (binoptab == rotr_optab
1540 && optab_handler (rotl_optab, mode) != CODE_FOR_nothing))
1541 && mclass == MODE_INT)
1542 {
1543 optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1544 rtx newop1;
1545 unsigned int bits = GET_MODE_PRECISION (mode);
1546
1547 if (CONST_INT_P (op1))
1548 newop1 = GEN_INT (bits - INTVAL (op1));
1549 else if (targetm.shift_truncation_mask (mode) == bits - 1)
1550 newop1 = negate_rtx (GET_MODE (op1), op1);
1551 else
1552 newop1 = expand_binop (GET_MODE (op1), sub_optab,
1553 GEN_INT (bits), op1,
1554 NULL_RTX, unsignedp, OPTAB_DIRECT);
1555
1556 temp = expand_binop_directly (mode, otheroptab, op0, newop1,
1557 target, unsignedp, methods, last);
1558 if (temp)
1559 return temp;
1560 }
1561
1562 /* If this is a multiply, see if we can do a widening operation that
1563 takes operands of this mode and makes a wider mode. */
1564
1565 if (binoptab == smul_optab
1566 && GET_MODE_2XWIDER_MODE (mode) != VOIDmode
1567 && (widening_optab_handler ((unsignedp ? umul_widen_optab
1568 : smul_widen_optab),
1569 GET_MODE_2XWIDER_MODE (mode), mode)
1570 != CODE_FOR_nothing))
1571 {
1572 temp = expand_binop (GET_MODE_2XWIDER_MODE (mode),
1573 unsignedp ? umul_widen_optab : smul_widen_optab,
1574 op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
1575
1576 if (temp != 0)
1577 {
1578 if (GET_MODE_CLASS (mode) == MODE_INT
1579 && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1580 return gen_lowpart (mode, temp);
1581 else
1582 return convert_to_mode (mode, temp, unsignedp);
1583 }
1584 }
1585
1586 /* If this is a vector shift by a scalar, see if we can do a vector
1587 shift by a vector. If so, broadcast the scalar into a vector. */
1588 if (mclass == MODE_VECTOR_INT)
1589 {
1590 optab otheroptab = unknown_optab;
1591
1592 if (binoptab == ashl_optab)
1593 otheroptab = vashl_optab;
1594 else if (binoptab == ashr_optab)
1595 otheroptab = vashr_optab;
1596 else if (binoptab == lshr_optab)
1597 otheroptab = vlshr_optab;
1598 else if (binoptab == rotl_optab)
1599 otheroptab = vrotl_optab;
1600 else if (binoptab == rotr_optab)
1601 otheroptab = vrotr_optab;
1602
1603 if (otheroptab && optab_handler (otheroptab, mode) != CODE_FOR_nothing)
1604 {
1605 rtx vop1 = expand_vector_broadcast (mode, op1);
1606 if (vop1)
1607 {
1608 temp = expand_binop_directly (mode, otheroptab, op0, vop1,
1609 target, unsignedp, methods, last);
1610 if (temp)
1611 return temp;
1612 }
1613 }
1614 }
1615
1616 /* Look for a wider mode of the same class for which we think we
1617 can open-code the operation. Check for a widening multiply at the
1618 wider mode as well. */
1619
1620 if (CLASS_HAS_WIDER_MODES_P (mclass)
1621 && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1622 for (wider_mode = GET_MODE_WIDER_MODE (mode);
1623 wider_mode != VOIDmode;
1624 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
1625 {
1626 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
1627 || (binoptab == smul_optab
1628 && GET_MODE_WIDER_MODE (wider_mode) != VOIDmode
1629 && (find_widening_optab_handler ((unsignedp
1630 ? umul_widen_optab
1631 : smul_widen_optab),
1632 GET_MODE_WIDER_MODE (wider_mode),
1633 mode, 0)
1634 != CODE_FOR_nothing)))
1635 {
1636 rtx xop0 = op0, xop1 = op1;
1637 int no_extend = 0;
1638
1639 /* For certain integer operations, we need not actually extend
1640 the narrow operands, as long as we will truncate
1641 the results to the same narrowness. */
1642
1643 if ((binoptab == ior_optab || binoptab == and_optab
1644 || binoptab == xor_optab
1645 || binoptab == add_optab || binoptab == sub_optab
1646 || binoptab == smul_optab || binoptab == ashl_optab)
1647 && mclass == MODE_INT)
1648 {
1649 no_extend = 1;
1650 xop0 = avoid_expensive_constant (mode, binoptab, 0,
1651 xop0, unsignedp);
1652 if (binoptab != ashl_optab)
1653 xop1 = avoid_expensive_constant (mode, binoptab, 1,
1654 xop1, unsignedp);
1655 }
1656
1657 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
1658
1659 /* The second operand of a shift must always be extended. */
1660 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1661 no_extend && binoptab != ashl_optab);
1662
1663 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1664 unsignedp, OPTAB_DIRECT);
1665 if (temp)
1666 {
1667 if (mclass != MODE_INT
1668 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1669 {
1670 if (target == 0)
1671 target = gen_reg_rtx (mode);
1672 convert_move (target, temp, 0);
1673 return target;
1674 }
1675 else
1676 return gen_lowpart (mode, temp);
1677 }
1678 else
1679 delete_insns_since (last);
1680 }
1681 }
1682
1683 /* If operation is commutative,
1684 try to make the first operand a register.
1685 Even better, try to make it the same as the target.
1686 Also try to make the last operand a constant. */
1687 if (commutative_optab_p (binoptab)
1688 && swap_commutative_operands_with_target (target, op0, op1))
1689 {
1690 temp = op1;
1691 op1 = op0;
1692 op0 = temp;
1693 }
1694
1695 /* These can be done a word at a time. */
1696 if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1697 && mclass == MODE_INT
1698 && GET_MODE_SIZE (mode) > UNITS_PER_WORD
1699 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1700 {
1701 int i;
1702 rtx insns;
1703
1704 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1705 won't be accurate, so use a new target. */
1706 if (target == 0
1707 || target == op0
1708 || target == op1
1709 || !valid_multiword_target_p (target))
1710 target = gen_reg_rtx (mode);
1711
1712 start_sequence ();
1713
1714 /* Do the actual arithmetic. */
1715 for (i = 0; i < GET_MODE_BITSIZE (mode) / BITS_PER_WORD; i++)
1716 {
1717 rtx target_piece = operand_subword (target, i, 1, mode);
1718 rtx x = expand_binop (word_mode, binoptab,
1719 operand_subword_force (op0, i, mode),
1720 operand_subword_force (op1, i, mode),
1721 target_piece, unsignedp, next_methods);
1722
1723 if (x == 0)
1724 break;
1725
1726 if (target_piece != x)
1727 emit_move_insn (target_piece, x);
1728 }
1729
1730 insns = get_insns ();
1731 end_sequence ();
1732
1733 if (i == GET_MODE_BITSIZE (mode) / BITS_PER_WORD)
1734 {
1735 emit_insn (insns);
1736 return target;
1737 }
1738 }
1739
1740 /* Synthesize double word shifts from single word shifts. */
1741 if ((binoptab == lshr_optab || binoptab == ashl_optab
1742 || binoptab == ashr_optab)
1743 && mclass == MODE_INT
1744 && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1745 && GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
1746 && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode)
1747 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
1748 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1749 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1750 {
1751 unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1752 enum machine_mode op1_mode;
1753
1754 double_shift_mask = targetm.shift_truncation_mask (mode);
1755 shift_mask = targetm.shift_truncation_mask (word_mode);
1756 op1_mode = GET_MODE (op1) != VOIDmode ? GET_MODE (op1) : word_mode;
1757
1758 /* Apply the truncation to constant shifts. */
1759 if (double_shift_mask > 0 && CONST_INT_P (op1))
1760 op1 = GEN_INT (INTVAL (op1) & double_shift_mask);
1761
1762 if (op1 == CONST0_RTX (op1_mode))
1763 return op0;
1764
1765 /* Make sure that this is a combination that expand_doubleword_shift
1766 can handle. See the comments there for details. */
1767 if (double_shift_mask == 0
1768 || (shift_mask == BITS_PER_WORD - 1
1769 && double_shift_mask == BITS_PER_WORD * 2 - 1))
1770 {
1771 rtx insns;
1772 rtx into_target, outof_target;
1773 rtx into_input, outof_input;
1774 int left_shift, outof_word;
1775
1776 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1777 won't be accurate, so use a new target. */
1778 if (target == 0
1779 || target == op0
1780 || target == op1
1781 || !valid_multiword_target_p (target))
1782 target = gen_reg_rtx (mode);
1783
1784 start_sequence ();
1785
1786 /* OUTOF_* is the word we are shifting bits away from, and
1787 INTO_* is the word that we are shifting bits towards, thus
1788 they differ depending on the direction of the shift and
1789 WORDS_BIG_ENDIAN. */
1790
1791 left_shift = binoptab == ashl_optab;
1792 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1793
1794 outof_target = operand_subword (target, outof_word, 1, mode);
1795 into_target = operand_subword (target, 1 - outof_word, 1, mode);
1796
1797 outof_input = operand_subword_force (op0, outof_word, mode);
1798 into_input = operand_subword_force (op0, 1 - outof_word, mode);
1799
1800 if (expand_doubleword_shift (op1_mode, binoptab,
1801 outof_input, into_input, op1,
1802 outof_target, into_target,
1803 unsignedp, next_methods, shift_mask))
1804 {
1805 insns = get_insns ();
1806 end_sequence ();
1807
1808 emit_insn (insns);
1809 return target;
1810 }
1811 end_sequence ();
1812 }
1813 }
1814
1815 /* Synthesize double word rotates from single word shifts. */
1816 if ((binoptab == rotl_optab || binoptab == rotr_optab)
1817 && mclass == MODE_INT
1818 && CONST_INT_P (op1)
1819 && GET_MODE_PRECISION (mode) == 2 * BITS_PER_WORD
1820 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1821 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1822 {
1823 rtx insns;
1824 rtx into_target, outof_target;
1825 rtx into_input, outof_input;
1826 rtx inter;
1827 int shift_count, left_shift, outof_word;
1828
1829 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1830 won't be accurate, so use a new target. Do this also if target is not
1831 a REG, first because having a register instead may open optimization
1832 opportunities, and second because if target and op0 happen to be MEMs
1833 designating the same location, we would risk clobbering it too early
1834 in the code sequence we generate below. */
1835 if (target == 0
1836 || target == op0
1837 || target == op1
1838 || !REG_P (target)
1839 || !valid_multiword_target_p (target))
1840 target = gen_reg_rtx (mode);
1841
1842 start_sequence ();
1843
1844 shift_count = INTVAL (op1);
1845
1846 /* OUTOF_* is the word we are shifting bits away from, and
1847 INTO_* is the word that we are shifting bits towards, thus
1848 they differ depending on the direction of the shift and
1849 WORDS_BIG_ENDIAN. */
1850
1851 left_shift = (binoptab == rotl_optab);
1852 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1853
1854 outof_target = operand_subword (target, outof_word, 1, mode);
1855 into_target = operand_subword (target, 1 - outof_word, 1, mode);
1856
1857 outof_input = operand_subword_force (op0, outof_word, mode);
1858 into_input = operand_subword_force (op0, 1 - outof_word, mode);
1859
1860 if (shift_count == BITS_PER_WORD)
1861 {
1862 /* This is just a word swap. */
1863 emit_move_insn (outof_target, into_input);
1864 emit_move_insn (into_target, outof_input);
1865 inter = const0_rtx;
1866 }
1867 else
1868 {
1869 rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1870 rtx first_shift_count, second_shift_count;
1871 optab reverse_unsigned_shift, unsigned_shift;
1872
1873 reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1874 ? lshr_optab : ashl_optab);
1875
1876 unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1877 ? ashl_optab : lshr_optab);
1878
1879 if (shift_count > BITS_PER_WORD)
1880 {
1881 first_shift_count = GEN_INT (shift_count - BITS_PER_WORD);
1882 second_shift_count = GEN_INT (2 * BITS_PER_WORD - shift_count);
1883 }
1884 else
1885 {
1886 first_shift_count = GEN_INT (BITS_PER_WORD - shift_count);
1887 second_shift_count = GEN_INT (shift_count);
1888 }
1889
1890 into_temp1 = expand_binop (word_mode, unsigned_shift,
1891 outof_input, first_shift_count,
1892 NULL_RTX, unsignedp, next_methods);
1893 into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1894 into_input, second_shift_count,
1895 NULL_RTX, unsignedp, next_methods);
1896
1897 if (into_temp1 != 0 && into_temp2 != 0)
1898 inter = expand_binop (word_mode, ior_optab, into_temp1, into_temp2,
1899 into_target, unsignedp, next_methods);
1900 else
1901 inter = 0;
1902
1903 if (inter != 0 && inter != into_target)
1904 emit_move_insn (into_target, inter);
1905
1906 outof_temp1 = expand_binop (word_mode, unsigned_shift,
1907 into_input, first_shift_count,
1908 NULL_RTX, unsignedp, next_methods);
1909 outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1910 outof_input, second_shift_count,
1911 NULL_RTX, unsignedp, next_methods);
1912
1913 if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
1914 inter = expand_binop (word_mode, ior_optab,
1915 outof_temp1, outof_temp2,
1916 outof_target, unsignedp, next_methods);
1917
1918 if (inter != 0 && inter != outof_target)
1919 emit_move_insn (outof_target, inter);
1920 }
1921
1922 insns = get_insns ();
1923 end_sequence ();
1924
1925 if (inter != 0)
1926 {
1927 emit_insn (insns);
1928 return target;
1929 }
1930 }
1931
1932 /* These can be done a word at a time by propagating carries. */
1933 if ((binoptab == add_optab || binoptab == sub_optab)
1934 && mclass == MODE_INT
1935 && GET_MODE_SIZE (mode) >= 2 * UNITS_PER_WORD
1936 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1937 {
1938 unsigned int i;
1939 optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
1940 const unsigned int nwords = GET_MODE_BITSIZE (mode) / BITS_PER_WORD;
1941 rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
1942 rtx xop0, xop1, xtarget;
1943
1944 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1945 value is one of those, use it. Otherwise, use 1 since it is the
1946 one easiest to get. */
1947 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1948 int normalizep = STORE_FLAG_VALUE;
1949 #else
1950 int normalizep = 1;
1951 #endif
1952
1953 /* Prepare the operands. */
1954 xop0 = force_reg (mode, op0);
1955 xop1 = force_reg (mode, op1);
1956
1957 xtarget = gen_reg_rtx (mode);
1958
1959 if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
1960 target = xtarget;
1961
1962 /* Indicate for flow that the entire target reg is being set. */
1963 if (REG_P (target))
1964 emit_clobber (xtarget);
1965
1966 /* Do the actual arithmetic. */
1967 for (i = 0; i < nwords; i++)
1968 {
1969 int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
1970 rtx target_piece = operand_subword (xtarget, index, 1, mode);
1971 rtx op0_piece = operand_subword_force (xop0, index, mode);
1972 rtx op1_piece = operand_subword_force (xop1, index, mode);
1973 rtx x;
1974
1975 /* Main add/subtract of the input operands. */
1976 x = expand_binop (word_mode, binoptab,
1977 op0_piece, op1_piece,
1978 target_piece, unsignedp, next_methods);
1979 if (x == 0)
1980 break;
1981
1982 if (i + 1 < nwords)
1983 {
1984 /* Store carry from main add/subtract. */
1985 carry_out = gen_reg_rtx (word_mode);
1986 carry_out = emit_store_flag_force (carry_out,
1987 (binoptab == add_optab
1988 ? LT : GT),
1989 x, op0_piece,
1990 word_mode, 1, normalizep);
1991 }
1992
1993 if (i > 0)
1994 {
1995 rtx newx;
1996
1997 /* Add/subtract previous carry to main result. */
1998 newx = expand_binop (word_mode,
1999 normalizep == 1 ? binoptab : otheroptab,
2000 x, carry_in,
2001 NULL_RTX, 1, next_methods);
2002
2003 if (i + 1 < nwords)
2004 {
2005 /* Get out carry from adding/subtracting carry in. */
2006 rtx carry_tmp = gen_reg_rtx (word_mode);
2007 carry_tmp = emit_store_flag_force (carry_tmp,
2008 (binoptab == add_optab
2009 ? LT : GT),
2010 newx, x,
2011 word_mode, 1, normalizep);
2012
2013 /* Logical-ior the two poss. carry together. */
2014 carry_out = expand_binop (word_mode, ior_optab,
2015 carry_out, carry_tmp,
2016 carry_out, 0, next_methods);
2017 if (carry_out == 0)
2018 break;
2019 }
2020 emit_move_insn (target_piece, newx);
2021 }
2022 else
2023 {
2024 if (x != target_piece)
2025 emit_move_insn (target_piece, x);
2026 }
2027
2028 carry_in = carry_out;
2029 }
2030
2031 if (i == GET_MODE_BITSIZE (mode) / (unsigned) BITS_PER_WORD)
2032 {
2033 if (optab_handler (mov_optab, mode) != CODE_FOR_nothing
2034 || ! rtx_equal_p (target, xtarget))
2035 {
2036 rtx temp = emit_move_insn (target, xtarget);
2037
2038 set_dst_reg_note (temp, REG_EQUAL,
2039 gen_rtx_fmt_ee (optab_to_code (binoptab),
2040 mode, copy_rtx (xop0),
2041 copy_rtx (xop1)),
2042 target);
2043 }
2044 else
2045 target = xtarget;
2046
2047 return target;
2048 }
2049
2050 else
2051 delete_insns_since (last);
2052 }
2053
2054 /* Attempt to synthesize double word multiplies using a sequence of word
2055 mode multiplications. We first attempt to generate a sequence using a
2056 more efficient unsigned widening multiply, and if that fails we then
2057 try using a signed widening multiply. */
2058
2059 if (binoptab == smul_optab
2060 && mclass == MODE_INT
2061 && GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
2062 && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
2063 && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
2064 {
2065 rtx product = NULL_RTX;
2066 if (widening_optab_handler (umul_widen_optab, mode, word_mode)
2067 != CODE_FOR_nothing)
2068 {
2069 product = expand_doubleword_mult (mode, op0, op1, target,
2070 true, methods);
2071 if (!product)
2072 delete_insns_since (last);
2073 }
2074
2075 if (product == NULL_RTX
2076 && widening_optab_handler (smul_widen_optab, mode, word_mode)
2077 != CODE_FOR_nothing)
2078 {
2079 product = expand_doubleword_mult (mode, op0, op1, target,
2080 false, methods);
2081 if (!product)
2082 delete_insns_since (last);
2083 }
2084
2085 if (product != NULL_RTX)
2086 {
2087 if (optab_handler (mov_optab, mode) != CODE_FOR_nothing)
2088 {
2089 temp = emit_move_insn (target ? target : product, product);
2090 set_dst_reg_note (temp,
2091 REG_EQUAL,
2092 gen_rtx_fmt_ee (MULT, mode,
2093 copy_rtx (op0),
2094 copy_rtx (op1)),
2095 target ? target : product);
2096 }
2097 return product;
2098 }
2099 }
2100
2101 /* It can't be open-coded in this mode.
2102 Use a library call if one is available and caller says that's ok. */
2103
2104 libfunc = optab_libfunc (binoptab, mode);
2105 if (libfunc
2106 && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
2107 {
2108 rtx insns;
2109 rtx op1x = op1;
2110 enum machine_mode op1_mode = mode;
2111 rtx value;
2112
2113 start_sequence ();
2114
2115 if (shift_optab_p (binoptab))
2116 {
2117 op1_mode = targetm.libgcc_shift_count_mode ();
2118 /* Specify unsigned here,
2119 since negative shift counts are meaningless. */
2120 op1x = convert_to_mode (op1_mode, op1, 1);
2121 }
2122
2123 if (GET_MODE (op0) != VOIDmode
2124 && GET_MODE (op0) != mode)
2125 op0 = convert_to_mode (mode, op0, unsignedp);
2126
2127 /* Pass 1 for NO_QUEUE so we don't lose any increments
2128 if the libcall is cse'd or moved. */
2129 value = emit_library_call_value (libfunc,
2130 NULL_RTX, LCT_CONST, mode, 2,
2131 op0, mode, op1x, op1_mode);
2132
2133 insns = get_insns ();
2134 end_sequence ();
2135
2136 target = gen_reg_rtx (mode);
2137 emit_libcall_block_1 (insns, target, value,
2138 gen_rtx_fmt_ee (optab_to_code (binoptab),
2139 mode, op0, op1),
2140 trapv_binoptab_p (binoptab));
2141
2142 return target;
2143 }
2144
2145 delete_insns_since (last);
2146
2147 /* It can't be done in this mode. Can we do it in a wider mode? */
2148
2149 if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
2150 || methods == OPTAB_MUST_WIDEN))
2151 {
2152 /* Caller says, don't even try. */
2153 delete_insns_since (entry_last);
2154 return 0;
2155 }
2156
2157 /* Compute the value of METHODS to pass to recursive calls.
2158 Don't allow widening to be tried recursively. */
2159
2160 methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
2161
2162 /* Look for a wider mode of the same class for which it appears we can do
2163 the operation. */
2164
2165 if (CLASS_HAS_WIDER_MODES_P (mclass))
2166 {
2167 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2168 wider_mode != VOIDmode;
2169 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2170 {
2171 if (find_widening_optab_handler (binoptab, wider_mode, mode, 1)
2172 != CODE_FOR_nothing
2173 || (methods == OPTAB_LIB
2174 && optab_libfunc (binoptab, wider_mode)))
2175 {
2176 rtx xop0 = op0, xop1 = op1;
2177 int no_extend = 0;
2178
2179 /* For certain integer operations, we need not actually extend
2180 the narrow operands, as long as we will truncate
2181 the results to the same narrowness. */
2182
2183 if ((binoptab == ior_optab || binoptab == and_optab
2184 || binoptab == xor_optab
2185 || binoptab == add_optab || binoptab == sub_optab
2186 || binoptab == smul_optab || binoptab == ashl_optab)
2187 && mclass == MODE_INT)
2188 no_extend = 1;
2189
2190 xop0 = widen_operand (xop0, wider_mode, mode,
2191 unsignedp, no_extend);
2192
2193 /* The second operand of a shift must always be extended. */
2194 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
2195 no_extend && binoptab != ashl_optab);
2196
2197 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
2198 unsignedp, methods);
2199 if (temp)
2200 {
2201 if (mclass != MODE_INT
2202 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2203 {
2204 if (target == 0)
2205 target = gen_reg_rtx (mode);
2206 convert_move (target, temp, 0);
2207 return target;
2208 }
2209 else
2210 return gen_lowpart (mode, temp);
2211 }
2212 else
2213 delete_insns_since (last);
2214 }
2215 }
2216 }
2217
2218 delete_insns_since (entry_last);
2219 return 0;
2220 }
2221 \f
2222 /* Expand a binary operator which has both signed and unsigned forms.
2223 UOPTAB is the optab for unsigned operations, and SOPTAB is for
2224 signed operations.
2225
2226 If we widen unsigned operands, we may use a signed wider operation instead
2227 of an unsigned wider operation, since the result would be the same. */
2228
2229 rtx
2230 sign_expand_binop (enum machine_mode mode, optab uoptab, optab soptab,
2231 rtx op0, rtx op1, rtx target, int unsignedp,
2232 enum optab_methods methods)
2233 {
2234 rtx temp;
2235 optab direct_optab = unsignedp ? uoptab : soptab;
2236 bool save_enable;
2237
2238 /* Do it without widening, if possible. */
2239 temp = expand_binop (mode, direct_optab, op0, op1, target,
2240 unsignedp, OPTAB_DIRECT);
2241 if (temp || methods == OPTAB_DIRECT)
2242 return temp;
2243
2244 /* Try widening to a signed int. Disable any direct use of any
2245 signed insn in the current mode. */
2246 save_enable = swap_optab_enable (soptab, mode, false);
2247
2248 temp = expand_binop (mode, soptab, op0, op1, target,
2249 unsignedp, OPTAB_WIDEN);
2250
2251 /* For unsigned operands, try widening to an unsigned int. */
2252 if (!temp && unsignedp)
2253 temp = expand_binop (mode, uoptab, op0, op1, target,
2254 unsignedp, OPTAB_WIDEN);
2255 if (temp || methods == OPTAB_WIDEN)
2256 goto egress;
2257
2258 /* Use the right width libcall if that exists. */
2259 temp = expand_binop (mode, direct_optab, op0, op1, target,
2260 unsignedp, OPTAB_LIB);
2261 if (temp || methods == OPTAB_LIB)
2262 goto egress;
2263
2264 /* Must widen and use a libcall, use either signed or unsigned. */
2265 temp = expand_binop (mode, soptab, op0, op1, target,
2266 unsignedp, methods);
2267 if (!temp && unsignedp)
2268 temp = expand_binop (mode, uoptab, op0, op1, target,
2269 unsignedp, methods);
2270
2271 egress:
2272 /* Undo the fiddling above. */
2273 if (save_enable)
2274 swap_optab_enable (soptab, mode, true);
2275 return temp;
2276 }
2277 \f
2278 /* Generate code to perform an operation specified by UNOPPTAB
2279 on operand OP0, with two results to TARG0 and TARG1.
2280 We assume that the order of the operands for the instruction
2281 is TARG0, TARG1, OP0.
2282
2283 Either TARG0 or TARG1 may be zero, but what that means is that
2284 the result is not actually wanted. We will generate it into
2285 a dummy pseudo-reg and discard it. They may not both be zero.
2286
2287 Returns 1 if this operation can be performed; 0 if not. */
2288
2289 int
2290 expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
2291 int unsignedp)
2292 {
2293 enum machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2294 enum mode_class mclass;
2295 enum machine_mode wider_mode;
2296 rtx entry_last = get_last_insn ();
2297 rtx last;
2298
2299 mclass = GET_MODE_CLASS (mode);
2300
2301 if (!targ0)
2302 targ0 = gen_reg_rtx (mode);
2303 if (!targ1)
2304 targ1 = gen_reg_rtx (mode);
2305
2306 /* Record where to go back to if we fail. */
2307 last = get_last_insn ();
2308
2309 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2310 {
2311 struct expand_operand ops[3];
2312 enum insn_code icode = optab_handler (unoptab, mode);
2313
2314 create_fixed_operand (&ops[0], targ0);
2315 create_fixed_operand (&ops[1], targ1);
2316 create_convert_operand_from (&ops[2], op0, mode, unsignedp);
2317 if (maybe_expand_insn (icode, 3, ops))
2318 return 1;
2319 }
2320
2321 /* It can't be done in this mode. Can we do it in a wider mode? */
2322
2323 if (CLASS_HAS_WIDER_MODES_P (mclass))
2324 {
2325 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2326 wider_mode != VOIDmode;
2327 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2328 {
2329 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2330 {
2331 rtx t0 = gen_reg_rtx (wider_mode);
2332 rtx t1 = gen_reg_rtx (wider_mode);
2333 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2334
2335 if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
2336 {
2337 convert_move (targ0, t0, unsignedp);
2338 convert_move (targ1, t1, unsignedp);
2339 return 1;
2340 }
2341 else
2342 delete_insns_since (last);
2343 }
2344 }
2345 }
2346
2347 delete_insns_since (entry_last);
2348 return 0;
2349 }
2350 \f
2351 /* Generate code to perform an operation specified by BINOPTAB
2352 on operands OP0 and OP1, with two results to TARG1 and TARG2.
2353 We assume that the order of the operands for the instruction
2354 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2355 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2356
2357 Either TARG0 or TARG1 may be zero, but what that means is that
2358 the result is not actually wanted. We will generate it into
2359 a dummy pseudo-reg and discard it. They may not both be zero.
2360
2361 Returns 1 if this operation can be performed; 0 if not. */
2362
2363 int
2364 expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2365 int unsignedp)
2366 {
2367 enum machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2368 enum mode_class mclass;
2369 enum machine_mode wider_mode;
2370 rtx entry_last = get_last_insn ();
2371 rtx last;
2372
2373 mclass = GET_MODE_CLASS (mode);
2374
2375 if (!targ0)
2376 targ0 = gen_reg_rtx (mode);
2377 if (!targ1)
2378 targ1 = gen_reg_rtx (mode);
2379
2380 /* Record where to go back to if we fail. */
2381 last = get_last_insn ();
2382
2383 if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
2384 {
2385 struct expand_operand ops[4];
2386 enum insn_code icode = optab_handler (binoptab, mode);
2387 enum machine_mode mode0 = insn_data[icode].operand[1].mode;
2388 enum machine_mode mode1 = insn_data[icode].operand[2].mode;
2389 rtx xop0 = op0, xop1 = op1;
2390
2391 /* If we are optimizing, force expensive constants into a register. */
2392 xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
2393 xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
2394
2395 create_fixed_operand (&ops[0], targ0);
2396 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2397 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
2398 create_fixed_operand (&ops[3], targ1);
2399 if (maybe_expand_insn (icode, 4, ops))
2400 return 1;
2401 delete_insns_since (last);
2402 }
2403
2404 /* It can't be done in this mode. Can we do it in a wider mode? */
2405
2406 if (CLASS_HAS_WIDER_MODES_P (mclass))
2407 {
2408 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2409 wider_mode != VOIDmode;
2410 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2411 {
2412 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
2413 {
2414 rtx t0 = gen_reg_rtx (wider_mode);
2415 rtx t1 = gen_reg_rtx (wider_mode);
2416 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2417 rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
2418
2419 if (expand_twoval_binop (binoptab, cop0, cop1,
2420 t0, t1, unsignedp))
2421 {
2422 convert_move (targ0, t0, unsignedp);
2423 convert_move (targ1, t1, unsignedp);
2424 return 1;
2425 }
2426 else
2427 delete_insns_since (last);
2428 }
2429 }
2430 }
2431
2432 delete_insns_since (entry_last);
2433 return 0;
2434 }
2435
2436 /* Expand the two-valued library call indicated by BINOPTAB, but
2437 preserve only one of the values. If TARG0 is non-NULL, the first
2438 value is placed into TARG0; otherwise the second value is placed
2439 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2440 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2441 This routine assumes that the value returned by the library call is
2442 as if the return value was of an integral mode twice as wide as the
2443 mode of OP0. Returns 1 if the call was successful. */
2444
2445 bool
2446 expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2447 rtx targ0, rtx targ1, enum rtx_code code)
2448 {
2449 enum machine_mode mode;
2450 enum machine_mode libval_mode;
2451 rtx libval;
2452 rtx insns;
2453 rtx libfunc;
2454
2455 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2456 gcc_assert (!targ0 != !targ1);
2457
2458 mode = GET_MODE (op0);
2459 libfunc = optab_libfunc (binoptab, mode);
2460 if (!libfunc)
2461 return false;
2462
2463 /* The value returned by the library function will have twice as
2464 many bits as the nominal MODE. */
2465 libval_mode = smallest_mode_for_size (2 * GET_MODE_BITSIZE (mode),
2466 MODE_INT);
2467 start_sequence ();
2468 libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
2469 libval_mode, 2,
2470 op0, mode,
2471 op1, mode);
2472 /* Get the part of VAL containing the value that we want. */
2473 libval = simplify_gen_subreg (mode, libval, libval_mode,
2474 targ0 ? 0 : GET_MODE_SIZE (mode));
2475 insns = get_insns ();
2476 end_sequence ();
2477 /* Move the into the desired location. */
2478 emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2479 gen_rtx_fmt_ee (code, mode, op0, op1));
2480
2481 return true;
2482 }
2483
2484 \f
2485 /* Wrapper around expand_unop which takes an rtx code to specify
2486 the operation to perform, not an optab pointer. All other
2487 arguments are the same. */
2488 rtx
2489 expand_simple_unop (enum machine_mode mode, enum rtx_code code, rtx op0,
2490 rtx target, int unsignedp)
2491 {
2492 optab unop = code_to_optab (code);
2493 gcc_assert (unop);
2494
2495 return expand_unop (mode, unop, op0, target, unsignedp);
2496 }
2497
2498 /* Try calculating
2499 (clz:narrow x)
2500 as
2501 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2502
2503 A similar operation can be used for clrsb. UNOPTAB says which operation
2504 we are trying to expand. */
2505 static rtx
2506 widen_leading (enum machine_mode mode, rtx op0, rtx target, optab unoptab)
2507 {
2508 enum mode_class mclass = GET_MODE_CLASS (mode);
2509 if (CLASS_HAS_WIDER_MODES_P (mclass))
2510 {
2511 enum machine_mode wider_mode;
2512 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2513 wider_mode != VOIDmode;
2514 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2515 {
2516 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2517 {
2518 rtx xop0, temp, last;
2519
2520 last = get_last_insn ();
2521
2522 if (target == 0)
2523 target = gen_reg_rtx (mode);
2524 xop0 = widen_operand (op0, wider_mode, mode,
2525 unoptab != clrsb_optab, false);
2526 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2527 unoptab != clrsb_optab);
2528 if (temp != 0)
2529 temp = expand_binop (wider_mode, sub_optab, temp,
2530 GEN_INT (GET_MODE_PRECISION (wider_mode)
2531 - GET_MODE_PRECISION (mode)),
2532 target, true, OPTAB_DIRECT);
2533 if (temp == 0)
2534 delete_insns_since (last);
2535
2536 return temp;
2537 }
2538 }
2539 }
2540 return 0;
2541 }
2542
2543 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2544 quantities, choosing which based on whether the high word is nonzero. */
2545 static rtx
2546 expand_doubleword_clz (enum machine_mode mode, rtx op0, rtx target)
2547 {
2548 rtx xop0 = force_reg (mode, op0);
2549 rtx subhi = gen_highpart (word_mode, xop0);
2550 rtx sublo = gen_lowpart (word_mode, xop0);
2551 rtx hi0_label = gen_label_rtx ();
2552 rtx after_label = gen_label_rtx ();
2553 rtx seq, temp, result;
2554
2555 /* If we were not given a target, use a word_mode register, not a
2556 'mode' register. The result will fit, and nobody is expecting
2557 anything bigger (the return type of __builtin_clz* is int). */
2558 if (!target)
2559 target = gen_reg_rtx (word_mode);
2560
2561 /* In any case, write to a word_mode scratch in both branches of the
2562 conditional, so we can ensure there is a single move insn setting
2563 'target' to tag a REG_EQUAL note on. */
2564 result = gen_reg_rtx (word_mode);
2565
2566 start_sequence ();
2567
2568 /* If the high word is not equal to zero,
2569 then clz of the full value is clz of the high word. */
2570 emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2571 word_mode, true, hi0_label);
2572
2573 temp = expand_unop_direct (word_mode, clz_optab, subhi, result, true);
2574 if (!temp)
2575 goto fail;
2576
2577 if (temp != result)
2578 convert_move (result, temp, true);
2579
2580 emit_jump_insn (gen_jump (after_label));
2581 emit_barrier ();
2582
2583 /* Else clz of the full value is clz of the low word plus the number
2584 of bits in the high word. */
2585 emit_label (hi0_label);
2586
2587 temp = expand_unop_direct (word_mode, clz_optab, sublo, 0, true);
2588 if (!temp)
2589 goto fail;
2590 temp = expand_binop (word_mode, add_optab, temp,
2591 GEN_INT (GET_MODE_BITSIZE (word_mode)),
2592 result, true, OPTAB_DIRECT);
2593 if (!temp)
2594 goto fail;
2595 if (temp != result)
2596 convert_move (result, temp, true);
2597
2598 emit_label (after_label);
2599 convert_move (target, result, true);
2600
2601 seq = get_insns ();
2602 end_sequence ();
2603
2604 add_equal_note (seq, target, CLZ, xop0, 0);
2605 emit_insn (seq);
2606 return target;
2607
2608 fail:
2609 end_sequence ();
2610 return 0;
2611 }
2612
2613 /* Try calculating
2614 (bswap:narrow x)
2615 as
2616 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2617 static rtx
2618 widen_bswap (enum machine_mode mode, rtx op0, rtx target)
2619 {
2620 enum mode_class mclass = GET_MODE_CLASS (mode);
2621 enum machine_mode wider_mode;
2622 rtx x, last;
2623
2624 if (!CLASS_HAS_WIDER_MODES_P (mclass))
2625 return NULL_RTX;
2626
2627 for (wider_mode = GET_MODE_WIDER_MODE (mode);
2628 wider_mode != VOIDmode;
2629 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2630 if (optab_handler (bswap_optab, wider_mode) != CODE_FOR_nothing)
2631 goto found;
2632 return NULL_RTX;
2633
2634 found:
2635 last = get_last_insn ();
2636
2637 x = widen_operand (op0, wider_mode, mode, true, true);
2638 x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true);
2639
2640 gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2641 && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2642 if (x != 0)
2643 x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2644 GET_MODE_BITSIZE (wider_mode)
2645 - GET_MODE_BITSIZE (mode),
2646 NULL_RTX, true);
2647
2648 if (x != 0)
2649 {
2650 if (target == 0)
2651 target = gen_reg_rtx (mode);
2652 emit_move_insn (target, gen_lowpart (mode, x));
2653 }
2654 else
2655 delete_insns_since (last);
2656
2657 return target;
2658 }
2659
2660 /* Try calculating bswap as two bswaps of two word-sized operands. */
2661
2662 static rtx
2663 expand_doubleword_bswap (enum machine_mode mode, rtx op, rtx target)
2664 {
2665 rtx t0, t1;
2666
2667 t1 = expand_unop (word_mode, bswap_optab,
2668 operand_subword_force (op, 0, mode), NULL_RTX, true);
2669 t0 = expand_unop (word_mode, bswap_optab,
2670 operand_subword_force (op, 1, mode), NULL_RTX, true);
2671
2672 if (target == 0 || !valid_multiword_target_p (target))
2673 target = gen_reg_rtx (mode);
2674 if (REG_P (target))
2675 emit_clobber (target);
2676 emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2677 emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2678
2679 return target;
2680 }
2681
2682 /* Try calculating (parity x) as (and (popcount x) 1), where
2683 popcount can also be done in a wider mode. */
2684 static rtx
2685 expand_parity (enum machine_mode mode, rtx op0, rtx target)
2686 {
2687 enum mode_class mclass = GET_MODE_CLASS (mode);
2688 if (CLASS_HAS_WIDER_MODES_P (mclass))
2689 {
2690 enum machine_mode wider_mode;
2691 for (wider_mode = mode; wider_mode != VOIDmode;
2692 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
2693 {
2694 if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
2695 {
2696 rtx xop0, temp, last;
2697
2698 last = get_last_insn ();
2699
2700 if (target == 0)
2701 target = gen_reg_rtx (mode);
2702 xop0 = widen_operand (op0, wider_mode, mode, true, false);
2703 temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
2704 true);
2705 if (temp != 0)
2706 temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
2707 target, true, OPTAB_DIRECT);
2708 if (temp == 0)
2709 delete_insns_since (last);
2710
2711 return temp;
2712 }
2713 }
2714 }
2715 return 0;
2716 }
2717
2718 /* Try calculating ctz(x) as K - clz(x & -x) ,
2719 where K is GET_MODE_PRECISION(mode) - 1.
2720
2721 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2722 don't have to worry about what the hardware does in that case. (If
2723 the clz instruction produces the usual value at 0, which is K, the
2724 result of this code sequence will be -1; expand_ffs, below, relies
2725 on this. It might be nice to have it be K instead, for consistency
2726 with the (very few) processors that provide a ctz with a defined
2727 value, but that would take one more instruction, and it would be
2728 less convenient for expand_ffs anyway. */
2729
2730 static rtx
2731 expand_ctz (enum machine_mode mode, rtx op0, rtx target)
2732 {
2733 rtx seq, temp;
2734
2735 if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2736 return 0;
2737
2738 start_sequence ();
2739
2740 temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
2741 if (temp)
2742 temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
2743 true, OPTAB_DIRECT);
2744 if (temp)
2745 temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
2746 if (temp)
2747 temp = expand_binop (mode, sub_optab, GEN_INT (GET_MODE_PRECISION (mode) - 1),
2748 temp, target,
2749 true, OPTAB_DIRECT);
2750 if (temp == 0)
2751 {
2752 end_sequence ();
2753 return 0;
2754 }
2755
2756 seq = get_insns ();
2757 end_sequence ();
2758
2759 add_equal_note (seq, temp, CTZ, op0, 0);
2760 emit_insn (seq);
2761 return temp;
2762 }
2763
2764
2765 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2766 else with the sequence used by expand_clz.
2767
2768 The ffs builtin promises to return zero for a zero value and ctz/clz
2769 may have an undefined value in that case. If they do not give us a
2770 convenient value, we have to generate a test and branch. */
2771 static rtx
2772 expand_ffs (enum machine_mode mode, rtx op0, rtx target)
2773 {
2774 HOST_WIDE_INT val = 0;
2775 bool defined_at_zero = false;
2776 rtx temp, seq;
2777
2778 if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
2779 {
2780 start_sequence ();
2781
2782 temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
2783 if (!temp)
2784 goto fail;
2785
2786 defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
2787 }
2788 else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
2789 {
2790 start_sequence ();
2791 temp = expand_ctz (mode, op0, 0);
2792 if (!temp)
2793 goto fail;
2794
2795 if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
2796 {
2797 defined_at_zero = true;
2798 val = (GET_MODE_PRECISION (mode) - 1) - val;
2799 }
2800 }
2801 else
2802 return 0;
2803
2804 if (defined_at_zero && val == -1)
2805 /* No correction needed at zero. */;
2806 else
2807 {
2808 /* We don't try to do anything clever with the situation found
2809 on some processors (eg Alpha) where ctz(0:mode) ==
2810 bitsize(mode). If someone can think of a way to send N to -1
2811 and leave alone all values in the range 0..N-1 (where N is a
2812 power of two), cheaper than this test-and-branch, please add it.
2813
2814 The test-and-branch is done after the operation itself, in case
2815 the operation sets condition codes that can be recycled for this.
2816 (This is true on i386, for instance.) */
2817
2818 rtx nonzero_label = gen_label_rtx ();
2819 emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
2820 mode, true, nonzero_label);
2821
2822 convert_move (temp, GEN_INT (-1), false);
2823 emit_label (nonzero_label);
2824 }
2825
2826 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2827 to produce a value in the range 0..bitsize. */
2828 temp = expand_binop (mode, add_optab, temp, GEN_INT (1),
2829 target, false, OPTAB_DIRECT);
2830 if (!temp)
2831 goto fail;
2832
2833 seq = get_insns ();
2834 end_sequence ();
2835
2836 add_equal_note (seq, temp, FFS, op0, 0);
2837 emit_insn (seq);
2838 return temp;
2839
2840 fail:
2841 end_sequence ();
2842 return 0;
2843 }
2844
2845 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2846 conditions, VAL may already be a SUBREG against which we cannot generate
2847 a further SUBREG. In this case, we expect forcing the value into a
2848 register will work around the situation. */
2849
2850 static rtx
2851 lowpart_subreg_maybe_copy (enum machine_mode omode, rtx val,
2852 enum machine_mode imode)
2853 {
2854 rtx ret;
2855 ret = lowpart_subreg (omode, val, imode);
2856 if (ret == NULL)
2857 {
2858 val = force_reg (imode, val);
2859 ret = lowpart_subreg (omode, val, imode);
2860 gcc_assert (ret != NULL);
2861 }
2862 return ret;
2863 }
2864
2865 /* Expand a floating point absolute value or negation operation via a
2866 logical operation on the sign bit. */
2867
2868 static rtx
2869 expand_absneg_bit (enum rtx_code code, enum machine_mode mode,
2870 rtx op0, rtx target)
2871 {
2872 const struct real_format *fmt;
2873 int bitpos, word, nwords, i;
2874 enum machine_mode imode;
2875 double_int mask;
2876 rtx temp, insns;
2877
2878 /* The format has to have a simple sign bit. */
2879 fmt = REAL_MODE_FORMAT (mode);
2880 if (fmt == NULL)
2881 return NULL_RTX;
2882
2883 bitpos = fmt->signbit_rw;
2884 if (bitpos < 0)
2885 return NULL_RTX;
2886
2887 /* Don't create negative zeros if the format doesn't support them. */
2888 if (code == NEG && !fmt->has_signed_zero)
2889 return NULL_RTX;
2890
2891 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
2892 {
2893 imode = int_mode_for_mode (mode);
2894 if (imode == BLKmode)
2895 return NULL_RTX;
2896 word = 0;
2897 nwords = 1;
2898 }
2899 else
2900 {
2901 imode = word_mode;
2902
2903 if (FLOAT_WORDS_BIG_ENDIAN)
2904 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
2905 else
2906 word = bitpos / BITS_PER_WORD;
2907 bitpos = bitpos % BITS_PER_WORD;
2908 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
2909 }
2910
2911 mask = double_int_setbit (double_int_zero, bitpos);
2912 if (code == ABS)
2913 mask = double_int_not (mask);
2914
2915 if (target == 0
2916 || target == op0
2917 || (nwords > 1 && !valid_multiword_target_p (target)))
2918 target = gen_reg_rtx (mode);
2919
2920 if (nwords > 1)
2921 {
2922 start_sequence ();
2923
2924 for (i = 0; i < nwords; ++i)
2925 {
2926 rtx targ_piece = operand_subword (target, i, 1, mode);
2927 rtx op0_piece = operand_subword_force (op0, i, mode);
2928
2929 if (i == word)
2930 {
2931 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2932 op0_piece,
2933 immed_double_int_const (mask, imode),
2934 targ_piece, 1, OPTAB_LIB_WIDEN);
2935 if (temp != targ_piece)
2936 emit_move_insn (targ_piece, temp);
2937 }
2938 else
2939 emit_move_insn (targ_piece, op0_piece);
2940 }
2941
2942 insns = get_insns ();
2943 end_sequence ();
2944
2945 emit_insn (insns);
2946 }
2947 else
2948 {
2949 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2950 gen_lowpart (imode, op0),
2951 immed_double_int_const (mask, imode),
2952 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
2953 target = lowpart_subreg_maybe_copy (mode, temp, imode);
2954
2955 set_dst_reg_note (get_last_insn (), REG_EQUAL,
2956 gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
2957 target);
2958 }
2959
2960 return target;
2961 }
2962
2963 /* As expand_unop, but will fail rather than attempt the operation in a
2964 different mode or with a libcall. */
2965 static rtx
2966 expand_unop_direct (enum machine_mode mode, optab unoptab, rtx op0, rtx target,
2967 int unsignedp)
2968 {
2969 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2970 {
2971 struct expand_operand ops[2];
2972 enum insn_code icode = optab_handler (unoptab, mode);
2973 rtx last = get_last_insn ();
2974 rtx pat;
2975
2976 create_output_operand (&ops[0], target, mode);
2977 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2978 pat = maybe_gen_insn (icode, 2, ops);
2979 if (pat)
2980 {
2981 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
2982 && ! add_equal_note (pat, ops[0].value, optab_to_code (unoptab),
2983 ops[1].value, NULL_RTX))
2984 {
2985 delete_insns_since (last);
2986 return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
2987 }
2988
2989 emit_insn (pat);
2990
2991 return ops[0].value;
2992 }
2993 }
2994 return 0;
2995 }
2996
2997 /* Generate code to perform an operation specified by UNOPTAB
2998 on operand OP0, with result having machine-mode MODE.
2999
3000 UNSIGNEDP is for the case where we have to widen the operands
3001 to perform the operation. It says to use zero-extension.
3002
3003 If TARGET is nonzero, the value
3004 is generated there, if it is convenient to do so.
3005 In all cases an rtx is returned for the locus of the value;
3006 this may or may not be TARGET. */
3007
3008 rtx
3009 expand_unop (enum machine_mode mode, optab unoptab, rtx op0, rtx target,
3010 int unsignedp)
3011 {
3012 enum mode_class mclass = GET_MODE_CLASS (mode);
3013 enum machine_mode wider_mode;
3014 rtx temp;
3015 rtx libfunc;
3016
3017 temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
3018 if (temp)
3019 return temp;
3020
3021 /* It can't be done in this mode. Can we open-code it in a wider mode? */
3022
3023 /* Widening (or narrowing) clz needs special treatment. */
3024 if (unoptab == clz_optab)
3025 {
3026 temp = widen_leading (mode, op0, target, unoptab);
3027 if (temp)
3028 return temp;
3029
3030 if (GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
3031 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3032 {
3033 temp = expand_doubleword_clz (mode, op0, target);
3034 if (temp)
3035 return temp;
3036 }
3037
3038 goto try_libcall;
3039 }
3040
3041 if (unoptab == clrsb_optab)
3042 {
3043 temp = widen_leading (mode, op0, target, unoptab);
3044 if (temp)
3045 return temp;
3046 goto try_libcall;
3047 }
3048
3049 /* Widening (or narrowing) bswap needs special treatment. */
3050 if (unoptab == bswap_optab)
3051 {
3052 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
3053 or ROTATERT. First try these directly; if this fails, then try the
3054 obvious pair of shifts with allowed widening, as this will probably
3055 be always more efficient than the other fallback methods. */
3056 if (mode == HImode)
3057 {
3058 rtx last, temp1, temp2;
3059
3060 if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
3061 {
3062 temp = expand_binop (mode, rotl_optab, op0, GEN_INT (8), target,
3063 unsignedp, OPTAB_DIRECT);
3064 if (temp)
3065 return temp;
3066 }
3067
3068 if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
3069 {
3070 temp = expand_binop (mode, rotr_optab, op0, GEN_INT (8), target,
3071 unsignedp, OPTAB_DIRECT);
3072 if (temp)
3073 return temp;
3074 }
3075
3076 last = get_last_insn ();
3077
3078 temp1 = expand_binop (mode, ashl_optab, op0, GEN_INT (8), NULL_RTX,
3079 unsignedp, OPTAB_WIDEN);
3080 temp2 = expand_binop (mode, lshr_optab, op0, GEN_INT (8), NULL_RTX,
3081 unsignedp, OPTAB_WIDEN);
3082 if (temp1 && temp2)
3083 {
3084 temp = expand_binop (mode, ior_optab, temp1, temp2, target,
3085 unsignedp, OPTAB_WIDEN);
3086 if (temp)
3087 return temp;
3088 }
3089
3090 delete_insns_since (last);
3091 }
3092
3093 temp = widen_bswap (mode, op0, target);
3094 if (temp)
3095 return temp;
3096
3097 if (GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
3098 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3099 {
3100 temp = expand_doubleword_bswap (mode, op0, target);
3101 if (temp)
3102 return temp;
3103 }
3104
3105 goto try_libcall;
3106 }
3107
3108 if (CLASS_HAS_WIDER_MODES_P (mclass))
3109 for (wider_mode = GET_MODE_WIDER_MODE (mode);
3110 wider_mode != VOIDmode;
3111 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
3112 {
3113 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
3114 {
3115 rtx xop0 = op0;
3116 rtx last = get_last_insn ();
3117
3118 /* For certain operations, we need not actually extend
3119 the narrow operand, as long as we will truncate the
3120 results to the same narrowness. */
3121
3122 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3123 (unoptab == neg_optab
3124 || unoptab == one_cmpl_optab)
3125 && mclass == MODE_INT);
3126
3127 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3128 unsignedp);
3129
3130 if (temp)
3131 {
3132 if (mclass != MODE_INT
3133 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
3134 {
3135 if (target == 0)
3136 target = gen_reg_rtx (mode);
3137 convert_move (target, temp, 0);
3138 return target;
3139 }
3140 else
3141 return gen_lowpart (mode, temp);
3142 }
3143 else
3144 delete_insns_since (last);
3145 }
3146 }
3147
3148 /* These can be done a word at a time. */
3149 if (unoptab == one_cmpl_optab
3150 && mclass == MODE_INT
3151 && GET_MODE_SIZE (mode) > UNITS_PER_WORD
3152 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3153 {
3154 int i;
3155 rtx insns;
3156
3157 if (target == 0 || target == op0 || !valid_multiword_target_p (target))
3158 target = gen_reg_rtx (mode);
3159
3160 start_sequence ();
3161
3162 /* Do the actual arithmetic. */
3163 for (i = 0; i < GET_MODE_BITSIZE (mode) / BITS_PER_WORD; i++)
3164 {
3165 rtx target_piece = operand_subword (target, i, 1, mode);
3166 rtx x = expand_unop (word_mode, unoptab,
3167 operand_subword_force (op0, i, mode),
3168 target_piece, unsignedp);
3169
3170 if (target_piece != x)
3171 emit_move_insn (target_piece, x);
3172 }
3173
3174 insns = get_insns ();
3175 end_sequence ();
3176
3177 emit_insn (insns);
3178 return target;
3179 }
3180
3181 if (optab_to_code (unoptab) == NEG)
3182 {
3183 /* Try negating floating point values by flipping the sign bit. */
3184 if (SCALAR_FLOAT_MODE_P (mode))
3185 {
3186 temp = expand_absneg_bit (NEG, mode, op0, target);
3187 if (temp)
3188 return temp;
3189 }
3190
3191 /* If there is no negation pattern, and we have no negative zero,
3192 try subtracting from zero. */
3193 if (!HONOR_SIGNED_ZEROS (mode))
3194 {
3195 temp = expand_binop (mode, (unoptab == negv_optab
3196 ? subv_optab : sub_optab),
3197 CONST0_RTX (mode), op0, target,
3198 unsignedp, OPTAB_DIRECT);
3199 if (temp)
3200 return temp;
3201 }
3202 }
3203
3204 /* Try calculating parity (x) as popcount (x) % 2. */
3205 if (unoptab == parity_optab)
3206 {
3207 temp = expand_parity (mode, op0, target);
3208 if (temp)
3209 return temp;
3210 }
3211
3212 /* Try implementing ffs (x) in terms of clz (x). */
3213 if (unoptab == ffs_optab)
3214 {
3215 temp = expand_ffs (mode, op0, target);
3216 if (temp)
3217 return temp;
3218 }
3219
3220 /* Try implementing ctz (x) in terms of clz (x). */
3221 if (unoptab == ctz_optab)
3222 {
3223 temp = expand_ctz (mode, op0, target);
3224 if (temp)
3225 return temp;
3226 }
3227
3228 try_libcall:
3229 /* Now try a library call in this mode. */
3230 libfunc = optab_libfunc (unoptab, mode);
3231 if (libfunc)
3232 {
3233 rtx insns;
3234 rtx value;
3235 rtx eq_value;
3236 enum machine_mode outmode = mode;
3237
3238 /* All of these functions return small values. Thus we choose to
3239 have them return something that isn't a double-word. */
3240 if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
3241 || unoptab == clrsb_optab || unoptab == popcount_optab
3242 || unoptab == parity_optab)
3243 outmode
3244 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
3245 optab_libfunc (unoptab, mode)));
3246
3247 start_sequence ();
3248
3249 /* Pass 1 for NO_QUEUE so we don't lose any increments
3250 if the libcall is cse'd or moved. */
3251 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
3252 1, op0, mode);
3253 insns = get_insns ();
3254 end_sequence ();
3255
3256 target = gen_reg_rtx (outmode);
3257 eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
3258 if (GET_MODE_SIZE (outmode) < GET_MODE_SIZE (mode))
3259 eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
3260 else if (GET_MODE_SIZE (outmode) > GET_MODE_SIZE (mode))
3261 eq_value = simplify_gen_unary (ZERO_EXTEND, outmode, eq_value, mode);
3262 emit_libcall_block_1 (insns, target, value, eq_value,
3263 trapv_unoptab_p (unoptab));
3264
3265 return target;
3266 }
3267
3268 /* It can't be done in this mode. Can we do it in a wider mode? */
3269
3270 if (CLASS_HAS_WIDER_MODES_P (mclass))
3271 {
3272 for (wider_mode = GET_MODE_WIDER_MODE (mode);
3273 wider_mode != VOIDmode;
3274 wider_mode = GET_MODE_WIDER_MODE (wider_mode))
3275 {
3276 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
3277 || optab_libfunc (unoptab, wider_mode))
3278 {
3279 rtx xop0 = op0;
3280 rtx last = get_last_insn ();
3281
3282 /* For certain operations, we need not actually extend
3283 the narrow operand, as long as we will truncate the
3284 results to the same narrowness. */
3285 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3286 (unoptab == neg_optab
3287 || unoptab == one_cmpl_optab
3288 || unoptab == bswap_optab)
3289 && mclass == MODE_INT);
3290
3291 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3292 unsignedp);
3293
3294 /* If we are generating clz using wider mode, adjust the
3295 result. Similarly for clrsb. */
3296 if ((unoptab == clz_optab || unoptab == clrsb_optab)
3297 && temp != 0)
3298 temp = expand_binop (wider_mode, sub_optab, temp,
3299 GEN_INT (GET_MODE_PRECISION (wider_mode)
3300 - GET_MODE_PRECISION (mode)),
3301 target, true, OPTAB_DIRECT);
3302
3303 /* Likewise for bswap. */
3304 if (unoptab == bswap_optab && temp != 0)
3305 {
3306 gcc_assert (GET_MODE_PRECISION (wider_mode)
3307 == GET_MODE_BITSIZE (wider_mode)
3308 && GET_MODE_PRECISION (mode)
3309 == GET_MODE_BITSIZE (mode));
3310
3311 temp = expand_shift (RSHIFT_EXPR, wider_mode, temp,
3312 GET_MODE_BITSIZE (wider_mode)
3313 - GET_MODE_BITSIZE (mode),
3314 NULL_RTX, true);
3315 }
3316
3317 if (temp)
3318 {
3319 if (mclass != MODE_INT)
3320 {
3321 if (target == 0)
3322 target = gen_reg_rtx (mode);
3323 convert_move (target, temp, 0);
3324 return target;
3325 }
3326 else
3327 return gen_lowpart (mode, temp);
3328 }
3329 else
3330 delete_insns_since (last);
3331 }
3332 }
3333 }
3334
3335 /* One final attempt at implementing negation via subtraction,
3336 this time allowing widening of the operand. */
3337 if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3338 {
3339 rtx temp;
3340 temp = expand_binop (mode,
3341 unoptab == negv_optab ? subv_optab : sub_optab,
3342 CONST0_RTX (mode), op0,
3343 target, unsignedp, OPTAB_LIB_WIDEN);
3344 if (temp)
3345 return temp;
3346 }
3347
3348 return 0;
3349 }
3350 \f
3351 /* Emit code to compute the absolute value of OP0, with result to
3352 TARGET if convenient. (TARGET may be 0.) The return value says
3353 where the result actually is to be found.
3354
3355 MODE is the mode of the operand; the mode of the result is
3356 different but can be deduced from MODE.
3357
3358 */
3359
3360 rtx
3361 expand_abs_nojump (enum machine_mode mode, rtx op0, rtx target,
3362 int result_unsignedp)
3363 {
3364 rtx temp;
3365
3366 if (! flag_trapv)
3367 result_unsignedp = 1;
3368
3369 /* First try to do it with a special abs instruction. */
3370 temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
3371 op0, target, 0);
3372 if (temp != 0)
3373 return temp;
3374
3375 /* For floating point modes, try clearing the sign bit. */
3376 if (SCALAR_FLOAT_MODE_P (mode))
3377 {
3378 temp = expand_absneg_bit (ABS, mode, op0, target);
3379 if (temp)
3380 return temp;
3381 }
3382
3383 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3384 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
3385 && !HONOR_SIGNED_ZEROS (mode))
3386 {
3387 rtx last = get_last_insn ();
3388
3389 temp = expand_unop (mode, neg_optab, op0, NULL_RTX, 0);
3390 if (temp != 0)
3391 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3392 OPTAB_WIDEN);
3393
3394 if (temp != 0)
3395 return temp;
3396
3397 delete_insns_since (last);
3398 }
3399
3400 /* If this machine has expensive jumps, we can do integer absolute
3401 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3402 where W is the width of MODE. */
3403
3404 if (GET_MODE_CLASS (mode) == MODE_INT
3405 && BRANCH_COST (optimize_insn_for_speed_p (),
3406 false) >= 2)
3407 {
3408 rtx extended = expand_shift (RSHIFT_EXPR, mode, op0,
3409 GET_MODE_PRECISION (mode) - 1,
3410 NULL_RTX, 0);
3411
3412 temp = expand_binop (mode, xor_optab, extended, op0, target, 0,
3413 OPTAB_LIB_WIDEN);
3414 if (temp != 0)
3415 temp = expand_binop (mode, result_unsignedp ? sub_optab : subv_optab,
3416 temp, extended, target, 0, OPTAB_LIB_WIDEN);
3417
3418 if (temp != 0)
3419 return temp;
3420 }
3421
3422 return NULL_RTX;
3423 }
3424
3425 rtx
3426 expand_abs (enum machine_mode mode, rtx op0, rtx target,
3427 int result_unsignedp, int safe)
3428 {
3429 rtx temp, op1;
3430
3431 if (! flag_trapv)
3432 result_unsignedp = 1;
3433
3434 temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3435 if (temp != 0)
3436 return temp;
3437
3438 /* If that does not win, use conditional jump and negate. */
3439
3440 /* It is safe to use the target if it is the same
3441 as the source if this is also a pseudo register */
3442 if (op0 == target && REG_P (op0)
3443 && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3444 safe = 1;
3445
3446 op1 = gen_label_rtx ();
3447 if (target == 0 || ! safe
3448 || GET_MODE (target) != mode
3449 || (MEM_P (target) && MEM_VOLATILE_P (target))
3450 || (REG_P (target)
3451 && REGNO (target) < FIRST_PSEUDO_REGISTER))
3452 target = gen_reg_rtx (mode);
3453
3454 emit_move_insn (target, op0);
3455 NO_DEFER_POP;
3456
3457 do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3458 NULL_RTX, NULL_RTX, op1, -1);
3459
3460 op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3461 target, target, 0);
3462 if (op0 != target)
3463 emit_move_insn (target, op0);
3464 emit_label (op1);
3465 OK_DEFER_POP;
3466 return target;
3467 }
3468
3469 /* Emit code to compute the one's complement absolute value of OP0
3470 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3471 (TARGET may be NULL_RTX.) The return value says where the result
3472 actually is to be found.
3473
3474 MODE is the mode of the operand; the mode of the result is
3475 different but can be deduced from MODE. */
3476
3477 rtx
3478 expand_one_cmpl_abs_nojump (enum machine_mode mode, rtx op0, rtx target)
3479 {
3480 rtx temp;
3481
3482 /* Not applicable for floating point modes. */
3483 if (FLOAT_MODE_P (mode))
3484 return NULL_RTX;
3485
3486 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3487 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
3488 {
3489 rtx last = get_last_insn ();
3490
3491 temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
3492 if (temp != 0)
3493 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3494 OPTAB_WIDEN);
3495
3496 if (temp != 0)
3497 return temp;
3498
3499 delete_insns_since (last);
3500 }
3501
3502 /* If this machine has expensive jumps, we can do one's complement
3503 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3504
3505 if (GET_MODE_CLASS (mode) == MODE_INT
3506 && BRANCH_COST (optimize_insn_for_speed_p (),
3507 false) >= 2)
3508 {
3509 rtx extended = expand_shift (RSHIFT_EXPR, mode, op0,
3510 GET_MODE_PRECISION (mode) - 1,
3511 NULL_RTX, 0);
3512
3513 temp = expand_binop (mode, xor_optab, extended, op0, target, 0,
3514 OPTAB_LIB_WIDEN);
3515
3516 if (temp != 0)
3517 return temp;
3518 }
3519
3520 return NULL_RTX;
3521 }
3522
3523 /* A subroutine of expand_copysign, perform the copysign operation using the
3524 abs and neg primitives advertised to exist on the target. The assumption
3525 is that we have a split register file, and leaving op0 in fp registers,
3526 and not playing with subregs so much, will help the register allocator. */
3527
3528 static rtx
3529 expand_copysign_absneg (enum machine_mode mode, rtx op0, rtx op1, rtx target,
3530 int bitpos, bool op0_is_abs)
3531 {
3532 enum machine_mode imode;
3533 enum insn_code icode;
3534 rtx sign, label;
3535
3536 if (target == op1)
3537 target = NULL_RTX;
3538
3539 /* Check if the back end provides an insn that handles signbit for the
3540 argument's mode. */
3541 icode = optab_handler (signbit_optab, mode);
3542 if (icode != CODE_FOR_nothing)
3543 {
3544 imode = insn_data[(int) icode].operand[0].mode;
3545 sign = gen_reg_rtx (imode);
3546 emit_unop_insn (icode, sign, op1, UNKNOWN);
3547 }
3548 else
3549 {
3550 double_int mask;
3551
3552 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3553 {
3554 imode = int_mode_for_mode (mode);
3555 if (imode == BLKmode)
3556 return NULL_RTX;
3557 op1 = gen_lowpart (imode, op1);
3558 }
3559 else
3560 {
3561 int word;
3562
3563 imode = word_mode;
3564 if (FLOAT_WORDS_BIG_ENDIAN)
3565 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3566 else
3567 word = bitpos / BITS_PER_WORD;
3568 bitpos = bitpos % BITS_PER_WORD;
3569 op1 = operand_subword_force (op1, word, mode);
3570 }
3571
3572 mask = double_int_setbit (double_int_zero, bitpos);
3573
3574 sign = expand_binop (imode, and_optab, op1,
3575 immed_double_int_const (mask, imode),
3576 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3577 }
3578
3579 if (!op0_is_abs)
3580 {
3581 op0 = expand_unop (mode, abs_optab, op0, target, 0);
3582 if (op0 == NULL)
3583 return NULL_RTX;
3584 target = op0;
3585 }
3586 else
3587 {
3588 if (target == NULL_RTX)
3589 target = copy_to_reg (op0);
3590 else
3591 emit_move_insn (target, op0);
3592 }
3593
3594 label = gen_label_rtx ();
3595 emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
3596
3597 if (GET_CODE (op0) == CONST_DOUBLE)
3598 op0 = simplify_unary_operation (NEG, mode, op0, mode);
3599 else
3600 op0 = expand_unop (mode, neg_optab, op0, target, 0);
3601 if (op0 != target)
3602 emit_move_insn (target, op0);
3603
3604 emit_label (label);
3605
3606 return target;
3607 }
3608
3609
3610 /* A subroutine of expand_copysign, perform the entire copysign operation
3611 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3612 is true if op0 is known to have its sign bit clear. */
3613
3614 static rtx
3615 expand_copysign_bit (enum machine_mode mode, rtx op0, rtx op1, rtx target,
3616 int bitpos, bool op0_is_abs)
3617 {
3618 enum machine_mode imode;
3619 double_int mask;
3620 int word, nwords, i;
3621 rtx temp, insns;
3622
3623 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3624 {
3625 imode = int_mode_for_mode (mode);
3626 if (imode == BLKmode)
3627 return NULL_RTX;
3628 word = 0;
3629 nwords = 1;
3630 }
3631 else
3632 {
3633 imode = word_mode;
3634
3635 if (FLOAT_WORDS_BIG_ENDIAN)
3636 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3637 else
3638 word = bitpos / BITS_PER_WORD;
3639 bitpos = bitpos % BITS_PER_WORD;
3640 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3641 }
3642
3643 mask = double_int_setbit (double_int_zero, bitpos);
3644
3645 if (target == 0
3646 || target == op0
3647 || target == op1
3648 || (nwords > 1 && !valid_multiword_target_p (target)))
3649 target = gen_reg_rtx (mode);
3650
3651 if (nwords > 1)
3652 {
3653 start_sequence ();
3654
3655 for (i = 0; i < nwords; ++i)
3656 {
3657 rtx targ_piece = operand_subword (target, i, 1, mode);
3658 rtx op0_piece = operand_subword_force (op0, i, mode);
3659
3660 if (i == word)
3661 {
3662 if (!op0_is_abs)
3663 op0_piece
3664 = expand_binop (imode, and_optab, op0_piece,
3665 immed_double_int_const (double_int_not (mask),
3666 imode),
3667 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3668
3669 op1 = expand_binop (imode, and_optab,
3670 operand_subword_force (op1, i, mode),
3671 immed_double_int_const (mask, imode),
3672 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3673
3674 temp = expand_binop (imode, ior_optab, op0_piece, op1,
3675 targ_piece, 1, OPTAB_LIB_WIDEN);
3676 if (temp != targ_piece)
3677 emit_move_insn (targ_piece, temp);
3678 }
3679 else
3680 emit_move_insn (targ_piece, op0_piece);
3681 }
3682
3683 insns = get_insns ();
3684 end_sequence ();
3685
3686 emit_insn (insns);
3687 }
3688 else
3689 {
3690 op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
3691 immed_double_int_const (mask, imode),
3692 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3693
3694 op0 = gen_lowpart (imode, op0);
3695 if (!op0_is_abs)
3696 op0 = expand_binop (imode, and_optab, op0,
3697 immed_double_int_const (double_int_not (mask),
3698 imode),
3699 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3700
3701 temp = expand_binop (imode, ior_optab, op0, op1,
3702 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
3703 target = lowpart_subreg_maybe_copy (mode, temp, imode);
3704 }
3705
3706 return target;
3707 }
3708
3709 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3710 scalar floating point mode. Return NULL if we do not know how to
3711 expand the operation inline. */
3712
3713 rtx
3714 expand_copysign (rtx op0, rtx op1, rtx target)
3715 {
3716 enum machine_mode mode = GET_MODE (op0);
3717 const struct real_format *fmt;
3718 bool op0_is_abs;
3719 rtx temp;
3720
3721 gcc_assert (SCALAR_FLOAT_MODE_P (mode));
3722 gcc_assert (GET_MODE (op1) == mode);
3723
3724 /* First try to do it with a special instruction. */
3725 temp = expand_binop (mode, copysign_optab, op0, op1,
3726 target, 0, OPTAB_DIRECT);
3727 if (temp)
3728 return temp;
3729
3730 fmt = REAL_MODE_FORMAT (mode);
3731 if (fmt == NULL || !fmt->has_signed_zero)
3732 return NULL_RTX;
3733
3734 op0_is_abs = false;
3735 if (GET_CODE (op0) == CONST_DOUBLE)
3736 {
3737 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
3738 op0 = simplify_unary_operation (ABS, mode, op0, mode);
3739 op0_is_abs = true;
3740 }
3741
3742 if (fmt->signbit_ro >= 0
3743 && (GET_CODE (op0) == CONST_DOUBLE
3744 || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
3745 && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
3746 {
3747 temp = expand_copysign_absneg (mode, op0, op1, target,
3748 fmt->signbit_ro, op0_is_abs);
3749 if (temp)
3750 return temp;
3751 }
3752
3753 if (fmt->signbit_rw < 0)
3754 return NULL_RTX;
3755 return expand_copysign_bit (mode, op0, op1, target,
3756 fmt->signbit_rw, op0_is_abs);
3757 }
3758 \f
3759 /* Generate an instruction whose insn-code is INSN_CODE,
3760 with two operands: an output TARGET and an input OP0.
3761 TARGET *must* be nonzero, and the output is always stored there.
3762 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3763 the value that is stored into TARGET.
3764
3765 Return false if expansion failed. */
3766
3767 bool
3768 maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
3769 enum rtx_code code)
3770 {
3771 struct expand_operand ops[2];
3772 rtx pat;
3773
3774 create_output_operand (&ops[0], target, GET_MODE (target));
3775 create_input_operand (&ops[1], op0, GET_MODE (op0));
3776 pat = maybe_gen_insn (icode, 2, ops);
3777 if (!pat)
3778 return false;
3779
3780 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX && code != UNKNOWN)
3781 add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX);
3782
3783 emit_insn (pat);
3784
3785 if (ops[0].value != target)
3786 emit_move_insn (target, ops[0].value);
3787 return true;
3788 }
3789 /* Generate an instruction whose insn-code is INSN_CODE,
3790 with two operands: an output TARGET and an input OP0.
3791 TARGET *must* be nonzero, and the output is always stored there.
3792 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3793 the value that is stored into TARGET. */
3794
3795 void
3796 emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
3797 {
3798 bool ok = maybe_emit_unop_insn (icode, target, op0, code);
3799 gcc_assert (ok);
3800 }
3801 \f
3802 struct no_conflict_data
3803 {
3804 rtx target, first, insn;
3805 bool must_stay;
3806 };
3807
3808 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
3809 the currently examined clobber / store has to stay in the list of
3810 insns that constitute the actual libcall block. */
3811 static void
3812 no_conflict_move_test (rtx dest, const_rtx set, void *p0)
3813 {
3814 struct no_conflict_data *p= (struct no_conflict_data *) p0;
3815
3816 /* If this inns directly contributes to setting the target, it must stay. */
3817 if (reg_overlap_mentioned_p (p->target, dest))
3818 p->must_stay = true;
3819 /* If we haven't committed to keeping any other insns in the list yet,
3820 there is nothing more to check. */
3821 else if (p->insn == p->first)
3822 return;
3823 /* If this insn sets / clobbers a register that feeds one of the insns
3824 already in the list, this insn has to stay too. */
3825 else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
3826 || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
3827 || reg_used_between_p (dest, p->first, p->insn)
3828 /* Likewise if this insn depends on a register set by a previous
3829 insn in the list, or if it sets a result (presumably a hard
3830 register) that is set or clobbered by a previous insn.
3831 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3832 SET_DEST perform the former check on the address, and the latter
3833 check on the MEM. */
3834 || (GET_CODE (set) == SET
3835 && (modified_in_p (SET_SRC (set), p->first)
3836 || modified_in_p (SET_DEST (set), p->first)
3837 || modified_between_p (SET_SRC (set), p->first, p->insn)
3838 || modified_between_p (SET_DEST (set), p->first, p->insn))))
3839 p->must_stay = true;
3840 }
3841
3842 \f
3843 /* Emit code to make a call to a constant function or a library call.
3844
3845 INSNS is a list containing all insns emitted in the call.
3846 These insns leave the result in RESULT. Our block is to copy RESULT
3847 to TARGET, which is logically equivalent to EQUIV.
3848
3849 We first emit any insns that set a pseudo on the assumption that these are
3850 loading constants into registers; doing so allows them to be safely cse'ed
3851 between blocks. Then we emit all the other insns in the block, followed by
3852 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3853 note with an operand of EQUIV. */
3854
3855 static void
3856 emit_libcall_block_1 (rtx insns, rtx target, rtx result, rtx equiv,
3857 bool equiv_may_trap)
3858 {
3859 rtx final_dest = target;
3860 rtx next, last, insn;
3861
3862 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3863 into a MEM later. Protect the libcall block from this change. */
3864 if (! REG_P (target) || REG_USERVAR_P (target))
3865 target = gen_reg_rtx (GET_MODE (target));
3866
3867 /* If we're using non-call exceptions, a libcall corresponding to an
3868 operation that may trap may also trap. */
3869 /* ??? See the comment in front of make_reg_eh_region_note. */
3870 if (cfun->can_throw_non_call_exceptions
3871 && (equiv_may_trap || may_trap_p (equiv)))
3872 {
3873 for (insn = insns; insn; insn = NEXT_INSN (insn))
3874 if (CALL_P (insn))
3875 {
3876 rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
3877 if (note)
3878 {
3879 int lp_nr = INTVAL (XEXP (note, 0));
3880 if (lp_nr == 0 || lp_nr == INT_MIN)
3881 remove_note (insn, note);
3882 }
3883 }
3884 }
3885 else
3886 {
3887 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3888 reg note to indicate that this call cannot throw or execute a nonlocal
3889 goto (unless there is already a REG_EH_REGION note, in which case
3890 we update it). */
3891 for (insn = insns; insn; insn = NEXT_INSN (insn))
3892 if (CALL_P (insn))
3893 make_reg_eh_region_note_nothrow_nononlocal (insn);
3894 }
3895
3896 /* First emit all insns that set pseudos. Remove them from the list as
3897 we go. Avoid insns that set pseudos which were referenced in previous
3898 insns. These can be generated by move_by_pieces, for example,
3899 to update an address. Similarly, avoid insns that reference things
3900 set in previous insns. */
3901
3902 for (insn = insns; insn; insn = next)
3903 {
3904 rtx set = single_set (insn);
3905
3906 next = NEXT_INSN (insn);
3907
3908 if (set != 0 && REG_P (SET_DEST (set))
3909 && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
3910 {
3911 struct no_conflict_data data;
3912
3913 data.target = const0_rtx;
3914 data.first = insns;
3915 data.insn = insn;
3916 data.must_stay = 0;
3917 note_stores (PATTERN (insn), no_conflict_move_test, &data);
3918 if (! data.must_stay)
3919 {
3920 if (PREV_INSN (insn))
3921 NEXT_INSN (PREV_INSN (insn)) = next;
3922 else
3923 insns = next;
3924
3925 if (next)
3926 PREV_INSN (next) = PREV_INSN (insn);
3927
3928 add_insn (insn);
3929 }
3930 }
3931
3932 /* Some ports use a loop to copy large arguments onto the stack.
3933 Don't move anything outside such a loop. */
3934 if (LABEL_P (insn))
3935 break;
3936 }
3937
3938 /* Write the remaining insns followed by the final copy. */
3939 for (insn = insns; insn; insn = next)
3940 {
3941 next = NEXT_INSN (insn);
3942
3943 add_insn (insn);
3944 }
3945
3946 last = emit_move_insn (target, result);
3947 set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
3948
3949 if (final_dest != target)
3950 emit_move_insn (final_dest, target);
3951 }
3952
3953 void
3954 emit_libcall_block (rtx insns, rtx target, rtx result, rtx equiv)
3955 {
3956 emit_libcall_block_1 (insns, target, result, equiv, false);
3957 }
3958 \f
3959 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
3960 PURPOSE describes how this comparison will be used. CODE is the rtx
3961 comparison code we will be using.
3962
3963 ??? Actually, CODE is slightly weaker than that. A target is still
3964 required to implement all of the normal bcc operations, but not
3965 required to implement all (or any) of the unordered bcc operations. */
3966
3967 int
3968 can_compare_p (enum rtx_code code, enum machine_mode mode,
3969 enum can_compare_purpose purpose)
3970 {
3971 rtx test;
3972 test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
3973 do
3974 {
3975 enum insn_code icode;
3976
3977 if (purpose == ccp_jump
3978 && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
3979 && insn_operand_matches (icode, 0, test))
3980 return 1;
3981 if (purpose == ccp_store_flag
3982 && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
3983 && insn_operand_matches (icode, 1, test))
3984 return 1;
3985 if (purpose == ccp_cmov
3986 && optab_handler (cmov_optab, mode) != CODE_FOR_nothing)
3987 return 1;
3988
3989 mode = GET_MODE_WIDER_MODE (mode);
3990 PUT_MODE (test, mode);
3991 }
3992 while (mode != VOIDmode);
3993
3994 return 0;
3995 }
3996
3997 /* This function is called when we are going to emit a compare instruction that
3998 compares the values found in *PX and *PY, using the rtl operator COMPARISON.
3999
4000 *PMODE is the mode of the inputs (in case they are const_int).
4001 *PUNSIGNEDP nonzero says that the operands are unsigned;
4002 this matters if they need to be widened (as given by METHODS).
4003
4004 If they have mode BLKmode, then SIZE specifies the size of both operands.
4005
4006 This function performs all the setup necessary so that the caller only has
4007 to emit a single comparison insn. This setup can involve doing a BLKmode
4008 comparison or emitting a library call to perform the comparison if no insn
4009 is available to handle it.
4010 The values which are passed in through pointers can be modified; the caller
4011 should perform the comparison on the modified values. Constant
4012 comparisons must have already been folded. */
4013
4014 static void
4015 prepare_cmp_insn (rtx x, rtx y, enum rtx_code comparison, rtx size,
4016 int unsignedp, enum optab_methods methods,
4017 rtx *ptest, enum machine_mode *pmode)
4018 {
4019 enum machine_mode mode = *pmode;
4020 rtx libfunc, test;
4021 enum machine_mode cmp_mode;
4022 enum mode_class mclass;
4023
4024 /* The other methods are not needed. */
4025 gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
4026 || methods == OPTAB_LIB_WIDEN);
4027
4028 /* If we are optimizing, force expensive constants into a register. */
4029 if (CONSTANT_P (x) && optimize
4030 && (rtx_cost (x, COMPARE, 0, optimize_insn_for_speed_p ())
4031 > COSTS_N_INSNS (1)))
4032 x = force_reg (mode, x);
4033
4034 if (CONSTANT_P (y) && optimize
4035 && (rtx_cost (y, COMPARE, 1, optimize_insn_for_speed_p ())
4036 > COSTS_N_INSNS (1)))
4037 y = force_reg (mode, y);
4038
4039 #ifdef HAVE_cc0
4040 /* Make sure if we have a canonical comparison. The RTL
4041 documentation states that canonical comparisons are required only
4042 for targets which have cc0. */
4043 gcc_assert (!CONSTANT_P (x) || CONSTANT_P (y));
4044 #endif
4045
4046 /* Don't let both operands fail to indicate the mode. */
4047 if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
4048 x = force_reg (mode, x);
4049 if (mode == VOIDmode)
4050 mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
4051
4052 /* Handle all BLKmode compares. */
4053
4054 if (mode == BLKmode)
4055 {
4056 enum machine_mode result_mode;
4057 enum insn_code cmp_code;
4058 tree length_type;
4059 rtx libfunc;
4060 rtx result;
4061 rtx opalign
4062 = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
4063
4064 gcc_assert (size);
4065
4066 /* Try to use a memory block compare insn - either cmpstr
4067 or cmpmem will do. */
4068 for (cmp_mode = GET_CLASS_NARROWEST_MODE (MODE_INT);
4069 cmp_mode != VOIDmode;
4070 cmp_mode = GET_MODE_WIDER_MODE (cmp_mode))
4071 {
4072 cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
4073 if (cmp_code == CODE_FOR_nothing)
4074 cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
4075 if (cmp_code == CODE_FOR_nothing)
4076 cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
4077 if (cmp_code == CODE_FOR_nothing)
4078 continue;
4079
4080 /* Must make sure the size fits the insn's mode. */
4081 if ((CONST_INT_P (size)
4082 && INTVAL (size) >= (1 << GET_MODE_BITSIZE (cmp_mode)))
4083 || (GET_MODE_BITSIZE (GET_MODE (size))
4084 > GET_MODE_BITSIZE (cmp_mode)))
4085 continue;
4086
4087 result_mode = insn_data[cmp_code].operand[0].mode;
4088 result = gen_reg_rtx (result_mode);
4089 size = convert_to_mode (cmp_mode, size, 1);
4090 emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
4091
4092 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
4093 *pmode = result_mode;
4094 return;
4095 }
4096
4097 if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
4098 goto fail;
4099
4100 /* Otherwise call a library function, memcmp. */
4101 libfunc = memcmp_libfunc;
4102 length_type = sizetype;
4103 result_mode = TYPE_MODE (integer_type_node);
4104 cmp_mode = TYPE_MODE (length_type);
4105 size = convert_to_mode (TYPE_MODE (length_type), size,
4106 TYPE_UNSIGNED (length_type));
4107
4108 result = emit_library_call_value (libfunc, 0, LCT_PURE,
4109 result_mode, 3,
4110 XEXP (x, 0), Pmode,
4111 XEXP (y, 0), Pmode,
4112 size, cmp_mode);
4113
4114 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
4115 *pmode = result_mode;
4116 return;
4117 }
4118
4119 /* Don't allow operands to the compare to trap, as that can put the
4120 compare and branch in different basic blocks. */
4121 if (cfun->can_throw_non_call_exceptions)
4122 {
4123 if (may_trap_p (x))
4124 x = force_reg (mode, x);
4125 if (may_trap_p (y))
4126 y = force_reg (mode, y);
4127 }
4128
4129 if (GET_MODE_CLASS (mode) == MODE_CC)
4130 {
4131 gcc_assert (can_compare_p (comparison, CCmode, ccp_jump));
4132 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4133 return;
4134 }
4135
4136 mclass = GET_MODE_CLASS (mode);
4137 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4138 cmp_mode = mode;
4139 do
4140 {
4141 enum insn_code icode;
4142 icode = optab_handler (cbranch_optab, cmp_mode);
4143 if (icode != CODE_FOR_nothing
4144 && insn_operand_matches (icode, 0, test))
4145 {
4146 rtx last = get_last_insn ();
4147 rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
4148 rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
4149 if (op0 && op1
4150 && insn_operand_matches (icode, 1, op0)
4151 && insn_operand_matches (icode, 2, op1))
4152 {
4153 XEXP (test, 0) = op0;
4154 XEXP (test, 1) = op1;
4155 *ptest = test;
4156 *pmode = cmp_mode;
4157 return;
4158 }
4159 delete_insns_since (last);
4160 }
4161
4162 if (methods == OPTAB_DIRECT || !CLASS_HAS_WIDER_MODES_P (mclass))
4163 break;
4164 cmp_mode = GET_MODE_WIDER_MODE (cmp_mode);
4165 }
4166 while (cmp_mode != VOIDmode);
4167
4168 if (methods != OPTAB_LIB_WIDEN)
4169 goto fail;
4170
4171 if (!SCALAR_FLOAT_MODE_P (mode))
4172 {
4173 rtx result;
4174 enum machine_mode ret_mode;
4175
4176 /* Handle a libcall just for the mode we are using. */
4177 libfunc = optab_libfunc (cmp_optab, mode);
4178 gcc_assert (libfunc);
4179
4180 /* If we want unsigned, and this mode has a distinct unsigned
4181 comparison routine, use that. */
4182 if (unsignedp)
4183 {
4184 rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
4185 if (ulibfunc)
4186 libfunc = ulibfunc;
4187 }
4188
4189 ret_mode = targetm.libgcc_cmp_return_mode ();
4190 result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4191 ret_mode, 2, x, mode, y, mode);
4192
4193 /* There are two kinds of comparison routines. Biased routines
4194 return 0/1/2, and unbiased routines return -1/0/1. Other parts
4195 of gcc expect that the comparison operation is equivalent
4196 to the modified comparison. For signed comparisons compare the
4197 result against 1 in the biased case, and zero in the unbiased
4198 case. For unsigned comparisons always compare against 1 after
4199 biasing the unbiased result by adding 1. This gives us a way to
4200 represent LTU.
4201 The comparisons in the fixed-point helper library are always
4202 biased. */
4203 x = result;
4204 y = const1_rtx;
4205
4206 if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
4207 {
4208 if (unsignedp)
4209 x = plus_constant (ret_mode, result, 1);
4210 else
4211 y = const0_rtx;
4212 }
4213
4214 *pmode = word_mode;
4215 prepare_cmp_insn (x, y, comparison, NULL_RTX, unsignedp, methods,
4216 ptest, pmode);
4217 }
4218 else
4219 prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
4220
4221 return;
4222
4223 fail:
4224 *ptest = NULL_RTX;
4225 }
4226
4227 /* Before emitting an insn with code ICODE, make sure that X, which is going
4228 to be used for operand OPNUM of the insn, is converted from mode MODE to
4229 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4230 that it is accepted by the operand predicate. Return the new value. */
4231
4232 rtx
4233 prepare_operand (enum insn_code icode, rtx x, int opnum, enum machine_mode mode,
4234 enum machine_mode wider_mode, int unsignedp)
4235 {
4236 if (mode != wider_mode)
4237 x = convert_modes (wider_mode, mode, x, unsignedp);
4238
4239 if (!insn_operand_matches (icode, opnum, x))
4240 {
4241 if (reload_completed)
4242 return NULL_RTX;
4243 x = copy_to_mode_reg (insn_data[(int) icode].operand[opnum].mode, x);
4244 }
4245
4246 return x;
4247 }
4248
4249 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4250 we can do the branch. */
4251
4252 static void
4253 emit_cmp_and_jump_insn_1 (rtx test, enum machine_mode mode, rtx label)
4254 {
4255 enum machine_mode optab_mode;
4256 enum mode_class mclass;
4257 enum insn_code icode;
4258
4259 mclass = GET_MODE_CLASS (mode);
4260 optab_mode = (mclass == MODE_CC) ? CCmode : mode;
4261 icode = optab_handler (cbranch_optab, optab_mode);
4262
4263 gcc_assert (icode != CODE_FOR_nothing);
4264 gcc_assert (insn_operand_matches (icode, 0, test));
4265 emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0), XEXP (test, 1), label));
4266 }
4267
4268 /* Generate code to compare X with Y so that the condition codes are
4269 set and to jump to LABEL if the condition is true. If X is a
4270 constant and Y is not a constant, then the comparison is swapped to
4271 ensure that the comparison RTL has the canonical form.
4272
4273 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4274 need to be widened. UNSIGNEDP is also used to select the proper
4275 branch condition code.
4276
4277 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4278
4279 MODE is the mode of the inputs (in case they are const_int).
4280
4281 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4282 It will be potentially converted into an unsigned variant based on
4283 UNSIGNEDP to select a proper jump instruction. */
4284
4285 void
4286 emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4287 enum machine_mode mode, int unsignedp, rtx label)
4288 {
4289 rtx op0 = x, op1 = y;
4290 rtx test;
4291
4292 /* Swap operands and condition to ensure canonical RTL. */
4293 if (swap_commutative_operands_p (x, y)
4294 && can_compare_p (swap_condition (comparison), mode, ccp_jump))
4295 {
4296 op0 = y, op1 = x;
4297 comparison = swap_condition (comparison);
4298 }
4299
4300 /* If OP0 is still a constant, then both X and Y must be constants
4301 or the opposite comparison is not supported. Force X into a register
4302 to create canonical RTL. */
4303 if (CONSTANT_P (op0))
4304 op0 = force_reg (mode, op0);
4305
4306 if (unsignedp)
4307 comparison = unsigned_condition (comparison);
4308
4309 prepare_cmp_insn (op0, op1, comparison, size, unsignedp, OPTAB_LIB_WIDEN,
4310 &test, &mode);
4311 emit_cmp_and_jump_insn_1 (test, mode, label);
4312 }
4313
4314 \f
4315 /* Emit a library call comparison between floating point X and Y.
4316 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4317
4318 static void
4319 prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
4320 rtx *ptest, enum machine_mode *pmode)
4321 {
4322 enum rtx_code swapped = swap_condition (comparison);
4323 enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
4324 enum machine_mode orig_mode = GET_MODE (x);
4325 enum machine_mode mode, cmp_mode;
4326 rtx true_rtx, false_rtx;
4327 rtx value, target, insns, equiv;
4328 rtx libfunc = 0;
4329 bool reversed_p = false;
4330 cmp_mode = targetm.libgcc_cmp_return_mode ();
4331
4332 for (mode = orig_mode;
4333 mode != VOIDmode;
4334 mode = GET_MODE_WIDER_MODE (mode))
4335 {
4336 if (code_to_optab (comparison)
4337 && (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
4338 break;
4339
4340 if (code_to_optab (swapped)
4341 && (libfunc = optab_libfunc (code_to_optab (swapped), mode)))
4342 {
4343 rtx tmp;
4344 tmp = x; x = y; y = tmp;
4345 comparison = swapped;
4346 break;
4347 }
4348
4349 if (code_to_optab (reversed)
4350 && (libfunc = optab_libfunc (code_to_optab (reversed), mode)))
4351 {
4352 comparison = reversed;
4353 reversed_p = true;
4354 break;
4355 }
4356 }
4357
4358 gcc_assert (mode != VOIDmode);
4359
4360 if (mode != orig_mode)
4361 {
4362 x = convert_to_mode (mode, x, 0);
4363 y = convert_to_mode (mode, y, 0);
4364 }
4365
4366 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4367 the RTL. The allows the RTL optimizers to delete the libcall if the
4368 condition can be determined at compile-time. */
4369 if (comparison == UNORDERED
4370 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4371 {
4372 true_rtx = const_true_rtx;
4373 false_rtx = const0_rtx;
4374 }
4375 else
4376 {
4377 switch (comparison)
4378 {
4379 case EQ:
4380 true_rtx = const0_rtx;
4381 false_rtx = const_true_rtx;
4382 break;
4383
4384 case NE:
4385 true_rtx = const_true_rtx;
4386 false_rtx = const0_rtx;
4387 break;
4388
4389 case GT:
4390 true_rtx = const1_rtx;
4391 false_rtx = const0_rtx;
4392 break;
4393
4394 case GE:
4395 true_rtx = const0_rtx;
4396 false_rtx = constm1_rtx;
4397 break;
4398
4399 case LT:
4400 true_rtx = constm1_rtx;
4401 false_rtx = const0_rtx;
4402 break;
4403
4404 case LE:
4405 true_rtx = const0_rtx;
4406 false_rtx = const1_rtx;
4407 break;
4408
4409 default:
4410 gcc_unreachable ();
4411 }
4412 }
4413
4414 if (comparison == UNORDERED)
4415 {
4416 rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x);
4417 equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y);
4418 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4419 temp, const_true_rtx, equiv);
4420 }
4421 else
4422 {
4423 equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y);
4424 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4425 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4426 equiv, true_rtx, false_rtx);
4427 }
4428
4429 start_sequence ();
4430 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4431 cmp_mode, 2, x, mode, y, mode);
4432 insns = get_insns ();
4433 end_sequence ();
4434
4435 target = gen_reg_rtx (cmp_mode);
4436 emit_libcall_block (insns, target, value, equiv);
4437
4438 if (comparison == UNORDERED
4439 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
4440 || reversed_p)
4441 *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
4442 else
4443 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
4444
4445 *pmode = cmp_mode;
4446 }
4447 \f
4448 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4449
4450 void
4451 emit_indirect_jump (rtx loc)
4452 {
4453 struct expand_operand ops[1];
4454
4455 create_address_operand (&ops[0], loc);
4456 expand_jump_insn (CODE_FOR_indirect_jump, 1, ops);
4457 emit_barrier ();
4458 }
4459 \f
4460 #ifdef HAVE_conditional_move
4461
4462 /* Emit a conditional move instruction if the machine supports one for that
4463 condition and machine mode.
4464
4465 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4466 the mode to use should they be constants. If it is VOIDmode, they cannot
4467 both be constants.
4468
4469 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4470 should be stored there. MODE is the mode to use should they be constants.
4471 If it is VOIDmode, they cannot both be constants.
4472
4473 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4474 is not supported. */
4475
4476 rtx
4477 emit_conditional_move (rtx target, enum rtx_code code, rtx op0, rtx op1,
4478 enum machine_mode cmode, rtx op2, rtx op3,
4479 enum machine_mode mode, int unsignedp)
4480 {
4481 rtx tem, comparison, last;
4482 enum insn_code icode;
4483 enum rtx_code reversed;
4484
4485 /* If one operand is constant, make it the second one. Only do this
4486 if the other operand is not constant as well. */
4487
4488 if (swap_commutative_operands_p (op0, op1))
4489 {
4490 tem = op0;
4491 op0 = op1;
4492 op1 = tem;
4493 code = swap_condition (code);
4494 }
4495
4496 /* get_condition will prefer to generate LT and GT even if the old
4497 comparison was against zero, so undo that canonicalization here since
4498 comparisons against zero are cheaper. */
4499 if (code == LT && op1 == const1_rtx)
4500 code = LE, op1 = const0_rtx;
4501 else if (code == GT && op1 == constm1_rtx)
4502 code = GE, op1 = const0_rtx;
4503
4504 if (cmode == VOIDmode)
4505 cmode = GET_MODE (op0);
4506
4507 if (swap_commutative_operands_p (op2, op3)
4508 && ((reversed = reversed_comparison_code_parts (code, op0, op1, NULL))
4509 != UNKNOWN))
4510 {
4511 tem = op2;
4512 op2 = op3;
4513 op3 = tem;
4514 code = reversed;
4515 }
4516
4517 if (mode == VOIDmode)
4518 mode = GET_MODE (op2);
4519
4520 icode = direct_optab_handler (movcc_optab, mode);
4521
4522 if (icode == CODE_FOR_nothing)
4523 return 0;
4524
4525 if (!target)
4526 target = gen_reg_rtx (mode);
4527
4528 code = unsignedp ? unsigned_condition (code) : code;
4529 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4530
4531 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4532 return NULL and let the caller figure out how best to deal with this
4533 situation. */
4534 if (!COMPARISON_P (comparison))
4535 return NULL_RTX;
4536
4537 do_pending_stack_adjust ();
4538 last = get_last_insn ();
4539 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4540 GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
4541 &comparison, &cmode);
4542 if (comparison)
4543 {
4544 struct expand_operand ops[4];
4545
4546 create_output_operand (&ops[0], target, mode);
4547 create_fixed_operand (&ops[1], comparison);
4548 create_input_operand (&ops[2], op2, mode);
4549 create_input_operand (&ops[3], op3, mode);
4550 if (maybe_expand_insn (icode, 4, ops))
4551 {
4552 if (ops[0].value != target)
4553 convert_move (target, ops[0].value, false);
4554 return target;
4555 }
4556 }
4557 delete_insns_since (last);
4558 return NULL_RTX;
4559 }
4560
4561 /* Return nonzero if a conditional move of mode MODE is supported.
4562
4563 This function is for combine so it can tell whether an insn that looks
4564 like a conditional move is actually supported by the hardware. If we
4565 guess wrong we lose a bit on optimization, but that's it. */
4566 /* ??? sparc64 supports conditionally moving integers values based on fp
4567 comparisons, and vice versa. How do we handle them? */
4568
4569 int
4570 can_conditionally_move_p (enum machine_mode mode)
4571 {
4572 if (direct_optab_handler (movcc_optab, mode) != CODE_FOR_nothing)
4573 return 1;
4574
4575 return 0;
4576 }
4577
4578 #endif /* HAVE_conditional_move */
4579
4580 /* Emit a conditional addition instruction if the machine supports one for that
4581 condition and machine mode.
4582
4583 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4584 the mode to use should they be constants. If it is VOIDmode, they cannot
4585 both be constants.
4586
4587 OP2 should be stored in TARGET if the comparison is true, otherwise OP2+OP3
4588 should be stored there. MODE is the mode to use should they be constants.
4589 If it is VOIDmode, they cannot both be constants.
4590
4591 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4592 is not supported. */
4593
4594 rtx
4595 emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
4596 enum machine_mode cmode, rtx op2, rtx op3,
4597 enum machine_mode mode, int unsignedp)
4598 {
4599 rtx tem, comparison, last;
4600 enum insn_code icode;
4601 enum rtx_code reversed;
4602
4603 /* If one operand is constant, make it the second one. Only do this
4604 if the other operand is not constant as well. */
4605
4606 if (swap_commutative_operands_p (op0, op1))
4607 {
4608 tem = op0;
4609 op0 = op1;
4610 op1 = tem;
4611 code = swap_condition (code);
4612 }
4613
4614 /* get_condition will prefer to generate LT and GT even if the old
4615 comparison was against zero, so undo that canonicalization here since
4616 comparisons against zero are cheaper. */
4617 if (code == LT && op1 == const1_rtx)
4618 code = LE, op1 = const0_rtx;
4619 else if (code == GT && op1 == constm1_rtx)
4620 code = GE, op1 = const0_rtx;
4621
4622 if (cmode == VOIDmode)
4623 cmode = GET_MODE (op0);
4624
4625 if (swap_commutative_operands_p (op2, op3)
4626 && ((reversed = reversed_comparison_code_parts (code, op0, op1, NULL))
4627 != UNKNOWN))
4628 {
4629 tem = op2;
4630 op2 = op3;
4631 op3 = tem;
4632 code = reversed;
4633 }
4634
4635 if (mode == VOIDmode)
4636 mode = GET_MODE (op2);
4637
4638 icode = optab_handler (addcc_optab, mode);
4639
4640 if (icode == CODE_FOR_nothing)
4641 return 0;
4642
4643 if (!target)
4644 target = gen_reg_rtx (mode);
4645
4646 code = unsignedp ? unsigned_condition (code) : code;
4647 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4648
4649 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4650 return NULL and let the caller figure out how best to deal with this
4651 situation. */
4652 if (!COMPARISON_P (comparison))
4653 return NULL_RTX;
4654
4655 do_pending_stack_adjust ();
4656 last = get_last_insn ();
4657 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4658 GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
4659 &comparison, &cmode);
4660 if (comparison)
4661 {
4662 struct expand_operand ops[4];
4663
4664 create_output_operand (&ops[0], target, mode);
4665 create_fixed_operand (&ops[1], comparison);
4666 create_input_operand (&ops[2], op2, mode);
4667 create_input_operand (&ops[3], op3, mode);
4668 if (maybe_expand_insn (icode, 4, ops))
4669 {
4670 if (ops[0].value != target)
4671 convert_move (target, ops[0].value, false);
4672 return target;
4673 }
4674 }
4675 delete_insns_since (last);
4676 return NULL_RTX;
4677 }
4678 \f
4679 /* These functions attempt to generate an insn body, rather than
4680 emitting the insn, but if the gen function already emits them, we
4681 make no attempt to turn them back into naked patterns. */
4682
4683 /* Generate and return an insn body to add Y to X. */
4684
4685 rtx
4686 gen_add2_insn (rtx x, rtx y)
4687 {
4688 enum insn_code icode = optab_handler (add_optab, GET_MODE (x));
4689
4690 gcc_assert (insn_operand_matches (icode, 0, x));
4691 gcc_assert (insn_operand_matches (icode, 1, x));
4692 gcc_assert (insn_operand_matches (icode, 2, y));
4693
4694 return GEN_FCN (icode) (x, x, y);
4695 }
4696
4697 /* Generate and return an insn body to add r1 and c,
4698 storing the result in r0. */
4699
4700 rtx
4701 gen_add3_insn (rtx r0, rtx r1, rtx c)
4702 {
4703 enum insn_code icode = optab_handler (add_optab, GET_MODE (r0));
4704
4705 if (icode == CODE_FOR_nothing
4706 || !insn_operand_matches (icode, 0, r0)
4707 || !insn_operand_matches (icode, 1, r1)
4708 || !insn_operand_matches (icode, 2, c))
4709 return NULL_RTX;
4710
4711 return GEN_FCN (icode) (r0, r1, c);
4712 }
4713
4714 int
4715 have_add2_insn (rtx x, rtx y)
4716 {
4717 enum insn_code icode;
4718
4719 gcc_assert (GET_MODE (x) != VOIDmode);
4720
4721 icode = optab_handler (add_optab, GET_MODE (x));
4722
4723 if (icode == CODE_FOR_nothing)
4724 return 0;
4725
4726 if (!insn_operand_matches (icode, 0, x)
4727 || !insn_operand_matches (icode, 1, x)
4728 || !insn_operand_matches (icode, 2, y))
4729 return 0;
4730
4731 return 1;
4732 }
4733
4734 /* Generate and return an insn body to subtract Y from X. */
4735
4736 rtx
4737 gen_sub2_insn (rtx x, rtx y)
4738 {
4739 enum insn_code icode = optab_handler (sub_optab, GET_MODE (x));
4740
4741 gcc_assert (insn_operand_matches (icode, 0, x));
4742 gcc_assert (insn_operand_matches (icode, 1, x));
4743 gcc_assert (insn_operand_matches (icode, 2, y));
4744
4745 return GEN_FCN (icode) (x, x, y);
4746 }
4747
4748 /* Generate and return an insn body to subtract r1 and c,
4749 storing the result in r0. */
4750
4751 rtx
4752 gen_sub3_insn (rtx r0, rtx r1, rtx c)
4753 {
4754 enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0));
4755
4756 if (icode == CODE_FOR_nothing
4757 || !insn_operand_matches (icode, 0, r0)
4758 || !insn_operand_matches (icode, 1, r1)
4759 || !insn_operand_matches (icode, 2, c))
4760 return NULL_RTX;
4761
4762 return GEN_FCN (icode) (r0, r1, c);
4763 }
4764
4765 int
4766 have_sub2_insn (rtx x, rtx y)
4767 {
4768 enum insn_code icode;
4769
4770 gcc_assert (GET_MODE (x) != VOIDmode);
4771
4772 icode = optab_handler (sub_optab, GET_MODE (x));
4773
4774 if (icode == CODE_FOR_nothing)
4775 return 0;
4776
4777 if (!insn_operand_matches (icode, 0, x)
4778 || !insn_operand_matches (icode, 1, x)
4779 || !insn_operand_matches (icode, 2, y))
4780 return 0;
4781
4782 return 1;
4783 }
4784
4785 /* Generate the body of an instruction to copy Y into X.
4786 It may be a list of insns, if one insn isn't enough. */
4787
4788 rtx
4789 gen_move_insn (rtx x, rtx y)
4790 {
4791 rtx seq;
4792
4793 start_sequence ();
4794 emit_move_insn_1 (x, y);
4795 seq = get_insns ();
4796 end_sequence ();
4797 return seq;
4798 }
4799 \f
4800 /* Return the insn code used to extend FROM_MODE to TO_MODE.
4801 UNSIGNEDP specifies zero-extension instead of sign-extension. If
4802 no such operation exists, CODE_FOR_nothing will be returned. */
4803
4804 enum insn_code
4805 can_extend_p (enum machine_mode to_mode, enum machine_mode from_mode,
4806 int unsignedp)
4807 {
4808 convert_optab tab;
4809 #ifdef HAVE_ptr_extend
4810 if (unsignedp < 0)
4811 return CODE_FOR_ptr_extend;
4812 #endif
4813
4814 tab = unsignedp ? zext_optab : sext_optab;
4815 return convert_optab_handler (tab, to_mode, from_mode);
4816 }
4817
4818 /* Generate the body of an insn to extend Y (with mode MFROM)
4819 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
4820
4821 rtx
4822 gen_extend_insn (rtx x, rtx y, enum machine_mode mto,
4823 enum machine_mode mfrom, int unsignedp)
4824 {
4825 enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
4826 return GEN_FCN (icode) (x, y);
4827 }
4828 \f
4829 /* can_fix_p and can_float_p say whether the target machine
4830 can directly convert a given fixed point type to
4831 a given floating point type, or vice versa.
4832 The returned value is the CODE_FOR_... value to use,
4833 or CODE_FOR_nothing if these modes cannot be directly converted.
4834
4835 *TRUNCP_PTR is set to 1 if it is necessary to output
4836 an explicit FTRUNC insn before the fix insn; otherwise 0. */
4837
4838 static enum insn_code
4839 can_fix_p (enum machine_mode fixmode, enum machine_mode fltmode,
4840 int unsignedp, int *truncp_ptr)
4841 {
4842 convert_optab tab;
4843 enum insn_code icode;
4844
4845 tab = unsignedp ? ufixtrunc_optab : sfixtrunc_optab;
4846 icode = convert_optab_handler (tab, fixmode, fltmode);
4847 if (icode != CODE_FOR_nothing)
4848 {
4849 *truncp_ptr = 0;
4850 return icode;
4851 }
4852
4853 /* FIXME: This requires a port to define both FIX and FTRUNC pattern
4854 for this to work. We need to rework the fix* and ftrunc* patterns
4855 and documentation. */
4856 tab = unsignedp ? ufix_optab : sfix_optab;
4857 icode = convert_optab_handler (tab, fixmode, fltmode);
4858 if (icode != CODE_FOR_nothing
4859 && optab_handler (ftrunc_optab, fltmode) != CODE_FOR_nothing)
4860 {
4861 *truncp_ptr = 1;
4862 return icode;
4863 }
4864
4865 *truncp_ptr = 0;
4866 return CODE_FOR_nothing;
4867 }
4868
4869 enum insn_code
4870 can_float_p (enum machine_mode fltmode, enum machine_mode fixmode,
4871 int unsignedp)
4872 {
4873 convert_optab tab;
4874
4875 tab = unsignedp ? ufloat_optab : sfloat_optab;
4876 return convert_optab_handler (tab, fltmode, fixmode);
4877 }
4878
4879 /* Function supportable_convert_operation
4880
4881 Check whether an operation represented by the code CODE is a
4882 convert operation that is supported by the target platform in
4883 vector form (i.e., when operating on arguments of type VECTYPE_IN
4884 producing a result of type VECTYPE_OUT).
4885
4886 Convert operations we currently support directly are FIX_TRUNC and FLOAT.
4887 This function checks if these operations are supported
4888 by the target platform either directly (via vector tree-codes), or via
4889 target builtins.
4890
4891 Output:
4892 - CODE1 is code of vector operation to be used when
4893 vectorizing the operation, if available.
4894 - DECL is decl of target builtin functions to be used
4895 when vectorizing the operation, if available. In this case,
4896 CODE1 is CALL_EXPR. */
4897
4898 bool
4899 supportable_convert_operation (enum tree_code code,
4900 tree vectype_out, tree vectype_in,
4901 tree *decl, enum tree_code *code1)
4902 {
4903 enum machine_mode m1,m2;
4904 int truncp;
4905
4906 m1 = TYPE_MODE (vectype_out);
4907 m2 = TYPE_MODE (vectype_in);
4908
4909 /* First check if we can done conversion directly. */
4910 if ((code == FIX_TRUNC_EXPR
4911 && can_fix_p (m1,m2,TYPE_UNSIGNED (vectype_out), &truncp)
4912 != CODE_FOR_nothing)
4913 || (code == FLOAT_EXPR
4914 && can_float_p (m1,m2,TYPE_UNSIGNED (vectype_in))
4915 != CODE_FOR_nothing))
4916 {
4917 *code1 = code;
4918 return true;
4919 }
4920
4921 /* Now check for builtin. */
4922 if (targetm.vectorize.builtin_conversion
4923 && targetm.vectorize.builtin_conversion (code, vectype_out, vectype_in))
4924 {
4925 *code1 = CALL_EXPR;
4926 *decl = targetm.vectorize.builtin_conversion (code, vectype_out, vectype_in);
4927 return true;
4928 }
4929 return false;
4930 }
4931
4932 \f
4933 /* Generate code to convert FROM to floating point
4934 and store in TO. FROM must be fixed point and not VOIDmode.
4935 UNSIGNEDP nonzero means regard FROM as unsigned.
4936 Normally this is done by correcting the final value
4937 if it is negative. */
4938
4939 void
4940 expand_float (rtx to, rtx from, int unsignedp)
4941 {
4942 enum insn_code icode;
4943 rtx target = to;
4944 enum machine_mode fmode, imode;
4945 bool can_do_signed = false;
4946
4947 /* Crash now, because we won't be able to decide which mode to use. */
4948 gcc_assert (GET_MODE (from) != VOIDmode);
4949
4950 /* Look for an insn to do the conversion. Do it in the specified
4951 modes if possible; otherwise convert either input, output or both to
4952 wider mode. If the integer mode is wider than the mode of FROM,
4953 we can do the conversion signed even if the input is unsigned. */
4954
4955 for (fmode = GET_MODE (to); fmode != VOIDmode;
4956 fmode = GET_MODE_WIDER_MODE (fmode))
4957 for (imode = GET_MODE (from); imode != VOIDmode;
4958 imode = GET_MODE_WIDER_MODE (imode))
4959 {
4960 int doing_unsigned = unsignedp;
4961
4962 if (fmode != GET_MODE (to)
4963 && significand_size (fmode) < GET_MODE_PRECISION (GET_MODE (from)))
4964 continue;
4965
4966 icode = can_float_p (fmode, imode, unsignedp);
4967 if (icode == CODE_FOR_nothing && unsignedp)
4968 {
4969 enum insn_code scode = can_float_p (fmode, imode, 0);
4970 if (scode != CODE_FOR_nothing)
4971 can_do_signed = true;
4972 if (imode != GET_MODE (from))
4973 icode = scode, doing_unsigned = 0;
4974 }
4975
4976 if (icode != CODE_FOR_nothing)
4977 {
4978 if (imode != GET_MODE (from))
4979 from = convert_to_mode (imode, from, unsignedp);
4980
4981 if (fmode != GET_MODE (to))
4982 target = gen_reg_rtx (fmode);
4983
4984 emit_unop_insn (icode, target, from,
4985 doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
4986
4987 if (target != to)
4988 convert_move (to, target, 0);
4989 return;
4990 }
4991 }
4992
4993 /* Unsigned integer, and no way to convert directly. Convert as signed,
4994 then unconditionally adjust the result. */
4995 if (unsignedp && can_do_signed)
4996 {
4997 rtx label = gen_label_rtx ();
4998 rtx temp;
4999 REAL_VALUE_TYPE offset;
5000
5001 /* Look for a usable floating mode FMODE wider than the source and at
5002 least as wide as the target. Using FMODE will avoid rounding woes
5003 with unsigned values greater than the signed maximum value. */
5004
5005 for (fmode = GET_MODE (to); fmode != VOIDmode;
5006 fmode = GET_MODE_WIDER_MODE (fmode))
5007 if (GET_MODE_PRECISION (GET_MODE (from)) < GET_MODE_BITSIZE (fmode)
5008 && can_float_p (fmode, GET_MODE (from), 0) != CODE_FOR_nothing)
5009 break;
5010
5011 if (fmode == VOIDmode)
5012 {
5013 /* There is no such mode. Pretend the target is wide enough. */
5014 fmode = GET_MODE (to);
5015
5016 /* Avoid double-rounding when TO is narrower than FROM. */
5017 if ((significand_size (fmode) + 1)
5018 < GET_MODE_PRECISION (GET_MODE (from)))
5019 {
5020 rtx temp1;
5021 rtx neglabel = gen_label_rtx ();
5022
5023 /* Don't use TARGET if it isn't a register, is a hard register,
5024 or is the wrong mode. */
5025 if (!REG_P (target)
5026 || REGNO (target) < FIRST_PSEUDO_REGISTER
5027 || GET_MODE (target) != fmode)
5028 target = gen_reg_rtx (fmode);
5029
5030 imode = GET_MODE (from);
5031 do_pending_stack_adjust ();
5032
5033 /* Test whether the sign bit is set. */
5034 emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode,
5035 0, neglabel);
5036
5037 /* The sign bit is not set. Convert as signed. */
5038 expand_float (target, from, 0);
5039 emit_jump_insn (gen_jump (label));
5040 emit_barrier ();
5041
5042 /* The sign bit is set.
5043 Convert to a usable (positive signed) value by shifting right
5044 one bit, while remembering if a nonzero bit was shifted
5045 out; i.e., compute (from & 1) | (from >> 1). */
5046
5047 emit_label (neglabel);
5048 temp = expand_binop (imode, and_optab, from, const1_rtx,
5049 NULL_RTX, 1, OPTAB_LIB_WIDEN);
5050 temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
5051 temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1,
5052 OPTAB_LIB_WIDEN);
5053 expand_float (target, temp, 0);
5054
5055 /* Multiply by 2 to undo the shift above. */
5056 temp = expand_binop (fmode, add_optab, target, target,
5057 target, 0, OPTAB_LIB_WIDEN);
5058 if (temp != target)
5059 emit_move_insn (target, temp);
5060
5061 do_pending_stack_adjust ();
5062 emit_label (label);
5063 goto done;
5064 }
5065 }
5066
5067 /* If we are about to do some arithmetic to correct for an
5068 unsigned operand, do it in a pseudo-register. */
5069
5070 if (GET_MODE (to) != fmode
5071 || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
5072 target = gen_reg_rtx (fmode);
5073
5074 /* Convert as signed integer to floating. */
5075 expand_float (target, from, 0);
5076
5077 /* If FROM is negative (and therefore TO is negative),
5078 correct its value by 2**bitwidth. */
5079
5080 do_pending_stack_adjust ();
5081 emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, GET_MODE (from),
5082 0, label);
5083
5084
5085 real_2expN (&offset, GET_MODE_PRECISION (GET_MODE (from)), fmode);
5086 temp = expand_binop (fmode, add_optab, target,
5087 CONST_DOUBLE_FROM_REAL_VALUE (offset, fmode),
5088 target, 0, OPTAB_LIB_WIDEN);
5089 if (temp != target)
5090 emit_move_insn (target, temp);
5091
5092 do_pending_stack_adjust ();
5093 emit_label (label);
5094 goto done;
5095 }
5096
5097 /* No hardware instruction available; call a library routine. */
5098 {
5099 rtx libfunc;
5100 rtx insns;
5101 rtx value;
5102 convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
5103
5104 if (GET_MODE_SIZE (GET_MODE (from)) < GET_MODE_SIZE (SImode))
5105 from = convert_to_mode (SImode, from, unsignedp);
5106
5107 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5108 gcc_assert (libfunc);
5109
5110 start_sequence ();
5111
5112 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5113 GET_MODE (to), 1, from,
5114 GET_MODE (from));
5115 insns = get_insns ();
5116 end_sequence ();
5117
5118 emit_libcall_block (insns, target, value,
5119 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
5120 GET_MODE (to), from));
5121 }
5122
5123 done:
5124
5125 /* Copy result to requested destination
5126 if we have been computing in a temp location. */
5127
5128 if (target != to)
5129 {
5130 if (GET_MODE (target) == GET_MODE (to))
5131 emit_move_insn (to, target);
5132 else
5133 convert_move (to, target, 0);
5134 }
5135 }
5136 \f
5137 /* Generate code to convert FROM to fixed point and store in TO. FROM
5138 must be floating point. */
5139
5140 void
5141 expand_fix (rtx to, rtx from, int unsignedp)
5142 {
5143 enum insn_code icode;
5144 rtx target = to;
5145 enum machine_mode fmode, imode;
5146 int must_trunc = 0;
5147
5148 /* We first try to find a pair of modes, one real and one integer, at
5149 least as wide as FROM and TO, respectively, in which we can open-code
5150 this conversion. If the integer mode is wider than the mode of TO,
5151 we can do the conversion either signed or unsigned. */
5152
5153 for (fmode = GET_MODE (from); fmode != VOIDmode;
5154 fmode = GET_MODE_WIDER_MODE (fmode))
5155 for (imode = GET_MODE (to); imode != VOIDmode;
5156 imode = GET_MODE_WIDER_MODE (imode))
5157 {
5158 int doing_unsigned = unsignedp;
5159
5160 icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
5161 if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
5162 icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
5163
5164 if (icode != CODE_FOR_nothing)
5165 {
5166 rtx last = get_last_insn ();
5167 if (fmode != GET_MODE (from))
5168 from = convert_to_mode (fmode, from, 0);
5169
5170 if (must_trunc)
5171 {
5172 rtx temp = gen_reg_rtx (GET_MODE (from));
5173 from = expand_unop (GET_MODE (from), ftrunc_optab, from,
5174 temp, 0);
5175 }
5176
5177 if (imode != GET_MODE (to))
5178 target = gen_reg_rtx (imode);
5179
5180 if (maybe_emit_unop_insn (icode, target, from,
5181 doing_unsigned ? UNSIGNED_FIX : FIX))
5182 {
5183 if (target != to)
5184 convert_move (to, target, unsignedp);
5185 return;
5186 }
5187 delete_insns_since (last);
5188 }
5189 }
5190
5191 /* For an unsigned conversion, there is one more way to do it.
5192 If we have a signed conversion, we generate code that compares
5193 the real value to the largest representable positive number. If if
5194 is smaller, the conversion is done normally. Otherwise, subtract
5195 one plus the highest signed number, convert, and add it back.
5196
5197 We only need to check all real modes, since we know we didn't find
5198 anything with a wider integer mode.
5199
5200 This code used to extend FP value into mode wider than the destination.
5201 This is needed for decimal float modes which cannot accurately
5202 represent one plus the highest signed number of the same size, but
5203 not for binary modes. Consider, for instance conversion from SFmode
5204 into DImode.
5205
5206 The hot path through the code is dealing with inputs smaller than 2^63
5207 and doing just the conversion, so there is no bits to lose.
5208
5209 In the other path we know the value is positive in the range 2^63..2^64-1
5210 inclusive. (as for other input overflow happens and result is undefined)
5211 So we know that the most important bit set in mantissa corresponds to
5212 2^63. The subtraction of 2^63 should not generate any rounding as it
5213 simply clears out that bit. The rest is trivial. */
5214
5215 if (unsignedp && GET_MODE_PRECISION (GET_MODE (to)) <= HOST_BITS_PER_WIDE_INT)
5216 for (fmode = GET_MODE (from); fmode != VOIDmode;
5217 fmode = GET_MODE_WIDER_MODE (fmode))
5218 if (CODE_FOR_nothing != can_fix_p (GET_MODE (to), fmode, 0, &must_trunc)
5219 && (!DECIMAL_FLOAT_MODE_P (fmode)
5220 || GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (GET_MODE (to))))
5221 {
5222 int bitsize;
5223 REAL_VALUE_TYPE offset;
5224 rtx limit, lab1, lab2, insn;
5225
5226 bitsize = GET_MODE_PRECISION (GET_MODE (to));
5227 real_2expN (&offset, bitsize - 1, fmode);
5228 limit = CONST_DOUBLE_FROM_REAL_VALUE (offset, fmode);
5229 lab1 = gen_label_rtx ();
5230 lab2 = gen_label_rtx ();
5231
5232 if (fmode != GET_MODE (from))
5233 from = convert_to_mode (fmode, from, 0);
5234
5235 /* See if we need to do the subtraction. */
5236 do_pending_stack_adjust ();
5237 emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX, GET_MODE (from),
5238 0, lab1);
5239
5240 /* If not, do the signed "fix" and branch around fixup code. */
5241 expand_fix (to, from, 0);
5242 emit_jump_insn (gen_jump (lab2));
5243 emit_barrier ();
5244
5245 /* Otherwise, subtract 2**(N-1), convert to signed number,
5246 then add 2**(N-1). Do the addition using XOR since this
5247 will often generate better code. */
5248 emit_label (lab1);
5249 target = expand_binop (GET_MODE (from), sub_optab, from, limit,
5250 NULL_RTX, 0, OPTAB_LIB_WIDEN);
5251 expand_fix (to, target, 0);
5252 target = expand_binop (GET_MODE (to), xor_optab, to,
5253 gen_int_mode
5254 ((HOST_WIDE_INT) 1 << (bitsize - 1),
5255 GET_MODE (to)),
5256 to, 1, OPTAB_LIB_WIDEN);
5257
5258 if (target != to)
5259 emit_move_insn (to, target);
5260
5261 emit_label (lab2);
5262
5263 if (optab_handler (mov_optab, GET_MODE (to)) != CODE_FOR_nothing)
5264 {
5265 /* Make a place for a REG_NOTE and add it. */
5266 insn = emit_move_insn (to, to);
5267 set_dst_reg_note (insn, REG_EQUAL,
5268 gen_rtx_fmt_e (UNSIGNED_FIX, GET_MODE (to),
5269 copy_rtx (from)),
5270 to);
5271 }
5272
5273 return;
5274 }
5275
5276 /* We can't do it with an insn, so use a library call. But first ensure
5277 that the mode of TO is at least as wide as SImode, since those are the
5278 only library calls we know about. */
5279
5280 if (GET_MODE_SIZE (GET_MODE (to)) < GET_MODE_SIZE (SImode))
5281 {
5282 target = gen_reg_rtx (SImode);
5283
5284 expand_fix (target, from, unsignedp);
5285 }
5286 else
5287 {
5288 rtx insns;
5289 rtx value;
5290 rtx libfunc;
5291
5292 convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
5293 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5294 gcc_assert (libfunc);
5295
5296 start_sequence ();
5297
5298 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5299 GET_MODE (to), 1, from,
5300 GET_MODE (from));
5301 insns = get_insns ();
5302 end_sequence ();
5303
5304 emit_libcall_block (insns, target, value,
5305 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
5306 GET_MODE (to), from));
5307 }
5308
5309 if (target != to)
5310 {
5311 if (GET_MODE (to) == GET_MODE (target))
5312 emit_move_insn (to, target);
5313 else
5314 convert_move (to, target, 0);
5315 }
5316 }
5317
5318 /* Generate code to convert FROM or TO a fixed-point.
5319 If UINTP is true, either TO or FROM is an unsigned integer.
5320 If SATP is true, we need to saturate the result. */
5321
5322 void
5323 expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
5324 {
5325 enum machine_mode to_mode = GET_MODE (to);
5326 enum machine_mode from_mode = GET_MODE (from);
5327 convert_optab tab;
5328 enum rtx_code this_code;
5329 enum insn_code code;
5330 rtx insns, value;
5331 rtx libfunc;
5332
5333 if (to_mode == from_mode)
5334 {
5335 emit_move_insn (to, from);
5336 return;
5337 }
5338
5339 if (uintp)
5340 {
5341 tab = satp ? satfractuns_optab : fractuns_optab;
5342 this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
5343 }
5344 else
5345 {
5346 tab = satp ? satfract_optab : fract_optab;
5347 this_code = satp ? SAT_FRACT : FRACT_CONVERT;
5348 }
5349 code = convert_optab_handler (tab, to_mode, from_mode);
5350 if (code != CODE_FOR_nothing)
5351 {
5352 emit_unop_insn (code, to, from, this_code);
5353 return;
5354 }
5355
5356 libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
5357 gcc_assert (libfunc);
5358
5359 start_sequence ();
5360 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
5361 1, from, from_mode);
5362 insns = get_insns ();
5363 end_sequence ();
5364
5365 emit_libcall_block (insns, to, value,
5366 gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
5367 }
5368
5369 /* Generate code to convert FROM to fixed point and store in TO. FROM
5370 must be floating point, TO must be signed. Use the conversion optab
5371 TAB to do the conversion. */
5372
5373 bool
5374 expand_sfix_optab (rtx to, rtx from, convert_optab tab)
5375 {
5376 enum insn_code icode;
5377 rtx target = to;
5378 enum machine_mode fmode, imode;
5379
5380 /* We first try to find a pair of modes, one real and one integer, at
5381 least as wide as FROM and TO, respectively, in which we can open-code
5382 this conversion. If the integer mode is wider than the mode of TO,
5383 we can do the conversion either signed or unsigned. */
5384
5385 for (fmode = GET_MODE (from); fmode != VOIDmode;
5386 fmode = GET_MODE_WIDER_MODE (fmode))
5387 for (imode = GET_MODE (to); imode != VOIDmode;
5388 imode = GET_MODE_WIDER_MODE (imode))
5389 {
5390 icode = convert_optab_handler (tab, imode, fmode);
5391 if (icode != CODE_FOR_nothing)
5392 {
5393 rtx last = get_last_insn ();
5394 if (fmode != GET_MODE (from))
5395 from = convert_to_mode (fmode, from, 0);
5396
5397 if (imode != GET_MODE (to))
5398 target = gen_reg_rtx (imode);
5399
5400 if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN))
5401 {
5402 delete_insns_since (last);
5403 continue;
5404 }
5405 if (target != to)
5406 convert_move (to, target, 0);
5407 return true;
5408 }
5409 }
5410
5411 return false;
5412 }
5413 \f
5414 /* Report whether we have an instruction to perform the operation
5415 specified by CODE on operands of mode MODE. */
5416 int
5417 have_insn_for (enum rtx_code code, enum machine_mode mode)
5418 {
5419 return (code_to_optab (code)
5420 && (optab_handler (code_to_optab (code), mode)
5421 != CODE_FOR_nothing));
5422 }
5423
5424 /* Initialize the libfunc fields of an entire group of entries in some
5425 optab. Each entry is set equal to a string consisting of a leading
5426 pair of underscores followed by a generic operation name followed by
5427 a mode name (downshifted to lowercase) followed by a single character
5428 representing the number of operands for the given operation (which is
5429 usually one of the characters '2', '3', or '4').
5430
5431 OPTABLE is the table in which libfunc fields are to be initialized.
5432 OPNAME is the generic (string) name of the operation.
5433 SUFFIX is the character which specifies the number of operands for
5434 the given generic operation.
5435 MODE is the mode to generate for.
5436 */
5437
5438 static void
5439 gen_libfunc (optab optable, const char *opname, int suffix,
5440 enum machine_mode mode)
5441 {
5442 unsigned opname_len = strlen (opname);
5443 const char *mname = GET_MODE_NAME (mode);
5444 unsigned mname_len = strlen (mname);
5445 int prefix_len = targetm.libfunc_gnu_prefix ? 6 : 2;
5446 int len = prefix_len + opname_len + mname_len + 1 + 1;
5447 char *libfunc_name = XALLOCAVEC (char, len);
5448 char *p;
5449 const char *q;
5450
5451 p = libfunc_name;
5452 *p++ = '_';
5453 *p++ = '_';
5454 if (targetm.libfunc_gnu_prefix)
5455 {
5456 *p++ = 'g';
5457 *p++ = 'n';
5458 *p++ = 'u';
5459 *p++ = '_';
5460 }
5461 for (q = opname; *q; )
5462 *p++ = *q++;
5463 for (q = mname; *q; q++)
5464 *p++ = TOLOWER (*q);
5465 *p++ = suffix;
5466 *p = '\0';
5467
5468 set_optab_libfunc (optable, mode,
5469 ggc_alloc_string (libfunc_name, p - libfunc_name));
5470 }
5471
5472 /* Like gen_libfunc, but verify that integer operation is involved. */
5473
5474 void
5475 gen_int_libfunc (optab optable, const char *opname, char suffix,
5476 enum machine_mode mode)
5477 {
5478 int maxsize = 2 * BITS_PER_WORD;
5479
5480 if (GET_MODE_CLASS (mode) != MODE_INT)
5481 return;
5482 if (maxsize < LONG_LONG_TYPE_SIZE)
5483 maxsize = LONG_LONG_TYPE_SIZE;
5484 if (GET_MODE_CLASS (mode) != MODE_INT
5485 || mode < word_mode || GET_MODE_BITSIZE (mode) > maxsize)
5486 return;
5487 gen_libfunc (optable, opname, suffix, mode);
5488 }
5489
5490 /* Like gen_libfunc, but verify that FP and set decimal prefix if needed. */
5491
5492 void
5493 gen_fp_libfunc (optab optable, const char *opname, char suffix,
5494 enum machine_mode mode)
5495 {
5496 char *dec_opname;
5497
5498 if (GET_MODE_CLASS (mode) == MODE_FLOAT)
5499 gen_libfunc (optable, opname, suffix, mode);
5500 if (DECIMAL_FLOAT_MODE_P (mode))
5501 {
5502 dec_opname = XALLOCAVEC (char, sizeof (DECIMAL_PREFIX) + strlen (opname));
5503 /* For BID support, change the name to have either a bid_ or dpd_ prefix
5504 depending on the low level floating format used. */
5505 memcpy (dec_opname, DECIMAL_PREFIX, sizeof (DECIMAL_PREFIX) - 1);
5506 strcpy (dec_opname + sizeof (DECIMAL_PREFIX) - 1, opname);
5507 gen_libfunc (optable, dec_opname, suffix, mode);
5508 }
5509 }
5510
5511 /* Like gen_libfunc, but verify that fixed-point operation is involved. */
5512
5513 void
5514 gen_fixed_libfunc (optab optable, const char *opname, char suffix,
5515 enum machine_mode mode)
5516 {
5517 if (!ALL_FIXED_POINT_MODE_P (mode))
5518 return;
5519 gen_libfunc (optable, opname, suffix, mode);
5520 }
5521
5522 /* Like gen_libfunc, but verify that signed fixed-point operation is
5523 involved. */
5524
5525 void
5526 gen_signed_fixed_libfunc (optab optable, const char *opname, char suffix,
5527 enum machine_mode mode)
5528 {
5529 if (!SIGNED_FIXED_POINT_MODE_P (mode))
5530 return;
5531 gen_libfunc (optable, opname, suffix, mode);
5532 }
5533
5534 /* Like gen_libfunc, but verify that unsigned fixed-point operation is
5535 involved. */
5536
5537 void
5538 gen_unsigned_fixed_libfunc (optab optable, const char *opname, char suffix,
5539 enum machine_mode mode)
5540 {
5541 if (!UNSIGNED_FIXED_POINT_MODE_P (mode))
5542 return;
5543 gen_libfunc (optable, opname, suffix, mode);
5544 }
5545
5546 /* Like gen_libfunc, but verify that FP or INT operation is involved. */
5547
5548 void
5549 gen_int_fp_libfunc (optab optable, const char *name, char suffix,
5550 enum machine_mode mode)
5551 {
5552 if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT)
5553 gen_fp_libfunc (optable, name, suffix, mode);
5554 if (INTEGRAL_MODE_P (mode))
5555 gen_int_libfunc (optable, name, suffix, mode);
5556 }
5557
5558 /* Like gen_libfunc, but verify that FP or INT operation is involved
5559 and add 'v' suffix for integer operation. */
5560
5561 void
5562 gen_intv_fp_libfunc (optab optable, const char *name, char suffix,
5563 enum machine_mode mode)
5564 {
5565 if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT)
5566 gen_fp_libfunc (optable, name, suffix, mode);
5567 if (GET_MODE_CLASS (mode) == MODE_INT)
5568 {
5569 int len = strlen (name);
5570 char *v_name = XALLOCAVEC (char, len + 2);
5571 strcpy (v_name, name);
5572 v_name[len] = 'v';
5573 v_name[len + 1] = 0;
5574 gen_int_libfunc (optable, v_name, suffix, mode);
5575 }
5576 }
5577
5578 /* Like gen_libfunc, but verify that FP or INT or FIXED operation is
5579 involved. */
5580
5581 void
5582 gen_int_fp_fixed_libfunc (optab optable, const char *name, char suffix,
5583 enum machine_mode mode)
5584 {
5585 if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT)
5586 gen_fp_libfunc (optable, name, suffix, mode);
5587 if (INTEGRAL_MODE_P (mode))
5588 gen_int_libfunc (optable, name, suffix, mode);
5589 if (ALL_FIXED_POINT_MODE_P (mode))
5590 gen_fixed_libfunc (optable, name, suffix, mode);
5591 }
5592
5593 /* Like gen_libfunc, but verify that FP or INT or signed FIXED operation is
5594 involved. */
5595
5596 void
5597 gen_int_fp_signed_fixed_libfunc (optab optable, const char *name, char suffix,
5598 enum machine_mode mode)
5599 {
5600 if (DECIMAL_FLOAT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_FLOAT)
5601 gen_fp_libfunc (optable, name, suffix, mode);
5602 if (INTEGRAL_MODE_P (mode))
5603 gen_int_libfunc (optable, name, suffix, mode);
5604 if (SIGNED_FIXED_POINT_MODE_P (mode))
5605 gen_signed_fixed_libfunc (optable, name, suffix, mode);
5606 }
5607
5608 /* Like gen_libfunc, but verify that INT or FIXED operation is
5609 involved. */
5610
5611 void
5612 gen_int_fixed_libfunc (optab optable, const char *name, char suffix,
5613 enum machine_mode mode)
5614 {
5615 if (INTEGRAL_MODE_P (mode))
5616 gen_int_libfunc (optable, name, suffix, mode);
5617 if (ALL_FIXED_POINT_MODE_P (mode))
5618 gen_fixed_libfunc (optable, name, suffix, mode);
5619 }
5620
5621 /* Like gen_libfunc, but verify that INT or signed FIXED operation is
5622 involved. */
5623
5624 void
5625 gen_int_signed_fixed_libfunc (optab optable, const char *name, char suffix,
5626 enum machine_mode mode)
5627 {
5628 if (INTEGRAL_MODE_P (mode))
5629 gen_int_libfunc (optable, name, suffix, mode);
5630 if (SIGNED_FIXED_POINT_MODE_P (mode))
5631 gen_signed_fixed_libfunc (optable, name, suffix, mode);
5632 }
5633
5634 /* Like gen_libfunc, but verify that INT or unsigned FIXED operation is
5635 involved. */
5636
5637 void
5638 gen_int_unsigned_fixed_libfunc (optab optable, const char *name, char suffix,
5639 enum machine_mode mode)
5640 {
5641 if (INTEGRAL_MODE_P (mode))
5642 gen_int_libfunc (optable, name, suffix, mode);
5643 if (UNSIGNED_FIXED_POINT_MODE_P (mode))
5644 gen_unsigned_fixed_libfunc (optable, name, suffix, mode);
5645 }
5646
5647 /* Initialize the libfunc fields of an entire group of entries of an
5648 inter-mode-class conversion optab. The string formation rules are
5649 similar to the ones for init_libfuncs, above, but instead of having
5650 a mode name and an operand count these functions have two mode names
5651 and no operand count. */
5652
5653 void
5654 gen_interclass_conv_libfunc (convert_optab tab,
5655 const char *opname,
5656 enum machine_mode tmode,
5657 enum machine_mode fmode)
5658 {
5659 size_t opname_len = strlen (opname);
5660 size_t mname_len = 0;
5661
5662 const char *fname, *tname;
5663 const char *q;
5664 int prefix_len = targetm.libfunc_gnu_prefix ? 6 : 2;
5665 char *libfunc_name, *suffix;
5666 char *nondec_name, *dec_name, *nondec_suffix, *dec_suffix;
5667 char *p;
5668
5669 /* If this is a decimal conversion, add the current BID vs. DPD prefix that
5670 depends on which underlying decimal floating point format is used. */
5671 const size_t dec_len = sizeof (DECIMAL_PREFIX) - 1;
5672
5673 mname_len = strlen (GET_MODE_NAME (tmode)) + strlen (GET_MODE_NAME (fmode));
5674
5675 nondec_name = XALLOCAVEC (char, prefix_len + opname_len + mname_len + 1 + 1);
5676 nondec_name[0] = '_';
5677 nondec_name[1] = '_';
5678 if (targetm.libfunc_gnu_prefix)
5679 {
5680 nondec_name[2] = 'g';
5681 nondec_name[3] = 'n';
5682 nondec_name[4] = 'u';
5683 nondec_name[5] = '_';
5684 }
5685
5686 memcpy (&nondec_name[prefix_len], opname, opname_len);
5687 nondec_suffix = nondec_name + opname_len + prefix_len;
5688
5689 dec_name = XALLOCAVEC (char, 2 + dec_len + opname_len + mname_len + 1 + 1);
5690 dec_name[0] = '_';
5691 dec_name[1] = '_';
5692 memcpy (&dec_name[2], DECIMAL_PREFIX, dec_len);
5693 memcpy (&dec_name[2+dec_len], opname, opname_len);
5694 dec_suffix = dec_name + dec_len + opname_len + 2;
5695
5696 fname = GET_MODE_NAME (fmode);
5697 tname = GET_MODE_NAME (tmode);
5698
5699 if (DECIMAL_FLOAT_MODE_P(fmode) || DECIMAL_FLOAT_MODE_P(tmode))
5700 {
5701 libfunc_name = dec_name;
5702 suffix = dec_suffix;
5703 }
5704 else
5705 {
5706 libfunc_name = nondec_name;
5707 suffix = nondec_suffix;
5708 }
5709
5710 p = suffix;
5711 for (q = fname; *q; p++, q++)
5712 *p = TOLOWER (*q);
5713 for (q = tname; *q; p++, q++)
5714 *p = TOLOWER (*q);
5715
5716 *p = '\0';
5717
5718 set_conv_libfunc (tab, tmode, fmode,
5719 ggc_alloc_string (libfunc_name, p - libfunc_name));
5720 }
5721
5722 /* Same as gen_interclass_conv_libfunc but verify that we are producing
5723 int->fp conversion. */
5724
5725 void
5726 gen_int_to_fp_conv_libfunc (convert_optab tab,
5727 const char *opname,
5728 enum machine_mode tmode,
5729 enum machine_mode fmode)
5730 {
5731 if (GET_MODE_CLASS (fmode) != MODE_INT)
5732 return;
5733 if (GET_MODE_CLASS (tmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (tmode))
5734 return;
5735 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5736 }
5737
5738 /* ufloat_optab is special by using floatun for FP and floatuns decimal fp
5739 naming scheme. */
5740
5741 void
5742 gen_ufloat_conv_libfunc (convert_optab tab,
5743 const char *opname ATTRIBUTE_UNUSED,
5744 enum machine_mode tmode,
5745 enum machine_mode fmode)
5746 {
5747 if (DECIMAL_FLOAT_MODE_P (tmode))
5748 gen_int_to_fp_conv_libfunc (tab, "floatuns", tmode, fmode);
5749 else
5750 gen_int_to_fp_conv_libfunc (tab, "floatun", tmode, fmode);
5751 }
5752
5753 /* Same as gen_interclass_conv_libfunc but verify that we are producing
5754 fp->int conversion. */
5755
5756 void
5757 gen_int_to_fp_nondecimal_conv_libfunc (convert_optab tab,
5758 const char *opname,
5759 enum machine_mode tmode,
5760 enum machine_mode fmode)
5761 {
5762 if (GET_MODE_CLASS (fmode) != MODE_INT)
5763 return;
5764 if (GET_MODE_CLASS (tmode) != MODE_FLOAT)
5765 return;
5766 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5767 }
5768
5769 /* Same as gen_interclass_conv_libfunc but verify that we are producing
5770 fp->int conversion with no decimal floating point involved. */
5771
5772 void
5773 gen_fp_to_int_conv_libfunc (convert_optab tab,
5774 const char *opname,
5775 enum machine_mode tmode,
5776 enum machine_mode fmode)
5777 {
5778 if (GET_MODE_CLASS (fmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (fmode))
5779 return;
5780 if (GET_MODE_CLASS (tmode) != MODE_INT)
5781 return;
5782 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5783 }
5784
5785 /* Initialize the libfunc fields of an of an intra-mode-class conversion optab.
5786 The string formation rules are
5787 similar to the ones for init_libfunc, above. */
5788
5789 void
5790 gen_intraclass_conv_libfunc (convert_optab tab, const char *opname,
5791 enum machine_mode tmode, enum machine_mode fmode)
5792 {
5793 size_t opname_len = strlen (opname);
5794 size_t mname_len = 0;
5795
5796 const char *fname, *tname;
5797 const char *q;
5798 int prefix_len = targetm.libfunc_gnu_prefix ? 6 : 2;
5799 char *nondec_name, *dec_name, *nondec_suffix, *dec_suffix;
5800 char *libfunc_name, *suffix;
5801 char *p;
5802
5803 /* If this is a decimal conversion, add the current BID vs. DPD prefix that
5804 depends on which underlying decimal floating point format is used. */
5805 const size_t dec_len = sizeof (DECIMAL_PREFIX) - 1;
5806
5807 mname_len = strlen (GET_MODE_NAME (tmode)) + strlen (GET_MODE_NAME (fmode));
5808
5809 nondec_name = XALLOCAVEC (char, 2 + opname_len + mname_len + 1 + 1);
5810 nondec_name[0] = '_';
5811 nondec_name[1] = '_';
5812 if (targetm.libfunc_gnu_prefix)
5813 {
5814 nondec_name[2] = 'g';
5815 nondec_name[3] = 'n';
5816 nondec_name[4] = 'u';
5817 nondec_name[5] = '_';
5818 }
5819 memcpy (&nondec_name[prefix_len], opname, opname_len);
5820 nondec_suffix = nondec_name + opname_len + prefix_len;
5821
5822 dec_name = XALLOCAVEC (char, 2 + dec_len + opname_len + mname_len + 1 + 1);
5823 dec_name[0] = '_';
5824 dec_name[1] = '_';
5825 memcpy (&dec_name[2], DECIMAL_PREFIX, dec_len);
5826 memcpy (&dec_name[2 + dec_len], opname, opname_len);
5827 dec_suffix = dec_name + dec_len + opname_len + 2;
5828
5829 fname = GET_MODE_NAME (fmode);
5830 tname = GET_MODE_NAME (tmode);
5831
5832 if (DECIMAL_FLOAT_MODE_P(fmode) || DECIMAL_FLOAT_MODE_P(tmode))
5833 {
5834 libfunc_name = dec_name;
5835 suffix = dec_suffix;
5836 }
5837 else
5838 {
5839 libfunc_name = nondec_name;
5840 suffix = nondec_suffix;
5841 }
5842
5843 p = suffix;
5844 for (q = fname; *q; p++, q++)
5845 *p = TOLOWER (*q);
5846 for (q = tname; *q; p++, q++)
5847 *p = TOLOWER (*q);
5848
5849 *p++ = '2';
5850 *p = '\0';
5851
5852 set_conv_libfunc (tab, tmode, fmode,
5853 ggc_alloc_string (libfunc_name, p - libfunc_name));
5854 }
5855
5856 /* Pick proper libcall for trunc_optab. We need to chose if we do
5857 truncation or extension and interclass or intraclass. */
5858
5859 void
5860 gen_trunc_conv_libfunc (convert_optab tab,
5861 const char *opname,
5862 enum machine_mode tmode,
5863 enum machine_mode fmode)
5864 {
5865 if (GET_MODE_CLASS (tmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (tmode))
5866 return;
5867 if (GET_MODE_CLASS (fmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (fmode))
5868 return;
5869 if (tmode == fmode)
5870 return;
5871
5872 if ((GET_MODE_CLASS (tmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (fmode))
5873 || (GET_MODE_CLASS (fmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (tmode)))
5874 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5875
5876 if (GET_MODE_PRECISION (fmode) <= GET_MODE_PRECISION (tmode))
5877 return;
5878
5879 if ((GET_MODE_CLASS (tmode) == MODE_FLOAT
5880 && GET_MODE_CLASS (fmode) == MODE_FLOAT)
5881 || (DECIMAL_FLOAT_MODE_P (fmode) && DECIMAL_FLOAT_MODE_P (tmode)))
5882 gen_intraclass_conv_libfunc (tab, opname, tmode, fmode);
5883 }
5884
5885 /* Pick proper libcall for extend_optab. We need to chose if we do
5886 truncation or extension and interclass or intraclass. */
5887
5888 void
5889 gen_extend_conv_libfunc (convert_optab tab,
5890 const char *opname ATTRIBUTE_UNUSED,
5891 enum machine_mode tmode,
5892 enum machine_mode fmode)
5893 {
5894 if (GET_MODE_CLASS (tmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (tmode))
5895 return;
5896 if (GET_MODE_CLASS (fmode) != MODE_FLOAT && !DECIMAL_FLOAT_MODE_P (fmode))
5897 return;
5898 if (tmode == fmode)
5899 return;
5900
5901 if ((GET_MODE_CLASS (tmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (fmode))
5902 || (GET_MODE_CLASS (fmode) == MODE_FLOAT && DECIMAL_FLOAT_MODE_P (tmode)))
5903 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5904
5905 if (GET_MODE_PRECISION (fmode) > GET_MODE_PRECISION (tmode))
5906 return;
5907
5908 if ((GET_MODE_CLASS (tmode) == MODE_FLOAT
5909 && GET_MODE_CLASS (fmode) == MODE_FLOAT)
5910 || (DECIMAL_FLOAT_MODE_P (fmode) && DECIMAL_FLOAT_MODE_P (tmode)))
5911 gen_intraclass_conv_libfunc (tab, opname, tmode, fmode);
5912 }
5913
5914 /* Pick proper libcall for fract_optab. We need to chose if we do
5915 interclass or intraclass. */
5916
5917 void
5918 gen_fract_conv_libfunc (convert_optab tab,
5919 const char *opname,
5920 enum machine_mode tmode,
5921 enum machine_mode fmode)
5922 {
5923 if (tmode == fmode)
5924 return;
5925 if (!(ALL_FIXED_POINT_MODE_P (tmode) || ALL_FIXED_POINT_MODE_P (fmode)))
5926 return;
5927
5928 if (GET_MODE_CLASS (tmode) == GET_MODE_CLASS (fmode))
5929 gen_intraclass_conv_libfunc (tab, opname, tmode, fmode);
5930 else
5931 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5932 }
5933
5934 /* Pick proper libcall for fractuns_optab. */
5935
5936 void
5937 gen_fractuns_conv_libfunc (convert_optab tab,
5938 const char *opname,
5939 enum machine_mode tmode,
5940 enum machine_mode fmode)
5941 {
5942 if (tmode == fmode)
5943 return;
5944 /* One mode must be a fixed-point mode, and the other must be an integer
5945 mode. */
5946 if (!((ALL_FIXED_POINT_MODE_P (tmode) && GET_MODE_CLASS (fmode) == MODE_INT)
5947 || (ALL_FIXED_POINT_MODE_P (fmode)
5948 && GET_MODE_CLASS (tmode) == MODE_INT)))
5949 return;
5950
5951 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5952 }
5953
5954 /* Pick proper libcall for satfract_optab. We need to chose if we do
5955 interclass or intraclass. */
5956
5957 void
5958 gen_satfract_conv_libfunc (convert_optab tab,
5959 const char *opname,
5960 enum machine_mode tmode,
5961 enum machine_mode fmode)
5962 {
5963 if (tmode == fmode)
5964 return;
5965 /* TMODE must be a fixed-point mode. */
5966 if (!ALL_FIXED_POINT_MODE_P (tmode))
5967 return;
5968
5969 if (GET_MODE_CLASS (tmode) == GET_MODE_CLASS (fmode))
5970 gen_intraclass_conv_libfunc (tab, opname, tmode, fmode);
5971 else
5972 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5973 }
5974
5975 /* Pick proper libcall for satfractuns_optab. */
5976
5977 void
5978 gen_satfractuns_conv_libfunc (convert_optab tab,
5979 const char *opname,
5980 enum machine_mode tmode,
5981 enum machine_mode fmode)
5982 {
5983 if (tmode == fmode)
5984 return;
5985 /* TMODE must be a fixed-point mode, and FMODE must be an integer mode. */
5986 if (!(ALL_FIXED_POINT_MODE_P (tmode) && GET_MODE_CLASS (fmode) == MODE_INT))
5987 return;
5988
5989 gen_interclass_conv_libfunc (tab, opname, tmode, fmode);
5990 }
5991
5992 /* A table of previously-created libfuncs, hashed by name. */
5993 static GTY ((param_is (union tree_node))) htab_t libfunc_decls;
5994
5995 /* Hashtable callbacks for libfunc_decls. */
5996
5997 static hashval_t
5998 libfunc_decl_hash (const void *entry)
5999 {
6000 return IDENTIFIER_HASH_VALUE (DECL_NAME ((const_tree) entry));
6001 }
6002
6003 static int
6004 libfunc_decl_eq (const void *entry1, const void *entry2)
6005 {
6006 return DECL_NAME ((const_tree) entry1) == (const_tree) entry2;
6007 }
6008
6009 /* Build a decl for a libfunc named NAME. */
6010
6011 tree
6012 build_libfunc_function (const char *name)
6013 {
6014 tree decl = build_decl (UNKNOWN_LOCATION, FUNCTION_DECL,
6015 get_identifier (name),
6016 build_function_type (integer_type_node, NULL_TREE));
6017 /* ??? We don't have any type information except for this is
6018 a function. Pretend this is "int foo()". */
6019 DECL_ARTIFICIAL (decl) = 1;
6020 DECL_EXTERNAL (decl) = 1;
6021 TREE_PUBLIC (decl) = 1;
6022 gcc_assert (DECL_ASSEMBLER_NAME (decl));
6023
6024 /* Zap the nonsensical SYMBOL_REF_DECL for this. What we're left with
6025 are the flags assigned by targetm.encode_section_info. */
6026 SET_SYMBOL_REF_DECL (XEXP (DECL_RTL (decl), 0), NULL);
6027
6028 return decl;
6029 }
6030
6031 rtx
6032 init_one_libfunc (const char *name)
6033 {
6034 tree id, decl;
6035 void **slot;
6036 hashval_t hash;
6037
6038 if (libfunc_decls == NULL)
6039 libfunc_decls = htab_create_ggc (37, libfunc_decl_hash,
6040 libfunc_decl_eq, NULL);
6041
6042 /* See if we have already created a libfunc decl for this function. */
6043 id = get_identifier (name);
6044 hash = IDENTIFIER_HASH_VALUE (id);
6045 slot = htab_find_slot_with_hash (libfunc_decls, id, hash, INSERT);
6046 decl = (tree) *slot;
6047 if (decl == NULL)
6048 {
6049 /* Create a new decl, so that it can be passed to
6050 targetm.encode_section_info. */
6051 decl = build_libfunc_function (name);
6052 *slot = decl;
6053 }
6054 return XEXP (DECL_RTL (decl), 0);
6055 }
6056
6057 /* Adjust the assembler name of libfunc NAME to ASMSPEC. */
6058
6059 rtx
6060 set_user_assembler_libfunc (const char *name, const char *asmspec)
6061 {
6062 tree id, decl;
6063 void **slot;
6064 hashval_t hash;
6065
6066 id = get_identifier (name);
6067 hash = IDENTIFIER_HASH_VALUE (id);
6068 slot = htab_find_slot_with_hash (libfunc_decls, id, hash, NO_INSERT);
6069 gcc_assert (slot);
6070 decl = (tree) *slot;
6071 set_user_assembler_name (decl, asmspec);
6072 return XEXP (DECL_RTL (decl), 0);
6073 }
6074
6075 /* Call this to reset the function entry for one optab (OPTABLE) in mode
6076 MODE to NAME, which should be either 0 or a string constant. */
6077 void
6078 set_optab_libfunc (optab op, enum machine_mode mode, const char *name)
6079 {
6080 rtx val;
6081 struct libfunc_entry e;
6082 struct libfunc_entry **slot;
6083
6084 e.op = op;
6085 e.mode1 = mode;
6086 e.mode2 = VOIDmode;
6087
6088 if (name)
6089 val = init_one_libfunc (name);
6090 else
6091 val = 0;
6092 slot = (struct libfunc_entry **) htab_find_slot (libfunc_hash, &e, INSERT);
6093 if (*slot == NULL)
6094 *slot = ggc_alloc_libfunc_entry ();
6095 (*slot)->op = op;
6096 (*slot)->mode1 = mode;
6097 (*slot)->mode2 = VOIDmode;
6098 (*slot)->libfunc = val;
6099 }
6100
6101 /* Call this to reset the function entry for one conversion optab
6102 (OPTABLE) from mode FMODE to mode TMODE to NAME, which should be
6103 either 0 or a string constant. */
6104 void
6105 set_conv_libfunc (convert_optab optab, enum machine_mode tmode,
6106 enum machine_mode fmode, const char *name)
6107 {
6108 rtx val;
6109 struct libfunc_entry e;
6110 struct libfunc_entry **slot;
6111
6112 e.op = optab;
6113 e.mode1 = tmode;
6114 e.mode2 = fmode;
6115
6116 if (name)
6117 val = init_one_libfunc (name);
6118 else
6119 val = 0;
6120 slot = (struct libfunc_entry **) htab_find_slot (libfunc_hash, &e, INSERT);
6121 if (*slot == NULL)
6122 *slot = ggc_alloc_libfunc_entry ();
6123 (*slot)->op = optab;
6124 (*slot)->mode1 = tmode;
6125 (*slot)->mode2 = fmode;
6126 (*slot)->libfunc = val;
6127 }
6128
6129 /* Call this to initialize the contents of the optabs
6130 appropriately for the current target machine. */
6131
6132 void
6133 init_optabs (void)
6134 {
6135 if (libfunc_hash)
6136 htab_empty (libfunc_hash);
6137 else
6138 libfunc_hash = htab_create_ggc (10, hash_libfunc, eq_libfunc, NULL);
6139
6140 /* Fill in the optabs with the insns we support. */
6141 init_all_optabs ();
6142
6143 /* The ffs function operates on `int'. Fall back on it if we do not
6144 have a libgcc2 function for that width. */
6145 if (INT_TYPE_SIZE < BITS_PER_WORD)
6146 set_optab_libfunc (ffs_optab, mode_for_size (INT_TYPE_SIZE, MODE_INT, 0),
6147 "ffs");
6148
6149 /* Explicitly initialize the bswap libfuncs since we need them to be
6150 valid for things other than word_mode. */
6151 if (targetm.libfunc_gnu_prefix)
6152 {
6153 set_optab_libfunc (bswap_optab, SImode, "__gnu_bswapsi2");
6154 set_optab_libfunc (bswap_optab, DImode, "__gnu_bswapdi2");
6155 }
6156 else
6157 {
6158 set_optab_libfunc (bswap_optab, SImode, "__bswapsi2");
6159 set_optab_libfunc (bswap_optab, DImode, "__bswapdi2");
6160 }
6161
6162 /* Use cabs for double complex abs, since systems generally have cabs.
6163 Don't define any libcall for float complex, so that cabs will be used. */
6164 if (complex_double_type_node)
6165 set_optab_libfunc (abs_optab, TYPE_MODE (complex_double_type_node),
6166 "cabs");
6167
6168 abort_libfunc = init_one_libfunc ("abort");
6169 memcpy_libfunc = init_one_libfunc ("memcpy");
6170 memmove_libfunc = init_one_libfunc ("memmove");
6171 memcmp_libfunc = init_one_libfunc ("memcmp");
6172 memset_libfunc = init_one_libfunc ("memset");
6173 setbits_libfunc = init_one_libfunc ("__setbits");
6174
6175 #ifndef DONT_USE_BUILTIN_SETJMP
6176 setjmp_libfunc = init_one_libfunc ("__builtin_setjmp");
6177 longjmp_libfunc = init_one_libfunc ("__builtin_longjmp");
6178 #else
6179 setjmp_libfunc = init_one_libfunc ("setjmp");
6180 longjmp_libfunc = init_one_libfunc ("longjmp");
6181 #endif
6182 unwind_sjlj_register_libfunc = init_one_libfunc ("_Unwind_SjLj_Register");
6183 unwind_sjlj_unregister_libfunc
6184 = init_one_libfunc ("_Unwind_SjLj_Unregister");
6185
6186 /* For function entry/exit instrumentation. */
6187 profile_function_entry_libfunc
6188 = init_one_libfunc ("__cyg_profile_func_enter");
6189 profile_function_exit_libfunc
6190 = init_one_libfunc ("__cyg_profile_func_exit");
6191
6192 gcov_flush_libfunc = init_one_libfunc ("__gcov_flush");
6193
6194 /* Allow the target to add more libcalls or rename some, etc. */
6195 targetm.init_libfuncs ();
6196 }
6197
6198 /* A helper function for init_sync_libfuncs. Using the basename BASE,
6199 install libfuncs into TAB for BASE_N for 1 <= N <= MAX. */
6200
6201 static void
6202 init_sync_libfuncs_1 (optab tab, const char *base, int max)
6203 {
6204 enum machine_mode mode;
6205 char buf[64];
6206 size_t len = strlen (base);
6207 int i;
6208
6209 gcc_assert (max <= 8);
6210 gcc_assert (len + 3 < sizeof (buf));
6211
6212 memcpy (buf, base, len);
6213 buf[len] = '_';
6214 buf[len + 1] = '0';
6215 buf[len + 2] = '\0';
6216
6217 mode = QImode;
6218 for (i = 1; i <= max; i *= 2)
6219 {
6220 buf[len + 1] = '0' + i;
6221 set_optab_libfunc (tab, mode, buf);
6222 mode = GET_MODE_2XWIDER_MODE (mode);
6223 }
6224 }
6225
6226 void
6227 init_sync_libfuncs (int max)
6228 {
6229 if (!flag_sync_libcalls)
6230 return;
6231
6232 init_sync_libfuncs_1 (sync_compare_and_swap_optab,
6233 "__sync_val_compare_and_swap", max);
6234 init_sync_libfuncs_1 (sync_lock_test_and_set_optab,
6235 "__sync_lock_test_and_set", max);
6236
6237 init_sync_libfuncs_1 (sync_old_add_optab, "__sync_fetch_and_add", max);
6238 init_sync_libfuncs_1 (sync_old_sub_optab, "__sync_fetch_and_sub", max);
6239 init_sync_libfuncs_1 (sync_old_ior_optab, "__sync_fetch_and_or", max);
6240 init_sync_libfuncs_1 (sync_old_and_optab, "__sync_fetch_and_and", max);
6241 init_sync_libfuncs_1 (sync_old_xor_optab, "__sync_fetch_and_xor", max);
6242 init_sync_libfuncs_1 (sync_old_nand_optab, "__sync_fetch_and_nand", max);
6243
6244 init_sync_libfuncs_1 (sync_new_add_optab, "__sync_add_and_fetch", max);
6245 init_sync_libfuncs_1 (sync_new_sub_optab, "__sync_sub_and_fetch", max);
6246 init_sync_libfuncs_1 (sync_new_ior_optab, "__sync_or_and_fetch", max);
6247 init_sync_libfuncs_1 (sync_new_and_optab, "__sync_and_and_fetch", max);
6248 init_sync_libfuncs_1 (sync_new_xor_optab, "__sync_xor_and_fetch", max);
6249 init_sync_libfuncs_1 (sync_new_nand_optab, "__sync_nand_and_fetch", max);
6250 }
6251
6252 /* Print information about the current contents of the optabs on
6253 STDERR. */
6254
6255 DEBUG_FUNCTION void
6256 debug_optab_libfuncs (void)
6257 {
6258 int i, j, k;
6259
6260 /* Dump the arithmetic optabs. */
6261 for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
6262 for (j = 0; j < NUM_MACHINE_MODES; ++j)
6263 {
6264 rtx l = optab_libfunc ((optab) i, (enum machine_mode) j);
6265 if (l)
6266 {
6267 gcc_assert (GET_CODE (l) == SYMBOL_REF);
6268 fprintf (stderr, "%s\t%s:\t%s\n",
6269 GET_RTX_NAME (optab_to_code ((optab) i)),
6270 GET_MODE_NAME (j),
6271 XSTR (l, 0));
6272 }
6273 }
6274
6275 /* Dump the conversion optabs. */
6276 for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
6277 for (j = 0; j < NUM_MACHINE_MODES; ++j)
6278 for (k = 0; k < NUM_MACHINE_MODES; ++k)
6279 {
6280 rtx l = convert_optab_libfunc ((optab) i, (enum machine_mode) j,
6281 (enum machine_mode) k);
6282 if (l)
6283 {
6284 gcc_assert (GET_CODE (l) == SYMBOL_REF);
6285 fprintf (stderr, "%s\t%s\t%s:\t%s\n",
6286 GET_RTX_NAME (optab_to_code ((optab) i)),
6287 GET_MODE_NAME (j),
6288 GET_MODE_NAME (k),
6289 XSTR (l, 0));
6290 }
6291 }
6292 }
6293
6294 \f
6295 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
6296 CODE. Return 0 on failure. */
6297
6298 rtx
6299 gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
6300 {
6301 enum machine_mode mode = GET_MODE (op1);
6302 enum insn_code icode;
6303 rtx insn;
6304 rtx trap_rtx;
6305
6306 if (mode == VOIDmode)
6307 return 0;
6308
6309 icode = optab_handler (ctrap_optab, mode);
6310 if (icode == CODE_FOR_nothing)
6311 return 0;
6312
6313 /* Some targets only accept a zero trap code. */
6314 if (!insn_operand_matches (icode, 3, tcode))
6315 return 0;
6316
6317 do_pending_stack_adjust ();
6318 start_sequence ();
6319 prepare_cmp_insn (op1, op2, code, NULL_RTX, false, OPTAB_DIRECT,
6320 &trap_rtx, &mode);
6321 if (!trap_rtx)
6322 insn = NULL_RTX;
6323 else
6324 insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
6325 tcode);
6326
6327 /* If that failed, then give up. */
6328 if (insn == 0)
6329 {
6330 end_sequence ();
6331 return 0;
6332 }
6333
6334 emit_insn (insn);
6335 insn = get_insns ();
6336 end_sequence ();
6337 return insn;
6338 }
6339
6340 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
6341 or unsigned operation code. */
6342
6343 static enum rtx_code
6344 get_rtx_code (enum tree_code tcode, bool unsignedp)
6345 {
6346 enum rtx_code code;
6347 switch (tcode)
6348 {
6349 case EQ_EXPR:
6350 code = EQ;
6351 break;
6352 case NE_EXPR:
6353 code = NE;
6354 break;
6355 case LT_EXPR:
6356 code = unsignedp ? LTU : LT;
6357 break;
6358 case LE_EXPR:
6359 code = unsignedp ? LEU : LE;
6360 break;
6361 case GT_EXPR:
6362 code = unsignedp ? GTU : GT;
6363 break;
6364 case GE_EXPR:
6365 code = unsignedp ? GEU : GE;
6366 break;
6367
6368 case UNORDERED_EXPR:
6369 code = UNORDERED;
6370 break;
6371 case ORDERED_EXPR:
6372 code = ORDERED;
6373 break;
6374 case UNLT_EXPR:
6375 code = UNLT;
6376 break;
6377 case UNLE_EXPR:
6378 code = UNLE;
6379 break;
6380 case UNGT_EXPR:
6381 code = UNGT;
6382 break;
6383 case UNGE_EXPR:
6384 code = UNGE;
6385 break;
6386 case UNEQ_EXPR:
6387 code = UNEQ;
6388 break;
6389 case LTGT_EXPR:
6390 code = LTGT;
6391 break;
6392
6393 default:
6394 gcc_unreachable ();
6395 }
6396 return code;
6397 }
6398
6399 /* Return comparison rtx for COND. Use UNSIGNEDP to select signed or
6400 unsigned operators. Do not generate compare instruction. */
6401
6402 static rtx
6403 vector_compare_rtx (tree cond, bool unsignedp, enum insn_code icode)
6404 {
6405 struct expand_operand ops[2];
6406 enum rtx_code rcode;
6407 tree t_op0, t_op1;
6408 rtx rtx_op0, rtx_op1;
6409
6410 /* This is unlikely. While generating VEC_COND_EXPR, auto vectorizer
6411 ensures that condition is a relational operation. */
6412 gcc_assert (COMPARISON_CLASS_P (cond));
6413
6414 rcode = get_rtx_code (TREE_CODE (cond), unsignedp);
6415 t_op0 = TREE_OPERAND (cond, 0);
6416 t_op1 = TREE_OPERAND (cond, 1);
6417
6418 /* Expand operands. */
6419 rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
6420 EXPAND_STACK_PARM);
6421 rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
6422 EXPAND_STACK_PARM);
6423
6424 create_input_operand (&ops[0], rtx_op0, GET_MODE (rtx_op0));
6425 create_input_operand (&ops[1], rtx_op1, GET_MODE (rtx_op1));
6426 if (!maybe_legitimize_operands (icode, 4, 2, ops))
6427 gcc_unreachable ();
6428 return gen_rtx_fmt_ee (rcode, VOIDmode, ops[0].value, ops[1].value);
6429 }
6430
6431 /* Return true if VEC_PERM_EXPR can be expanded using SIMD extensions
6432 of the CPU. SEL may be NULL, which stands for an unknown constant. */
6433
6434 bool
6435 can_vec_perm_p (enum machine_mode mode, bool variable,
6436 const unsigned char *sel)
6437 {
6438 enum machine_mode qimode;
6439
6440 /* If the target doesn't implement a vector mode for the vector type,
6441 then no operations are supported. */
6442 if (!VECTOR_MODE_P (mode))
6443 return false;
6444
6445 if (!variable)
6446 {
6447 if (direct_optab_handler (vec_perm_const_optab, mode) != CODE_FOR_nothing
6448 && (sel == NULL
6449 || targetm.vectorize.vec_perm_const_ok == NULL
6450 || targetm.vectorize.vec_perm_const_ok (mode, sel)))
6451 return true;
6452 }
6453
6454 if (direct_optab_handler (vec_perm_optab, mode) != CODE_FOR_nothing)
6455 return true;
6456
6457 /* We allow fallback to a QI vector mode, and adjust the mask. */
6458 if (GET_MODE_INNER (mode) == QImode)
6459 return false;
6460 qimode = mode_for_vector (QImode, GET_MODE_SIZE (mode));
6461 if (!VECTOR_MODE_P (qimode))
6462 return false;
6463
6464 /* ??? For completeness, we ought to check the QImode version of
6465 vec_perm_const_optab. But all users of this implicit lowering
6466 feature implement the variable vec_perm_optab. */
6467 if (direct_optab_handler (vec_perm_optab, qimode) == CODE_FOR_nothing)
6468 return false;
6469
6470 /* In order to support the lowering of variable permutations,
6471 we need to support shifts and adds. */
6472 if (variable)
6473 {
6474 if (GET_MODE_UNIT_SIZE (mode) > 2
6475 && optab_handler (ashl_optab, mode) == CODE_FOR_nothing
6476 && optab_handler (vashl_optab, mode) == CODE_FOR_nothing)
6477 return false;
6478 if (optab_handler (add_optab, qimode) == CODE_FOR_nothing)
6479 return false;
6480 }
6481
6482 return true;
6483 }
6484
6485 /* A subroutine of expand_vec_perm for expanding one vec_perm insn. */
6486
6487 static rtx
6488 expand_vec_perm_1 (enum insn_code icode, rtx target,
6489 rtx v0, rtx v1, rtx sel)
6490 {
6491 enum machine_mode tmode = GET_MODE (target);
6492 enum machine_mode smode = GET_MODE (sel);
6493 struct expand_operand ops[4];
6494
6495 create_output_operand (&ops[0], target, tmode);
6496 create_input_operand (&ops[3], sel, smode);
6497
6498 /* Make an effort to preserve v0 == v1. The target expander is able to
6499 rely on this to determine if we're permuting a single input operand. */
6500 if (rtx_equal_p (v0, v1))
6501 {
6502 if (!insn_operand_matches (icode, 1, v0))
6503 v0 = force_reg (tmode, v0);
6504 gcc_checking_assert (insn_operand_matches (icode, 1, v0));
6505 gcc_checking_assert (insn_operand_matches (icode, 2, v0));
6506
6507 create_fixed_operand (&ops[1], v0);
6508 create_fixed_operand (&ops[2], v0);
6509 }
6510 else
6511 {
6512 create_input_operand (&ops[1], v0, tmode);
6513 create_input_operand (&ops[2], v1, tmode);
6514 }
6515
6516 if (maybe_expand_insn (icode, 4, ops))
6517 return ops[0].value;
6518 return NULL_RTX;
6519 }
6520
6521 /* Generate instructions for vec_perm optab given its mode
6522 and three operands. */
6523
6524 rtx
6525 expand_vec_perm (enum machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
6526 {
6527 enum insn_code icode;
6528 enum machine_mode qimode;
6529 unsigned int i, w, e, u;
6530 rtx tmp, sel_qi = NULL;
6531 rtvec vec;
6532
6533 if (!target || GET_MODE (target) != mode)
6534 target = gen_reg_rtx (mode);
6535
6536 w = GET_MODE_SIZE (mode);
6537 e = GET_MODE_NUNITS (mode);
6538 u = GET_MODE_UNIT_SIZE (mode);
6539
6540 /* Set QIMODE to a different vector mode with byte elements.
6541 If no such mode, or if MODE already has byte elements, use VOIDmode. */
6542 qimode = VOIDmode;
6543 if (GET_MODE_INNER (mode) != QImode)
6544 {
6545 qimode = mode_for_vector (QImode, w);
6546 if (!VECTOR_MODE_P (qimode))
6547 qimode = VOIDmode;
6548 }
6549
6550 /* If the input is a constant, expand it specially. */
6551 gcc_assert (GET_MODE_CLASS (GET_MODE (sel)) == MODE_VECTOR_INT);
6552 if (GET_CODE (sel) == CONST_VECTOR)
6553 {
6554 icode = direct_optab_handler (vec_perm_const_optab, mode);
6555 if (icode != CODE_FOR_nothing)
6556 {
6557 tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
6558 if (tmp)
6559 return tmp;
6560 }
6561
6562 /* Fall back to a constant byte-based permutation. */
6563 if (qimode != VOIDmode)
6564 {
6565 vec = rtvec_alloc (w);
6566 for (i = 0; i < e; ++i)
6567 {
6568 unsigned int j, this_e;
6569
6570 this_e = INTVAL (CONST_VECTOR_ELT (sel, i));
6571 this_e &= 2 * e - 1;
6572 this_e *= u;
6573
6574 for (j = 0; j < u; ++j)
6575 RTVEC_ELT (vec, i * u + j) = GEN_INT (this_e + j);
6576 }
6577 sel_qi = gen_rtx_CONST_VECTOR (qimode, vec);
6578
6579 icode = direct_optab_handler (vec_perm_const_optab, qimode);
6580 if (icode != CODE_FOR_nothing)
6581 {
6582 tmp = expand_vec_perm_1 (icode, gen_lowpart (qimode, target),
6583 gen_lowpart (qimode, v0),
6584 gen_lowpart (qimode, v1), sel_qi);
6585 if (tmp)
6586 return gen_lowpart (mode, tmp);
6587 }
6588 }
6589 }
6590
6591 /* Otherwise expand as a fully variable permuation. */
6592 icode = direct_optab_handler (vec_perm_optab, mode);
6593 if (icode != CODE_FOR_nothing)
6594 {
6595 tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
6596 if (tmp)
6597 return tmp;
6598 }
6599
6600 /* As a special case to aid several targets, lower the element-based
6601 permutation to a byte-based permutation and try again. */
6602 if (qimode == VOIDmode)
6603 return NULL_RTX;
6604 icode = direct_optab_handler (vec_perm_optab, qimode);
6605 if (icode == CODE_FOR_nothing)
6606 return NULL_RTX;
6607
6608 if (sel_qi == NULL)
6609 {
6610 /* Multiply each element by its byte size. */
6611 enum machine_mode selmode = GET_MODE (sel);
6612 if (u == 2)
6613 sel = expand_simple_binop (selmode, PLUS, sel, sel,
6614 sel, 0, OPTAB_DIRECT);
6615 else
6616 sel = expand_simple_binop (selmode, ASHIFT, sel,
6617 GEN_INT (exact_log2 (u)),
6618 sel, 0, OPTAB_DIRECT);
6619 gcc_assert (sel != NULL);
6620
6621 /* Broadcast the low byte each element into each of its bytes. */
6622 vec = rtvec_alloc (w);
6623 for (i = 0; i < w; ++i)
6624 {
6625 int this_e = i / u * u;
6626 if (BYTES_BIG_ENDIAN)
6627 this_e += u - 1;
6628 RTVEC_ELT (vec, i) = GEN_INT (this_e);
6629 }
6630 tmp = gen_rtx_CONST_VECTOR (qimode, vec);
6631 sel = gen_lowpart (qimode, sel);
6632 sel = expand_vec_perm (qimode, sel, sel, tmp, NULL);
6633 gcc_assert (sel != NULL);
6634
6635 /* Add the byte offset to each byte element. */
6636 /* Note that the definition of the indicies here is memory ordering,
6637 so there should be no difference between big and little endian. */
6638 vec = rtvec_alloc (w);
6639 for (i = 0; i < w; ++i)
6640 RTVEC_ELT (vec, i) = GEN_INT (i % u);
6641 tmp = gen_rtx_CONST_VECTOR (qimode, vec);
6642 sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
6643 sel, 0, OPTAB_DIRECT);
6644 gcc_assert (sel_qi != NULL);
6645 }
6646
6647 tmp = expand_vec_perm_1 (icode, gen_lowpart (qimode, target),
6648 gen_lowpart (qimode, v0),
6649 gen_lowpart (qimode, v1), sel_qi);
6650 if (tmp)
6651 tmp = gen_lowpart (mode, tmp);
6652 return tmp;
6653 }
6654
6655 /* Return insn code for a conditional operator with a comparison in
6656 mode CMODE, unsigned if UNS is true, resulting in a value of mode VMODE. */
6657
6658 static inline enum insn_code
6659 get_vcond_icode (enum machine_mode vmode, enum machine_mode cmode, bool uns)
6660 {
6661 enum insn_code icode = CODE_FOR_nothing;
6662 if (uns)
6663 icode = convert_optab_handler (vcondu_optab, vmode, cmode);
6664 else
6665 icode = convert_optab_handler (vcond_optab, vmode, cmode);
6666 return icode;
6667 }
6668
6669 /* Return TRUE iff, appropriate vector insns are available
6670 for vector cond expr with vector type VALUE_TYPE and a comparison
6671 with operand vector types in CMP_OP_TYPE. */
6672
6673 bool
6674 expand_vec_cond_expr_p (tree value_type, tree cmp_op_type)
6675 {
6676 enum machine_mode value_mode = TYPE_MODE (value_type);
6677 enum machine_mode cmp_op_mode = TYPE_MODE (cmp_op_type);
6678 if (GET_MODE_SIZE (value_mode) != GET_MODE_SIZE (cmp_op_mode)
6679 || GET_MODE_NUNITS (value_mode) != GET_MODE_NUNITS (cmp_op_mode)
6680 || get_vcond_icode (TYPE_MODE (value_type), TYPE_MODE (cmp_op_type),
6681 TYPE_UNSIGNED (cmp_op_type)) == CODE_FOR_nothing)
6682 return false;
6683 return true;
6684 }
6685
6686 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
6687 three operands. */
6688
6689 rtx
6690 expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
6691 rtx target)
6692 {
6693 struct expand_operand ops[6];
6694 enum insn_code icode;
6695 rtx comparison, rtx_op1, rtx_op2;
6696 enum machine_mode mode = TYPE_MODE (vec_cond_type);
6697 enum machine_mode cmp_op_mode;
6698 bool unsignedp;
6699
6700 gcc_assert (COMPARISON_CLASS_P (op0));
6701
6702 unsignedp = TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (op0, 0)));
6703 cmp_op_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (op0, 0)));
6704
6705 gcc_assert (GET_MODE_SIZE (mode) == GET_MODE_SIZE (cmp_op_mode)
6706 && GET_MODE_NUNITS (mode) == GET_MODE_NUNITS (cmp_op_mode));
6707
6708 icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
6709 if (icode == CODE_FOR_nothing)
6710 return 0;
6711
6712 comparison = vector_compare_rtx (op0, unsignedp, icode);
6713 rtx_op1 = expand_normal (op1);
6714 rtx_op2 = expand_normal (op2);
6715
6716 create_output_operand (&ops[0], target, mode);
6717 create_input_operand (&ops[1], rtx_op1, mode);
6718 create_input_operand (&ops[2], rtx_op2, mode);
6719 create_fixed_operand (&ops[3], comparison);
6720 create_fixed_operand (&ops[4], XEXP (comparison, 0));
6721 create_fixed_operand (&ops[5], XEXP (comparison, 1));
6722 expand_insn (icode, 6, ops);
6723 return ops[0].value;
6724 }
6725
6726 /* Return non-zero if a highpart multiply is supported of can be synthisized.
6727 For the benefit of expand_mult_highpart, the return value is 1 for direct,
6728 2 for even/odd widening, and 3 for hi/lo widening. */
6729
6730 int
6731 can_mult_highpart_p (enum machine_mode mode, bool uns_p)
6732 {
6733 optab op;
6734 unsigned char *sel;
6735 unsigned i, nunits;
6736
6737 op = uns_p ? umul_highpart_optab : smul_highpart_optab;
6738 if (optab_handler (op, mode) != CODE_FOR_nothing)
6739 return 1;
6740
6741 /* If the mode is an integral vector, synth from widening operations. */
6742 if (GET_MODE_CLASS (mode) != MODE_VECTOR_INT)
6743 return 0;
6744
6745 nunits = GET_MODE_NUNITS (mode);
6746 sel = XALLOCAVEC (unsigned char, nunits);
6747
6748 op = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
6749 if (optab_handler (op, mode) != CODE_FOR_nothing)
6750 {
6751 op = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
6752 if (optab_handler (op, mode) != CODE_FOR_nothing)
6753 {
6754 for (i = 0; i < nunits; ++i)
6755 sel[i] = !BYTES_BIG_ENDIAN + (i & ~1) + ((i & 1) ? nunits : 0);
6756 if (can_vec_perm_p (mode, false, sel))
6757 return 2;
6758 }
6759 }
6760
6761 op = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
6762 if (optab_handler (op, mode) != CODE_FOR_nothing)
6763 {
6764 op = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
6765 if (optab_handler (op, mode) != CODE_FOR_nothing)
6766 {
6767 for (i = 0; i < nunits; ++i)
6768 sel[i] = 2 * i + (BYTES_BIG_ENDIAN ? 0 : 1);
6769 if (can_vec_perm_p (mode, false, sel))
6770 return 3;
6771 }
6772 }
6773
6774 return 0;
6775 }
6776
6777 /* Expand a highpart multiply. */
6778
6779 rtx
6780 expand_mult_highpart (enum machine_mode mode, rtx op0, rtx op1,
6781 rtx target, bool uns_p)
6782 {
6783 struct expand_operand eops[3];
6784 enum insn_code icode;
6785 int method, i, nunits;
6786 enum machine_mode wmode;
6787 rtx m1, m2, perm;
6788 optab tab1, tab2;
6789 rtvec v;
6790
6791 method = can_mult_highpart_p (mode, uns_p);
6792 switch (method)
6793 {
6794 case 0:
6795 return NULL_RTX;
6796 case 1:
6797 tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
6798 return expand_binop (mode, tab1, op0, op1, target, uns_p,
6799 OPTAB_LIB_WIDEN);
6800 case 2:
6801 tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
6802 tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
6803 break;
6804 case 3:
6805 tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
6806 tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
6807 if (BYTES_BIG_ENDIAN)
6808 {
6809 optab t = tab1;
6810 tab1 = tab2;
6811 tab2 = t;
6812 }
6813 break;
6814 default:
6815 gcc_unreachable ();
6816 }
6817
6818 icode = optab_handler (tab1, mode);
6819 nunits = GET_MODE_NUNITS (mode);
6820 wmode = insn_data[icode].operand[0].mode;
6821 gcc_checking_assert (2 * GET_MODE_NUNITS (wmode) == nunits);
6822 gcc_checking_assert (GET_MODE_SIZE (wmode) == GET_MODE_SIZE (mode));
6823
6824 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
6825 create_input_operand (&eops[1], op0, mode);
6826 create_input_operand (&eops[2], op1, mode);
6827 expand_insn (icode, 3, eops);
6828 m1 = gen_lowpart (mode, eops[0].value);
6829
6830 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
6831 create_input_operand (&eops[1], op0, mode);
6832 create_input_operand (&eops[2], op1, mode);
6833 expand_insn (optab_handler (tab2, mode), 3, eops);
6834 m2 = gen_lowpart (mode, eops[0].value);
6835
6836 v = rtvec_alloc (nunits);
6837 if (method == 2)
6838 {
6839 for (i = 0; i < nunits; ++i)
6840 RTVEC_ELT (v, i) = GEN_INT (!BYTES_BIG_ENDIAN + (i & ~1)
6841 + ((i & 1) ? nunits : 0));
6842 }
6843 else
6844 {
6845 for (i = 0; i < nunits; ++i)
6846 RTVEC_ELT (v, i) = GEN_INT (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
6847 }
6848 perm = gen_rtx_CONST_VECTOR (mode, v);
6849
6850 return expand_vec_perm (mode, m1, m2, perm, target);
6851 }
6852 \f
6853 /* Return true if there is a compare_and_swap pattern. */
6854
6855 bool
6856 can_compare_and_swap_p (enum machine_mode mode, bool allow_libcall)
6857 {
6858 enum insn_code icode;
6859
6860 /* Check for __atomic_compare_and_swap. */
6861 icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
6862 if (icode != CODE_FOR_nothing)
6863 return true;
6864
6865 /* Check for __sync_compare_and_swap. */
6866 icode = optab_handler (sync_compare_and_swap_optab, mode);
6867 if (icode != CODE_FOR_nothing)
6868 return true;
6869 if (allow_libcall && optab_libfunc (sync_compare_and_swap_optab, mode))
6870 return true;
6871
6872 /* No inline compare and swap. */
6873 return false;
6874 }
6875
6876 /* Return true if an atomic exchange can be performed. */
6877
6878 bool
6879 can_atomic_exchange_p (enum machine_mode mode, bool allow_libcall)
6880 {
6881 enum insn_code icode;
6882
6883 /* Check for __atomic_exchange. */
6884 icode = direct_optab_handler (atomic_exchange_optab, mode);
6885 if (icode != CODE_FOR_nothing)
6886 return true;
6887
6888 /* Don't check __sync_test_and_set, as on some platforms that
6889 has reduced functionality. Targets that really do support
6890 a proper exchange should simply be updated to the __atomics. */
6891
6892 return can_compare_and_swap_p (mode, allow_libcall);
6893 }
6894
6895
6896 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
6897 pattern. */
6898
6899 static void
6900 find_cc_set (rtx x, const_rtx pat, void *data)
6901 {
6902 if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
6903 && GET_CODE (pat) == SET)
6904 {
6905 rtx *p_cc_reg = (rtx *) data;
6906 gcc_assert (!*p_cc_reg);
6907 *p_cc_reg = x;
6908 }
6909 }
6910
6911 /* This is a helper function for the other atomic operations. This function
6912 emits a loop that contains SEQ that iterates until a compare-and-swap
6913 operation at the end succeeds. MEM is the memory to be modified. SEQ is
6914 a set of instructions that takes a value from OLD_REG as an input and
6915 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
6916 set to the current contents of MEM. After SEQ, a compare-and-swap will
6917 attempt to update MEM with NEW_REG. The function returns true when the
6918 loop was generated successfully. */
6919
6920 static bool
6921 expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
6922 {
6923 enum machine_mode mode = GET_MODE (mem);
6924 rtx label, cmp_reg, success, oldval;
6925
6926 /* The loop we want to generate looks like
6927
6928 cmp_reg = mem;
6929 label:
6930 old_reg = cmp_reg;
6931 seq;
6932 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
6933 if (success)
6934 goto label;
6935
6936 Note that we only do the plain load from memory once. Subsequent
6937 iterations use the value loaded by the compare-and-swap pattern. */
6938
6939 label = gen_label_rtx ();
6940 cmp_reg = gen_reg_rtx (mode);
6941
6942 emit_move_insn (cmp_reg, mem);
6943 emit_label (label);
6944 emit_move_insn (old_reg, cmp_reg);
6945 if (seq)
6946 emit_insn (seq);
6947
6948 success = NULL_RTX;
6949 oldval = cmp_reg;
6950 if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
6951 new_reg, false, MEMMODEL_SEQ_CST,
6952 MEMMODEL_RELAXED))
6953 return false;
6954
6955 if (oldval != cmp_reg)
6956 emit_move_insn (cmp_reg, oldval);
6957
6958 /* ??? Mark this jump predicted not taken? */
6959 emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
6960 GET_MODE (success), 1, label);
6961 return true;
6962 }
6963
6964
6965 /* This function tries to emit an atomic_exchange intruction. VAL is written
6966 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
6967 using TARGET if possible. */
6968
6969 static rtx
6970 maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
6971 {
6972 enum machine_mode mode = GET_MODE (mem);
6973 enum insn_code icode;
6974
6975 /* If the target supports the exchange directly, great. */
6976 icode = direct_optab_handler (atomic_exchange_optab, mode);
6977 if (icode != CODE_FOR_nothing)
6978 {
6979 struct expand_operand ops[4];
6980
6981 create_output_operand (&ops[0], target, mode);
6982 create_fixed_operand (&ops[1], mem);
6983 /* VAL may have been promoted to a wider mode. Shrink it if so. */
6984 create_convert_operand_to (&ops[2], val, mode, true);
6985 create_integer_operand (&ops[3], model);
6986 if (maybe_expand_insn (icode, 4, ops))
6987 return ops[0].value;
6988 }
6989
6990 return NULL_RTX;
6991 }
6992
6993 /* This function tries to implement an atomic exchange operation using
6994 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
6995 The previous contents of *MEM are returned, using TARGET if possible.
6996 Since this instructionn is an acquire barrier only, stronger memory
6997 models may require additional barriers to be emitted. */
6998
6999 static rtx
7000 maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
7001 enum memmodel model)
7002 {
7003 enum machine_mode mode = GET_MODE (mem);
7004 enum insn_code icode;
7005 rtx last_insn = get_last_insn ();
7006
7007 icode = optab_handler (sync_lock_test_and_set_optab, mode);
7008
7009 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
7010 exists, and the memory model is stronger than acquire, add a release
7011 barrier before the instruction. */
7012
7013 if (model == MEMMODEL_SEQ_CST
7014 || model == MEMMODEL_RELEASE
7015 || model == MEMMODEL_ACQ_REL)
7016 expand_mem_thread_fence (model);
7017
7018 if (icode != CODE_FOR_nothing)
7019 {
7020 struct expand_operand ops[3];
7021 create_output_operand (&ops[0], target, mode);
7022 create_fixed_operand (&ops[1], mem);
7023 /* VAL may have been promoted to a wider mode. Shrink it if so. */
7024 create_convert_operand_to (&ops[2], val, mode, true);
7025 if (maybe_expand_insn (icode, 3, ops))
7026 return ops[0].value;
7027 }
7028
7029 /* If an external test-and-set libcall is provided, use that instead of
7030 any external compare-and-swap that we might get from the compare-and-
7031 swap-loop expansion later. */
7032 if (!can_compare_and_swap_p (mode, false))
7033 {
7034 rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
7035 if (libfunc != NULL)
7036 {
7037 rtx addr;
7038
7039 addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7040 return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7041 mode, 2, addr, ptr_mode,
7042 val, mode);
7043 }
7044 }
7045
7046 /* If the test_and_set can't be emitted, eliminate any barrier that might
7047 have been emitted. */
7048 delete_insns_since (last_insn);
7049 return NULL_RTX;
7050 }
7051
7052 /* This function tries to implement an atomic exchange operation using a
7053 compare_and_swap loop. VAL is written to *MEM. The previous contents of
7054 *MEM are returned, using TARGET if possible. No memory model is required
7055 since a compare_and_swap loop is seq-cst. */
7056
7057 static rtx
7058 maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
7059 {
7060 enum machine_mode mode = GET_MODE (mem);
7061
7062 if (can_compare_and_swap_p (mode, true))
7063 {
7064 if (!target || !register_operand (target, mode))
7065 target = gen_reg_rtx (mode);
7066 if (GET_MODE (val) != VOIDmode && GET_MODE (val) != mode)
7067 val = convert_modes (mode, GET_MODE (val), val, 1);
7068 if (expand_compare_and_swap_loop (mem, target, val, NULL_RTX))
7069 return target;
7070 }
7071
7072 return NULL_RTX;
7073 }
7074
7075 /* This function tries to implement an atomic test-and-set operation
7076 using the atomic_test_and_set instruction pattern. A boolean value
7077 is returned from the operation, using TARGET if possible. */
7078
7079 #ifndef HAVE_atomic_test_and_set
7080 #define HAVE_atomic_test_and_set 0
7081 #define CODE_FOR_atomic_test_and_set CODE_FOR_nothing
7082 #endif
7083
7084 static rtx
7085 maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7086 {
7087 enum machine_mode pat_bool_mode;
7088 struct expand_operand ops[3];
7089
7090 if (!HAVE_atomic_test_and_set)
7091 return NULL_RTX;
7092
7093 /* While we always get QImode from __atomic_test_and_set, we get
7094 other memory modes from __sync_lock_test_and_set. Note that we
7095 use no endian adjustment here. This matches the 4.6 behavior
7096 in the Sparc backend. */
7097 gcc_checking_assert
7098 (insn_data[CODE_FOR_atomic_test_and_set].operand[1].mode == QImode);
7099 if (GET_MODE (mem) != QImode)
7100 mem = adjust_address_nv (mem, QImode, 0);
7101
7102 pat_bool_mode = insn_data[CODE_FOR_atomic_test_and_set].operand[0].mode;
7103 create_output_operand (&ops[0], target, pat_bool_mode);
7104 create_fixed_operand (&ops[1], mem);
7105 create_integer_operand (&ops[2], model);
7106
7107 if (maybe_expand_insn (CODE_FOR_atomic_test_and_set, 3, ops))
7108 return ops[0].value;
7109 return NULL_RTX;
7110 }
7111
7112 /* This function expands the legacy _sync_lock test_and_set operation which is
7113 generally an atomic exchange. Some limited targets only allow the
7114 constant 1 to be stored. This is an ACQUIRE operation.
7115
7116 TARGET is an optional place to stick the return value.
7117 MEM is where VAL is stored. */
7118
7119 rtx
7120 expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
7121 {
7122 rtx ret;
7123
7124 /* Try an atomic_exchange first. */
7125 ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_ACQUIRE);
7126 if (ret)
7127 return ret;
7128
7129 ret = maybe_emit_sync_lock_test_and_set (target, mem, val, MEMMODEL_ACQUIRE);
7130 if (ret)
7131 return ret;
7132
7133 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7134 if (ret)
7135 return ret;
7136
7137 /* If there are no other options, try atomic_test_and_set if the value
7138 being stored is 1. */
7139 if (val == const1_rtx)
7140 ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_ACQUIRE);
7141
7142 return ret;
7143 }
7144
7145 /* This function expands the atomic test_and_set operation:
7146 atomically store a boolean TRUE into MEM and return the previous value.
7147
7148 MEMMODEL is the memory model variant to use.
7149 TARGET is an optional place to stick the return value. */
7150
7151 rtx
7152 expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7153 {
7154 enum machine_mode mode = GET_MODE (mem);
7155 rtx ret, trueval, subtarget;
7156
7157 ret = maybe_emit_atomic_test_and_set (target, mem, model);
7158 if (ret)
7159 return ret;
7160
7161 /* Be binary compatible with non-default settings of trueval, and different
7162 cpu revisions. E.g. one revision may have atomic-test-and-set, but
7163 another only has atomic-exchange. */
7164 if (targetm.atomic_test_and_set_trueval == 1)
7165 {
7166 trueval = const1_rtx;
7167 subtarget = target ? target : gen_reg_rtx (mode);
7168 }
7169 else
7170 {
7171 trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
7172 subtarget = gen_reg_rtx (mode);
7173 }
7174
7175 /* Try the atomic-exchange optab... */
7176 ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model);
7177
7178 /* ... then an atomic-compare-and-swap loop ... */
7179 if (!ret)
7180 ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval);
7181
7182 /* ... before trying the vaguely defined legacy lock_test_and_set. */
7183 if (!ret)
7184 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model);
7185
7186 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
7187 things with the value 1. Thus we try again without trueval. */
7188 if (!ret && targetm.atomic_test_and_set_trueval != 1)
7189 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model);
7190
7191 /* Failing all else, assume a single threaded environment and simply
7192 perform the operation. */
7193 if (!ret)
7194 {
7195 emit_move_insn (subtarget, mem);
7196 emit_move_insn (mem, trueval);
7197 ret = subtarget;
7198 }
7199
7200 /* Recall that have to return a boolean value; rectify if trueval
7201 is not exactly one. */
7202 if (targetm.atomic_test_and_set_trueval != 1)
7203 ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
7204
7205 return ret;
7206 }
7207
7208 /* This function expands the atomic exchange operation:
7209 atomically store VAL in MEM and return the previous value in MEM.
7210
7211 MEMMODEL is the memory model variant to use.
7212 TARGET is an optional place to stick the return value. */
7213
7214 rtx
7215 expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7216 {
7217 rtx ret;
7218
7219 ret = maybe_emit_atomic_exchange (target, mem, val, model);
7220
7221 /* Next try a compare-and-swap loop for the exchange. */
7222 if (!ret)
7223 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7224
7225 return ret;
7226 }
7227
7228 /* This function expands the atomic compare exchange operation:
7229
7230 *PTARGET_BOOL is an optional place to store the boolean success/failure.
7231 *PTARGET_OVAL is an optional place to store the old value from memory.
7232 Both target parameters may be NULL to indicate that we do not care about
7233 that return value. Both target parameters are updated on success to
7234 the actual location of the corresponding result.
7235
7236 MEMMODEL is the memory model variant to use.
7237
7238 The return value of the function is true for success. */
7239
7240 bool
7241 expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
7242 rtx mem, rtx expected, rtx desired,
7243 bool is_weak, enum memmodel succ_model,
7244 enum memmodel fail_model)
7245 {
7246 enum machine_mode mode = GET_MODE (mem);
7247 struct expand_operand ops[8];
7248 enum insn_code icode;
7249 rtx target_oval, target_bool = NULL_RTX;
7250 rtx libfunc;
7251
7252 /* Load expected into a register for the compare and swap. */
7253 if (MEM_P (expected))
7254 expected = copy_to_reg (expected);
7255
7256 /* Make sure we always have some place to put the return oldval.
7257 Further, make sure that place is distinct from the input expected,
7258 just in case we need that path down below. */
7259 if (ptarget_oval == NULL
7260 || (target_oval = *ptarget_oval) == NULL
7261 || reg_overlap_mentioned_p (expected, target_oval))
7262 target_oval = gen_reg_rtx (mode);
7263
7264 icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
7265 if (icode != CODE_FOR_nothing)
7266 {
7267 enum machine_mode bool_mode = insn_data[icode].operand[0].mode;
7268
7269 /* Make sure we always have a place for the bool operand. */
7270 if (ptarget_bool == NULL
7271 || (target_bool = *ptarget_bool) == NULL
7272 || GET_MODE (target_bool) != bool_mode)
7273 target_bool = gen_reg_rtx (bool_mode);
7274
7275 /* Emit the compare_and_swap. */
7276 create_output_operand (&ops[0], target_bool, bool_mode);
7277 create_output_operand (&ops[1], target_oval, mode);
7278 create_fixed_operand (&ops[2], mem);
7279 create_convert_operand_to (&ops[3], expected, mode, true);
7280 create_convert_operand_to (&ops[4], desired, mode, true);
7281 create_integer_operand (&ops[5], is_weak);
7282 create_integer_operand (&ops[6], succ_model);
7283 create_integer_operand (&ops[7], fail_model);
7284 expand_insn (icode, 8, ops);
7285
7286 /* Return success/failure. */
7287 target_bool = ops[0].value;
7288 target_oval = ops[1].value;
7289 goto success;
7290 }
7291
7292 /* Otherwise fall back to the original __sync_val_compare_and_swap
7293 which is always seq-cst. */
7294 icode = optab_handler (sync_compare_and_swap_optab, mode);
7295 if (icode != CODE_FOR_nothing)
7296 {
7297 rtx cc_reg;
7298
7299 create_output_operand (&ops[0], target_oval, mode);
7300 create_fixed_operand (&ops[1], mem);
7301 create_convert_operand_to (&ops[2], expected, mode, true);
7302 create_convert_operand_to (&ops[3], desired, mode, true);
7303 if (!maybe_expand_insn (icode, 4, ops))
7304 return false;
7305
7306 target_oval = ops[0].value;
7307
7308 /* If the caller isn't interested in the boolean return value,
7309 skip the computation of it. */
7310 if (ptarget_bool == NULL)
7311 goto success;
7312
7313 /* Otherwise, work out if the compare-and-swap succeeded. */
7314 cc_reg = NULL_RTX;
7315 if (have_insn_for (COMPARE, CCmode))
7316 note_stores (PATTERN (get_last_insn ()), find_cc_set, &cc_reg);
7317 if (cc_reg)
7318 {
7319 target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
7320 const0_rtx, VOIDmode, 0, 1);
7321 goto success;
7322 }
7323 goto success_bool_from_val;
7324 }
7325
7326 /* Also check for library support for __sync_val_compare_and_swap. */
7327 libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
7328 if (libfunc != NULL)
7329 {
7330 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7331 target_oval = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7332 mode, 3, addr, ptr_mode,
7333 expected, mode, desired, mode);
7334
7335 /* Compute the boolean return value only if requested. */
7336 if (ptarget_bool)
7337 goto success_bool_from_val;
7338 else
7339 goto success;
7340 }
7341
7342 /* Failure. */
7343 return false;
7344
7345 success_bool_from_val:
7346 target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
7347 expected, VOIDmode, 1, 1);
7348 success:
7349 /* Make sure that the oval output winds up where the caller asked. */
7350 if (ptarget_oval)
7351 *ptarget_oval = target_oval;
7352 if (ptarget_bool)
7353 *ptarget_bool = target_bool;
7354 return true;
7355 }
7356
7357 /* Generate asm volatile("" : : : "memory") as the memory barrier. */
7358
7359 static void
7360 expand_asm_memory_barrier (void)
7361 {
7362 rtx asm_op, clob;
7363
7364 asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, empty_string, empty_string, 0,
7365 rtvec_alloc (0), rtvec_alloc (0),
7366 rtvec_alloc (0), UNKNOWN_LOCATION);
7367 MEM_VOLATILE_P (asm_op) = 1;
7368
7369 clob = gen_rtx_SCRATCH (VOIDmode);
7370 clob = gen_rtx_MEM (BLKmode, clob);
7371 clob = gen_rtx_CLOBBER (VOIDmode, clob);
7372
7373 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
7374 }
7375
7376 /* This routine will either emit the mem_thread_fence pattern or issue a
7377 sync_synchronize to generate a fence for memory model MEMMODEL. */
7378
7379 #ifndef HAVE_mem_thread_fence
7380 # define HAVE_mem_thread_fence 0
7381 # define gen_mem_thread_fence(x) (gcc_unreachable (), NULL_RTX)
7382 #endif
7383 #ifndef HAVE_memory_barrier
7384 # define HAVE_memory_barrier 0
7385 # define gen_memory_barrier() (gcc_unreachable (), NULL_RTX)
7386 #endif
7387
7388 void
7389 expand_mem_thread_fence (enum memmodel model)
7390 {
7391 if (HAVE_mem_thread_fence)
7392 emit_insn (gen_mem_thread_fence (GEN_INT (model)));
7393 else if (model != MEMMODEL_RELAXED)
7394 {
7395 if (HAVE_memory_barrier)
7396 emit_insn (gen_memory_barrier ());
7397 else if (synchronize_libfunc != NULL_RTX)
7398 emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode, 0);
7399 else
7400 expand_asm_memory_barrier ();
7401 }
7402 }
7403
7404 /* This routine will either emit the mem_signal_fence pattern or issue a
7405 sync_synchronize to generate a fence for memory model MEMMODEL. */
7406
7407 #ifndef HAVE_mem_signal_fence
7408 # define HAVE_mem_signal_fence 0
7409 # define gen_mem_signal_fence(x) (gcc_unreachable (), NULL_RTX)
7410 #endif
7411
7412 void
7413 expand_mem_signal_fence (enum memmodel model)
7414 {
7415 if (HAVE_mem_signal_fence)
7416 emit_insn (gen_mem_signal_fence (GEN_INT (model)));
7417 else if (model != MEMMODEL_RELAXED)
7418 {
7419 /* By default targets are coherent between a thread and the signal
7420 handler running on the same thread. Thus this really becomes a
7421 compiler barrier, in that stores must not be sunk past
7422 (or raised above) a given point. */
7423 expand_asm_memory_barrier ();
7424 }
7425 }
7426
7427 /* This function expands the atomic load operation:
7428 return the atomically loaded value in MEM.
7429
7430 MEMMODEL is the memory model variant to use.
7431 TARGET is an option place to stick the return value. */
7432
7433 rtx
7434 expand_atomic_load (rtx target, rtx mem, enum memmodel model)
7435 {
7436 enum machine_mode mode = GET_MODE (mem);
7437 enum insn_code icode;
7438
7439 /* If the target supports the load directly, great. */
7440 icode = direct_optab_handler (atomic_load_optab, mode);
7441 if (icode != CODE_FOR_nothing)
7442 {
7443 struct expand_operand ops[3];
7444
7445 create_output_operand (&ops[0], target, mode);
7446 create_fixed_operand (&ops[1], mem);
7447 create_integer_operand (&ops[2], model);
7448 if (maybe_expand_insn (icode, 3, ops))
7449 return ops[0].value;
7450 }
7451
7452 /* If the size of the object is greater than word size on this target,
7453 then we assume that a load will not be atomic. */
7454 if (GET_MODE_PRECISION (mode) > BITS_PER_WORD)
7455 {
7456 /* Issue val = compare_and_swap (mem, 0, 0).
7457 This may cause the occasional harmless store of 0 when the value is
7458 already 0, but it seems to be OK according to the standards guys. */
7459 if (expand_atomic_compare_and_swap (NULL, &target, mem, const0_rtx,
7460 const0_rtx, false, model, model))
7461 return target;
7462 else
7463 /* Otherwise there is no atomic load, leave the library call. */
7464 return NULL_RTX;
7465 }
7466
7467 /* Otherwise assume loads are atomic, and emit the proper barriers. */
7468 if (!target || target == const0_rtx)
7469 target = gen_reg_rtx (mode);
7470
7471 /* Emit the appropriate barrier before the load. */
7472 expand_mem_thread_fence (model);
7473
7474 emit_move_insn (target, mem);
7475
7476 /* For SEQ_CST, also emit a barrier after the load. */
7477 if (model == MEMMODEL_SEQ_CST)
7478 expand_mem_thread_fence (model);
7479
7480 return target;
7481 }
7482
7483 /* This function expands the atomic store operation:
7484 Atomically store VAL in MEM.
7485 MEMMODEL is the memory model variant to use.
7486 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
7487 function returns const0_rtx if a pattern was emitted. */
7488
7489 rtx
7490 expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
7491 {
7492 enum machine_mode mode = GET_MODE (mem);
7493 enum insn_code icode;
7494 struct expand_operand ops[3];
7495
7496 /* If the target supports the store directly, great. */
7497 icode = direct_optab_handler (atomic_store_optab, mode);
7498 if (icode != CODE_FOR_nothing)
7499 {
7500 create_fixed_operand (&ops[0], mem);
7501 create_input_operand (&ops[1], val, mode);
7502 create_integer_operand (&ops[2], model);
7503 if (maybe_expand_insn (icode, 3, ops))
7504 return const0_rtx;
7505 }
7506
7507 /* If using __sync_lock_release is a viable alternative, try it. */
7508 if (use_release)
7509 {
7510 icode = direct_optab_handler (sync_lock_release_optab, mode);
7511 if (icode != CODE_FOR_nothing)
7512 {
7513 create_fixed_operand (&ops[0], mem);
7514 create_input_operand (&ops[1], const0_rtx, mode);
7515 if (maybe_expand_insn (icode, 2, ops))
7516 {
7517 /* lock_release is only a release barrier. */
7518 if (model == MEMMODEL_SEQ_CST)
7519 expand_mem_thread_fence (model);
7520 return const0_rtx;
7521 }
7522 }
7523 }
7524
7525 /* If the size of the object is greater than word size on this target,
7526 a default store will not be atomic, Try a mem_exchange and throw away
7527 the result. If that doesn't work, don't do anything. */
7528 if (GET_MODE_PRECISION(mode) > BITS_PER_WORD)
7529 {
7530 rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
7531 if (!target)
7532 target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem, val);
7533 if (target)
7534 return const0_rtx;
7535 else
7536 return NULL_RTX;
7537 }
7538
7539 /* If there is no mem_store, default to a move with barriers */
7540 if (model == MEMMODEL_SEQ_CST || model == MEMMODEL_RELEASE)
7541 expand_mem_thread_fence (model);
7542
7543 emit_move_insn (mem, val);
7544
7545 /* For SEQ_CST, also emit a barrier after the load. */
7546 if (model == MEMMODEL_SEQ_CST)
7547 expand_mem_thread_fence (model);
7548
7549 return const0_rtx;
7550 }
7551
7552
7553 /* Structure containing the pointers and values required to process the
7554 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
7555
7556 struct atomic_op_functions
7557 {
7558 direct_optab mem_fetch_before;
7559 direct_optab mem_fetch_after;
7560 direct_optab mem_no_result;
7561 optab fetch_before;
7562 optab fetch_after;
7563 direct_optab no_result;
7564 enum rtx_code reverse_code;
7565 };
7566
7567
7568 /* Fill in structure pointed to by OP with the various optab entries for an
7569 operation of type CODE. */
7570
7571 static void
7572 get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
7573 {
7574 gcc_assert (op!= NULL);
7575
7576 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
7577 in the source code during compilation, and the optab entries are not
7578 computable until runtime. Fill in the values at runtime. */
7579 switch (code)
7580 {
7581 case PLUS:
7582 op->mem_fetch_before = atomic_fetch_add_optab;
7583 op->mem_fetch_after = atomic_add_fetch_optab;
7584 op->mem_no_result = atomic_add_optab;
7585 op->fetch_before = sync_old_add_optab;
7586 op->fetch_after = sync_new_add_optab;
7587 op->no_result = sync_add_optab;
7588 op->reverse_code = MINUS;
7589 break;
7590 case MINUS:
7591 op->mem_fetch_before = atomic_fetch_sub_optab;
7592 op->mem_fetch_after = atomic_sub_fetch_optab;
7593 op->mem_no_result = atomic_sub_optab;
7594 op->fetch_before = sync_old_sub_optab;
7595 op->fetch_after = sync_new_sub_optab;
7596 op->no_result = sync_sub_optab;
7597 op->reverse_code = PLUS;
7598 break;
7599 case XOR:
7600 op->mem_fetch_before = atomic_fetch_xor_optab;
7601 op->mem_fetch_after = atomic_xor_fetch_optab;
7602 op->mem_no_result = atomic_xor_optab;
7603 op->fetch_before = sync_old_xor_optab;
7604 op->fetch_after = sync_new_xor_optab;
7605 op->no_result = sync_xor_optab;
7606 op->reverse_code = XOR;
7607 break;
7608 case AND:
7609 op->mem_fetch_before = atomic_fetch_and_optab;
7610 op->mem_fetch_after = atomic_and_fetch_optab;
7611 op->mem_no_result = atomic_and_optab;
7612 op->fetch_before = sync_old_and_optab;
7613 op->fetch_after = sync_new_and_optab;
7614 op->no_result = sync_and_optab;
7615 op->reverse_code = UNKNOWN;
7616 break;
7617 case IOR:
7618 op->mem_fetch_before = atomic_fetch_or_optab;
7619 op->mem_fetch_after = atomic_or_fetch_optab;
7620 op->mem_no_result = atomic_or_optab;
7621 op->fetch_before = sync_old_ior_optab;
7622 op->fetch_after = sync_new_ior_optab;
7623 op->no_result = sync_ior_optab;
7624 op->reverse_code = UNKNOWN;
7625 break;
7626 case NOT:
7627 op->mem_fetch_before = atomic_fetch_nand_optab;
7628 op->mem_fetch_after = atomic_nand_fetch_optab;
7629 op->mem_no_result = atomic_nand_optab;
7630 op->fetch_before = sync_old_nand_optab;
7631 op->fetch_after = sync_new_nand_optab;
7632 op->no_result = sync_nand_optab;
7633 op->reverse_code = UNKNOWN;
7634 break;
7635 default:
7636 gcc_unreachable ();
7637 }
7638 }
7639
7640 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
7641 using memory order MODEL. If AFTER is true the operation needs to return
7642 the value of *MEM after the operation, otherwise the previous value.
7643 TARGET is an optional place to place the result. The result is unused if
7644 it is const0_rtx.
7645 Return the result if there is a better sequence, otherwise NULL_RTX. */
7646
7647 static rtx
7648 maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
7649 enum memmodel model, bool after)
7650 {
7651 /* If the value is prefetched, or not used, it may be possible to replace
7652 the sequence with a native exchange operation. */
7653 if (!after || target == const0_rtx)
7654 {
7655 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
7656 if (code == AND && val == const0_rtx)
7657 {
7658 if (target == const0_rtx)
7659 target = gen_reg_rtx (GET_MODE (mem));
7660 return maybe_emit_atomic_exchange (target, mem, val, model);
7661 }
7662
7663 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
7664 if (code == IOR && val == constm1_rtx)
7665 {
7666 if (target == const0_rtx)
7667 target = gen_reg_rtx (GET_MODE (mem));
7668 return maybe_emit_atomic_exchange (target, mem, val, model);
7669 }
7670 }
7671
7672 return NULL_RTX;
7673 }
7674
7675 /* Try to emit an instruction for a specific operation varaition.
7676 OPTAB contains the OP functions.
7677 TARGET is an optional place to return the result. const0_rtx means unused.
7678 MEM is the memory location to operate on.
7679 VAL is the value to use in the operation.
7680 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
7681 MODEL is the memory model, if used.
7682 AFTER is true if the returned result is the value after the operation. */
7683
7684 static rtx
7685 maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
7686 rtx val, bool use_memmodel, enum memmodel model, bool after)
7687 {
7688 enum machine_mode mode = GET_MODE (mem);
7689 struct expand_operand ops[4];
7690 enum insn_code icode;
7691 int op_counter = 0;
7692 int num_ops;
7693
7694 /* Check to see if there is a result returned. */
7695 if (target == const0_rtx)
7696 {
7697 if (use_memmodel)
7698 {
7699 icode = direct_optab_handler (optab->mem_no_result, mode);
7700 create_integer_operand (&ops[2], model);
7701 num_ops = 3;
7702 }
7703 else
7704 {
7705 icode = direct_optab_handler (optab->no_result, mode);
7706 num_ops = 2;
7707 }
7708 }
7709 /* Otherwise, we need to generate a result. */
7710 else
7711 {
7712 if (use_memmodel)
7713 {
7714 icode = direct_optab_handler (after ? optab->mem_fetch_after
7715 : optab->mem_fetch_before, mode);
7716 create_integer_operand (&ops[3], model);
7717 num_ops = 4;
7718 }
7719 else
7720 {
7721 icode = optab_handler (after ? optab->fetch_after
7722 : optab->fetch_before, mode);
7723 num_ops = 3;
7724 }
7725 create_output_operand (&ops[op_counter++], target, mode);
7726 }
7727 if (icode == CODE_FOR_nothing)
7728 return NULL_RTX;
7729
7730 create_fixed_operand (&ops[op_counter++], mem);
7731 /* VAL may have been promoted to a wider mode. Shrink it if so. */
7732 create_convert_operand_to (&ops[op_counter++], val, mode, true);
7733
7734 if (maybe_expand_insn (icode, num_ops, ops))
7735 return (target == const0_rtx ? const0_rtx : ops[0].value);
7736
7737 return NULL_RTX;
7738 }
7739
7740
7741 /* This function expands an atomic fetch_OP or OP_fetch operation:
7742 TARGET is an option place to stick the return value. const0_rtx indicates
7743 the result is unused.
7744 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7745 CODE is the operation being performed (OP)
7746 MEMMODEL is the memory model variant to use.
7747 AFTER is true to return the result of the operation (OP_fetch).
7748 AFTER is false to return the value before the operation (fetch_OP). */
7749 rtx
7750 expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
7751 enum memmodel model, bool after)
7752 {
7753 enum machine_mode mode = GET_MODE (mem);
7754 struct atomic_op_functions optab;
7755 rtx result;
7756 bool unused_result = (target == const0_rtx);
7757
7758 get_atomic_op_for_code (&optab, code);
7759
7760 /* Check to see if there are any better instructions. */
7761 result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
7762 if (result)
7763 return result;
7764
7765 /* Check for the case where the result isn't used and try those patterns. */
7766 if (unused_result)
7767 {
7768 /* Try the memory model variant first. */
7769 result = maybe_emit_op (&optab, target, mem, val, true, model, true);
7770 if (result)
7771 return result;
7772
7773 /* Next try the old style withuot a memory model. */
7774 result = maybe_emit_op (&optab, target, mem, val, false, model, true);
7775 if (result)
7776 return result;
7777
7778 /* There is no no-result pattern, so try patterns with a result. */
7779 target = NULL_RTX;
7780 }
7781
7782 /* Try the __atomic version. */
7783 result = maybe_emit_op (&optab, target, mem, val, true, model, after);
7784 if (result)
7785 return result;
7786
7787 /* Try the older __sync version. */
7788 result = maybe_emit_op (&optab, target, mem, val, false, model, after);
7789 if (result)
7790 return result;
7791
7792 /* If the fetch value can be calculated from the other variation of fetch,
7793 try that operation. */
7794 if (after || unused_result || optab.reverse_code != UNKNOWN)
7795 {
7796 /* Try the __atomic version, then the older __sync version. */
7797 result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
7798 if (!result)
7799 result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
7800
7801 if (result)
7802 {
7803 /* If the result isn't used, no need to do compensation code. */
7804 if (unused_result)
7805 return result;
7806
7807 /* Issue compensation code. Fetch_after == fetch_before OP val.
7808 Fetch_before == after REVERSE_OP val. */
7809 if (!after)
7810 code = optab.reverse_code;
7811 if (code == NOT)
7812 {
7813 result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
7814 true, OPTAB_LIB_WIDEN);
7815 result = expand_simple_unop (mode, NOT, result, target, true);
7816 }
7817 else
7818 result = expand_simple_binop (mode, code, result, val, target,
7819 true, OPTAB_LIB_WIDEN);
7820 return result;
7821 }
7822 }
7823
7824 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
7825 if (!can_compare_and_swap_p (mode, false))
7826 {
7827 rtx libfunc;
7828 bool fixup = false;
7829
7830 libfunc = optab_libfunc (after ? optab.fetch_after
7831 : optab.fetch_before, mode);
7832 if (libfunc == NULL
7833 && (after || unused_result || optab.reverse_code != UNKNOWN))
7834 {
7835 fixup = true;
7836 if (!after)
7837 code = optab.reverse_code;
7838 libfunc = optab_libfunc (after ? optab.fetch_before
7839 : optab.fetch_after, mode);
7840 }
7841 if (libfunc != NULL)
7842 {
7843 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7844 result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
7845 2, addr, ptr_mode, val, mode);
7846
7847 if (!unused_result && fixup)
7848 result = expand_simple_binop (mode, code, result, val, target,
7849 true, OPTAB_LIB_WIDEN);
7850 return result;
7851 }
7852 }
7853
7854 /* If nothing else has succeeded, default to a compare and swap loop. */
7855 if (can_compare_and_swap_p (mode, true))
7856 {
7857 rtx insn;
7858 rtx t0 = gen_reg_rtx (mode), t1;
7859
7860 start_sequence ();
7861
7862 /* If the result is used, get a register for it. */
7863 if (!unused_result)
7864 {
7865 if (!target || !register_operand (target, mode))
7866 target = gen_reg_rtx (mode);
7867 /* If fetch_before, copy the value now. */
7868 if (!after)
7869 emit_move_insn (target, t0);
7870 }
7871 else
7872 target = const0_rtx;
7873
7874 t1 = t0;
7875 if (code == NOT)
7876 {
7877 t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
7878 true, OPTAB_LIB_WIDEN);
7879 t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
7880 }
7881 else
7882 t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
7883 OPTAB_LIB_WIDEN);
7884
7885 /* For after, copy the value now. */
7886 if (!unused_result && after)
7887 emit_move_insn (target, t1);
7888 insn = get_insns ();
7889 end_sequence ();
7890
7891 if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
7892 return target;
7893 }
7894
7895 return NULL_RTX;
7896 }
7897 \f
7898 /* Return true if OPERAND is suitable for operand number OPNO of
7899 instruction ICODE. */
7900
7901 bool
7902 insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
7903 {
7904 return (!insn_data[(int) icode].operand[opno].predicate
7905 || (insn_data[(int) icode].operand[opno].predicate
7906 (operand, insn_data[(int) icode].operand[opno].mode)));
7907 }
7908 \f
7909 /* TARGET is a target of a multiword operation that we are going to
7910 implement as a series of word-mode operations. Return true if
7911 TARGET is suitable for this purpose. */
7912
7913 bool
7914 valid_multiword_target_p (rtx target)
7915 {
7916 enum machine_mode mode;
7917 int i;
7918
7919 mode = GET_MODE (target);
7920 for (i = 0; i < GET_MODE_SIZE (mode); i += UNITS_PER_WORD)
7921 if (!validate_subreg (word_mode, mode, target, i))
7922 return false;
7923 return true;
7924 }
7925
7926 /* Like maybe_legitimize_operand, but do not change the code of the
7927 current rtx value. */
7928
7929 static bool
7930 maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
7931 struct expand_operand *op)
7932 {
7933 /* See if the operand matches in its current form. */
7934 if (insn_operand_matches (icode, opno, op->value))
7935 return true;
7936
7937 /* If the operand is a memory whose address has no side effects,
7938 try forcing the address into a non-virtual pseudo register.
7939 The check for side effects is important because copy_to_mode_reg
7940 cannot handle things like auto-modified addresses. */
7941 if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
7942 {
7943 rtx addr, mem;
7944
7945 mem = op->value;
7946 addr = XEXP (mem, 0);
7947 if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
7948 && !side_effects_p (addr))
7949 {
7950 rtx last;
7951 enum machine_mode mode;
7952
7953 last = get_last_insn ();
7954 mode = get_address_mode (mem);
7955 mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
7956 if (insn_operand_matches (icode, opno, mem))
7957 {
7958 op->value = mem;
7959 return true;
7960 }
7961 delete_insns_since (last);
7962 }
7963 }
7964
7965 return false;
7966 }
7967
7968 /* Try to make OP match operand OPNO of instruction ICODE. Return true
7969 on success, storing the new operand value back in OP. */
7970
7971 static bool
7972 maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
7973 struct expand_operand *op)
7974 {
7975 enum machine_mode mode, imode;
7976 bool old_volatile_ok, result;
7977
7978 mode = op->mode;
7979 switch (op->type)
7980 {
7981 case EXPAND_FIXED:
7982 old_volatile_ok = volatile_ok;
7983 volatile_ok = true;
7984 result = maybe_legitimize_operand_same_code (icode, opno, op);
7985 volatile_ok = old_volatile_ok;
7986 return result;
7987
7988 case EXPAND_OUTPUT:
7989 gcc_assert (mode != VOIDmode);
7990 if (op->value
7991 && op->value != const0_rtx
7992 && GET_MODE (op->value) == mode
7993 && maybe_legitimize_operand_same_code (icode, opno, op))
7994 return true;
7995
7996 op->value = gen_reg_rtx (mode);
7997 break;
7998
7999 case EXPAND_INPUT:
8000 input:
8001 gcc_assert (mode != VOIDmode);
8002 gcc_assert (GET_MODE (op->value) == VOIDmode
8003 || GET_MODE (op->value) == mode);
8004 if (maybe_legitimize_operand_same_code (icode, opno, op))
8005 return true;
8006
8007 op->value = copy_to_mode_reg (mode, op->value);
8008 break;
8009
8010 case EXPAND_CONVERT_TO:
8011 gcc_assert (mode != VOIDmode);
8012 op->value = convert_to_mode (mode, op->value, op->unsigned_p);
8013 goto input;
8014
8015 case EXPAND_CONVERT_FROM:
8016 if (GET_MODE (op->value) != VOIDmode)
8017 mode = GET_MODE (op->value);
8018 else
8019 /* The caller must tell us what mode this value has. */
8020 gcc_assert (mode != VOIDmode);
8021
8022 imode = insn_data[(int) icode].operand[opno].mode;
8023 if (imode != VOIDmode && imode != mode)
8024 {
8025 op->value = convert_modes (imode, mode, op->value, op->unsigned_p);
8026 mode = imode;
8027 }
8028 goto input;
8029
8030 case EXPAND_ADDRESS:
8031 gcc_assert (mode != VOIDmode);
8032 op->value = convert_memory_address (mode, op->value);
8033 goto input;
8034
8035 case EXPAND_INTEGER:
8036 mode = insn_data[(int) icode].operand[opno].mode;
8037 if (mode != VOIDmode && const_int_operand (op->value, mode))
8038 goto input;
8039 break;
8040 }
8041 return insn_operand_matches (icode, opno, op->value);
8042 }
8043
8044 /* Make OP describe an input operand that should have the same value
8045 as VALUE, after any mode conversion that the target might request.
8046 TYPE is the type of VALUE. */
8047
8048 void
8049 create_convert_operand_from_type (struct expand_operand *op,
8050 rtx value, tree type)
8051 {
8052 create_convert_operand_from (op, value, TYPE_MODE (type),
8053 TYPE_UNSIGNED (type));
8054 }
8055
8056 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
8057 of instruction ICODE. Return true on success, leaving the new operand
8058 values in the OPS themselves. Emit no code on failure. */
8059
8060 bool
8061 maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
8062 unsigned int nops, struct expand_operand *ops)
8063 {
8064 rtx last;
8065 unsigned int i;
8066
8067 last = get_last_insn ();
8068 for (i = 0; i < nops; i++)
8069 if (!maybe_legitimize_operand (icode, opno + i, &ops[i]))
8070 {
8071 delete_insns_since (last);
8072 return false;
8073 }
8074 return true;
8075 }
8076
8077 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
8078 as its operands. Return the instruction pattern on success,
8079 and emit any necessary set-up code. Return null and emit no
8080 code on failure. */
8081
8082 rtx
8083 maybe_gen_insn (enum insn_code icode, unsigned int nops,
8084 struct expand_operand *ops)
8085 {
8086 gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
8087 if (!maybe_legitimize_operands (icode, 0, nops, ops))
8088 return NULL_RTX;
8089
8090 switch (nops)
8091 {
8092 case 1:
8093 return GEN_FCN (icode) (ops[0].value);
8094 case 2:
8095 return GEN_FCN (icode) (ops[0].value, ops[1].value);
8096 case 3:
8097 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
8098 case 4:
8099 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8100 ops[3].value);
8101 case 5:
8102 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8103 ops[3].value, ops[4].value);
8104 case 6:
8105 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8106 ops[3].value, ops[4].value, ops[5].value);
8107 case 7:
8108 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8109 ops[3].value, ops[4].value, ops[5].value,
8110 ops[6].value);
8111 case 8:
8112 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8113 ops[3].value, ops[4].value, ops[5].value,
8114 ops[6].value, ops[7].value);
8115 }
8116 gcc_unreachable ();
8117 }
8118
8119 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
8120 as its operands. Return true on success and emit no code on failure. */
8121
8122 bool
8123 maybe_expand_insn (enum insn_code icode, unsigned int nops,
8124 struct expand_operand *ops)
8125 {
8126 rtx pat = maybe_gen_insn (icode, nops, ops);
8127 if (pat)
8128 {
8129 emit_insn (pat);
8130 return true;
8131 }
8132 return false;
8133 }
8134
8135 /* Like maybe_expand_insn, but for jumps. */
8136
8137 bool
8138 maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
8139 struct expand_operand *ops)
8140 {
8141 rtx pat = maybe_gen_insn (icode, nops, ops);
8142 if (pat)
8143 {
8144 emit_jump_insn (pat);
8145 return true;
8146 }
8147 return false;
8148 }
8149
8150 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
8151 as its operands. */
8152
8153 void
8154 expand_insn (enum insn_code icode, unsigned int nops,
8155 struct expand_operand *ops)
8156 {
8157 if (!maybe_expand_insn (icode, nops, ops))
8158 gcc_unreachable ();
8159 }
8160
8161 /* Like expand_insn, but for jumps. */
8162
8163 void
8164 expand_jump_insn (enum insn_code icode, unsigned int nops,
8165 struct expand_operand *ops)
8166 {
8167 if (!maybe_expand_jump_insn (icode, nops, ops))
8168 gcc_unreachable ();
8169 }
8170
8171 #include "gt-optabs.h"