Fix ICE on unsupported FP comparison
[gcc.git] / gcc / optabs.c
1 /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 Copyright (C) 1987-2020 Free Software Foundation, Inc.
3
4 This file is part of GCC.
5
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
9 version.
10
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 for more details.
15
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
19
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "backend.h"
25 #include "target.h"
26 #include "rtl.h"
27 #include "tree.h"
28 #include "memmodel.h"
29 #include "predict.h"
30 #include "tm_p.h"
31 #include "expmed.h"
32 #include "optabs.h"
33 #include "emit-rtl.h"
34 #include "recog.h"
35 #include "diagnostic-core.h"
36 #include "rtx-vector-builder.h"
37
38 /* Include insn-config.h before expr.h so that HAVE_conditional_move
39 is properly defined. */
40 #include "stor-layout.h"
41 #include "except.h"
42 #include "dojump.h"
43 #include "explow.h"
44 #include "expr.h"
45 #include "optabs-tree.h"
46 #include "libfuncs.h"
47
48 static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
49 machine_mode *);
50 static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int);
51 static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool);
52
53 /* Debug facility for use in GDB. */
54 void debug_optab_libfuncs (void);
55 \f
56 /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
57 the result of operation CODE applied to OP0 (and OP1 if it is a binary
58 operation). OP0_MODE is OP0's mode.
59
60 If the last insn does not set TARGET, don't do anything, but return 1.
61
62 If the last insn or a previous insn sets TARGET and TARGET is one of OP0
63 or OP1, don't add the REG_EQUAL note but return 0. Our caller can then
64 try again, ensuring that TARGET is not one of the operands. */
65
66 static int
67 add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0,
68 rtx op1, machine_mode op0_mode)
69 {
70 rtx_insn *last_insn;
71 rtx set;
72 rtx note;
73
74 gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
75
76 if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
77 && GET_RTX_CLASS (code) != RTX_BIN_ARITH
78 && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
79 && GET_RTX_CLASS (code) != RTX_COMPARE
80 && GET_RTX_CLASS (code) != RTX_UNARY)
81 return 1;
82
83 if (GET_CODE (target) == ZERO_EXTRACT)
84 return 1;
85
86 for (last_insn = insns;
87 NEXT_INSN (last_insn) != NULL_RTX;
88 last_insn = NEXT_INSN (last_insn))
89 ;
90
91 /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
92 a value changing in the insn, so the note would be invalid for CSE. */
93 if (reg_overlap_mentioned_p (target, op0)
94 || (op1 && reg_overlap_mentioned_p (target, op1)))
95 {
96 if (MEM_P (target)
97 && (rtx_equal_p (target, op0)
98 || (op1 && rtx_equal_p (target, op1))))
99 {
100 /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
101 over expanding it as temp = MEM op X, MEM = temp. If the target
102 supports MEM = MEM op X instructions, it is sometimes too hard
103 to reconstruct that form later, especially if X is also a memory,
104 and due to multiple occurrences of addresses the address might
105 be forced into register unnecessarily.
106 Note that not emitting the REG_EQUIV note might inhibit
107 CSE in some cases. */
108 set = single_set (last_insn);
109 if (set
110 && GET_CODE (SET_SRC (set)) == code
111 && MEM_P (SET_DEST (set))
112 && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
113 || (op1 && rtx_equal_p (SET_DEST (set),
114 XEXP (SET_SRC (set), 1)))))
115 return 1;
116 }
117 return 0;
118 }
119
120 set = set_for_reg_notes (last_insn);
121 if (set == NULL_RTX)
122 return 1;
123
124 if (! rtx_equal_p (SET_DEST (set), target)
125 /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
126 && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
127 || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
128 return 1;
129
130 if (GET_RTX_CLASS (code) == RTX_UNARY)
131 switch (code)
132 {
133 case FFS:
134 case CLZ:
135 case CTZ:
136 case CLRSB:
137 case POPCOUNT:
138 case PARITY:
139 case BSWAP:
140 if (op0_mode != VOIDmode && GET_MODE (target) != op0_mode)
141 {
142 note = gen_rtx_fmt_e (code, op0_mode, copy_rtx (op0));
143 if (GET_MODE_UNIT_SIZE (op0_mode)
144 > GET_MODE_UNIT_SIZE (GET_MODE (target)))
145 note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
146 note, op0_mode);
147 else
148 note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
149 note, op0_mode);
150 break;
151 }
152 /* FALLTHRU */
153 default:
154 note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
155 break;
156 }
157 else
158 note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
159
160 set_unique_reg_note (last_insn, REG_EQUAL, note);
161
162 return 1;
163 }
164 \f
165 /* Given two input operands, OP0 and OP1, determine what the correct from_mode
166 for a widening operation would be. In most cases this would be OP0, but if
167 that's a constant it'll be VOIDmode, which isn't useful. */
168
169 static machine_mode
170 widened_mode (machine_mode to_mode, rtx op0, rtx op1)
171 {
172 machine_mode m0 = GET_MODE (op0);
173 machine_mode m1 = GET_MODE (op1);
174 machine_mode result;
175
176 if (m0 == VOIDmode && m1 == VOIDmode)
177 return to_mode;
178 else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1))
179 result = m1;
180 else
181 result = m0;
182
183 if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode))
184 return to_mode;
185
186 return result;
187 }
188 \f
189 /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
190 says whether OP is signed or unsigned. NO_EXTEND is nonzero if we need
191 not actually do a sign-extend or zero-extend, but can leave the
192 higher-order bits of the result rtx undefined, for example, in the case
193 of logical operations, but not right shifts. */
194
195 static rtx
196 widen_operand (rtx op, machine_mode mode, machine_mode oldmode,
197 int unsignedp, int no_extend)
198 {
199 rtx result;
200 scalar_int_mode int_mode;
201
202 /* If we don't have to extend and this is a constant, return it. */
203 if (no_extend && GET_MODE (op) == VOIDmode)
204 return op;
205
206 /* If we must extend do so. If OP is a SUBREG for a promoted object, also
207 extend since it will be more efficient to do so unless the signedness of
208 a promoted object differs from our extension. */
209 if (! no_extend
210 || !is_a <scalar_int_mode> (mode, &int_mode)
211 || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
212 && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
213 return convert_modes (mode, oldmode, op, unsignedp);
214
215 /* If MODE is no wider than a single word, we return a lowpart or paradoxical
216 SUBREG. */
217 if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD)
218 return gen_lowpart (int_mode, force_reg (GET_MODE (op), op));
219
220 /* Otherwise, get an object of MODE, clobber it, and set the low-order
221 part to OP. */
222
223 result = gen_reg_rtx (int_mode);
224 emit_clobber (result);
225 emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
226 return result;
227 }
228 \f
229 /* Expand vector widening operations.
230
231 There are two different classes of operations handled here:
232 1) Operations whose result is wider than all the arguments to the operation.
233 Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
234 In this case OP0 and optionally OP1 would be initialized,
235 but WIDE_OP wouldn't (not relevant for this case).
236 2) Operations whose result is of the same size as the last argument to the
237 operation, but wider than all the other arguments to the operation.
238 Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
239 In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
240
241 E.g, when called to expand the following operations, this is how
242 the arguments will be initialized:
243 nops OP0 OP1 WIDE_OP
244 widening-sum 2 oprnd0 - oprnd1
245 widening-dot-product 3 oprnd0 oprnd1 oprnd2
246 widening-mult 2 oprnd0 oprnd1 -
247 type-promotion (vec-unpack) 1 oprnd0 - - */
248
249 rtx
250 expand_widen_pattern_expr (sepops ops, rtx op0, rtx op1, rtx wide_op,
251 rtx target, int unsignedp)
252 {
253 class expand_operand eops[4];
254 tree oprnd0, oprnd1, oprnd2;
255 machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
256 optab widen_pattern_optab;
257 enum insn_code icode;
258 int nops = TREE_CODE_LENGTH (ops->code);
259 int op;
260 bool sbool = false;
261
262 oprnd0 = ops->op0;
263 tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
264 if (ops->code == VEC_UNPACK_FIX_TRUNC_HI_EXPR
265 || ops->code == VEC_UNPACK_FIX_TRUNC_LO_EXPR)
266 /* The sign is from the result type rather than operand's type
267 for these ops. */
268 widen_pattern_optab
269 = optab_for_tree_code (ops->code, ops->type, optab_default);
270 else if ((ops->code == VEC_UNPACK_HI_EXPR
271 || ops->code == VEC_UNPACK_LO_EXPR)
272 && VECTOR_BOOLEAN_TYPE_P (ops->type)
273 && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0))
274 && TYPE_MODE (ops->type) == TYPE_MODE (TREE_TYPE (oprnd0))
275 && SCALAR_INT_MODE_P (TYPE_MODE (ops->type)))
276 {
277 /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
278 the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
279 vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
280 the pattern number of elements in the wider vector. */
281 widen_pattern_optab
282 = (ops->code == VEC_UNPACK_HI_EXPR
283 ? vec_unpacks_sbool_hi_optab : vec_unpacks_sbool_lo_optab);
284 sbool = true;
285 }
286 else
287 widen_pattern_optab
288 = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
289 if (ops->code == WIDEN_MULT_PLUS_EXPR
290 || ops->code == WIDEN_MULT_MINUS_EXPR)
291 icode = find_widening_optab_handler (widen_pattern_optab,
292 TYPE_MODE (TREE_TYPE (ops->op2)),
293 tmode0);
294 else
295 icode = optab_handler (widen_pattern_optab, tmode0);
296 gcc_assert (icode != CODE_FOR_nothing);
297
298 if (nops >= 2)
299 {
300 oprnd1 = ops->op1;
301 tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
302 }
303 else if (sbool)
304 {
305 nops = 2;
306 op1 = GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0)).to_constant ());
307 tmode1 = tmode0;
308 }
309
310 /* The last operand is of a wider mode than the rest of the operands. */
311 if (nops == 2)
312 wmode = tmode1;
313 else if (nops == 3)
314 {
315 gcc_assert (tmode1 == tmode0);
316 gcc_assert (op1);
317 oprnd2 = ops->op2;
318 wmode = TYPE_MODE (TREE_TYPE (oprnd2));
319 }
320
321 op = 0;
322 create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
323 create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
324 if (op1)
325 create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
326 if (wide_op)
327 create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
328 expand_insn (icode, op, eops);
329 return eops[0].value;
330 }
331
332 /* Generate code to perform an operation specified by TERNARY_OPTAB
333 on operands OP0, OP1 and OP2, with result having machine-mode MODE.
334
335 UNSIGNEDP is for the case where we have to widen the operands
336 to perform the operation. It says to use zero-extension.
337
338 If TARGET is nonzero, the value
339 is generated there, if it is convenient to do so.
340 In all cases an rtx is returned for the locus of the value;
341 this may or may not be TARGET. */
342
343 rtx
344 expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0,
345 rtx op1, rtx op2, rtx target, int unsignedp)
346 {
347 class expand_operand ops[4];
348 enum insn_code icode = optab_handler (ternary_optab, mode);
349
350 gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
351
352 create_output_operand (&ops[0], target, mode);
353 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
354 create_convert_operand_from (&ops[2], op1, mode, unsignedp);
355 create_convert_operand_from (&ops[3], op2, mode, unsignedp);
356 expand_insn (icode, 4, ops);
357 return ops[0].value;
358 }
359
360
361 /* Like expand_binop, but return a constant rtx if the result can be
362 calculated at compile time. The arguments and return value are
363 otherwise the same as for expand_binop. */
364
365 rtx
366 simplify_expand_binop (machine_mode mode, optab binoptab,
367 rtx op0, rtx op1, rtx target, int unsignedp,
368 enum optab_methods methods)
369 {
370 if (CONSTANT_P (op0) && CONSTANT_P (op1))
371 {
372 rtx x = simplify_binary_operation (optab_to_code (binoptab),
373 mode, op0, op1);
374 if (x)
375 return x;
376 }
377
378 return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
379 }
380
381 /* Like simplify_expand_binop, but always put the result in TARGET.
382 Return true if the expansion succeeded. */
383
384 bool
385 force_expand_binop (machine_mode mode, optab binoptab,
386 rtx op0, rtx op1, rtx target, int unsignedp,
387 enum optab_methods methods)
388 {
389 rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
390 target, unsignedp, methods);
391 if (x == 0)
392 return false;
393 if (x != target)
394 emit_move_insn (target, x);
395 return true;
396 }
397
398 /* Create a new vector value in VMODE with all elements set to OP. The
399 mode of OP must be the element mode of VMODE. If OP is a constant,
400 then the return value will be a constant. */
401
402 rtx
403 expand_vector_broadcast (machine_mode vmode, rtx op)
404 {
405 int n;
406 rtvec vec;
407
408 gcc_checking_assert (VECTOR_MODE_P (vmode));
409
410 if (valid_for_const_vector_p (vmode, op))
411 return gen_const_vec_duplicate (vmode, op);
412
413 insn_code icode = optab_handler (vec_duplicate_optab, vmode);
414 if (icode != CODE_FOR_nothing)
415 {
416 class expand_operand ops[2];
417 create_output_operand (&ops[0], NULL_RTX, vmode);
418 create_input_operand (&ops[1], op, GET_MODE (op));
419 expand_insn (icode, 2, ops);
420 return ops[0].value;
421 }
422
423 if (!GET_MODE_NUNITS (vmode).is_constant (&n))
424 return NULL;
425
426 /* ??? If the target doesn't have a vec_init, then we have no easy way
427 of performing this operation. Most of this sort of generic support
428 is hidden away in the vector lowering support in gimple. */
429 icode = convert_optab_handler (vec_init_optab, vmode,
430 GET_MODE_INNER (vmode));
431 if (icode == CODE_FOR_nothing)
432 return NULL;
433
434 vec = rtvec_alloc (n);
435 for (int i = 0; i < n; ++i)
436 RTVEC_ELT (vec, i) = op;
437 rtx ret = gen_reg_rtx (vmode);
438 emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
439
440 return ret;
441 }
442
443 /* This subroutine of expand_doubleword_shift handles the cases in which
444 the effective shift value is >= BITS_PER_WORD. The arguments and return
445 value are the same as for the parent routine, except that SUPERWORD_OP1
446 is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
447 INTO_TARGET may be null if the caller has decided to calculate it. */
448
449 static bool
450 expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
451 rtx outof_target, rtx into_target,
452 int unsignedp, enum optab_methods methods)
453 {
454 if (into_target != 0)
455 if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
456 into_target, unsignedp, methods))
457 return false;
458
459 if (outof_target != 0)
460 {
461 /* For a signed right shift, we must fill OUTOF_TARGET with copies
462 of the sign bit, otherwise we must fill it with zeros. */
463 if (binoptab != ashr_optab)
464 emit_move_insn (outof_target, CONST0_RTX (word_mode));
465 else
466 if (!force_expand_binop (word_mode, binoptab, outof_input,
467 gen_int_shift_amount (word_mode,
468 BITS_PER_WORD - 1),
469 outof_target, unsignedp, methods))
470 return false;
471 }
472 return true;
473 }
474
475 /* This subroutine of expand_doubleword_shift handles the cases in which
476 the effective shift value is < BITS_PER_WORD. The arguments and return
477 value are the same as for the parent routine. */
478
479 static bool
480 expand_subword_shift (scalar_int_mode op1_mode, optab binoptab,
481 rtx outof_input, rtx into_input, rtx op1,
482 rtx outof_target, rtx into_target,
483 int unsignedp, enum optab_methods methods,
484 unsigned HOST_WIDE_INT shift_mask)
485 {
486 optab reverse_unsigned_shift, unsigned_shift;
487 rtx tmp, carries;
488
489 reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
490 unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
491
492 /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
493 We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
494 the opposite direction to BINOPTAB. */
495 if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
496 {
497 carries = outof_input;
498 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
499 op1_mode), op1_mode);
500 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
501 0, true, methods);
502 }
503 else
504 {
505 /* We must avoid shifting by BITS_PER_WORD bits since that is either
506 the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
507 has unknown behavior. Do a single shift first, then shift by the
508 remainder. It's OK to use ~OP1 as the remainder if shift counts
509 are truncated to the mode size. */
510 carries = expand_binop (word_mode, reverse_unsigned_shift,
511 outof_input, const1_rtx, 0, unsignedp, methods);
512 if (shift_mask == BITS_PER_WORD - 1)
513 {
514 tmp = immed_wide_int_const
515 (wi::minus_one (GET_MODE_PRECISION (op1_mode)), op1_mode);
516 tmp = simplify_expand_binop (op1_mode, xor_optab, op1, tmp,
517 0, true, methods);
518 }
519 else
520 {
521 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
522 op1_mode), op1_mode);
523 tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
524 0, true, methods);
525 }
526 }
527 if (tmp == 0 || carries == 0)
528 return false;
529 carries = expand_binop (word_mode, reverse_unsigned_shift,
530 carries, tmp, 0, unsignedp, methods);
531 if (carries == 0)
532 return false;
533
534 /* Shift INTO_INPUT logically by OP1. This is the last use of INTO_INPUT
535 so the result can go directly into INTO_TARGET if convenient. */
536 tmp = expand_binop (word_mode, unsigned_shift, into_input, op1,
537 into_target, unsignedp, methods);
538 if (tmp == 0)
539 return false;
540
541 /* Now OR in the bits carried over from OUTOF_INPUT. */
542 if (!force_expand_binop (word_mode, ior_optab, tmp, carries,
543 into_target, unsignedp, methods))
544 return false;
545
546 /* Use a standard word_mode shift for the out-of half. */
547 if (outof_target != 0)
548 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
549 outof_target, unsignedp, methods))
550 return false;
551
552 return true;
553 }
554
555
556 /* Try implementing expand_doubleword_shift using conditional moves.
557 The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
558 otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
559 are the shift counts to use in the former and latter case. All other
560 arguments are the same as the parent routine. */
561
562 static bool
563 expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab,
564 enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
565 rtx outof_input, rtx into_input,
566 rtx subword_op1, rtx superword_op1,
567 rtx outof_target, rtx into_target,
568 int unsignedp, enum optab_methods methods,
569 unsigned HOST_WIDE_INT shift_mask)
570 {
571 rtx outof_superword, into_superword;
572
573 /* Put the superword version of the output into OUTOF_SUPERWORD and
574 INTO_SUPERWORD. */
575 outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
576 if (outof_target != 0 && subword_op1 == superword_op1)
577 {
578 /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
579 OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
580 into_superword = outof_target;
581 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
582 outof_superword, 0, unsignedp, methods))
583 return false;
584 }
585 else
586 {
587 into_superword = gen_reg_rtx (word_mode);
588 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
589 outof_superword, into_superword,
590 unsignedp, methods))
591 return false;
592 }
593
594 /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
595 if (!expand_subword_shift (op1_mode, binoptab,
596 outof_input, into_input, subword_op1,
597 outof_target, into_target,
598 unsignedp, methods, shift_mask))
599 return false;
600
601 /* Select between them. Do the INTO half first because INTO_SUPERWORD
602 might be the current value of OUTOF_TARGET. */
603 if (!emit_conditional_move (into_target, cmp_code, cmp1, cmp2, op1_mode,
604 into_target, into_superword, word_mode, false))
605 return false;
606
607 if (outof_target != 0)
608 if (!emit_conditional_move (outof_target, cmp_code, cmp1, cmp2, op1_mode,
609 outof_target, outof_superword,
610 word_mode, false))
611 return false;
612
613 return true;
614 }
615
616 /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
617 OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
618 input operand; the shift moves bits in the direction OUTOF_INPUT->
619 INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
620 of the target. OP1 is the shift count and OP1_MODE is its mode.
621 If OP1 is constant, it will have been truncated as appropriate
622 and is known to be nonzero.
623
624 If SHIFT_MASK is zero, the result of word shifts is undefined when the
625 shift count is outside the range [0, BITS_PER_WORD). This routine must
626 avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
627
628 If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
629 masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
630 fill with zeros or sign bits as appropriate.
631
632 If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
633 a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
634 Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
635 In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
636 are undefined.
637
638 BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
639 may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
640 OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
641 function wants to calculate it itself.
642
643 Return true if the shift could be successfully synthesized. */
644
645 static bool
646 expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab,
647 rtx outof_input, rtx into_input, rtx op1,
648 rtx outof_target, rtx into_target,
649 int unsignedp, enum optab_methods methods,
650 unsigned HOST_WIDE_INT shift_mask)
651 {
652 rtx superword_op1, tmp, cmp1, cmp2;
653 enum rtx_code cmp_code;
654
655 /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
656 fill the result with sign or zero bits as appropriate. If so, the value
657 of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
658 this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
659 and INTO_INPUT), then emit code to set up OUTOF_TARGET.
660
661 This isn't worthwhile for constant shifts since the optimizers will
662 cope better with in-range shift counts. */
663 if (shift_mask >= BITS_PER_WORD
664 && outof_target != 0
665 && !CONSTANT_P (op1))
666 {
667 if (!expand_doubleword_shift (op1_mode, binoptab,
668 outof_input, into_input, op1,
669 0, into_target,
670 unsignedp, methods, shift_mask))
671 return false;
672 if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
673 outof_target, unsignedp, methods))
674 return false;
675 return true;
676 }
677
678 /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
679 is true when the effective shift value is less than BITS_PER_WORD.
680 Set SUPERWORD_OP1 to the shift count that should be used to shift
681 OUTOF_INPUT into INTO_TARGET when the condition is false. */
682 tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode);
683 if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
684 {
685 /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
686 is a subword shift count. */
687 cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
688 0, true, methods);
689 cmp2 = CONST0_RTX (op1_mode);
690 cmp_code = EQ;
691 superword_op1 = op1;
692 }
693 else
694 {
695 /* Set CMP1 to OP1 - BITS_PER_WORD. */
696 cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
697 0, true, methods);
698 cmp2 = CONST0_RTX (op1_mode);
699 cmp_code = LT;
700 superword_op1 = cmp1;
701 }
702 if (cmp1 == 0)
703 return false;
704
705 /* If we can compute the condition at compile time, pick the
706 appropriate subroutine. */
707 tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
708 if (tmp != 0 && CONST_INT_P (tmp))
709 {
710 if (tmp == const0_rtx)
711 return expand_superword_shift (binoptab, outof_input, superword_op1,
712 outof_target, into_target,
713 unsignedp, methods);
714 else
715 return expand_subword_shift (op1_mode, binoptab,
716 outof_input, into_input, op1,
717 outof_target, into_target,
718 unsignedp, methods, shift_mask);
719 }
720
721 /* Try using conditional moves to generate straight-line code. */
722 if (HAVE_conditional_move)
723 {
724 rtx_insn *start = get_last_insn ();
725 if (expand_doubleword_shift_condmove (op1_mode, binoptab,
726 cmp_code, cmp1, cmp2,
727 outof_input, into_input,
728 op1, superword_op1,
729 outof_target, into_target,
730 unsignedp, methods, shift_mask))
731 return true;
732 delete_insns_since (start);
733 }
734
735 /* As a last resort, use branches to select the correct alternative. */
736 rtx_code_label *subword_label = gen_label_rtx ();
737 rtx_code_label *done_label = gen_label_rtx ();
738
739 NO_DEFER_POP;
740 do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
741 0, 0, subword_label,
742 profile_probability::uninitialized ());
743 OK_DEFER_POP;
744
745 if (!expand_superword_shift (binoptab, outof_input, superword_op1,
746 outof_target, into_target,
747 unsignedp, methods))
748 return false;
749
750 emit_jump_insn (targetm.gen_jump (done_label));
751 emit_barrier ();
752 emit_label (subword_label);
753
754 if (!expand_subword_shift (op1_mode, binoptab,
755 outof_input, into_input, op1,
756 outof_target, into_target,
757 unsignedp, methods, shift_mask))
758 return false;
759
760 emit_label (done_label);
761 return true;
762 }
763 \f
764 /* Subroutine of expand_binop. Perform a double word multiplication of
765 operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
766 as the target's word_mode. This function return NULL_RTX if anything
767 goes wrong, in which case it may have already emitted instructions
768 which need to be deleted.
769
770 If we want to multiply two two-word values and have normal and widening
771 multiplies of single-word values, we can do this with three smaller
772 multiplications.
773
774 The multiplication proceeds as follows:
775 _______________________
776 [__op0_high_|__op0_low__]
777 _______________________
778 * [__op1_high_|__op1_low__]
779 _______________________________________________
780 _______________________
781 (1) [__op0_low__*__op1_low__]
782 _______________________
783 (2a) [__op0_low__*__op1_high_]
784 _______________________
785 (2b) [__op0_high_*__op1_low__]
786 _______________________
787 (3) [__op0_high_*__op1_high_]
788
789
790 This gives a 4-word result. Since we are only interested in the
791 lower 2 words, partial result (3) and the upper words of (2a) and
792 (2b) don't need to be calculated. Hence (2a) and (2b) can be
793 calculated using non-widening multiplication.
794
795 (1), however, needs to be calculated with an unsigned widening
796 multiplication. If this operation is not directly supported we
797 try using a signed widening multiplication and adjust the result.
798 This adjustment works as follows:
799
800 If both operands are positive then no adjustment is needed.
801
802 If the operands have different signs, for example op0_low < 0 and
803 op1_low >= 0, the instruction treats the most significant bit of
804 op0_low as a sign bit instead of a bit with significance
805 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
806 with 2**BITS_PER_WORD - op0_low, and two's complements the
807 result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
808 the result.
809
810 Similarly, if both operands are negative, we need to add
811 (op0_low + op1_low) * 2**BITS_PER_WORD.
812
813 We use a trick to adjust quickly. We logically shift op0_low right
814 (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
815 op0_high (op1_high) before it is used to calculate 2b (2a). If no
816 logical shift exists, we do an arithmetic right shift and subtract
817 the 0 or -1. */
818
819 static rtx
820 expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
821 bool umulp, enum optab_methods methods)
822 {
823 int low = (WORDS_BIG_ENDIAN ? 1 : 0);
824 int high = (WORDS_BIG_ENDIAN ? 0 : 1);
825 rtx wordm1 = (umulp ? NULL_RTX
826 : gen_int_shift_amount (word_mode, BITS_PER_WORD - 1));
827 rtx product, adjust, product_high, temp;
828
829 rtx op0_high = operand_subword_force (op0, high, mode);
830 rtx op0_low = operand_subword_force (op0, low, mode);
831 rtx op1_high = operand_subword_force (op1, high, mode);
832 rtx op1_low = operand_subword_force (op1, low, mode);
833
834 /* If we're using an unsigned multiply to directly compute the product
835 of the low-order words of the operands and perform any required
836 adjustments of the operands, we begin by trying two more multiplications
837 and then computing the appropriate sum.
838
839 We have checked above that the required addition is provided.
840 Full-word addition will normally always succeed, especially if
841 it is provided at all, so we don't worry about its failure. The
842 multiplication may well fail, however, so we do handle that. */
843
844 if (!umulp)
845 {
846 /* ??? This could be done with emit_store_flag where available. */
847 temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
848 NULL_RTX, 1, methods);
849 if (temp)
850 op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
851 NULL_RTX, 0, OPTAB_DIRECT);
852 else
853 {
854 temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
855 NULL_RTX, 0, methods);
856 if (!temp)
857 return NULL_RTX;
858 op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
859 NULL_RTX, 0, OPTAB_DIRECT);
860 }
861
862 if (!op0_high)
863 return NULL_RTX;
864 }
865
866 adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
867 NULL_RTX, 0, OPTAB_DIRECT);
868 if (!adjust)
869 return NULL_RTX;
870
871 /* OP0_HIGH should now be dead. */
872
873 if (!umulp)
874 {
875 /* ??? This could be done with emit_store_flag where available. */
876 temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
877 NULL_RTX, 1, methods);
878 if (temp)
879 op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
880 NULL_RTX, 0, OPTAB_DIRECT);
881 else
882 {
883 temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
884 NULL_RTX, 0, methods);
885 if (!temp)
886 return NULL_RTX;
887 op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
888 NULL_RTX, 0, OPTAB_DIRECT);
889 }
890
891 if (!op1_high)
892 return NULL_RTX;
893 }
894
895 temp = expand_binop (word_mode, smul_optab, op1_high, op0_low,
896 NULL_RTX, 0, OPTAB_DIRECT);
897 if (!temp)
898 return NULL_RTX;
899
900 /* OP1_HIGH should now be dead. */
901
902 adjust = expand_binop (word_mode, add_optab, adjust, temp,
903 NULL_RTX, 0, OPTAB_DIRECT);
904
905 if (target && !REG_P (target))
906 target = NULL_RTX;
907
908 /* *_widen_optab needs to determine operand mode, make sure at least
909 one operand has non-VOID mode. */
910 if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode)
911 op0_low = force_reg (word_mode, op0_low);
912
913 if (umulp)
914 product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
915 target, 1, OPTAB_DIRECT);
916 else
917 product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
918 target, 1, OPTAB_DIRECT);
919
920 if (!product)
921 return NULL_RTX;
922
923 product_high = operand_subword (product, high, 1, mode);
924 adjust = expand_binop (word_mode, add_optab, product_high, adjust,
925 NULL_RTX, 0, OPTAB_DIRECT);
926 emit_move_insn (product_high, adjust);
927 return product;
928 }
929 \f
930 /* Wrapper around expand_binop which takes an rtx code to specify
931 the operation to perform, not an optab pointer. All other
932 arguments are the same. */
933 rtx
934 expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0,
935 rtx op1, rtx target, int unsignedp,
936 enum optab_methods methods)
937 {
938 optab binop = code_to_optab (code);
939 gcc_assert (binop);
940
941 return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
942 }
943
944 /* Return whether OP0 and OP1 should be swapped when expanding a commutative
945 binop. Order them according to commutative_operand_precedence and, if
946 possible, try to put TARGET or a pseudo first. */
947 static bool
948 swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
949 {
950 int op0_prec = commutative_operand_precedence (op0);
951 int op1_prec = commutative_operand_precedence (op1);
952
953 if (op0_prec < op1_prec)
954 return true;
955
956 if (op0_prec > op1_prec)
957 return false;
958
959 /* With equal precedence, both orders are ok, but it is better if the
960 first operand is TARGET, or if both TARGET and OP0 are pseudos. */
961 if (target == 0 || REG_P (target))
962 return (REG_P (op1) && !REG_P (op0)) || target == op1;
963 else
964 return rtx_equal_p (op1, target);
965 }
966
967 /* Return true if BINOPTAB implements a shift operation. */
968
969 static bool
970 shift_optab_p (optab binoptab)
971 {
972 switch (optab_to_code (binoptab))
973 {
974 case ASHIFT:
975 case SS_ASHIFT:
976 case US_ASHIFT:
977 case ASHIFTRT:
978 case LSHIFTRT:
979 case ROTATE:
980 case ROTATERT:
981 return true;
982
983 default:
984 return false;
985 }
986 }
987
988 /* Return true if BINOPTAB implements a commutative binary operation. */
989
990 static bool
991 commutative_optab_p (optab binoptab)
992 {
993 return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
994 || binoptab == smul_widen_optab
995 || binoptab == umul_widen_optab
996 || binoptab == smul_highpart_optab
997 || binoptab == umul_highpart_optab);
998 }
999
1000 /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1001 optimizing, and if the operand is a constant that costs more than
1002 1 instruction, force the constant into a register and return that
1003 register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1004
1005 static rtx
1006 avoid_expensive_constant (machine_mode mode, optab binoptab,
1007 int opn, rtx x, bool unsignedp)
1008 {
1009 bool speed = optimize_insn_for_speed_p ();
1010
1011 if (mode != VOIDmode
1012 && optimize
1013 && CONSTANT_P (x)
1014 && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed)
1015 > set_src_cost (x, mode, speed)))
1016 {
1017 if (CONST_INT_P (x))
1018 {
1019 HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
1020 if (intval != INTVAL (x))
1021 x = GEN_INT (intval);
1022 }
1023 else
1024 x = convert_modes (mode, VOIDmode, x, unsignedp);
1025 x = force_reg (mode, x);
1026 }
1027 return x;
1028 }
1029
1030 /* Helper function for expand_binop: handle the case where there
1031 is an insn ICODE that directly implements the indicated operation.
1032 Returns null if this is not possible. */
1033 static rtx
1034 expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab,
1035 rtx op0, rtx op1,
1036 rtx target, int unsignedp, enum optab_methods methods,
1037 rtx_insn *last)
1038 {
1039 machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
1040 machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
1041 machine_mode mode0, mode1, tmp_mode;
1042 class expand_operand ops[3];
1043 bool commutative_p;
1044 rtx_insn *pat;
1045 rtx xop0 = op0, xop1 = op1;
1046 bool canonicalize_op1 = false;
1047
1048 /* If it is a commutative operator and the modes would match
1049 if we would swap the operands, we can save the conversions. */
1050 commutative_p = commutative_optab_p (binoptab);
1051 if (commutative_p
1052 && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1053 && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode1)
1054 std::swap (xop0, xop1);
1055
1056 /* If we are optimizing, force expensive constants into a register. */
1057 xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
1058 if (!shift_optab_p (binoptab))
1059 xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
1060 else
1061 /* Shifts and rotates often use a different mode for op1 from op0;
1062 for VOIDmode constants we don't know the mode, so force it
1063 to be canonicalized using convert_modes. */
1064 canonicalize_op1 = true;
1065
1066 /* In case the insn wants input operands in modes different from
1067 those of the actual operands, convert the operands. It would
1068 seem that we don't need to convert CONST_INTs, but we do, so
1069 that they're properly zero-extended, sign-extended or truncated
1070 for their mode. */
1071
1072 mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1073 if (xmode0 != VOIDmode && xmode0 != mode0)
1074 {
1075 xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
1076 mode0 = xmode0;
1077 }
1078
1079 mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1)
1080 ? GET_MODE (xop1) : mode);
1081 if (xmode1 != VOIDmode && xmode1 != mode1)
1082 {
1083 xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
1084 mode1 = xmode1;
1085 }
1086
1087 /* If operation is commutative,
1088 try to make the first operand a register.
1089 Even better, try to make it the same as the target.
1090 Also try to make the last operand a constant. */
1091 if (commutative_p
1092 && swap_commutative_operands_with_target (target, xop0, xop1))
1093 std::swap (xop0, xop1);
1094
1095 /* Now, if insn's predicates don't allow our operands, put them into
1096 pseudo regs. */
1097
1098 if (binoptab == vec_pack_trunc_optab
1099 || binoptab == vec_pack_usat_optab
1100 || binoptab == vec_pack_ssat_optab
1101 || binoptab == vec_pack_ufix_trunc_optab
1102 || binoptab == vec_pack_sfix_trunc_optab
1103 || binoptab == vec_packu_float_optab
1104 || binoptab == vec_packs_float_optab)
1105 {
1106 /* The mode of the result is different then the mode of the
1107 arguments. */
1108 tmp_mode = insn_data[(int) icode].operand[0].mode;
1109 if (VECTOR_MODE_P (mode)
1110 && maybe_ne (GET_MODE_NUNITS (tmp_mode), 2 * GET_MODE_NUNITS (mode)))
1111 {
1112 delete_insns_since (last);
1113 return NULL_RTX;
1114 }
1115 }
1116 else
1117 tmp_mode = mode;
1118
1119 create_output_operand (&ops[0], target, tmp_mode);
1120 create_input_operand (&ops[1], xop0, mode0);
1121 create_input_operand (&ops[2], xop1, mode1);
1122 pat = maybe_gen_insn (icode, 3, ops);
1123 if (pat)
1124 {
1125 /* If PAT is composed of more than one insn, try to add an appropriate
1126 REG_EQUAL note to it. If we can't because TEMP conflicts with an
1127 operand, call expand_binop again, this time without a target. */
1128 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
1129 && ! add_equal_note (pat, ops[0].value,
1130 optab_to_code (binoptab),
1131 ops[1].value, ops[2].value, mode0))
1132 {
1133 delete_insns_since (last);
1134 return expand_binop (mode, binoptab, op0, op1, NULL_RTX,
1135 unsignedp, methods);
1136 }
1137
1138 emit_insn (pat);
1139 return ops[0].value;
1140 }
1141 delete_insns_since (last);
1142 return NULL_RTX;
1143 }
1144
1145 /* Generate code to perform an operation specified by BINOPTAB
1146 on operands OP0 and OP1, with result having machine-mode MODE.
1147
1148 UNSIGNEDP is for the case where we have to widen the operands
1149 to perform the operation. It says to use zero-extension.
1150
1151 If TARGET is nonzero, the value
1152 is generated there, if it is convenient to do so.
1153 In all cases an rtx is returned for the locus of the value;
1154 this may or may not be TARGET. */
1155
1156 rtx
1157 expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1,
1158 rtx target, int unsignedp, enum optab_methods methods)
1159 {
1160 enum optab_methods next_methods
1161 = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1162 ? OPTAB_WIDEN : methods);
1163 enum mode_class mclass;
1164 enum insn_code icode;
1165 machine_mode wider_mode;
1166 scalar_int_mode int_mode;
1167 rtx libfunc;
1168 rtx temp;
1169 rtx_insn *entry_last = get_last_insn ();
1170 rtx_insn *last;
1171
1172 mclass = GET_MODE_CLASS (mode);
1173
1174 /* If subtracting an integer constant, convert this into an addition of
1175 the negated constant. */
1176
1177 if (binoptab == sub_optab && CONST_INT_P (op1))
1178 {
1179 op1 = negate_rtx (mode, op1);
1180 binoptab = add_optab;
1181 }
1182 /* For shifts, constant invalid op1 might be expanded from different
1183 mode than MODE. As those are invalid, force them to a register
1184 to avoid further problems during expansion. */
1185 else if (CONST_INT_P (op1)
1186 && shift_optab_p (binoptab)
1187 && UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode)))
1188 {
1189 op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode));
1190 op1 = force_reg (GET_MODE_INNER (mode), op1);
1191 }
1192
1193 /* Record where to delete back to if we backtrack. */
1194 last = get_last_insn ();
1195
1196 /* If we can do it with a three-operand insn, do so. */
1197
1198 if (methods != OPTAB_MUST_WIDEN)
1199 {
1200 if (convert_optab_p (binoptab))
1201 {
1202 machine_mode from_mode = widened_mode (mode, op0, op1);
1203 icode = find_widening_optab_handler (binoptab, mode, from_mode);
1204 }
1205 else
1206 icode = optab_handler (binoptab, mode);
1207 if (icode != CODE_FOR_nothing)
1208 {
1209 temp = expand_binop_directly (icode, mode, binoptab, op0, op1,
1210 target, unsignedp, methods, last);
1211 if (temp)
1212 return temp;
1213 }
1214 }
1215
1216 /* If we were trying to rotate, and that didn't work, try rotating
1217 the other direction before falling back to shifts and bitwise-or. */
1218 if (((binoptab == rotl_optab
1219 && (icode = optab_handler (rotr_optab, mode)) != CODE_FOR_nothing)
1220 || (binoptab == rotr_optab
1221 && (icode = optab_handler (rotl_optab, mode)) != CODE_FOR_nothing))
1222 && is_int_mode (mode, &int_mode))
1223 {
1224 optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1225 rtx newop1;
1226 unsigned int bits = GET_MODE_PRECISION (int_mode);
1227
1228 if (CONST_INT_P (op1))
1229 newop1 = gen_int_shift_amount (int_mode, bits - INTVAL (op1));
1230 else if (targetm.shift_truncation_mask (int_mode) == bits - 1)
1231 newop1 = negate_rtx (GET_MODE (op1), op1);
1232 else
1233 newop1 = expand_binop (GET_MODE (op1), sub_optab,
1234 gen_int_mode (bits, GET_MODE (op1)), op1,
1235 NULL_RTX, unsignedp, OPTAB_DIRECT);
1236
1237 temp = expand_binop_directly (icode, int_mode, otheroptab, op0, newop1,
1238 target, unsignedp, methods, last);
1239 if (temp)
1240 return temp;
1241 }
1242
1243 /* If this is a multiply, see if we can do a widening operation that
1244 takes operands of this mode and makes a wider mode. */
1245
1246 if (binoptab == smul_optab
1247 && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode)
1248 && (convert_optab_handler ((unsignedp
1249 ? umul_widen_optab
1250 : smul_widen_optab),
1251 wider_mode, mode) != CODE_FOR_nothing))
1252 {
1253 /* *_widen_optab needs to determine operand mode, make sure at least
1254 one operand has non-VOID mode. */
1255 if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
1256 op0 = force_reg (mode, op0);
1257 temp = expand_binop (wider_mode,
1258 unsignedp ? umul_widen_optab : smul_widen_optab,
1259 op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
1260
1261 if (temp != 0)
1262 {
1263 if (GET_MODE_CLASS (mode) == MODE_INT
1264 && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1265 return gen_lowpart (mode, temp);
1266 else
1267 return convert_to_mode (mode, temp, unsignedp);
1268 }
1269 }
1270
1271 /* If this is a vector shift by a scalar, see if we can do a vector
1272 shift by a vector. If so, broadcast the scalar into a vector. */
1273 if (mclass == MODE_VECTOR_INT)
1274 {
1275 optab otheroptab = unknown_optab;
1276
1277 if (binoptab == ashl_optab)
1278 otheroptab = vashl_optab;
1279 else if (binoptab == ashr_optab)
1280 otheroptab = vashr_optab;
1281 else if (binoptab == lshr_optab)
1282 otheroptab = vlshr_optab;
1283 else if (binoptab == rotl_optab)
1284 otheroptab = vrotl_optab;
1285 else if (binoptab == rotr_optab)
1286 otheroptab = vrotr_optab;
1287
1288 if (otheroptab
1289 && (icode = optab_handler (otheroptab, mode)) != CODE_FOR_nothing)
1290 {
1291 /* The scalar may have been extended to be too wide. Truncate
1292 it back to the proper size to fit in the broadcast vector. */
1293 scalar_mode inner_mode = GET_MODE_INNER (mode);
1294 if (!CONST_INT_P (op1)
1295 && (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (op1)))
1296 > GET_MODE_BITSIZE (inner_mode)))
1297 op1 = force_reg (inner_mode,
1298 simplify_gen_unary (TRUNCATE, inner_mode, op1,
1299 GET_MODE (op1)));
1300 rtx vop1 = expand_vector_broadcast (mode, op1);
1301 if (vop1)
1302 {
1303 temp = expand_binop_directly (icode, mode, otheroptab, op0, vop1,
1304 target, unsignedp, methods, last);
1305 if (temp)
1306 return temp;
1307 }
1308 }
1309 }
1310
1311 /* Look for a wider mode of the same class for which we think we
1312 can open-code the operation. Check for a widening multiply at the
1313 wider mode as well. */
1314
1315 if (CLASS_HAS_WIDER_MODES_P (mclass)
1316 && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1317 FOR_EACH_WIDER_MODE (wider_mode, mode)
1318 {
1319 machine_mode next_mode;
1320 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
1321 || (binoptab == smul_optab
1322 && GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode)
1323 && (find_widening_optab_handler ((unsignedp
1324 ? umul_widen_optab
1325 : smul_widen_optab),
1326 next_mode, mode)
1327 != CODE_FOR_nothing)))
1328 {
1329 rtx xop0 = op0, xop1 = op1;
1330 int no_extend = 0;
1331
1332 /* For certain integer operations, we need not actually extend
1333 the narrow operands, as long as we will truncate
1334 the results to the same narrowness. */
1335
1336 if ((binoptab == ior_optab || binoptab == and_optab
1337 || binoptab == xor_optab
1338 || binoptab == add_optab || binoptab == sub_optab
1339 || binoptab == smul_optab || binoptab == ashl_optab)
1340 && mclass == MODE_INT)
1341 {
1342 no_extend = 1;
1343 xop0 = avoid_expensive_constant (mode, binoptab, 0,
1344 xop0, unsignedp);
1345 if (binoptab != ashl_optab)
1346 xop1 = avoid_expensive_constant (mode, binoptab, 1,
1347 xop1, unsignedp);
1348 }
1349
1350 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
1351
1352 /* The second operand of a shift must always be extended. */
1353 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1354 no_extend && binoptab != ashl_optab);
1355
1356 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1357 unsignedp, OPTAB_DIRECT);
1358 if (temp)
1359 {
1360 if (mclass != MODE_INT
1361 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1362 {
1363 if (target == 0)
1364 target = gen_reg_rtx (mode);
1365 convert_move (target, temp, 0);
1366 return target;
1367 }
1368 else
1369 return gen_lowpart (mode, temp);
1370 }
1371 else
1372 delete_insns_since (last);
1373 }
1374 }
1375
1376 /* If operation is commutative,
1377 try to make the first operand a register.
1378 Even better, try to make it the same as the target.
1379 Also try to make the last operand a constant. */
1380 if (commutative_optab_p (binoptab)
1381 && swap_commutative_operands_with_target (target, op0, op1))
1382 std::swap (op0, op1);
1383
1384 /* These can be done a word at a time. */
1385 if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1386 && is_int_mode (mode, &int_mode)
1387 && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
1388 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1389 {
1390 int i;
1391 rtx_insn *insns;
1392
1393 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1394 won't be accurate, so use a new target. */
1395 if (target == 0
1396 || target == op0
1397 || target == op1
1398 || !valid_multiword_target_p (target))
1399 target = gen_reg_rtx (int_mode);
1400
1401 start_sequence ();
1402
1403 /* Do the actual arithmetic. */
1404 machine_mode op0_mode = GET_MODE (op0);
1405 machine_mode op1_mode = GET_MODE (op1);
1406 if (op0_mode == VOIDmode)
1407 op0_mode = int_mode;
1408 if (op1_mode == VOIDmode)
1409 op1_mode = int_mode;
1410 for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
1411 {
1412 rtx target_piece = operand_subword (target, i, 1, int_mode);
1413 rtx x = expand_binop (word_mode, binoptab,
1414 operand_subword_force (op0, i, op0_mode),
1415 operand_subword_force (op1, i, op1_mode),
1416 target_piece, unsignedp, next_methods);
1417
1418 if (x == 0)
1419 break;
1420
1421 if (target_piece != x)
1422 emit_move_insn (target_piece, x);
1423 }
1424
1425 insns = get_insns ();
1426 end_sequence ();
1427
1428 if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD)
1429 {
1430 emit_insn (insns);
1431 return target;
1432 }
1433 }
1434
1435 /* Synthesize double word shifts from single word shifts. */
1436 if ((binoptab == lshr_optab || binoptab == ashl_optab
1437 || binoptab == ashr_optab)
1438 && is_int_mode (mode, &int_mode)
1439 && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1440 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1441 && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode)
1442 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
1443 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1444 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1445 {
1446 unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1447 scalar_int_mode op1_mode;
1448
1449 double_shift_mask = targetm.shift_truncation_mask (int_mode);
1450 shift_mask = targetm.shift_truncation_mask (word_mode);
1451 op1_mode = (GET_MODE (op1) != VOIDmode
1452 ? as_a <scalar_int_mode> (GET_MODE (op1))
1453 : word_mode);
1454
1455 /* Apply the truncation to constant shifts. */
1456 if (double_shift_mask > 0 && CONST_INT_P (op1))
1457 op1 = gen_int_mode (INTVAL (op1) & double_shift_mask, op1_mode);
1458
1459 if (op1 == CONST0_RTX (op1_mode))
1460 return op0;
1461
1462 /* Make sure that this is a combination that expand_doubleword_shift
1463 can handle. See the comments there for details. */
1464 if (double_shift_mask == 0
1465 || (shift_mask == BITS_PER_WORD - 1
1466 && double_shift_mask == BITS_PER_WORD * 2 - 1))
1467 {
1468 rtx_insn *insns;
1469 rtx into_target, outof_target;
1470 rtx into_input, outof_input;
1471 int left_shift, outof_word;
1472
1473 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1474 won't be accurate, so use a new target. */
1475 if (target == 0
1476 || target == op0
1477 || target == op1
1478 || !valid_multiword_target_p (target))
1479 target = gen_reg_rtx (int_mode);
1480
1481 start_sequence ();
1482
1483 /* OUTOF_* is the word we are shifting bits away from, and
1484 INTO_* is the word that we are shifting bits towards, thus
1485 they differ depending on the direction of the shift and
1486 WORDS_BIG_ENDIAN. */
1487
1488 left_shift = binoptab == ashl_optab;
1489 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1490
1491 outof_target = operand_subword (target, outof_word, 1, int_mode);
1492 into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1493
1494 outof_input = operand_subword_force (op0, outof_word, int_mode);
1495 into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1496
1497 if (expand_doubleword_shift (op1_mode, binoptab,
1498 outof_input, into_input, op1,
1499 outof_target, into_target,
1500 unsignedp, next_methods, shift_mask))
1501 {
1502 insns = get_insns ();
1503 end_sequence ();
1504
1505 emit_insn (insns);
1506 return target;
1507 }
1508 end_sequence ();
1509 }
1510 }
1511
1512 /* Synthesize double word rotates from single word shifts. */
1513 if ((binoptab == rotl_optab || binoptab == rotr_optab)
1514 && is_int_mode (mode, &int_mode)
1515 && CONST_INT_P (op1)
1516 && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD
1517 && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1518 && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1519 {
1520 rtx_insn *insns;
1521 rtx into_target, outof_target;
1522 rtx into_input, outof_input;
1523 rtx inter;
1524 int shift_count, left_shift, outof_word;
1525
1526 /* If TARGET is the same as one of the operands, the REG_EQUAL note
1527 won't be accurate, so use a new target. Do this also if target is not
1528 a REG, first because having a register instead may open optimization
1529 opportunities, and second because if target and op0 happen to be MEMs
1530 designating the same location, we would risk clobbering it too early
1531 in the code sequence we generate below. */
1532 if (target == 0
1533 || target == op0
1534 || target == op1
1535 || !REG_P (target)
1536 || !valid_multiword_target_p (target))
1537 target = gen_reg_rtx (int_mode);
1538
1539 start_sequence ();
1540
1541 shift_count = INTVAL (op1);
1542
1543 /* OUTOF_* is the word we are shifting bits away from, and
1544 INTO_* is the word that we are shifting bits towards, thus
1545 they differ depending on the direction of the shift and
1546 WORDS_BIG_ENDIAN. */
1547
1548 left_shift = (binoptab == rotl_optab);
1549 outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1550
1551 outof_target = operand_subword (target, outof_word, 1, int_mode);
1552 into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1553
1554 outof_input = operand_subword_force (op0, outof_word, int_mode);
1555 into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1556
1557 if (shift_count == BITS_PER_WORD)
1558 {
1559 /* This is just a word swap. */
1560 emit_move_insn (outof_target, into_input);
1561 emit_move_insn (into_target, outof_input);
1562 inter = const0_rtx;
1563 }
1564 else
1565 {
1566 rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1567 HOST_WIDE_INT first_shift_count, second_shift_count;
1568 optab reverse_unsigned_shift, unsigned_shift;
1569
1570 reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1571 ? lshr_optab : ashl_optab);
1572
1573 unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1574 ? ashl_optab : lshr_optab);
1575
1576 if (shift_count > BITS_PER_WORD)
1577 {
1578 first_shift_count = shift_count - BITS_PER_WORD;
1579 second_shift_count = 2 * BITS_PER_WORD - shift_count;
1580 }
1581 else
1582 {
1583 first_shift_count = BITS_PER_WORD - shift_count;
1584 second_shift_count = shift_count;
1585 }
1586 rtx first_shift_count_rtx
1587 = gen_int_shift_amount (word_mode, first_shift_count);
1588 rtx second_shift_count_rtx
1589 = gen_int_shift_amount (word_mode, second_shift_count);
1590
1591 into_temp1 = expand_binop (word_mode, unsigned_shift,
1592 outof_input, first_shift_count_rtx,
1593 NULL_RTX, unsignedp, next_methods);
1594 into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1595 into_input, second_shift_count_rtx,
1596 NULL_RTX, unsignedp, next_methods);
1597
1598 if (into_temp1 != 0 && into_temp2 != 0)
1599 inter = expand_binop (word_mode, ior_optab, into_temp1, into_temp2,
1600 into_target, unsignedp, next_methods);
1601 else
1602 inter = 0;
1603
1604 if (inter != 0 && inter != into_target)
1605 emit_move_insn (into_target, inter);
1606
1607 outof_temp1 = expand_binop (word_mode, unsigned_shift,
1608 into_input, first_shift_count_rtx,
1609 NULL_RTX, unsignedp, next_methods);
1610 outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1611 outof_input, second_shift_count_rtx,
1612 NULL_RTX, unsignedp, next_methods);
1613
1614 if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
1615 inter = expand_binop (word_mode, ior_optab,
1616 outof_temp1, outof_temp2,
1617 outof_target, unsignedp, next_methods);
1618
1619 if (inter != 0 && inter != outof_target)
1620 emit_move_insn (outof_target, inter);
1621 }
1622
1623 insns = get_insns ();
1624 end_sequence ();
1625
1626 if (inter != 0)
1627 {
1628 emit_insn (insns);
1629 return target;
1630 }
1631 }
1632
1633 /* These can be done a word at a time by propagating carries. */
1634 if ((binoptab == add_optab || binoptab == sub_optab)
1635 && is_int_mode (mode, &int_mode)
1636 && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD
1637 && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1638 {
1639 unsigned int i;
1640 optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
1641 const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD;
1642 rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
1643 rtx xop0, xop1, xtarget;
1644
1645 /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
1646 value is one of those, use it. Otherwise, use 1 since it is the
1647 one easiest to get. */
1648 #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
1649 int normalizep = STORE_FLAG_VALUE;
1650 #else
1651 int normalizep = 1;
1652 #endif
1653
1654 /* Prepare the operands. */
1655 xop0 = force_reg (int_mode, op0);
1656 xop1 = force_reg (int_mode, op1);
1657
1658 xtarget = gen_reg_rtx (int_mode);
1659
1660 if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
1661 target = xtarget;
1662
1663 /* Indicate for flow that the entire target reg is being set. */
1664 if (REG_P (target))
1665 emit_clobber (xtarget);
1666
1667 /* Do the actual arithmetic. */
1668 for (i = 0; i < nwords; i++)
1669 {
1670 int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
1671 rtx target_piece = operand_subword (xtarget, index, 1, int_mode);
1672 rtx op0_piece = operand_subword_force (xop0, index, int_mode);
1673 rtx op1_piece = operand_subword_force (xop1, index, int_mode);
1674 rtx x;
1675
1676 /* Main add/subtract of the input operands. */
1677 x = expand_binop (word_mode, binoptab,
1678 op0_piece, op1_piece,
1679 target_piece, unsignedp, next_methods);
1680 if (x == 0)
1681 break;
1682
1683 if (i + 1 < nwords)
1684 {
1685 /* Store carry from main add/subtract. */
1686 carry_out = gen_reg_rtx (word_mode);
1687 carry_out = emit_store_flag_force (carry_out,
1688 (binoptab == add_optab
1689 ? LT : GT),
1690 x, op0_piece,
1691 word_mode, 1, normalizep);
1692 }
1693
1694 if (i > 0)
1695 {
1696 rtx newx;
1697
1698 /* Add/subtract previous carry to main result. */
1699 newx = expand_binop (word_mode,
1700 normalizep == 1 ? binoptab : otheroptab,
1701 x, carry_in,
1702 NULL_RTX, 1, next_methods);
1703
1704 if (i + 1 < nwords)
1705 {
1706 /* Get out carry from adding/subtracting carry in. */
1707 rtx carry_tmp = gen_reg_rtx (word_mode);
1708 carry_tmp = emit_store_flag_force (carry_tmp,
1709 (binoptab == add_optab
1710 ? LT : GT),
1711 newx, x,
1712 word_mode, 1, normalizep);
1713
1714 /* Logical-ior the two poss. carry together. */
1715 carry_out = expand_binop (word_mode, ior_optab,
1716 carry_out, carry_tmp,
1717 carry_out, 0, next_methods);
1718 if (carry_out == 0)
1719 break;
1720 }
1721 emit_move_insn (target_piece, newx);
1722 }
1723 else
1724 {
1725 if (x != target_piece)
1726 emit_move_insn (target_piece, x);
1727 }
1728
1729 carry_in = carry_out;
1730 }
1731
1732 if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD)
1733 {
1734 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing
1735 || ! rtx_equal_p (target, xtarget))
1736 {
1737 rtx_insn *temp = emit_move_insn (target, xtarget);
1738
1739 set_dst_reg_note (temp, REG_EQUAL,
1740 gen_rtx_fmt_ee (optab_to_code (binoptab),
1741 int_mode, copy_rtx (xop0),
1742 copy_rtx (xop1)),
1743 target);
1744 }
1745 else
1746 target = xtarget;
1747
1748 return target;
1749 }
1750
1751 else
1752 delete_insns_since (last);
1753 }
1754
1755 /* Attempt to synthesize double word multiplies using a sequence of word
1756 mode multiplications. We first attempt to generate a sequence using a
1757 more efficient unsigned widening multiply, and if that fails we then
1758 try using a signed widening multiply. */
1759
1760 if (binoptab == smul_optab
1761 && is_int_mode (mode, &int_mode)
1762 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1763 && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
1764 && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
1765 {
1766 rtx product = NULL_RTX;
1767 if (convert_optab_handler (umul_widen_optab, int_mode, word_mode)
1768 != CODE_FOR_nothing)
1769 {
1770 product = expand_doubleword_mult (int_mode, op0, op1, target,
1771 true, methods);
1772 if (!product)
1773 delete_insns_since (last);
1774 }
1775
1776 if (product == NULL_RTX
1777 && (convert_optab_handler (smul_widen_optab, int_mode, word_mode)
1778 != CODE_FOR_nothing))
1779 {
1780 product = expand_doubleword_mult (int_mode, op0, op1, target,
1781 false, methods);
1782 if (!product)
1783 delete_insns_since (last);
1784 }
1785
1786 if (product != NULL_RTX)
1787 {
1788 if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
1789 {
1790 rtx_insn *move = emit_move_insn (target ? target : product,
1791 product);
1792 set_dst_reg_note (move,
1793 REG_EQUAL,
1794 gen_rtx_fmt_ee (MULT, int_mode,
1795 copy_rtx (op0),
1796 copy_rtx (op1)),
1797 target ? target : product);
1798 }
1799 return product;
1800 }
1801 }
1802
1803 /* It can't be open-coded in this mode.
1804 Use a library call if one is available and caller says that's ok. */
1805
1806 libfunc = optab_libfunc (binoptab, mode);
1807 if (libfunc
1808 && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
1809 {
1810 rtx_insn *insns;
1811 rtx op1x = op1;
1812 machine_mode op1_mode = mode;
1813 rtx value;
1814
1815 start_sequence ();
1816
1817 if (shift_optab_p (binoptab))
1818 {
1819 op1_mode = targetm.libgcc_shift_count_mode ();
1820 /* Specify unsigned here,
1821 since negative shift counts are meaningless. */
1822 op1x = convert_to_mode (op1_mode, op1, 1);
1823 }
1824
1825 if (GET_MODE (op0) != VOIDmode
1826 && GET_MODE (op0) != mode)
1827 op0 = convert_to_mode (mode, op0, unsignedp);
1828
1829 /* Pass 1 for NO_QUEUE so we don't lose any increments
1830 if the libcall is cse'd or moved. */
1831 value = emit_library_call_value (libfunc,
1832 NULL_RTX, LCT_CONST, mode,
1833 op0, mode, op1x, op1_mode);
1834
1835 insns = get_insns ();
1836 end_sequence ();
1837
1838 bool trapv = trapv_binoptab_p (binoptab);
1839 target = gen_reg_rtx (mode);
1840 emit_libcall_block_1 (insns, target, value,
1841 trapv ? NULL_RTX
1842 : gen_rtx_fmt_ee (optab_to_code (binoptab),
1843 mode, op0, op1), trapv);
1844
1845 return target;
1846 }
1847
1848 delete_insns_since (last);
1849
1850 /* It can't be done in this mode. Can we do it in a wider mode? */
1851
1852 if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
1853 || methods == OPTAB_MUST_WIDEN))
1854 {
1855 /* Caller says, don't even try. */
1856 delete_insns_since (entry_last);
1857 return 0;
1858 }
1859
1860 /* Compute the value of METHODS to pass to recursive calls.
1861 Don't allow widening to be tried recursively. */
1862
1863 methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
1864
1865 /* Look for a wider mode of the same class for which it appears we can do
1866 the operation. */
1867
1868 if (CLASS_HAS_WIDER_MODES_P (mclass))
1869 {
1870 /* This code doesn't make sense for conversion optabs, since we
1871 wouldn't then want to extend the operands to be the same size
1872 as the result. */
1873 gcc_assert (!convert_optab_p (binoptab));
1874 FOR_EACH_WIDER_MODE (wider_mode, mode)
1875 {
1876 if (optab_handler (binoptab, wider_mode)
1877 || (methods == OPTAB_LIB
1878 && optab_libfunc (binoptab, wider_mode)))
1879 {
1880 rtx xop0 = op0, xop1 = op1;
1881 int no_extend = 0;
1882
1883 /* For certain integer operations, we need not actually extend
1884 the narrow operands, as long as we will truncate
1885 the results to the same narrowness. */
1886
1887 if ((binoptab == ior_optab || binoptab == and_optab
1888 || binoptab == xor_optab
1889 || binoptab == add_optab || binoptab == sub_optab
1890 || binoptab == smul_optab || binoptab == ashl_optab)
1891 && mclass == MODE_INT)
1892 no_extend = 1;
1893
1894 xop0 = widen_operand (xop0, wider_mode, mode,
1895 unsignedp, no_extend);
1896
1897 /* The second operand of a shift must always be extended. */
1898 xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1899 no_extend && binoptab != ashl_optab);
1900
1901 temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1902 unsignedp, methods);
1903 if (temp)
1904 {
1905 if (mclass != MODE_INT
1906 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1907 {
1908 if (target == 0)
1909 target = gen_reg_rtx (mode);
1910 convert_move (target, temp, 0);
1911 return target;
1912 }
1913 else
1914 return gen_lowpart (mode, temp);
1915 }
1916 else
1917 delete_insns_since (last);
1918 }
1919 }
1920 }
1921
1922 delete_insns_since (entry_last);
1923 return 0;
1924 }
1925 \f
1926 /* Expand a binary operator which has both signed and unsigned forms.
1927 UOPTAB is the optab for unsigned operations, and SOPTAB is for
1928 signed operations.
1929
1930 If we widen unsigned operands, we may use a signed wider operation instead
1931 of an unsigned wider operation, since the result would be the same. */
1932
1933 rtx
1934 sign_expand_binop (machine_mode mode, optab uoptab, optab soptab,
1935 rtx op0, rtx op1, rtx target, int unsignedp,
1936 enum optab_methods methods)
1937 {
1938 rtx temp;
1939 optab direct_optab = unsignedp ? uoptab : soptab;
1940 bool save_enable;
1941
1942 /* Do it without widening, if possible. */
1943 temp = expand_binop (mode, direct_optab, op0, op1, target,
1944 unsignedp, OPTAB_DIRECT);
1945 if (temp || methods == OPTAB_DIRECT)
1946 return temp;
1947
1948 /* Try widening to a signed int. Disable any direct use of any
1949 signed insn in the current mode. */
1950 save_enable = swap_optab_enable (soptab, mode, false);
1951
1952 temp = expand_binop (mode, soptab, op0, op1, target,
1953 unsignedp, OPTAB_WIDEN);
1954
1955 /* For unsigned operands, try widening to an unsigned int. */
1956 if (!temp && unsignedp)
1957 temp = expand_binop (mode, uoptab, op0, op1, target,
1958 unsignedp, OPTAB_WIDEN);
1959 if (temp || methods == OPTAB_WIDEN)
1960 goto egress;
1961
1962 /* Use the right width libcall if that exists. */
1963 temp = expand_binop (mode, direct_optab, op0, op1, target,
1964 unsignedp, OPTAB_LIB);
1965 if (temp || methods == OPTAB_LIB)
1966 goto egress;
1967
1968 /* Must widen and use a libcall, use either signed or unsigned. */
1969 temp = expand_binop (mode, soptab, op0, op1, target,
1970 unsignedp, methods);
1971 if (!temp && unsignedp)
1972 temp = expand_binop (mode, uoptab, op0, op1, target,
1973 unsignedp, methods);
1974
1975 egress:
1976 /* Undo the fiddling above. */
1977 if (save_enable)
1978 swap_optab_enable (soptab, mode, true);
1979 return temp;
1980 }
1981 \f
1982 /* Generate code to perform an operation specified by UNOPPTAB
1983 on operand OP0, with two results to TARG0 and TARG1.
1984 We assume that the order of the operands for the instruction
1985 is TARG0, TARG1, OP0.
1986
1987 Either TARG0 or TARG1 may be zero, but what that means is that
1988 the result is not actually wanted. We will generate it into
1989 a dummy pseudo-reg and discard it. They may not both be zero.
1990
1991 Returns 1 if this operation can be performed; 0 if not. */
1992
1993 int
1994 expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
1995 int unsignedp)
1996 {
1997 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
1998 enum mode_class mclass;
1999 machine_mode wider_mode;
2000 rtx_insn *entry_last = get_last_insn ();
2001 rtx_insn *last;
2002
2003 mclass = GET_MODE_CLASS (mode);
2004
2005 if (!targ0)
2006 targ0 = gen_reg_rtx (mode);
2007 if (!targ1)
2008 targ1 = gen_reg_rtx (mode);
2009
2010 /* Record where to go back to if we fail. */
2011 last = get_last_insn ();
2012
2013 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2014 {
2015 class expand_operand ops[3];
2016 enum insn_code icode = optab_handler (unoptab, mode);
2017
2018 create_fixed_operand (&ops[0], targ0);
2019 create_fixed_operand (&ops[1], targ1);
2020 create_convert_operand_from (&ops[2], op0, mode, unsignedp);
2021 if (maybe_expand_insn (icode, 3, ops))
2022 return 1;
2023 }
2024
2025 /* It can't be done in this mode. Can we do it in a wider mode? */
2026
2027 if (CLASS_HAS_WIDER_MODES_P (mclass))
2028 {
2029 FOR_EACH_WIDER_MODE (wider_mode, mode)
2030 {
2031 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2032 {
2033 rtx t0 = gen_reg_rtx (wider_mode);
2034 rtx t1 = gen_reg_rtx (wider_mode);
2035 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2036
2037 if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
2038 {
2039 convert_move (targ0, t0, unsignedp);
2040 convert_move (targ1, t1, unsignedp);
2041 return 1;
2042 }
2043 else
2044 delete_insns_since (last);
2045 }
2046 }
2047 }
2048
2049 delete_insns_since (entry_last);
2050 return 0;
2051 }
2052 \f
2053 /* Generate code to perform an operation specified by BINOPTAB
2054 on operands OP0 and OP1, with two results to TARG1 and TARG2.
2055 We assume that the order of the operands for the instruction
2056 is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2057 [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2058
2059 Either TARG0 or TARG1 may be zero, but what that means is that
2060 the result is not actually wanted. We will generate it into
2061 a dummy pseudo-reg and discard it. They may not both be zero.
2062
2063 Returns 1 if this operation can be performed; 0 if not. */
2064
2065 int
2066 expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2067 int unsignedp)
2068 {
2069 machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2070 enum mode_class mclass;
2071 machine_mode wider_mode;
2072 rtx_insn *entry_last = get_last_insn ();
2073 rtx_insn *last;
2074
2075 mclass = GET_MODE_CLASS (mode);
2076
2077 if (!targ0)
2078 targ0 = gen_reg_rtx (mode);
2079 if (!targ1)
2080 targ1 = gen_reg_rtx (mode);
2081
2082 /* Record where to go back to if we fail. */
2083 last = get_last_insn ();
2084
2085 if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
2086 {
2087 class expand_operand ops[4];
2088 enum insn_code icode = optab_handler (binoptab, mode);
2089 machine_mode mode0 = insn_data[icode].operand[1].mode;
2090 machine_mode mode1 = insn_data[icode].operand[2].mode;
2091 rtx xop0 = op0, xop1 = op1;
2092
2093 /* If we are optimizing, force expensive constants into a register. */
2094 xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
2095 xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
2096
2097 create_fixed_operand (&ops[0], targ0);
2098 create_convert_operand_from (&ops[1], xop0, mode, unsignedp);
2099 create_convert_operand_from (&ops[2], xop1, mode, unsignedp);
2100 create_fixed_operand (&ops[3], targ1);
2101 if (maybe_expand_insn (icode, 4, ops))
2102 return 1;
2103 delete_insns_since (last);
2104 }
2105
2106 /* It can't be done in this mode. Can we do it in a wider mode? */
2107
2108 if (CLASS_HAS_WIDER_MODES_P (mclass))
2109 {
2110 FOR_EACH_WIDER_MODE (wider_mode, mode)
2111 {
2112 if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
2113 {
2114 rtx t0 = gen_reg_rtx (wider_mode);
2115 rtx t1 = gen_reg_rtx (wider_mode);
2116 rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2117 rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
2118
2119 if (expand_twoval_binop (binoptab, cop0, cop1,
2120 t0, t1, unsignedp))
2121 {
2122 convert_move (targ0, t0, unsignedp);
2123 convert_move (targ1, t1, unsignedp);
2124 return 1;
2125 }
2126 else
2127 delete_insns_since (last);
2128 }
2129 }
2130 }
2131
2132 delete_insns_since (entry_last);
2133 return 0;
2134 }
2135
2136 /* Expand the two-valued library call indicated by BINOPTAB, but
2137 preserve only one of the values. If TARG0 is non-NULL, the first
2138 value is placed into TARG0; otherwise the second value is placed
2139 into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2140 value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2141 This routine assumes that the value returned by the library call is
2142 as if the return value was of an integral mode twice as wide as the
2143 mode of OP0. Returns 1 if the call was successful. */
2144
2145 bool
2146 expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2147 rtx targ0, rtx targ1, enum rtx_code code)
2148 {
2149 machine_mode mode;
2150 machine_mode libval_mode;
2151 rtx libval;
2152 rtx_insn *insns;
2153 rtx libfunc;
2154
2155 /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2156 gcc_assert (!targ0 != !targ1);
2157
2158 mode = GET_MODE (op0);
2159 libfunc = optab_libfunc (binoptab, mode);
2160 if (!libfunc)
2161 return false;
2162
2163 /* The value returned by the library function will have twice as
2164 many bits as the nominal MODE. */
2165 libval_mode = smallest_int_mode_for_size (2 * GET_MODE_BITSIZE (mode));
2166 start_sequence ();
2167 libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
2168 libval_mode,
2169 op0, mode,
2170 op1, mode);
2171 /* Get the part of VAL containing the value that we want. */
2172 libval = simplify_gen_subreg (mode, libval, libval_mode,
2173 targ0 ? 0 : GET_MODE_SIZE (mode));
2174 insns = get_insns ();
2175 end_sequence ();
2176 /* Move the into the desired location. */
2177 emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2178 gen_rtx_fmt_ee (code, mode, op0, op1));
2179
2180 return true;
2181 }
2182
2183 \f
2184 /* Wrapper around expand_unop which takes an rtx code to specify
2185 the operation to perform, not an optab pointer. All other
2186 arguments are the same. */
2187 rtx
2188 expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0,
2189 rtx target, int unsignedp)
2190 {
2191 optab unop = code_to_optab (code);
2192 gcc_assert (unop);
2193
2194 return expand_unop (mode, unop, op0, target, unsignedp);
2195 }
2196
2197 /* Try calculating
2198 (clz:narrow x)
2199 as
2200 (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2201
2202 A similar operation can be used for clrsb. UNOPTAB says which operation
2203 we are trying to expand. */
2204 static rtx
2205 widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab)
2206 {
2207 opt_scalar_int_mode wider_mode_iter;
2208 FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2209 {
2210 scalar_int_mode wider_mode = wider_mode_iter.require ();
2211 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2212 {
2213 rtx xop0, temp;
2214 rtx_insn *last;
2215
2216 last = get_last_insn ();
2217
2218 if (target == 0)
2219 target = gen_reg_rtx (mode);
2220 xop0 = widen_operand (op0, wider_mode, mode,
2221 unoptab != clrsb_optab, false);
2222 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2223 unoptab != clrsb_optab);
2224 if (temp != 0)
2225 temp = expand_binop
2226 (wider_mode, sub_optab, temp,
2227 gen_int_mode (GET_MODE_PRECISION (wider_mode)
2228 - GET_MODE_PRECISION (mode),
2229 wider_mode),
2230 target, true, OPTAB_DIRECT);
2231 if (temp == 0)
2232 delete_insns_since (last);
2233
2234 return temp;
2235 }
2236 }
2237 return 0;
2238 }
2239
2240 /* Try calculating clz of a double-word quantity as two clz's of word-sized
2241 quantities, choosing which based on whether the high word is nonzero. */
2242 static rtx
2243 expand_doubleword_clz (scalar_int_mode mode, rtx op0, rtx target)
2244 {
2245 rtx xop0 = force_reg (mode, op0);
2246 rtx subhi = gen_highpart (word_mode, xop0);
2247 rtx sublo = gen_lowpart (word_mode, xop0);
2248 rtx_code_label *hi0_label = gen_label_rtx ();
2249 rtx_code_label *after_label = gen_label_rtx ();
2250 rtx_insn *seq;
2251 rtx temp, result;
2252
2253 /* If we were not given a target, use a word_mode register, not a
2254 'mode' register. The result will fit, and nobody is expecting
2255 anything bigger (the return type of __builtin_clz* is int). */
2256 if (!target)
2257 target = gen_reg_rtx (word_mode);
2258
2259 /* In any case, write to a word_mode scratch in both branches of the
2260 conditional, so we can ensure there is a single move insn setting
2261 'target' to tag a REG_EQUAL note on. */
2262 result = gen_reg_rtx (word_mode);
2263
2264 start_sequence ();
2265
2266 /* If the high word is not equal to zero,
2267 then clz of the full value is clz of the high word. */
2268 emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2269 word_mode, true, hi0_label);
2270
2271 temp = expand_unop_direct (word_mode, clz_optab, subhi, result, true);
2272 if (!temp)
2273 goto fail;
2274
2275 if (temp != result)
2276 convert_move (result, temp, true);
2277
2278 emit_jump_insn (targetm.gen_jump (after_label));
2279 emit_barrier ();
2280
2281 /* Else clz of the full value is clz of the low word plus the number
2282 of bits in the high word. */
2283 emit_label (hi0_label);
2284
2285 temp = expand_unop_direct (word_mode, clz_optab, sublo, 0, true);
2286 if (!temp)
2287 goto fail;
2288 temp = expand_binop (word_mode, add_optab, temp,
2289 gen_int_mode (GET_MODE_BITSIZE (word_mode), word_mode),
2290 result, true, OPTAB_DIRECT);
2291 if (!temp)
2292 goto fail;
2293 if (temp != result)
2294 convert_move (result, temp, true);
2295
2296 emit_label (after_label);
2297 convert_move (target, result, true);
2298
2299 seq = get_insns ();
2300 end_sequence ();
2301
2302 add_equal_note (seq, target, CLZ, xop0, NULL_RTX, mode);
2303 emit_insn (seq);
2304 return target;
2305
2306 fail:
2307 end_sequence ();
2308 return 0;
2309 }
2310
2311 /* Try calculating popcount of a double-word quantity as two popcount's of
2312 word-sized quantities and summing up the results. */
2313 static rtx
2314 expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target)
2315 {
2316 rtx t0, t1, t;
2317 rtx_insn *seq;
2318
2319 start_sequence ();
2320
2321 t0 = expand_unop_direct (word_mode, popcount_optab,
2322 operand_subword_force (op0, 0, mode), NULL_RTX,
2323 true);
2324 t1 = expand_unop_direct (word_mode, popcount_optab,
2325 operand_subword_force (op0, 1, mode), NULL_RTX,
2326 true);
2327 if (!t0 || !t1)
2328 {
2329 end_sequence ();
2330 return NULL_RTX;
2331 }
2332
2333 /* If we were not given a target, use a word_mode register, not a
2334 'mode' register. The result will fit, and nobody is expecting
2335 anything bigger (the return type of __builtin_popcount* is int). */
2336 if (!target)
2337 target = gen_reg_rtx (word_mode);
2338
2339 t = expand_binop (word_mode, add_optab, t0, t1, target, 0, OPTAB_DIRECT);
2340
2341 seq = get_insns ();
2342 end_sequence ();
2343
2344 add_equal_note (seq, t, POPCOUNT, op0, NULL_RTX, mode);
2345 emit_insn (seq);
2346 return t;
2347 }
2348
2349 /* Try calculating
2350 (parity:wide x)
2351 as
2352 (parity:narrow (low (x) ^ high (x))) */
2353 static rtx
2354 expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target)
2355 {
2356 rtx t = expand_binop (word_mode, xor_optab,
2357 operand_subword_force (op0, 0, mode),
2358 operand_subword_force (op0, 1, mode),
2359 NULL_RTX, 0, OPTAB_DIRECT);
2360 return expand_unop (word_mode, parity_optab, t, target, true);
2361 }
2362
2363 /* Try calculating
2364 (bswap:narrow x)
2365 as
2366 (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2367 static rtx
2368 widen_bswap (scalar_int_mode mode, rtx op0, rtx target)
2369 {
2370 rtx x;
2371 rtx_insn *last;
2372 opt_scalar_int_mode wider_mode_iter;
2373
2374 FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2375 if (optab_handler (bswap_optab, wider_mode_iter.require ())
2376 != CODE_FOR_nothing)
2377 break;
2378
2379 if (!wider_mode_iter.exists ())
2380 return NULL_RTX;
2381
2382 scalar_int_mode wider_mode = wider_mode_iter.require ();
2383 last = get_last_insn ();
2384
2385 x = widen_operand (op0, wider_mode, mode, true, true);
2386 x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true);
2387
2388 gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2389 && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2390 if (x != 0)
2391 x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2392 GET_MODE_BITSIZE (wider_mode)
2393 - GET_MODE_BITSIZE (mode),
2394 NULL_RTX, true);
2395
2396 if (x != 0)
2397 {
2398 if (target == 0)
2399 target = gen_reg_rtx (mode);
2400 emit_move_insn (target, gen_lowpart (mode, x));
2401 }
2402 else
2403 delete_insns_since (last);
2404
2405 return target;
2406 }
2407
2408 /* Try calculating bswap as two bswaps of two word-sized operands. */
2409
2410 static rtx
2411 expand_doubleword_bswap (machine_mode mode, rtx op, rtx target)
2412 {
2413 rtx t0, t1;
2414
2415 t1 = expand_unop (word_mode, bswap_optab,
2416 operand_subword_force (op, 0, mode), NULL_RTX, true);
2417 t0 = expand_unop (word_mode, bswap_optab,
2418 operand_subword_force (op, 1, mode), NULL_RTX, true);
2419
2420 if (target == 0 || !valid_multiword_target_p (target))
2421 target = gen_reg_rtx (mode);
2422 if (REG_P (target))
2423 emit_clobber (target);
2424 emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2425 emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2426
2427 return target;
2428 }
2429
2430 /* Try calculating (parity x) as (and (popcount x) 1), where
2431 popcount can also be done in a wider mode. */
2432 static rtx
2433 expand_parity (scalar_int_mode mode, rtx op0, rtx target)
2434 {
2435 enum mode_class mclass = GET_MODE_CLASS (mode);
2436 opt_scalar_int_mode wider_mode_iter;
2437 FOR_EACH_MODE_FROM (wider_mode_iter, mode)
2438 {
2439 scalar_int_mode wider_mode = wider_mode_iter.require ();
2440 if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
2441 {
2442 rtx xop0, temp;
2443 rtx_insn *last;
2444
2445 last = get_last_insn ();
2446
2447 if (target == 0 || GET_MODE (target) != wider_mode)
2448 target = gen_reg_rtx (wider_mode);
2449
2450 xop0 = widen_operand (op0, wider_mode, mode, true, false);
2451 temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
2452 true);
2453 if (temp != 0)
2454 temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
2455 target, true, OPTAB_DIRECT);
2456
2457 if (temp)
2458 {
2459 if (mclass != MODE_INT
2460 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2461 return convert_to_mode (mode, temp, 0);
2462 else
2463 return gen_lowpart (mode, temp);
2464 }
2465 else
2466 delete_insns_since (last);
2467 }
2468 }
2469 return 0;
2470 }
2471
2472 /* Try calculating ctz(x) as K - clz(x & -x) ,
2473 where K is GET_MODE_PRECISION(mode) - 1.
2474
2475 Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2476 don't have to worry about what the hardware does in that case. (If
2477 the clz instruction produces the usual value at 0, which is K, the
2478 result of this code sequence will be -1; expand_ffs, below, relies
2479 on this. It might be nice to have it be K instead, for consistency
2480 with the (very few) processors that provide a ctz with a defined
2481 value, but that would take one more instruction, and it would be
2482 less convenient for expand_ffs anyway. */
2483
2484 static rtx
2485 expand_ctz (scalar_int_mode mode, rtx op0, rtx target)
2486 {
2487 rtx_insn *seq;
2488 rtx temp;
2489
2490 if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2491 return 0;
2492
2493 start_sequence ();
2494
2495 temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
2496 if (temp)
2497 temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
2498 true, OPTAB_DIRECT);
2499 if (temp)
2500 temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
2501 if (temp)
2502 temp = expand_binop (mode, sub_optab,
2503 gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
2504 temp, target,
2505 true, OPTAB_DIRECT);
2506 if (temp == 0)
2507 {
2508 end_sequence ();
2509 return 0;
2510 }
2511
2512 seq = get_insns ();
2513 end_sequence ();
2514
2515 add_equal_note (seq, temp, CTZ, op0, NULL_RTX, mode);
2516 emit_insn (seq);
2517 return temp;
2518 }
2519
2520
2521 /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
2522 else with the sequence used by expand_clz.
2523
2524 The ffs builtin promises to return zero for a zero value and ctz/clz
2525 may have an undefined value in that case. If they do not give us a
2526 convenient value, we have to generate a test and branch. */
2527 static rtx
2528 expand_ffs (scalar_int_mode mode, rtx op0, rtx target)
2529 {
2530 HOST_WIDE_INT val = 0;
2531 bool defined_at_zero = false;
2532 rtx temp;
2533 rtx_insn *seq;
2534
2535 if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
2536 {
2537 start_sequence ();
2538
2539 temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
2540 if (!temp)
2541 goto fail;
2542
2543 defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
2544 }
2545 else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
2546 {
2547 start_sequence ();
2548 temp = expand_ctz (mode, op0, 0);
2549 if (!temp)
2550 goto fail;
2551
2552 if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
2553 {
2554 defined_at_zero = true;
2555 val = (GET_MODE_PRECISION (mode) - 1) - val;
2556 }
2557 }
2558 else
2559 return 0;
2560
2561 if (defined_at_zero && val == -1)
2562 /* No correction needed at zero. */;
2563 else
2564 {
2565 /* We don't try to do anything clever with the situation found
2566 on some processors (eg Alpha) where ctz(0:mode) ==
2567 bitsize(mode). If someone can think of a way to send N to -1
2568 and leave alone all values in the range 0..N-1 (where N is a
2569 power of two), cheaper than this test-and-branch, please add it.
2570
2571 The test-and-branch is done after the operation itself, in case
2572 the operation sets condition codes that can be recycled for this.
2573 (This is true on i386, for instance.) */
2574
2575 rtx_code_label *nonzero_label = gen_label_rtx ();
2576 emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
2577 mode, true, nonzero_label);
2578
2579 convert_move (temp, GEN_INT (-1), false);
2580 emit_label (nonzero_label);
2581 }
2582
2583 /* temp now has a value in the range -1..bitsize-1. ffs is supposed
2584 to produce a value in the range 0..bitsize. */
2585 temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode),
2586 target, false, OPTAB_DIRECT);
2587 if (!temp)
2588 goto fail;
2589
2590 seq = get_insns ();
2591 end_sequence ();
2592
2593 add_equal_note (seq, temp, FFS, op0, NULL_RTX, mode);
2594 emit_insn (seq);
2595 return temp;
2596
2597 fail:
2598 end_sequence ();
2599 return 0;
2600 }
2601
2602 /* Extract the OMODE lowpart from VAL, which has IMODE. Under certain
2603 conditions, VAL may already be a SUBREG against which we cannot generate
2604 a further SUBREG. In this case, we expect forcing the value into a
2605 register will work around the situation. */
2606
2607 static rtx
2608 lowpart_subreg_maybe_copy (machine_mode omode, rtx val,
2609 machine_mode imode)
2610 {
2611 rtx ret;
2612 ret = lowpart_subreg (omode, val, imode);
2613 if (ret == NULL)
2614 {
2615 val = force_reg (imode, val);
2616 ret = lowpart_subreg (omode, val, imode);
2617 gcc_assert (ret != NULL);
2618 }
2619 return ret;
2620 }
2621
2622 /* Expand a floating point absolute value or negation operation via a
2623 logical operation on the sign bit. */
2624
2625 static rtx
2626 expand_absneg_bit (enum rtx_code code, scalar_float_mode mode,
2627 rtx op0, rtx target)
2628 {
2629 const struct real_format *fmt;
2630 int bitpos, word, nwords, i;
2631 scalar_int_mode imode;
2632 rtx temp;
2633 rtx_insn *insns;
2634
2635 /* The format has to have a simple sign bit. */
2636 fmt = REAL_MODE_FORMAT (mode);
2637 if (fmt == NULL)
2638 return NULL_RTX;
2639
2640 bitpos = fmt->signbit_rw;
2641 if (bitpos < 0)
2642 return NULL_RTX;
2643
2644 /* Don't create negative zeros if the format doesn't support them. */
2645 if (code == NEG && !fmt->has_signed_zero)
2646 return NULL_RTX;
2647
2648 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
2649 {
2650 if (!int_mode_for_mode (mode).exists (&imode))
2651 return NULL_RTX;
2652 word = 0;
2653 nwords = 1;
2654 }
2655 else
2656 {
2657 imode = word_mode;
2658
2659 if (FLOAT_WORDS_BIG_ENDIAN)
2660 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
2661 else
2662 word = bitpos / BITS_PER_WORD;
2663 bitpos = bitpos % BITS_PER_WORD;
2664 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
2665 }
2666
2667 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
2668 if (code == ABS)
2669 mask = ~mask;
2670
2671 if (target == 0
2672 || target == op0
2673 || (nwords > 1 && !valid_multiword_target_p (target)))
2674 target = gen_reg_rtx (mode);
2675
2676 if (nwords > 1)
2677 {
2678 start_sequence ();
2679
2680 for (i = 0; i < nwords; ++i)
2681 {
2682 rtx targ_piece = operand_subword (target, i, 1, mode);
2683 rtx op0_piece = operand_subword_force (op0, i, mode);
2684
2685 if (i == word)
2686 {
2687 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2688 op0_piece,
2689 immed_wide_int_const (mask, imode),
2690 targ_piece, 1, OPTAB_LIB_WIDEN);
2691 if (temp != targ_piece)
2692 emit_move_insn (targ_piece, temp);
2693 }
2694 else
2695 emit_move_insn (targ_piece, op0_piece);
2696 }
2697
2698 insns = get_insns ();
2699 end_sequence ();
2700
2701 emit_insn (insns);
2702 }
2703 else
2704 {
2705 temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
2706 gen_lowpart (imode, op0),
2707 immed_wide_int_const (mask, imode),
2708 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
2709 target = lowpart_subreg_maybe_copy (mode, temp, imode);
2710
2711 set_dst_reg_note (get_last_insn (), REG_EQUAL,
2712 gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
2713 target);
2714 }
2715
2716 return target;
2717 }
2718
2719 /* As expand_unop, but will fail rather than attempt the operation in a
2720 different mode or with a libcall. */
2721 static rtx
2722 expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target,
2723 int unsignedp)
2724 {
2725 if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2726 {
2727 class expand_operand ops[2];
2728 enum insn_code icode = optab_handler (unoptab, mode);
2729 rtx_insn *last = get_last_insn ();
2730 rtx_insn *pat;
2731
2732 create_output_operand (&ops[0], target, mode);
2733 create_convert_operand_from (&ops[1], op0, mode, unsignedp);
2734 pat = maybe_gen_insn (icode, 2, ops);
2735 if (pat)
2736 {
2737 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
2738 && ! add_equal_note (pat, ops[0].value,
2739 optab_to_code (unoptab),
2740 ops[1].value, NULL_RTX, mode))
2741 {
2742 delete_insns_since (last);
2743 return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
2744 }
2745
2746 emit_insn (pat);
2747
2748 return ops[0].value;
2749 }
2750 }
2751 return 0;
2752 }
2753
2754 /* Generate code to perform an operation specified by UNOPTAB
2755 on operand OP0, with result having machine-mode MODE.
2756
2757 UNSIGNEDP is for the case where we have to widen the operands
2758 to perform the operation. It says to use zero-extension.
2759
2760 If TARGET is nonzero, the value
2761 is generated there, if it is convenient to do so.
2762 In all cases an rtx is returned for the locus of the value;
2763 this may or may not be TARGET. */
2764
2765 rtx
2766 expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target,
2767 int unsignedp)
2768 {
2769 enum mode_class mclass = GET_MODE_CLASS (mode);
2770 machine_mode wider_mode;
2771 scalar_int_mode int_mode;
2772 scalar_float_mode float_mode;
2773 rtx temp;
2774 rtx libfunc;
2775
2776 temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
2777 if (temp)
2778 return temp;
2779
2780 /* It can't be done in this mode. Can we open-code it in a wider mode? */
2781
2782 /* Widening (or narrowing) clz needs special treatment. */
2783 if (unoptab == clz_optab)
2784 {
2785 if (is_a <scalar_int_mode> (mode, &int_mode))
2786 {
2787 temp = widen_leading (int_mode, op0, target, unoptab);
2788 if (temp)
2789 return temp;
2790
2791 if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2792 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2793 {
2794 temp = expand_doubleword_clz (int_mode, op0, target);
2795 if (temp)
2796 return temp;
2797 }
2798 }
2799
2800 goto try_libcall;
2801 }
2802
2803 if (unoptab == clrsb_optab)
2804 {
2805 if (is_a <scalar_int_mode> (mode, &int_mode))
2806 {
2807 temp = widen_leading (int_mode, op0, target, unoptab);
2808 if (temp)
2809 return temp;
2810 }
2811 goto try_libcall;
2812 }
2813
2814 if (unoptab == popcount_optab
2815 && is_a <scalar_int_mode> (mode, &int_mode)
2816 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2817 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing
2818 && optimize_insn_for_speed_p ())
2819 {
2820 temp = expand_doubleword_popcount (int_mode, op0, target);
2821 if (temp)
2822 return temp;
2823 }
2824
2825 if (unoptab == parity_optab
2826 && is_a <scalar_int_mode> (mode, &int_mode)
2827 && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2828 && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
2829 || optab_handler (popcount_optab, word_mode) != CODE_FOR_nothing)
2830 && optimize_insn_for_speed_p ())
2831 {
2832 temp = expand_doubleword_parity (int_mode, op0, target);
2833 if (temp)
2834 return temp;
2835 }
2836
2837 /* Widening (or narrowing) bswap needs special treatment. */
2838 if (unoptab == bswap_optab)
2839 {
2840 /* HImode is special because in this mode BSWAP is equivalent to ROTATE
2841 or ROTATERT. First try these directly; if this fails, then try the
2842 obvious pair of shifts with allowed widening, as this will probably
2843 be always more efficient than the other fallback methods. */
2844 if (mode == HImode)
2845 {
2846 rtx_insn *last;
2847 rtx temp1, temp2;
2848
2849 if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
2850 {
2851 temp = expand_binop (mode, rotl_optab, op0,
2852 gen_int_shift_amount (mode, 8),
2853 target, unsignedp, OPTAB_DIRECT);
2854 if (temp)
2855 return temp;
2856 }
2857
2858 if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
2859 {
2860 temp = expand_binop (mode, rotr_optab, op0,
2861 gen_int_shift_amount (mode, 8),
2862 target, unsignedp, OPTAB_DIRECT);
2863 if (temp)
2864 return temp;
2865 }
2866
2867 last = get_last_insn ();
2868
2869 temp1 = expand_binop (mode, ashl_optab, op0,
2870 gen_int_shift_amount (mode, 8), NULL_RTX,
2871 unsignedp, OPTAB_WIDEN);
2872 temp2 = expand_binop (mode, lshr_optab, op0,
2873 gen_int_shift_amount (mode, 8), NULL_RTX,
2874 unsignedp, OPTAB_WIDEN);
2875 if (temp1 && temp2)
2876 {
2877 temp = expand_binop (mode, ior_optab, temp1, temp2, target,
2878 unsignedp, OPTAB_WIDEN);
2879 if (temp)
2880 return temp;
2881 }
2882
2883 delete_insns_since (last);
2884 }
2885
2886 if (is_a <scalar_int_mode> (mode, &int_mode))
2887 {
2888 temp = widen_bswap (int_mode, op0, target);
2889 if (temp)
2890 return temp;
2891
2892 if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2893 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2894 {
2895 temp = expand_doubleword_bswap (mode, op0, target);
2896 if (temp)
2897 return temp;
2898 }
2899 }
2900
2901 goto try_libcall;
2902 }
2903
2904 if (CLASS_HAS_WIDER_MODES_P (mclass))
2905 FOR_EACH_WIDER_MODE (wider_mode, mode)
2906 {
2907 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2908 {
2909 rtx xop0 = op0;
2910 rtx_insn *last = get_last_insn ();
2911
2912 /* For certain operations, we need not actually extend
2913 the narrow operand, as long as we will truncate the
2914 results to the same narrowness. */
2915
2916 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
2917 (unoptab == neg_optab
2918 || unoptab == one_cmpl_optab)
2919 && mclass == MODE_INT);
2920
2921 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2922 unsignedp);
2923
2924 if (temp)
2925 {
2926 if (mclass != MODE_INT
2927 || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2928 {
2929 if (target == 0)
2930 target = gen_reg_rtx (mode);
2931 convert_move (target, temp, 0);
2932 return target;
2933 }
2934 else
2935 return gen_lowpart (mode, temp);
2936 }
2937 else
2938 delete_insns_since (last);
2939 }
2940 }
2941
2942 /* These can be done a word at a time. */
2943 if (unoptab == one_cmpl_optab
2944 && is_int_mode (mode, &int_mode)
2945 && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
2946 && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2947 {
2948 int i;
2949 rtx_insn *insns;
2950
2951 if (target == 0 || target == op0 || !valid_multiword_target_p (target))
2952 target = gen_reg_rtx (int_mode);
2953
2954 start_sequence ();
2955
2956 /* Do the actual arithmetic. */
2957 for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
2958 {
2959 rtx target_piece = operand_subword (target, i, 1, int_mode);
2960 rtx x = expand_unop (word_mode, unoptab,
2961 operand_subword_force (op0, i, int_mode),
2962 target_piece, unsignedp);
2963
2964 if (target_piece != x)
2965 emit_move_insn (target_piece, x);
2966 }
2967
2968 insns = get_insns ();
2969 end_sequence ();
2970
2971 emit_insn (insns);
2972 return target;
2973 }
2974
2975 /* Emit ~op0 as op0 ^ -1. */
2976 if (unoptab == one_cmpl_optab
2977 && (SCALAR_INT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
2978 && optab_handler (xor_optab, mode) != CODE_FOR_nothing)
2979 {
2980 temp = expand_binop (mode, xor_optab, op0, CONSTM1_RTX (mode),
2981 target, unsignedp, OPTAB_DIRECT);
2982 if (temp)
2983 return temp;
2984 }
2985
2986 if (optab_to_code (unoptab) == NEG)
2987 {
2988 /* Try negating floating point values by flipping the sign bit. */
2989 if (is_a <scalar_float_mode> (mode, &float_mode))
2990 {
2991 temp = expand_absneg_bit (NEG, float_mode, op0, target);
2992 if (temp)
2993 return temp;
2994 }
2995
2996 /* If there is no negation pattern, and we have no negative zero,
2997 try subtracting from zero. */
2998 if (!HONOR_SIGNED_ZEROS (mode))
2999 {
3000 temp = expand_binop (mode, (unoptab == negv_optab
3001 ? subv_optab : sub_optab),
3002 CONST0_RTX (mode), op0, target,
3003 unsignedp, OPTAB_DIRECT);
3004 if (temp)
3005 return temp;
3006 }
3007 }
3008
3009 /* Try calculating parity (x) as popcount (x) % 2. */
3010 if (unoptab == parity_optab && is_a <scalar_int_mode> (mode, &int_mode))
3011 {
3012 temp = expand_parity (int_mode, op0, target);
3013 if (temp)
3014 return temp;
3015 }
3016
3017 /* Try implementing ffs (x) in terms of clz (x). */
3018 if (unoptab == ffs_optab && is_a <scalar_int_mode> (mode, &int_mode))
3019 {
3020 temp = expand_ffs (int_mode, op0, target);
3021 if (temp)
3022 return temp;
3023 }
3024
3025 /* Try implementing ctz (x) in terms of clz (x). */
3026 if (unoptab == ctz_optab && is_a <scalar_int_mode> (mode, &int_mode))
3027 {
3028 temp = expand_ctz (int_mode, op0, target);
3029 if (temp)
3030 return temp;
3031 }
3032
3033 try_libcall:
3034 /* Now try a library call in this mode. */
3035 libfunc = optab_libfunc (unoptab, mode);
3036 if (libfunc)
3037 {
3038 rtx_insn *insns;
3039 rtx value;
3040 rtx eq_value;
3041 machine_mode outmode = mode;
3042
3043 /* All of these functions return small values. Thus we choose to
3044 have them return something that isn't a double-word. */
3045 if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
3046 || unoptab == clrsb_optab || unoptab == popcount_optab
3047 || unoptab == parity_optab)
3048 outmode
3049 = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
3050 optab_libfunc (unoptab, mode)));
3051
3052 start_sequence ();
3053
3054 /* Pass 1 for NO_QUEUE so we don't lose any increments
3055 if the libcall is cse'd or moved. */
3056 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
3057 op0, mode);
3058 insns = get_insns ();
3059 end_sequence ();
3060
3061 target = gen_reg_rtx (outmode);
3062 bool trapv = trapv_unoptab_p (unoptab);
3063 if (trapv)
3064 eq_value = NULL_RTX;
3065 else
3066 {
3067 eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
3068 if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode))
3069 eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
3070 else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode))
3071 eq_value = simplify_gen_unary (ZERO_EXTEND,
3072 outmode, eq_value, mode);
3073 }
3074 emit_libcall_block_1 (insns, target, value, eq_value, trapv);
3075
3076 return target;
3077 }
3078
3079 /* It can't be done in this mode. Can we do it in a wider mode? */
3080
3081 if (CLASS_HAS_WIDER_MODES_P (mclass))
3082 {
3083 FOR_EACH_WIDER_MODE (wider_mode, mode)
3084 {
3085 if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
3086 || optab_libfunc (unoptab, wider_mode))
3087 {
3088 rtx xop0 = op0;
3089 rtx_insn *last = get_last_insn ();
3090
3091 /* For certain operations, we need not actually extend
3092 the narrow operand, as long as we will truncate the
3093 results to the same narrowness. */
3094 xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3095 (unoptab == neg_optab
3096 || unoptab == one_cmpl_optab
3097 || unoptab == bswap_optab)
3098 && mclass == MODE_INT);
3099
3100 temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3101 unsignedp);
3102
3103 /* If we are generating clz using wider mode, adjust the
3104 result. Similarly for clrsb. */
3105 if ((unoptab == clz_optab || unoptab == clrsb_optab)
3106 && temp != 0)
3107 {
3108 scalar_int_mode wider_int_mode
3109 = as_a <scalar_int_mode> (wider_mode);
3110 int_mode = as_a <scalar_int_mode> (mode);
3111 temp = expand_binop
3112 (wider_mode, sub_optab, temp,
3113 gen_int_mode (GET_MODE_PRECISION (wider_int_mode)
3114 - GET_MODE_PRECISION (int_mode),
3115 wider_int_mode),
3116 target, true, OPTAB_DIRECT);
3117 }
3118
3119 /* Likewise for bswap. */
3120 if (unoptab == bswap_optab && temp != 0)
3121 {
3122 scalar_int_mode wider_int_mode
3123 = as_a <scalar_int_mode> (wider_mode);
3124 int_mode = as_a <scalar_int_mode> (mode);
3125 gcc_assert (GET_MODE_PRECISION (wider_int_mode)
3126 == GET_MODE_BITSIZE (wider_int_mode)
3127 && GET_MODE_PRECISION (int_mode)
3128 == GET_MODE_BITSIZE (int_mode));
3129
3130 temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp,
3131 GET_MODE_BITSIZE (wider_int_mode)
3132 - GET_MODE_BITSIZE (int_mode),
3133 NULL_RTX, true);
3134 }
3135
3136 if (temp)
3137 {
3138 if (mclass != MODE_INT)
3139 {
3140 if (target == 0)
3141 target = gen_reg_rtx (mode);
3142 convert_move (target, temp, 0);
3143 return target;
3144 }
3145 else
3146 return gen_lowpart (mode, temp);
3147 }
3148 else
3149 delete_insns_since (last);
3150 }
3151 }
3152 }
3153
3154 /* One final attempt at implementing negation via subtraction,
3155 this time allowing widening of the operand. */
3156 if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3157 {
3158 rtx temp;
3159 temp = expand_binop (mode,
3160 unoptab == negv_optab ? subv_optab : sub_optab,
3161 CONST0_RTX (mode), op0,
3162 target, unsignedp, OPTAB_LIB_WIDEN);
3163 if (temp)
3164 return temp;
3165 }
3166
3167 return 0;
3168 }
3169 \f
3170 /* Emit code to compute the absolute value of OP0, with result to
3171 TARGET if convenient. (TARGET may be 0.) The return value says
3172 where the result actually is to be found.
3173
3174 MODE is the mode of the operand; the mode of the result is
3175 different but can be deduced from MODE.
3176
3177 */
3178
3179 rtx
3180 expand_abs_nojump (machine_mode mode, rtx op0, rtx target,
3181 int result_unsignedp)
3182 {
3183 rtx temp;
3184
3185 if (GET_MODE_CLASS (mode) != MODE_INT
3186 || ! flag_trapv)
3187 result_unsignedp = 1;
3188
3189 /* First try to do it with a special abs instruction. */
3190 temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
3191 op0, target, 0);
3192 if (temp != 0)
3193 return temp;
3194
3195 /* For floating point modes, try clearing the sign bit. */
3196 scalar_float_mode float_mode;
3197 if (is_a <scalar_float_mode> (mode, &float_mode))
3198 {
3199 temp = expand_absneg_bit (ABS, float_mode, op0, target);
3200 if (temp)
3201 return temp;
3202 }
3203
3204 /* If we have a MAX insn, we can do this as MAX (x, -x). */
3205 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
3206 && !HONOR_SIGNED_ZEROS (mode))
3207 {
3208 rtx_insn *last = get_last_insn ();
3209
3210 temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3211 op0, NULL_RTX, 0);
3212 if (temp != 0)
3213 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3214 OPTAB_WIDEN);
3215
3216 if (temp != 0)
3217 return temp;
3218
3219 delete_insns_since (last);
3220 }
3221
3222 /* If this machine has expensive jumps, we can do integer absolute
3223 value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3224 where W is the width of MODE. */
3225
3226 scalar_int_mode int_mode;
3227 if (is_int_mode (mode, &int_mode)
3228 && BRANCH_COST (optimize_insn_for_speed_p (),
3229 false) >= 2)
3230 {
3231 rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3232 GET_MODE_PRECISION (int_mode) - 1,
3233 NULL_RTX, 0);
3234
3235 temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3236 OPTAB_LIB_WIDEN);
3237 if (temp != 0)
3238 temp = expand_binop (int_mode,
3239 result_unsignedp ? sub_optab : subv_optab,
3240 temp, extended, target, 0, OPTAB_LIB_WIDEN);
3241
3242 if (temp != 0)
3243 return temp;
3244 }
3245
3246 return NULL_RTX;
3247 }
3248
3249 rtx
3250 expand_abs (machine_mode mode, rtx op0, rtx target,
3251 int result_unsignedp, int safe)
3252 {
3253 rtx temp;
3254 rtx_code_label *op1;
3255
3256 if (GET_MODE_CLASS (mode) != MODE_INT
3257 || ! flag_trapv)
3258 result_unsignedp = 1;
3259
3260 temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3261 if (temp != 0)
3262 return temp;
3263
3264 /* If that does not win, use conditional jump and negate. */
3265
3266 /* It is safe to use the target if it is the same
3267 as the source if this is also a pseudo register */
3268 if (op0 == target && REG_P (op0)
3269 && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3270 safe = 1;
3271
3272 op1 = gen_label_rtx ();
3273 if (target == 0 || ! safe
3274 || GET_MODE (target) != mode
3275 || (MEM_P (target) && MEM_VOLATILE_P (target))
3276 || (REG_P (target)
3277 && REGNO (target) < FIRST_PSEUDO_REGISTER))
3278 target = gen_reg_rtx (mode);
3279
3280 emit_move_insn (target, op0);
3281 NO_DEFER_POP;
3282
3283 do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3284 NULL_RTX, NULL, op1,
3285 profile_probability::uninitialized ());
3286
3287 op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3288 target, target, 0);
3289 if (op0 != target)
3290 emit_move_insn (target, op0);
3291 emit_label (op1);
3292 OK_DEFER_POP;
3293 return target;
3294 }
3295
3296 /* Emit code to compute the one's complement absolute value of OP0
3297 (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3298 (TARGET may be NULL_RTX.) The return value says where the result
3299 actually is to be found.
3300
3301 MODE is the mode of the operand; the mode of the result is
3302 different but can be deduced from MODE. */
3303
3304 rtx
3305 expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target)
3306 {
3307 rtx temp;
3308
3309 /* Not applicable for floating point modes. */
3310 if (FLOAT_MODE_P (mode))
3311 return NULL_RTX;
3312
3313 /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3314 if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
3315 {
3316 rtx_insn *last = get_last_insn ();
3317
3318 temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
3319 if (temp != 0)
3320 temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3321 OPTAB_WIDEN);
3322
3323 if (temp != 0)
3324 return temp;
3325
3326 delete_insns_since (last);
3327 }
3328
3329 /* If this machine has expensive jumps, we can do one's complement
3330 absolute value of X as (((signed) x >> (W-1)) ^ x). */
3331
3332 scalar_int_mode int_mode;
3333 if (is_int_mode (mode, &int_mode)
3334 && BRANCH_COST (optimize_insn_for_speed_p (),
3335 false) >= 2)
3336 {
3337 rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3338 GET_MODE_PRECISION (int_mode) - 1,
3339 NULL_RTX, 0);
3340
3341 temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3342 OPTAB_LIB_WIDEN);
3343
3344 if (temp != 0)
3345 return temp;
3346 }
3347
3348 return NULL_RTX;
3349 }
3350
3351 /* A subroutine of expand_copysign, perform the copysign operation using the
3352 abs and neg primitives advertised to exist on the target. The assumption
3353 is that we have a split register file, and leaving op0 in fp registers,
3354 and not playing with subregs so much, will help the register allocator. */
3355
3356 static rtx
3357 expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3358 int bitpos, bool op0_is_abs)
3359 {
3360 scalar_int_mode imode;
3361 enum insn_code icode;
3362 rtx sign;
3363 rtx_code_label *label;
3364
3365 if (target == op1)
3366 target = NULL_RTX;
3367
3368 /* Check if the back end provides an insn that handles signbit for the
3369 argument's mode. */
3370 icode = optab_handler (signbit_optab, mode);
3371 if (icode != CODE_FOR_nothing)
3372 {
3373 imode = as_a <scalar_int_mode> (insn_data[(int) icode].operand[0].mode);
3374 sign = gen_reg_rtx (imode);
3375 emit_unop_insn (icode, sign, op1, UNKNOWN);
3376 }
3377 else
3378 {
3379 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3380 {
3381 if (!int_mode_for_mode (mode).exists (&imode))
3382 return NULL_RTX;
3383 op1 = gen_lowpart (imode, op1);
3384 }
3385 else
3386 {
3387 int word;
3388
3389 imode = word_mode;
3390 if (FLOAT_WORDS_BIG_ENDIAN)
3391 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3392 else
3393 word = bitpos / BITS_PER_WORD;
3394 bitpos = bitpos % BITS_PER_WORD;
3395 op1 = operand_subword_force (op1, word, mode);
3396 }
3397
3398 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3399 sign = expand_binop (imode, and_optab, op1,
3400 immed_wide_int_const (mask, imode),
3401 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3402 }
3403
3404 if (!op0_is_abs)
3405 {
3406 op0 = expand_unop (mode, abs_optab, op0, target, 0);
3407 if (op0 == NULL)
3408 return NULL_RTX;
3409 target = op0;
3410 }
3411 else
3412 {
3413 if (target == NULL_RTX)
3414 target = copy_to_reg (op0);
3415 else
3416 emit_move_insn (target, op0);
3417 }
3418
3419 label = gen_label_rtx ();
3420 emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
3421
3422 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3423 op0 = simplify_unary_operation (NEG, mode, op0, mode);
3424 else
3425 op0 = expand_unop (mode, neg_optab, op0, target, 0);
3426 if (op0 != target)
3427 emit_move_insn (target, op0);
3428
3429 emit_label (label);
3430
3431 return target;
3432 }
3433
3434
3435 /* A subroutine of expand_copysign, perform the entire copysign operation
3436 with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3437 is true if op0 is known to have its sign bit clear. */
3438
3439 static rtx
3440 expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3441 int bitpos, bool op0_is_abs)
3442 {
3443 scalar_int_mode imode;
3444 int word, nwords, i;
3445 rtx temp;
3446 rtx_insn *insns;
3447
3448 if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3449 {
3450 if (!int_mode_for_mode (mode).exists (&imode))
3451 return NULL_RTX;
3452 word = 0;
3453 nwords = 1;
3454 }
3455 else
3456 {
3457 imode = word_mode;
3458
3459 if (FLOAT_WORDS_BIG_ENDIAN)
3460 word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3461 else
3462 word = bitpos / BITS_PER_WORD;
3463 bitpos = bitpos % BITS_PER_WORD;
3464 nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3465 }
3466
3467 wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3468
3469 if (target == 0
3470 || target == op0
3471 || target == op1
3472 || (nwords > 1 && !valid_multiword_target_p (target)))
3473 target = gen_reg_rtx (mode);
3474
3475 if (nwords > 1)
3476 {
3477 start_sequence ();
3478
3479 for (i = 0; i < nwords; ++i)
3480 {
3481 rtx targ_piece = operand_subword (target, i, 1, mode);
3482 rtx op0_piece = operand_subword_force (op0, i, mode);
3483
3484 if (i == word)
3485 {
3486 if (!op0_is_abs)
3487 op0_piece
3488 = expand_binop (imode, and_optab, op0_piece,
3489 immed_wide_int_const (~mask, imode),
3490 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3491 op1 = expand_binop (imode, and_optab,
3492 operand_subword_force (op1, i, mode),
3493 immed_wide_int_const (mask, imode),
3494 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3495
3496 temp = expand_binop (imode, ior_optab, op0_piece, op1,
3497 targ_piece, 1, OPTAB_LIB_WIDEN);
3498 if (temp != targ_piece)
3499 emit_move_insn (targ_piece, temp);
3500 }
3501 else
3502 emit_move_insn (targ_piece, op0_piece);
3503 }
3504
3505 insns = get_insns ();
3506 end_sequence ();
3507
3508 emit_insn (insns);
3509 }
3510 else
3511 {
3512 op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
3513 immed_wide_int_const (mask, imode),
3514 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3515
3516 op0 = gen_lowpart (imode, op0);
3517 if (!op0_is_abs)
3518 op0 = expand_binop (imode, and_optab, op0,
3519 immed_wide_int_const (~mask, imode),
3520 NULL_RTX, 1, OPTAB_LIB_WIDEN);
3521
3522 temp = expand_binop (imode, ior_optab, op0, op1,
3523 gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
3524 target = lowpart_subreg_maybe_copy (mode, temp, imode);
3525 }
3526
3527 return target;
3528 }
3529
3530 /* Expand the C99 copysign operation. OP0 and OP1 must be the same
3531 scalar floating point mode. Return NULL if we do not know how to
3532 expand the operation inline. */
3533
3534 rtx
3535 expand_copysign (rtx op0, rtx op1, rtx target)
3536 {
3537 scalar_float_mode mode;
3538 const struct real_format *fmt;
3539 bool op0_is_abs;
3540 rtx temp;
3541
3542 mode = as_a <scalar_float_mode> (GET_MODE (op0));
3543 gcc_assert (GET_MODE (op1) == mode);
3544
3545 /* First try to do it with a special instruction. */
3546 temp = expand_binop (mode, copysign_optab, op0, op1,
3547 target, 0, OPTAB_DIRECT);
3548 if (temp)
3549 return temp;
3550
3551 fmt = REAL_MODE_FORMAT (mode);
3552 if (fmt == NULL || !fmt->has_signed_zero)
3553 return NULL_RTX;
3554
3555 op0_is_abs = false;
3556 if (CONST_DOUBLE_AS_FLOAT_P (op0))
3557 {
3558 if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
3559 op0 = simplify_unary_operation (ABS, mode, op0, mode);
3560 op0_is_abs = true;
3561 }
3562
3563 if (fmt->signbit_ro >= 0
3564 && (CONST_DOUBLE_AS_FLOAT_P (op0)
3565 || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
3566 && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
3567 {
3568 temp = expand_copysign_absneg (mode, op0, op1, target,
3569 fmt->signbit_ro, op0_is_abs);
3570 if (temp)
3571 return temp;
3572 }
3573
3574 if (fmt->signbit_rw < 0)
3575 return NULL_RTX;
3576 return expand_copysign_bit (mode, op0, op1, target,
3577 fmt->signbit_rw, op0_is_abs);
3578 }
3579 \f
3580 /* Generate an instruction whose insn-code is INSN_CODE,
3581 with two operands: an output TARGET and an input OP0.
3582 TARGET *must* be nonzero, and the output is always stored there.
3583 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3584 the value that is stored into TARGET.
3585
3586 Return false if expansion failed. */
3587
3588 bool
3589 maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
3590 enum rtx_code code)
3591 {
3592 class expand_operand ops[2];
3593 rtx_insn *pat;
3594
3595 create_output_operand (&ops[0], target, GET_MODE (target));
3596 create_input_operand (&ops[1], op0, GET_MODE (op0));
3597 pat = maybe_gen_insn (icode, 2, ops);
3598 if (!pat)
3599 return false;
3600
3601 if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
3602 && code != UNKNOWN)
3603 add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX,
3604 GET_MODE (op0));
3605
3606 emit_insn (pat);
3607
3608 if (ops[0].value != target)
3609 emit_move_insn (target, ops[0].value);
3610 return true;
3611 }
3612 /* Generate an instruction whose insn-code is INSN_CODE,
3613 with two operands: an output TARGET and an input OP0.
3614 TARGET *must* be nonzero, and the output is always stored there.
3615 CODE is an rtx code such that (CODE OP0) is an rtx that describes
3616 the value that is stored into TARGET. */
3617
3618 void
3619 emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
3620 {
3621 bool ok = maybe_emit_unop_insn (icode, target, op0, code);
3622 gcc_assert (ok);
3623 }
3624 \f
3625 struct no_conflict_data
3626 {
3627 rtx target;
3628 rtx_insn *first, *insn;
3629 bool must_stay;
3630 };
3631
3632 /* Called via note_stores by emit_libcall_block. Set P->must_stay if
3633 the currently examined clobber / store has to stay in the list of
3634 insns that constitute the actual libcall block. */
3635 static void
3636 no_conflict_move_test (rtx dest, const_rtx set, void *p0)
3637 {
3638 struct no_conflict_data *p= (struct no_conflict_data *) p0;
3639
3640 /* If this inns directly contributes to setting the target, it must stay. */
3641 if (reg_overlap_mentioned_p (p->target, dest))
3642 p->must_stay = true;
3643 /* If we haven't committed to keeping any other insns in the list yet,
3644 there is nothing more to check. */
3645 else if (p->insn == p->first)
3646 return;
3647 /* If this insn sets / clobbers a register that feeds one of the insns
3648 already in the list, this insn has to stay too. */
3649 else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
3650 || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
3651 || reg_used_between_p (dest, p->first, p->insn)
3652 /* Likewise if this insn depends on a register set by a previous
3653 insn in the list, or if it sets a result (presumably a hard
3654 register) that is set or clobbered by a previous insn.
3655 N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
3656 SET_DEST perform the former check on the address, and the latter
3657 check on the MEM. */
3658 || (GET_CODE (set) == SET
3659 && (modified_in_p (SET_SRC (set), p->first)
3660 || modified_in_p (SET_DEST (set), p->first)
3661 || modified_between_p (SET_SRC (set), p->first, p->insn)
3662 || modified_between_p (SET_DEST (set), p->first, p->insn))))
3663 p->must_stay = true;
3664 }
3665
3666 \f
3667 /* Emit code to make a call to a constant function or a library call.
3668
3669 INSNS is a list containing all insns emitted in the call.
3670 These insns leave the result in RESULT. Our block is to copy RESULT
3671 to TARGET, which is logically equivalent to EQUIV.
3672
3673 We first emit any insns that set a pseudo on the assumption that these are
3674 loading constants into registers; doing so allows them to be safely cse'ed
3675 between blocks. Then we emit all the other insns in the block, followed by
3676 an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
3677 note with an operand of EQUIV. */
3678
3679 static void
3680 emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv,
3681 bool equiv_may_trap)
3682 {
3683 rtx final_dest = target;
3684 rtx_insn *next, *last, *insn;
3685
3686 /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
3687 into a MEM later. Protect the libcall block from this change. */
3688 if (! REG_P (target) || REG_USERVAR_P (target))
3689 target = gen_reg_rtx (GET_MODE (target));
3690
3691 /* If we're using non-call exceptions, a libcall corresponding to an
3692 operation that may trap may also trap. */
3693 /* ??? See the comment in front of make_reg_eh_region_note. */
3694 if (cfun->can_throw_non_call_exceptions
3695 && (equiv_may_trap || may_trap_p (equiv)))
3696 {
3697 for (insn = insns; insn; insn = NEXT_INSN (insn))
3698 if (CALL_P (insn))
3699 {
3700 rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
3701 if (note)
3702 {
3703 int lp_nr = INTVAL (XEXP (note, 0));
3704 if (lp_nr == 0 || lp_nr == INT_MIN)
3705 remove_note (insn, note);
3706 }
3707 }
3708 }
3709 else
3710 {
3711 /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
3712 reg note to indicate that this call cannot throw or execute a nonlocal
3713 goto (unless there is already a REG_EH_REGION note, in which case
3714 we update it). */
3715 for (insn = insns; insn; insn = NEXT_INSN (insn))
3716 if (CALL_P (insn))
3717 make_reg_eh_region_note_nothrow_nononlocal (insn);
3718 }
3719
3720 /* First emit all insns that set pseudos. Remove them from the list as
3721 we go. Avoid insns that set pseudos which were referenced in previous
3722 insns. These can be generated by move_by_pieces, for example,
3723 to update an address. Similarly, avoid insns that reference things
3724 set in previous insns. */
3725
3726 for (insn = insns; insn; insn = next)
3727 {
3728 rtx set = single_set (insn);
3729
3730 next = NEXT_INSN (insn);
3731
3732 if (set != 0 && REG_P (SET_DEST (set))
3733 && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
3734 {
3735 struct no_conflict_data data;
3736
3737 data.target = const0_rtx;
3738 data.first = insns;
3739 data.insn = insn;
3740 data.must_stay = 0;
3741 note_stores (insn, no_conflict_move_test, &data);
3742 if (! data.must_stay)
3743 {
3744 if (PREV_INSN (insn))
3745 SET_NEXT_INSN (PREV_INSN (insn)) = next;
3746 else
3747 insns = next;
3748
3749 if (next)
3750 SET_PREV_INSN (next) = PREV_INSN (insn);
3751
3752 add_insn (insn);
3753 }
3754 }
3755
3756 /* Some ports use a loop to copy large arguments onto the stack.
3757 Don't move anything outside such a loop. */
3758 if (LABEL_P (insn))
3759 break;
3760 }
3761
3762 /* Write the remaining insns followed by the final copy. */
3763 for (insn = insns; insn; insn = next)
3764 {
3765 next = NEXT_INSN (insn);
3766
3767 add_insn (insn);
3768 }
3769
3770 last = emit_move_insn (target, result);
3771 if (equiv)
3772 set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
3773
3774 if (final_dest != target)
3775 emit_move_insn (final_dest, target);
3776 }
3777
3778 void
3779 emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv)
3780 {
3781 emit_libcall_block_1 (insns, target, result, equiv, false);
3782 }
3783 \f
3784 /* Nonzero if we can perform a comparison of mode MODE straightforwardly.
3785 PURPOSE describes how this comparison will be used. CODE is the rtx
3786 comparison code we will be using.
3787
3788 ??? Actually, CODE is slightly weaker than that. A target is still
3789 required to implement all of the normal bcc operations, but not
3790 required to implement all (or any) of the unordered bcc operations. */
3791
3792 int
3793 can_compare_p (enum rtx_code code, machine_mode mode,
3794 enum can_compare_purpose purpose)
3795 {
3796 rtx test;
3797 test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
3798 do
3799 {
3800 enum insn_code icode;
3801
3802 if (purpose == ccp_jump
3803 && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
3804 && insn_operand_matches (icode, 0, test))
3805 return 1;
3806 if (purpose == ccp_store_flag
3807 && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
3808 && insn_operand_matches (icode, 1, test))
3809 return 1;
3810 if (purpose == ccp_cmov
3811 && optab_handler (cmov_optab, mode) != CODE_FOR_nothing)
3812 return 1;
3813
3814 mode = GET_MODE_WIDER_MODE (mode).else_void ();
3815 PUT_MODE (test, mode);
3816 }
3817 while (mode != VOIDmode);
3818
3819 return 0;
3820 }
3821
3822 /* Return whether the backend can emit a vector comparison for code CODE,
3823 comparing operands of mode CMP_OP_MODE and producing a result with
3824 VALUE_MODE. */
3825
3826 bool
3827 can_vcond_compare_p (enum rtx_code code, machine_mode value_mode,
3828 machine_mode cmp_op_mode)
3829 {
3830 enum insn_code icode;
3831 bool unsigned_p = (code == LTU || code == LEU || code == GTU || code == GEU);
3832 rtx reg1 = alloca_raw_REG (cmp_op_mode, LAST_VIRTUAL_REGISTER + 1);
3833 rtx reg2 = alloca_raw_REG (cmp_op_mode, LAST_VIRTUAL_REGISTER + 2);
3834 rtx test = alloca_rtx_fmt_ee (code, value_mode, reg1, reg2);
3835
3836 return (icode = get_vcond_icode (value_mode, cmp_op_mode, unsigned_p))
3837 != CODE_FOR_nothing
3838 && insn_operand_matches (icode, 3, test);
3839 }
3840
3841 /* This function is called when we are going to emit a compare instruction that
3842 compares the values found in X and Y, using the rtl operator COMPARISON.
3843
3844 If they have mode BLKmode, then SIZE specifies the size of both operands.
3845
3846 UNSIGNEDP nonzero says that the operands are unsigned;
3847 this matters if they need to be widened (as given by METHODS).
3848
3849 *PTEST is where the resulting comparison RTX is returned or NULL_RTX
3850 if we failed to produce one.
3851
3852 *PMODE is the mode of the inputs (in case they are const_int).
3853
3854 This function performs all the setup necessary so that the caller only has
3855 to emit a single comparison insn. This setup can involve doing a BLKmode
3856 comparison or emitting a library call to perform the comparison if no insn
3857 is available to handle it.
3858 The values which are passed in through pointers can be modified; the caller
3859 should perform the comparison on the modified values. Constant
3860 comparisons must have already been folded. */
3861
3862 static void
3863 prepare_cmp_insn (rtx x, rtx y, enum rtx_code comparison, rtx size,
3864 int unsignedp, enum optab_methods methods,
3865 rtx *ptest, machine_mode *pmode)
3866 {
3867 machine_mode mode = *pmode;
3868 rtx libfunc, test;
3869 machine_mode cmp_mode;
3870 enum mode_class mclass;
3871
3872 /* The other methods are not needed. */
3873 gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
3874 || methods == OPTAB_LIB_WIDEN);
3875
3876 if (CONST_SCALAR_INT_P (y))
3877 canonicalize_comparison (mode, &comparison, &y);
3878
3879 /* If we are optimizing, force expensive constants into a register. */
3880 if (CONSTANT_P (x) && optimize
3881 && (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ())
3882 > COSTS_N_INSNS (1)))
3883 x = force_reg (mode, x);
3884
3885 if (CONSTANT_P (y) && optimize
3886 && (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ())
3887 > COSTS_N_INSNS (1)))
3888 y = force_reg (mode, y);
3889
3890 #if HAVE_cc0
3891 /* Make sure if we have a canonical comparison. The RTL
3892 documentation states that canonical comparisons are required only
3893 for targets which have cc0. */
3894 gcc_assert (!CONSTANT_P (x) || CONSTANT_P (y));
3895 #endif
3896
3897 /* Don't let both operands fail to indicate the mode. */
3898 if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
3899 x = force_reg (mode, x);
3900 if (mode == VOIDmode)
3901 mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
3902
3903 /* Handle all BLKmode compares. */
3904
3905 if (mode == BLKmode)
3906 {
3907 machine_mode result_mode;
3908 enum insn_code cmp_code;
3909 rtx result;
3910 rtx opalign
3911 = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
3912
3913 gcc_assert (size);
3914
3915 /* Try to use a memory block compare insn - either cmpstr
3916 or cmpmem will do. */
3917 opt_scalar_int_mode cmp_mode_iter;
3918 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
3919 {
3920 scalar_int_mode cmp_mode = cmp_mode_iter.require ();
3921 cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
3922 if (cmp_code == CODE_FOR_nothing)
3923 cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
3924 if (cmp_code == CODE_FOR_nothing)
3925 cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
3926 if (cmp_code == CODE_FOR_nothing)
3927 continue;
3928
3929 /* Must make sure the size fits the insn's mode. */
3930 if (CONST_INT_P (size)
3931 ? UINTVAL (size) > GET_MODE_MASK (cmp_mode)
3932 : (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (size)))
3933 > GET_MODE_BITSIZE (cmp_mode)))
3934 continue;
3935
3936 result_mode = insn_data[cmp_code].operand[0].mode;
3937 result = gen_reg_rtx (result_mode);
3938 size = convert_to_mode (cmp_mode, size, 1);
3939 emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
3940
3941 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
3942 *pmode = result_mode;
3943 return;
3944 }
3945
3946 if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
3947 goto fail;
3948
3949 /* Otherwise call a library function. */
3950 result = emit_block_comp_via_libcall (x, y, size);
3951
3952 x = result;
3953 y = const0_rtx;
3954 mode = TYPE_MODE (integer_type_node);
3955 methods = OPTAB_LIB_WIDEN;
3956 unsignedp = false;
3957 }
3958
3959 /* Don't allow operands to the compare to trap, as that can put the
3960 compare and branch in different basic blocks. */
3961 if (cfun->can_throw_non_call_exceptions)
3962 {
3963 if (may_trap_p (x))
3964 x = copy_to_reg (x);
3965 if (may_trap_p (y))
3966 y = copy_to_reg (y);
3967 }
3968
3969 if (GET_MODE_CLASS (mode) == MODE_CC)
3970 {
3971 enum insn_code icode = optab_handler (cbranch_optab, CCmode);
3972 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
3973 gcc_assert (icode != CODE_FOR_nothing
3974 && insn_operand_matches (icode, 0, test));
3975 *ptest = test;
3976 return;
3977 }
3978
3979 mclass = GET_MODE_CLASS (mode);
3980 test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
3981 FOR_EACH_MODE_FROM (cmp_mode, mode)
3982 {
3983 enum insn_code icode;
3984 icode = optab_handler (cbranch_optab, cmp_mode);
3985 if (icode != CODE_FOR_nothing
3986 && insn_operand_matches (icode, 0, test))
3987 {
3988 rtx_insn *last = get_last_insn ();
3989 rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
3990 rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
3991 if (op0 && op1
3992 && insn_operand_matches (icode, 1, op0)
3993 && insn_operand_matches (icode, 2, op1))
3994 {
3995 XEXP (test, 0) = op0;
3996 XEXP (test, 1) = op1;
3997 *ptest = test;
3998 *pmode = cmp_mode;
3999 return;
4000 }
4001 delete_insns_since (last);
4002 }
4003
4004 if (methods == OPTAB_DIRECT || !CLASS_HAS_WIDER_MODES_P (mclass))
4005 break;
4006 }
4007
4008 if (methods != OPTAB_LIB_WIDEN)
4009 goto fail;
4010
4011 if (SCALAR_FLOAT_MODE_P (mode))
4012 {
4013 /* Small trick if UNORDERED isn't implemented by the hardware. */
4014 if (comparison == UNORDERED && rtx_equal_p (x, y))
4015 {
4016 prepare_cmp_insn (x, y, UNLT, NULL_RTX, unsignedp, OPTAB_WIDEN,
4017 ptest, pmode);
4018 if (*ptest)
4019 return;
4020 }
4021
4022 prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
4023 }
4024 else
4025 {
4026 rtx result;
4027 machine_mode ret_mode;
4028
4029 /* Handle a libcall just for the mode we are using. */
4030 libfunc = optab_libfunc (cmp_optab, mode);
4031 gcc_assert (libfunc);
4032
4033 /* If we want unsigned, and this mode has a distinct unsigned
4034 comparison routine, use that. */
4035 if (unsignedp)
4036 {
4037 rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
4038 if (ulibfunc)
4039 libfunc = ulibfunc;
4040 }
4041
4042 ret_mode = targetm.libgcc_cmp_return_mode ();
4043 result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4044 ret_mode, x, mode, y, mode);
4045
4046 /* There are two kinds of comparison routines. Biased routines
4047 return 0/1/2, and unbiased routines return -1/0/1. Other parts
4048 of gcc expect that the comparison operation is equivalent
4049 to the modified comparison. For signed comparisons compare the
4050 result against 1 in the biased case, and zero in the unbiased
4051 case. For unsigned comparisons always compare against 1 after
4052 biasing the unbiased result by adding 1. This gives us a way to
4053 represent LTU.
4054 The comparisons in the fixed-point helper library are always
4055 biased. */
4056 x = result;
4057 y = const1_rtx;
4058
4059 if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
4060 {
4061 if (unsignedp)
4062 x = plus_constant (ret_mode, result, 1);
4063 else
4064 y = const0_rtx;
4065 }
4066
4067 *pmode = ret_mode;
4068 prepare_cmp_insn (x, y, comparison, NULL_RTX, unsignedp, methods,
4069 ptest, pmode);
4070 }
4071
4072 return;
4073
4074 fail:
4075 *ptest = NULL_RTX;
4076 }
4077
4078 /* Before emitting an insn with code ICODE, make sure that X, which is going
4079 to be used for operand OPNUM of the insn, is converted from mode MODE to
4080 WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4081 that it is accepted by the operand predicate. Return the new value. */
4082
4083 rtx
4084 prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode,
4085 machine_mode wider_mode, int unsignedp)
4086 {
4087 if (mode != wider_mode)
4088 x = convert_modes (wider_mode, mode, x, unsignedp);
4089
4090 if (!insn_operand_matches (icode, opnum, x))
4091 {
4092 machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode;
4093 if (reload_completed)
4094 return NULL_RTX;
4095 if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode)
4096 return NULL_RTX;
4097 x = copy_to_mode_reg (op_mode, x);
4098 }
4099
4100 return x;
4101 }
4102
4103 /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4104 we can do the branch. */
4105
4106 static void
4107 emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label,
4108 profile_probability prob)
4109 {
4110 machine_mode optab_mode;
4111 enum mode_class mclass;
4112 enum insn_code icode;
4113 rtx_insn *insn;
4114
4115 mclass = GET_MODE_CLASS (mode);
4116 optab_mode = (mclass == MODE_CC) ? CCmode : mode;
4117 icode = optab_handler (cbranch_optab, optab_mode);
4118
4119 gcc_assert (icode != CODE_FOR_nothing);
4120 gcc_assert (insn_operand_matches (icode, 0, test));
4121 insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
4122 XEXP (test, 1), label));
4123 if (prob.initialized_p ()
4124 && profile_status_for_fn (cfun) != PROFILE_ABSENT
4125 && insn
4126 && JUMP_P (insn)
4127 && any_condjump_p (insn)
4128 && !find_reg_note (insn, REG_BR_PROB, 0))
4129 add_reg_br_prob_note (insn, prob);
4130 }
4131
4132 /* Generate code to compare X with Y so that the condition codes are
4133 set and to jump to LABEL if the condition is true. If X is a
4134 constant and Y is not a constant, then the comparison is swapped to
4135 ensure that the comparison RTL has the canonical form.
4136
4137 UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4138 need to be widened. UNSIGNEDP is also used to select the proper
4139 branch condition code.
4140
4141 If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4142
4143 MODE is the mode of the inputs (in case they are const_int).
4144
4145 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4146 It will be potentially converted into an unsigned variant based on
4147 UNSIGNEDP to select a proper jump instruction.
4148
4149 PROB is the probability of jumping to LABEL. */
4150
4151 void
4152 emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4153 machine_mode mode, int unsignedp, rtx label,
4154 profile_probability prob)
4155 {
4156 rtx op0 = x, op1 = y;
4157 rtx test;
4158
4159 /* Swap operands and condition to ensure canonical RTL. */
4160 if (swap_commutative_operands_p (x, y)
4161 && can_compare_p (swap_condition (comparison), mode, ccp_jump))
4162 {
4163 op0 = y, op1 = x;
4164 comparison = swap_condition (comparison);
4165 }
4166
4167 /* If OP0 is still a constant, then both X and Y must be constants
4168 or the opposite comparison is not supported. Force X into a register
4169 to create canonical RTL. */
4170 if (CONSTANT_P (op0))
4171 op0 = force_reg (mode, op0);
4172
4173 if (unsignedp)
4174 comparison = unsigned_condition (comparison);
4175
4176 prepare_cmp_insn (op0, op1, comparison, size, unsignedp, OPTAB_LIB_WIDEN,
4177 &test, &mode);
4178 emit_cmp_and_jump_insn_1 (test, mode, label, prob);
4179 }
4180
4181 \f
4182 /* Emit a library call comparison between floating point X and Y.
4183 COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
4184
4185 static void
4186 prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
4187 rtx *ptest, machine_mode *pmode)
4188 {
4189 enum rtx_code swapped = swap_condition (comparison);
4190 enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
4191 machine_mode orig_mode = GET_MODE (x);
4192 machine_mode mode;
4193 rtx true_rtx, false_rtx;
4194 rtx value, target, equiv;
4195 rtx_insn *insns;
4196 rtx libfunc = 0;
4197 bool reversed_p = false;
4198 scalar_int_mode cmp_mode = targetm.libgcc_cmp_return_mode ();
4199
4200 FOR_EACH_MODE_FROM (mode, orig_mode)
4201 {
4202 if (code_to_optab (comparison)
4203 && (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
4204 break;
4205
4206 if (code_to_optab (swapped)
4207 && (libfunc = optab_libfunc (code_to_optab (swapped), mode)))
4208 {
4209 std::swap (x, y);
4210 comparison = swapped;
4211 break;
4212 }
4213
4214 if (code_to_optab (reversed)
4215 && (libfunc = optab_libfunc (code_to_optab (reversed), mode)))
4216 {
4217 comparison = reversed;
4218 reversed_p = true;
4219 break;
4220 }
4221 }
4222
4223 gcc_assert (mode != VOIDmode);
4224
4225 if (mode != orig_mode)
4226 {
4227 x = convert_to_mode (mode, x, 0);
4228 y = convert_to_mode (mode, y, 0);
4229 }
4230
4231 /* Attach a REG_EQUAL note describing the semantics of the libcall to
4232 the RTL. The allows the RTL optimizers to delete the libcall if the
4233 condition can be determined at compile-time. */
4234 if (comparison == UNORDERED
4235 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4236 {
4237 true_rtx = const_true_rtx;
4238 false_rtx = const0_rtx;
4239 }
4240 else
4241 {
4242 switch (comparison)
4243 {
4244 case EQ:
4245 true_rtx = const0_rtx;
4246 false_rtx = const_true_rtx;
4247 break;
4248
4249 case NE:
4250 true_rtx = const_true_rtx;
4251 false_rtx = const0_rtx;
4252 break;
4253
4254 case GT:
4255 true_rtx = const1_rtx;
4256 false_rtx = const0_rtx;
4257 break;
4258
4259 case GE:
4260 true_rtx = const0_rtx;
4261 false_rtx = constm1_rtx;
4262 break;
4263
4264 case LT:
4265 true_rtx = constm1_rtx;
4266 false_rtx = const0_rtx;
4267 break;
4268
4269 case LE:
4270 true_rtx = const0_rtx;
4271 false_rtx = const1_rtx;
4272 break;
4273
4274 default:
4275 gcc_unreachable ();
4276 }
4277 }
4278
4279 if (comparison == UNORDERED)
4280 {
4281 rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x);
4282 equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y);
4283 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4284 temp, const_true_rtx, equiv);
4285 }
4286 else
4287 {
4288 equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y);
4289 if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
4290 equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
4291 equiv, true_rtx, false_rtx);
4292 }
4293
4294 start_sequence ();
4295 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4296 cmp_mode, x, mode, y, mode);
4297 insns = get_insns ();
4298 end_sequence ();
4299
4300 target = gen_reg_rtx (cmp_mode);
4301 emit_libcall_block (insns, target, value, equiv);
4302
4303 if (comparison == UNORDERED
4304 || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
4305 || reversed_p)
4306 *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
4307 else
4308 *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
4309
4310 *pmode = cmp_mode;
4311 }
4312 \f
4313 /* Generate code to indirectly jump to a location given in the rtx LOC. */
4314
4315 void
4316 emit_indirect_jump (rtx loc)
4317 {
4318 if (!targetm.have_indirect_jump ())
4319 sorry ("indirect jumps are not available on this target");
4320 else
4321 {
4322 class expand_operand ops[1];
4323 create_address_operand (&ops[0], loc);
4324 expand_jump_insn (targetm.code_for_indirect_jump, 1, ops);
4325 emit_barrier ();
4326 }
4327 }
4328 \f
4329
4330 /* Emit a conditional move instruction if the machine supports one for that
4331 condition and machine mode.
4332
4333 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4334 the mode to use should they be constants. If it is VOIDmode, they cannot
4335 both be constants.
4336
4337 OP2 should be stored in TARGET if the comparison is true, otherwise OP3
4338 should be stored there. MODE is the mode to use should they be constants.
4339 If it is VOIDmode, they cannot both be constants.
4340
4341 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4342 is not supported. */
4343
4344 rtx
4345 emit_conditional_move (rtx target, enum rtx_code code, rtx op0, rtx op1,
4346 machine_mode cmode, rtx op2, rtx op3,
4347 machine_mode mode, int unsignedp)
4348 {
4349 rtx comparison;
4350 rtx_insn *last;
4351 enum insn_code icode;
4352 enum rtx_code reversed;
4353
4354 /* If the two source operands are identical, that's just a move. */
4355
4356 if (rtx_equal_p (op2, op3))
4357 {
4358 if (!target)
4359 target = gen_reg_rtx (mode);
4360
4361 emit_move_insn (target, op3);
4362 return target;
4363 }
4364
4365 /* If one operand is constant, make it the second one. Only do this
4366 if the other operand is not constant as well. */
4367
4368 if (swap_commutative_operands_p (op0, op1))
4369 {
4370 std::swap (op0, op1);
4371 code = swap_condition (code);
4372 }
4373
4374 /* get_condition will prefer to generate LT and GT even if the old
4375 comparison was against zero, so undo that canonicalization here since
4376 comparisons against zero are cheaper. */
4377 if (code == LT && op1 == const1_rtx)
4378 code = LE, op1 = const0_rtx;
4379 else if (code == GT && op1 == constm1_rtx)
4380 code = GE, op1 = const0_rtx;
4381
4382 if (cmode == VOIDmode)
4383 cmode = GET_MODE (op0);
4384
4385 enum rtx_code orig_code = code;
4386 bool swapped = false;
4387 if (swap_commutative_operands_p (op2, op3)
4388 && ((reversed = reversed_comparison_code_parts (code, op0, op1, NULL))
4389 != UNKNOWN))
4390 {
4391 std::swap (op2, op3);
4392 code = reversed;
4393 swapped = true;
4394 }
4395
4396 if (mode == VOIDmode)
4397 mode = GET_MODE (op2);
4398
4399 icode = direct_optab_handler (movcc_optab, mode);
4400
4401 if (icode == CODE_FOR_nothing)
4402 return NULL_RTX;
4403
4404 if (!target)
4405 target = gen_reg_rtx (mode);
4406
4407 for (int pass = 0; ; pass++)
4408 {
4409 code = unsignedp ? unsigned_condition (code) : code;
4410 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4411
4412 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4413 punt and let the caller figure out how best to deal with this
4414 situation. */
4415 if (COMPARISON_P (comparison))
4416 {
4417 saved_pending_stack_adjust save;
4418 save_pending_stack_adjust (&save);
4419 last = get_last_insn ();
4420 do_pending_stack_adjust ();
4421 machine_mode cmpmode = cmode;
4422 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4423 GET_CODE (comparison), NULL_RTX, unsignedp,
4424 OPTAB_WIDEN, &comparison, &cmpmode);
4425 if (comparison)
4426 {
4427 class expand_operand ops[4];
4428
4429 create_output_operand (&ops[0], target, mode);
4430 create_fixed_operand (&ops[1], comparison);
4431 create_input_operand (&ops[2], op2, mode);
4432 create_input_operand (&ops[3], op3, mode);
4433 if (maybe_expand_insn (icode, 4, ops))
4434 {
4435 if (ops[0].value != target)
4436 convert_move (target, ops[0].value, false);
4437 return target;
4438 }
4439 }
4440 delete_insns_since (last);
4441 restore_pending_stack_adjust (&save);
4442 }
4443
4444 if (pass == 1)
4445 return NULL_RTX;
4446
4447 /* If the preferred op2/op3 order is not usable, retry with other
4448 operand order, perhaps it will expand successfully. */
4449 if (swapped)
4450 code = orig_code;
4451 else if ((reversed = reversed_comparison_code_parts (orig_code, op0, op1,
4452 NULL))
4453 != UNKNOWN)
4454 code = reversed;
4455 else
4456 return NULL_RTX;
4457 std::swap (op2, op3);
4458 }
4459 }
4460
4461
4462 /* Emit a conditional negate or bitwise complement using the
4463 negcc or notcc optabs if available. Return NULL_RTX if such operations
4464 are not available. Otherwise return the RTX holding the result.
4465 TARGET is the desired destination of the result. COMP is the comparison
4466 on which to negate. If COND is true move into TARGET the negation
4467 or bitwise complement of OP1. Otherwise move OP2 into TARGET.
4468 CODE is either NEG or NOT. MODE is the machine mode in which the
4469 operation is performed. */
4470
4471 rtx
4472 emit_conditional_neg_or_complement (rtx target, rtx_code code,
4473 machine_mode mode, rtx cond, rtx op1,
4474 rtx op2)
4475 {
4476 optab op = unknown_optab;
4477 if (code == NEG)
4478 op = negcc_optab;
4479 else if (code == NOT)
4480 op = notcc_optab;
4481 else
4482 gcc_unreachable ();
4483
4484 insn_code icode = direct_optab_handler (op, mode);
4485
4486 if (icode == CODE_FOR_nothing)
4487 return NULL_RTX;
4488
4489 if (!target)
4490 target = gen_reg_rtx (mode);
4491
4492 rtx_insn *last = get_last_insn ();
4493 class expand_operand ops[4];
4494
4495 create_output_operand (&ops[0], target, mode);
4496 create_fixed_operand (&ops[1], cond);
4497 create_input_operand (&ops[2], op1, mode);
4498 create_input_operand (&ops[3], op2, mode);
4499
4500 if (maybe_expand_insn (icode, 4, ops))
4501 {
4502 if (ops[0].value != target)
4503 convert_move (target, ops[0].value, false);
4504
4505 return target;
4506 }
4507 delete_insns_since (last);
4508 return NULL_RTX;
4509 }
4510
4511 /* Emit a conditional addition instruction if the machine supports one for that
4512 condition and machine mode.
4513
4514 OP0 and OP1 are the operands that should be compared using CODE. CMODE is
4515 the mode to use should they be constants. If it is VOIDmode, they cannot
4516 both be constants.
4517
4518 OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
4519 should be stored there. MODE is the mode to use should they be constants.
4520 If it is VOIDmode, they cannot both be constants.
4521
4522 The result is either TARGET (perhaps modified) or NULL_RTX if the operation
4523 is not supported. */
4524
4525 rtx
4526 emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
4527 machine_mode cmode, rtx op2, rtx op3,
4528 machine_mode mode, int unsignedp)
4529 {
4530 rtx comparison;
4531 rtx_insn *last;
4532 enum insn_code icode;
4533
4534 /* If one operand is constant, make it the second one. Only do this
4535 if the other operand is not constant as well. */
4536
4537 if (swap_commutative_operands_p (op0, op1))
4538 {
4539 std::swap (op0, op1);
4540 code = swap_condition (code);
4541 }
4542
4543 /* get_condition will prefer to generate LT and GT even if the old
4544 comparison was against zero, so undo that canonicalization here since
4545 comparisons against zero are cheaper. */
4546 if (code == LT && op1 == const1_rtx)
4547 code = LE, op1 = const0_rtx;
4548 else if (code == GT && op1 == constm1_rtx)
4549 code = GE, op1 = const0_rtx;
4550
4551 if (cmode == VOIDmode)
4552 cmode = GET_MODE (op0);
4553
4554 if (mode == VOIDmode)
4555 mode = GET_MODE (op2);
4556
4557 icode = optab_handler (addcc_optab, mode);
4558
4559 if (icode == CODE_FOR_nothing)
4560 return 0;
4561
4562 if (!target)
4563 target = gen_reg_rtx (mode);
4564
4565 code = unsignedp ? unsigned_condition (code) : code;
4566 comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
4567
4568 /* We can get const0_rtx or const_true_rtx in some circumstances. Just
4569 return NULL and let the caller figure out how best to deal with this
4570 situation. */
4571 if (!COMPARISON_P (comparison))
4572 return NULL_RTX;
4573
4574 do_pending_stack_adjust ();
4575 last = get_last_insn ();
4576 prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1),
4577 GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
4578 &comparison, &cmode);
4579 if (comparison)
4580 {
4581 class expand_operand ops[4];
4582
4583 create_output_operand (&ops[0], target, mode);
4584 create_fixed_operand (&ops[1], comparison);
4585 create_input_operand (&ops[2], op2, mode);
4586 create_input_operand (&ops[3], op3, mode);
4587 if (maybe_expand_insn (icode, 4, ops))
4588 {
4589 if (ops[0].value != target)
4590 convert_move (target, ops[0].value, false);
4591 return target;
4592 }
4593 }
4594 delete_insns_since (last);
4595 return NULL_RTX;
4596 }
4597 \f
4598 /* These functions attempt to generate an insn body, rather than
4599 emitting the insn, but if the gen function already emits them, we
4600 make no attempt to turn them back into naked patterns. */
4601
4602 /* Generate and return an insn body to add Y to X. */
4603
4604 rtx_insn *
4605 gen_add2_insn (rtx x, rtx y)
4606 {
4607 enum insn_code icode = optab_handler (add_optab, GET_MODE (x));
4608
4609 gcc_assert (insn_operand_matches (icode, 0, x));
4610 gcc_assert (insn_operand_matches (icode, 1, x));
4611 gcc_assert (insn_operand_matches (icode, 2, y));
4612
4613 return GEN_FCN (icode) (x, x, y);
4614 }
4615
4616 /* Generate and return an insn body to add r1 and c,
4617 storing the result in r0. */
4618
4619 rtx_insn *
4620 gen_add3_insn (rtx r0, rtx r1, rtx c)
4621 {
4622 enum insn_code icode = optab_handler (add_optab, GET_MODE (r0));
4623
4624 if (icode == CODE_FOR_nothing
4625 || !insn_operand_matches (icode, 0, r0)
4626 || !insn_operand_matches (icode, 1, r1)
4627 || !insn_operand_matches (icode, 2, c))
4628 return NULL;
4629
4630 return GEN_FCN (icode) (r0, r1, c);
4631 }
4632
4633 int
4634 have_add2_insn (rtx x, rtx y)
4635 {
4636 enum insn_code icode;
4637
4638 gcc_assert (GET_MODE (x) != VOIDmode);
4639
4640 icode = optab_handler (add_optab, GET_MODE (x));
4641
4642 if (icode == CODE_FOR_nothing)
4643 return 0;
4644
4645 if (!insn_operand_matches (icode, 0, x)
4646 || !insn_operand_matches (icode, 1, x)
4647 || !insn_operand_matches (icode, 2, y))
4648 return 0;
4649
4650 return 1;
4651 }
4652
4653 /* Generate and return an insn body to add Y to X. */
4654
4655 rtx_insn *
4656 gen_addptr3_insn (rtx x, rtx y, rtx z)
4657 {
4658 enum insn_code icode = optab_handler (addptr3_optab, GET_MODE (x));
4659
4660 gcc_assert (insn_operand_matches (icode, 0, x));
4661 gcc_assert (insn_operand_matches (icode, 1, y));
4662 gcc_assert (insn_operand_matches (icode, 2, z));
4663
4664 return GEN_FCN (icode) (x, y, z);
4665 }
4666
4667 /* Return true if the target implements an addptr pattern and X, Y,
4668 and Z are valid for the pattern predicates. */
4669
4670 int
4671 have_addptr3_insn (rtx x, rtx y, rtx z)
4672 {
4673 enum insn_code icode;
4674
4675 gcc_assert (GET_MODE (x) != VOIDmode);
4676
4677 icode = optab_handler (addptr3_optab, GET_MODE (x));
4678
4679 if (icode == CODE_FOR_nothing)
4680 return 0;
4681
4682 if (!insn_operand_matches (icode, 0, x)
4683 || !insn_operand_matches (icode, 1, y)
4684 || !insn_operand_matches (icode, 2, z))
4685 return 0;
4686
4687 return 1;
4688 }
4689
4690 /* Generate and return an insn body to subtract Y from X. */
4691
4692 rtx_insn *
4693 gen_sub2_insn (rtx x, rtx y)
4694 {
4695 enum insn_code icode = optab_handler (sub_optab, GET_MODE (x));
4696
4697 gcc_assert (insn_operand_matches (icode, 0, x));
4698 gcc_assert (insn_operand_matches (icode, 1, x));
4699 gcc_assert (insn_operand_matches (icode, 2, y));
4700
4701 return GEN_FCN (icode) (x, x, y);
4702 }
4703
4704 /* Generate and return an insn body to subtract r1 and c,
4705 storing the result in r0. */
4706
4707 rtx_insn *
4708 gen_sub3_insn (rtx r0, rtx r1, rtx c)
4709 {
4710 enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0));
4711
4712 if (icode == CODE_FOR_nothing
4713 || !insn_operand_matches (icode, 0, r0)
4714 || !insn_operand_matches (icode, 1, r1)
4715 || !insn_operand_matches (icode, 2, c))
4716 return NULL;
4717
4718 return GEN_FCN (icode) (r0, r1, c);
4719 }
4720
4721 int
4722 have_sub2_insn (rtx x, rtx y)
4723 {
4724 enum insn_code icode;
4725
4726 gcc_assert (GET_MODE (x) != VOIDmode);
4727
4728 icode = optab_handler (sub_optab, GET_MODE (x));
4729
4730 if (icode == CODE_FOR_nothing)
4731 return 0;
4732
4733 if (!insn_operand_matches (icode, 0, x)
4734 || !insn_operand_matches (icode, 1, x)
4735 || !insn_operand_matches (icode, 2, y))
4736 return 0;
4737
4738 return 1;
4739 }
4740 \f
4741 /* Generate the body of an insn to extend Y (with mode MFROM)
4742 into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
4743
4744 rtx_insn *
4745 gen_extend_insn (rtx x, rtx y, machine_mode mto,
4746 machine_mode mfrom, int unsignedp)
4747 {
4748 enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
4749 return GEN_FCN (icode) (x, y);
4750 }
4751 \f
4752 /* Generate code to convert FROM to floating point
4753 and store in TO. FROM must be fixed point and not VOIDmode.
4754 UNSIGNEDP nonzero means regard FROM as unsigned.
4755 Normally this is done by correcting the final value
4756 if it is negative. */
4757
4758 void
4759 expand_float (rtx to, rtx from, int unsignedp)
4760 {
4761 enum insn_code icode;
4762 rtx target = to;
4763 scalar_mode from_mode, to_mode;
4764 machine_mode fmode, imode;
4765 bool can_do_signed = false;
4766
4767 /* Crash now, because we won't be able to decide which mode to use. */
4768 gcc_assert (GET_MODE (from) != VOIDmode);
4769
4770 /* Look for an insn to do the conversion. Do it in the specified
4771 modes if possible; otherwise convert either input, output or both to
4772 wider mode. If the integer mode is wider than the mode of FROM,
4773 we can do the conversion signed even if the input is unsigned. */
4774
4775 FOR_EACH_MODE_FROM (fmode, GET_MODE (to))
4776 FOR_EACH_MODE_FROM (imode, GET_MODE (from))
4777 {
4778 int doing_unsigned = unsignedp;
4779
4780 if (fmode != GET_MODE (to)
4781 && (significand_size (fmode)
4782 < GET_MODE_UNIT_PRECISION (GET_MODE (from))))
4783 continue;
4784
4785 icode = can_float_p (fmode, imode, unsignedp);
4786 if (icode == CODE_FOR_nothing && unsignedp)
4787 {
4788 enum insn_code scode = can_float_p (fmode, imode, 0);
4789 if (scode != CODE_FOR_nothing)
4790 can_do_signed = true;
4791 if (imode != GET_MODE (from))
4792 icode = scode, doing_unsigned = 0;
4793 }
4794
4795 if (icode != CODE_FOR_nothing)
4796 {
4797 if (imode != GET_MODE (from))
4798 from = convert_to_mode (imode, from, unsignedp);
4799
4800 if (fmode != GET_MODE (to))
4801 target = gen_reg_rtx (fmode);
4802
4803 emit_unop_insn (icode, target, from,
4804 doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
4805
4806 if (target != to)
4807 convert_move (to, target, 0);
4808 return;
4809 }
4810 }
4811
4812 /* Unsigned integer, and no way to convert directly. Convert as signed,
4813 then unconditionally adjust the result. */
4814 if (unsignedp
4815 && can_do_signed
4816 && is_a <scalar_mode> (GET_MODE (to), &to_mode)
4817 && is_a <scalar_mode> (GET_MODE (from), &from_mode))
4818 {
4819 opt_scalar_mode fmode_iter;
4820 rtx_code_label *label = gen_label_rtx ();
4821 rtx temp;
4822 REAL_VALUE_TYPE offset;
4823
4824 /* Look for a usable floating mode FMODE wider than the source and at
4825 least as wide as the target. Using FMODE will avoid rounding woes
4826 with unsigned values greater than the signed maximum value. */
4827
4828 FOR_EACH_MODE_FROM (fmode_iter, to_mode)
4829 {
4830 scalar_mode fmode = fmode_iter.require ();
4831 if (GET_MODE_PRECISION (from_mode) < GET_MODE_BITSIZE (fmode)
4832 && can_float_p (fmode, from_mode, 0) != CODE_FOR_nothing)
4833 break;
4834 }
4835
4836 if (!fmode_iter.exists (&fmode))
4837 {
4838 /* There is no such mode. Pretend the target is wide enough. */
4839 fmode = to_mode;
4840
4841 /* Avoid double-rounding when TO is narrower than FROM. */
4842 if ((significand_size (fmode) + 1)
4843 < GET_MODE_PRECISION (from_mode))
4844 {
4845 rtx temp1;
4846 rtx_code_label *neglabel = gen_label_rtx ();
4847
4848 /* Don't use TARGET if it isn't a register, is a hard register,
4849 or is the wrong mode. */
4850 if (!REG_P (target)
4851 || REGNO (target) < FIRST_PSEUDO_REGISTER
4852 || GET_MODE (target) != fmode)
4853 target = gen_reg_rtx (fmode);
4854
4855 imode = from_mode;
4856 do_pending_stack_adjust ();
4857
4858 /* Test whether the sign bit is set. */
4859 emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode,
4860 0, neglabel);
4861
4862 /* The sign bit is not set. Convert as signed. */
4863 expand_float (target, from, 0);
4864 emit_jump_insn (targetm.gen_jump (label));
4865 emit_barrier ();
4866
4867 /* The sign bit is set.
4868 Convert to a usable (positive signed) value by shifting right
4869 one bit, while remembering if a nonzero bit was shifted
4870 out; i.e., compute (from & 1) | (from >> 1). */
4871
4872 emit_label (neglabel);
4873 temp = expand_binop (imode, and_optab, from, const1_rtx,
4874 NULL_RTX, 1, OPTAB_LIB_WIDEN);
4875 temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
4876 temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1,
4877 OPTAB_LIB_WIDEN);
4878 expand_float (target, temp, 0);
4879
4880 /* Multiply by 2 to undo the shift above. */
4881 temp = expand_binop (fmode, add_optab, target, target,
4882 target, 0, OPTAB_LIB_WIDEN);
4883 if (temp != target)
4884 emit_move_insn (target, temp);
4885
4886 do_pending_stack_adjust ();
4887 emit_label (label);
4888 goto done;
4889 }
4890 }
4891
4892 /* If we are about to do some arithmetic to correct for an
4893 unsigned operand, do it in a pseudo-register. */
4894
4895 if (to_mode != fmode
4896 || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
4897 target = gen_reg_rtx (fmode);
4898
4899 /* Convert as signed integer to floating. */
4900 expand_float (target, from, 0);
4901
4902 /* If FROM is negative (and therefore TO is negative),
4903 correct its value by 2**bitwidth. */
4904
4905 do_pending_stack_adjust ();
4906 emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, from_mode,
4907 0, label);
4908
4909
4910 real_2expN (&offset, GET_MODE_PRECISION (from_mode), fmode);
4911 temp = expand_binop (fmode, add_optab, target,
4912 const_double_from_real_value (offset, fmode),
4913 target, 0, OPTAB_LIB_WIDEN);
4914 if (temp != target)
4915 emit_move_insn (target, temp);
4916
4917 do_pending_stack_adjust ();
4918 emit_label (label);
4919 goto done;
4920 }
4921
4922 /* No hardware instruction available; call a library routine. */
4923 {
4924 rtx libfunc;
4925 rtx_insn *insns;
4926 rtx value;
4927 convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
4928
4929 if (is_narrower_int_mode (GET_MODE (from), SImode))
4930 from = convert_to_mode (SImode, from, unsignedp);
4931
4932 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
4933 gcc_assert (libfunc);
4934
4935 start_sequence ();
4936
4937 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4938 GET_MODE (to), from, GET_MODE (from));
4939 insns = get_insns ();
4940 end_sequence ();
4941
4942 emit_libcall_block (insns, target, value,
4943 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
4944 GET_MODE (to), from));
4945 }
4946
4947 done:
4948
4949 /* Copy result to requested destination
4950 if we have been computing in a temp location. */
4951
4952 if (target != to)
4953 {
4954 if (GET_MODE (target) == GET_MODE (to))
4955 emit_move_insn (to, target);
4956 else
4957 convert_move (to, target, 0);
4958 }
4959 }
4960 \f
4961 /* Generate code to convert FROM to fixed point and store in TO. FROM
4962 must be floating point. */
4963
4964 void
4965 expand_fix (rtx to, rtx from, int unsignedp)
4966 {
4967 enum insn_code icode;
4968 rtx target = to;
4969 machine_mode fmode, imode;
4970 opt_scalar_mode fmode_iter;
4971 bool must_trunc = false;
4972
4973 /* We first try to find a pair of modes, one real and one integer, at
4974 least as wide as FROM and TO, respectively, in which we can open-code
4975 this conversion. If the integer mode is wider than the mode of TO,
4976 we can do the conversion either signed or unsigned. */
4977
4978 FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
4979 FOR_EACH_MODE_FROM (imode, GET_MODE (to))
4980 {
4981 int doing_unsigned = unsignedp;
4982
4983 icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
4984 if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
4985 icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
4986
4987 if (icode != CODE_FOR_nothing)
4988 {
4989 rtx_insn *last = get_last_insn ();
4990 if (fmode != GET_MODE (from))
4991 from = convert_to_mode (fmode, from, 0);
4992
4993 if (must_trunc)
4994 {
4995 rtx temp = gen_reg_rtx (GET_MODE (from));
4996 from = expand_unop (GET_MODE (from), ftrunc_optab, from,
4997 temp, 0);
4998 }
4999
5000 if (imode != GET_MODE (to))
5001 target = gen_reg_rtx (imode);
5002
5003 if (maybe_emit_unop_insn (icode, target, from,
5004 doing_unsigned ? UNSIGNED_FIX : FIX))
5005 {
5006 if (target != to)
5007 convert_move (to, target, unsignedp);
5008 return;
5009 }
5010 delete_insns_since (last);
5011 }
5012 }
5013
5014 /* For an unsigned conversion, there is one more way to do it.
5015 If we have a signed conversion, we generate code that compares
5016 the real value to the largest representable positive number. If if
5017 is smaller, the conversion is done normally. Otherwise, subtract
5018 one plus the highest signed number, convert, and add it back.
5019
5020 We only need to check all real modes, since we know we didn't find
5021 anything with a wider integer mode.
5022
5023 This code used to extend FP value into mode wider than the destination.
5024 This is needed for decimal float modes which cannot accurately
5025 represent one plus the highest signed number of the same size, but
5026 not for binary modes. Consider, for instance conversion from SFmode
5027 into DImode.
5028
5029 The hot path through the code is dealing with inputs smaller than 2^63
5030 and doing just the conversion, so there is no bits to lose.
5031
5032 In the other path we know the value is positive in the range 2^63..2^64-1
5033 inclusive. (as for other input overflow happens and result is undefined)
5034 So we know that the most important bit set in mantissa corresponds to
5035 2^63. The subtraction of 2^63 should not generate any rounding as it
5036 simply clears out that bit. The rest is trivial. */
5037
5038 scalar_int_mode to_mode;
5039 if (unsignedp
5040 && is_a <scalar_int_mode> (GET_MODE (to), &to_mode)
5041 && HWI_COMPUTABLE_MODE_P (to_mode))
5042 FOR_EACH_MODE_FROM (fmode_iter, as_a <scalar_mode> (GET_MODE (from)))
5043 {
5044 scalar_mode fmode = fmode_iter.require ();
5045 if (CODE_FOR_nothing != can_fix_p (to_mode, fmode,
5046 0, &must_trunc)
5047 && (!DECIMAL_FLOAT_MODE_P (fmode)
5048 || (GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (to_mode))))
5049 {
5050 int bitsize;
5051 REAL_VALUE_TYPE offset;
5052 rtx limit;
5053 rtx_code_label *lab1, *lab2;
5054 rtx_insn *insn;
5055
5056 bitsize = GET_MODE_PRECISION (to_mode);
5057 real_2expN (&offset, bitsize - 1, fmode);
5058 limit = const_double_from_real_value (offset, fmode);
5059 lab1 = gen_label_rtx ();
5060 lab2 = gen_label_rtx ();
5061
5062 if (fmode != GET_MODE (from))
5063 from = convert_to_mode (fmode, from, 0);
5064
5065 /* See if we need to do the subtraction. */
5066 do_pending_stack_adjust ();
5067 emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX,
5068 GET_MODE (from), 0, lab1);
5069
5070 /* If not, do the signed "fix" and branch around fixup code. */
5071 expand_fix (to, from, 0);
5072 emit_jump_insn (targetm.gen_jump (lab2));
5073 emit_barrier ();
5074
5075 /* Otherwise, subtract 2**(N-1), convert to signed number,
5076 then add 2**(N-1). Do the addition using XOR since this
5077 will often generate better code. */
5078 emit_label (lab1);
5079 target = expand_binop (GET_MODE (from), sub_optab, from, limit,
5080 NULL_RTX, 0, OPTAB_LIB_WIDEN);
5081 expand_fix (to, target, 0);
5082 target = expand_binop (to_mode, xor_optab, to,
5083 gen_int_mode
5084 (HOST_WIDE_INT_1 << (bitsize - 1),
5085 to_mode),
5086 to, 1, OPTAB_LIB_WIDEN);
5087
5088 if (target != to)
5089 emit_move_insn (to, target);
5090
5091 emit_label (lab2);
5092
5093 if (optab_handler (mov_optab, to_mode) != CODE_FOR_nothing)
5094 {
5095 /* Make a place for a REG_NOTE and add it. */
5096 insn = emit_move_insn (to, to);
5097 set_dst_reg_note (insn, REG_EQUAL,
5098 gen_rtx_fmt_e (UNSIGNED_FIX, to_mode,
5099 copy_rtx (from)),
5100 to);
5101 }
5102
5103 return;
5104 }
5105 }
5106
5107 /* We can't do it with an insn, so use a library call. But first ensure
5108 that the mode of TO is at least as wide as SImode, since those are the
5109 only library calls we know about. */
5110
5111 if (is_narrower_int_mode (GET_MODE (to), SImode))
5112 {
5113 target = gen_reg_rtx (SImode);
5114
5115 expand_fix (target, from, unsignedp);
5116 }
5117 else
5118 {
5119 rtx_insn *insns;
5120 rtx value;
5121 rtx libfunc;
5122
5123 convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
5124 libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5125 gcc_assert (libfunc);
5126
5127 start_sequence ();
5128
5129 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5130 GET_MODE (to), from, GET_MODE (from));
5131 insns = get_insns ();
5132 end_sequence ();
5133
5134 emit_libcall_block (insns, target, value,
5135 gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
5136 GET_MODE (to), from));
5137 }
5138
5139 if (target != to)
5140 {
5141 if (GET_MODE (to) == GET_MODE (target))
5142 emit_move_insn (to, target);
5143 else
5144 convert_move (to, target, 0);
5145 }
5146 }
5147
5148
5149 /* Promote integer arguments for a libcall if necessary.
5150 emit_library_call_value cannot do the promotion because it does not
5151 know if it should do a signed or unsigned promotion. This is because
5152 there are no tree types defined for libcalls. */
5153
5154 static rtx
5155 prepare_libcall_arg (rtx arg, int uintp)
5156 {
5157 scalar_int_mode mode;
5158 machine_mode arg_mode;
5159 if (is_a <scalar_int_mode> (GET_MODE (arg), &mode))
5160 {
5161 /* If we need to promote the integer function argument we need to do
5162 it here instead of inside emit_library_call_value because in
5163 emit_library_call_value we don't know if we should do a signed or
5164 unsigned promotion. */
5165
5166 int unsigned_p = 0;
5167 arg_mode = promote_function_mode (NULL_TREE, mode,
5168 &unsigned_p, NULL_TREE, 0);
5169 if (arg_mode != mode)
5170 return convert_to_mode (arg_mode, arg, uintp);
5171 }
5172 return arg;
5173 }
5174
5175 /* Generate code to convert FROM or TO a fixed-point.
5176 If UINTP is true, either TO or FROM is an unsigned integer.
5177 If SATP is true, we need to saturate the result. */
5178
5179 void
5180 expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
5181 {
5182 machine_mode to_mode = GET_MODE (to);
5183 machine_mode from_mode = GET_MODE (from);
5184 convert_optab tab;
5185 enum rtx_code this_code;
5186 enum insn_code code;
5187 rtx_insn *insns;
5188 rtx value;
5189 rtx libfunc;
5190
5191 if (to_mode == from_mode)
5192 {
5193 emit_move_insn (to, from);
5194 return;
5195 }
5196
5197 if (uintp)
5198 {
5199 tab = satp ? satfractuns_optab : fractuns_optab;
5200 this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
5201 }
5202 else
5203 {
5204 tab = satp ? satfract_optab : fract_optab;
5205 this_code = satp ? SAT_FRACT : FRACT_CONVERT;
5206 }
5207 code = convert_optab_handler (tab, to_mode, from_mode);
5208 if (code != CODE_FOR_nothing)
5209 {
5210 emit_unop_insn (code, to, from, this_code);
5211 return;
5212 }
5213
5214 libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
5215 gcc_assert (libfunc);
5216
5217 from = prepare_libcall_arg (from, uintp);
5218 from_mode = GET_MODE (from);
5219
5220 start_sequence ();
5221 value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
5222 from, from_mode);
5223 insns = get_insns ();
5224 end_sequence ();
5225
5226 emit_libcall_block (insns, to, value,
5227 gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
5228 }
5229
5230 /* Generate code to convert FROM to fixed point and store in TO. FROM
5231 must be floating point, TO must be signed. Use the conversion optab
5232 TAB to do the conversion. */
5233
5234 bool
5235 expand_sfix_optab (rtx to, rtx from, convert_optab tab)
5236 {
5237 enum insn_code icode;
5238 rtx target = to;
5239 machine_mode fmode, imode;
5240
5241 /* We first try to find a pair of modes, one real and one integer, at
5242 least as wide as FROM and TO, respectively, in which we can open-code
5243 this conversion. If the integer mode is wider than the mode of TO,
5244 we can do the conversion either signed or unsigned. */
5245
5246 FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
5247 FOR_EACH_MODE_FROM (imode, GET_MODE (to))
5248 {
5249 icode = convert_optab_handler (tab, imode, fmode);
5250 if (icode != CODE_FOR_nothing)
5251 {
5252 rtx_insn *last = get_last_insn ();
5253 if (fmode != GET_MODE (from))
5254 from = convert_to_mode (fmode, from, 0);
5255
5256 if (imode != GET_MODE (to))
5257 target = gen_reg_rtx (imode);
5258
5259 if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN))
5260 {
5261 delete_insns_since (last);
5262 continue;
5263 }
5264 if (target != to)
5265 convert_move (to, target, 0);
5266 return true;
5267 }
5268 }
5269
5270 return false;
5271 }
5272 \f
5273 /* Report whether we have an instruction to perform the operation
5274 specified by CODE on operands of mode MODE. */
5275 int
5276 have_insn_for (enum rtx_code code, machine_mode mode)
5277 {
5278 return (code_to_optab (code)
5279 && (optab_handler (code_to_optab (code), mode)
5280 != CODE_FOR_nothing));
5281 }
5282
5283 /* Print information about the current contents of the optabs on
5284 STDERR. */
5285
5286 DEBUG_FUNCTION void
5287 debug_optab_libfuncs (void)
5288 {
5289 int i, j, k;
5290
5291 /* Dump the arithmetic optabs. */
5292 for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
5293 for (j = 0; j < NUM_MACHINE_MODES; ++j)
5294 {
5295 rtx l = optab_libfunc ((optab) i, (machine_mode) j);
5296 if (l)
5297 {
5298 gcc_assert (GET_CODE (l) == SYMBOL_REF);
5299 fprintf (stderr, "%s\t%s:\t%s\n",
5300 GET_RTX_NAME (optab_to_code ((optab) i)),
5301 GET_MODE_NAME (j),
5302 XSTR (l, 0));
5303 }
5304 }
5305
5306 /* Dump the conversion optabs. */
5307 for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
5308 for (j = 0; j < NUM_MACHINE_MODES; ++j)
5309 for (k = 0; k < NUM_MACHINE_MODES; ++k)
5310 {
5311 rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j,
5312 (machine_mode) k);
5313 if (l)
5314 {
5315 gcc_assert (GET_CODE (l) == SYMBOL_REF);
5316 fprintf (stderr, "%s\t%s\t%s:\t%s\n",
5317 GET_RTX_NAME (optab_to_code ((optab) i)),
5318 GET_MODE_NAME (j),
5319 GET_MODE_NAME (k),
5320 XSTR (l, 0));
5321 }
5322 }
5323 }
5324
5325 /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
5326 CODE. Return 0 on failure. */
5327
5328 rtx_insn *
5329 gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
5330 {
5331 machine_mode mode = GET_MODE (op1);
5332 enum insn_code icode;
5333 rtx_insn *insn;
5334 rtx trap_rtx;
5335
5336 if (mode == VOIDmode)
5337 return 0;
5338
5339 icode = optab_handler (ctrap_optab, mode);
5340 if (icode == CODE_FOR_nothing)
5341 return 0;
5342
5343 /* Some targets only accept a zero trap code. */
5344 if (!insn_operand_matches (icode, 3, tcode))
5345 return 0;
5346
5347 do_pending_stack_adjust ();
5348 start_sequence ();
5349 prepare_cmp_insn (op1, op2, code, NULL_RTX, false, OPTAB_DIRECT,
5350 &trap_rtx, &mode);
5351 if (!trap_rtx)
5352 insn = NULL;
5353 else
5354 insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
5355 tcode);
5356
5357 /* If that failed, then give up. */
5358 if (insn == 0)
5359 {
5360 end_sequence ();
5361 return 0;
5362 }
5363
5364 emit_insn (insn);
5365 insn = get_insns ();
5366 end_sequence ();
5367 return insn;
5368 }
5369
5370 /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
5371 or unsigned operation code. */
5372
5373 enum rtx_code
5374 get_rtx_code (enum tree_code tcode, bool unsignedp)
5375 {
5376 enum rtx_code code;
5377 switch (tcode)
5378 {
5379 case EQ_EXPR:
5380 code = EQ;
5381 break;
5382 case NE_EXPR:
5383 code = NE;
5384 break;
5385 case LT_EXPR:
5386 code = unsignedp ? LTU : LT;
5387 break;
5388 case LE_EXPR:
5389 code = unsignedp ? LEU : LE;
5390 break;
5391 case GT_EXPR:
5392 code = unsignedp ? GTU : GT;
5393 break;
5394 case GE_EXPR:
5395 code = unsignedp ? GEU : GE;
5396 break;
5397
5398 case UNORDERED_EXPR:
5399 code = UNORDERED;
5400 break;
5401 case ORDERED_EXPR:
5402 code = ORDERED;
5403 break;
5404 case UNLT_EXPR:
5405 code = UNLT;
5406 break;
5407 case UNLE_EXPR:
5408 code = UNLE;
5409 break;
5410 case UNGT_EXPR:
5411 code = UNGT;
5412 break;
5413 case UNGE_EXPR:
5414 code = UNGE;
5415 break;
5416 case UNEQ_EXPR:
5417 code = UNEQ;
5418 break;
5419 case LTGT_EXPR:
5420 code = LTGT;
5421 break;
5422
5423 case BIT_AND_EXPR:
5424 code = AND;
5425 break;
5426
5427 case BIT_IOR_EXPR:
5428 code = IOR;
5429 break;
5430
5431 default:
5432 gcc_unreachable ();
5433 }
5434 return code;
5435 }
5436
5437 /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
5438 select signed or unsigned operators. OPNO holds the index of the
5439 first comparison operand for insn ICODE. Do not generate the
5440 compare instruction itself. */
5441
5442 static rtx
5443 vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode,
5444 tree t_op0, tree t_op1, bool unsignedp,
5445 enum insn_code icode, unsigned int opno)
5446 {
5447 class expand_operand ops[2];
5448 rtx rtx_op0, rtx_op1;
5449 machine_mode m0, m1;
5450 enum rtx_code rcode = get_rtx_code (tcode, unsignedp);
5451
5452 gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
5453
5454 /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
5455 has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
5456 cases, use the original mode. */
5457 rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
5458 EXPAND_STACK_PARM);
5459 m0 = GET_MODE (rtx_op0);
5460 if (m0 == VOIDmode)
5461 m0 = TYPE_MODE (TREE_TYPE (t_op0));
5462
5463 rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
5464 EXPAND_STACK_PARM);
5465 m1 = GET_MODE (rtx_op1);
5466 if (m1 == VOIDmode)
5467 m1 = TYPE_MODE (TREE_TYPE (t_op1));
5468
5469 create_input_operand (&ops[0], rtx_op0, m0);
5470 create_input_operand (&ops[1], rtx_op1, m1);
5471 if (!maybe_legitimize_operands (icode, opno, 2, ops))
5472 gcc_unreachable ();
5473 return gen_rtx_fmt_ee (rcode, cmp_mode, ops[0].value, ops[1].value);
5474 }
5475
5476 /* Check if vec_perm mask SEL is a constant equivalent to a shift of
5477 the first vec_perm operand, assuming the second operand (for left shift
5478 first operand) is a constant vector of zeros. Return the shift distance
5479 in bits if so, or NULL_RTX if the vec_perm is not a shift. MODE is the
5480 mode of the value being shifted. SHIFT_OPTAB is vec_shr_optab for right
5481 shift or vec_shl_optab for left shift. */
5482 static rtx
5483 shift_amt_for_vec_perm_mask (machine_mode mode, const vec_perm_indices &sel,
5484 optab shift_optab)
5485 {
5486 unsigned int bitsize = GET_MODE_UNIT_BITSIZE (mode);
5487 poly_int64 first = sel[0];
5488 if (maybe_ge (sel[0], GET_MODE_NUNITS (mode)))
5489 return NULL_RTX;
5490
5491 if (shift_optab == vec_shl_optab)
5492 {
5493 unsigned int nelt;
5494 if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
5495 return NULL_RTX;
5496 unsigned firstidx = 0;
5497 for (unsigned int i = 0; i < nelt; i++)
5498 {
5499 if (known_eq (sel[i], nelt))
5500 {
5501 if (i == 0 || firstidx)
5502 return NULL_RTX;
5503 firstidx = i;
5504 }
5505 else if (firstidx
5506 ? maybe_ne (sel[i], nelt + i - firstidx)
5507 : maybe_ge (sel[i], nelt))
5508 return NULL_RTX;
5509 }
5510
5511 if (firstidx == 0)
5512 return NULL_RTX;
5513 first = firstidx;
5514 }
5515 else if (!sel.series_p (0, 1, first, 1))
5516 {
5517 unsigned int nelt;
5518 if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
5519 return NULL_RTX;
5520 for (unsigned int i = 1; i < nelt; i++)
5521 {
5522 poly_int64 expected = i + first;
5523 /* Indices into the second vector are all equivalent. */
5524 if (maybe_lt (sel[i], nelt)
5525 ? maybe_ne (sel[i], expected)
5526 : maybe_lt (expected, nelt))
5527 return NULL_RTX;
5528 }
5529 }
5530
5531 return gen_int_shift_amount (mode, first * bitsize);
5532 }
5533
5534 /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
5535
5536 static rtx
5537 expand_vec_perm_1 (enum insn_code icode, rtx target,
5538 rtx v0, rtx v1, rtx sel)
5539 {
5540 machine_mode tmode = GET_MODE (target);
5541 machine_mode smode = GET_MODE (sel);
5542 class expand_operand ops[4];
5543
5544 gcc_assert (GET_MODE_CLASS (smode) == MODE_VECTOR_INT
5545 || related_int_vector_mode (tmode).require () == smode);
5546 create_output_operand (&ops[0], target, tmode);
5547 create_input_operand (&ops[3], sel, smode);
5548
5549 /* Make an effort to preserve v0 == v1. The target expander is able to
5550 rely on this to determine if we're permuting a single input operand. */
5551 if (rtx_equal_p (v0, v1))
5552 {
5553 if (!insn_operand_matches (icode, 1, v0))
5554 v0 = force_reg (tmode, v0);
5555 gcc_checking_assert (insn_operand_matches (icode, 1, v0));
5556 gcc_checking_assert (insn_operand_matches (icode, 2, v0));
5557
5558 create_fixed_operand (&ops[1], v0);
5559 create_fixed_operand (&ops[2], v0);
5560 }
5561 else
5562 {
5563 create_input_operand (&ops[1], v0, tmode);
5564 create_input_operand (&ops[2], v1, tmode);
5565 }
5566
5567 if (maybe_expand_insn (icode, 4, ops))
5568 return ops[0].value;
5569 return NULL_RTX;
5570 }
5571
5572 /* Implement a permutation of vectors v0 and v1 using the permutation
5573 vector in SEL and return the result. Use TARGET to hold the result
5574 if nonnull and convenient.
5575
5576 MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
5577 is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
5578 to have a particular mode. */
5579
5580 rtx
5581 expand_vec_perm_const (machine_mode mode, rtx v0, rtx v1,
5582 const vec_perm_builder &sel, machine_mode sel_mode,
5583 rtx target)
5584 {
5585 if (!target || !register_operand (target, mode))
5586 target = gen_reg_rtx (mode);
5587
5588 /* Set QIMODE to a different vector mode with byte elements.
5589 If no such mode, or if MODE already has byte elements, use VOIDmode. */
5590 machine_mode qimode;
5591 if (!qimode_for_vec_perm (mode).exists (&qimode))
5592 qimode = VOIDmode;
5593
5594 rtx_insn *last = get_last_insn ();
5595
5596 bool single_arg_p = rtx_equal_p (v0, v1);
5597 /* Always specify two input vectors here and leave the target to handle
5598 cases in which the inputs are equal. Not all backends can cope with
5599 the single-input representation when testing for a double-input
5600 target instruction. */
5601 vec_perm_indices indices (sel, 2, GET_MODE_NUNITS (mode));
5602
5603 /* See if this can be handled with a vec_shr or vec_shl. We only do this
5604 if the second (for vec_shr) or first (for vec_shl) vector is all
5605 zeroes. */
5606 insn_code shift_code = CODE_FOR_nothing;
5607 insn_code shift_code_qi = CODE_FOR_nothing;
5608 optab shift_optab = unknown_optab;
5609 rtx v2 = v0;
5610 if (v1 == CONST0_RTX (GET_MODE (v1)))
5611 shift_optab = vec_shr_optab;
5612 else if (v0 == CONST0_RTX (GET_MODE (v0)))
5613 {
5614 shift_optab = vec_shl_optab;
5615 v2 = v1;
5616 }
5617 if (shift_optab != unknown_optab)
5618 {
5619 shift_code = optab_handler (shift_optab, mode);
5620 shift_code_qi = ((qimode != VOIDmode && qimode != mode)
5621 ? optab_handler (shift_optab, qimode)
5622 : CODE_FOR_nothing);
5623 }
5624 if (shift_code != CODE_FOR_nothing || shift_code_qi != CODE_FOR_nothing)
5625 {
5626 rtx shift_amt = shift_amt_for_vec_perm_mask (mode, indices, shift_optab);
5627 if (shift_amt)
5628 {
5629 class expand_operand ops[3];
5630 if (shift_code != CODE_FOR_nothing)
5631 {
5632 create_output_operand (&ops[0], target, mode);
5633 create_input_operand (&ops[1], v2, mode);
5634 create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
5635 if (maybe_expand_insn (shift_code, 3, ops))
5636 return ops[0].value;
5637 }
5638 if (shift_code_qi != CODE_FOR_nothing)
5639 {
5640 rtx tmp = gen_reg_rtx (qimode);
5641 create_output_operand (&ops[0], tmp, qimode);
5642 create_input_operand (&ops[1], gen_lowpart (qimode, v2), qimode);
5643 create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
5644 if (maybe_expand_insn (shift_code_qi, 3, ops))
5645 return gen_lowpart (mode, ops[0].value);
5646 }
5647 }
5648 }
5649
5650 if (targetm.vectorize.vec_perm_const != NULL)
5651 {
5652 v0 = force_reg (mode, v0);
5653 if (single_arg_p)
5654 v1 = v0;
5655 else
5656 v1 = force_reg (mode, v1);
5657
5658 if (targetm.vectorize.vec_perm_const (mode, target, v0, v1, indices))
5659 return target;
5660 }
5661
5662 /* Fall back to a constant byte-based permutation. */
5663 vec_perm_indices qimode_indices;
5664 rtx target_qi = NULL_RTX, v0_qi = NULL_RTX, v1_qi = NULL_RTX;
5665 if (qimode != VOIDmode)
5666 {
5667 qimode_indices.new_expanded_vector (indices, GET_MODE_UNIT_SIZE (mode));
5668 target_qi = gen_reg_rtx (qimode);
5669 v0_qi = gen_lowpart (qimode, v0);
5670 v1_qi = gen_lowpart (qimode, v1);
5671 if (targetm.vectorize.vec_perm_const != NULL
5672 && targetm.vectorize.vec_perm_const (qimode, target_qi, v0_qi,
5673 v1_qi, qimode_indices))
5674 return gen_lowpart (mode, target_qi);
5675 }
5676
5677 /* Otherwise expand as a fully variable permuation. */
5678
5679 /* The optabs are only defined for selectors with the same width
5680 as the values being permuted. */
5681 machine_mode required_sel_mode;
5682 if (!related_int_vector_mode (mode).exists (&required_sel_mode))
5683 {
5684 delete_insns_since (last);
5685 return NULL_RTX;
5686 }
5687
5688 /* We know that it is semantically valid to treat SEL as having SEL_MODE.
5689 If that isn't the mode we want then we need to prove that using
5690 REQUIRED_SEL_MODE is OK. */
5691 if (sel_mode != required_sel_mode)
5692 {
5693 if (!selector_fits_mode_p (required_sel_mode, indices))
5694 {
5695 delete_insns_since (last);
5696 return NULL_RTX;
5697 }
5698 sel_mode = required_sel_mode;
5699 }
5700
5701 insn_code icode = direct_optab_handler (vec_perm_optab, mode);
5702 if (icode != CODE_FOR_nothing)
5703 {
5704 rtx sel_rtx = vec_perm_indices_to_rtx (sel_mode, indices);
5705 rtx tmp = expand_vec_perm_1 (icode, target, v0, v1, sel_rtx);
5706 if (tmp)
5707 return tmp;
5708 }
5709
5710 if (qimode != VOIDmode
5711 && selector_fits_mode_p (qimode, qimode_indices))
5712 {
5713 icode = direct_optab_handler (vec_perm_optab, qimode);
5714 if (icode != CODE_FOR_nothing)
5715 {
5716 rtx sel_qi = vec_perm_indices_to_rtx (qimode, qimode_indices);
5717 rtx tmp = expand_vec_perm_1 (icode, target_qi, v0_qi, v1_qi, sel_qi);
5718 if (tmp)
5719 return gen_lowpart (mode, tmp);
5720 }
5721 }
5722
5723 delete_insns_since (last);
5724 return NULL_RTX;
5725 }
5726
5727 /* Implement a permutation of vectors v0 and v1 using the permutation
5728 vector in SEL and return the result. Use TARGET to hold the result
5729 if nonnull and convenient.
5730
5731 MODE is the mode of the vectors being permuted (V0 and V1).
5732 SEL must have the integer equivalent of MODE and is known to be
5733 unsuitable for permutes with a constant permutation vector. */
5734
5735 rtx
5736 expand_vec_perm_var (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
5737 {
5738 enum insn_code icode;
5739 unsigned int i, u;
5740 rtx tmp, sel_qi;
5741
5742 u = GET_MODE_UNIT_SIZE (mode);
5743
5744 if (!target || GET_MODE (target) != mode)
5745 target = gen_reg_rtx (mode);
5746
5747 icode = direct_optab_handler (vec_perm_optab, mode);
5748 if (icode != CODE_FOR_nothing)
5749 {
5750 tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
5751 if (tmp)
5752 return tmp;
5753 }
5754
5755 /* As a special case to aid several targets, lower the element-based
5756 permutation to a byte-based permutation and try again. */
5757 machine_mode qimode;
5758 if (!qimode_for_vec_perm (mode).exists (&qimode)
5759 || maybe_gt (GET_MODE_NUNITS (qimode), GET_MODE_MASK (QImode) + 1))
5760 return NULL_RTX;
5761 icode = direct_optab_handler (vec_perm_optab, qimode);
5762 if (icode == CODE_FOR_nothing)
5763 return NULL_RTX;
5764
5765 /* Multiply each element by its byte size. */
5766 machine_mode selmode = GET_MODE (sel);
5767 if (u == 2)
5768 sel = expand_simple_binop (selmode, PLUS, sel, sel,
5769 NULL, 0, OPTAB_DIRECT);
5770 else
5771 sel = expand_simple_binop (selmode, ASHIFT, sel,
5772 gen_int_shift_amount (selmode, exact_log2 (u)),
5773 NULL, 0, OPTAB_DIRECT);
5774 gcc_assert (sel != NULL);
5775
5776 /* Broadcast the low byte each element into each of its bytes.
5777 The encoding has U interleaved stepped patterns, one for each
5778 byte of an element. */
5779 vec_perm_builder const_sel (GET_MODE_SIZE (mode), u, 3);
5780 unsigned int low_byte_in_u = BYTES_BIG_ENDIAN ? u - 1 : 0;
5781 for (i = 0; i < 3; ++i)
5782 for (unsigned int j = 0; j < u; ++j)
5783 const_sel.quick_push (i * u + low_byte_in_u);
5784 sel = gen_lowpart (qimode, sel);
5785 sel = expand_vec_perm_const (qimode, sel, sel, const_sel, qimode, NULL);
5786 gcc_assert (sel != NULL);
5787
5788 /* Add the byte offset to each byte element. */
5789 /* Note that the definition of the indicies here is memory ordering,
5790 so there should be no difference between big and little endian. */
5791 rtx_vector_builder byte_indices (qimode, u, 1);
5792 for (i = 0; i < u; ++i)
5793 byte_indices.quick_push (GEN_INT (i));
5794 tmp = byte_indices.build ();
5795 sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
5796 sel, 0, OPTAB_DIRECT);
5797 gcc_assert (sel_qi != NULL);
5798
5799 tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
5800 tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
5801 gen_lowpart (qimode, v1), sel_qi);
5802 if (tmp)
5803 tmp = gen_lowpart (mode, tmp);
5804 return tmp;
5805 }
5806
5807 /* Generate insns for a VEC_COND_EXPR with mask, given its TYPE and its
5808 three operands. */
5809
5810 rtx
5811 expand_vec_cond_mask_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
5812 rtx target)
5813 {
5814 class expand_operand ops[4];
5815 machine_mode mode = TYPE_MODE (vec_cond_type);
5816 machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
5817 enum insn_code icode = get_vcond_mask_icode (mode, mask_mode);
5818 rtx mask, rtx_op1, rtx_op2;
5819
5820 if (icode == CODE_FOR_nothing)
5821 return 0;
5822
5823 mask = expand_normal (op0);
5824 rtx_op1 = expand_normal (op1);
5825 rtx_op2 = expand_normal (op2);
5826
5827 mask = force_reg (mask_mode, mask);
5828 rtx_op1 = force_reg (GET_MODE (rtx_op1), rtx_op1);
5829
5830 create_output_operand (&ops[0], target, mode);
5831 create_input_operand (&ops[1], rtx_op1, mode);
5832 create_input_operand (&ops[2], rtx_op2, mode);
5833 create_input_operand (&ops[3], mask, mask_mode);
5834 expand_insn (icode, 4, ops);
5835
5836 return ops[0].value;
5837 }
5838
5839 /* Generate insns for a VEC_COND_EXPR, given its TYPE and its
5840 three operands. */
5841
5842 rtx
5843 expand_vec_cond_expr (tree vec_cond_type, tree op0, tree op1, tree op2,
5844 rtx target)
5845 {
5846 class expand_operand ops[6];
5847 enum insn_code icode;
5848 rtx comparison, rtx_op1, rtx_op2;
5849 machine_mode mode = TYPE_MODE (vec_cond_type);
5850 machine_mode cmp_op_mode;
5851 bool unsignedp;
5852 tree op0a, op0b;
5853 enum tree_code tcode;
5854
5855 if (COMPARISON_CLASS_P (op0))
5856 {
5857 op0a = TREE_OPERAND (op0, 0);
5858 op0b = TREE_OPERAND (op0, 1);
5859 tcode = TREE_CODE (op0);
5860 }
5861 else
5862 {
5863 gcc_assert (VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (op0)));
5864 if (get_vcond_mask_icode (mode, TYPE_MODE (TREE_TYPE (op0)))
5865 != CODE_FOR_nothing)
5866 return expand_vec_cond_mask_expr (vec_cond_type, op0, op1,
5867 op2, target);
5868 /* Fake op0 < 0. */
5869 else
5870 {
5871 gcc_assert (GET_MODE_CLASS (TYPE_MODE (TREE_TYPE (op0)))
5872 == MODE_VECTOR_INT);
5873 op0a = op0;
5874 op0b = build_zero_cst (TREE_TYPE (op0));
5875 tcode = LT_EXPR;
5876 }
5877 }
5878 cmp_op_mode = TYPE_MODE (TREE_TYPE (op0a));
5879 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
5880
5881
5882 gcc_assert (known_eq (GET_MODE_SIZE (mode), GET_MODE_SIZE (cmp_op_mode))
5883 && known_eq (GET_MODE_NUNITS (mode),
5884 GET_MODE_NUNITS (cmp_op_mode)));
5885
5886 icode = get_vcond_icode (mode, cmp_op_mode, unsignedp);
5887 if (icode == CODE_FOR_nothing)
5888 {
5889 if (tcode == LT_EXPR
5890 && op0a == op0
5891 && TREE_CODE (op0) == VECTOR_CST)
5892 {
5893 /* A VEC_COND_EXPR condition could be folded from EQ_EXPR/NE_EXPR
5894 into a constant when only get_vcond_eq_icode is supported.
5895 Verify < 0 and != 0 behave the same and change it to NE_EXPR. */
5896 unsigned HOST_WIDE_INT nelts;
5897 if (!VECTOR_CST_NELTS (op0).is_constant (&nelts))
5898 {
5899 if (VECTOR_CST_STEPPED_P (op0))
5900 return 0;
5901 nelts = vector_cst_encoded_nelts (op0);
5902 }
5903 for (unsigned int i = 0; i < nelts; ++i)
5904 if (tree_int_cst_sgn (vector_cst_elt (op0, i)) == 1)
5905 return 0;
5906 tcode = NE_EXPR;
5907 }
5908 if (tcode == EQ_EXPR || tcode == NE_EXPR)
5909 icode = get_vcond_eq_icode (mode, cmp_op_mode);
5910 if (icode == CODE_FOR_nothing)
5911 return 0;
5912 }
5913
5914 comparison = vector_compare_rtx (VOIDmode, tcode, op0a, op0b, unsignedp,
5915 icode, 4);
5916 rtx_op1 = expand_normal (op1);
5917 rtx_op2 = expand_normal (op2);
5918
5919 create_output_operand (&ops[0], target, mode);
5920 create_input_operand (&ops[1], rtx_op1, mode);
5921 create_input_operand (&ops[2], rtx_op2, mode);
5922 create_fixed_operand (&ops[3], comparison);
5923 create_fixed_operand (&ops[4], XEXP (comparison, 0));
5924 create_fixed_operand (&ops[5], XEXP (comparison, 1));
5925 expand_insn (icode, 6, ops);
5926 return ops[0].value;
5927 }
5928
5929 /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
5930 Use TARGET for the result if nonnull and convenient. */
5931
5932 rtx
5933 expand_vec_series_expr (machine_mode vmode, rtx op0, rtx op1, rtx target)
5934 {
5935 class expand_operand ops[3];
5936 enum insn_code icode;
5937 machine_mode emode = GET_MODE_INNER (vmode);
5938
5939 icode = direct_optab_handler (vec_series_optab, vmode);
5940 gcc_assert (icode != CODE_FOR_nothing);
5941
5942 create_output_operand (&ops[0], target, vmode);
5943 create_input_operand (&ops[1], op0, emode);
5944 create_input_operand (&ops[2], op1, emode);
5945
5946 expand_insn (icode, 3, ops);
5947 return ops[0].value;
5948 }
5949
5950 /* Generate insns for a vector comparison into a mask. */
5951
5952 rtx
5953 expand_vec_cmp_expr (tree type, tree exp, rtx target)
5954 {
5955 class expand_operand ops[4];
5956 enum insn_code icode;
5957 rtx comparison;
5958 machine_mode mask_mode = TYPE_MODE (type);
5959 machine_mode vmode;
5960 bool unsignedp;
5961 tree op0a, op0b;
5962 enum tree_code tcode;
5963
5964 op0a = TREE_OPERAND (exp, 0);
5965 op0b = TREE_OPERAND (exp, 1);
5966 tcode = TREE_CODE (exp);
5967
5968 unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
5969 vmode = TYPE_MODE (TREE_TYPE (op0a));
5970
5971 icode = get_vec_cmp_icode (vmode, mask_mode, unsignedp);
5972 if (icode == CODE_FOR_nothing)
5973 {
5974 if (tcode == EQ_EXPR || tcode == NE_EXPR)
5975 icode = get_vec_cmp_eq_icode (vmode, mask_mode);
5976 if (icode == CODE_FOR_nothing)
5977 return 0;
5978 }
5979
5980 comparison = vector_compare_rtx (mask_mode, tcode, op0a, op0b,
5981 unsignedp, icode, 2);
5982 create_output_operand (&ops[0], target, mask_mode);
5983 create_fixed_operand (&ops[1], comparison);
5984 create_fixed_operand (&ops[2], XEXP (comparison, 0));
5985 create_fixed_operand (&ops[3], XEXP (comparison, 1));
5986 expand_insn (icode, 4, ops);
5987 return ops[0].value;
5988 }
5989
5990 /* Expand a highpart multiply. */
5991
5992 rtx
5993 expand_mult_highpart (machine_mode mode, rtx op0, rtx op1,
5994 rtx target, bool uns_p)
5995 {
5996 class expand_operand eops[3];
5997 enum insn_code icode;
5998 int method, i;
5999 machine_mode wmode;
6000 rtx m1, m2;
6001 optab tab1, tab2;
6002
6003 method = can_mult_highpart_p (mode, uns_p);
6004 switch (method)
6005 {
6006 case 0:
6007 return NULL_RTX;
6008 case 1:
6009 tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
6010 return expand_binop (mode, tab1, op0, op1, target, uns_p,
6011 OPTAB_LIB_WIDEN);
6012 case 2:
6013 tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
6014 tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
6015 break;
6016 case 3:
6017 tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
6018 tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
6019 if (BYTES_BIG_ENDIAN)
6020 std::swap (tab1, tab2);
6021 break;
6022 default:
6023 gcc_unreachable ();
6024 }
6025
6026 icode = optab_handler (tab1, mode);
6027 wmode = insn_data[icode].operand[0].mode;
6028 gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode),
6029 GET_MODE_NUNITS (mode)));
6030 gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode), GET_MODE_SIZE (mode)));
6031
6032 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
6033 create_input_operand (&eops[1], op0, mode);
6034 create_input_operand (&eops[2], op1, mode);
6035 expand_insn (icode, 3, eops);
6036 m1 = gen_lowpart (mode, eops[0].value);
6037
6038 create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
6039 create_input_operand (&eops[1], op0, mode);
6040 create_input_operand (&eops[2], op1, mode);
6041 expand_insn (optab_handler (tab2, mode), 3, eops);
6042 m2 = gen_lowpart (mode, eops[0].value);
6043
6044 vec_perm_builder sel;
6045 if (method == 2)
6046 {
6047 /* The encoding has 2 interleaved stepped patterns. */
6048 sel.new_vector (GET_MODE_NUNITS (mode), 2, 3);
6049 for (i = 0; i < 6; ++i)
6050 sel.quick_push (!BYTES_BIG_ENDIAN + (i & ~1)
6051 + ((i & 1) ? GET_MODE_NUNITS (mode) : 0));
6052 }
6053 else
6054 {
6055 /* The encoding has a single interleaved stepped pattern. */
6056 sel.new_vector (GET_MODE_NUNITS (mode), 1, 3);
6057 for (i = 0; i < 3; ++i)
6058 sel.quick_push (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
6059 }
6060
6061 return expand_vec_perm_const (mode, m1, m2, sel, BLKmode, target);
6062 }
6063 \f
6064 /* Helper function to find the MODE_CC set in a sync_compare_and_swap
6065 pattern. */
6066
6067 static void
6068 find_cc_set (rtx x, const_rtx pat, void *data)
6069 {
6070 if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
6071 && GET_CODE (pat) == SET)
6072 {
6073 rtx *p_cc_reg = (rtx *) data;
6074 gcc_assert (!*p_cc_reg);
6075 *p_cc_reg = x;
6076 }
6077 }
6078
6079 /* This is a helper function for the other atomic operations. This function
6080 emits a loop that contains SEQ that iterates until a compare-and-swap
6081 operation at the end succeeds. MEM is the memory to be modified. SEQ is
6082 a set of instructions that takes a value from OLD_REG as an input and
6083 produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
6084 set to the current contents of MEM. After SEQ, a compare-and-swap will
6085 attempt to update MEM with NEW_REG. The function returns true when the
6086 loop was generated successfully. */
6087
6088 static bool
6089 expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
6090 {
6091 machine_mode mode = GET_MODE (mem);
6092 rtx_code_label *label;
6093 rtx cmp_reg, success, oldval;
6094
6095 /* The loop we want to generate looks like
6096
6097 cmp_reg = mem;
6098 label:
6099 old_reg = cmp_reg;
6100 seq;
6101 (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
6102 if (success)
6103 goto label;
6104
6105 Note that we only do the plain load from memory once. Subsequent
6106 iterations use the value loaded by the compare-and-swap pattern. */
6107
6108 label = gen_label_rtx ();
6109 cmp_reg = gen_reg_rtx (mode);
6110
6111 emit_move_insn (cmp_reg, mem);
6112 emit_label (label);
6113 emit_move_insn (old_reg, cmp_reg);
6114 if (seq)
6115 emit_insn (seq);
6116
6117 success = NULL_RTX;
6118 oldval = cmp_reg;
6119 if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
6120 new_reg, false, MEMMODEL_SYNC_SEQ_CST,
6121 MEMMODEL_RELAXED))
6122 return false;
6123
6124 if (oldval != cmp_reg)
6125 emit_move_insn (cmp_reg, oldval);
6126
6127 /* Mark this jump predicted not taken. */
6128 emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
6129 GET_MODE (success), 1, label,
6130 profile_probability::guessed_never ());
6131 return true;
6132 }
6133
6134
6135 /* This function tries to emit an atomic_exchange intruction. VAL is written
6136 to *MEM using memory model MODEL. The previous contents of *MEM are returned,
6137 using TARGET if possible. */
6138
6139 static rtx
6140 maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
6141 {
6142 machine_mode mode = GET_MODE (mem);
6143 enum insn_code icode;
6144
6145 /* If the target supports the exchange directly, great. */
6146 icode = direct_optab_handler (atomic_exchange_optab, mode);
6147 if (icode != CODE_FOR_nothing)
6148 {
6149 class expand_operand ops[4];
6150
6151 create_output_operand (&ops[0], target, mode);
6152 create_fixed_operand (&ops[1], mem);
6153 create_input_operand (&ops[2], val, mode);
6154 create_integer_operand (&ops[3], model);
6155 if (maybe_expand_insn (icode, 4, ops))
6156 return ops[0].value;
6157 }
6158
6159 return NULL_RTX;
6160 }
6161
6162 /* This function tries to implement an atomic exchange operation using
6163 __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
6164 The previous contents of *MEM are returned, using TARGET if possible.
6165 Since this instructionn is an acquire barrier only, stronger memory
6166 models may require additional barriers to be emitted. */
6167
6168 static rtx
6169 maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
6170 enum memmodel model)
6171 {
6172 machine_mode mode = GET_MODE (mem);
6173 enum insn_code icode;
6174 rtx_insn *last_insn = get_last_insn ();
6175
6176 icode = optab_handler (sync_lock_test_and_set_optab, mode);
6177
6178 /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
6179 exists, and the memory model is stronger than acquire, add a release
6180 barrier before the instruction. */
6181
6182 if (is_mm_seq_cst (model) || is_mm_release (model) || is_mm_acq_rel (model))
6183 expand_mem_thread_fence (model);
6184
6185 if (icode != CODE_FOR_nothing)
6186 {
6187 class expand_operand ops[3];
6188 create_output_operand (&ops[0], target, mode);
6189 create_fixed_operand (&ops[1], mem);
6190 create_input_operand (&ops[2], val, mode);
6191 if (maybe_expand_insn (icode, 3, ops))
6192 return ops[0].value;
6193 }
6194
6195 /* If an external test-and-set libcall is provided, use that instead of
6196 any external compare-and-swap that we might get from the compare-and-
6197 swap-loop expansion later. */
6198 if (!can_compare_and_swap_p (mode, false))
6199 {
6200 rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
6201 if (libfunc != NULL)
6202 {
6203 rtx addr;
6204
6205 addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
6206 return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
6207 mode, addr, ptr_mode,
6208 val, mode);
6209 }
6210 }
6211
6212 /* If the test_and_set can't be emitted, eliminate any barrier that might
6213 have been emitted. */
6214 delete_insns_since (last_insn);
6215 return NULL_RTX;
6216 }
6217
6218 /* This function tries to implement an atomic exchange operation using a
6219 compare_and_swap loop. VAL is written to *MEM. The previous contents of
6220 *MEM are returned, using TARGET if possible. No memory model is required
6221 since a compare_and_swap loop is seq-cst. */
6222
6223 static rtx
6224 maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
6225 {
6226 machine_mode mode = GET_MODE (mem);
6227
6228 if (can_compare_and_swap_p (mode, true))
6229 {
6230 if (!target || !register_operand (target, mode))
6231 target = gen_reg_rtx (mode);
6232 if (expand_compare_and_swap_loop (mem, target, val, NULL_RTX))
6233 return target;
6234 }
6235
6236 return NULL_RTX;
6237 }
6238
6239 /* This function tries to implement an atomic test-and-set operation
6240 using the atomic_test_and_set instruction pattern. A boolean value
6241 is returned from the operation, using TARGET if possible. */
6242
6243 static rtx
6244 maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
6245 {
6246 machine_mode pat_bool_mode;
6247 class expand_operand ops[3];
6248
6249 if (!targetm.have_atomic_test_and_set ())
6250 return NULL_RTX;
6251
6252 /* While we always get QImode from __atomic_test_and_set, we get
6253 other memory modes from __sync_lock_test_and_set. Note that we
6254 use no endian adjustment here. This matches the 4.6 behavior
6255 in the Sparc backend. */
6256 enum insn_code icode = targetm.code_for_atomic_test_and_set;
6257 gcc_checking_assert (insn_data[icode].operand[1].mode == QImode);
6258 if (GET_MODE (mem) != QImode)
6259 mem = adjust_address_nv (mem, QImode, 0);
6260
6261 pat_bool_mode = insn_data[icode].operand[0].mode;
6262 create_output_operand (&ops[0], target, pat_bool_mode);
6263 create_fixed_operand (&ops[1], mem);
6264 create_integer_operand (&ops[2], model);
6265
6266 if (maybe_expand_insn (icode, 3, ops))
6267 return ops[0].value;
6268 return NULL_RTX;
6269 }
6270
6271 /* This function expands the legacy _sync_lock test_and_set operation which is
6272 generally an atomic exchange. Some limited targets only allow the
6273 constant 1 to be stored. This is an ACQUIRE operation.
6274
6275 TARGET is an optional place to stick the return value.
6276 MEM is where VAL is stored. */
6277
6278 rtx
6279 expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
6280 {
6281 rtx ret;
6282
6283 /* Try an atomic_exchange first. */
6284 ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_SYNC_ACQUIRE);
6285 if (ret)
6286 return ret;
6287
6288 ret = maybe_emit_sync_lock_test_and_set (target, mem, val,
6289 MEMMODEL_SYNC_ACQUIRE);
6290 if (ret)
6291 return ret;
6292
6293 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
6294 if (ret)
6295 return ret;
6296
6297 /* If there are no other options, try atomic_test_and_set if the value
6298 being stored is 1. */
6299 if (val == const1_rtx)
6300 ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_SYNC_ACQUIRE);
6301
6302 return ret;
6303 }
6304
6305 /* This function expands the atomic test_and_set operation:
6306 atomically store a boolean TRUE into MEM and return the previous value.
6307
6308 MEMMODEL is the memory model variant to use.
6309 TARGET is an optional place to stick the return value. */
6310
6311 rtx
6312 expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
6313 {
6314 machine_mode mode = GET_MODE (mem);
6315 rtx ret, trueval, subtarget;
6316
6317 ret = maybe_emit_atomic_test_and_set (target, mem, model);
6318 if (ret)
6319 return ret;
6320
6321 /* Be binary compatible with non-default settings of trueval, and different
6322 cpu revisions. E.g. one revision may have atomic-test-and-set, but
6323 another only has atomic-exchange. */
6324 if (targetm.atomic_test_and_set_trueval == 1)
6325 {
6326 trueval = const1_rtx;
6327 subtarget = target ? target : gen_reg_rtx (mode);
6328 }
6329 else
6330 {
6331 trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
6332 subtarget = gen_reg_rtx (mode);
6333 }
6334
6335 /* Try the atomic-exchange optab... */
6336 ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model);
6337
6338 /* ... then an atomic-compare-and-swap loop ... */
6339 if (!ret)
6340 ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval);
6341
6342 /* ... before trying the vaguely defined legacy lock_test_and_set. */
6343 if (!ret)
6344 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model);
6345
6346 /* Recall that the legacy lock_test_and_set optab was allowed to do magic
6347 things with the value 1. Thus we try again without trueval. */
6348 if (!ret && targetm.atomic_test_and_set_trueval != 1)
6349 ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model);
6350
6351 /* Failing all else, assume a single threaded environment and simply
6352 perform the operation. */
6353 if (!ret)
6354 {
6355 /* If the result is ignored skip the move to target. */
6356 if (subtarget != const0_rtx)
6357 emit_move_insn (subtarget, mem);
6358
6359 emit_move_insn (mem, trueval);
6360 ret = subtarget;
6361 }
6362
6363 /* Recall that have to return a boolean value; rectify if trueval
6364 is not exactly one. */
6365 if (targetm.atomic_test_and_set_trueval != 1)
6366 ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
6367
6368 return ret;
6369 }
6370
6371 /* This function expands the atomic exchange operation:
6372 atomically store VAL in MEM and return the previous value in MEM.
6373
6374 MEMMODEL is the memory model variant to use.
6375 TARGET is an optional place to stick the return value. */
6376
6377 rtx
6378 expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
6379 {
6380 machine_mode mode = GET_MODE (mem);
6381 rtx ret;
6382
6383 /* If loads are not atomic for the required size and we are not called to
6384 provide a __sync builtin, do not do anything so that we stay consistent
6385 with atomic loads of the same size. */
6386 if (!can_atomic_load_p (mode) && !is_mm_sync (model))
6387 return NULL_RTX;
6388
6389 ret = maybe_emit_atomic_exchange (target, mem, val, model);
6390
6391 /* Next try a compare-and-swap loop for the exchange. */
6392 if (!ret)
6393 ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
6394
6395 return ret;
6396 }
6397
6398 /* This function expands the atomic compare exchange operation:
6399
6400 *PTARGET_BOOL is an optional place to store the boolean success/failure.
6401 *PTARGET_OVAL is an optional place to store the old value from memory.
6402 Both target parameters may be NULL or const0_rtx to indicate that we do
6403 not care about that return value. Both target parameters are updated on
6404 success to the actual location of the corresponding result.
6405
6406 MEMMODEL is the memory model variant to use.
6407
6408 The return value of the function is true for success. */
6409
6410 bool
6411 expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
6412 rtx mem, rtx expected, rtx desired,
6413 bool is_weak, enum memmodel succ_model,
6414 enum memmodel fail_model)
6415 {
6416 machine_mode mode = GET_MODE (mem);
6417 class expand_operand ops[8];
6418 enum insn_code icode;
6419 rtx target_oval, target_bool = NULL_RTX;
6420 rtx libfunc;
6421
6422 /* If loads are not atomic for the required size and we are not called to
6423 provide a __sync builtin, do not do anything so that we stay consistent
6424 with atomic loads of the same size. */
6425 if (!can_atomic_load_p (mode) && !is_mm_sync (succ_model))
6426 return false;
6427
6428 /* Load expected into a register for the compare and swap. */
6429 if (MEM_P (expected))
6430 expected = copy_to_reg (expected);
6431
6432 /* Make sure we always have some place to put the return oldval.
6433 Further, make sure that place is distinct from the input expected,
6434 just in case we need that path down below. */
6435 if (ptarget_oval && *ptarget_oval == const0_rtx)
6436 ptarget_oval = NULL;
6437
6438 if (ptarget_oval == NULL
6439 || (target_oval = *ptarget_oval) == NULL
6440 || reg_overlap_mentioned_p (expected, target_oval))
6441 target_oval = gen_reg_rtx (mode);
6442
6443 icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
6444 if (icode != CODE_FOR_nothing)
6445 {
6446 machine_mode bool_mode = insn_data[icode].operand[0].mode;
6447
6448 if (ptarget_bool && *ptarget_bool == const0_rtx)
6449 ptarget_bool = NULL;
6450
6451 /* Make sure we always have a place for the bool operand. */
6452 if (ptarget_bool == NULL
6453 || (target_bool = *ptarget_bool) == NULL
6454 || GET_MODE (target_bool) != bool_mode)
6455 target_bool = gen_reg_rtx (bool_mode);
6456
6457 /* Emit the compare_and_swap. */
6458 create_output_operand (&ops[0], target_bool, bool_mode);
6459 create_output_operand (&ops[1], target_oval, mode);
6460 create_fixed_operand (&ops[2], mem);
6461 create_input_operand (&ops[3], expected, mode);
6462 create_input_operand (&ops[4], desired, mode);
6463 create_integer_operand (&ops[5], is_weak);
6464 create_integer_operand (&ops[6], succ_model);
6465 create_integer_operand (&ops[7], fail_model);
6466 if (maybe_expand_insn (icode, 8, ops))
6467 {
6468 /* Return success/failure. */
6469 target_bool = ops[0].value;
6470 target_oval = ops[1].value;
6471 goto success;
6472 }
6473 }
6474
6475 /* Otherwise fall back to the original __sync_val_compare_and_swap
6476 which is always seq-cst. */
6477 icode = optab_handler (sync_compare_and_swap_optab, mode);
6478 if (icode != CODE_FOR_nothing)
6479 {
6480 rtx cc_reg;
6481
6482 create_output_operand (&ops[0], target_oval, mode);
6483 create_fixed_operand (&ops[1], mem);
6484 create_input_operand (&ops[2], expected, mode);
6485 create_input_operand (&ops[3], desired, mode);
6486 if (!maybe_expand_insn (icode, 4, ops))
6487 return false;
6488
6489 target_oval = ops[0].value;
6490
6491 /* If the caller isn't interested in the boolean return value,
6492 skip the computation of it. */
6493 if (ptarget_bool == NULL)
6494 goto success;
6495
6496 /* Otherwise, work out if the compare-and-swap succeeded. */
6497 cc_reg = NULL_RTX;
6498 if (have_insn_for (COMPARE, CCmode))
6499 note_stores (get_last_insn (), find_cc_set, &cc_reg);
6500 if (cc_reg)
6501 {
6502 target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
6503 const0_rtx, VOIDmode, 0, 1);
6504 goto success;
6505 }
6506 goto success_bool_from_val;
6507 }
6508
6509 /* Also check for library support for __sync_val_compare_and_swap. */
6510 libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
6511 if (libfunc != NULL)
6512 {
6513 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
6514 rtx target = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
6515 mode, addr, ptr_mode,
6516 expected, mode, desired, mode);
6517 emit_move_insn (target_oval, target);
6518
6519 /* Compute the boolean return value only if requested. */
6520 if (ptarget_bool)
6521 goto success_bool_from_val;
6522 else
6523 goto success;
6524 }
6525
6526 /* Failure. */
6527 return false;
6528
6529 success_bool_from_val:
6530 target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
6531 expected, VOIDmode, 1, 1);
6532 success:
6533 /* Make sure that the oval output winds up where the caller asked. */
6534 if (ptarget_oval)
6535 *ptarget_oval = target_oval;
6536 if (ptarget_bool)
6537 *ptarget_bool = target_bool;
6538 return true;
6539 }
6540
6541 /* Generate asm volatile("" : : : "memory") as the memory blockage. */
6542
6543 static void
6544 expand_asm_memory_blockage (void)
6545 {
6546 rtx asm_op, clob;
6547
6548 asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
6549 rtvec_alloc (0), rtvec_alloc (0),
6550 rtvec_alloc (0), UNKNOWN_LOCATION);
6551 MEM_VOLATILE_P (asm_op) = 1;
6552
6553 clob = gen_rtx_SCRATCH (VOIDmode);
6554 clob = gen_rtx_MEM (BLKmode, clob);
6555 clob = gen_rtx_CLOBBER (VOIDmode, clob);
6556
6557 emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
6558 }
6559
6560 /* Do not propagate memory accesses across this point. */
6561
6562 static void
6563 expand_memory_blockage (void)
6564 {
6565 if (targetm.have_memory_blockage ())
6566 emit_insn (targetm.gen_memory_blockage ());
6567 else
6568 expand_asm_memory_blockage ();
6569 }
6570
6571 /* This routine will either emit the mem_thread_fence pattern or issue a
6572 sync_synchronize to generate a fence for memory model MEMMODEL. */
6573
6574 void
6575 expand_mem_thread_fence (enum memmodel model)
6576 {
6577 if (is_mm_relaxed (model))
6578 return;
6579 if (targetm.have_mem_thread_fence ())
6580 {
6581 emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
6582 expand_memory_blockage ();
6583 }
6584 else if (targetm.have_memory_barrier ())
6585 emit_insn (targetm.gen_memory_barrier ());
6586 else if (synchronize_libfunc != NULL_RTX)
6587 emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode);
6588 else
6589 expand_memory_blockage ();
6590 }
6591
6592 /* Emit a signal fence with given memory model. */
6593
6594 void
6595 expand_mem_signal_fence (enum memmodel model)
6596 {
6597 /* No machine barrier is required to implement a signal fence, but
6598 a compiler memory barrier must be issued, except for relaxed MM. */
6599 if (!is_mm_relaxed (model))
6600 expand_memory_blockage ();
6601 }
6602
6603 /* This function expands the atomic load operation:
6604 return the atomically loaded value in MEM.
6605
6606 MEMMODEL is the memory model variant to use.
6607 TARGET is an option place to stick the return value. */
6608
6609 rtx
6610 expand_atomic_load (rtx target, rtx mem, enum memmodel model)
6611 {
6612 machine_mode mode = GET_MODE (mem);
6613 enum insn_code icode;
6614
6615 /* If the target supports the load directly, great. */
6616 icode = direct_optab_handler (atomic_load_optab, mode);
6617 if (icode != CODE_FOR_nothing)
6618 {
6619 class expand_operand ops[3];
6620 rtx_insn *last = get_last_insn ();
6621 if (is_mm_seq_cst (model))
6622 expand_memory_blockage ();
6623
6624 create_output_operand (&ops[0], target, mode);
6625 create_fixed_operand (&ops[1], mem);
6626 create_integer_operand (&ops[2], model);
6627 if (maybe_expand_insn (icode, 3, ops))
6628 {
6629 if (!is_mm_relaxed (model))
6630 expand_memory_blockage ();
6631 return ops[0].value;
6632 }
6633 delete_insns_since (last);
6634 }
6635
6636 /* If the size of the object is greater than word size on this target,
6637 then we assume that a load will not be atomic. We could try to
6638 emulate a load with a compare-and-swap operation, but the store that
6639 doing this could result in would be incorrect if this is a volatile
6640 atomic load or targetting read-only-mapped memory. */
6641 if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
6642 /* If there is no atomic load, leave the library call. */
6643 return NULL_RTX;
6644
6645 /* Otherwise assume loads are atomic, and emit the proper barriers. */
6646 if (!target || target == const0_rtx)
6647 target = gen_reg_rtx (mode);
6648
6649 /* For SEQ_CST, emit a barrier before the load. */
6650 if (is_mm_seq_cst (model))
6651 expand_mem_thread_fence (model);
6652
6653 emit_move_insn (target, mem);
6654
6655 /* Emit the appropriate barrier after the load. */
6656 expand_mem_thread_fence (model);
6657
6658 return target;
6659 }
6660
6661 /* This function expands the atomic store operation:
6662 Atomically store VAL in MEM.
6663 MEMMODEL is the memory model variant to use.
6664 USE_RELEASE is true if __sync_lock_release can be used as a fall back.
6665 function returns const0_rtx if a pattern was emitted. */
6666
6667 rtx
6668 expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
6669 {
6670 machine_mode mode = GET_MODE (mem);
6671 enum insn_code icode;
6672 class expand_operand ops[3];
6673
6674 /* If the target supports the store directly, great. */
6675 icode = direct_optab_handler (atomic_store_optab, mode);
6676 if (icode != CODE_FOR_nothing)
6677 {
6678 rtx_insn *last = get_last_insn ();
6679 if (!is_mm_relaxed (model))
6680 expand_memory_blockage ();
6681 create_fixed_operand (&ops[0], mem);
6682 create_input_operand (&ops[1], val, mode);
6683 create_integer_operand (&ops[2], model);
6684 if (maybe_expand_insn (icode, 3, ops))
6685 {
6686 if (is_mm_seq_cst (model))
6687 expand_memory_blockage ();
6688 return const0_rtx;
6689 }
6690 delete_insns_since (last);
6691 }
6692
6693 /* If using __sync_lock_release is a viable alternative, try it.
6694 Note that this will not be set to true if we are expanding a generic
6695 __atomic_store_n. */
6696 if (use_release)
6697 {
6698 icode = direct_optab_handler (sync_lock_release_optab, mode);
6699 if (icode != CODE_FOR_nothing)
6700 {
6701 create_fixed_operand (&ops[0], mem);
6702 create_input_operand (&ops[1], const0_rtx, mode);
6703 if (maybe_expand_insn (icode, 2, ops))
6704 {
6705 /* lock_release is only a release barrier. */
6706 if (is_mm_seq_cst (model))
6707 expand_mem_thread_fence (model);
6708 return const0_rtx;
6709 }
6710 }
6711 }
6712
6713 /* If the size of the object is greater than word size on this target,
6714 a default store will not be atomic. */
6715 if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
6716 {
6717 /* If loads are atomic or we are called to provide a __sync builtin,
6718 we can try a atomic_exchange and throw away the result. Otherwise,
6719 don't do anything so that we do not create an inconsistency between
6720 loads and stores. */
6721 if (can_atomic_load_p (mode) || is_mm_sync (model))
6722 {
6723 rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
6724 if (!target)
6725 target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem,
6726 val);
6727 if (target)
6728 return const0_rtx;
6729 }
6730 return NULL_RTX;
6731 }
6732
6733 /* Otherwise assume stores are atomic, and emit the proper barriers. */
6734 expand_mem_thread_fence (model);
6735
6736 emit_move_insn (mem, val);
6737
6738 /* For SEQ_CST, also emit a barrier after the store. */
6739 if (is_mm_seq_cst (model))
6740 expand_mem_thread_fence (model);
6741
6742 return const0_rtx;
6743 }
6744
6745
6746 /* Structure containing the pointers and values required to process the
6747 various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
6748
6749 struct atomic_op_functions
6750 {
6751 direct_optab mem_fetch_before;
6752 direct_optab mem_fetch_after;
6753 direct_optab mem_no_result;
6754 optab fetch_before;
6755 optab fetch_after;
6756 direct_optab no_result;
6757 enum rtx_code reverse_code;
6758 };
6759
6760
6761 /* Fill in structure pointed to by OP with the various optab entries for an
6762 operation of type CODE. */
6763
6764 static void
6765 get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
6766 {
6767 gcc_assert (op!= NULL);
6768
6769 /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
6770 in the source code during compilation, and the optab entries are not
6771 computable until runtime. Fill in the values at runtime. */
6772 switch (code)
6773 {
6774 case PLUS:
6775 op->mem_fetch_before = atomic_fetch_add_optab;
6776 op->mem_fetch_after = atomic_add_fetch_optab;
6777 op->mem_no_result = atomic_add_optab;
6778 op->fetch_before = sync_old_add_optab;
6779 op->fetch_after = sync_new_add_optab;
6780 op->no_result = sync_add_optab;
6781 op->reverse_code = MINUS;
6782 break;
6783 case MINUS:
6784 op->mem_fetch_before = atomic_fetch_sub_optab;
6785 op->mem_fetch_after = atomic_sub_fetch_optab;
6786 op->mem_no_result = atomic_sub_optab;
6787 op->fetch_before = sync_old_sub_optab;
6788 op->fetch_after = sync_new_sub_optab;
6789 op->no_result = sync_sub_optab;
6790 op->reverse_code = PLUS;
6791 break;
6792 case XOR:
6793 op->mem_fetch_before = atomic_fetch_xor_optab;
6794 op->mem_fetch_after = atomic_xor_fetch_optab;
6795 op->mem_no_result = atomic_xor_optab;
6796 op->fetch_before = sync_old_xor_optab;
6797 op->fetch_after = sync_new_xor_optab;
6798 op->no_result = sync_xor_optab;
6799 op->reverse_code = XOR;
6800 break;
6801 case AND:
6802 op->mem_fetch_before = atomic_fetch_and_optab;
6803 op->mem_fetch_after = atomic_and_fetch_optab;
6804 op->mem_no_result = atomic_and_optab;
6805 op->fetch_before = sync_old_and_optab;
6806 op->fetch_after = sync_new_and_optab;
6807 op->no_result = sync_and_optab;
6808 op->reverse_code = UNKNOWN;
6809 break;
6810 case IOR:
6811 op->mem_fetch_before = atomic_fetch_or_optab;
6812 op->mem_fetch_after = atomic_or_fetch_optab;
6813 op->mem_no_result = atomic_or_optab;
6814 op->fetch_before = sync_old_ior_optab;
6815 op->fetch_after = sync_new_ior_optab;
6816 op->no_result = sync_ior_optab;
6817 op->reverse_code = UNKNOWN;
6818 break;
6819 case NOT:
6820 op->mem_fetch_before = atomic_fetch_nand_optab;
6821 op->mem_fetch_after = atomic_nand_fetch_optab;
6822 op->mem_no_result = atomic_nand_optab;
6823 op->fetch_before = sync_old_nand_optab;
6824 op->fetch_after = sync_new_nand_optab;
6825 op->no_result = sync_nand_optab;
6826 op->reverse_code = UNKNOWN;
6827 break;
6828 default:
6829 gcc_unreachable ();
6830 }
6831 }
6832
6833 /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
6834 using memory order MODEL. If AFTER is true the operation needs to return
6835 the value of *MEM after the operation, otherwise the previous value.
6836 TARGET is an optional place to place the result. The result is unused if
6837 it is const0_rtx.
6838 Return the result if there is a better sequence, otherwise NULL_RTX. */
6839
6840 static rtx
6841 maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
6842 enum memmodel model, bool after)
6843 {
6844 /* If the value is prefetched, or not used, it may be possible to replace
6845 the sequence with a native exchange operation. */
6846 if (!after || target == const0_rtx)
6847 {
6848 /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
6849 if (code == AND && val == const0_rtx)
6850 {
6851 if (target == const0_rtx)
6852 target = gen_reg_rtx (GET_MODE (mem));
6853 return maybe_emit_atomic_exchange (target, mem, val, model);
6854 }
6855
6856 /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
6857 if (code == IOR && val == constm1_rtx)
6858 {
6859 if (target == const0_rtx)
6860 target = gen_reg_rtx (GET_MODE (mem));
6861 return maybe_emit_atomic_exchange (target, mem, val, model);
6862 }
6863 }
6864
6865 return NULL_RTX;
6866 }
6867
6868 /* Try to emit an instruction for a specific operation varaition.
6869 OPTAB contains the OP functions.
6870 TARGET is an optional place to return the result. const0_rtx means unused.
6871 MEM is the memory location to operate on.
6872 VAL is the value to use in the operation.
6873 USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
6874 MODEL is the memory model, if used.
6875 AFTER is true if the returned result is the value after the operation. */
6876
6877 static rtx
6878 maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
6879 rtx val, bool use_memmodel, enum memmodel model, bool after)
6880 {
6881 machine_mode mode = GET_MODE (mem);
6882 class expand_operand ops[4];
6883 enum insn_code icode;
6884 int op_counter = 0;
6885 int num_ops;
6886
6887 /* Check to see if there is a result returned. */
6888 if (target == const0_rtx)
6889 {
6890 if (use_memmodel)
6891 {
6892 icode = direct_optab_handler (optab->mem_no_result, mode);
6893 create_integer_operand (&ops[2], model);
6894 num_ops = 3;
6895 }
6896 else
6897 {
6898 icode = direct_optab_handler (optab->no_result, mode);
6899 num_ops = 2;
6900 }
6901 }
6902 /* Otherwise, we need to generate a result. */
6903 else
6904 {
6905 if (use_memmodel)
6906 {
6907 icode = direct_optab_handler (after ? optab->mem_fetch_after
6908 : optab->mem_fetch_before, mode);
6909 create_integer_operand (&ops[3], model);
6910 num_ops = 4;
6911 }
6912 else
6913 {
6914 icode = optab_handler (after ? optab->fetch_after
6915 : optab->fetch_before, mode);
6916 num_ops = 3;
6917 }
6918 create_output_operand (&ops[op_counter++], target, mode);
6919 }
6920 if (icode == CODE_FOR_nothing)
6921 return NULL_RTX;
6922
6923 create_fixed_operand (&ops[op_counter++], mem);
6924 /* VAL may have been promoted to a wider mode. Shrink it if so. */
6925 create_convert_operand_to (&ops[op_counter++], val, mode, true);
6926
6927 if (maybe_expand_insn (icode, num_ops, ops))
6928 return (target == const0_rtx ? const0_rtx : ops[0].value);
6929
6930 return NULL_RTX;
6931 }
6932
6933
6934 /* This function expands an atomic fetch_OP or OP_fetch operation:
6935 TARGET is an option place to stick the return value. const0_rtx indicates
6936 the result is unused.
6937 atomically fetch MEM, perform the operation with VAL and return it to MEM.
6938 CODE is the operation being performed (OP)
6939 MEMMODEL is the memory model variant to use.
6940 AFTER is true to return the result of the operation (OP_fetch).
6941 AFTER is false to return the value before the operation (fetch_OP).
6942
6943 This function will *only* generate instructions if there is a direct
6944 optab. No compare and swap loops or libcalls will be generated. */
6945
6946 static rtx
6947 expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
6948 enum rtx_code code, enum memmodel model,
6949 bool after)
6950 {
6951 machine_mode mode = GET_MODE (mem);
6952 struct atomic_op_functions optab;
6953 rtx result;
6954 bool unused_result = (target == const0_rtx);
6955
6956 get_atomic_op_for_code (&optab, code);
6957
6958 /* Check to see if there are any better instructions. */
6959 result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
6960 if (result)
6961 return result;
6962
6963 /* Check for the case where the result isn't used and try those patterns. */
6964 if (unused_result)
6965 {
6966 /* Try the memory model variant first. */
6967 result = maybe_emit_op (&optab, target, mem, val, true, model, true);
6968 if (result)
6969 return result;
6970
6971 /* Next try the old style withuot a memory model. */
6972 result = maybe_emit_op (&optab, target, mem, val, false, model, true);
6973 if (result)
6974 return result;
6975
6976 /* There is no no-result pattern, so try patterns with a result. */
6977 target = NULL_RTX;
6978 }
6979
6980 /* Try the __atomic version. */
6981 result = maybe_emit_op (&optab, target, mem, val, true, model, after);
6982 if (result)
6983 return result;
6984
6985 /* Try the older __sync version. */
6986 result = maybe_emit_op (&optab, target, mem, val, false, model, after);
6987 if (result)
6988 return result;
6989
6990 /* If the fetch value can be calculated from the other variation of fetch,
6991 try that operation. */
6992 if (after || unused_result || optab.reverse_code != UNKNOWN)
6993 {
6994 /* Try the __atomic version, then the older __sync version. */
6995 result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
6996 if (!result)
6997 result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
6998
6999 if (result)
7000 {
7001 /* If the result isn't used, no need to do compensation code. */
7002 if (unused_result)
7003 return result;
7004
7005 /* Issue compensation code. Fetch_after == fetch_before OP val.
7006 Fetch_before == after REVERSE_OP val. */
7007 if (!after)
7008 code = optab.reverse_code;
7009 if (code == NOT)
7010 {
7011 result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
7012 true, OPTAB_LIB_WIDEN);
7013 result = expand_simple_unop (mode, NOT, result, target, true);
7014 }
7015 else
7016 result = expand_simple_binop (mode, code, result, val, target,
7017 true, OPTAB_LIB_WIDEN);
7018 return result;
7019 }
7020 }
7021
7022 /* No direct opcode can be generated. */
7023 return NULL_RTX;
7024 }
7025
7026
7027
7028 /* This function expands an atomic fetch_OP or OP_fetch operation:
7029 TARGET is an option place to stick the return value. const0_rtx indicates
7030 the result is unused.
7031 atomically fetch MEM, perform the operation with VAL and return it to MEM.
7032 CODE is the operation being performed (OP)
7033 MEMMODEL is the memory model variant to use.
7034 AFTER is true to return the result of the operation (OP_fetch).
7035 AFTER is false to return the value before the operation (fetch_OP). */
7036 rtx
7037 expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
7038 enum memmodel model, bool after)
7039 {
7040 machine_mode mode = GET_MODE (mem);
7041 rtx result;
7042 bool unused_result = (target == const0_rtx);
7043
7044 /* If loads are not atomic for the required size and we are not called to
7045 provide a __sync builtin, do not do anything so that we stay consistent
7046 with atomic loads of the same size. */
7047 if (!can_atomic_load_p (mode) && !is_mm_sync (model))
7048 return NULL_RTX;
7049
7050 result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
7051 after);
7052
7053 if (result)
7054 return result;
7055
7056 /* Add/sub can be implemented by doing the reverse operation with -(val). */
7057 if (code == PLUS || code == MINUS)
7058 {
7059 rtx tmp;
7060 enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
7061
7062 start_sequence ();
7063 tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true);
7064 result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse,
7065 model, after);
7066 if (result)
7067 {
7068 /* PLUS worked so emit the insns and return. */
7069 tmp = get_insns ();
7070 end_sequence ();
7071 emit_insn (tmp);
7072 return result;
7073 }
7074
7075 /* PLUS did not work, so throw away the negation code and continue. */
7076 end_sequence ();
7077 }
7078
7079 /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
7080 if (!can_compare_and_swap_p (mode, false))
7081 {
7082 rtx libfunc;
7083 bool fixup = false;
7084 enum rtx_code orig_code = code;
7085 struct atomic_op_functions optab;
7086
7087 get_atomic_op_for_code (&optab, code);
7088 libfunc = optab_libfunc (after ? optab.fetch_after
7089 : optab.fetch_before, mode);
7090 if (libfunc == NULL
7091 && (after || unused_result || optab.reverse_code != UNKNOWN))
7092 {
7093 fixup = true;
7094 if (!after)
7095 code = optab.reverse_code;
7096 libfunc = optab_libfunc (after ? optab.fetch_before
7097 : optab.fetch_after, mode);
7098 }
7099 if (libfunc != NULL)
7100 {
7101 rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7102 result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
7103 addr, ptr_mode, val, mode);
7104
7105 if (!unused_result && fixup)
7106 result = expand_simple_binop (mode, code, result, val, target,
7107 true, OPTAB_LIB_WIDEN);
7108 return result;
7109 }
7110
7111 /* We need the original code for any further attempts. */
7112 code = orig_code;
7113 }
7114
7115 /* If nothing else has succeeded, default to a compare and swap loop. */
7116 if (can_compare_and_swap_p (mode, true))
7117 {
7118 rtx_insn *insn;
7119 rtx t0 = gen_reg_rtx (mode), t1;
7120
7121 start_sequence ();
7122
7123 /* If the result is used, get a register for it. */
7124 if (!unused_result)
7125 {
7126 if (!target || !register_operand (target, mode))
7127 target = gen_reg_rtx (mode);
7128 /* If fetch_before, copy the value now. */
7129 if (!after)
7130 emit_move_insn (target, t0);
7131 }
7132 else
7133 target = const0_rtx;
7134
7135 t1 = t0;
7136 if (code == NOT)
7137 {
7138 t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
7139 true, OPTAB_LIB_WIDEN);
7140 t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
7141 }
7142 else
7143 t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
7144 OPTAB_LIB_WIDEN);
7145
7146 /* For after, copy the value now. */
7147 if (!unused_result && after)
7148 emit_move_insn (target, t1);
7149 insn = get_insns ();
7150 end_sequence ();
7151
7152 if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
7153 return target;
7154 }
7155
7156 return NULL_RTX;
7157 }
7158 \f
7159 /* Return true if OPERAND is suitable for operand number OPNO of
7160 instruction ICODE. */
7161
7162 bool
7163 insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
7164 {
7165 return (!insn_data[(int) icode].operand[opno].predicate
7166 || (insn_data[(int) icode].operand[opno].predicate
7167 (operand, insn_data[(int) icode].operand[opno].mode)));
7168 }
7169 \f
7170 /* TARGET is a target of a multiword operation that we are going to
7171 implement as a series of word-mode operations. Return true if
7172 TARGET is suitable for this purpose. */
7173
7174 bool
7175 valid_multiword_target_p (rtx target)
7176 {
7177 machine_mode mode;
7178 int i, size;
7179
7180 mode = GET_MODE (target);
7181 if (!GET_MODE_SIZE (mode).is_constant (&size))
7182 return false;
7183 for (i = 0; i < size; i += UNITS_PER_WORD)
7184 if (!validate_subreg (word_mode, mode, target, i))
7185 return false;
7186 return true;
7187 }
7188
7189 /* Make OP describe an input operand that has value INTVAL and that has
7190 no inherent mode. This function should only be used for operands that
7191 are always expand-time constants. The backend may request that INTVAL
7192 be copied into a different kind of rtx, but it must specify the mode
7193 of that rtx if so. */
7194
7195 void
7196 create_integer_operand (class expand_operand *op, poly_int64 intval)
7197 {
7198 create_expand_operand (op, EXPAND_INTEGER,
7199 gen_int_mode (intval, MAX_MODE_INT),
7200 VOIDmode, false, intval);
7201 }
7202
7203 /* Like maybe_legitimize_operand, but do not change the code of the
7204 current rtx value. */
7205
7206 static bool
7207 maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
7208 class expand_operand *op)
7209 {
7210 /* See if the operand matches in its current form. */
7211 if (insn_operand_matches (icode, opno, op->value))
7212 return true;
7213
7214 /* If the operand is a memory whose address has no side effects,
7215 try forcing the address into a non-virtual pseudo register.
7216 The check for side effects is important because copy_to_mode_reg
7217 cannot handle things like auto-modified addresses. */
7218 if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
7219 {
7220 rtx addr, mem;
7221
7222 mem = op->value;
7223 addr = XEXP (mem, 0);
7224 if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
7225 && !side_effects_p (addr))
7226 {
7227 rtx_insn *last;
7228 machine_mode mode;
7229
7230 last = get_last_insn ();
7231 mode = get_address_mode (mem);
7232 mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
7233 if (insn_operand_matches (icode, opno, mem))
7234 {
7235 op->value = mem;
7236 return true;
7237 }
7238 delete_insns_since (last);
7239 }
7240 }
7241
7242 return false;
7243 }
7244
7245 /* Try to make OP match operand OPNO of instruction ICODE. Return true
7246 on success, storing the new operand value back in OP. */
7247
7248 static bool
7249 maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
7250 class expand_operand *op)
7251 {
7252 machine_mode mode, imode, tmode;
7253
7254 mode = op->mode;
7255 switch (op->type)
7256 {
7257 case EXPAND_FIXED:
7258 {
7259 temporary_volatile_ok v (true);
7260 return maybe_legitimize_operand_same_code (icode, opno, op);
7261 }
7262
7263 case EXPAND_OUTPUT:
7264 gcc_assert (mode != VOIDmode);
7265 if (op->value
7266 && op->value != const0_rtx
7267 && GET_MODE (op->value) == mode
7268 && maybe_legitimize_operand_same_code (icode, opno, op))
7269 return true;
7270
7271 op->value = gen_reg_rtx (mode);
7272 op->target = 0;
7273 break;
7274
7275 case EXPAND_INPUT:
7276 input:
7277 gcc_assert (mode != VOIDmode);
7278 gcc_assert (GET_MODE (op->value) == VOIDmode
7279 || GET_MODE (op->value) == mode);
7280 if (maybe_legitimize_operand_same_code (icode, opno, op))
7281 return true;
7282
7283 op->value = copy_to_mode_reg (mode, op->value);
7284 break;
7285
7286 case EXPAND_CONVERT_TO:
7287 gcc_assert (mode != VOIDmode);
7288 op->value = convert_to_mode (mode, op->value, op->unsigned_p);
7289 goto input;
7290
7291 case EXPAND_CONVERT_FROM:
7292 if (GET_MODE (op->value) != VOIDmode)
7293 mode = GET_MODE (op->value);
7294 else
7295 /* The caller must tell us what mode this value has. */
7296 gcc_assert (mode != VOIDmode);
7297
7298 imode = insn_data[(int) icode].operand[opno].mode;
7299 tmode = (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode)
7300 ? GET_MODE_INNER (imode) : imode);
7301 if (tmode != VOIDmode && tmode != mode)
7302 {
7303 op->value = convert_modes (tmode, mode, op->value, op->unsigned_p);
7304 mode = tmode;
7305 }
7306 if (imode != VOIDmode && imode != mode)
7307 {
7308 gcc_assert (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode));
7309 op->value = expand_vector_broadcast (imode, op->value);
7310 mode = imode;
7311 }
7312 goto input;
7313
7314 case EXPAND_ADDRESS:
7315 op->value = convert_memory_address (as_a <scalar_int_mode> (mode),
7316 op->value);
7317 goto input;
7318
7319 case EXPAND_INTEGER:
7320 mode = insn_data[(int) icode].operand[opno].mode;
7321 if (mode != VOIDmode
7322 && known_eq (trunc_int_for_mode (op->int_value, mode),
7323 op->int_value))
7324 {
7325 op->value = gen_int_mode (op->int_value, mode);
7326 goto input;
7327 }
7328 break;
7329 }
7330 return insn_operand_matches (icode, opno, op->value);
7331 }
7332
7333 /* Make OP describe an input operand that should have the same value
7334 as VALUE, after any mode conversion that the target might request.
7335 TYPE is the type of VALUE. */
7336
7337 void
7338 create_convert_operand_from_type (class expand_operand *op,
7339 rtx value, tree type)
7340 {
7341 create_convert_operand_from (op, value, TYPE_MODE (type),
7342 TYPE_UNSIGNED (type));
7343 }
7344
7345 /* Return true if the requirements on operands OP1 and OP2 of instruction
7346 ICODE are similar enough for the result of legitimizing OP1 to be
7347 reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
7348 with OP1 and OP2 respectively. */
7349
7350 static inline bool
7351 can_reuse_operands_p (enum insn_code icode,
7352 unsigned int opno1, unsigned int opno2,
7353 const class expand_operand *op1,
7354 const class expand_operand *op2)
7355 {
7356 /* Check requirements that are common to all types. */
7357 if (op1->type != op2->type
7358 || op1->mode != op2->mode
7359 || (insn_data[(int) icode].operand[opno1].mode
7360 != insn_data[(int) icode].operand[opno2].mode))
7361 return false;
7362
7363 /* Check the requirements for specific types. */
7364 switch (op1->type)
7365 {
7366 case EXPAND_OUTPUT:
7367 /* Outputs must remain distinct. */
7368 return false;
7369
7370 case EXPAND_FIXED:
7371 case EXPAND_INPUT:
7372 case EXPAND_ADDRESS:
7373 case EXPAND_INTEGER:
7374 return true;
7375
7376 case EXPAND_CONVERT_TO:
7377 case EXPAND_CONVERT_FROM:
7378 return op1->unsigned_p == op2->unsigned_p;
7379 }
7380 gcc_unreachable ();
7381 }
7382
7383 /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
7384 of instruction ICODE. Return true on success, leaving the new operand
7385 values in the OPS themselves. Emit no code on failure. */
7386
7387 bool
7388 maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
7389 unsigned int nops, class expand_operand *ops)
7390 {
7391 rtx_insn *last = get_last_insn ();
7392 rtx *orig_values = XALLOCAVEC (rtx, nops);
7393 for (unsigned int i = 0; i < nops; i++)
7394 {
7395 orig_values[i] = ops[i].value;
7396
7397 /* First try reusing the result of an earlier legitimization.
7398 This avoids duplicate rtl and ensures that tied operands
7399 remain tied.
7400
7401 This search is linear, but NOPS is bounded at compile time
7402 to a small number (current a single digit). */
7403 unsigned int j = 0;
7404 for (; j < i; ++j)
7405 if (can_reuse_operands_p (icode, opno + j, opno + i, &ops[j], &ops[i])
7406 && rtx_equal_p (orig_values[j], orig_values[i])
7407 && ops[j].value
7408 && insn_operand_matches (icode, opno + i, ops[j].value))
7409 {
7410 ops[i].value = copy_rtx (ops[j].value);
7411 break;
7412 }
7413
7414 /* Otherwise try legitimizing the operand on its own. */
7415 if (j == i && !maybe_legitimize_operand (icode, opno + i, &ops[i]))
7416 {
7417 delete_insns_since (last);
7418 return false;
7419 }
7420 }
7421 return true;
7422 }
7423
7424 /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
7425 as its operands. Return the instruction pattern on success,
7426 and emit any necessary set-up code. Return null and emit no
7427 code on failure. */
7428
7429 rtx_insn *
7430 maybe_gen_insn (enum insn_code icode, unsigned int nops,
7431 class expand_operand *ops)
7432 {
7433 gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
7434 if (!maybe_legitimize_operands (icode, 0, nops, ops))
7435 return NULL;
7436
7437 switch (nops)
7438 {
7439 case 1:
7440 return GEN_FCN (icode) (ops[0].value);
7441 case 2:
7442 return GEN_FCN (icode) (ops[0].value, ops[1].value);
7443 case 3:
7444 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
7445 case 4:
7446 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7447 ops[3].value);
7448 case 5:
7449 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7450 ops[3].value, ops[4].value);
7451 case 6:
7452 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7453 ops[3].value, ops[4].value, ops[5].value);
7454 case 7:
7455 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7456 ops[3].value, ops[4].value, ops[5].value,
7457 ops[6].value);
7458 case 8:
7459 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7460 ops[3].value, ops[4].value, ops[5].value,
7461 ops[6].value, ops[7].value);
7462 case 9:
7463 return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
7464 ops[3].value, ops[4].value, ops[5].value,
7465 ops[6].value, ops[7].value, ops[8].value);
7466 }
7467 gcc_unreachable ();
7468 }
7469
7470 /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
7471 as its operands. Return true on success and emit no code on failure. */
7472
7473 bool
7474 maybe_expand_insn (enum insn_code icode, unsigned int nops,
7475 class expand_operand *ops)
7476 {
7477 rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
7478 if (pat)
7479 {
7480 emit_insn (pat);
7481 return true;
7482 }
7483 return false;
7484 }
7485
7486 /* Like maybe_expand_insn, but for jumps. */
7487
7488 bool
7489 maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
7490 class expand_operand *ops)
7491 {
7492 rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
7493 if (pat)
7494 {
7495 emit_jump_insn (pat);
7496 return true;
7497 }
7498 return false;
7499 }
7500
7501 /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
7502 as its operands. */
7503
7504 void
7505 expand_insn (enum insn_code icode, unsigned int nops,
7506 class expand_operand *ops)
7507 {
7508 if (!maybe_expand_insn (icode, nops, ops))
7509 gcc_unreachable ();
7510 }
7511
7512 /* Like expand_insn, but for jumps. */
7513
7514 void
7515 expand_jump_insn (enum insn_code icode, unsigned int nops,
7516 class expand_operand *ops)
7517 {
7518 if (!maybe_expand_jump_insn (icode, nops, ops))
7519 gcc_unreachable ();
7520 }