1 /* Medium-level subroutines: convert bit-field store and extract
2 and shifts, multiplies and divides to rtl instructions.
3 Copyright (C) 1987-2015 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
26 #include "diagnostic-core.h"
31 #include "double-int.h"
38 #include "fold-const.h"
39 #include "stor-layout.h"
42 #include "insn-config.h"
44 #include "hard-reg-set.h"
46 #include "statistics.h"
48 #include "fixed-value.h"
57 #include "insn-codes.h"
60 #include "langhooks.h"
62 #include "basic-block.h"
66 struct target_expmed default_target_expmed
;
68 struct target_expmed
*this_target_expmed
= &default_target_expmed
;
71 static void store_fixed_bit_field (rtx
, unsigned HOST_WIDE_INT
,
72 unsigned HOST_WIDE_INT
,
73 unsigned HOST_WIDE_INT
,
74 unsigned HOST_WIDE_INT
,
76 static void store_fixed_bit_field_1 (rtx
, unsigned HOST_WIDE_INT
,
77 unsigned HOST_WIDE_INT
,
79 static void store_split_bit_field (rtx
, unsigned HOST_WIDE_INT
,
80 unsigned HOST_WIDE_INT
,
81 unsigned HOST_WIDE_INT
,
82 unsigned HOST_WIDE_INT
,
84 static rtx
extract_fixed_bit_field (machine_mode
, rtx
,
85 unsigned HOST_WIDE_INT
,
86 unsigned HOST_WIDE_INT
, rtx
, int);
87 static rtx
extract_fixed_bit_field_1 (machine_mode
, rtx
,
88 unsigned HOST_WIDE_INT
,
89 unsigned HOST_WIDE_INT
, rtx
, int);
90 static rtx
lshift_value (machine_mode
, unsigned HOST_WIDE_INT
, int);
91 static rtx
extract_split_bit_field (rtx
, unsigned HOST_WIDE_INT
,
92 unsigned HOST_WIDE_INT
, int);
93 static void do_cmp_and_jump (rtx
, rtx
, enum rtx_code
, machine_mode
, rtx_code_label
*);
94 static rtx
expand_smod_pow2 (machine_mode
, rtx
, HOST_WIDE_INT
);
95 static rtx
expand_sdiv_pow2 (machine_mode
, rtx
, HOST_WIDE_INT
);
97 /* Return a constant integer mask value of mode MODE with BITSIZE ones
98 followed by BITPOS zeros, or the complement of that if COMPLEMENT.
99 The mask is truncated if necessary to the width of mode MODE. The
100 mask is zero-extended if BITSIZE+BITPOS is too small for MODE. */
103 mask_rtx (machine_mode mode
, int bitpos
, int bitsize
, bool complement
)
105 return immed_wide_int_const
106 (wi::shifted_mask (bitpos
, bitsize
, complement
,
107 GET_MODE_PRECISION (mode
)), mode
);
110 /* Test whether a value is zero of a power of two. */
111 #define EXACT_POWER_OF_2_OR_ZERO_P(x) \
112 (((x) & ((x) - (unsigned HOST_WIDE_INT) 1)) == 0)
114 struct init_expmed_rtl
135 rtx pow2
[MAX_BITS_PER_WORD
];
136 rtx cint
[MAX_BITS_PER_WORD
];
140 init_expmed_one_conv (struct init_expmed_rtl
*all
, machine_mode to_mode
,
141 machine_mode from_mode
, bool speed
)
143 int to_size
, from_size
;
146 to_size
= GET_MODE_PRECISION (to_mode
);
147 from_size
= GET_MODE_PRECISION (from_mode
);
149 /* Most partial integers have a precision less than the "full"
150 integer it requires for storage. In case one doesn't, for
151 comparison purposes here, reduce the bit size by one in that
153 if (GET_MODE_CLASS (to_mode
) == MODE_PARTIAL_INT
154 && exact_log2 (to_size
) != -1)
156 if (GET_MODE_CLASS (from_mode
) == MODE_PARTIAL_INT
157 && exact_log2 (from_size
) != -1)
160 /* Assume cost of zero-extend and sign-extend is the same. */
161 which
= (to_size
< from_size
? all
->trunc
: all
->zext
);
163 PUT_MODE (all
->reg
, from_mode
);
164 set_convert_cost (to_mode
, from_mode
, speed
, set_src_cost (which
, speed
));
168 init_expmed_one_mode (struct init_expmed_rtl
*all
,
169 machine_mode mode
, int speed
)
171 int m
, n
, mode_bitsize
;
172 machine_mode mode_from
;
174 mode_bitsize
= GET_MODE_UNIT_BITSIZE (mode
);
176 PUT_MODE (all
->reg
, mode
);
177 PUT_MODE (all
->plus
, mode
);
178 PUT_MODE (all
->neg
, mode
);
179 PUT_MODE (all
->mult
, mode
);
180 PUT_MODE (all
->sdiv
, mode
);
181 PUT_MODE (all
->udiv
, mode
);
182 PUT_MODE (all
->sdiv_32
, mode
);
183 PUT_MODE (all
->smod_32
, mode
);
184 PUT_MODE (all
->wide_trunc
, mode
);
185 PUT_MODE (all
->shift
, mode
);
186 PUT_MODE (all
->shift_mult
, mode
);
187 PUT_MODE (all
->shift_add
, mode
);
188 PUT_MODE (all
->shift_sub0
, mode
);
189 PUT_MODE (all
->shift_sub1
, mode
);
190 PUT_MODE (all
->zext
, mode
);
191 PUT_MODE (all
->trunc
, mode
);
193 set_add_cost (speed
, mode
, set_src_cost (all
->plus
, speed
));
194 set_neg_cost (speed
, mode
, set_src_cost (all
->neg
, speed
));
195 set_mul_cost (speed
, mode
, set_src_cost (all
->mult
, speed
));
196 set_sdiv_cost (speed
, mode
, set_src_cost (all
->sdiv
, speed
));
197 set_udiv_cost (speed
, mode
, set_src_cost (all
->udiv
, speed
));
199 set_sdiv_pow2_cheap (speed
, mode
, (set_src_cost (all
->sdiv_32
, speed
)
200 <= 2 * add_cost (speed
, mode
)));
201 set_smod_pow2_cheap (speed
, mode
, (set_src_cost (all
->smod_32
, speed
)
202 <= 4 * add_cost (speed
, mode
)));
204 set_shift_cost (speed
, mode
, 0, 0);
206 int cost
= add_cost (speed
, mode
);
207 set_shiftadd_cost (speed
, mode
, 0, cost
);
208 set_shiftsub0_cost (speed
, mode
, 0, cost
);
209 set_shiftsub1_cost (speed
, mode
, 0, cost
);
212 n
= MIN (MAX_BITS_PER_WORD
, mode_bitsize
);
213 for (m
= 1; m
< n
; m
++)
215 XEXP (all
->shift
, 1) = all
->cint
[m
];
216 XEXP (all
->shift_mult
, 1) = all
->pow2
[m
];
218 set_shift_cost (speed
, mode
, m
, set_src_cost (all
->shift
, speed
));
219 set_shiftadd_cost (speed
, mode
, m
, set_src_cost (all
->shift_add
, speed
));
220 set_shiftsub0_cost (speed
, mode
, m
, set_src_cost (all
->shift_sub0
, speed
));
221 set_shiftsub1_cost (speed
, mode
, m
, set_src_cost (all
->shift_sub1
, speed
));
224 if (SCALAR_INT_MODE_P (mode
))
226 for (mode_from
= MIN_MODE_INT
; mode_from
<= MAX_MODE_INT
;
227 mode_from
= (machine_mode
)(mode_from
+ 1))
228 init_expmed_one_conv (all
, mode
, mode_from
, speed
);
230 if (GET_MODE_CLASS (mode
) == MODE_INT
)
232 machine_mode wider_mode
= GET_MODE_WIDER_MODE (mode
);
233 if (wider_mode
!= VOIDmode
)
235 PUT_MODE (all
->zext
, wider_mode
);
236 PUT_MODE (all
->wide_mult
, wider_mode
);
237 PUT_MODE (all
->wide_lshr
, wider_mode
);
238 XEXP (all
->wide_lshr
, 1) = GEN_INT (mode_bitsize
);
240 set_mul_widen_cost (speed
, wider_mode
,
241 set_src_cost (all
->wide_mult
, speed
));
242 set_mul_highpart_cost (speed
, mode
,
243 set_src_cost (all
->wide_trunc
, speed
));
251 struct init_expmed_rtl all
;
252 machine_mode mode
= QImode
;
255 memset (&all
, 0, sizeof all
);
256 for (m
= 1; m
< MAX_BITS_PER_WORD
; m
++)
258 all
.pow2
[m
] = GEN_INT ((HOST_WIDE_INT
) 1 << m
);
259 all
.cint
[m
] = GEN_INT (m
);
262 /* Avoid using hard regs in ways which may be unsupported. */
263 all
.reg
= gen_rtx_raw_REG (mode
, LAST_VIRTUAL_REGISTER
+ 1);
264 all
.plus
= gen_rtx_PLUS (mode
, all
.reg
, all
.reg
);
265 all
.neg
= gen_rtx_NEG (mode
, all
.reg
);
266 all
.mult
= gen_rtx_MULT (mode
, all
.reg
, all
.reg
);
267 all
.sdiv
= gen_rtx_DIV (mode
, all
.reg
, all
.reg
);
268 all
.udiv
= gen_rtx_UDIV (mode
, all
.reg
, all
.reg
);
269 all
.sdiv_32
= gen_rtx_DIV (mode
, all
.reg
, all
.pow2
[5]);
270 all
.smod_32
= gen_rtx_MOD (mode
, all
.reg
, all
.pow2
[5]);
271 all
.zext
= gen_rtx_ZERO_EXTEND (mode
, all
.reg
);
272 all
.wide_mult
= gen_rtx_MULT (mode
, all
.zext
, all
.zext
);
273 all
.wide_lshr
= gen_rtx_LSHIFTRT (mode
, all
.wide_mult
, all
.reg
);
274 all
.wide_trunc
= gen_rtx_TRUNCATE (mode
, all
.wide_lshr
);
275 all
.shift
= gen_rtx_ASHIFT (mode
, all
.reg
, all
.reg
);
276 all
.shift_mult
= gen_rtx_MULT (mode
, all
.reg
, all
.reg
);
277 all
.shift_add
= gen_rtx_PLUS (mode
, all
.shift_mult
, all
.reg
);
278 all
.shift_sub0
= gen_rtx_MINUS (mode
, all
.shift_mult
, all
.reg
);
279 all
.shift_sub1
= gen_rtx_MINUS (mode
, all
.reg
, all
.shift_mult
);
280 all
.trunc
= gen_rtx_TRUNCATE (mode
, all
.reg
);
282 for (speed
= 0; speed
< 2; speed
++)
284 crtl
->maybe_hot_insn_p
= speed
;
285 set_zero_cost (speed
, set_src_cost (const0_rtx
, speed
));
287 for (mode
= MIN_MODE_INT
; mode
<= MAX_MODE_INT
;
288 mode
= (machine_mode
)(mode
+ 1))
289 init_expmed_one_mode (&all
, mode
, speed
);
291 if (MIN_MODE_PARTIAL_INT
!= VOIDmode
)
292 for (mode
= MIN_MODE_PARTIAL_INT
; mode
<= MAX_MODE_PARTIAL_INT
;
293 mode
= (machine_mode
)(mode
+ 1))
294 init_expmed_one_mode (&all
, mode
, speed
);
296 if (MIN_MODE_VECTOR_INT
!= VOIDmode
)
297 for (mode
= MIN_MODE_VECTOR_INT
; mode
<= MAX_MODE_VECTOR_INT
;
298 mode
= (machine_mode
)(mode
+ 1))
299 init_expmed_one_mode (&all
, mode
, speed
);
302 if (alg_hash_used_p ())
304 struct alg_hash_entry
*p
= alg_hash_entry_ptr (0);
305 memset (p
, 0, sizeof (*p
) * NUM_ALG_HASH_ENTRIES
);
308 set_alg_hash_used_p (true);
309 default_rtl_profile ();
311 ggc_free (all
.trunc
);
312 ggc_free (all
.shift_sub1
);
313 ggc_free (all
.shift_sub0
);
314 ggc_free (all
.shift_add
);
315 ggc_free (all
.shift_mult
);
316 ggc_free (all
.shift
);
317 ggc_free (all
.wide_trunc
);
318 ggc_free (all
.wide_lshr
);
319 ggc_free (all
.wide_mult
);
321 ggc_free (all
.smod_32
);
322 ggc_free (all
.sdiv_32
);
331 /* Return an rtx representing minus the value of X.
332 MODE is the intended mode of the result,
333 useful if X is a CONST_INT. */
336 negate_rtx (machine_mode mode
, rtx x
)
338 rtx result
= simplify_unary_operation (NEG
, mode
, x
, mode
);
341 result
= expand_unop (mode
, neg_optab
, x
, NULL_RTX
, 0);
346 /* Adjust bitfield memory MEM so that it points to the first unit of mode
347 MODE that contains a bitfield of size BITSIZE at bit position BITNUM.
348 If MODE is BLKmode, return a reference to every byte in the bitfield.
349 Set *NEW_BITNUM to the bit position of the field within the new memory. */
352 narrow_bit_field_mem (rtx mem
, machine_mode mode
,
353 unsigned HOST_WIDE_INT bitsize
,
354 unsigned HOST_WIDE_INT bitnum
,
355 unsigned HOST_WIDE_INT
*new_bitnum
)
359 *new_bitnum
= bitnum
% BITS_PER_UNIT
;
360 HOST_WIDE_INT offset
= bitnum
/ BITS_PER_UNIT
;
361 HOST_WIDE_INT size
= ((*new_bitnum
+ bitsize
+ BITS_PER_UNIT
- 1)
363 return adjust_bitfield_address_size (mem
, mode
, offset
, size
);
367 unsigned int unit
= GET_MODE_BITSIZE (mode
);
368 *new_bitnum
= bitnum
% unit
;
369 HOST_WIDE_INT offset
= (bitnum
- *new_bitnum
) / BITS_PER_UNIT
;
370 return adjust_bitfield_address (mem
, mode
, offset
);
374 /* The caller wants to perform insertion or extraction PATTERN on a
375 bitfield of size BITSIZE at BITNUM bits into memory operand OP0.
376 BITREGION_START and BITREGION_END are as for store_bit_field
377 and FIELDMODE is the natural mode of the field.
379 Search for a mode that is compatible with the memory access
380 restrictions and (where applicable) with a register insertion or
381 extraction. Return the new memory on success, storing the adjusted
382 bit position in *NEW_BITNUM. Return null otherwise. */
385 adjust_bit_field_mem_for_reg (enum extraction_pattern pattern
,
386 rtx op0
, HOST_WIDE_INT bitsize
,
387 HOST_WIDE_INT bitnum
,
388 unsigned HOST_WIDE_INT bitregion_start
,
389 unsigned HOST_WIDE_INT bitregion_end
,
390 machine_mode fieldmode
,
391 unsigned HOST_WIDE_INT
*new_bitnum
)
393 bit_field_mode_iterator
iter (bitsize
, bitnum
, bitregion_start
,
394 bitregion_end
, MEM_ALIGN (op0
),
395 MEM_VOLATILE_P (op0
));
396 machine_mode best_mode
;
397 if (iter
.next_mode (&best_mode
))
399 /* We can use a memory in BEST_MODE. See whether this is true for
400 any wider modes. All other things being equal, we prefer to
401 use the widest mode possible because it tends to expose more
402 CSE opportunities. */
403 if (!iter
.prefer_smaller_modes ())
405 /* Limit the search to the mode required by the corresponding
406 register insertion or extraction instruction, if any. */
407 machine_mode limit_mode
= word_mode
;
408 extraction_insn insn
;
409 if (get_best_reg_extraction_insn (&insn
, pattern
,
410 GET_MODE_BITSIZE (best_mode
),
412 limit_mode
= insn
.field_mode
;
414 machine_mode wider_mode
;
415 while (iter
.next_mode (&wider_mode
)
416 && GET_MODE_SIZE (wider_mode
) <= GET_MODE_SIZE (limit_mode
))
417 best_mode
= wider_mode
;
419 return narrow_bit_field_mem (op0
, best_mode
, bitsize
, bitnum
,
425 /* Return true if a bitfield of size BITSIZE at bit number BITNUM within
426 a structure of mode STRUCT_MODE represents a lowpart subreg. The subreg
427 offset is then BITNUM / BITS_PER_UNIT. */
430 lowpart_bit_field_p (unsigned HOST_WIDE_INT bitnum
,
431 unsigned HOST_WIDE_INT bitsize
,
432 machine_mode struct_mode
)
434 if (BYTES_BIG_ENDIAN
)
435 return (bitnum
% BITS_PER_UNIT
== 0
436 && (bitnum
+ bitsize
== GET_MODE_BITSIZE (struct_mode
)
437 || (bitnum
+ bitsize
) % BITS_PER_WORD
== 0));
439 return bitnum
% BITS_PER_WORD
== 0;
442 /* Return true if -fstrict-volatile-bitfields applies to an access of OP0
443 containing BITSIZE bits starting at BITNUM, with field mode FIELDMODE.
444 Return false if the access would touch memory outside the range
445 BITREGION_START to BITREGION_END for conformance to the C++ memory
449 strict_volatile_bitfield_p (rtx op0
, unsigned HOST_WIDE_INT bitsize
,
450 unsigned HOST_WIDE_INT bitnum
,
451 machine_mode fieldmode
,
452 unsigned HOST_WIDE_INT bitregion_start
,
453 unsigned HOST_WIDE_INT bitregion_end
)
455 unsigned HOST_WIDE_INT modesize
= GET_MODE_BITSIZE (fieldmode
);
457 /* -fstrict-volatile-bitfields must be enabled and we must have a
460 || !MEM_VOLATILE_P (op0
)
461 || flag_strict_volatile_bitfields
<= 0)
464 /* Non-integral modes likely only happen with packed structures.
466 if (!SCALAR_INT_MODE_P (fieldmode
))
469 /* The bit size must not be larger than the field mode, and
470 the field mode must not be larger than a word. */
471 if (bitsize
> modesize
|| modesize
> BITS_PER_WORD
)
474 /* Check for cases of unaligned fields that must be split. */
475 if (bitnum
% modesize
+ bitsize
> modesize
)
478 /* The memory must be sufficiently aligned for a MODESIZE access.
479 This condition guarantees, that the memory access will not
480 touch anything after the end of the structure. */
481 if (MEM_ALIGN (op0
) < modesize
)
484 /* Check for cases where the C++ memory model applies. */
485 if (bitregion_end
!= 0
486 && (bitnum
- bitnum
% modesize
< bitregion_start
487 || bitnum
- bitnum
% modesize
+ modesize
- 1 > bitregion_end
))
493 /* Return true if OP is a memory and if a bitfield of size BITSIZE at
494 bit number BITNUM can be treated as a simple value of mode MODE. */
497 simple_mem_bitfield_p (rtx op0
, unsigned HOST_WIDE_INT bitsize
,
498 unsigned HOST_WIDE_INT bitnum
, machine_mode mode
)
501 && bitnum
% BITS_PER_UNIT
== 0
502 && bitsize
== GET_MODE_BITSIZE (mode
)
503 && (!SLOW_UNALIGNED_ACCESS (mode
, MEM_ALIGN (op0
))
504 || (bitnum
% GET_MODE_ALIGNMENT (mode
) == 0
505 && MEM_ALIGN (op0
) >= GET_MODE_ALIGNMENT (mode
))));
508 /* Try to use instruction INSV to store VALUE into a field of OP0.
509 BITSIZE and BITNUM are as for store_bit_field. */
512 store_bit_field_using_insv (const extraction_insn
*insv
, rtx op0
,
513 unsigned HOST_WIDE_INT bitsize
,
514 unsigned HOST_WIDE_INT bitnum
,
517 struct expand_operand ops
[4];
520 rtx_insn
*last
= get_last_insn ();
521 bool copy_back
= false;
523 machine_mode op_mode
= insv
->field_mode
;
524 unsigned int unit
= GET_MODE_BITSIZE (op_mode
);
525 if (bitsize
== 0 || bitsize
> unit
)
529 /* Get a reference to the first byte of the field. */
530 xop0
= narrow_bit_field_mem (xop0
, insv
->struct_mode
, bitsize
, bitnum
,
534 /* Convert from counting within OP0 to counting in OP_MODE. */
535 if (BYTES_BIG_ENDIAN
)
536 bitnum
+= unit
- GET_MODE_BITSIZE (GET_MODE (op0
));
538 /* If xop0 is a register, we need it in OP_MODE
539 to make it acceptable to the format of insv. */
540 if (GET_CODE (xop0
) == SUBREG
)
541 /* We can't just change the mode, because this might clobber op0,
542 and we will need the original value of op0 if insv fails. */
543 xop0
= gen_rtx_SUBREG (op_mode
, SUBREG_REG (xop0
), SUBREG_BYTE (xop0
));
544 if (REG_P (xop0
) && GET_MODE (xop0
) != op_mode
)
545 xop0
= gen_lowpart_SUBREG (op_mode
, xop0
);
548 /* If the destination is a paradoxical subreg such that we need a
549 truncate to the inner mode, perform the insertion on a temporary and
550 truncate the result to the original destination. Note that we can't
551 just truncate the paradoxical subreg as (truncate:N (subreg:W (reg:N
552 X) 0)) is (reg:N X). */
553 if (GET_CODE (xop0
) == SUBREG
554 && REG_P (SUBREG_REG (xop0
))
555 && !TRULY_NOOP_TRUNCATION_MODES_P (GET_MODE (SUBREG_REG (xop0
)),
558 rtx tem
= gen_reg_rtx (op_mode
);
559 emit_move_insn (tem
, xop0
);
564 /* There are similar overflow check at the start of store_bit_field_1,
565 but that only check the situation where the field lies completely
566 outside the register, while there do have situation where the field
567 lies partialy in the register, we need to adjust bitsize for this
568 partial overflow situation. Without this fix, pr48335-2.c on big-endian
569 will broken on those arch support bit insert instruction, like arm, aarch64
571 if (bitsize
+ bitnum
> unit
&& bitnum
< unit
)
573 warning (OPT_Wextra
, "write of %wu-bit data outside the bound of "
574 "destination object, data truncated into %wu-bit",
575 bitsize
, unit
- bitnum
);
576 bitsize
= unit
- bitnum
;
579 /* If BITS_BIG_ENDIAN is zero on a BYTES_BIG_ENDIAN machine, we count
580 "backwards" from the size of the unit we are inserting into.
581 Otherwise, we count bits from the most significant on a
582 BYTES/BITS_BIG_ENDIAN machine. */
584 if (BITS_BIG_ENDIAN
!= BYTES_BIG_ENDIAN
)
585 bitnum
= unit
- bitsize
- bitnum
;
587 /* Convert VALUE to op_mode (which insv insn wants) in VALUE1. */
589 if (GET_MODE (value
) != op_mode
)
591 if (GET_MODE_BITSIZE (GET_MODE (value
)) >= bitsize
)
593 /* Optimization: Don't bother really extending VALUE
594 if it has all the bits we will actually use. However,
595 if we must narrow it, be sure we do it correctly. */
597 if (GET_MODE_SIZE (GET_MODE (value
)) < GET_MODE_SIZE (op_mode
))
601 tmp
= simplify_subreg (op_mode
, value1
, GET_MODE (value
), 0);
603 tmp
= simplify_gen_subreg (op_mode
,
604 force_reg (GET_MODE (value
),
606 GET_MODE (value
), 0);
610 value1
= gen_lowpart (op_mode
, value1
);
612 else if (CONST_INT_P (value
))
613 value1
= gen_int_mode (INTVAL (value
), op_mode
);
615 /* Parse phase is supposed to make VALUE's data type
616 match that of the component reference, which is a type
617 at least as wide as the field; so VALUE should have
618 a mode that corresponds to that type. */
619 gcc_assert (CONSTANT_P (value
));
622 create_fixed_operand (&ops
[0], xop0
);
623 create_integer_operand (&ops
[1], bitsize
);
624 create_integer_operand (&ops
[2], bitnum
);
625 create_input_operand (&ops
[3], value1
, op_mode
);
626 if (maybe_expand_insn (insv
->icode
, 4, ops
))
629 convert_move (op0
, xop0
, true);
632 delete_insns_since (last
);
636 /* A subroutine of store_bit_field, with the same arguments. Return true
637 if the operation could be implemented.
639 If FALLBACK_P is true, fall back to store_fixed_bit_field if we have
640 no other way of implementing the operation. If FALLBACK_P is false,
641 return false instead. */
644 store_bit_field_1 (rtx str_rtx
, unsigned HOST_WIDE_INT bitsize
,
645 unsigned HOST_WIDE_INT bitnum
,
646 unsigned HOST_WIDE_INT bitregion_start
,
647 unsigned HOST_WIDE_INT bitregion_end
,
648 machine_mode fieldmode
,
649 rtx value
, bool fallback_p
)
654 while (GET_CODE (op0
) == SUBREG
)
656 /* The following line once was done only if WORDS_BIG_ENDIAN,
657 but I think that is a mistake. WORDS_BIG_ENDIAN is
658 meaningful at a much higher level; when structures are copied
659 between memory and regs, the higher-numbered regs
660 always get higher addresses. */
661 int inner_mode_size
= GET_MODE_SIZE (GET_MODE (SUBREG_REG (op0
)));
662 int outer_mode_size
= GET_MODE_SIZE (GET_MODE (op0
));
665 /* Paradoxical subregs need special handling on big endian machines. */
666 if (SUBREG_BYTE (op0
) == 0 && inner_mode_size
< outer_mode_size
)
668 int difference
= inner_mode_size
- outer_mode_size
;
670 if (WORDS_BIG_ENDIAN
)
671 byte_offset
+= (difference
/ UNITS_PER_WORD
) * UNITS_PER_WORD
;
672 if (BYTES_BIG_ENDIAN
)
673 byte_offset
+= difference
% UNITS_PER_WORD
;
676 byte_offset
= SUBREG_BYTE (op0
);
678 bitnum
+= byte_offset
* BITS_PER_UNIT
;
679 op0
= SUBREG_REG (op0
);
682 /* No action is needed if the target is a register and if the field
683 lies completely outside that register. This can occur if the source
684 code contains an out-of-bounds access to a small array. */
685 if (REG_P (op0
) && bitnum
>= GET_MODE_BITSIZE (GET_MODE (op0
)))
688 /* Use vec_set patterns for inserting parts of vectors whenever
690 if (VECTOR_MODE_P (GET_MODE (op0
))
692 && optab_handler (vec_set_optab
, GET_MODE (op0
)) != CODE_FOR_nothing
693 && fieldmode
== GET_MODE_INNER (GET_MODE (op0
))
694 && bitsize
== GET_MODE_BITSIZE (GET_MODE_INNER (GET_MODE (op0
)))
695 && !(bitnum
% GET_MODE_BITSIZE (GET_MODE_INNER (GET_MODE (op0
)))))
697 struct expand_operand ops
[3];
698 machine_mode outermode
= GET_MODE (op0
);
699 machine_mode innermode
= GET_MODE_INNER (outermode
);
700 enum insn_code icode
= optab_handler (vec_set_optab
, outermode
);
701 int pos
= bitnum
/ GET_MODE_BITSIZE (innermode
);
703 create_fixed_operand (&ops
[0], op0
);
704 create_input_operand (&ops
[1], value
, innermode
);
705 create_integer_operand (&ops
[2], pos
);
706 if (maybe_expand_insn (icode
, 3, ops
))
710 /* If the target is a register, overwriting the entire object, or storing
711 a full-word or multi-word field can be done with just a SUBREG. */
713 && bitsize
== GET_MODE_BITSIZE (fieldmode
)
714 && ((bitsize
== GET_MODE_BITSIZE (GET_MODE (op0
)) && bitnum
== 0)
715 || (bitsize
% BITS_PER_WORD
== 0 && bitnum
% BITS_PER_WORD
== 0)))
717 /* Use the subreg machinery either to narrow OP0 to the required
718 words or to cope with mode punning between equal-sized modes.
719 In the latter case, use subreg on the rhs side, not lhs. */
722 if (bitsize
== GET_MODE_BITSIZE (GET_MODE (op0
)))
724 sub
= simplify_gen_subreg (GET_MODE (op0
), value
, fieldmode
, 0);
727 emit_move_insn (op0
, sub
);
733 sub
= simplify_gen_subreg (fieldmode
, op0
, GET_MODE (op0
),
734 bitnum
/ BITS_PER_UNIT
);
737 emit_move_insn (sub
, value
);
743 /* If the target is memory, storing any naturally aligned field can be
744 done with a simple store. For targets that support fast unaligned
745 memory, any naturally sized, unit aligned field can be done directly. */
746 if (simple_mem_bitfield_p (op0
, bitsize
, bitnum
, fieldmode
))
748 op0
= adjust_bitfield_address (op0
, fieldmode
, bitnum
/ BITS_PER_UNIT
);
749 emit_move_insn (op0
, value
);
753 /* Make sure we are playing with integral modes. Pun with subregs
754 if we aren't. This must come after the entire register case above,
755 since that case is valid for any mode. The following cases are only
756 valid for integral modes. */
758 machine_mode imode
= int_mode_for_mode (GET_MODE (op0
));
759 if (imode
!= GET_MODE (op0
))
762 op0
= adjust_bitfield_address_size (op0
, imode
, 0, MEM_SIZE (op0
));
765 gcc_assert (imode
!= BLKmode
);
766 op0
= gen_lowpart (imode
, op0
);
771 /* Storing an lsb-aligned field in a register
772 can be done with a movstrict instruction. */
775 && lowpart_bit_field_p (bitnum
, bitsize
, GET_MODE (op0
))
776 && bitsize
== GET_MODE_BITSIZE (fieldmode
)
777 && optab_handler (movstrict_optab
, fieldmode
) != CODE_FOR_nothing
)
779 struct expand_operand ops
[2];
780 enum insn_code icode
= optab_handler (movstrict_optab
, fieldmode
);
782 unsigned HOST_WIDE_INT subreg_off
;
784 if (GET_CODE (arg0
) == SUBREG
)
786 /* Else we've got some float mode source being extracted into
787 a different float mode destination -- this combination of
788 subregs results in Severe Tire Damage. */
789 gcc_assert (GET_MODE (SUBREG_REG (arg0
)) == fieldmode
790 || GET_MODE_CLASS (fieldmode
) == MODE_INT
791 || GET_MODE_CLASS (fieldmode
) == MODE_PARTIAL_INT
);
792 arg0
= SUBREG_REG (arg0
);
795 subreg_off
= bitnum
/ BITS_PER_UNIT
;
796 if (validate_subreg (fieldmode
, GET_MODE (arg0
), arg0
, subreg_off
))
798 arg0
= gen_rtx_SUBREG (fieldmode
, arg0
, subreg_off
);
800 create_fixed_operand (&ops
[0], arg0
);
801 /* Shrink the source operand to FIELDMODE. */
802 create_convert_operand_to (&ops
[1], value
, fieldmode
, false);
803 if (maybe_expand_insn (icode
, 2, ops
))
808 /* Handle fields bigger than a word. */
810 if (bitsize
> BITS_PER_WORD
)
812 /* Here we transfer the words of the field
813 in the order least significant first.
814 This is because the most significant word is the one which may
816 However, only do that if the value is not BLKmode. */
818 unsigned int backwards
= WORDS_BIG_ENDIAN
&& fieldmode
!= BLKmode
;
819 unsigned int nwords
= (bitsize
+ (BITS_PER_WORD
- 1)) / BITS_PER_WORD
;
823 /* This is the mode we must force value to, so that there will be enough
824 subwords to extract. Note that fieldmode will often (always?) be
825 VOIDmode, because that is what store_field uses to indicate that this
826 is a bit field, but passing VOIDmode to operand_subword_force
828 fieldmode
= GET_MODE (value
);
829 if (fieldmode
== VOIDmode
)
830 fieldmode
= smallest_mode_for_size (nwords
* BITS_PER_WORD
, MODE_INT
);
832 last
= get_last_insn ();
833 for (i
= 0; i
< nwords
; i
++)
835 /* If I is 0, use the low-order word in both field and target;
836 if I is 1, use the next to lowest word; and so on. */
837 unsigned int wordnum
= (backwards
838 ? GET_MODE_SIZE (fieldmode
) / UNITS_PER_WORD
841 unsigned int bit_offset
= (backwards
842 ? MAX ((int) bitsize
- ((int) i
+ 1)
845 : (int) i
* BITS_PER_WORD
);
846 rtx value_word
= operand_subword_force (value
, wordnum
, fieldmode
);
847 unsigned HOST_WIDE_INT new_bitsize
=
848 MIN (BITS_PER_WORD
, bitsize
- i
* BITS_PER_WORD
);
850 /* If the remaining chunk doesn't have full wordsize we have
851 to make sure that for big endian machines the higher order
853 if (new_bitsize
< BITS_PER_WORD
&& BYTES_BIG_ENDIAN
&& !backwards
)
854 value_word
= simplify_expand_binop (word_mode
, lshr_optab
,
856 GEN_INT (BITS_PER_WORD
861 if (!store_bit_field_1 (op0
, new_bitsize
,
863 bitregion_start
, bitregion_end
,
865 value_word
, fallback_p
))
867 delete_insns_since (last
);
874 /* If VALUE has a floating-point or complex mode, access it as an
875 integer of the corresponding size. This can occur on a machine
876 with 64 bit registers that uses SFmode for float. It can also
877 occur for unaligned float or complex fields. */
879 if (GET_MODE (value
) != VOIDmode
880 && GET_MODE_CLASS (GET_MODE (value
)) != MODE_INT
881 && GET_MODE_CLASS (GET_MODE (value
)) != MODE_PARTIAL_INT
)
883 value
= gen_reg_rtx (int_mode_for_mode (GET_MODE (value
)));
884 emit_move_insn (gen_lowpart (GET_MODE (orig_value
), value
), orig_value
);
887 /* If OP0 is a multi-word register, narrow it to the affected word.
888 If the region spans two words, defer to store_split_bit_field. */
889 if (!MEM_P (op0
) && GET_MODE_SIZE (GET_MODE (op0
)) > UNITS_PER_WORD
)
891 op0
= simplify_gen_subreg (word_mode
, op0
, GET_MODE (op0
),
892 bitnum
/ BITS_PER_WORD
* UNITS_PER_WORD
);
894 bitnum
%= BITS_PER_WORD
;
895 if (bitnum
+ bitsize
> BITS_PER_WORD
)
900 store_split_bit_field (op0
, bitsize
, bitnum
, bitregion_start
,
901 bitregion_end
, value
);
906 /* From here on we can assume that the field to be stored in fits
907 within a word. If the destination is a register, it too fits
910 extraction_insn insv
;
912 && get_best_reg_extraction_insn (&insv
, EP_insv
,
913 GET_MODE_BITSIZE (GET_MODE (op0
)),
915 && store_bit_field_using_insv (&insv
, op0
, bitsize
, bitnum
, value
))
918 /* If OP0 is a memory, try copying it to a register and seeing if a
919 cheap register alternative is available. */
922 if (get_best_mem_extraction_insn (&insv
, EP_insv
, bitsize
, bitnum
,
924 && store_bit_field_using_insv (&insv
, op0
, bitsize
, bitnum
, value
))
927 rtx_insn
*last
= get_last_insn ();
929 /* Try loading part of OP0 into a register, inserting the bitfield
930 into that, and then copying the result back to OP0. */
931 unsigned HOST_WIDE_INT bitpos
;
932 rtx xop0
= adjust_bit_field_mem_for_reg (EP_insv
, op0
, bitsize
, bitnum
,
933 bitregion_start
, bitregion_end
,
937 rtx tempreg
= copy_to_reg (xop0
);
938 if (store_bit_field_1 (tempreg
, bitsize
, bitpos
,
939 bitregion_start
, bitregion_end
,
940 fieldmode
, orig_value
, false))
942 emit_move_insn (xop0
, tempreg
);
945 delete_insns_since (last
);
952 store_fixed_bit_field (op0
, bitsize
, bitnum
, bitregion_start
,
953 bitregion_end
, value
);
957 /* Generate code to store value from rtx VALUE
958 into a bit-field within structure STR_RTX
959 containing BITSIZE bits starting at bit BITNUM.
961 BITREGION_START is bitpos of the first bitfield in this region.
962 BITREGION_END is the bitpos of the ending bitfield in this region.
963 These two fields are 0, if the C++ memory model does not apply,
964 or we are not interested in keeping track of bitfield regions.
966 FIELDMODE is the machine-mode of the FIELD_DECL node for this field. */
969 store_bit_field (rtx str_rtx
, unsigned HOST_WIDE_INT bitsize
,
970 unsigned HOST_WIDE_INT bitnum
,
971 unsigned HOST_WIDE_INT bitregion_start
,
972 unsigned HOST_WIDE_INT bitregion_end
,
973 machine_mode fieldmode
,
976 /* Handle -fstrict-volatile-bitfields in the cases where it applies. */
977 if (strict_volatile_bitfield_p (str_rtx
, bitsize
, bitnum
, fieldmode
,
978 bitregion_start
, bitregion_end
))
980 /* Storing of a full word can be done with a simple store.
981 We know here that the field can be accessed with one single
982 instruction. For targets that support unaligned memory,
983 an unaligned access may be necessary. */
984 if (bitsize
== GET_MODE_BITSIZE (fieldmode
))
986 str_rtx
= adjust_bitfield_address (str_rtx
, fieldmode
,
987 bitnum
/ BITS_PER_UNIT
);
988 gcc_assert (bitnum
% BITS_PER_UNIT
== 0);
989 emit_move_insn (str_rtx
, value
);
995 str_rtx
= narrow_bit_field_mem (str_rtx
, fieldmode
, bitsize
, bitnum
,
997 gcc_assert (bitnum
+ bitsize
<= GET_MODE_BITSIZE (fieldmode
));
998 temp
= copy_to_reg (str_rtx
);
999 if (!store_bit_field_1 (temp
, bitsize
, bitnum
, 0, 0,
1000 fieldmode
, value
, true))
1003 emit_move_insn (str_rtx
, temp
);
1009 /* Under the C++0x memory model, we must not touch bits outside the
1010 bit region. Adjust the address to start at the beginning of the
1012 if (MEM_P (str_rtx
) && bitregion_start
> 0)
1014 machine_mode bestmode
;
1015 HOST_WIDE_INT offset
, size
;
1017 gcc_assert ((bitregion_start
% BITS_PER_UNIT
) == 0);
1019 offset
= bitregion_start
/ BITS_PER_UNIT
;
1020 bitnum
-= bitregion_start
;
1021 size
= (bitnum
+ bitsize
+ BITS_PER_UNIT
- 1) / BITS_PER_UNIT
;
1022 bitregion_end
-= bitregion_start
;
1023 bitregion_start
= 0;
1024 bestmode
= get_best_mode (bitsize
, bitnum
,
1025 bitregion_start
, bitregion_end
,
1026 MEM_ALIGN (str_rtx
), VOIDmode
,
1027 MEM_VOLATILE_P (str_rtx
));
1028 str_rtx
= adjust_bitfield_address_size (str_rtx
, bestmode
, offset
, size
);
1031 if (!store_bit_field_1 (str_rtx
, bitsize
, bitnum
,
1032 bitregion_start
, bitregion_end
,
1033 fieldmode
, value
, true))
1037 /* Use shifts and boolean operations to store VALUE into a bit field of
1038 width BITSIZE in OP0, starting at bit BITNUM. */
1041 store_fixed_bit_field (rtx op0
, unsigned HOST_WIDE_INT bitsize
,
1042 unsigned HOST_WIDE_INT bitnum
,
1043 unsigned HOST_WIDE_INT bitregion_start
,
1044 unsigned HOST_WIDE_INT bitregion_end
,
1047 /* There is a case not handled here:
1048 a structure with a known alignment of just a halfword
1049 and a field split across two aligned halfwords within the structure.
1050 Or likewise a structure with a known alignment of just a byte
1051 and a field split across two bytes.
1052 Such cases are not supposed to be able to occur. */
1056 machine_mode mode
= GET_MODE (op0
);
1057 if (GET_MODE_BITSIZE (mode
) == 0
1058 || GET_MODE_BITSIZE (mode
) > GET_MODE_BITSIZE (word_mode
))
1060 mode
= get_best_mode (bitsize
, bitnum
, bitregion_start
, bitregion_end
,
1061 MEM_ALIGN (op0
), mode
, MEM_VOLATILE_P (op0
));
1063 if (mode
== VOIDmode
)
1065 /* The only way this should occur is if the field spans word
1067 store_split_bit_field (op0
, bitsize
, bitnum
, bitregion_start
,
1068 bitregion_end
, value
);
1072 op0
= narrow_bit_field_mem (op0
, mode
, bitsize
, bitnum
, &bitnum
);
1075 store_fixed_bit_field_1 (op0
, bitsize
, bitnum
, value
);
1078 /* Helper function for store_fixed_bit_field, stores
1079 the bit field always using the MODE of OP0. */
1082 store_fixed_bit_field_1 (rtx op0
, unsigned HOST_WIDE_INT bitsize
,
1083 unsigned HOST_WIDE_INT bitnum
,
1091 mode
= GET_MODE (op0
);
1092 gcc_assert (SCALAR_INT_MODE_P (mode
));
1094 /* Note that bitsize + bitnum can be greater than GET_MODE_BITSIZE (mode)
1095 for invalid input, such as f5 from gcc.dg/pr48335-2.c. */
1097 if (BYTES_BIG_ENDIAN
)
1098 /* BITNUM is the distance between our msb
1099 and that of the containing datum.
1100 Convert it to the distance from the lsb. */
1101 bitnum
= GET_MODE_BITSIZE (mode
) - bitsize
- bitnum
;
1103 /* Now BITNUM is always the distance between our lsb
1106 /* Shift VALUE left by BITNUM bits. If VALUE is not constant,
1107 we must first convert its mode to MODE. */
1109 if (CONST_INT_P (value
))
1111 unsigned HOST_WIDE_INT v
= UINTVAL (value
);
1113 if (bitsize
< HOST_BITS_PER_WIDE_INT
)
1114 v
&= ((unsigned HOST_WIDE_INT
) 1 << bitsize
) - 1;
1118 else if ((bitsize
< HOST_BITS_PER_WIDE_INT
1119 && v
== ((unsigned HOST_WIDE_INT
) 1 << bitsize
) - 1)
1120 || (bitsize
== HOST_BITS_PER_WIDE_INT
1121 && v
== (unsigned HOST_WIDE_INT
) -1))
1124 value
= lshift_value (mode
, v
, bitnum
);
1128 int must_and
= (GET_MODE_BITSIZE (GET_MODE (value
)) != bitsize
1129 && bitnum
+ bitsize
!= GET_MODE_BITSIZE (mode
));
1131 if (GET_MODE (value
) != mode
)
1132 value
= convert_to_mode (mode
, value
, 1);
1135 value
= expand_binop (mode
, and_optab
, value
,
1136 mask_rtx (mode
, 0, bitsize
, 0),
1137 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
1139 value
= expand_shift (LSHIFT_EXPR
, mode
, value
,
1140 bitnum
, NULL_RTX
, 1);
1143 /* Now clear the chosen bits in OP0,
1144 except that if VALUE is -1 we need not bother. */
1145 /* We keep the intermediates in registers to allow CSE to combine
1146 consecutive bitfield assignments. */
1148 temp
= force_reg (mode
, op0
);
1152 temp
= expand_binop (mode
, and_optab
, temp
,
1153 mask_rtx (mode
, bitnum
, bitsize
, 1),
1154 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
1155 temp
= force_reg (mode
, temp
);
1158 /* Now logical-or VALUE into OP0, unless it is zero. */
1162 temp
= expand_binop (mode
, ior_optab
, temp
, value
,
1163 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
1164 temp
= force_reg (mode
, temp
);
1169 op0
= copy_rtx (op0
);
1170 emit_move_insn (op0
, temp
);
1174 /* Store a bit field that is split across multiple accessible memory objects.
1176 OP0 is the REG, SUBREG or MEM rtx for the first of the objects.
1177 BITSIZE is the field width; BITPOS the position of its first bit
1179 VALUE is the value to store.
1181 This does not yet handle fields wider than BITS_PER_WORD. */
1184 store_split_bit_field (rtx op0
, unsigned HOST_WIDE_INT bitsize
,
1185 unsigned HOST_WIDE_INT bitpos
,
1186 unsigned HOST_WIDE_INT bitregion_start
,
1187 unsigned HOST_WIDE_INT bitregion_end
,
1191 unsigned int bitsdone
= 0;
1193 /* Make sure UNIT isn't larger than BITS_PER_WORD, we can only handle that
1195 if (REG_P (op0
) || GET_CODE (op0
) == SUBREG
)
1196 unit
= BITS_PER_WORD
;
1198 unit
= MIN (MEM_ALIGN (op0
), BITS_PER_WORD
);
1200 /* If OP0 is a memory with a mode, then UNIT must not be larger than
1201 OP0's mode as well. Otherwise, store_fixed_bit_field will call us
1202 again, and we will mutually recurse forever. */
1203 if (MEM_P (op0
) && GET_MODE_BITSIZE (GET_MODE (op0
)) > 0)
1204 unit
= MIN (unit
, GET_MODE_BITSIZE (GET_MODE (op0
)));
1206 /* If VALUE is a constant other than a CONST_INT, get it into a register in
1207 WORD_MODE. If we can do this using gen_lowpart_common, do so. Note
1208 that VALUE might be a floating-point constant. */
1209 if (CONSTANT_P (value
) && !CONST_INT_P (value
))
1211 rtx word
= gen_lowpart_common (word_mode
, value
);
1213 if (word
&& (value
!= word
))
1216 value
= gen_lowpart_common (word_mode
,
1217 force_reg (GET_MODE (value
) != VOIDmode
1219 : word_mode
, value
));
1222 while (bitsdone
< bitsize
)
1224 unsigned HOST_WIDE_INT thissize
;
1226 unsigned HOST_WIDE_INT thispos
;
1227 unsigned HOST_WIDE_INT offset
;
1229 offset
= (bitpos
+ bitsdone
) / unit
;
1230 thispos
= (bitpos
+ bitsdone
) % unit
;
1232 /* When region of bytes we can touch is restricted, decrease
1233 UNIT close to the end of the region as needed. If op0 is a REG
1234 or SUBREG of REG, don't do this, as there can't be data races
1235 on a register and we can expand shorter code in some cases. */
1237 && unit
> BITS_PER_UNIT
1238 && bitpos
+ bitsdone
- thispos
+ unit
> bitregion_end
+ 1
1240 && (GET_CODE (op0
) != SUBREG
|| !REG_P (SUBREG_REG (op0
))))
1246 /* THISSIZE must not overrun a word boundary. Otherwise,
1247 store_fixed_bit_field will call us again, and we will mutually
1249 thissize
= MIN (bitsize
- bitsdone
, BITS_PER_WORD
);
1250 thissize
= MIN (thissize
, unit
- thispos
);
1252 if (BYTES_BIG_ENDIAN
)
1254 /* Fetch successively less significant portions. */
1255 if (CONST_INT_P (value
))
1256 part
= GEN_INT (((unsigned HOST_WIDE_INT
) (INTVAL (value
))
1257 >> (bitsize
- bitsdone
- thissize
))
1258 & (((HOST_WIDE_INT
) 1 << thissize
) - 1));
1261 int total_bits
= GET_MODE_BITSIZE (GET_MODE (value
));
1262 /* The args are chosen so that the last part includes the
1263 lsb. Give extract_bit_field the value it needs (with
1264 endianness compensation) to fetch the piece we want. */
1265 part
= extract_fixed_bit_field (word_mode
, value
, thissize
,
1266 total_bits
- bitsize
+ bitsdone
,
1272 /* Fetch successively more significant portions. */
1273 if (CONST_INT_P (value
))
1274 part
= GEN_INT (((unsigned HOST_WIDE_INT
) (INTVAL (value
))
1276 & (((HOST_WIDE_INT
) 1 << thissize
) - 1));
1278 part
= extract_fixed_bit_field (word_mode
, value
, thissize
,
1279 bitsdone
, NULL_RTX
, 1);
1282 /* If OP0 is a register, then handle OFFSET here.
1284 When handling multiword bitfields, extract_bit_field may pass
1285 down a word_mode SUBREG of a larger REG for a bitfield that actually
1286 crosses a word boundary. Thus, for a SUBREG, we must find
1287 the current word starting from the base register. */
1288 if (GET_CODE (op0
) == SUBREG
)
1290 int word_offset
= (SUBREG_BYTE (op0
) / UNITS_PER_WORD
)
1291 + (offset
* unit
/ BITS_PER_WORD
);
1292 machine_mode sub_mode
= GET_MODE (SUBREG_REG (op0
));
1293 if (sub_mode
!= BLKmode
&& GET_MODE_SIZE (sub_mode
) < UNITS_PER_WORD
)
1294 word
= word_offset
? const0_rtx
: op0
;
1296 word
= operand_subword_force (SUBREG_REG (op0
), word_offset
,
1297 GET_MODE (SUBREG_REG (op0
)));
1298 offset
&= BITS_PER_WORD
/ unit
- 1;
1300 else if (REG_P (op0
))
1302 machine_mode op0_mode
= GET_MODE (op0
);
1303 if (op0_mode
!= BLKmode
&& GET_MODE_SIZE (op0_mode
) < UNITS_PER_WORD
)
1304 word
= offset
? const0_rtx
: op0
;
1306 word
= operand_subword_force (op0
, offset
* unit
/ BITS_PER_WORD
,
1308 offset
&= BITS_PER_WORD
/ unit
- 1;
1313 /* OFFSET is in UNITs, and UNIT is in bits. If WORD is const0_rtx,
1314 it is just an out-of-bounds access. Ignore it. */
1315 if (word
!= const0_rtx
)
1316 store_fixed_bit_field (word
, thissize
, offset
* unit
+ thispos
,
1317 bitregion_start
, bitregion_end
, part
);
1318 bitsdone
+= thissize
;
1322 /* A subroutine of extract_bit_field_1 that converts return value X
1323 to either MODE or TMODE. MODE, TMODE and UNSIGNEDP are arguments
1324 to extract_bit_field. */
1327 convert_extracted_bit_field (rtx x
, machine_mode mode
,
1328 machine_mode tmode
, bool unsignedp
)
1330 if (GET_MODE (x
) == tmode
|| GET_MODE (x
) == mode
)
1333 /* If the x mode is not a scalar integral, first convert to the
1334 integer mode of that size and then access it as a floating-point
1335 value via a SUBREG. */
1336 if (!SCALAR_INT_MODE_P (tmode
))
1340 smode
= mode_for_size (GET_MODE_BITSIZE (tmode
), MODE_INT
, 0);
1341 x
= convert_to_mode (smode
, x
, unsignedp
);
1342 x
= force_reg (smode
, x
);
1343 return gen_lowpart (tmode
, x
);
1346 return convert_to_mode (tmode
, x
, unsignedp
);
1349 /* Try to use an ext(z)v pattern to extract a field from OP0.
1350 Return the extracted value on success, otherwise return null.
1351 EXT_MODE is the mode of the extraction and the other arguments
1352 are as for extract_bit_field. */
1355 extract_bit_field_using_extv (const extraction_insn
*extv
, rtx op0
,
1356 unsigned HOST_WIDE_INT bitsize
,
1357 unsigned HOST_WIDE_INT bitnum
,
1358 int unsignedp
, rtx target
,
1359 machine_mode mode
, machine_mode tmode
)
1361 struct expand_operand ops
[4];
1362 rtx spec_target
= target
;
1363 rtx spec_target_subreg
= 0;
1364 machine_mode ext_mode
= extv
->field_mode
;
1365 unsigned unit
= GET_MODE_BITSIZE (ext_mode
);
1367 if (bitsize
== 0 || unit
< bitsize
)
1371 /* Get a reference to the first byte of the field. */
1372 op0
= narrow_bit_field_mem (op0
, extv
->struct_mode
, bitsize
, bitnum
,
1376 /* Convert from counting within OP0 to counting in EXT_MODE. */
1377 if (BYTES_BIG_ENDIAN
)
1378 bitnum
+= unit
- GET_MODE_BITSIZE (GET_MODE (op0
));
1380 /* If op0 is a register, we need it in EXT_MODE to make it
1381 acceptable to the format of ext(z)v. */
1382 if (GET_CODE (op0
) == SUBREG
&& GET_MODE (op0
) != ext_mode
)
1384 if (REG_P (op0
) && GET_MODE (op0
) != ext_mode
)
1385 op0
= gen_lowpart_SUBREG (ext_mode
, op0
);
1388 /* If BITS_BIG_ENDIAN is zero on a BYTES_BIG_ENDIAN machine, we count
1389 "backwards" from the size of the unit we are extracting from.
1390 Otherwise, we count bits from the most significant on a
1391 BYTES/BITS_BIG_ENDIAN machine. */
1393 if (BITS_BIG_ENDIAN
!= BYTES_BIG_ENDIAN
)
1394 bitnum
= unit
- bitsize
- bitnum
;
1397 target
= spec_target
= gen_reg_rtx (tmode
);
1399 if (GET_MODE (target
) != ext_mode
)
1401 /* Don't use LHS paradoxical subreg if explicit truncation is needed
1402 between the mode of the extraction (word_mode) and the target
1403 mode. Instead, create a temporary and use convert_move to set
1406 && TRULY_NOOP_TRUNCATION_MODES_P (GET_MODE (target
), ext_mode
))
1408 target
= gen_lowpart (ext_mode
, target
);
1409 if (GET_MODE_PRECISION (ext_mode
)
1410 > GET_MODE_PRECISION (GET_MODE (spec_target
)))
1411 spec_target_subreg
= target
;
1414 target
= gen_reg_rtx (ext_mode
);
1417 create_output_operand (&ops
[0], target
, ext_mode
);
1418 create_fixed_operand (&ops
[1], op0
);
1419 create_integer_operand (&ops
[2], bitsize
);
1420 create_integer_operand (&ops
[3], bitnum
);
1421 if (maybe_expand_insn (extv
->icode
, 4, ops
))
1423 target
= ops
[0].value
;
1424 if (target
== spec_target
)
1426 if (target
== spec_target_subreg
)
1428 return convert_extracted_bit_field (target
, mode
, tmode
, unsignedp
);
1433 /* A subroutine of extract_bit_field, with the same arguments.
1434 If FALLBACK_P is true, fall back to extract_fixed_bit_field
1435 if we can find no other means of implementing the operation.
1436 if FALLBACK_P is false, return NULL instead. */
1439 extract_bit_field_1 (rtx str_rtx
, unsigned HOST_WIDE_INT bitsize
,
1440 unsigned HOST_WIDE_INT bitnum
, int unsignedp
, rtx target
,
1441 machine_mode mode
, machine_mode tmode
,
1445 machine_mode int_mode
;
1448 if (tmode
== VOIDmode
)
1451 while (GET_CODE (op0
) == SUBREG
)
1453 bitnum
+= SUBREG_BYTE (op0
) * BITS_PER_UNIT
;
1454 op0
= SUBREG_REG (op0
);
1457 /* If we have an out-of-bounds access to a register, just return an
1458 uninitialized register of the required mode. This can occur if the
1459 source code contains an out-of-bounds access to a small array. */
1460 if (REG_P (op0
) && bitnum
>= GET_MODE_BITSIZE (GET_MODE (op0
)))
1461 return gen_reg_rtx (tmode
);
1464 && mode
== GET_MODE (op0
)
1466 && bitsize
== GET_MODE_BITSIZE (GET_MODE (op0
)))
1468 /* We're trying to extract a full register from itself. */
1472 /* See if we can get a better vector mode before extracting. */
1473 if (VECTOR_MODE_P (GET_MODE (op0
))
1475 && GET_MODE_INNER (GET_MODE (op0
)) != tmode
)
1477 machine_mode new_mode
;
1479 if (GET_MODE_CLASS (tmode
) == MODE_FLOAT
)
1480 new_mode
= MIN_MODE_VECTOR_FLOAT
;
1481 else if (GET_MODE_CLASS (tmode
) == MODE_FRACT
)
1482 new_mode
= MIN_MODE_VECTOR_FRACT
;
1483 else if (GET_MODE_CLASS (tmode
) == MODE_UFRACT
)
1484 new_mode
= MIN_MODE_VECTOR_UFRACT
;
1485 else if (GET_MODE_CLASS (tmode
) == MODE_ACCUM
)
1486 new_mode
= MIN_MODE_VECTOR_ACCUM
;
1487 else if (GET_MODE_CLASS (tmode
) == MODE_UACCUM
)
1488 new_mode
= MIN_MODE_VECTOR_UACCUM
;
1490 new_mode
= MIN_MODE_VECTOR_INT
;
1492 for (; new_mode
!= VOIDmode
; new_mode
= GET_MODE_WIDER_MODE (new_mode
))
1493 if (GET_MODE_SIZE (new_mode
) == GET_MODE_SIZE (GET_MODE (op0
))
1494 && targetm
.vector_mode_supported_p (new_mode
))
1496 if (new_mode
!= VOIDmode
)
1497 op0
= gen_lowpart (new_mode
, op0
);
1500 /* Use vec_extract patterns for extracting parts of vectors whenever
1502 if (VECTOR_MODE_P (GET_MODE (op0
))
1504 && optab_handler (vec_extract_optab
, GET_MODE (op0
)) != CODE_FOR_nothing
1505 && ((bitnum
+ bitsize
- 1) / GET_MODE_BITSIZE (GET_MODE_INNER (GET_MODE (op0
)))
1506 == bitnum
/ GET_MODE_BITSIZE (GET_MODE_INNER (GET_MODE (op0
)))))
1508 struct expand_operand ops
[3];
1509 machine_mode outermode
= GET_MODE (op0
);
1510 machine_mode innermode
= GET_MODE_INNER (outermode
);
1511 enum insn_code icode
= optab_handler (vec_extract_optab
, outermode
);
1512 unsigned HOST_WIDE_INT pos
= bitnum
/ GET_MODE_BITSIZE (innermode
);
1514 create_output_operand (&ops
[0], target
, innermode
);
1515 create_input_operand (&ops
[1], op0
, outermode
);
1516 create_integer_operand (&ops
[2], pos
);
1517 if (maybe_expand_insn (icode
, 3, ops
))
1519 target
= ops
[0].value
;
1520 if (GET_MODE (target
) != mode
)
1521 return gen_lowpart (tmode
, target
);
1526 /* Make sure we are playing with integral modes. Pun with subregs
1529 machine_mode imode
= int_mode_for_mode (GET_MODE (op0
));
1530 if (imode
!= GET_MODE (op0
))
1533 op0
= adjust_bitfield_address_size (op0
, imode
, 0, MEM_SIZE (op0
));
1534 else if (imode
!= BLKmode
)
1536 op0
= gen_lowpart (imode
, op0
);
1538 /* If we got a SUBREG, force it into a register since we
1539 aren't going to be able to do another SUBREG on it. */
1540 if (GET_CODE (op0
) == SUBREG
)
1541 op0
= force_reg (imode
, op0
);
1543 else if (REG_P (op0
))
1546 imode
= smallest_mode_for_size (GET_MODE_BITSIZE (GET_MODE (op0
)),
1548 reg
= gen_reg_rtx (imode
);
1549 subreg
= gen_lowpart_SUBREG (GET_MODE (op0
), reg
);
1550 emit_move_insn (subreg
, op0
);
1552 bitnum
+= SUBREG_BYTE (subreg
) * BITS_PER_UNIT
;
1556 HOST_WIDE_INT size
= GET_MODE_SIZE (GET_MODE (op0
));
1557 rtx mem
= assign_stack_temp (GET_MODE (op0
), size
);
1558 emit_move_insn (mem
, op0
);
1559 op0
= adjust_bitfield_address_size (mem
, BLKmode
, 0, size
);
1564 /* ??? We currently assume TARGET is at least as big as BITSIZE.
1565 If that's wrong, the solution is to test for it and set TARGET to 0
1568 /* Get the mode of the field to use for atomic access or subreg
1571 if (SCALAR_INT_MODE_P (tmode
))
1573 machine_mode try_mode
= mode_for_size (bitsize
,
1574 GET_MODE_CLASS (tmode
), 0);
1575 if (try_mode
!= BLKmode
)
1578 gcc_assert (mode1
!= BLKmode
);
1580 /* Extraction of a full MODE1 value can be done with a subreg as long
1581 as the least significant bit of the value is the least significant
1582 bit of either OP0 or a word of OP0. */
1584 && lowpart_bit_field_p (bitnum
, bitsize
, GET_MODE (op0
))
1585 && bitsize
== GET_MODE_BITSIZE (mode1
)
1586 && TRULY_NOOP_TRUNCATION_MODES_P (mode1
, GET_MODE (op0
)))
1588 rtx sub
= simplify_gen_subreg (mode1
, op0
, GET_MODE (op0
),
1589 bitnum
/ BITS_PER_UNIT
);
1591 return convert_extracted_bit_field (sub
, mode
, tmode
, unsignedp
);
1594 /* Extraction of a full MODE1 value can be done with a load as long as
1595 the field is on a byte boundary and is sufficiently aligned. */
1596 if (simple_mem_bitfield_p (op0
, bitsize
, bitnum
, mode1
))
1598 op0
= adjust_bitfield_address (op0
, mode1
, bitnum
/ BITS_PER_UNIT
);
1599 return convert_extracted_bit_field (op0
, mode
, tmode
, unsignedp
);
1602 /* Handle fields bigger than a word. */
1604 if (bitsize
> BITS_PER_WORD
)
1606 /* Here we transfer the words of the field
1607 in the order least significant first.
1608 This is because the most significant word is the one which may
1609 be less than full. */
1611 unsigned int backwards
= WORDS_BIG_ENDIAN
;
1612 unsigned int nwords
= (bitsize
+ (BITS_PER_WORD
- 1)) / BITS_PER_WORD
;
1616 if (target
== 0 || !REG_P (target
) || !valid_multiword_target_p (target
))
1617 target
= gen_reg_rtx (mode
);
1619 /* Indicate for flow that the entire target reg is being set. */
1620 emit_clobber (target
);
1622 last
= get_last_insn ();
1623 for (i
= 0; i
< nwords
; i
++)
1625 /* If I is 0, use the low-order word in both field and target;
1626 if I is 1, use the next to lowest word; and so on. */
1627 /* Word number in TARGET to use. */
1628 unsigned int wordnum
1630 ? GET_MODE_SIZE (GET_MODE (target
)) / UNITS_PER_WORD
- i
- 1
1632 /* Offset from start of field in OP0. */
1633 unsigned int bit_offset
= (backwards
1634 ? MAX ((int) bitsize
- ((int) i
+ 1)
1637 : (int) i
* BITS_PER_WORD
);
1638 rtx target_part
= operand_subword (target
, wordnum
, 1, VOIDmode
);
1640 = extract_bit_field_1 (op0
, MIN (BITS_PER_WORD
,
1641 bitsize
- i
* BITS_PER_WORD
),
1642 bitnum
+ bit_offset
, 1, target_part
,
1643 mode
, word_mode
, fallback_p
);
1645 gcc_assert (target_part
);
1648 delete_insns_since (last
);
1652 if (result_part
!= target_part
)
1653 emit_move_insn (target_part
, result_part
);
1658 /* Unless we've filled TARGET, the upper regs in a multi-reg value
1659 need to be zero'd out. */
1660 if (GET_MODE_SIZE (GET_MODE (target
)) > nwords
* UNITS_PER_WORD
)
1662 unsigned int i
, total_words
;
1664 total_words
= GET_MODE_SIZE (GET_MODE (target
)) / UNITS_PER_WORD
;
1665 for (i
= nwords
; i
< total_words
; i
++)
1667 (operand_subword (target
,
1668 backwards
? total_words
- i
- 1 : i
,
1675 /* Signed bit field: sign-extend with two arithmetic shifts. */
1676 target
= expand_shift (LSHIFT_EXPR
, mode
, target
,
1677 GET_MODE_BITSIZE (mode
) - bitsize
, NULL_RTX
, 0);
1678 return expand_shift (RSHIFT_EXPR
, mode
, target
,
1679 GET_MODE_BITSIZE (mode
) - bitsize
, NULL_RTX
, 0);
1682 /* If OP0 is a multi-word register, narrow it to the affected word.
1683 If the region spans two words, defer to extract_split_bit_field. */
1684 if (!MEM_P (op0
) && GET_MODE_SIZE (GET_MODE (op0
)) > UNITS_PER_WORD
)
1686 op0
= simplify_gen_subreg (word_mode
, op0
, GET_MODE (op0
),
1687 bitnum
/ BITS_PER_WORD
* UNITS_PER_WORD
);
1688 bitnum
%= BITS_PER_WORD
;
1689 if (bitnum
+ bitsize
> BITS_PER_WORD
)
1693 target
= extract_split_bit_field (op0
, bitsize
, bitnum
, unsignedp
);
1694 return convert_extracted_bit_field (target
, mode
, tmode
, unsignedp
);
1698 /* From here on we know the desired field is smaller than a word.
1699 If OP0 is a register, it too fits within a word. */
1700 enum extraction_pattern pattern
= unsignedp
? EP_extzv
: EP_extv
;
1701 extraction_insn extv
;
1703 /* ??? We could limit the structure size to the part of OP0 that
1704 contains the field, with appropriate checks for endianness
1705 and TRULY_NOOP_TRUNCATION. */
1706 && get_best_reg_extraction_insn (&extv
, pattern
,
1707 GET_MODE_BITSIZE (GET_MODE (op0
)),
1710 rtx result
= extract_bit_field_using_extv (&extv
, op0
, bitsize
, bitnum
,
1711 unsignedp
, target
, mode
,
1717 /* If OP0 is a memory, try copying it to a register and seeing if a
1718 cheap register alternative is available. */
1721 if (get_best_mem_extraction_insn (&extv
, pattern
, bitsize
, bitnum
,
1724 rtx result
= extract_bit_field_using_extv (&extv
, op0
, bitsize
,
1732 rtx_insn
*last
= get_last_insn ();
1734 /* Try loading part of OP0 into a register and extracting the
1735 bitfield from that. */
1736 unsigned HOST_WIDE_INT bitpos
;
1737 rtx xop0
= adjust_bit_field_mem_for_reg (pattern
, op0
, bitsize
, bitnum
,
1738 0, 0, tmode
, &bitpos
);
1741 xop0
= copy_to_reg (xop0
);
1742 rtx result
= extract_bit_field_1 (xop0
, bitsize
, bitpos
,
1744 mode
, tmode
, false);
1747 delete_insns_since (last
);
1754 /* Find a correspondingly-sized integer field, so we can apply
1755 shifts and masks to it. */
1756 int_mode
= int_mode_for_mode (tmode
);
1757 if (int_mode
== BLKmode
)
1758 int_mode
= int_mode_for_mode (mode
);
1759 /* Should probably push op0 out to memory and then do a load. */
1760 gcc_assert (int_mode
!= BLKmode
);
1762 target
= extract_fixed_bit_field (int_mode
, op0
, bitsize
, bitnum
,
1764 return convert_extracted_bit_field (target
, mode
, tmode
, unsignedp
);
1767 /* Generate code to extract a byte-field from STR_RTX
1768 containing BITSIZE bits, starting at BITNUM,
1769 and put it in TARGET if possible (if TARGET is nonzero).
1770 Regardless of TARGET, we return the rtx for where the value is placed.
1772 STR_RTX is the structure containing the byte (a REG or MEM).
1773 UNSIGNEDP is nonzero if this is an unsigned bit field.
1774 MODE is the natural mode of the field value once extracted.
1775 TMODE is the mode the caller would like the value to have;
1776 but the value may be returned with type MODE instead.
1778 If a TARGET is specified and we can store in it at no extra cost,
1779 we do so, and return TARGET.
1780 Otherwise, we return a REG of mode TMODE or MODE, with TMODE preferred
1781 if they are equally easy. */
1784 extract_bit_field (rtx str_rtx
, unsigned HOST_WIDE_INT bitsize
,
1785 unsigned HOST_WIDE_INT bitnum
, int unsignedp
, rtx target
,
1786 machine_mode mode
, machine_mode tmode
)
1790 /* Handle -fstrict-volatile-bitfields in the cases where it applies. */
1791 if (GET_MODE_BITSIZE (GET_MODE (str_rtx
)) > 0)
1792 mode1
= GET_MODE (str_rtx
);
1793 else if (target
&& GET_MODE_BITSIZE (GET_MODE (target
)) > 0)
1794 mode1
= GET_MODE (target
);
1798 if (strict_volatile_bitfield_p (str_rtx
, bitsize
, bitnum
, mode1
, 0, 0))
1800 /* Extraction of a full MODE1 value can be done with a simple load.
1801 We know here that the field can be accessed with one single
1802 instruction. For targets that support unaligned memory,
1803 an unaligned access may be necessary. */
1804 if (bitsize
== GET_MODE_BITSIZE (mode1
))
1806 rtx result
= adjust_bitfield_address (str_rtx
, mode1
,
1807 bitnum
/ BITS_PER_UNIT
);
1808 gcc_assert (bitnum
% BITS_PER_UNIT
== 0);
1809 return convert_extracted_bit_field (result
, mode
, tmode
, unsignedp
);
1812 str_rtx
= narrow_bit_field_mem (str_rtx
, mode1
, bitsize
, bitnum
,
1814 gcc_assert (bitnum
+ bitsize
<= GET_MODE_BITSIZE (mode1
));
1815 str_rtx
= copy_to_reg (str_rtx
);
1818 return extract_bit_field_1 (str_rtx
, bitsize
, bitnum
, unsignedp
,
1819 target
, mode
, tmode
, true);
1822 /* Use shifts and boolean operations to extract a field of BITSIZE bits
1823 from bit BITNUM of OP0.
1825 UNSIGNEDP is nonzero for an unsigned bit field (don't sign-extend value).
1826 If TARGET is nonzero, attempts to store the value there
1827 and return TARGET, but this is not guaranteed.
1828 If TARGET is not used, create a pseudo-reg of mode TMODE for the value. */
1831 extract_fixed_bit_field (machine_mode tmode
, rtx op0
,
1832 unsigned HOST_WIDE_INT bitsize
,
1833 unsigned HOST_WIDE_INT bitnum
, rtx target
,
1839 = get_best_mode (bitsize
, bitnum
, 0, 0, MEM_ALIGN (op0
), word_mode
,
1840 MEM_VOLATILE_P (op0
));
1842 if (mode
== VOIDmode
)
1843 /* The only way this should occur is if the field spans word
1845 return extract_split_bit_field (op0
, bitsize
, bitnum
, unsignedp
);
1847 op0
= narrow_bit_field_mem (op0
, mode
, bitsize
, bitnum
, &bitnum
);
1850 return extract_fixed_bit_field_1 (tmode
, op0
, bitsize
, bitnum
,
1854 /* Helper function for extract_fixed_bit_field, extracts
1855 the bit field always using the MODE of OP0. */
1858 extract_fixed_bit_field_1 (machine_mode tmode
, rtx op0
,
1859 unsigned HOST_WIDE_INT bitsize
,
1860 unsigned HOST_WIDE_INT bitnum
, rtx target
,
1863 machine_mode mode
= GET_MODE (op0
);
1864 gcc_assert (SCALAR_INT_MODE_P (mode
));
1866 /* Note that bitsize + bitnum can be greater than GET_MODE_BITSIZE (mode)
1867 for invalid input, such as extract equivalent of f5 from
1868 gcc.dg/pr48335-2.c. */
1870 if (BYTES_BIG_ENDIAN
)
1871 /* BITNUM is the distance between our msb and that of OP0.
1872 Convert it to the distance from the lsb. */
1873 bitnum
= GET_MODE_BITSIZE (mode
) - bitsize
- bitnum
;
1875 /* Now BITNUM is always the distance between the field's lsb and that of OP0.
1876 We have reduced the big-endian case to the little-endian case. */
1882 /* If the field does not already start at the lsb,
1883 shift it so it does. */
1884 /* Maybe propagate the target for the shift. */
1885 rtx subtarget
= (target
!= 0 && REG_P (target
) ? target
: 0);
1888 op0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, bitnum
, subtarget
, 1);
1890 /* Convert the value to the desired mode. */
1892 op0
= convert_to_mode (tmode
, op0
, 1);
1894 /* Unless the msb of the field used to be the msb when we shifted,
1895 mask out the upper bits. */
1897 if (GET_MODE_BITSIZE (mode
) != bitnum
+ bitsize
)
1898 return expand_binop (GET_MODE (op0
), and_optab
, op0
,
1899 mask_rtx (GET_MODE (op0
), 0, bitsize
, 0),
1900 target
, 1, OPTAB_LIB_WIDEN
);
1904 /* To extract a signed bit-field, first shift its msb to the msb of the word,
1905 then arithmetic-shift its lsb to the lsb of the word. */
1906 op0
= force_reg (mode
, op0
);
1908 /* Find the narrowest integer mode that contains the field. */
1910 for (mode
= GET_CLASS_NARROWEST_MODE (MODE_INT
); mode
!= VOIDmode
;
1911 mode
= GET_MODE_WIDER_MODE (mode
))
1912 if (GET_MODE_BITSIZE (mode
) >= bitsize
+ bitnum
)
1914 op0
= convert_to_mode (mode
, op0
, 0);
1921 if (GET_MODE_BITSIZE (mode
) != (bitsize
+ bitnum
))
1923 int amount
= GET_MODE_BITSIZE (mode
) - (bitsize
+ bitnum
);
1924 /* Maybe propagate the target for the shift. */
1925 rtx subtarget
= (target
!= 0 && REG_P (target
) ? target
: 0);
1926 op0
= expand_shift (LSHIFT_EXPR
, mode
, op0
, amount
, subtarget
, 1);
1929 return expand_shift (RSHIFT_EXPR
, mode
, op0
,
1930 GET_MODE_BITSIZE (mode
) - bitsize
, target
, 0);
1933 /* Return a constant integer (CONST_INT or CONST_DOUBLE) rtx with the value
1937 lshift_value (machine_mode mode
, unsigned HOST_WIDE_INT value
,
1940 return immed_wide_int_const (wi::lshift (value
, bitpos
), mode
);
1943 /* Extract a bit field that is split across two words
1944 and return an RTX for the result.
1946 OP0 is the REG, SUBREG or MEM rtx for the first of the two words.
1947 BITSIZE is the field width; BITPOS, position of its first bit, in the word.
1948 UNSIGNEDP is 1 if should zero-extend the contents; else sign-extend. */
1951 extract_split_bit_field (rtx op0
, unsigned HOST_WIDE_INT bitsize
,
1952 unsigned HOST_WIDE_INT bitpos
, int unsignedp
)
1955 unsigned int bitsdone
= 0;
1956 rtx result
= NULL_RTX
;
1959 /* Make sure UNIT isn't larger than BITS_PER_WORD, we can only handle that
1961 if (REG_P (op0
) || GET_CODE (op0
) == SUBREG
)
1962 unit
= BITS_PER_WORD
;
1964 unit
= MIN (MEM_ALIGN (op0
), BITS_PER_WORD
);
1966 while (bitsdone
< bitsize
)
1968 unsigned HOST_WIDE_INT thissize
;
1970 unsigned HOST_WIDE_INT thispos
;
1971 unsigned HOST_WIDE_INT offset
;
1973 offset
= (bitpos
+ bitsdone
) / unit
;
1974 thispos
= (bitpos
+ bitsdone
) % unit
;
1976 /* THISSIZE must not overrun a word boundary. Otherwise,
1977 extract_fixed_bit_field will call us again, and we will mutually
1979 thissize
= MIN (bitsize
- bitsdone
, BITS_PER_WORD
);
1980 thissize
= MIN (thissize
, unit
- thispos
);
1982 /* If OP0 is a register, then handle OFFSET here.
1984 When handling multiword bitfields, extract_bit_field may pass
1985 down a word_mode SUBREG of a larger REG for a bitfield that actually
1986 crosses a word boundary. Thus, for a SUBREG, we must find
1987 the current word starting from the base register. */
1988 if (GET_CODE (op0
) == SUBREG
)
1990 int word_offset
= (SUBREG_BYTE (op0
) / UNITS_PER_WORD
) + offset
;
1991 word
= operand_subword_force (SUBREG_REG (op0
), word_offset
,
1992 GET_MODE (SUBREG_REG (op0
)));
1995 else if (REG_P (op0
))
1997 word
= operand_subword_force (op0
, offset
, GET_MODE (op0
));
2003 /* Extract the parts in bit-counting order,
2004 whose meaning is determined by BYTES_PER_UNIT.
2005 OFFSET is in UNITs, and UNIT is in bits. */
2006 part
= extract_fixed_bit_field (word_mode
, word
, thissize
,
2007 offset
* unit
+ thispos
, 0, 1);
2008 bitsdone
+= thissize
;
2010 /* Shift this part into place for the result. */
2011 if (BYTES_BIG_ENDIAN
)
2013 if (bitsize
!= bitsdone
)
2014 part
= expand_shift (LSHIFT_EXPR
, word_mode
, part
,
2015 bitsize
- bitsdone
, 0, 1);
2019 if (bitsdone
!= thissize
)
2020 part
= expand_shift (LSHIFT_EXPR
, word_mode
, part
,
2021 bitsdone
- thissize
, 0, 1);
2027 /* Combine the parts with bitwise or. This works
2028 because we extracted each part as an unsigned bit field. */
2029 result
= expand_binop (word_mode
, ior_optab
, part
, result
, NULL_RTX
, 1,
2035 /* Unsigned bit field: we are done. */
2038 /* Signed bit field: sign-extend with two arithmetic shifts. */
2039 result
= expand_shift (LSHIFT_EXPR
, word_mode
, result
,
2040 BITS_PER_WORD
- bitsize
, NULL_RTX
, 0);
2041 return expand_shift (RSHIFT_EXPR
, word_mode
, result
,
2042 BITS_PER_WORD
- bitsize
, NULL_RTX
, 0);
2045 /* Try to read the low bits of SRC as an rvalue of mode MODE, preserving
2046 the bit pattern. SRC_MODE is the mode of SRC; if this is smaller than
2047 MODE, fill the upper bits with zeros. Fail if the layout of either
2048 mode is unknown (as for CC modes) or if the extraction would involve
2049 unprofitable mode punning. Return the value on success, otherwise
2052 This is different from gen_lowpart* in these respects:
2054 - the returned value must always be considered an rvalue
2056 - when MODE is wider than SRC_MODE, the extraction involves
2059 - when MODE is smaller than SRC_MODE, the extraction involves
2060 a truncation (and is thus subject to TRULY_NOOP_TRUNCATION).
2062 In other words, this routine performs a computation, whereas the
2063 gen_lowpart* routines are conceptually lvalue or rvalue subreg
2067 extract_low_bits (machine_mode mode
, machine_mode src_mode
, rtx src
)
2069 machine_mode int_mode
, src_int_mode
;
2071 if (mode
== src_mode
)
2074 if (CONSTANT_P (src
))
2076 /* simplify_gen_subreg can't be used here, as if simplify_subreg
2077 fails, it will happily create (subreg (symbol_ref)) or similar
2079 unsigned int byte
= subreg_lowpart_offset (mode
, src_mode
);
2080 rtx ret
= simplify_subreg (mode
, src
, src_mode
, byte
);
2084 if (GET_MODE (src
) == VOIDmode
2085 || !validate_subreg (mode
, src_mode
, src
, byte
))
2088 src
= force_reg (GET_MODE (src
), src
);
2089 return gen_rtx_SUBREG (mode
, src
, byte
);
2092 if (GET_MODE_CLASS (mode
) == MODE_CC
|| GET_MODE_CLASS (src_mode
) == MODE_CC
)
2095 if (GET_MODE_BITSIZE (mode
) == GET_MODE_BITSIZE (src_mode
)
2096 && MODES_TIEABLE_P (mode
, src_mode
))
2098 rtx x
= gen_lowpart_common (mode
, src
);
2103 src_int_mode
= int_mode_for_mode (src_mode
);
2104 int_mode
= int_mode_for_mode (mode
);
2105 if (src_int_mode
== BLKmode
|| int_mode
== BLKmode
)
2108 if (!MODES_TIEABLE_P (src_int_mode
, src_mode
))
2110 if (!MODES_TIEABLE_P (int_mode
, mode
))
2113 src
= gen_lowpart (src_int_mode
, src
);
2114 src
= convert_modes (int_mode
, src_int_mode
, src
, true);
2115 src
= gen_lowpart (mode
, src
);
2119 /* Add INC into TARGET. */
2122 expand_inc (rtx target
, rtx inc
)
2124 rtx value
= expand_binop (GET_MODE (target
), add_optab
,
2126 target
, 0, OPTAB_LIB_WIDEN
);
2127 if (value
!= target
)
2128 emit_move_insn (target
, value
);
2131 /* Subtract DEC from TARGET. */
2134 expand_dec (rtx target
, rtx dec
)
2136 rtx value
= expand_binop (GET_MODE (target
), sub_optab
,
2138 target
, 0, OPTAB_LIB_WIDEN
);
2139 if (value
!= target
)
2140 emit_move_insn (target
, value
);
2143 /* Output a shift instruction for expression code CODE,
2144 with SHIFTED being the rtx for the value to shift,
2145 and AMOUNT the rtx for the amount to shift by.
2146 Store the result in the rtx TARGET, if that is convenient.
2147 If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2148 Return the rtx for where the value is. */
2151 expand_shift_1 (enum tree_code code
, machine_mode mode
, rtx shifted
,
2152 rtx amount
, rtx target
, int unsignedp
)
2155 int left
= (code
== LSHIFT_EXPR
|| code
== LROTATE_EXPR
);
2156 int rotate
= (code
== LROTATE_EXPR
|| code
== RROTATE_EXPR
);
2157 optab lshift_optab
= ashl_optab
;
2158 optab rshift_arith_optab
= ashr_optab
;
2159 optab rshift_uns_optab
= lshr_optab
;
2160 optab lrotate_optab
= rotl_optab
;
2161 optab rrotate_optab
= rotr_optab
;
2162 machine_mode op1_mode
;
2163 machine_mode scalar_mode
= mode
;
2165 bool speed
= optimize_insn_for_speed_p ();
2167 if (VECTOR_MODE_P (mode
))
2168 scalar_mode
= GET_MODE_INNER (mode
);
2170 op1_mode
= GET_MODE (op1
);
2172 /* Determine whether the shift/rotate amount is a vector, or scalar. If the
2173 shift amount is a vector, use the vector/vector shift patterns. */
2174 if (VECTOR_MODE_P (mode
) && VECTOR_MODE_P (op1_mode
))
2176 lshift_optab
= vashl_optab
;
2177 rshift_arith_optab
= vashr_optab
;
2178 rshift_uns_optab
= vlshr_optab
;
2179 lrotate_optab
= vrotl_optab
;
2180 rrotate_optab
= vrotr_optab
;
2183 /* Previously detected shift-counts computed by NEGATE_EXPR
2184 and shifted in the other direction; but that does not work
2187 if (SHIFT_COUNT_TRUNCATED
)
2189 if (CONST_INT_P (op1
)
2190 && ((unsigned HOST_WIDE_INT
) INTVAL (op1
) >=
2191 (unsigned HOST_WIDE_INT
) GET_MODE_BITSIZE (scalar_mode
)))
2192 op1
= GEN_INT ((unsigned HOST_WIDE_INT
) INTVAL (op1
)
2193 % GET_MODE_BITSIZE (scalar_mode
));
2194 else if (GET_CODE (op1
) == SUBREG
2195 && subreg_lowpart_p (op1
)
2196 && SCALAR_INT_MODE_P (GET_MODE (SUBREG_REG (op1
)))
2197 && SCALAR_INT_MODE_P (GET_MODE (op1
)))
2198 op1
= SUBREG_REG (op1
);
2201 /* Canonicalize rotates by constant amount. If op1 is bitsize / 2,
2202 prefer left rotation, if op1 is from bitsize / 2 + 1 to
2203 bitsize - 1, use other direction of rotate with 1 .. bitsize / 2 - 1
2206 && CONST_INT_P (op1
)
2207 && IN_RANGE (INTVAL (op1
), GET_MODE_BITSIZE (scalar_mode
) / 2 + left
,
2208 GET_MODE_BITSIZE (scalar_mode
) - 1))
2210 op1
= GEN_INT (GET_MODE_BITSIZE (scalar_mode
) - INTVAL (op1
));
2212 code
= left
? LROTATE_EXPR
: RROTATE_EXPR
;
2215 /* Rotation of 16bit values by 8 bits is effectively equivalent to a bswaphi.
2216 Note that this is not the case for bigger values. For instance a rotation
2217 of 0x01020304 by 16 bits gives 0x03040102 which is different from
2218 0x04030201 (bswapsi). */
2220 && CONST_INT_P (op1
)
2221 && INTVAL (op1
) == BITS_PER_UNIT
2222 && GET_MODE_SIZE (scalar_mode
) == 2
2223 && optab_handler (bswap_optab
, HImode
) != CODE_FOR_nothing
)
2224 return expand_unop (HImode
, bswap_optab
, shifted
, NULL_RTX
,
2227 if (op1
== const0_rtx
)
2230 /* Check whether its cheaper to implement a left shift by a constant
2231 bit count by a sequence of additions. */
2232 if (code
== LSHIFT_EXPR
2233 && CONST_INT_P (op1
)
2235 && INTVAL (op1
) < GET_MODE_PRECISION (scalar_mode
)
2236 && INTVAL (op1
) < MAX_BITS_PER_WORD
2237 && (shift_cost (speed
, mode
, INTVAL (op1
))
2238 > INTVAL (op1
) * add_cost (speed
, mode
))
2239 && shift_cost (speed
, mode
, INTVAL (op1
)) != MAX_COST
)
2242 for (i
= 0; i
< INTVAL (op1
); i
++)
2244 temp
= force_reg (mode
, shifted
);
2245 shifted
= expand_binop (mode
, add_optab
, temp
, temp
, NULL_RTX
,
2246 unsignedp
, OPTAB_LIB_WIDEN
);
2251 for (attempt
= 0; temp
== 0 && attempt
< 3; attempt
++)
2253 enum optab_methods methods
;
2256 methods
= OPTAB_DIRECT
;
2257 else if (attempt
== 1)
2258 methods
= OPTAB_WIDEN
;
2260 methods
= OPTAB_LIB_WIDEN
;
2264 /* Widening does not work for rotation. */
2265 if (methods
== OPTAB_WIDEN
)
2267 else if (methods
== OPTAB_LIB_WIDEN
)
2269 /* If we have been unable to open-code this by a rotation,
2270 do it as the IOR of two shifts. I.e., to rotate A
2272 (A << N) | ((unsigned) A >> ((-N) & (C - 1)))
2273 where C is the bitsize of A.
2275 It is theoretically possible that the target machine might
2276 not be able to perform either shift and hence we would
2277 be making two libcalls rather than just the one for the
2278 shift (similarly if IOR could not be done). We will allow
2279 this extremely unlikely lossage to avoid complicating the
2282 rtx subtarget
= target
== shifted
? 0 : target
;
2283 rtx new_amount
, other_amount
;
2287 if (op1
== const0_rtx
)
2289 else if (CONST_INT_P (op1
))
2290 other_amount
= GEN_INT (GET_MODE_BITSIZE (scalar_mode
)
2295 = simplify_gen_unary (NEG
, GET_MODE (op1
),
2296 op1
, GET_MODE (op1
));
2297 HOST_WIDE_INT mask
= GET_MODE_PRECISION (scalar_mode
) - 1;
2299 = simplify_gen_binary (AND
, GET_MODE (op1
), other_amount
,
2300 gen_int_mode (mask
, GET_MODE (op1
)));
2303 shifted
= force_reg (mode
, shifted
);
2305 temp
= expand_shift_1 (left
? LSHIFT_EXPR
: RSHIFT_EXPR
,
2306 mode
, shifted
, new_amount
, 0, 1);
2307 temp1
= expand_shift_1 (left
? RSHIFT_EXPR
: LSHIFT_EXPR
,
2308 mode
, shifted
, other_amount
,
2310 return expand_binop (mode
, ior_optab
, temp
, temp1
, target
,
2311 unsignedp
, methods
);
2314 temp
= expand_binop (mode
,
2315 left
? lrotate_optab
: rrotate_optab
,
2316 shifted
, op1
, target
, unsignedp
, methods
);
2319 temp
= expand_binop (mode
,
2320 left
? lshift_optab
: rshift_uns_optab
,
2321 shifted
, op1
, target
, unsignedp
, methods
);
2323 /* Do arithmetic shifts.
2324 Also, if we are going to widen the operand, we can just as well
2325 use an arithmetic right-shift instead of a logical one. */
2326 if (temp
== 0 && ! rotate
2327 && (! unsignedp
|| (! left
&& methods
== OPTAB_WIDEN
)))
2329 enum optab_methods methods1
= methods
;
2331 /* If trying to widen a log shift to an arithmetic shift,
2332 don't accept an arithmetic shift of the same size. */
2334 methods1
= OPTAB_MUST_WIDEN
;
2336 /* Arithmetic shift */
2338 temp
= expand_binop (mode
,
2339 left
? lshift_optab
: rshift_arith_optab
,
2340 shifted
, op1
, target
, unsignedp
, methods1
);
2343 /* We used to try extzv here for logical right shifts, but that was
2344 only useful for one machine, the VAX, and caused poor code
2345 generation there for lshrdi3, so the code was deleted and a
2346 define_expand for lshrsi3 was added to vax.md. */
2353 /* Output a shift instruction for expression code CODE,
2354 with SHIFTED being the rtx for the value to shift,
2355 and AMOUNT the amount to shift by.
2356 Store the result in the rtx TARGET, if that is convenient.
2357 If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2358 Return the rtx for where the value is. */
2361 expand_shift (enum tree_code code
, machine_mode mode
, rtx shifted
,
2362 int amount
, rtx target
, int unsignedp
)
2364 return expand_shift_1 (code
, mode
,
2365 shifted
, GEN_INT (amount
), target
, unsignedp
);
2368 /* Output a shift instruction for expression code CODE,
2369 with SHIFTED being the rtx for the value to shift,
2370 and AMOUNT the tree for the amount to shift by.
2371 Store the result in the rtx TARGET, if that is convenient.
2372 If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
2373 Return the rtx for where the value is. */
2376 expand_variable_shift (enum tree_code code
, machine_mode mode
, rtx shifted
,
2377 tree amount
, rtx target
, int unsignedp
)
2379 return expand_shift_1 (code
, mode
,
2380 shifted
, expand_normal (amount
), target
, unsignedp
);
2384 /* Indicates the type of fixup needed after a constant multiplication.
2385 BASIC_VARIANT means no fixup is needed, NEGATE_VARIANT means that
2386 the result should be negated, and ADD_VARIANT means that the
2387 multiplicand should be added to the result. */
2388 enum mult_variant
{basic_variant
, negate_variant
, add_variant
};
2390 static void synth_mult (struct algorithm
*, unsigned HOST_WIDE_INT
,
2391 const struct mult_cost
*, machine_mode mode
);
2392 static bool choose_mult_variant (machine_mode
, HOST_WIDE_INT
,
2393 struct algorithm
*, enum mult_variant
*, int);
2394 static rtx
expand_mult_const (machine_mode
, rtx
, HOST_WIDE_INT
, rtx
,
2395 const struct algorithm
*, enum mult_variant
);
2396 static unsigned HOST_WIDE_INT
invert_mod2n (unsigned HOST_WIDE_INT
, int);
2397 static rtx
extract_high_half (machine_mode
, rtx
);
2398 static rtx
expmed_mult_highpart (machine_mode
, rtx
, rtx
, rtx
, int, int);
2399 static rtx
expmed_mult_highpart_optab (machine_mode
, rtx
, rtx
, rtx
,
2401 /* Compute and return the best algorithm for multiplying by T.
2402 The algorithm must cost less than cost_limit
2403 If retval.cost >= COST_LIMIT, no algorithm was found and all
2404 other field of the returned struct are undefined.
2405 MODE is the machine mode of the multiplication. */
2408 synth_mult (struct algorithm
*alg_out
, unsigned HOST_WIDE_INT t
,
2409 const struct mult_cost
*cost_limit
, machine_mode mode
)
2412 struct algorithm
*alg_in
, *best_alg
;
2413 struct mult_cost best_cost
;
2414 struct mult_cost new_limit
;
2415 int op_cost
, op_latency
;
2416 unsigned HOST_WIDE_INT orig_t
= t
;
2417 unsigned HOST_WIDE_INT q
;
2418 int maxm
, hash_index
;
2419 bool cache_hit
= false;
2420 enum alg_code cache_alg
= alg_zero
;
2421 bool speed
= optimize_insn_for_speed_p ();
2423 struct alg_hash_entry
*entry_ptr
;
2425 /* Indicate that no algorithm is yet found. If no algorithm
2426 is found, this value will be returned and indicate failure. */
2427 alg_out
->cost
.cost
= cost_limit
->cost
+ 1;
2428 alg_out
->cost
.latency
= cost_limit
->latency
+ 1;
2430 if (cost_limit
->cost
< 0
2431 || (cost_limit
->cost
== 0 && cost_limit
->latency
<= 0))
2434 /* Be prepared for vector modes. */
2435 imode
= GET_MODE_INNER (mode
);
2436 if (imode
== VOIDmode
)
2439 maxm
= MIN (BITS_PER_WORD
, GET_MODE_BITSIZE (imode
));
2441 /* Restrict the bits of "t" to the multiplication's mode. */
2442 t
&= GET_MODE_MASK (imode
);
2444 /* t == 1 can be done in zero cost. */
2448 alg_out
->cost
.cost
= 0;
2449 alg_out
->cost
.latency
= 0;
2450 alg_out
->op
[0] = alg_m
;
2454 /* t == 0 sometimes has a cost. If it does and it exceeds our limit,
2458 if (MULT_COST_LESS (cost_limit
, zero_cost (speed
)))
2463 alg_out
->cost
.cost
= zero_cost (speed
);
2464 alg_out
->cost
.latency
= zero_cost (speed
);
2465 alg_out
->op
[0] = alg_zero
;
2470 /* We'll be needing a couple extra algorithm structures now. */
2472 alg_in
= XALLOCA (struct algorithm
);
2473 best_alg
= XALLOCA (struct algorithm
);
2474 best_cost
= *cost_limit
;
2476 /* Compute the hash index. */
2477 hash_index
= (t
^ (unsigned int) mode
^ (speed
* 256)) % NUM_ALG_HASH_ENTRIES
;
2479 /* See if we already know what to do for T. */
2480 entry_ptr
= alg_hash_entry_ptr (hash_index
);
2481 if (entry_ptr
->t
== t
2482 && entry_ptr
->mode
== mode
2483 && entry_ptr
->mode
== mode
2484 && entry_ptr
->speed
== speed
2485 && entry_ptr
->alg
!= alg_unknown
)
2487 cache_alg
= entry_ptr
->alg
;
2489 if (cache_alg
== alg_impossible
)
2491 /* The cache tells us that it's impossible to synthesize
2492 multiplication by T within entry_ptr->cost. */
2493 if (!CHEAPER_MULT_COST (&entry_ptr
->cost
, cost_limit
))
2494 /* COST_LIMIT is at least as restrictive as the one
2495 recorded in the hash table, in which case we have no
2496 hope of synthesizing a multiplication. Just
2500 /* If we get here, COST_LIMIT is less restrictive than the
2501 one recorded in the hash table, so we may be able to
2502 synthesize a multiplication. Proceed as if we didn't
2503 have the cache entry. */
2507 if (CHEAPER_MULT_COST (cost_limit
, &entry_ptr
->cost
))
2508 /* The cached algorithm shows that this multiplication
2509 requires more cost than COST_LIMIT. Just return. This
2510 way, we don't clobber this cache entry with
2511 alg_impossible but retain useful information. */
2523 goto do_alg_addsub_t_m2
;
2525 case alg_add_factor
:
2526 case alg_sub_factor
:
2527 goto do_alg_addsub_factor
;
2530 goto do_alg_add_t2_m
;
2533 goto do_alg_sub_t2_m
;
2541 /* If we have a group of zero bits at the low-order part of T, try
2542 multiplying by the remaining bits and then doing a shift. */
2547 m
= floor_log2 (t
& -t
); /* m = number of low zero bits */
2551 /* The function expand_shift will choose between a shift and
2552 a sequence of additions, so the observed cost is given as
2553 MIN (m * add_cost(speed, mode), shift_cost(speed, mode, m)). */
2554 op_cost
= m
* add_cost (speed
, mode
);
2555 if (shift_cost (speed
, mode
, m
) < op_cost
)
2556 op_cost
= shift_cost (speed
, mode
, m
);
2557 new_limit
.cost
= best_cost
.cost
- op_cost
;
2558 new_limit
.latency
= best_cost
.latency
- op_cost
;
2559 synth_mult (alg_in
, q
, &new_limit
, mode
);
2561 alg_in
->cost
.cost
+= op_cost
;
2562 alg_in
->cost
.latency
+= op_cost
;
2563 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
2565 best_cost
= alg_in
->cost
;
2566 std::swap (alg_in
, best_alg
);
2567 best_alg
->log
[best_alg
->ops
] = m
;
2568 best_alg
->op
[best_alg
->ops
] = alg_shift
;
2571 /* See if treating ORIG_T as a signed number yields a better
2572 sequence. Try this sequence only for a negative ORIG_T
2573 as it would be useless for a non-negative ORIG_T. */
2574 if ((HOST_WIDE_INT
) orig_t
< 0)
2576 /* Shift ORIG_T as follows because a right shift of a
2577 negative-valued signed type is implementation
2579 q
= ~(~orig_t
>> m
);
2580 /* The function expand_shift will choose between a shift
2581 and a sequence of additions, so the observed cost is
2582 given as MIN (m * add_cost(speed, mode),
2583 shift_cost(speed, mode, m)). */
2584 op_cost
= m
* add_cost (speed
, mode
);
2585 if (shift_cost (speed
, mode
, m
) < op_cost
)
2586 op_cost
= shift_cost (speed
, mode
, m
);
2587 new_limit
.cost
= best_cost
.cost
- op_cost
;
2588 new_limit
.latency
= best_cost
.latency
- op_cost
;
2589 synth_mult (alg_in
, q
, &new_limit
, mode
);
2591 alg_in
->cost
.cost
+= op_cost
;
2592 alg_in
->cost
.latency
+= op_cost
;
2593 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
2595 best_cost
= alg_in
->cost
;
2596 std::swap (alg_in
, best_alg
);
2597 best_alg
->log
[best_alg
->ops
] = m
;
2598 best_alg
->op
[best_alg
->ops
] = alg_shift
;
2606 /* If we have an odd number, add or subtract one. */
2609 unsigned HOST_WIDE_INT w
;
2612 for (w
= 1; (w
& t
) != 0; w
<<= 1)
2614 /* If T was -1, then W will be zero after the loop. This is another
2615 case where T ends with ...111. Handling this with (T + 1) and
2616 subtract 1 produces slightly better code and results in algorithm
2617 selection much faster than treating it like the ...0111 case
2621 /* Reject the case where t is 3.
2622 Thus we prefer addition in that case. */
2625 /* T ends with ...111. Multiply by (T + 1) and subtract T. */
2627 op_cost
= add_cost (speed
, mode
);
2628 new_limit
.cost
= best_cost
.cost
- op_cost
;
2629 new_limit
.latency
= best_cost
.latency
- op_cost
;
2630 synth_mult (alg_in
, t
+ 1, &new_limit
, mode
);
2632 alg_in
->cost
.cost
+= op_cost
;
2633 alg_in
->cost
.latency
+= op_cost
;
2634 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
2636 best_cost
= alg_in
->cost
;
2637 std::swap (alg_in
, best_alg
);
2638 best_alg
->log
[best_alg
->ops
] = 0;
2639 best_alg
->op
[best_alg
->ops
] = alg_sub_t_m2
;
2644 /* T ends with ...01 or ...011. Multiply by (T - 1) and add T. */
2646 op_cost
= add_cost (speed
, mode
);
2647 new_limit
.cost
= best_cost
.cost
- op_cost
;
2648 new_limit
.latency
= best_cost
.latency
- op_cost
;
2649 synth_mult (alg_in
, t
- 1, &new_limit
, mode
);
2651 alg_in
->cost
.cost
+= op_cost
;
2652 alg_in
->cost
.latency
+= op_cost
;
2653 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
2655 best_cost
= alg_in
->cost
;
2656 std::swap (alg_in
, best_alg
);
2657 best_alg
->log
[best_alg
->ops
] = 0;
2658 best_alg
->op
[best_alg
->ops
] = alg_add_t_m2
;
2662 /* We may be able to calculate a * -7, a * -15, a * -31, etc
2663 quickly with a - a * n for some appropriate constant n. */
2664 m
= exact_log2 (-orig_t
+ 1);
2665 if (m
>= 0 && m
< maxm
)
2667 op_cost
= add_cost (speed
, mode
) + shift_cost (speed
, mode
, m
);
2668 /* If the target has a cheap shift-and-subtract insn use
2669 that in preference to a shift insn followed by a sub insn.
2670 Assume that the shift-and-sub is "atomic" with a latency
2671 equal to it's cost, otherwise assume that on superscalar
2672 hardware the shift may be executed concurrently with the
2673 earlier steps in the algorithm. */
2674 if (shiftsub1_cost (speed
, mode
, m
) <= op_cost
)
2676 op_cost
= shiftsub1_cost (speed
, mode
, m
);
2677 op_latency
= op_cost
;
2680 op_latency
= add_cost (speed
, mode
);
2682 new_limit
.cost
= best_cost
.cost
- op_cost
;
2683 new_limit
.latency
= best_cost
.latency
- op_latency
;
2684 synth_mult (alg_in
, (unsigned HOST_WIDE_INT
) (-orig_t
+ 1) >> m
,
2687 alg_in
->cost
.cost
+= op_cost
;
2688 alg_in
->cost
.latency
+= op_latency
;
2689 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
2691 best_cost
= alg_in
->cost
;
2692 std::swap (alg_in
, best_alg
);
2693 best_alg
->log
[best_alg
->ops
] = m
;
2694 best_alg
->op
[best_alg
->ops
] = alg_sub_t_m2
;
2702 /* Look for factors of t of the form
2703 t = q(2**m +- 1), 2 <= m <= floor(log2(t - 1)).
2704 If we find such a factor, we can multiply by t using an algorithm that
2705 multiplies by q, shift the result by m and add/subtract it to itself.
2707 We search for large factors first and loop down, even if large factors
2708 are less probable than small; if we find a large factor we will find a
2709 good sequence quickly, and therefore be able to prune (by decreasing
2710 COST_LIMIT) the search. */
2712 do_alg_addsub_factor
:
2713 for (m
= floor_log2 (t
- 1); m
>= 2; m
--)
2715 unsigned HOST_WIDE_INT d
;
2717 d
= ((unsigned HOST_WIDE_INT
) 1 << m
) + 1;
2718 if (t
% d
== 0 && t
> d
&& m
< maxm
2719 && (!cache_hit
|| cache_alg
== alg_add_factor
))
2721 op_cost
= add_cost (speed
, mode
) + shift_cost (speed
, mode
, m
);
2722 if (shiftadd_cost (speed
, mode
, m
) <= op_cost
)
2723 op_cost
= shiftadd_cost (speed
, mode
, m
);
2725 op_latency
= op_cost
;
2728 new_limit
.cost
= best_cost
.cost
- op_cost
;
2729 new_limit
.latency
= best_cost
.latency
- op_latency
;
2730 synth_mult (alg_in
, t
/ d
, &new_limit
, mode
);
2732 alg_in
->cost
.cost
+= op_cost
;
2733 alg_in
->cost
.latency
+= op_latency
;
2734 if (alg_in
->cost
.latency
< op_cost
)
2735 alg_in
->cost
.latency
= op_cost
;
2736 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
2738 best_cost
= alg_in
->cost
;
2739 std::swap (alg_in
, best_alg
);
2740 best_alg
->log
[best_alg
->ops
] = m
;
2741 best_alg
->op
[best_alg
->ops
] = alg_add_factor
;
2743 /* Other factors will have been taken care of in the recursion. */
2747 d
= ((unsigned HOST_WIDE_INT
) 1 << m
) - 1;
2748 if (t
% d
== 0 && t
> d
&& m
< maxm
2749 && (!cache_hit
|| cache_alg
== alg_sub_factor
))
2751 op_cost
= add_cost (speed
, mode
) + shift_cost (speed
, mode
, m
);
2752 if (shiftsub0_cost (speed
, mode
, m
) <= op_cost
)
2753 op_cost
= shiftsub0_cost (speed
, mode
, m
);
2755 op_latency
= op_cost
;
2757 new_limit
.cost
= best_cost
.cost
- op_cost
;
2758 new_limit
.latency
= best_cost
.latency
- op_latency
;
2759 synth_mult (alg_in
, t
/ d
, &new_limit
, mode
);
2761 alg_in
->cost
.cost
+= op_cost
;
2762 alg_in
->cost
.latency
+= op_latency
;
2763 if (alg_in
->cost
.latency
< op_cost
)
2764 alg_in
->cost
.latency
= op_cost
;
2765 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
2767 best_cost
= alg_in
->cost
;
2768 std::swap (alg_in
, best_alg
);
2769 best_alg
->log
[best_alg
->ops
] = m
;
2770 best_alg
->op
[best_alg
->ops
] = alg_sub_factor
;
2778 /* Try shift-and-add (load effective address) instructions,
2779 i.e. do a*3, a*5, a*9. */
2786 if (m
>= 0 && m
< maxm
)
2788 op_cost
= shiftadd_cost (speed
, mode
, m
);
2789 new_limit
.cost
= best_cost
.cost
- op_cost
;
2790 new_limit
.latency
= best_cost
.latency
- op_cost
;
2791 synth_mult (alg_in
, (t
- 1) >> m
, &new_limit
, mode
);
2793 alg_in
->cost
.cost
+= op_cost
;
2794 alg_in
->cost
.latency
+= op_cost
;
2795 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
2797 best_cost
= alg_in
->cost
;
2798 std::swap (alg_in
, best_alg
);
2799 best_alg
->log
[best_alg
->ops
] = m
;
2800 best_alg
->op
[best_alg
->ops
] = alg_add_t2_m
;
2810 if (m
>= 0 && m
< maxm
)
2812 op_cost
= shiftsub0_cost (speed
, mode
, m
);
2813 new_limit
.cost
= best_cost
.cost
- op_cost
;
2814 new_limit
.latency
= best_cost
.latency
- op_cost
;
2815 synth_mult (alg_in
, (t
+ 1) >> m
, &new_limit
, mode
);
2817 alg_in
->cost
.cost
+= op_cost
;
2818 alg_in
->cost
.latency
+= op_cost
;
2819 if (CHEAPER_MULT_COST (&alg_in
->cost
, &best_cost
))
2821 best_cost
= alg_in
->cost
;
2822 std::swap (alg_in
, best_alg
);
2823 best_alg
->log
[best_alg
->ops
] = m
;
2824 best_alg
->op
[best_alg
->ops
] = alg_sub_t2_m
;
2832 /* If best_cost has not decreased, we have not found any algorithm. */
2833 if (!CHEAPER_MULT_COST (&best_cost
, cost_limit
))
2835 /* We failed to find an algorithm. Record alg_impossible for
2836 this case (that is, <T, MODE, COST_LIMIT>) so that next time
2837 we are asked to find an algorithm for T within the same or
2838 lower COST_LIMIT, we can immediately return to the
2841 entry_ptr
->mode
= mode
;
2842 entry_ptr
->speed
= speed
;
2843 entry_ptr
->alg
= alg_impossible
;
2844 entry_ptr
->cost
= *cost_limit
;
2848 /* Cache the result. */
2852 entry_ptr
->mode
= mode
;
2853 entry_ptr
->speed
= speed
;
2854 entry_ptr
->alg
= best_alg
->op
[best_alg
->ops
];
2855 entry_ptr
->cost
.cost
= best_cost
.cost
;
2856 entry_ptr
->cost
.latency
= best_cost
.latency
;
2859 /* If we are getting a too long sequence for `struct algorithm'
2860 to record, make this search fail. */
2861 if (best_alg
->ops
== MAX_BITS_PER_WORD
)
2864 /* Copy the algorithm from temporary space to the space at alg_out.
2865 We avoid using structure assignment because the majority of
2866 best_alg is normally undefined, and this is a critical function. */
2867 alg_out
->ops
= best_alg
->ops
+ 1;
2868 alg_out
->cost
= best_cost
;
2869 memcpy (alg_out
->op
, best_alg
->op
,
2870 alg_out
->ops
* sizeof *alg_out
->op
);
2871 memcpy (alg_out
->log
, best_alg
->log
,
2872 alg_out
->ops
* sizeof *alg_out
->log
);
2875 /* Find the cheapest way of multiplying a value of mode MODE by VAL.
2876 Try three variations:
2878 - a shift/add sequence based on VAL itself
2879 - a shift/add sequence based on -VAL, followed by a negation
2880 - a shift/add sequence based on VAL - 1, followed by an addition.
2882 Return true if the cheapest of these cost less than MULT_COST,
2883 describing the algorithm in *ALG and final fixup in *VARIANT. */
2886 choose_mult_variant (machine_mode mode
, HOST_WIDE_INT val
,
2887 struct algorithm
*alg
, enum mult_variant
*variant
,
2890 struct algorithm alg2
;
2891 struct mult_cost limit
;
2893 bool speed
= optimize_insn_for_speed_p ();
2895 /* Fail quickly for impossible bounds. */
2899 /* Ensure that mult_cost provides a reasonable upper bound.
2900 Any constant multiplication can be performed with less
2901 than 2 * bits additions. */
2902 op_cost
= 2 * GET_MODE_UNIT_BITSIZE (mode
) * add_cost (speed
, mode
);
2903 if (mult_cost
> op_cost
)
2904 mult_cost
= op_cost
;
2906 *variant
= basic_variant
;
2907 limit
.cost
= mult_cost
;
2908 limit
.latency
= mult_cost
;
2909 synth_mult (alg
, val
, &limit
, mode
);
2911 /* This works only if the inverted value actually fits in an
2913 if (HOST_BITS_PER_INT
>= GET_MODE_UNIT_BITSIZE (mode
))
2915 op_cost
= neg_cost (speed
, mode
);
2916 if (MULT_COST_LESS (&alg
->cost
, mult_cost
))
2918 limit
.cost
= alg
->cost
.cost
- op_cost
;
2919 limit
.latency
= alg
->cost
.latency
- op_cost
;
2923 limit
.cost
= mult_cost
- op_cost
;
2924 limit
.latency
= mult_cost
- op_cost
;
2927 synth_mult (&alg2
, -val
, &limit
, mode
);
2928 alg2
.cost
.cost
+= op_cost
;
2929 alg2
.cost
.latency
+= op_cost
;
2930 if (CHEAPER_MULT_COST (&alg2
.cost
, &alg
->cost
))
2931 *alg
= alg2
, *variant
= negate_variant
;
2934 /* This proves very useful for division-by-constant. */
2935 op_cost
= add_cost (speed
, mode
);
2936 if (MULT_COST_LESS (&alg
->cost
, mult_cost
))
2938 limit
.cost
= alg
->cost
.cost
- op_cost
;
2939 limit
.latency
= alg
->cost
.latency
- op_cost
;
2943 limit
.cost
= mult_cost
- op_cost
;
2944 limit
.latency
= mult_cost
- op_cost
;
2947 synth_mult (&alg2
, val
- 1, &limit
, mode
);
2948 alg2
.cost
.cost
+= op_cost
;
2949 alg2
.cost
.latency
+= op_cost
;
2950 if (CHEAPER_MULT_COST (&alg2
.cost
, &alg
->cost
))
2951 *alg
= alg2
, *variant
= add_variant
;
2953 return MULT_COST_LESS (&alg
->cost
, mult_cost
);
2956 /* A subroutine of expand_mult, used for constant multiplications.
2957 Multiply OP0 by VAL in mode MODE, storing the result in TARGET if
2958 convenient. Use the shift/add sequence described by ALG and apply
2959 the final fixup specified by VARIANT. */
2962 expand_mult_const (machine_mode mode
, rtx op0
, HOST_WIDE_INT val
,
2963 rtx target
, const struct algorithm
*alg
,
2964 enum mult_variant variant
)
2966 HOST_WIDE_INT val_so_far
;
2972 /* Avoid referencing memory over and over and invalid sharing
2974 op0
= force_reg (mode
, op0
);
2976 /* ACCUM starts out either as OP0 or as a zero, depending on
2977 the first operation. */
2979 if (alg
->op
[0] == alg_zero
)
2981 accum
= copy_to_mode_reg (mode
, CONST0_RTX (mode
));
2984 else if (alg
->op
[0] == alg_m
)
2986 accum
= copy_to_mode_reg (mode
, op0
);
2992 for (opno
= 1; opno
< alg
->ops
; opno
++)
2994 int log
= alg
->log
[opno
];
2995 rtx shift_subtarget
= optimize
? 0 : accum
;
2997 = (opno
== alg
->ops
- 1 && target
!= 0 && variant
!= add_variant
3000 rtx accum_target
= optimize
? 0 : accum
;
3003 switch (alg
->op
[opno
])
3006 tem
= expand_shift (LSHIFT_EXPR
, mode
, accum
, log
, NULL_RTX
, 0);
3007 /* REG_EQUAL note will be attached to the following insn. */
3008 emit_move_insn (accum
, tem
);
3013 tem
= expand_shift (LSHIFT_EXPR
, mode
, op0
, log
, NULL_RTX
, 0);
3014 accum
= force_operand (gen_rtx_PLUS (mode
, accum
, tem
),
3015 add_target
? add_target
: accum_target
);
3016 val_so_far
+= (HOST_WIDE_INT
) 1 << log
;
3020 tem
= expand_shift (LSHIFT_EXPR
, mode
, op0
, log
, NULL_RTX
, 0);
3021 accum
= force_operand (gen_rtx_MINUS (mode
, accum
, tem
),
3022 add_target
? add_target
: accum_target
);
3023 val_so_far
-= (HOST_WIDE_INT
) 1 << log
;
3027 accum
= expand_shift (LSHIFT_EXPR
, mode
, accum
,
3028 log
, shift_subtarget
, 0);
3029 accum
= force_operand (gen_rtx_PLUS (mode
, accum
, op0
),
3030 add_target
? add_target
: accum_target
);
3031 val_so_far
= (val_so_far
<< log
) + 1;
3035 accum
= expand_shift (LSHIFT_EXPR
, mode
, accum
,
3036 log
, shift_subtarget
, 0);
3037 accum
= force_operand (gen_rtx_MINUS (mode
, accum
, op0
),
3038 add_target
? add_target
: accum_target
);
3039 val_so_far
= (val_so_far
<< log
) - 1;
3042 case alg_add_factor
:
3043 tem
= expand_shift (LSHIFT_EXPR
, mode
, accum
, log
, NULL_RTX
, 0);
3044 accum
= force_operand (gen_rtx_PLUS (mode
, accum
, tem
),
3045 add_target
? add_target
: accum_target
);
3046 val_so_far
+= val_so_far
<< log
;
3049 case alg_sub_factor
:
3050 tem
= expand_shift (LSHIFT_EXPR
, mode
, accum
, log
, NULL_RTX
, 0);
3051 accum
= force_operand (gen_rtx_MINUS (mode
, tem
, accum
),
3053 ? add_target
: (optimize
? 0 : tem
)));
3054 val_so_far
= (val_so_far
<< log
) - val_so_far
;
3061 if (SCALAR_INT_MODE_P (mode
))
3063 /* Write a REG_EQUAL note on the last insn so that we can cse
3064 multiplication sequences. Note that if ACCUM is a SUBREG,
3065 we've set the inner register and must properly indicate that. */
3066 tem
= op0
, nmode
= mode
;
3067 accum_inner
= accum
;
3068 if (GET_CODE (accum
) == SUBREG
)
3070 accum_inner
= SUBREG_REG (accum
);
3071 nmode
= GET_MODE (accum_inner
);
3072 tem
= gen_lowpart (nmode
, op0
);
3075 insn
= get_last_insn ();
3076 set_dst_reg_note (insn
, REG_EQUAL
,
3077 gen_rtx_MULT (nmode
, tem
,
3078 gen_int_mode (val_so_far
, nmode
)),
3083 if (variant
== negate_variant
)
3085 val_so_far
= -val_so_far
;
3086 accum
= expand_unop (mode
, neg_optab
, accum
, target
, 0);
3088 else if (variant
== add_variant
)
3090 val_so_far
= val_so_far
+ 1;
3091 accum
= force_operand (gen_rtx_PLUS (mode
, accum
, op0
), target
);
3094 /* Compare only the bits of val and val_so_far that are significant
3095 in the result mode, to avoid sign-/zero-extension confusion. */
3096 nmode
= GET_MODE_INNER (mode
);
3097 if (nmode
== VOIDmode
)
3099 val
&= GET_MODE_MASK (nmode
);
3100 val_so_far
&= GET_MODE_MASK (nmode
);
3101 gcc_assert (val
== val_so_far
);
3106 /* Perform a multiplication and return an rtx for the result.
3107 MODE is mode of value; OP0 and OP1 are what to multiply (rtx's);
3108 TARGET is a suggestion for where to store the result (an rtx).
3110 We check specially for a constant integer as OP1.
3111 If you want this check for OP0 as well, then before calling
3112 you should swap the two operands if OP0 would be constant. */
3115 expand_mult (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
3118 enum mult_variant variant
;
3119 struct algorithm algorithm
;
3122 bool speed
= optimize_insn_for_speed_p ();
3123 bool do_trapv
= flag_trapv
&& SCALAR_INT_MODE_P (mode
) && !unsignedp
;
3125 if (CONSTANT_P (op0
))
3126 std::swap (op0
, op1
);
3128 /* For vectors, there are several simplifications that can be made if
3129 all elements of the vector constant are identical. */
3131 if (GET_CODE (op1
) == CONST_VECTOR
)
3133 int i
, n
= CONST_VECTOR_NUNITS (op1
);
3134 scalar_op1
= CONST_VECTOR_ELT (op1
, 0);
3135 for (i
= 1; i
< n
; ++i
)
3136 if (!rtx_equal_p (scalar_op1
, CONST_VECTOR_ELT (op1
, i
)))
3140 if (INTEGRAL_MODE_P (mode
))
3143 HOST_WIDE_INT coeff
;
3147 if (op1
== CONST0_RTX (mode
))
3149 if (op1
== CONST1_RTX (mode
))
3151 if (op1
== CONSTM1_RTX (mode
))
3152 return expand_unop (mode
, do_trapv
? negv_optab
: neg_optab
,
3158 /* If mode is integer vector mode, check if the backend supports
3159 vector lshift (by scalar or vector) at all. If not, we can't use
3160 synthetized multiply. */
3161 if (GET_MODE_CLASS (mode
) == MODE_VECTOR_INT
3162 && optab_handler (vashl_optab
, mode
) == CODE_FOR_nothing
3163 && optab_handler (ashl_optab
, mode
) == CODE_FOR_nothing
)
3166 /* These are the operations that are potentially turned into
3167 a sequence of shifts and additions. */
3168 mode_bitsize
= GET_MODE_UNIT_BITSIZE (mode
);
3170 /* synth_mult does an `unsigned int' multiply. As long as the mode is
3171 less than or equal in size to `unsigned int' this doesn't matter.
3172 If the mode is larger than `unsigned int', then synth_mult works
3173 only if the constant value exactly fits in an `unsigned int' without
3174 any truncation. This means that multiplying by negative values does
3175 not work; results are off by 2^32 on a 32 bit machine. */
3176 if (CONST_INT_P (scalar_op1
))
3178 coeff
= INTVAL (scalar_op1
);
3181 #if TARGET_SUPPORTS_WIDE_INT
3182 else if (CONST_WIDE_INT_P (scalar_op1
))
3184 else if (CONST_DOUBLE_AS_INT_P (scalar_op1
))
3187 int shift
= wi::exact_log2 (std::make_pair (scalar_op1
, mode
));
3188 /* Perfect power of 2 (other than 1, which is handled above). */
3190 return expand_shift (LSHIFT_EXPR
, mode
, op0
,
3191 shift
, target
, unsignedp
);
3198 /* We used to test optimize here, on the grounds that it's better to
3199 produce a smaller program when -O is not used. But this causes
3200 such a terrible slowdown sometimes that it seems better to always
3203 /* Special case powers of two. */
3204 if (EXACT_POWER_OF_2_OR_ZERO_P (coeff
)
3205 && !(is_neg
&& mode_bitsize
> HOST_BITS_PER_WIDE_INT
))
3206 return expand_shift (LSHIFT_EXPR
, mode
, op0
,
3207 floor_log2 (coeff
), target
, unsignedp
);
3209 fake_reg
= gen_raw_REG (mode
, LAST_VIRTUAL_REGISTER
+ 1);
3211 /* Attempt to handle multiplication of DImode values by negative
3212 coefficients, by performing the multiplication by a positive
3213 multiplier and then inverting the result. */
3214 if (is_neg
&& mode_bitsize
> HOST_BITS_PER_WIDE_INT
)
3216 /* Its safe to use -coeff even for INT_MIN, as the
3217 result is interpreted as an unsigned coefficient.
3218 Exclude cost of op0 from max_cost to match the cost
3219 calculation of the synth_mult. */
3220 coeff
= -(unsigned HOST_WIDE_INT
) coeff
;
3221 max_cost
= (set_src_cost (gen_rtx_MULT (mode
, fake_reg
, op1
), speed
)
3222 - neg_cost (speed
, mode
));
3226 /* Special case powers of two. */
3227 if (EXACT_POWER_OF_2_OR_ZERO_P (coeff
))
3229 rtx temp
= expand_shift (LSHIFT_EXPR
, mode
, op0
,
3230 floor_log2 (coeff
), target
, unsignedp
);
3231 return expand_unop (mode
, neg_optab
, temp
, target
, 0);
3234 if (choose_mult_variant (mode
, coeff
, &algorithm
, &variant
,
3237 rtx temp
= expand_mult_const (mode
, op0
, coeff
, NULL_RTX
,
3238 &algorithm
, variant
);
3239 return expand_unop (mode
, neg_optab
, temp
, target
, 0);
3244 /* Exclude cost of op0 from max_cost to match the cost
3245 calculation of the synth_mult. */
3246 max_cost
= set_src_cost (gen_rtx_MULT (mode
, fake_reg
, op1
), speed
);
3247 if (choose_mult_variant (mode
, coeff
, &algorithm
, &variant
, max_cost
))
3248 return expand_mult_const (mode
, op0
, coeff
, target
,
3249 &algorithm
, variant
);
3253 /* Expand x*2.0 as x+x. */
3254 if (CONST_DOUBLE_AS_FLOAT_P (scalar_op1
))
3257 REAL_VALUE_FROM_CONST_DOUBLE (d
, scalar_op1
);
3259 if (REAL_VALUES_EQUAL (d
, dconst2
))
3261 op0
= force_reg (GET_MODE (op0
), op0
);
3262 return expand_binop (mode
, add_optab
, op0
, op0
,
3263 target
, unsignedp
, OPTAB_LIB_WIDEN
);
3268 /* This used to use umul_optab if unsigned, but for non-widening multiply
3269 there is no difference between signed and unsigned. */
3270 op0
= expand_binop (mode
, do_trapv
? smulv_optab
: smul_optab
,
3271 op0
, op1
, target
, unsignedp
, OPTAB_LIB_WIDEN
);
3276 /* Return a cost estimate for multiplying a register by the given
3277 COEFFicient in the given MODE and SPEED. */
3280 mult_by_coeff_cost (HOST_WIDE_INT coeff
, machine_mode mode
, bool speed
)
3283 struct algorithm algorithm
;
3284 enum mult_variant variant
;
3286 rtx fake_reg
= gen_raw_REG (mode
, LAST_VIRTUAL_REGISTER
+ 1);
3287 max_cost
= set_src_cost (gen_rtx_MULT (mode
, fake_reg
, fake_reg
), speed
);
3288 if (choose_mult_variant (mode
, coeff
, &algorithm
, &variant
, max_cost
))
3289 return algorithm
.cost
.cost
;
3294 /* Perform a widening multiplication and return an rtx for the result.
3295 MODE is mode of value; OP0 and OP1 are what to multiply (rtx's);
3296 TARGET is a suggestion for where to store the result (an rtx).
3297 THIS_OPTAB is the optab we should use, it must be either umul_widen_optab
3298 or smul_widen_optab.
3300 We check specially for a constant integer as OP1, comparing the
3301 cost of a widening multiply against the cost of a sequence of shifts
3305 expand_widening_mult (machine_mode mode
, rtx op0
, rtx op1
, rtx target
,
3306 int unsignedp
, optab this_optab
)
3308 bool speed
= optimize_insn_for_speed_p ();
3311 if (CONST_INT_P (op1
)
3312 && GET_MODE (op0
) != VOIDmode
3313 && (cop1
= convert_modes (mode
, GET_MODE (op0
), op1
,
3314 this_optab
== umul_widen_optab
))
3315 && CONST_INT_P (cop1
)
3316 && (INTVAL (cop1
) >= 0
3317 || HWI_COMPUTABLE_MODE_P (mode
)))
3319 HOST_WIDE_INT coeff
= INTVAL (cop1
);
3321 enum mult_variant variant
;
3322 struct algorithm algorithm
;
3325 return CONST0_RTX (mode
);
3327 /* Special case powers of two. */
3328 if (EXACT_POWER_OF_2_OR_ZERO_P (coeff
))
3330 op0
= convert_to_mode (mode
, op0
, this_optab
== umul_widen_optab
);
3331 return expand_shift (LSHIFT_EXPR
, mode
, op0
,
3332 floor_log2 (coeff
), target
, unsignedp
);
3335 /* Exclude cost of op0 from max_cost to match the cost
3336 calculation of the synth_mult. */
3337 max_cost
= mul_widen_cost (speed
, mode
);
3338 if (choose_mult_variant (mode
, coeff
, &algorithm
, &variant
,
3341 op0
= convert_to_mode (mode
, op0
, this_optab
== umul_widen_optab
);
3342 return expand_mult_const (mode
, op0
, coeff
, target
,
3343 &algorithm
, variant
);
3346 return expand_binop (mode
, this_optab
, op0
, op1
, target
,
3347 unsignedp
, OPTAB_LIB_WIDEN
);
3350 /* Choose a minimal N + 1 bit approximation to 1/D that can be used to
3351 replace division by D, and put the least significant N bits of the result
3352 in *MULTIPLIER_PTR and return the most significant bit.
3354 The width of operations is N (should be <= HOST_BITS_PER_WIDE_INT), the
3355 needed precision is in PRECISION (should be <= N).
3357 PRECISION should be as small as possible so this function can choose
3358 multiplier more freely.
3360 The rounded-up logarithm of D is placed in *lgup_ptr. A shift count that
3361 is to be used for a final right shift is placed in *POST_SHIFT_PTR.
3363 Using this function, x/D will be equal to (x * m) >> (*POST_SHIFT_PTR),
3364 where m is the full HOST_BITS_PER_WIDE_INT + 1 bit multiplier. */
3366 unsigned HOST_WIDE_INT
3367 choose_multiplier (unsigned HOST_WIDE_INT d
, int n
, int precision
,
3368 unsigned HOST_WIDE_INT
*multiplier_ptr
,
3369 int *post_shift_ptr
, int *lgup_ptr
)
3371 int lgup
, post_shift
;
3374 /* lgup = ceil(log2(divisor)); */
3375 lgup
= ceil_log2 (d
);
3377 gcc_assert (lgup
<= n
);
3380 pow2
= n
+ lgup
- precision
;
3382 /* mlow = 2^(N + lgup)/d */
3383 wide_int val
= wi::set_bit_in_zero (pow
, HOST_BITS_PER_DOUBLE_INT
);
3384 wide_int mlow
= wi::udiv_trunc (val
, d
);
3386 /* mhigh = (2^(N + lgup) + 2^(N + lgup - precision))/d */
3387 val
|= wi::set_bit_in_zero (pow2
, HOST_BITS_PER_DOUBLE_INT
);
3388 wide_int mhigh
= wi::udiv_trunc (val
, d
);
3390 /* If precision == N, then mlow, mhigh exceed 2^N
3391 (but they do not exceed 2^(N+1)). */
3393 /* Reduce to lowest terms. */
3394 for (post_shift
= lgup
; post_shift
> 0; post_shift
--)
3396 unsigned HOST_WIDE_INT ml_lo
= wi::extract_uhwi (mlow
, 1,
3397 HOST_BITS_PER_WIDE_INT
);
3398 unsigned HOST_WIDE_INT mh_lo
= wi::extract_uhwi (mhigh
, 1,
3399 HOST_BITS_PER_WIDE_INT
);
3403 mlow
= wi::uhwi (ml_lo
, HOST_BITS_PER_DOUBLE_INT
);
3404 mhigh
= wi::uhwi (mh_lo
, HOST_BITS_PER_DOUBLE_INT
);
3407 *post_shift_ptr
= post_shift
;
3409 if (n
< HOST_BITS_PER_WIDE_INT
)
3411 unsigned HOST_WIDE_INT mask
= ((unsigned HOST_WIDE_INT
) 1 << n
) - 1;
3412 *multiplier_ptr
= mhigh
.to_uhwi () & mask
;
3413 return mhigh
.to_uhwi () >= mask
;
3417 *multiplier_ptr
= mhigh
.to_uhwi ();
3418 return wi::extract_uhwi (mhigh
, HOST_BITS_PER_WIDE_INT
, 1);
3422 /* Compute the inverse of X mod 2**n, i.e., find Y such that X * Y is
3423 congruent to 1 (mod 2**N). */
3425 static unsigned HOST_WIDE_INT
3426 invert_mod2n (unsigned HOST_WIDE_INT x
, int n
)
3428 /* Solve x*y == 1 (mod 2^n), where x is odd. Return y. */
3430 /* The algorithm notes that the choice y = x satisfies
3431 x*y == 1 mod 2^3, since x is assumed odd.
3432 Each iteration doubles the number of bits of significance in y. */
3434 unsigned HOST_WIDE_INT mask
;
3435 unsigned HOST_WIDE_INT y
= x
;
3438 mask
= (n
== HOST_BITS_PER_WIDE_INT
3439 ? ~(unsigned HOST_WIDE_INT
) 0
3440 : ((unsigned HOST_WIDE_INT
) 1 << n
) - 1);
3444 y
= y
* (2 - x
*y
) & mask
; /* Modulo 2^N */
3450 /* Emit code to adjust ADJ_OPERAND after multiplication of wrong signedness
3451 flavor of OP0 and OP1. ADJ_OPERAND is already the high half of the
3452 product OP0 x OP1. If UNSIGNEDP is nonzero, adjust the signed product
3453 to become unsigned, if UNSIGNEDP is zero, adjust the unsigned product to
3456 The result is put in TARGET if that is convenient.
3458 MODE is the mode of operation. */
3461 expand_mult_highpart_adjust (machine_mode mode
, rtx adj_operand
, rtx op0
,
3462 rtx op1
, rtx target
, int unsignedp
)
3465 enum rtx_code adj_code
= unsignedp
? PLUS
: MINUS
;
3467 tem
= expand_shift (RSHIFT_EXPR
, mode
, op0
,
3468 GET_MODE_BITSIZE (mode
) - 1, NULL_RTX
, 0);
3469 tem
= expand_and (mode
, tem
, op1
, NULL_RTX
);
3471 = force_operand (gen_rtx_fmt_ee (adj_code
, mode
, adj_operand
, tem
),
3474 tem
= expand_shift (RSHIFT_EXPR
, mode
, op1
,
3475 GET_MODE_BITSIZE (mode
) - 1, NULL_RTX
, 0);
3476 tem
= expand_and (mode
, tem
, op0
, NULL_RTX
);
3477 target
= force_operand (gen_rtx_fmt_ee (adj_code
, mode
, adj_operand
, tem
),
3483 /* Subroutine of expmed_mult_highpart. Return the MODE high part of OP. */
3486 extract_high_half (machine_mode mode
, rtx op
)
3488 machine_mode wider_mode
;
3490 if (mode
== word_mode
)
3491 return gen_highpart (mode
, op
);
3493 gcc_assert (!SCALAR_FLOAT_MODE_P (mode
));
3495 wider_mode
= GET_MODE_WIDER_MODE (mode
);
3496 op
= expand_shift (RSHIFT_EXPR
, wider_mode
, op
,
3497 GET_MODE_BITSIZE (mode
), 0, 1);
3498 return convert_modes (mode
, wider_mode
, op
, 0);
3501 /* Like expmed_mult_highpart, but only consider using a multiplication
3502 optab. OP1 is an rtx for the constant operand. */
3505 expmed_mult_highpart_optab (machine_mode mode
, rtx op0
, rtx op1
,
3506 rtx target
, int unsignedp
, int max_cost
)
3508 rtx narrow_op1
= gen_int_mode (INTVAL (op1
), mode
);
3509 machine_mode wider_mode
;
3513 bool speed
= optimize_insn_for_speed_p ();
3515 gcc_assert (!SCALAR_FLOAT_MODE_P (mode
));
3517 wider_mode
= GET_MODE_WIDER_MODE (mode
);
3518 size
= GET_MODE_BITSIZE (mode
);
3520 /* Firstly, try using a multiplication insn that only generates the needed
3521 high part of the product, and in the sign flavor of unsignedp. */
3522 if (mul_highpart_cost (speed
, mode
) < max_cost
)
3524 moptab
= unsignedp
? umul_highpart_optab
: smul_highpart_optab
;
3525 tem
= expand_binop (mode
, moptab
, op0
, narrow_op1
, target
,
3526 unsignedp
, OPTAB_DIRECT
);
3531 /* Secondly, same as above, but use sign flavor opposite of unsignedp.
3532 Need to adjust the result after the multiplication. */
3533 if (size
- 1 < BITS_PER_WORD
3534 && (mul_highpart_cost (speed
, mode
)
3535 + 2 * shift_cost (speed
, mode
, size
-1)
3536 + 4 * add_cost (speed
, mode
) < max_cost
))
3538 moptab
= unsignedp
? smul_highpart_optab
: umul_highpart_optab
;
3539 tem
= expand_binop (mode
, moptab
, op0
, narrow_op1
, target
,
3540 unsignedp
, OPTAB_DIRECT
);
3542 /* We used the wrong signedness. Adjust the result. */
3543 return expand_mult_highpart_adjust (mode
, tem
, op0
, narrow_op1
,
3547 /* Try widening multiplication. */
3548 moptab
= unsignedp
? umul_widen_optab
: smul_widen_optab
;
3549 if (widening_optab_handler (moptab
, wider_mode
, mode
) != CODE_FOR_nothing
3550 && mul_widen_cost (speed
, wider_mode
) < max_cost
)
3552 tem
= expand_binop (wider_mode
, moptab
, op0
, narrow_op1
, 0,
3553 unsignedp
, OPTAB_WIDEN
);
3555 return extract_high_half (mode
, tem
);
3558 /* Try widening the mode and perform a non-widening multiplication. */
3559 if (optab_handler (smul_optab
, wider_mode
) != CODE_FOR_nothing
3560 && size
- 1 < BITS_PER_WORD
3561 && (mul_cost (speed
, wider_mode
) + shift_cost (speed
, mode
, size
-1)
3567 /* We need to widen the operands, for example to ensure the
3568 constant multiplier is correctly sign or zero extended.
3569 Use a sequence to clean-up any instructions emitted by
3570 the conversions if things don't work out. */
3572 wop0
= convert_modes (wider_mode
, mode
, op0
, unsignedp
);
3573 wop1
= convert_modes (wider_mode
, mode
, op1
, unsignedp
);
3574 tem
= expand_binop (wider_mode
, smul_optab
, wop0
, wop1
, 0,
3575 unsignedp
, OPTAB_WIDEN
);
3576 insns
= get_insns ();
3582 return extract_high_half (mode
, tem
);
3586 /* Try widening multiplication of opposite signedness, and adjust. */
3587 moptab
= unsignedp
? smul_widen_optab
: umul_widen_optab
;
3588 if (widening_optab_handler (moptab
, wider_mode
, mode
) != CODE_FOR_nothing
3589 && size
- 1 < BITS_PER_WORD
3590 && (mul_widen_cost (speed
, wider_mode
)
3591 + 2 * shift_cost (speed
, mode
, size
-1)
3592 + 4 * add_cost (speed
, mode
) < max_cost
))
3594 tem
= expand_binop (wider_mode
, moptab
, op0
, narrow_op1
,
3595 NULL_RTX
, ! unsignedp
, OPTAB_WIDEN
);
3598 tem
= extract_high_half (mode
, tem
);
3599 /* We used the wrong signedness. Adjust the result. */
3600 return expand_mult_highpart_adjust (mode
, tem
, op0
, narrow_op1
,
3608 /* Emit code to multiply OP0 and OP1 (where OP1 is an integer constant),
3609 putting the high half of the result in TARGET if that is convenient,
3610 and return where the result is. If the operation can not be performed,
3613 MODE is the mode of operation and result.
3615 UNSIGNEDP nonzero means unsigned multiply.
3617 MAX_COST is the total allowed cost for the expanded RTL. */
3620 expmed_mult_highpart (machine_mode mode
, rtx op0
, rtx op1
,
3621 rtx target
, int unsignedp
, int max_cost
)
3623 machine_mode wider_mode
= GET_MODE_WIDER_MODE (mode
);
3624 unsigned HOST_WIDE_INT cnst1
;
3626 bool sign_adjust
= false;
3627 enum mult_variant variant
;
3628 struct algorithm alg
;
3630 bool speed
= optimize_insn_for_speed_p ();
3632 gcc_assert (!SCALAR_FLOAT_MODE_P (mode
));
3633 /* We can't support modes wider than HOST_BITS_PER_INT. */
3634 gcc_assert (HWI_COMPUTABLE_MODE_P (mode
));
3636 cnst1
= INTVAL (op1
) & GET_MODE_MASK (mode
);
3638 /* We can't optimize modes wider than BITS_PER_WORD.
3639 ??? We might be able to perform double-word arithmetic if
3640 mode == word_mode, however all the cost calculations in
3641 synth_mult etc. assume single-word operations. */
3642 if (GET_MODE_BITSIZE (wider_mode
) > BITS_PER_WORD
)
3643 return expmed_mult_highpart_optab (mode
, op0
, op1
, target
,
3644 unsignedp
, max_cost
);
3646 extra_cost
= shift_cost (speed
, mode
, GET_MODE_BITSIZE (mode
) - 1);
3648 /* Check whether we try to multiply by a negative constant. */
3649 if (!unsignedp
&& ((cnst1
>> (GET_MODE_BITSIZE (mode
) - 1)) & 1))
3652 extra_cost
+= add_cost (speed
, mode
);
3655 /* See whether shift/add multiplication is cheap enough. */
3656 if (choose_mult_variant (wider_mode
, cnst1
, &alg
, &variant
,
3657 max_cost
- extra_cost
))
3659 /* See whether the specialized multiplication optabs are
3660 cheaper than the shift/add version. */
3661 tem
= expmed_mult_highpart_optab (mode
, op0
, op1
, target
, unsignedp
,
3662 alg
.cost
.cost
+ extra_cost
);
3666 tem
= convert_to_mode (wider_mode
, op0
, unsignedp
);
3667 tem
= expand_mult_const (wider_mode
, tem
, cnst1
, 0, &alg
, variant
);
3668 tem
= extract_high_half (mode
, tem
);
3670 /* Adjust result for signedness. */
3672 tem
= force_operand (gen_rtx_MINUS (mode
, tem
, op0
), tem
);
3676 return expmed_mult_highpart_optab (mode
, op0
, op1
, target
,
3677 unsignedp
, max_cost
);
3681 /* Expand signed modulus of OP0 by a power of two D in mode MODE. */
3684 expand_smod_pow2 (machine_mode mode
, rtx op0
, HOST_WIDE_INT d
)
3686 rtx result
, temp
, shift
;
3687 rtx_code_label
*label
;
3689 int prec
= GET_MODE_PRECISION (mode
);
3691 logd
= floor_log2 (d
);
3692 result
= gen_reg_rtx (mode
);
3694 /* Avoid conditional branches when they're expensive. */
3695 if (BRANCH_COST (optimize_insn_for_speed_p (), false) >= 2
3696 && optimize_insn_for_speed_p ())
3698 rtx signmask
= emit_store_flag (result
, LT
, op0
, const0_rtx
,
3702 HOST_WIDE_INT masklow
= ((HOST_WIDE_INT
) 1 << logd
) - 1;
3703 signmask
= force_reg (mode
, signmask
);
3704 shift
= GEN_INT (GET_MODE_BITSIZE (mode
) - logd
);
3706 /* Use the rtx_cost of a LSHIFTRT instruction to determine
3707 which instruction sequence to use. If logical right shifts
3708 are expensive the use 2 XORs, 2 SUBs and an AND, otherwise
3709 use a LSHIFTRT, 1 ADD, 1 SUB and an AND. */
3711 temp
= gen_rtx_LSHIFTRT (mode
, result
, shift
);
3712 if (optab_handler (lshr_optab
, mode
) == CODE_FOR_nothing
3713 || (set_src_cost (temp
, optimize_insn_for_speed_p ())
3714 > COSTS_N_INSNS (2)))
3716 temp
= expand_binop (mode
, xor_optab
, op0
, signmask
,
3717 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3718 temp
= expand_binop (mode
, sub_optab
, temp
, signmask
,
3719 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3720 temp
= expand_binop (mode
, and_optab
, temp
,
3721 gen_int_mode (masklow
, mode
),
3722 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3723 temp
= expand_binop (mode
, xor_optab
, temp
, signmask
,
3724 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3725 temp
= expand_binop (mode
, sub_optab
, temp
, signmask
,
3726 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3730 signmask
= expand_binop (mode
, lshr_optab
, signmask
, shift
,
3731 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3732 signmask
= force_reg (mode
, signmask
);
3734 temp
= expand_binop (mode
, add_optab
, op0
, signmask
,
3735 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3736 temp
= expand_binop (mode
, and_optab
, temp
,
3737 gen_int_mode (masklow
, mode
),
3738 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3739 temp
= expand_binop (mode
, sub_optab
, temp
, signmask
,
3740 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
3746 /* Mask contains the mode's signbit and the significant bits of the
3747 modulus. By including the signbit in the operation, many targets
3748 can avoid an explicit compare operation in the following comparison
3750 wide_int mask
= wi::mask (logd
, false, prec
);
3751 mask
= wi::set_bit (mask
, prec
- 1);
3753 temp
= expand_binop (mode
, and_optab
, op0
,
3754 immed_wide_int_const (mask
, mode
),
3755 result
, 1, OPTAB_LIB_WIDEN
);
3757 emit_move_insn (result
, temp
);
3759 label
= gen_label_rtx ();
3760 do_cmp_and_jump (result
, const0_rtx
, GE
, mode
, label
);
3762 temp
= expand_binop (mode
, sub_optab
, result
, const1_rtx
, result
,
3763 0, OPTAB_LIB_WIDEN
);
3765 mask
= wi::mask (logd
, true, prec
);
3766 temp
= expand_binop (mode
, ior_optab
, temp
,
3767 immed_wide_int_const (mask
, mode
),
3768 result
, 1, OPTAB_LIB_WIDEN
);
3769 temp
= expand_binop (mode
, add_optab
, temp
, const1_rtx
, result
,
3770 0, OPTAB_LIB_WIDEN
);
3772 emit_move_insn (result
, temp
);
3777 /* Expand signed division of OP0 by a power of two D in mode MODE.
3778 This routine is only called for positive values of D. */
3781 expand_sdiv_pow2 (machine_mode mode
, rtx op0
, HOST_WIDE_INT d
)
3784 rtx_code_label
*label
;
3787 logd
= floor_log2 (d
);
3790 && BRANCH_COST (optimize_insn_for_speed_p (),
3793 temp
= gen_reg_rtx (mode
);
3794 temp
= emit_store_flag (temp
, LT
, op0
, const0_rtx
, mode
, 0, 1);
3795 temp
= expand_binop (mode
, add_optab
, temp
, op0
, NULL_RTX
,
3796 0, OPTAB_LIB_WIDEN
);
3797 return expand_shift (RSHIFT_EXPR
, mode
, temp
, logd
, NULL_RTX
, 0);
3800 #ifdef HAVE_conditional_move
3801 if (BRANCH_COST (optimize_insn_for_speed_p (), false)
3807 temp2
= copy_to_mode_reg (mode
, op0
);
3808 temp
= expand_binop (mode
, add_optab
, temp2
, gen_int_mode (d
- 1, mode
),
3809 NULL_RTX
, 0, OPTAB_LIB_WIDEN
);
3810 temp
= force_reg (mode
, temp
);
3812 /* Construct "temp2 = (temp2 < 0) ? temp : temp2". */
3813 temp2
= emit_conditional_move (temp2
, LT
, temp2
, const0_rtx
,
3814 mode
, temp
, temp2
, mode
, 0);
3817 rtx_insn
*seq
= get_insns ();
3820 return expand_shift (RSHIFT_EXPR
, mode
, temp2
, logd
, NULL_RTX
, 0);
3826 if (BRANCH_COST (optimize_insn_for_speed_p (),
3829 int ushift
= GET_MODE_BITSIZE (mode
) - logd
;
3831 temp
= gen_reg_rtx (mode
);
3832 temp
= emit_store_flag (temp
, LT
, op0
, const0_rtx
, mode
, 0, -1);
3833 if (GET_MODE_BITSIZE (mode
) >= BITS_PER_WORD
3834 || shift_cost (optimize_insn_for_speed_p (), mode
, ushift
)
3835 > COSTS_N_INSNS (1))
3836 temp
= expand_binop (mode
, and_optab
, temp
, gen_int_mode (d
- 1, mode
),
3837 NULL_RTX
, 0, OPTAB_LIB_WIDEN
);
3839 temp
= expand_shift (RSHIFT_EXPR
, mode
, temp
,
3840 ushift
, NULL_RTX
, 1);
3841 temp
= expand_binop (mode
, add_optab
, temp
, op0
, NULL_RTX
,
3842 0, OPTAB_LIB_WIDEN
);
3843 return expand_shift (RSHIFT_EXPR
, mode
, temp
, logd
, NULL_RTX
, 0);
3846 label
= gen_label_rtx ();
3847 temp
= copy_to_mode_reg (mode
, op0
);
3848 do_cmp_and_jump (temp
, const0_rtx
, GE
, mode
, label
);
3849 expand_inc (temp
, gen_int_mode (d
- 1, mode
));
3851 return expand_shift (RSHIFT_EXPR
, mode
, temp
, logd
, NULL_RTX
, 0);
3854 /* Emit the code to divide OP0 by OP1, putting the result in TARGET
3855 if that is convenient, and returning where the result is.
3856 You may request either the quotient or the remainder as the result;
3857 specify REM_FLAG nonzero to get the remainder.
3859 CODE is the expression code for which kind of division this is;
3860 it controls how rounding is done. MODE is the machine mode to use.
3861 UNSIGNEDP nonzero means do unsigned division. */
3863 /* ??? For CEIL_MOD_EXPR, can compute incorrect remainder with ANDI
3864 and then correct it by or'ing in missing high bits
3865 if result of ANDI is nonzero.
3866 For ROUND_MOD_EXPR, can use ANDI and then sign-extend the result.
3867 This could optimize to a bfexts instruction.
3868 But C doesn't use these operations, so their optimizations are
3870 /* ??? For modulo, we don't actually need the highpart of the first product,
3871 the low part will do nicely. And for small divisors, the second multiply
3872 can also be a low-part only multiply or even be completely left out.
3873 E.g. to calculate the remainder of a division by 3 with a 32 bit
3874 multiply, multiply with 0x55555556 and extract the upper two bits;
3875 the result is exact for inputs up to 0x1fffffff.
3876 The input range can be reduced by using cross-sum rules.
3877 For odd divisors >= 3, the following table gives right shift counts
3878 so that if a number is shifted by an integer multiple of the given
3879 amount, the remainder stays the same:
3880 2, 4, 3, 6, 10, 12, 4, 8, 18, 6, 11, 20, 18, 0, 5, 10, 12, 0, 12, 20,
3881 14, 12, 23, 21, 8, 0, 20, 18, 0, 0, 6, 12, 0, 22, 0, 18, 20, 30, 0, 0,
3882 0, 8, 0, 11, 12, 10, 36, 0, 30, 0, 0, 12, 0, 0, 0, 0, 44, 12, 24, 0,
3883 20, 0, 7, 14, 0, 18, 36, 0, 0, 46, 60, 0, 42, 0, 15, 24, 20, 0, 0, 33,
3884 0, 20, 0, 0, 18, 0, 60, 0, 0, 0, 0, 0, 40, 18, 0, 0, 12
3886 Cross-sum rules for even numbers can be derived by leaving as many bits
3887 to the right alone as the divisor has zeros to the right.
3888 E.g. if x is an unsigned 32 bit number:
3889 (x mod 12) == (((x & 1023) + ((x >> 8) & ~3)) * 0x15555558 >> 2 * 3) >> 28
3893 expand_divmod (int rem_flag
, enum tree_code code
, machine_mode mode
,
3894 rtx op0
, rtx op1
, rtx target
, int unsignedp
)
3896 machine_mode compute_mode
;
3898 rtx quotient
= 0, remainder
= 0;
3902 optab optab1
, optab2
;
3903 int op1_is_constant
, op1_is_pow2
= 0;
3904 int max_cost
, extra_cost
;
3905 static HOST_WIDE_INT last_div_const
= 0;
3906 bool speed
= optimize_insn_for_speed_p ();
3908 op1_is_constant
= CONST_INT_P (op1
);
3909 if (op1_is_constant
)
3911 unsigned HOST_WIDE_INT ext_op1
= UINTVAL (op1
);
3913 ext_op1
&= GET_MODE_MASK (mode
);
3914 op1_is_pow2
= ((EXACT_POWER_OF_2_OR_ZERO_P (ext_op1
)
3915 || (! unsignedp
&& EXACT_POWER_OF_2_OR_ZERO_P (-ext_op1
))));
3919 This is the structure of expand_divmod:
3921 First comes code to fix up the operands so we can perform the operations
3922 correctly and efficiently.
3924 Second comes a switch statement with code specific for each rounding mode.
3925 For some special operands this code emits all RTL for the desired
3926 operation, for other cases, it generates only a quotient and stores it in
3927 QUOTIENT. The case for trunc division/remainder might leave quotient = 0,
3928 to indicate that it has not done anything.
3930 Last comes code that finishes the operation. If QUOTIENT is set and
3931 REM_FLAG is set, the remainder is computed as OP0 - QUOTIENT * OP1. If
3932 QUOTIENT is not set, it is computed using trunc rounding.
3934 We try to generate special code for division and remainder when OP1 is a
3935 constant. If |OP1| = 2**n we can use shifts and some other fast
3936 operations. For other values of OP1, we compute a carefully selected
3937 fixed-point approximation m = 1/OP1, and generate code that multiplies OP0
3940 In all cases but EXACT_DIV_EXPR, this multiplication requires the upper
3941 half of the product. Different strategies for generating the product are
3942 implemented in expmed_mult_highpart.
3944 If what we actually want is the remainder, we generate that by another
3945 by-constant multiplication and a subtraction. */
3947 /* We shouldn't be called with OP1 == const1_rtx, but some of the
3948 code below will malfunction if we are, so check here and handle
3949 the special case if so. */
3950 if (op1
== const1_rtx
)
3951 return rem_flag
? const0_rtx
: op0
;
3953 /* When dividing by -1, we could get an overflow.
3954 negv_optab can handle overflows. */
3955 if (! unsignedp
&& op1
== constm1_rtx
)
3959 return expand_unop (mode
, flag_trapv
&& GET_MODE_CLASS (mode
) == MODE_INT
3960 ? negv_optab
: neg_optab
, op0
, target
, 0);
3964 /* Don't use the function value register as a target
3965 since we have to read it as well as write it,
3966 and function-inlining gets confused by this. */
3967 && ((REG_P (target
) && REG_FUNCTION_VALUE_P (target
))
3968 /* Don't clobber an operand while doing a multi-step calculation. */
3969 || ((rem_flag
|| op1_is_constant
)
3970 && (reg_mentioned_p (target
, op0
)
3971 || (MEM_P (op0
) && MEM_P (target
))))
3972 || reg_mentioned_p (target
, op1
)
3973 || (MEM_P (op1
) && MEM_P (target
))))
3976 /* Get the mode in which to perform this computation. Normally it will
3977 be MODE, but sometimes we can't do the desired operation in MODE.
3978 If so, pick a wider mode in which we can do the operation. Convert
3979 to that mode at the start to avoid repeated conversions.
3981 First see what operations we need. These depend on the expression
3982 we are evaluating. (We assume that divxx3 insns exist under the
3983 same conditions that modxx3 insns and that these insns don't normally
3984 fail. If these assumptions are not correct, we may generate less
3985 efficient code in some cases.)
3987 Then see if we find a mode in which we can open-code that operation
3988 (either a division, modulus, or shift). Finally, check for the smallest
3989 mode for which we can do the operation with a library call. */
3991 /* We might want to refine this now that we have division-by-constant
3992 optimization. Since expmed_mult_highpart tries so many variants, it is
3993 not straightforward to generalize this. Maybe we should make an array
3994 of possible modes in init_expmed? Save this for GCC 2.7. */
3996 optab1
= ((op1_is_pow2
&& op1
!= const0_rtx
)
3997 ? (unsignedp
? lshr_optab
: ashr_optab
)
3998 : (unsignedp
? udiv_optab
: sdiv_optab
));
3999 optab2
= ((op1_is_pow2
&& op1
!= const0_rtx
)
4001 : (unsignedp
? udivmod_optab
: sdivmod_optab
));
4003 for (compute_mode
= mode
; compute_mode
!= VOIDmode
;
4004 compute_mode
= GET_MODE_WIDER_MODE (compute_mode
))
4005 if (optab_handler (optab1
, compute_mode
) != CODE_FOR_nothing
4006 || optab_handler (optab2
, compute_mode
) != CODE_FOR_nothing
)
4009 if (compute_mode
== VOIDmode
)
4010 for (compute_mode
= mode
; compute_mode
!= VOIDmode
;
4011 compute_mode
= GET_MODE_WIDER_MODE (compute_mode
))
4012 if (optab_libfunc (optab1
, compute_mode
)
4013 || optab_libfunc (optab2
, compute_mode
))
4016 /* If we still couldn't find a mode, use MODE, but expand_binop will
4018 if (compute_mode
== VOIDmode
)
4019 compute_mode
= mode
;
4021 if (target
&& GET_MODE (target
) == compute_mode
)
4024 tquotient
= gen_reg_rtx (compute_mode
);
4026 size
= GET_MODE_BITSIZE (compute_mode
);
4028 /* It should be possible to restrict the precision to GET_MODE_BITSIZE
4029 (mode), and thereby get better code when OP1 is a constant. Do that
4030 later. It will require going over all usages of SIZE below. */
4031 size
= GET_MODE_BITSIZE (mode
);
4034 /* Only deduct something for a REM if the last divide done was
4035 for a different constant. Then set the constant of the last
4037 max_cost
= (unsignedp
4038 ? udiv_cost (speed
, compute_mode
)
4039 : sdiv_cost (speed
, compute_mode
));
4040 if (rem_flag
&& ! (last_div_const
!= 0 && op1_is_constant
4041 && INTVAL (op1
) == last_div_const
))
4042 max_cost
-= (mul_cost (speed
, compute_mode
)
4043 + add_cost (speed
, compute_mode
));
4045 last_div_const
= ! rem_flag
&& op1_is_constant
? INTVAL (op1
) : 0;
4047 /* Now convert to the best mode to use. */
4048 if (compute_mode
!= mode
)
4050 op0
= convert_modes (compute_mode
, mode
, op0
, unsignedp
);
4051 op1
= convert_modes (compute_mode
, mode
, op1
, unsignedp
);
4053 /* convert_modes may have placed op1 into a register, so we
4054 must recompute the following. */
4055 op1_is_constant
= CONST_INT_P (op1
);
4056 op1_is_pow2
= (op1_is_constant
4057 && ((EXACT_POWER_OF_2_OR_ZERO_P (INTVAL (op1
))
4059 && EXACT_POWER_OF_2_OR_ZERO_P (-UINTVAL (op1
))))));
4062 /* If one of the operands is a volatile MEM, copy it into a register. */
4064 if (MEM_P (op0
) && MEM_VOLATILE_P (op0
))
4065 op0
= force_reg (compute_mode
, op0
);
4066 if (MEM_P (op1
) && MEM_VOLATILE_P (op1
))
4067 op1
= force_reg (compute_mode
, op1
);
4069 /* If we need the remainder or if OP1 is constant, we need to
4070 put OP0 in a register in case it has any queued subexpressions. */
4071 if (rem_flag
|| op1_is_constant
)
4072 op0
= force_reg (compute_mode
, op0
);
4074 last
= get_last_insn ();
4076 /* Promote floor rounding to trunc rounding for unsigned operations. */
4079 if (code
== FLOOR_DIV_EXPR
)
4080 code
= TRUNC_DIV_EXPR
;
4081 if (code
== FLOOR_MOD_EXPR
)
4082 code
= TRUNC_MOD_EXPR
;
4083 if (code
== EXACT_DIV_EXPR
&& op1_is_pow2
)
4084 code
= TRUNC_DIV_EXPR
;
4087 if (op1
!= const0_rtx
)
4090 case TRUNC_MOD_EXPR
:
4091 case TRUNC_DIV_EXPR
:
4092 if (op1_is_constant
)
4096 unsigned HOST_WIDE_INT mh
, ml
;
4097 int pre_shift
, post_shift
;
4099 unsigned HOST_WIDE_INT d
= (INTVAL (op1
)
4100 & GET_MODE_MASK (compute_mode
));
4102 if (EXACT_POWER_OF_2_OR_ZERO_P (d
))
4104 pre_shift
= floor_log2 (d
);
4107 unsigned HOST_WIDE_INT mask
4108 = ((unsigned HOST_WIDE_INT
) 1 << pre_shift
) - 1;
4110 = expand_binop (compute_mode
, and_optab
, op0
,
4111 gen_int_mode (mask
, compute_mode
),
4115 return gen_lowpart (mode
, remainder
);
4117 quotient
= expand_shift (RSHIFT_EXPR
, compute_mode
, op0
,
4118 pre_shift
, tquotient
, 1);
4120 else if (size
<= HOST_BITS_PER_WIDE_INT
)
4122 if (d
>= ((unsigned HOST_WIDE_INT
) 1 << (size
- 1)))
4124 /* Most significant bit of divisor is set; emit an scc
4126 quotient
= emit_store_flag_force (tquotient
, GEU
, op0
, op1
,
4127 compute_mode
, 1, 1);
4131 /* Find a suitable multiplier and right shift count
4132 instead of multiplying with D. */
4134 mh
= choose_multiplier (d
, size
, size
,
4135 &ml
, &post_shift
, &dummy
);
4137 /* If the suggested multiplier is more than SIZE bits,
4138 we can do better for even divisors, using an
4139 initial right shift. */
4140 if (mh
!= 0 && (d
& 1) == 0)
4142 pre_shift
= floor_log2 (d
& -d
);
4143 mh
= choose_multiplier (d
>> pre_shift
, size
,
4145 &ml
, &post_shift
, &dummy
);
4155 if (post_shift
- 1 >= BITS_PER_WORD
)
4159 = (shift_cost (speed
, compute_mode
, post_shift
- 1)
4160 + shift_cost (speed
, compute_mode
, 1)
4161 + 2 * add_cost (speed
, compute_mode
));
4162 t1
= expmed_mult_highpart
4164 gen_int_mode (ml
, compute_mode
),
4165 NULL_RTX
, 1, max_cost
- extra_cost
);
4168 t2
= force_operand (gen_rtx_MINUS (compute_mode
,
4171 t3
= expand_shift (RSHIFT_EXPR
, compute_mode
,
4172 t2
, 1, NULL_RTX
, 1);
4173 t4
= force_operand (gen_rtx_PLUS (compute_mode
,
4176 quotient
= expand_shift
4177 (RSHIFT_EXPR
, compute_mode
, t4
,
4178 post_shift
- 1, tquotient
, 1);
4184 if (pre_shift
>= BITS_PER_WORD
4185 || post_shift
>= BITS_PER_WORD
)
4189 (RSHIFT_EXPR
, compute_mode
, op0
,
4190 pre_shift
, NULL_RTX
, 1);
4192 = (shift_cost (speed
, compute_mode
, pre_shift
)
4193 + shift_cost (speed
, compute_mode
, post_shift
));
4194 t2
= expmed_mult_highpart
4196 gen_int_mode (ml
, compute_mode
),
4197 NULL_RTX
, 1, max_cost
- extra_cost
);
4200 quotient
= expand_shift
4201 (RSHIFT_EXPR
, compute_mode
, t2
,
4202 post_shift
, tquotient
, 1);
4206 else /* Too wide mode to use tricky code */
4209 insn
= get_last_insn ();
4211 set_dst_reg_note (insn
, REG_EQUAL
,
4212 gen_rtx_UDIV (compute_mode
, op0
, op1
),
4215 else /* TRUNC_DIV, signed */
4217 unsigned HOST_WIDE_INT ml
;
4218 int lgup
, post_shift
;
4220 HOST_WIDE_INT d
= INTVAL (op1
);
4221 unsigned HOST_WIDE_INT abs_d
;
4223 /* Since d might be INT_MIN, we have to cast to
4224 unsigned HOST_WIDE_INT before negating to avoid
4225 undefined signed overflow. */
4227 ? (unsigned HOST_WIDE_INT
) d
4228 : - (unsigned HOST_WIDE_INT
) d
);
4230 /* n rem d = n rem -d */
4231 if (rem_flag
&& d
< 0)
4234 op1
= gen_int_mode (abs_d
, compute_mode
);
4240 quotient
= expand_unop (compute_mode
, neg_optab
, op0
,
4242 else if (HOST_BITS_PER_WIDE_INT
>= size
4243 && abs_d
== (unsigned HOST_WIDE_INT
) 1 << (size
- 1))
4245 /* This case is not handled correctly below. */
4246 quotient
= emit_store_flag (tquotient
, EQ
, op0
, op1
,
4247 compute_mode
, 1, 1);
4251 else if (EXACT_POWER_OF_2_OR_ZERO_P (d
)
4253 ? smod_pow2_cheap (speed
, compute_mode
)
4254 : sdiv_pow2_cheap (speed
, compute_mode
))
4255 /* We assume that cheap metric is true if the
4256 optab has an expander for this mode. */
4257 && ((optab_handler ((rem_flag
? smod_optab
4260 != CODE_FOR_nothing
)
4261 || (optab_handler (sdivmod_optab
,
4263 != CODE_FOR_nothing
)))
4265 else if (EXACT_POWER_OF_2_OR_ZERO_P (abs_d
))
4269 remainder
= expand_smod_pow2 (compute_mode
, op0
, d
);
4271 return gen_lowpart (mode
, remainder
);
4274 if (sdiv_pow2_cheap (speed
, compute_mode
)
4275 && ((optab_handler (sdiv_optab
, compute_mode
)
4276 != CODE_FOR_nothing
)
4277 || (optab_handler (sdivmod_optab
, compute_mode
)
4278 != CODE_FOR_nothing
)))
4279 quotient
= expand_divmod (0, TRUNC_DIV_EXPR
,
4281 gen_int_mode (abs_d
,
4285 quotient
= expand_sdiv_pow2 (compute_mode
, op0
, abs_d
);
4287 /* We have computed OP0 / abs(OP1). If OP1 is negative,
4288 negate the quotient. */
4291 insn
= get_last_insn ();
4293 && abs_d
< ((unsigned HOST_WIDE_INT
) 1
4294 << (HOST_BITS_PER_WIDE_INT
- 1)))
4295 set_dst_reg_note (insn
, REG_EQUAL
,
4296 gen_rtx_DIV (compute_mode
, op0
,
4302 quotient
= expand_unop (compute_mode
, neg_optab
,
4303 quotient
, quotient
, 0);
4306 else if (size
<= HOST_BITS_PER_WIDE_INT
)
4308 choose_multiplier (abs_d
, size
, size
- 1,
4309 &ml
, &post_shift
, &lgup
);
4310 if (ml
< (unsigned HOST_WIDE_INT
) 1 << (size
- 1))
4314 if (post_shift
>= BITS_PER_WORD
4315 || size
- 1 >= BITS_PER_WORD
)
4318 extra_cost
= (shift_cost (speed
, compute_mode
, post_shift
)
4319 + shift_cost (speed
, compute_mode
, size
- 1)
4320 + add_cost (speed
, compute_mode
));
4321 t1
= expmed_mult_highpart
4322 (compute_mode
, op0
, gen_int_mode (ml
, compute_mode
),
4323 NULL_RTX
, 0, max_cost
- extra_cost
);
4327 (RSHIFT_EXPR
, compute_mode
, t1
,
4328 post_shift
, NULL_RTX
, 0);
4330 (RSHIFT_EXPR
, compute_mode
, op0
,
4331 size
- 1, NULL_RTX
, 0);
4334 = force_operand (gen_rtx_MINUS (compute_mode
,
4339 = force_operand (gen_rtx_MINUS (compute_mode
,
4347 if (post_shift
>= BITS_PER_WORD
4348 || size
- 1 >= BITS_PER_WORD
)
4351 ml
|= (~(unsigned HOST_WIDE_INT
) 0) << (size
- 1);
4352 mlr
= gen_int_mode (ml
, compute_mode
);
4353 extra_cost
= (shift_cost (speed
, compute_mode
, post_shift
)
4354 + shift_cost (speed
, compute_mode
, size
- 1)
4355 + 2 * add_cost (speed
, compute_mode
));
4356 t1
= expmed_mult_highpart (compute_mode
, op0
, mlr
,
4358 max_cost
- extra_cost
);
4361 t2
= force_operand (gen_rtx_PLUS (compute_mode
,
4365 (RSHIFT_EXPR
, compute_mode
, t2
,
4366 post_shift
, NULL_RTX
, 0);
4368 (RSHIFT_EXPR
, compute_mode
, op0
,
4369 size
- 1, NULL_RTX
, 0);
4372 = force_operand (gen_rtx_MINUS (compute_mode
,
4377 = force_operand (gen_rtx_MINUS (compute_mode
,
4382 else /* Too wide mode to use tricky code */
4385 insn
= get_last_insn ();
4387 set_dst_reg_note (insn
, REG_EQUAL
,
4388 gen_rtx_DIV (compute_mode
, op0
, op1
),
4394 delete_insns_since (last
);
4397 case FLOOR_DIV_EXPR
:
4398 case FLOOR_MOD_EXPR
:
4399 /* We will come here only for signed operations. */
4400 if (op1_is_constant
&& HOST_BITS_PER_WIDE_INT
>= size
)
4402 unsigned HOST_WIDE_INT mh
, ml
;
4403 int pre_shift
, lgup
, post_shift
;
4404 HOST_WIDE_INT d
= INTVAL (op1
);
4408 /* We could just as easily deal with negative constants here,
4409 but it does not seem worth the trouble for GCC 2.6. */
4410 if (EXACT_POWER_OF_2_OR_ZERO_P (d
))
4412 pre_shift
= floor_log2 (d
);
4415 unsigned HOST_WIDE_INT mask
4416 = ((unsigned HOST_WIDE_INT
) 1 << pre_shift
) - 1;
4417 remainder
= expand_binop
4418 (compute_mode
, and_optab
, op0
,
4419 gen_int_mode (mask
, compute_mode
),
4420 remainder
, 0, OPTAB_LIB_WIDEN
);
4422 return gen_lowpart (mode
, remainder
);
4424 quotient
= expand_shift
4425 (RSHIFT_EXPR
, compute_mode
, op0
,
4426 pre_shift
, tquotient
, 0);
4432 mh
= choose_multiplier (d
, size
, size
- 1,
4433 &ml
, &post_shift
, &lgup
);
4436 if (post_shift
< BITS_PER_WORD
4437 && size
- 1 < BITS_PER_WORD
)
4440 (RSHIFT_EXPR
, compute_mode
, op0
,
4441 size
- 1, NULL_RTX
, 0);
4442 t2
= expand_binop (compute_mode
, xor_optab
, op0
, t1
,
4443 NULL_RTX
, 0, OPTAB_WIDEN
);
4444 extra_cost
= (shift_cost (speed
, compute_mode
, post_shift
)
4445 + shift_cost (speed
, compute_mode
, size
- 1)
4446 + 2 * add_cost (speed
, compute_mode
));
4447 t3
= expmed_mult_highpart
4448 (compute_mode
, t2
, gen_int_mode (ml
, compute_mode
),
4449 NULL_RTX
, 1, max_cost
- extra_cost
);
4453 (RSHIFT_EXPR
, compute_mode
, t3
,
4454 post_shift
, NULL_RTX
, 1);
4455 quotient
= expand_binop (compute_mode
, xor_optab
,
4456 t4
, t1
, tquotient
, 0,
4464 rtx nsign
, t1
, t2
, t3
, t4
;
4465 t1
= force_operand (gen_rtx_PLUS (compute_mode
,
4466 op0
, constm1_rtx
), NULL_RTX
);
4467 t2
= expand_binop (compute_mode
, ior_optab
, op0
, t1
, NULL_RTX
,
4469 nsign
= expand_shift
4470 (RSHIFT_EXPR
, compute_mode
, t2
,
4471 size
- 1, NULL_RTX
, 0);
4472 t3
= force_operand (gen_rtx_MINUS (compute_mode
, t1
, nsign
),
4474 t4
= expand_divmod (0, TRUNC_DIV_EXPR
, compute_mode
, t3
, op1
,
4479 t5
= expand_unop (compute_mode
, one_cmpl_optab
, nsign
,
4481 quotient
= force_operand (gen_rtx_PLUS (compute_mode
,
4490 delete_insns_since (last
);
4492 /* Try using an instruction that produces both the quotient and
4493 remainder, using truncation. We can easily compensate the quotient
4494 or remainder to get floor rounding, once we have the remainder.
4495 Notice that we compute also the final remainder value here,
4496 and return the result right away. */
4497 if (target
== 0 || GET_MODE (target
) != compute_mode
)
4498 target
= gen_reg_rtx (compute_mode
);
4503 = REG_P (target
) ? target
: gen_reg_rtx (compute_mode
);
4504 quotient
= gen_reg_rtx (compute_mode
);
4509 = REG_P (target
) ? target
: gen_reg_rtx (compute_mode
);
4510 remainder
= gen_reg_rtx (compute_mode
);
4513 if (expand_twoval_binop (sdivmod_optab
, op0
, op1
,
4514 quotient
, remainder
, 0))
4516 /* This could be computed with a branch-less sequence.
4517 Save that for later. */
4519 rtx_code_label
*label
= gen_label_rtx ();
4520 do_cmp_and_jump (remainder
, const0_rtx
, EQ
, compute_mode
, label
);
4521 tem
= expand_binop (compute_mode
, xor_optab
, op0
, op1
,
4522 NULL_RTX
, 0, OPTAB_WIDEN
);
4523 do_cmp_and_jump (tem
, const0_rtx
, GE
, compute_mode
, label
);
4524 expand_dec (quotient
, const1_rtx
);
4525 expand_inc (remainder
, op1
);
4527 return gen_lowpart (mode
, rem_flag
? remainder
: quotient
);
4530 /* No luck with division elimination or divmod. Have to do it
4531 by conditionally adjusting op0 *and* the result. */
4533 rtx_code_label
*label1
, *label2
, *label3
, *label4
, *label5
;
4537 quotient
= gen_reg_rtx (compute_mode
);
4538 adjusted_op0
= copy_to_mode_reg (compute_mode
, op0
);
4539 label1
= gen_label_rtx ();
4540 label2
= gen_label_rtx ();
4541 label3
= gen_label_rtx ();
4542 label4
= gen_label_rtx ();
4543 label5
= gen_label_rtx ();
4544 do_cmp_and_jump (op1
, const0_rtx
, LT
, compute_mode
, label2
);
4545 do_cmp_and_jump (adjusted_op0
, const0_rtx
, LT
, compute_mode
, label1
);
4546 tem
= expand_binop (compute_mode
, sdiv_optab
, adjusted_op0
, op1
,
4547 quotient
, 0, OPTAB_LIB_WIDEN
);
4548 if (tem
!= quotient
)
4549 emit_move_insn (quotient
, tem
);
4550 emit_jump_insn (gen_jump (label5
));
4552 emit_label (label1
);
4553 expand_inc (adjusted_op0
, const1_rtx
);
4554 emit_jump_insn (gen_jump (label4
));
4556 emit_label (label2
);
4557 do_cmp_and_jump (adjusted_op0
, const0_rtx
, GT
, compute_mode
, label3
);
4558 tem
= expand_binop (compute_mode
, sdiv_optab
, adjusted_op0
, op1
,
4559 quotient
, 0, OPTAB_LIB_WIDEN
);
4560 if (tem
!= quotient
)
4561 emit_move_insn (quotient
, tem
);
4562 emit_jump_insn (gen_jump (label5
));
4564 emit_label (label3
);
4565 expand_dec (adjusted_op0
, const1_rtx
);
4566 emit_label (label4
);
4567 tem
= expand_binop (compute_mode
, sdiv_optab
, adjusted_op0
, op1
,
4568 quotient
, 0, OPTAB_LIB_WIDEN
);
4569 if (tem
!= quotient
)
4570 emit_move_insn (quotient
, tem
);
4571 expand_dec (quotient
, const1_rtx
);
4572 emit_label (label5
);
4580 if (op1_is_constant
&& EXACT_POWER_OF_2_OR_ZERO_P (INTVAL (op1
)))
4583 unsigned HOST_WIDE_INT d
= INTVAL (op1
);
4584 t1
= expand_shift (RSHIFT_EXPR
, compute_mode
, op0
,
4585 floor_log2 (d
), tquotient
, 1);
4586 t2
= expand_binop (compute_mode
, and_optab
, op0
,
4587 gen_int_mode (d
- 1, compute_mode
),
4588 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4589 t3
= gen_reg_rtx (compute_mode
);
4590 t3
= emit_store_flag (t3
, NE
, t2
, const0_rtx
,
4591 compute_mode
, 1, 1);
4594 rtx_code_label
*lab
;
4595 lab
= gen_label_rtx ();
4596 do_cmp_and_jump (t2
, const0_rtx
, EQ
, compute_mode
, lab
);
4597 expand_inc (t1
, const1_rtx
);
4602 quotient
= force_operand (gen_rtx_PLUS (compute_mode
,
4608 /* Try using an instruction that produces both the quotient and
4609 remainder, using truncation. We can easily compensate the
4610 quotient or remainder to get ceiling rounding, once we have the
4611 remainder. Notice that we compute also the final remainder
4612 value here, and return the result right away. */
4613 if (target
== 0 || GET_MODE (target
) != compute_mode
)
4614 target
= gen_reg_rtx (compute_mode
);
4618 remainder
= (REG_P (target
)
4619 ? target
: gen_reg_rtx (compute_mode
));
4620 quotient
= gen_reg_rtx (compute_mode
);
4624 quotient
= (REG_P (target
)
4625 ? target
: gen_reg_rtx (compute_mode
));
4626 remainder
= gen_reg_rtx (compute_mode
);
4629 if (expand_twoval_binop (udivmod_optab
, op0
, op1
, quotient
,
4632 /* This could be computed with a branch-less sequence.
4633 Save that for later. */
4634 rtx_code_label
*label
= gen_label_rtx ();
4635 do_cmp_and_jump (remainder
, const0_rtx
, EQ
,
4636 compute_mode
, label
);
4637 expand_inc (quotient
, const1_rtx
);
4638 expand_dec (remainder
, op1
);
4640 return gen_lowpart (mode
, rem_flag
? remainder
: quotient
);
4643 /* No luck with division elimination or divmod. Have to do it
4644 by conditionally adjusting op0 *and* the result. */
4646 rtx_code_label
*label1
, *label2
;
4647 rtx adjusted_op0
, tem
;
4649 quotient
= gen_reg_rtx (compute_mode
);
4650 adjusted_op0
= copy_to_mode_reg (compute_mode
, op0
);
4651 label1
= gen_label_rtx ();
4652 label2
= gen_label_rtx ();
4653 do_cmp_and_jump (adjusted_op0
, const0_rtx
, NE
,
4654 compute_mode
, label1
);
4655 emit_move_insn (quotient
, const0_rtx
);
4656 emit_jump_insn (gen_jump (label2
));
4658 emit_label (label1
);
4659 expand_dec (adjusted_op0
, const1_rtx
);
4660 tem
= expand_binop (compute_mode
, udiv_optab
, adjusted_op0
, op1
,
4661 quotient
, 1, OPTAB_LIB_WIDEN
);
4662 if (tem
!= quotient
)
4663 emit_move_insn (quotient
, tem
);
4664 expand_inc (quotient
, const1_rtx
);
4665 emit_label (label2
);
4670 if (op1_is_constant
&& EXACT_POWER_OF_2_OR_ZERO_P (INTVAL (op1
))
4671 && INTVAL (op1
) >= 0)
4673 /* This is extremely similar to the code for the unsigned case
4674 above. For 2.7 we should merge these variants, but for
4675 2.6.1 I don't want to touch the code for unsigned since that
4676 get used in C. The signed case will only be used by other
4680 unsigned HOST_WIDE_INT d
= INTVAL (op1
);
4681 t1
= expand_shift (RSHIFT_EXPR
, compute_mode
, op0
,
4682 floor_log2 (d
), tquotient
, 0);
4683 t2
= expand_binop (compute_mode
, and_optab
, op0
,
4684 gen_int_mode (d
- 1, compute_mode
),
4685 NULL_RTX
, 1, OPTAB_LIB_WIDEN
);
4686 t3
= gen_reg_rtx (compute_mode
);
4687 t3
= emit_store_flag (t3
, NE
, t2
, const0_rtx
,
4688 compute_mode
, 1, 1);
4691 rtx_code_label
*lab
;
4692 lab
= gen_label_rtx ();
4693 do_cmp_and_jump (t2
, const0_rtx
, EQ
, compute_mode
, lab
);
4694 expand_inc (t1
, const1_rtx
);
4699 quotient
= force_operand (gen_rtx_PLUS (compute_mode
,
4705 /* Try using an instruction that produces both the quotient and
4706 remainder, using truncation. We can easily compensate the
4707 quotient or remainder to get ceiling rounding, once we have the
4708 remainder. Notice that we compute also the final remainder
4709 value here, and return the result right away. */
4710 if (target
== 0 || GET_MODE (target
) != compute_mode
)
4711 target
= gen_reg_rtx (compute_mode
);
4714 remainder
= (REG_P (target
)
4715 ? target
: gen_reg_rtx (compute_mode
));
4716 quotient
= gen_reg_rtx (compute_mode
);
4720 quotient
= (REG_P (target
)
4721 ? target
: gen_reg_rtx (compute_mode
));
4722 remainder
= gen_reg_rtx (compute_mode
);
4725 if (expand_twoval_binop (sdivmod_optab
, op0
, op1
, quotient
,
4728 /* This could be computed with a branch-less sequence.
4729 Save that for later. */
4731 rtx_code_label
*label
= gen_label_rtx ();
4732 do_cmp_and_jump (remainder
, const0_rtx
, EQ
,
4733 compute_mode
, label
);
4734 tem
= expand_binop (compute_mode
, xor_optab
, op0
, op1
,
4735 NULL_RTX
, 0, OPTAB_WIDEN
);
4736 do_cmp_and_jump (tem
, const0_rtx
, LT
, compute_mode
, label
);
4737 expand_inc (quotient
, const1_rtx
);
4738 expand_dec (remainder
, op1
);
4740 return gen_lowpart (mode
, rem_flag
? remainder
: quotient
);
4743 /* No luck with division elimination or divmod. Have to do it
4744 by conditionally adjusting op0 *and* the result. */
4746 rtx_code_label
*label1
, *label2
, *label3
, *label4
, *label5
;
4750 quotient
= gen_reg_rtx (compute_mode
);
4751 adjusted_op0
= copy_to_mode_reg (compute_mode
, op0
);
4752 label1
= gen_label_rtx ();
4753 label2
= gen_label_rtx ();
4754 label3
= gen_label_rtx ();
4755 label4
= gen_label_rtx ();
4756 label5
= gen_label_rtx ();
4757 do_cmp_and_jump (op1
, const0_rtx
, LT
, compute_mode
, label2
);
4758 do_cmp_and_jump (adjusted_op0
, const0_rtx
, GT
,
4759 compute_mode
, label1
);
4760 tem
= expand_binop (compute_mode
, sdiv_optab
, adjusted_op0
, op1
,
4761 quotient
, 0, OPTAB_LIB_WIDEN
);
4762 if (tem
!= quotient
)
4763 emit_move_insn (quotient
, tem
);
4764 emit_jump_insn (gen_jump (label5
));
4766 emit_label (label1
);
4767 expand_dec (adjusted_op0
, const1_rtx
);
4768 emit_jump_insn (gen_jump (label4
));
4770 emit_label (label2
);
4771 do_cmp_and_jump (adjusted_op0
, const0_rtx
, LT
,
4772 compute_mode
, label3
);
4773 tem
= expand_binop (compute_mode
, sdiv_optab
, adjusted_op0
, op1
,
4774 quotient
, 0, OPTAB_LIB_WIDEN
);
4775 if (tem
!= quotient
)
4776 emit_move_insn (quotient
, tem
);
4777 emit_jump_insn (gen_jump (label5
));
4779 emit_label (label3
);
4780 expand_inc (adjusted_op0
, const1_rtx
);
4781 emit_label (label4
);
4782 tem
= expand_binop (compute_mode
, sdiv_optab
, adjusted_op0
, op1
,
4783 quotient
, 0, OPTAB_LIB_WIDEN
);
4784 if (tem
!= quotient
)
4785 emit_move_insn (quotient
, tem
);
4786 expand_inc (quotient
, const1_rtx
);
4787 emit_label (label5
);
4792 case EXACT_DIV_EXPR
:
4793 if (op1_is_constant
&& HOST_BITS_PER_WIDE_INT
>= size
)
4795 HOST_WIDE_INT d
= INTVAL (op1
);
4796 unsigned HOST_WIDE_INT ml
;
4800 pre_shift
= floor_log2 (d
& -d
);
4801 ml
= invert_mod2n (d
>> pre_shift
, size
);
4802 t1
= expand_shift (RSHIFT_EXPR
, compute_mode
, op0
,
4803 pre_shift
, NULL_RTX
, unsignedp
);
4804 quotient
= expand_mult (compute_mode
, t1
,
4805 gen_int_mode (ml
, compute_mode
),
4808 insn
= get_last_insn ();
4809 set_dst_reg_note (insn
, REG_EQUAL
,
4810 gen_rtx_fmt_ee (unsignedp
? UDIV
: DIV
,
4811 compute_mode
, op0
, op1
),
4816 case ROUND_DIV_EXPR
:
4817 case ROUND_MOD_EXPR
:
4821 rtx_code_label
*label
;
4822 label
= gen_label_rtx ();
4823 quotient
= gen_reg_rtx (compute_mode
);
4824 remainder
= gen_reg_rtx (compute_mode
);
4825 if (expand_twoval_binop (udivmod_optab
, op0
, op1
, quotient
, remainder
, 1) == 0)
4828 quotient
= expand_binop (compute_mode
, udiv_optab
, op0
, op1
,
4829 quotient
, 1, OPTAB_LIB_WIDEN
);
4830 tem
= expand_mult (compute_mode
, quotient
, op1
, NULL_RTX
, 1);
4831 remainder
= expand_binop (compute_mode
, sub_optab
, op0
, tem
,
4832 remainder
, 1, OPTAB_LIB_WIDEN
);
4834 tem
= plus_constant (compute_mode
, op1
, -1);
4835 tem
= expand_shift (RSHIFT_EXPR
, compute_mode
, tem
, 1, NULL_RTX
, 1);
4836 do_cmp_and_jump (remainder
, tem
, LEU
, compute_mode
, label
);
4837 expand_inc (quotient
, const1_rtx
);
4838 expand_dec (remainder
, op1
);
4843 rtx abs_rem
, abs_op1
, tem
, mask
;
4844 rtx_code_label
*label
;
4845 label
= gen_label_rtx ();
4846 quotient
= gen_reg_rtx (compute_mode
);
4847 remainder
= gen_reg_rtx (compute_mode
);
4848 if (expand_twoval_binop (sdivmod_optab
, op0
, op1
, quotient
, remainder
, 0) == 0)
4851 quotient
= expand_binop (compute_mode
, sdiv_optab
, op0
, op1
,
4852 quotient
, 0, OPTAB_LIB_WIDEN
);
4853 tem
= expand_mult (compute_mode
, quotient
, op1
, NULL_RTX
, 0);
4854 remainder
= expand_binop (compute_mode
, sub_optab
, op0
, tem
,
4855 remainder
, 0, OPTAB_LIB_WIDEN
);
4857 abs_rem
= expand_abs (compute_mode
, remainder
, NULL_RTX
, 1, 0);
4858 abs_op1
= expand_abs (compute_mode
, op1
, NULL_RTX
, 1, 0);
4859 tem
= expand_shift (LSHIFT_EXPR
, compute_mode
, abs_rem
,
4861 do_cmp_and_jump (tem
, abs_op1
, LTU
, compute_mode
, label
);
4862 tem
= expand_binop (compute_mode
, xor_optab
, op0
, op1
,
4863 NULL_RTX
, 0, OPTAB_WIDEN
);
4864 mask
= expand_shift (RSHIFT_EXPR
, compute_mode
, tem
,
4865 size
- 1, NULL_RTX
, 0);
4866 tem
= expand_binop (compute_mode
, xor_optab
, mask
, const1_rtx
,
4867 NULL_RTX
, 0, OPTAB_WIDEN
);
4868 tem
= expand_binop (compute_mode
, sub_optab
, tem
, mask
,
4869 NULL_RTX
, 0, OPTAB_WIDEN
);
4870 expand_inc (quotient
, tem
);
4871 tem
= expand_binop (compute_mode
, xor_optab
, mask
, op1
,
4872 NULL_RTX
, 0, OPTAB_WIDEN
);
4873 tem
= expand_binop (compute_mode
, sub_optab
, tem
, mask
,
4874 NULL_RTX
, 0, OPTAB_WIDEN
);
4875 expand_dec (remainder
, tem
);
4878 return gen_lowpart (mode
, rem_flag
? remainder
: quotient
);
4886 if (target
&& GET_MODE (target
) != compute_mode
)
4891 /* Try to produce the remainder without producing the quotient.
4892 If we seem to have a divmod pattern that does not require widening,
4893 don't try widening here. We should really have a WIDEN argument
4894 to expand_twoval_binop, since what we'd really like to do here is
4895 1) try a mod insn in compute_mode
4896 2) try a divmod insn in compute_mode
4897 3) try a div insn in compute_mode and multiply-subtract to get
4899 4) try the same things with widening allowed. */
4901 = sign_expand_binop (compute_mode
, umod_optab
, smod_optab
,
4904 ((optab_handler (optab2
, compute_mode
)
4905 != CODE_FOR_nothing
)
4906 ? OPTAB_DIRECT
: OPTAB_WIDEN
));
4909 /* No luck there. Can we do remainder and divide at once
4910 without a library call? */
4911 remainder
= gen_reg_rtx (compute_mode
);
4912 if (! expand_twoval_binop ((unsignedp
4916 NULL_RTX
, remainder
, unsignedp
))
4921 return gen_lowpart (mode
, remainder
);
4924 /* Produce the quotient. Try a quotient insn, but not a library call.
4925 If we have a divmod in this mode, use it in preference to widening
4926 the div (for this test we assume it will not fail). Note that optab2
4927 is set to the one of the two optabs that the call below will use. */
4929 = sign_expand_binop (compute_mode
, udiv_optab
, sdiv_optab
,
4930 op0
, op1
, rem_flag
? NULL_RTX
: target
,
4932 ((optab_handler (optab2
, compute_mode
)
4933 != CODE_FOR_nothing
)
4934 ? OPTAB_DIRECT
: OPTAB_WIDEN
));
4938 /* No luck there. Try a quotient-and-remainder insn,
4939 keeping the quotient alone. */
4940 quotient
= gen_reg_rtx (compute_mode
);
4941 if (! expand_twoval_binop (unsignedp
? udivmod_optab
: sdivmod_optab
,
4943 quotient
, NULL_RTX
, unsignedp
))
4947 /* Still no luck. If we are not computing the remainder,
4948 use a library call for the quotient. */
4949 quotient
= sign_expand_binop (compute_mode
,
4950 udiv_optab
, sdiv_optab
,
4952 unsignedp
, OPTAB_LIB_WIDEN
);
4959 if (target
&& GET_MODE (target
) != compute_mode
)
4964 /* No divide instruction either. Use library for remainder. */
4965 remainder
= sign_expand_binop (compute_mode
, umod_optab
, smod_optab
,
4967 unsignedp
, OPTAB_LIB_WIDEN
);
4968 /* No remainder function. Try a quotient-and-remainder
4969 function, keeping the remainder. */
4972 remainder
= gen_reg_rtx (compute_mode
);
4973 if (!expand_twoval_binop_libfunc
4974 (unsignedp
? udivmod_optab
: sdivmod_optab
,
4976 NULL_RTX
, remainder
,
4977 unsignedp
? UMOD
: MOD
))
4978 remainder
= NULL_RTX
;
4983 /* We divided. Now finish doing X - Y * (X / Y). */
4984 remainder
= expand_mult (compute_mode
, quotient
, op1
,
4985 NULL_RTX
, unsignedp
);
4986 remainder
= expand_binop (compute_mode
, sub_optab
, op0
,
4987 remainder
, target
, unsignedp
,
4992 return gen_lowpart (mode
, rem_flag
? remainder
: quotient
);
4995 /* Return a tree node with data type TYPE, describing the value of X.
4996 Usually this is an VAR_DECL, if there is no obvious better choice.
4997 X may be an expression, however we only support those expressions
4998 generated by loop.c. */
5001 make_tree (tree type
, rtx x
)
5005 switch (GET_CODE (x
))
5008 case CONST_WIDE_INT
:
5009 t
= wide_int_to_tree (type
, std::make_pair (x
, TYPE_MODE (type
)));
5013 STATIC_ASSERT (HOST_BITS_PER_WIDE_INT
* 2 <= MAX_BITSIZE_MODE_ANY_INT
);
5014 if (TARGET_SUPPORTS_WIDE_INT
== 0 && GET_MODE (x
) == VOIDmode
)
5015 t
= wide_int_to_tree (type
,
5016 wide_int::from_array (&CONST_DOUBLE_LOW (x
), 2,
5017 HOST_BITS_PER_WIDE_INT
* 2));
5022 REAL_VALUE_FROM_CONST_DOUBLE (d
, x
);
5023 t
= build_real (type
, d
);
5030 int units
= CONST_VECTOR_NUNITS (x
);
5031 tree itype
= TREE_TYPE (type
);
5035 /* Build a tree with vector elements. */
5036 elts
= XALLOCAVEC (tree
, units
);
5037 for (i
= units
- 1; i
>= 0; --i
)
5039 rtx elt
= CONST_VECTOR_ELT (x
, i
);
5040 elts
[i
] = make_tree (itype
, elt
);
5043 return build_vector (type
, elts
);
5047 return fold_build2 (PLUS_EXPR
, type
, make_tree (type
, XEXP (x
, 0)),
5048 make_tree (type
, XEXP (x
, 1)));
5051 return fold_build2 (MINUS_EXPR
, type
, make_tree (type
, XEXP (x
, 0)),
5052 make_tree (type
, XEXP (x
, 1)));
5055 return fold_build1 (NEGATE_EXPR
, type
, make_tree (type
, XEXP (x
, 0)));
5058 return fold_build2 (MULT_EXPR
, type
, make_tree (type
, XEXP (x
, 0)),
5059 make_tree (type
, XEXP (x
, 1)));
5062 return fold_build2 (LSHIFT_EXPR
, type
, make_tree (type
, XEXP (x
, 0)),
5063 make_tree (type
, XEXP (x
, 1)));
5066 t
= unsigned_type_for (type
);
5067 return fold_convert (type
, build2 (RSHIFT_EXPR
, t
,
5068 make_tree (t
, XEXP (x
, 0)),
5069 make_tree (type
, XEXP (x
, 1))));
5072 t
= signed_type_for (type
);
5073 return fold_convert (type
, build2 (RSHIFT_EXPR
, t
,
5074 make_tree (t
, XEXP (x
, 0)),
5075 make_tree (type
, XEXP (x
, 1))));
5078 if (TREE_CODE (type
) != REAL_TYPE
)
5079 t
= signed_type_for (type
);
5083 return fold_convert (type
, build2 (TRUNC_DIV_EXPR
, t
,
5084 make_tree (t
, XEXP (x
, 0)),
5085 make_tree (t
, XEXP (x
, 1))));
5087 t
= unsigned_type_for (type
);
5088 return fold_convert (type
, build2 (TRUNC_DIV_EXPR
, t
,
5089 make_tree (t
, XEXP (x
, 0)),
5090 make_tree (t
, XEXP (x
, 1))));
5094 t
= lang_hooks
.types
.type_for_mode (GET_MODE (XEXP (x
, 0)),
5095 GET_CODE (x
) == ZERO_EXTEND
);
5096 return fold_convert (type
, make_tree (t
, XEXP (x
, 0)));
5099 return make_tree (type
, XEXP (x
, 0));
5102 t
= SYMBOL_REF_DECL (x
);
5104 return fold_convert (type
, build_fold_addr_expr (t
));
5105 /* else fall through. */
5108 t
= build_decl (RTL_LOCATION (x
), VAR_DECL
, NULL_TREE
, type
);
5110 /* If TYPE is a POINTER_TYPE, we might need to convert X from
5111 address mode to pointer mode. */
5112 if (POINTER_TYPE_P (type
))
5113 x
= convert_memory_address_addr_space
5114 (TYPE_MODE (type
), x
, TYPE_ADDR_SPACE (TREE_TYPE (type
)));
5116 /* Note that we do *not* use SET_DECL_RTL here, because we do not
5117 want set_decl_rtl to go adjusting REG_ATTRS for this temporary. */
5118 t
->decl_with_rtl
.rtl
= x
;
5124 /* Compute the logical-and of OP0 and OP1, storing it in TARGET
5125 and returning TARGET.
5127 If TARGET is 0, a pseudo-register or constant is returned. */
5130 expand_and (machine_mode mode
, rtx op0
, rtx op1
, rtx target
)
5134 if (GET_MODE (op0
) == VOIDmode
&& GET_MODE (op1
) == VOIDmode
)
5135 tem
= simplify_binary_operation (AND
, mode
, op0
, op1
);
5137 tem
= expand_binop (mode
, and_optab
, op0
, op1
, target
, 0, OPTAB_LIB_WIDEN
);
5141 else if (tem
!= target
)
5142 emit_move_insn (target
, tem
);
5146 /* Helper function for emit_store_flag. */
5148 emit_cstore (rtx target
, enum insn_code icode
, enum rtx_code code
,
5149 machine_mode mode
, machine_mode compare_mode
,
5150 int unsignedp
, rtx x
, rtx y
, int normalizep
,
5151 machine_mode target_mode
)
5153 struct expand_operand ops
[4];
5154 rtx op0
, comparison
, subtarget
;
5156 machine_mode result_mode
= targetm
.cstore_mode (icode
);
5158 last
= get_last_insn ();
5159 x
= prepare_operand (icode
, x
, 2, mode
, compare_mode
, unsignedp
);
5160 y
= prepare_operand (icode
, y
, 3, mode
, compare_mode
, unsignedp
);
5163 delete_insns_since (last
);
5167 if (target_mode
== VOIDmode
)
5168 target_mode
= result_mode
;
5170 target
= gen_reg_rtx (target_mode
);
5172 comparison
= gen_rtx_fmt_ee (code
, result_mode
, x
, y
);
5174 create_output_operand (&ops
[0], optimize
? NULL_RTX
: target
, result_mode
);
5175 create_fixed_operand (&ops
[1], comparison
);
5176 create_fixed_operand (&ops
[2], x
);
5177 create_fixed_operand (&ops
[3], y
);
5178 if (!maybe_expand_insn (icode
, 4, ops
))
5180 delete_insns_since (last
);
5183 subtarget
= ops
[0].value
;
5185 /* If we are converting to a wider mode, first convert to
5186 TARGET_MODE, then normalize. This produces better combining
5187 opportunities on machines that have a SIGN_EXTRACT when we are
5188 testing a single bit. This mostly benefits the 68k.
5190 If STORE_FLAG_VALUE does not have the sign bit set when
5191 interpreted in MODE, we can do this conversion as unsigned, which
5192 is usually more efficient. */
5193 if (GET_MODE_SIZE (target_mode
) > GET_MODE_SIZE (result_mode
))
5195 convert_move (target
, subtarget
,
5196 val_signbit_known_clear_p (result_mode
,
5199 result_mode
= target_mode
;
5204 /* If we want to keep subexpressions around, don't reuse our last
5209 /* Now normalize to the proper value in MODE. Sometimes we don't
5210 have to do anything. */
5211 if (normalizep
== 0 || normalizep
== STORE_FLAG_VALUE
)
5213 /* STORE_FLAG_VALUE might be the most negative number, so write
5214 the comparison this way to avoid a compiler-time warning. */
5215 else if (- normalizep
== STORE_FLAG_VALUE
)
5216 op0
= expand_unop (result_mode
, neg_optab
, op0
, subtarget
, 0);
5218 /* We don't want to use STORE_FLAG_VALUE < 0 below since this makes
5219 it hard to use a value of just the sign bit due to ANSI integer
5220 constant typing rules. */
5221 else if (val_signbit_known_set_p (result_mode
, STORE_FLAG_VALUE
))
5222 op0
= expand_shift (RSHIFT_EXPR
, result_mode
, op0
,
5223 GET_MODE_BITSIZE (result_mode
) - 1, subtarget
,
5227 gcc_assert (STORE_FLAG_VALUE
& 1);
5229 op0
= expand_and (result_mode
, op0
, const1_rtx
, subtarget
);
5230 if (normalizep
== -1)
5231 op0
= expand_unop (result_mode
, neg_optab
, op0
, op0
, 0);
5234 /* If we were converting to a smaller mode, do the conversion now. */
5235 if (target_mode
!= result_mode
)
5237 convert_move (target
, op0
, 0);
5245 /* A subroutine of emit_store_flag only including "tricks" that do not
5246 need a recursive call. These are kept separate to avoid infinite
5250 emit_store_flag_1 (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
5251 machine_mode mode
, int unsignedp
, int normalizep
,
5252 machine_mode target_mode
)
5255 enum insn_code icode
;
5256 machine_mode compare_mode
;
5257 enum mode_class mclass
;
5258 enum rtx_code scode
;
5261 code
= unsigned_condition (code
);
5262 scode
= swap_condition (code
);
5264 /* If one operand is constant, make it the second one. Only do this
5265 if the other operand is not constant as well. */
5267 if (swap_commutative_operands_p (op0
, op1
))
5269 std::swap (op0
, op1
);
5270 code
= swap_condition (code
);
5273 if (mode
== VOIDmode
)
5274 mode
= GET_MODE (op0
);
5276 /* For some comparisons with 1 and -1, we can convert this to
5277 comparisons with zero. This will often produce more opportunities for
5278 store-flag insns. */
5283 if (op1
== const1_rtx
)
5284 op1
= const0_rtx
, code
= LE
;
5287 if (op1
== constm1_rtx
)
5288 op1
= const0_rtx
, code
= LT
;
5291 if (op1
== const1_rtx
)
5292 op1
= const0_rtx
, code
= GT
;
5295 if (op1
== constm1_rtx
)
5296 op1
= const0_rtx
, code
= GE
;
5299 if (op1
== const1_rtx
)
5300 op1
= const0_rtx
, code
= NE
;
5303 if (op1
== const1_rtx
)
5304 op1
= const0_rtx
, code
= EQ
;
5310 /* If we are comparing a double-word integer with zero or -1, we can
5311 convert the comparison into one involving a single word. */
5312 if (GET_MODE_BITSIZE (mode
) == BITS_PER_WORD
* 2
5313 && GET_MODE_CLASS (mode
) == MODE_INT
5314 && (!MEM_P (op0
) || ! MEM_VOLATILE_P (op0
)))
5317 if ((code
== EQ
|| code
== NE
)
5318 && (op1
== const0_rtx
|| op1
== constm1_rtx
))
5322 /* Do a logical OR or AND of the two words and compare the
5324 op00
= simplify_gen_subreg (word_mode
, op0
, mode
, 0);
5325 op01
= simplify_gen_subreg (word_mode
, op0
, mode
, UNITS_PER_WORD
);
5326 tem
= expand_binop (word_mode
,
5327 op1
== const0_rtx
? ior_optab
: and_optab
,
5328 op00
, op01
, NULL_RTX
, unsignedp
,
5332 tem
= emit_store_flag (NULL_RTX
, code
, tem
, op1
, word_mode
,
5333 unsignedp
, normalizep
);
5335 else if ((code
== LT
|| code
== GE
) && op1
== const0_rtx
)
5339 /* If testing the sign bit, can just test on high word. */
5340 op0h
= simplify_gen_subreg (word_mode
, op0
, mode
,
5341 subreg_highpart_offset (word_mode
,
5343 tem
= emit_store_flag (NULL_RTX
, code
, op0h
, op1
, word_mode
,
5344 unsignedp
, normalizep
);
5351 if (target_mode
== VOIDmode
|| GET_MODE (tem
) == target_mode
)
5354 target
= gen_reg_rtx (target_mode
);
5356 convert_move (target
, tem
,
5357 !val_signbit_known_set_p (word_mode
,
5358 (normalizep
? normalizep
5359 : STORE_FLAG_VALUE
)));
5364 /* If this is A < 0 or A >= 0, we can do this by taking the ones
5365 complement of A (for GE) and shifting the sign bit to the low bit. */
5366 if (op1
== const0_rtx
&& (code
== LT
|| code
== GE
)
5367 && GET_MODE_CLASS (mode
) == MODE_INT
5368 && (normalizep
|| STORE_FLAG_VALUE
== 1
5369 || val_signbit_p (mode
, STORE_FLAG_VALUE
)))
5376 /* If the result is to be wider than OP0, it is best to convert it
5377 first. If it is to be narrower, it is *incorrect* to convert it
5379 else if (GET_MODE_SIZE (target_mode
) > GET_MODE_SIZE (mode
))
5381 op0
= convert_modes (target_mode
, mode
, op0
, 0);
5385 if (target_mode
!= mode
)
5389 op0
= expand_unop (mode
, one_cmpl_optab
, op0
,
5390 ((STORE_FLAG_VALUE
== 1 || normalizep
)
5391 ? 0 : subtarget
), 0);
5393 if (STORE_FLAG_VALUE
== 1 || normalizep
)
5394 /* If we are supposed to produce a 0/1 value, we want to do
5395 a logical shift from the sign bit to the low-order bit; for
5396 a -1/0 value, we do an arithmetic shift. */
5397 op0
= expand_shift (RSHIFT_EXPR
, mode
, op0
,
5398 GET_MODE_BITSIZE (mode
) - 1,
5399 subtarget
, normalizep
!= -1);
5401 if (mode
!= target_mode
)
5402 op0
= convert_modes (target_mode
, mode
, op0
, 0);
5407 mclass
= GET_MODE_CLASS (mode
);
5408 for (compare_mode
= mode
; compare_mode
!= VOIDmode
;
5409 compare_mode
= GET_MODE_WIDER_MODE (compare_mode
))
5411 machine_mode optab_mode
= mclass
== MODE_CC
? CCmode
: compare_mode
;
5412 icode
= optab_handler (cstore_optab
, optab_mode
);
5413 if (icode
!= CODE_FOR_nothing
)
5415 do_pending_stack_adjust ();
5416 rtx tem
= emit_cstore (target
, icode
, code
, mode
, compare_mode
,
5417 unsignedp
, op0
, op1
, normalizep
, target_mode
);
5421 if (GET_MODE_CLASS (mode
) == MODE_FLOAT
)
5423 tem
= emit_cstore (target
, icode
, scode
, mode
, compare_mode
,
5424 unsignedp
, op1
, op0
, normalizep
, target_mode
);
5435 /* Emit a store-flags instruction for comparison CODE on OP0 and OP1
5436 and storing in TARGET. Normally return TARGET.
5437 Return 0 if that cannot be done.
5439 MODE is the mode to use for OP0 and OP1 should they be CONST_INTs. If
5440 it is VOIDmode, they cannot both be CONST_INT.
5442 UNSIGNEDP is for the case where we have to widen the operands
5443 to perform the operation. It says to use zero-extension.
5445 NORMALIZEP is 1 if we should convert the result to be either zero
5446 or one. Normalize is -1 if we should convert the result to be
5447 either zero or -1. If NORMALIZEP is zero, the result will be left
5448 "raw" out of the scc insn. */
5451 emit_store_flag (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
5452 machine_mode mode
, int unsignedp
, int normalizep
)
5454 machine_mode target_mode
= target
? GET_MODE (target
) : VOIDmode
;
5455 enum rtx_code rcode
;
5460 /* If we compare constants, we shouldn't use a store-flag operation,
5461 but a constant load. We can get there via the vanilla route that
5462 usually generates a compare-branch sequence, but will in this case
5463 fold the comparison to a constant, and thus elide the branch. */
5464 if (CONSTANT_P (op0
) && CONSTANT_P (op1
))
5467 tem
= emit_store_flag_1 (target
, code
, op0
, op1
, mode
, unsignedp
, normalizep
,
5472 /* If we reached here, we can't do this with a scc insn, however there
5473 are some comparisons that can be done in other ways. Don't do any
5474 of these cases if branches are very cheap. */
5475 if (BRANCH_COST (optimize_insn_for_speed_p (), false) == 0)
5478 /* See what we need to return. We can only return a 1, -1, or the
5481 if (normalizep
== 0)
5483 if (STORE_FLAG_VALUE
== 1 || STORE_FLAG_VALUE
== -1)
5484 normalizep
= STORE_FLAG_VALUE
;
5486 else if (val_signbit_p (mode
, STORE_FLAG_VALUE
))
5492 last
= get_last_insn ();
5494 /* If optimizing, use different pseudo registers for each insn, instead
5495 of reusing the same pseudo. This leads to better CSE, but slows
5496 down the compiler, since there are more pseudos */
5497 subtarget
= (!optimize
5498 && (target_mode
== mode
)) ? target
: NULL_RTX
;
5499 trueval
= GEN_INT (normalizep
? normalizep
: STORE_FLAG_VALUE
);
5501 /* For floating-point comparisons, try the reverse comparison or try
5502 changing the "orderedness" of the comparison. */
5503 if (GET_MODE_CLASS (mode
) == MODE_FLOAT
)
5505 enum rtx_code first_code
;
5508 rcode
= reverse_condition_maybe_unordered (code
);
5509 if (can_compare_p (rcode
, mode
, ccp_store_flag
)
5510 && (code
== ORDERED
|| code
== UNORDERED
5511 || (! HONOR_NANS (mode
) && (code
== LTGT
|| code
== UNEQ
))
5512 || (! HONOR_SNANS (mode
) && (code
== EQ
|| code
== NE
))))
5514 int want_add
= ((STORE_FLAG_VALUE
== 1 && normalizep
== -1)
5515 || (STORE_FLAG_VALUE
== -1 && normalizep
== 1));
5517 /* For the reverse comparison, use either an addition or a XOR. */
5519 && rtx_cost (GEN_INT (normalizep
), PLUS
, 1,
5520 optimize_insn_for_speed_p ()) == 0)
5522 tem
= emit_store_flag_1 (subtarget
, rcode
, op0
, op1
, mode
, 0,
5523 STORE_FLAG_VALUE
, target_mode
);
5525 return expand_binop (target_mode
, add_optab
, tem
,
5526 gen_int_mode (normalizep
, target_mode
),
5527 target
, 0, OPTAB_WIDEN
);
5530 && rtx_cost (trueval
, XOR
, 1,
5531 optimize_insn_for_speed_p ()) == 0)
5533 tem
= emit_store_flag_1 (subtarget
, rcode
, op0
, op1
, mode
, 0,
5534 normalizep
, target_mode
);
5536 return expand_binop (target_mode
, xor_optab
, tem
, trueval
,
5537 target
, INTVAL (trueval
) >= 0, OPTAB_WIDEN
);
5541 delete_insns_since (last
);
5543 /* Cannot split ORDERED and UNORDERED, only try the above trick. */
5544 if (code
== ORDERED
|| code
== UNORDERED
)
5547 and_them
= split_comparison (code
, mode
, &first_code
, &code
);
5549 /* If there are no NaNs, the first comparison should always fall through.
5550 Effectively change the comparison to the other one. */
5551 if (!HONOR_NANS (mode
))
5553 gcc_assert (first_code
== (and_them
? ORDERED
: UNORDERED
));
5554 return emit_store_flag_1 (target
, code
, op0
, op1
, mode
, 0, normalizep
,
5558 #ifdef HAVE_conditional_move
5559 /* Try using a setcc instruction for ORDERED/UNORDERED, followed by a
5560 conditional move. */
5561 tem
= emit_store_flag_1 (subtarget
, first_code
, op0
, op1
, mode
, 0,
5562 normalizep
, target_mode
);
5567 tem
= emit_conditional_move (target
, code
, op0
, op1
, mode
,
5568 tem
, const0_rtx
, GET_MODE (tem
), 0);
5570 tem
= emit_conditional_move (target
, code
, op0
, op1
, mode
,
5571 trueval
, tem
, GET_MODE (tem
), 0);
5574 delete_insns_since (last
);
5581 /* The remaining tricks only apply to integer comparisons. */
5583 if (GET_MODE_CLASS (mode
) != MODE_INT
)
5586 /* If this is an equality comparison of integers, we can try to exclusive-or
5587 (or subtract) the two operands and use a recursive call to try the
5588 comparison with zero. Don't do any of these cases if branches are
5591 if ((code
== EQ
|| code
== NE
) && op1
!= const0_rtx
)
5593 tem
= expand_binop (mode
, xor_optab
, op0
, op1
, subtarget
, 1,
5597 tem
= expand_binop (mode
, sub_optab
, op0
, op1
, subtarget
, 1,
5600 tem
= emit_store_flag (target
, code
, tem
, const0_rtx
,
5601 mode
, unsignedp
, normalizep
);
5605 delete_insns_since (last
);
5608 /* For integer comparisons, try the reverse comparison. However, for
5609 small X and if we'd have anyway to extend, implementing "X != 0"
5610 as "-(int)X >> 31" is still cheaper than inverting "(int)X == 0". */
5611 rcode
= reverse_condition (code
);
5612 if (can_compare_p (rcode
, mode
, ccp_store_flag
)
5613 && ! (optab_handler (cstore_optab
, mode
) == CODE_FOR_nothing
5615 && GET_MODE_SIZE (mode
) < UNITS_PER_WORD
5616 && op1
== const0_rtx
))
5618 int want_add
= ((STORE_FLAG_VALUE
== 1 && normalizep
== -1)
5619 || (STORE_FLAG_VALUE
== -1 && normalizep
== 1));
5621 /* Again, for the reverse comparison, use either an addition or a XOR. */
5623 && rtx_cost (GEN_INT (normalizep
), PLUS
, 1,
5624 optimize_insn_for_speed_p ()) == 0)
5626 tem
= emit_store_flag_1 (subtarget
, rcode
, op0
, op1
, mode
, 0,
5627 STORE_FLAG_VALUE
, target_mode
);
5629 tem
= expand_binop (target_mode
, add_optab
, tem
,
5630 gen_int_mode (normalizep
, target_mode
),
5631 target
, 0, OPTAB_WIDEN
);
5634 && rtx_cost (trueval
, XOR
, 1,
5635 optimize_insn_for_speed_p ()) == 0)
5637 tem
= emit_store_flag_1 (subtarget
, rcode
, op0
, op1
, mode
, 0,
5638 normalizep
, target_mode
);
5640 tem
= expand_binop (target_mode
, xor_optab
, tem
, trueval
, target
,
5641 INTVAL (trueval
) >= 0, OPTAB_WIDEN
);
5646 delete_insns_since (last
);
5649 /* Some other cases we can do are EQ, NE, LE, and GT comparisons with
5650 the constant zero. Reject all other comparisons at this point. Only
5651 do LE and GT if branches are expensive since they are expensive on
5652 2-operand machines. */
5654 if (op1
!= const0_rtx
5655 || (code
!= EQ
&& code
!= NE
5656 && (BRANCH_COST (optimize_insn_for_speed_p (),
5657 false) <= 1 || (code
!= LE
&& code
!= GT
))))
5660 /* Try to put the result of the comparison in the sign bit. Assume we can't
5661 do the necessary operation below. */
5665 /* To see if A <= 0, compute (A | (A - 1)). A <= 0 iff that result has
5666 the sign bit set. */
5670 /* This is destructive, so SUBTARGET can't be OP0. */
5671 if (rtx_equal_p (subtarget
, op0
))
5674 tem
= expand_binop (mode
, sub_optab
, op0
, const1_rtx
, subtarget
, 0,
5677 tem
= expand_binop (mode
, ior_optab
, op0
, tem
, subtarget
, 0,
5681 /* To see if A > 0, compute (((signed) A) << BITS) - A, where BITS is the
5682 number of bits in the mode of OP0, minus one. */
5686 if (rtx_equal_p (subtarget
, op0
))
5689 tem
= expand_shift (RSHIFT_EXPR
, mode
, op0
,
5690 GET_MODE_BITSIZE (mode
) - 1,
5692 tem
= expand_binop (mode
, sub_optab
, tem
, op0
, subtarget
, 0,
5696 if (code
== EQ
|| code
== NE
)
5698 /* For EQ or NE, one way to do the comparison is to apply an operation
5699 that converts the operand into a positive number if it is nonzero
5700 or zero if it was originally zero. Then, for EQ, we subtract 1 and
5701 for NE we negate. This puts the result in the sign bit. Then we
5702 normalize with a shift, if needed.
5704 Two operations that can do the above actions are ABS and FFS, so try
5705 them. If that doesn't work, and MODE is smaller than a full word,
5706 we can use zero-extension to the wider mode (an unsigned conversion)
5707 as the operation. */
5709 /* Note that ABS doesn't yield a positive number for INT_MIN, but
5710 that is compensated by the subsequent overflow when subtracting
5713 if (optab_handler (abs_optab
, mode
) != CODE_FOR_nothing
)
5714 tem
= expand_unop (mode
, abs_optab
, op0
, subtarget
, 1);
5715 else if (optab_handler (ffs_optab
, mode
) != CODE_FOR_nothing
)
5716 tem
= expand_unop (mode
, ffs_optab
, op0
, subtarget
, 1);
5717 else if (GET_MODE_SIZE (mode
) < UNITS_PER_WORD
)
5719 tem
= convert_modes (word_mode
, mode
, op0
, 1);
5726 tem
= expand_binop (mode
, sub_optab
, tem
, const1_rtx
, subtarget
,
5729 tem
= expand_unop (mode
, neg_optab
, tem
, subtarget
, 0);
5732 /* If we couldn't do it that way, for NE we can "or" the two's complement
5733 of the value with itself. For EQ, we take the one's complement of
5734 that "or", which is an extra insn, so we only handle EQ if branches
5739 || BRANCH_COST (optimize_insn_for_speed_p (),
5742 if (rtx_equal_p (subtarget
, op0
))
5745 tem
= expand_unop (mode
, neg_optab
, op0
, subtarget
, 0);
5746 tem
= expand_binop (mode
, ior_optab
, tem
, op0
, subtarget
, 0,
5749 if (tem
&& code
== EQ
)
5750 tem
= expand_unop (mode
, one_cmpl_optab
, tem
, subtarget
, 0);
5754 if (tem
&& normalizep
)
5755 tem
= expand_shift (RSHIFT_EXPR
, mode
, tem
,
5756 GET_MODE_BITSIZE (mode
) - 1,
5757 subtarget
, normalizep
== 1);
5763 else if (GET_MODE (tem
) != target_mode
)
5765 convert_move (target
, tem
, 0);
5768 else if (!subtarget
)
5770 emit_move_insn (target
, tem
);
5775 delete_insns_since (last
);
5780 /* Like emit_store_flag, but always succeeds. */
5783 emit_store_flag_force (rtx target
, enum rtx_code code
, rtx op0
, rtx op1
,
5784 machine_mode mode
, int unsignedp
, int normalizep
)
5787 rtx_code_label
*label
;
5788 rtx trueval
, falseval
;
5790 /* First see if emit_store_flag can do the job. */
5791 tem
= emit_store_flag (target
, code
, op0
, op1
, mode
, unsignedp
, normalizep
);
5796 target
= gen_reg_rtx (word_mode
);
5798 /* If this failed, we have to do this with set/compare/jump/set code.
5799 For foo != 0, if foo is in OP0, just replace it with 1 if nonzero. */
5800 trueval
= normalizep
? GEN_INT (normalizep
) : const1_rtx
;
5802 && GET_MODE_CLASS (mode
) == MODE_INT
5805 && op1
== const0_rtx
)
5807 label
= gen_label_rtx ();
5808 do_compare_rtx_and_jump (target
, const0_rtx
, EQ
, unsignedp
,
5809 mode
, NULL_RTX
, NULL_RTX
, label
, -1);
5810 emit_move_insn (target
, trueval
);
5816 || reg_mentioned_p (target
, op0
) || reg_mentioned_p (target
, op1
))
5817 target
= gen_reg_rtx (GET_MODE (target
));
5819 /* Jump in the right direction if the target cannot implement CODE
5820 but can jump on its reverse condition. */
5821 falseval
= const0_rtx
;
5822 if (! can_compare_p (code
, mode
, ccp_jump
)
5823 && (! FLOAT_MODE_P (mode
)
5824 || code
== ORDERED
|| code
== UNORDERED
5825 || (! HONOR_NANS (mode
) && (code
== LTGT
|| code
== UNEQ
))
5826 || (! HONOR_SNANS (mode
) && (code
== EQ
|| code
== NE
))))
5828 enum rtx_code rcode
;
5829 if (FLOAT_MODE_P (mode
))
5830 rcode
= reverse_condition_maybe_unordered (code
);
5832 rcode
= reverse_condition (code
);
5834 /* Canonicalize to UNORDERED for the libcall. */
5835 if (can_compare_p (rcode
, mode
, ccp_jump
)
5836 || (code
== ORDERED
&& ! can_compare_p (ORDERED
, mode
, ccp_jump
)))
5839 trueval
= const0_rtx
;
5844 emit_move_insn (target
, trueval
);
5845 label
= gen_label_rtx ();
5846 do_compare_rtx_and_jump (op0
, op1
, code
, unsignedp
, mode
, NULL_RTX
,
5847 NULL_RTX
, label
, -1);
5849 emit_move_insn (target
, falseval
);
5855 /* Perform possibly multi-word comparison and conditional jump to LABEL
5856 if ARG1 OP ARG2 true where ARG1 and ARG2 are of mode MODE. This is
5857 now a thin wrapper around do_compare_rtx_and_jump. */
5860 do_cmp_and_jump (rtx arg1
, rtx arg2
, enum rtx_code op
, machine_mode mode
,
5861 rtx_code_label
*label
)
5863 int unsignedp
= (op
== LTU
|| op
== LEU
|| op
== GTU
|| op
== GEU
);
5864 do_compare_rtx_and_jump (arg1
, arg2
, op
, unsignedp
, mode
,
5865 NULL_RTX
, NULL_RTX
, label
, -1);