2 Copyright (C) 2011-2020 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
29 #include "stringpool.h"
31 #include "tree-ssanames.h"
36 #include "diagnostic-core.h"
37 #include "fold-const.h"
38 #include "internal-fn.h"
39 #include "stor-layout.h"
42 #include "stringpool.h"
48 #include "optabs-tree.h"
49 #include "gimple-ssa.h"
50 #include "tree-phinodes.h"
51 #include "ssa-iterators.h"
53 /* The names of each internal function, indexed by function number. */
54 const char *const internal_fn_name_array
[] = {
55 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
56 #include "internal-fn.def"
60 /* The ECF_* flags of each internal function, indexed by function number. */
61 const int internal_fn_flags_array
[] = {
62 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
63 #include "internal-fn.def"
67 /* Return the internal function called NAME, or IFN_LAST if there's
71 lookup_internal_fn (const char *name
)
73 typedef hash_map
<nofree_string_hash
, internal_fn
> name_to_fn_map_type
;
74 static name_to_fn_map_type
*name_to_fn_map
;
78 name_to_fn_map
= new name_to_fn_map_type (IFN_LAST
);
79 for (unsigned int i
= 0; i
< IFN_LAST
; ++i
)
80 name_to_fn_map
->put (internal_fn_name (internal_fn (i
)),
83 internal_fn
*entry
= name_to_fn_map
->get (name
);
84 return entry
? *entry
: IFN_LAST
;
87 /* Fnspec of each internal function, indexed by function number. */
88 const_tree internal_fn_fnspec_array
[IFN_LAST
+ 1];
93 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
94 if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
95 build_string ((int) sizeof (FNSPEC), FNSPEC ? FNSPEC : "");
96 #include "internal-fn.def"
97 internal_fn_fnspec_array
[IFN_LAST
] = 0;
100 /* Create static initializers for the information returned by
101 direct_internal_fn. */
102 #define not_direct { -2, -2, false }
103 #define mask_load_direct { -1, 2, false }
104 #define load_lanes_direct { -1, -1, false }
105 #define mask_load_lanes_direct { -1, -1, false }
106 #define gather_load_direct { 3, 1, false }
107 #define mask_store_direct { 3, 2, false }
108 #define store_lanes_direct { 0, 0, false }
109 #define mask_store_lanes_direct { 0, 0, false }
110 #define scatter_store_direct { 3, 1, false }
111 #define unary_direct { 0, 0, true }
112 #define binary_direct { 0, 0, true }
113 #define ternary_direct { 0, 0, true }
114 #define cond_unary_direct { 1, 1, true }
115 #define cond_binary_direct { 1, 1, true }
116 #define cond_ternary_direct { 1, 1, true }
117 #define while_direct { 0, 2, false }
118 #define fold_extract_direct { 2, 2, false }
119 #define fold_left_direct { 1, 1, false }
120 #define mask_fold_left_direct { 1, 1, false }
121 #define check_ptrs_direct { 0, 0, false }
123 const direct_internal_fn_info direct_internal_fn_array
[IFN_LAST
+ 1] = {
124 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
125 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
126 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
127 UNSIGNED_OPTAB, TYPE) TYPE##_direct,
128 #include "internal-fn.def"
132 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
133 for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none. */
135 static enum insn_code
136 get_multi_vector_move (tree array_type
, convert_optab optab
)
141 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
142 imode
= TYPE_MODE (array_type
);
143 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
145 return convert_optab_handler (optab
, imode
, vmode
);
148 /* Expand LOAD_LANES call STMT using optab OPTAB. */
151 expand_load_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
153 class expand_operand ops
[2];
157 lhs
= gimple_call_lhs (stmt
);
158 rhs
= gimple_call_arg (stmt
, 0);
159 type
= TREE_TYPE (lhs
);
161 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
162 mem
= expand_normal (rhs
);
164 gcc_assert (MEM_P (mem
));
165 PUT_MODE (mem
, TYPE_MODE (type
));
167 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
168 create_fixed_operand (&ops
[1], mem
);
169 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
172 /* Expand STORE_LANES call STMT using optab OPTAB. */
175 expand_store_lanes_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
177 class expand_operand ops
[2];
181 lhs
= gimple_call_lhs (stmt
);
182 rhs
= gimple_call_arg (stmt
, 0);
183 type
= TREE_TYPE (rhs
);
185 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
186 reg
= expand_normal (rhs
);
188 gcc_assert (MEM_P (target
));
189 PUT_MODE (target
, TYPE_MODE (type
));
191 create_fixed_operand (&ops
[0], target
);
192 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
193 expand_insn (get_multi_vector_move (type
, optab
), 2, ops
);
197 expand_ANNOTATE (internal_fn
, gcall
*)
202 /* This should get expanded in omp_device_lower pass. */
205 expand_GOMP_USE_SIMT (internal_fn
, gcall
*)
210 /* This should get expanded in omp_device_lower pass. */
213 expand_GOMP_SIMT_ENTER (internal_fn
, gcall
*)
218 /* Allocate per-lane storage and begin non-uniform execution region. */
221 expand_GOMP_SIMT_ENTER_ALLOC (internal_fn
, gcall
*stmt
)
224 tree lhs
= gimple_call_lhs (stmt
);
226 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
228 target
= gen_reg_rtx (Pmode
);
229 rtx size
= expand_normal (gimple_call_arg (stmt
, 0));
230 rtx align
= expand_normal (gimple_call_arg (stmt
, 1));
231 class expand_operand ops
[3];
232 create_output_operand (&ops
[0], target
, Pmode
);
233 create_input_operand (&ops
[1], size
, Pmode
);
234 create_input_operand (&ops
[2], align
, Pmode
);
235 gcc_assert (targetm
.have_omp_simt_enter ());
236 expand_insn (targetm
.code_for_omp_simt_enter
, 3, ops
);
239 /* Deallocate per-lane storage and leave non-uniform execution region. */
242 expand_GOMP_SIMT_EXIT (internal_fn
, gcall
*stmt
)
244 gcc_checking_assert (!gimple_call_lhs (stmt
));
245 rtx arg
= expand_normal (gimple_call_arg (stmt
, 0));
246 class expand_operand ops
[1];
247 create_input_operand (&ops
[0], arg
, Pmode
);
248 gcc_assert (targetm
.have_omp_simt_exit ());
249 expand_insn (targetm
.code_for_omp_simt_exit
, 1, ops
);
252 /* Lane index on SIMT targets: thread index in the warp on NVPTX. On targets
253 without SIMT execution this should be expanded in omp_device_lower pass. */
256 expand_GOMP_SIMT_LANE (internal_fn
, gcall
*stmt
)
258 tree lhs
= gimple_call_lhs (stmt
);
262 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
263 gcc_assert (targetm
.have_omp_simt_lane ());
264 emit_insn (targetm
.gen_omp_simt_lane (target
));
267 /* This should get expanded in omp_device_lower pass. */
270 expand_GOMP_SIMT_VF (internal_fn
, gcall
*)
275 /* Lane index of the first SIMT lane that supplies a non-zero argument.
276 This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
277 lane that executed the last iteration for handling OpenMP lastprivate. */
280 expand_GOMP_SIMT_LAST_LANE (internal_fn
, gcall
*stmt
)
282 tree lhs
= gimple_call_lhs (stmt
);
286 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
287 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
288 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
289 class expand_operand ops
[2];
290 create_output_operand (&ops
[0], target
, mode
);
291 create_input_operand (&ops
[1], cond
, mode
);
292 gcc_assert (targetm
.have_omp_simt_last_lane ());
293 expand_insn (targetm
.code_for_omp_simt_last_lane
, 2, ops
);
296 /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered". */
299 expand_GOMP_SIMT_ORDERED_PRED (internal_fn
, gcall
*stmt
)
301 tree lhs
= gimple_call_lhs (stmt
);
305 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
306 rtx ctr
= expand_normal (gimple_call_arg (stmt
, 0));
307 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
308 class expand_operand ops
[2];
309 create_output_operand (&ops
[0], target
, mode
);
310 create_input_operand (&ops
[1], ctr
, mode
);
311 gcc_assert (targetm
.have_omp_simt_ordered ());
312 expand_insn (targetm
.code_for_omp_simt_ordered
, 2, ops
);
315 /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
316 any lane supplies a non-zero argument. */
319 expand_GOMP_SIMT_VOTE_ANY (internal_fn
, gcall
*stmt
)
321 tree lhs
= gimple_call_lhs (stmt
);
325 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
326 rtx cond
= expand_normal (gimple_call_arg (stmt
, 0));
327 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
328 class expand_operand ops
[2];
329 create_output_operand (&ops
[0], target
, mode
);
330 create_input_operand (&ops
[1], cond
, mode
);
331 gcc_assert (targetm
.have_omp_simt_vote_any ());
332 expand_insn (targetm
.code_for_omp_simt_vote_any
, 2, ops
);
335 /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
336 is destination lane index XOR given offset. */
339 expand_GOMP_SIMT_XCHG_BFLY (internal_fn
, gcall
*stmt
)
341 tree lhs
= gimple_call_lhs (stmt
);
345 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
346 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
347 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
348 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
349 class expand_operand ops
[3];
350 create_output_operand (&ops
[0], target
, mode
);
351 create_input_operand (&ops
[1], src
, mode
);
352 create_input_operand (&ops
[2], idx
, SImode
);
353 gcc_assert (targetm
.have_omp_simt_xchg_bfly ());
354 expand_insn (targetm
.code_for_omp_simt_xchg_bfly
, 3, ops
);
357 /* Exchange between SIMT lanes according to given source lane index. */
360 expand_GOMP_SIMT_XCHG_IDX (internal_fn
, gcall
*stmt
)
362 tree lhs
= gimple_call_lhs (stmt
);
366 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
367 rtx src
= expand_normal (gimple_call_arg (stmt
, 0));
368 rtx idx
= expand_normal (gimple_call_arg (stmt
, 1));
369 machine_mode mode
= TYPE_MODE (TREE_TYPE (lhs
));
370 class expand_operand ops
[3];
371 create_output_operand (&ops
[0], target
, mode
);
372 create_input_operand (&ops
[1], src
, mode
);
373 create_input_operand (&ops
[2], idx
, SImode
);
374 gcc_assert (targetm
.have_omp_simt_xchg_idx ());
375 expand_insn (targetm
.code_for_omp_simt_xchg_idx
, 3, ops
);
378 /* This should get expanded in adjust_simduid_builtins. */
381 expand_GOMP_SIMD_LANE (internal_fn
, gcall
*)
386 /* This should get expanded in adjust_simduid_builtins. */
389 expand_GOMP_SIMD_VF (internal_fn
, gcall
*)
394 /* This should get expanded in adjust_simduid_builtins. */
397 expand_GOMP_SIMD_LAST_LANE (internal_fn
, gcall
*)
402 /* This should get expanded in adjust_simduid_builtins. */
405 expand_GOMP_SIMD_ORDERED_START (internal_fn
, gcall
*)
410 /* This should get expanded in adjust_simduid_builtins. */
413 expand_GOMP_SIMD_ORDERED_END (internal_fn
, gcall
*)
418 /* This should get expanded in the sanopt pass. */
421 expand_UBSAN_NULL (internal_fn
, gcall
*)
426 /* This should get expanded in the sanopt pass. */
429 expand_UBSAN_BOUNDS (internal_fn
, gcall
*)
434 /* This should get expanded in the sanopt pass. */
437 expand_UBSAN_VPTR (internal_fn
, gcall
*)
442 /* This should get expanded in the sanopt pass. */
445 expand_UBSAN_PTR (internal_fn
, gcall
*)
450 /* This should get expanded in the sanopt pass. */
453 expand_UBSAN_OBJECT_SIZE (internal_fn
, gcall
*)
458 /* This should get expanded in the sanopt pass. */
461 expand_ASAN_CHECK (internal_fn
, gcall
*)
466 /* This should get expanded in the sanopt pass. */
469 expand_ASAN_MARK (internal_fn
, gcall
*)
474 /* This should get expanded in the sanopt pass. */
477 expand_ASAN_POISON (internal_fn
, gcall
*)
482 /* This should get expanded in the sanopt pass. */
485 expand_ASAN_POISON_USE (internal_fn
, gcall
*)
490 /* This should get expanded in the tsan pass. */
493 expand_TSAN_FUNC_EXIT (internal_fn
, gcall
*)
498 /* This should get expanded in the lower pass. */
501 expand_FALLTHROUGH (internal_fn
, gcall
*call
)
503 error_at (gimple_location (call
),
504 "invalid use of attribute %<fallthrough%>");
507 /* Return minimum precision needed to represent all values
508 of ARG in SIGNed integral type. */
511 get_min_precision (tree arg
, signop sign
)
513 int prec
= TYPE_PRECISION (TREE_TYPE (arg
));
515 signop orig_sign
= sign
;
516 if (TREE_CODE (arg
) == INTEGER_CST
)
519 if (TYPE_SIGN (TREE_TYPE (arg
)) != sign
)
521 widest_int w
= wi::to_widest (arg
);
522 w
= wi::ext (w
, prec
, sign
);
523 p
= wi::min_precision (w
, sign
);
526 p
= wi::min_precision (wi::to_wide (arg
), sign
);
527 return MIN (p
, prec
);
529 while (CONVERT_EXPR_P (arg
)
530 && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg
, 0)))
531 && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg
, 0))) <= prec
)
533 arg
= TREE_OPERAND (arg
, 0);
534 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
536 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
538 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
539 return prec
+ (orig_sign
!= sign
);
540 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
543 return prec
+ (orig_sign
!= sign
);
545 if (TREE_CODE (arg
) != SSA_NAME
)
546 return prec
+ (orig_sign
!= sign
);
547 wide_int arg_min
, arg_max
;
548 while (get_range_info (arg
, &arg_min
, &arg_max
) != VR_RANGE
)
550 gimple
*g
= SSA_NAME_DEF_STMT (arg
);
551 if (is_gimple_assign (g
)
552 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g
)))
554 tree t
= gimple_assign_rhs1 (g
);
555 if (INTEGRAL_TYPE_P (TREE_TYPE (t
))
556 && TYPE_PRECISION (TREE_TYPE (t
)) <= prec
)
559 if (TYPE_PRECISION (TREE_TYPE (arg
)) < prec
)
561 if (TYPE_UNSIGNED (TREE_TYPE (arg
)))
563 else if (sign
== UNSIGNED
&& get_range_pos_neg (arg
) != 1)
564 return prec
+ (orig_sign
!= sign
);
565 prec
= TYPE_PRECISION (TREE_TYPE (arg
));
568 return prec
+ (orig_sign
!= sign
);
572 return prec
+ (orig_sign
!= sign
);
574 if (sign
== TYPE_SIGN (TREE_TYPE (arg
)))
576 int p1
= wi::min_precision (arg_min
, sign
);
577 int p2
= wi::min_precision (arg_max
, sign
);
579 prec
= MIN (prec
, p1
);
581 else if (sign
== UNSIGNED
&& !wi::neg_p (arg_min
, SIGNED
))
583 int p
= wi::min_precision (arg_max
, UNSIGNED
);
584 prec
= MIN (prec
, p
);
586 return prec
+ (orig_sign
!= sign
);
589 /* Helper for expand_*_overflow. Set the __imag__ part to true
590 (1 except for signed:1 type, in which case store -1). */
593 expand_arith_set_overflow (tree lhs
, rtx target
)
595 if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
))) == 1
596 && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
))))
597 write_complex_part (target
, constm1_rtx
, true);
599 write_complex_part (target
, const1_rtx
, true);
602 /* Helper for expand_*_overflow. Store RES into the __real__ part
603 of TARGET. If RES has larger MODE than __real__ part of TARGET,
604 set the __imag__ part to 1 if RES doesn't fit into it. Similarly
605 if LHS has smaller precision than its mode. */
608 expand_arith_overflow_result_store (tree lhs
, rtx target
,
609 scalar_int_mode mode
, rtx res
)
611 scalar_int_mode tgtmode
612 = as_a
<scalar_int_mode
> (GET_MODE_INNER (GET_MODE (target
)));
616 rtx_code_label
*done_label
= gen_label_rtx ();
617 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
618 lres
= convert_modes (tgtmode
, mode
, res
, uns
);
619 gcc_assert (GET_MODE_PRECISION (tgtmode
) < GET_MODE_PRECISION (mode
));
620 do_compare_rtx_and_jump (res
, convert_modes (mode
, tgtmode
, lres
, uns
),
621 EQ
, true, mode
, NULL_RTX
, NULL
, done_label
,
622 profile_probability::very_likely ());
623 expand_arith_set_overflow (lhs
, target
);
624 emit_label (done_label
);
626 int prec
= TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs
)));
627 int tgtprec
= GET_MODE_PRECISION (tgtmode
);
630 rtx_code_label
*done_label
= gen_label_rtx ();
631 int uns
= TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs
)));
636 = immed_wide_int_const (wi::shifted_mask (0, prec
, false, tgtprec
),
638 lres
= expand_simple_binop (tgtmode
, AND
, res
, mask
, NULL_RTX
,
639 true, OPTAB_LIB_WIDEN
);
643 lres
= expand_shift (LSHIFT_EXPR
, tgtmode
, res
, tgtprec
- prec
,
645 lres
= expand_shift (RSHIFT_EXPR
, tgtmode
, lres
, tgtprec
- prec
,
648 do_compare_rtx_and_jump (res
, lres
,
649 EQ
, true, tgtmode
, NULL_RTX
, NULL
, done_label
,
650 profile_probability::very_likely ());
651 expand_arith_set_overflow (lhs
, target
);
652 emit_label (done_label
);
654 write_complex_part (target
, lres
, false);
657 /* Helper for expand_*_overflow. Store RES into TARGET. */
660 expand_ubsan_result_store (rtx target
, rtx res
)
662 if (GET_CODE (target
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (target
))
663 /* If this is a scalar in a register that is stored in a wider mode
664 than the declared mode, compute the result into its declared mode
665 and then convert to the wider mode. Our value is the computed
667 convert_move (SUBREG_REG (target
), res
, SUBREG_PROMOTED_SIGN (target
));
669 emit_move_insn (target
, res
);
672 /* Add sub/add overflow checking to the statement STMT.
673 CODE says whether the operation is +, or -. */
676 expand_addsub_overflow (location_t loc
, tree_code code
, tree lhs
,
677 tree arg0
, tree arg1
, bool unsr_p
, bool uns0_p
,
678 bool uns1_p
, bool is_ubsan
, tree
*datap
)
680 rtx res
, target
= NULL_RTX
;
682 rtx_code_label
*done_label
= gen_label_rtx ();
683 rtx_code_label
*do_error
= gen_label_rtx ();
684 do_pending_stack_adjust ();
685 rtx op0
= expand_normal (arg0
);
686 rtx op1
= expand_normal (arg1
);
687 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
688 int prec
= GET_MODE_PRECISION (mode
);
689 rtx sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
693 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
697 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
699 write_complex_part (target
, const0_rtx
, true);
702 /* We assume both operands and result have the same precision
703 here (GET_MODE_BITSIZE (mode)), S stands for signed type
704 with that precision, U for unsigned type with that precision,
705 sgn for unsigned most significant bit in that precision.
706 s1 is signed first operand, u1 is unsigned first operand,
707 s2 is signed second operand, u2 is unsigned second operand,
708 sr is signed result, ur is unsigned result and the following
709 rules say how to compute result (which is always result of
710 the operands as if both were unsigned, cast to the right
711 signedness) and how to compute whether operation overflowed.
714 res = (S) ((U) s1 + (U) s2)
715 ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
717 res = (S) ((U) s1 - (U) s2)
718 ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
721 ovf = res < u1 (or jump on carry, but RTL opts will handle it)
724 ovf = res > u1 (or jump on carry, but RTL opts will handle it)
726 res = (S) ((U) s1 + u2)
727 ovf = ((U) res ^ sgn) < u2
732 ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
734 res = (S) ((U) s1 - u2)
735 ovf = u2 > ((U) s1 ^ sgn)
738 ovf = s1 < 0 || u2 > (U) s1
741 ovf = u1 >= ((U) s2 ^ sgn)
746 ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
748 res = (U) s1 + (U) s2
749 ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
752 ovf = (U) res < u2 || res < 0
755 ovf = u1 >= u2 ? res < 0 : res >= 0
757 res = (U) s1 - (U) s2
758 ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0) */
760 if (code
== PLUS_EXPR
&& uns0_p
&& !uns1_p
)
762 /* PLUS_EXPR is commutative, if operand signedness differs,
763 canonicalize to the first operand being signed and second
764 unsigned to simplify following code. */
765 std::swap (op0
, op1
);
766 std::swap (arg0
, arg1
);
772 if (uns0_p
&& uns1_p
&& unsr_p
)
774 insn_code icode
= optab_handler (code
== PLUS_EXPR
? uaddv4_optab
775 : usubv4_optab
, mode
);
776 if (icode
!= CODE_FOR_nothing
)
778 class expand_operand ops
[4];
779 rtx_insn
*last
= get_last_insn ();
781 res
= gen_reg_rtx (mode
);
782 create_output_operand (&ops
[0], res
, mode
);
783 create_input_operand (&ops
[1], op0
, mode
);
784 create_input_operand (&ops
[2], op1
, mode
);
785 create_fixed_operand (&ops
[3], do_error
);
786 if (maybe_expand_insn (icode
, 4, ops
))
788 last
= get_last_insn ();
789 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
791 && any_condjump_p (last
)
792 && !find_reg_note (last
, REG_BR_PROB
, 0))
793 add_reg_br_prob_note (last
,
794 profile_probability::very_unlikely ());
795 emit_jump (done_label
);
799 delete_insns_since (last
);
802 /* Compute the operation. On RTL level, the addition is always
804 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
805 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
807 /* For PLUS_EXPR, the operation is commutative, so we can pick
808 operand to compare against. For prec <= BITS_PER_WORD, I think
809 preferring REG operand is better over CONST_INT, because
810 the CONST_INT might enlarge the instruction or CSE would need
811 to figure out we'd already loaded it into a register before.
812 For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
813 as then the multi-word comparison can be perhaps simplified. */
814 if (code
== PLUS_EXPR
815 && (prec
<= BITS_PER_WORD
816 ? (CONST_SCALAR_INT_P (op0
) && REG_P (op1
))
817 : CONST_SCALAR_INT_P (op1
)))
819 do_compare_rtx_and_jump (res
, tem
, code
== PLUS_EXPR
? GEU
: LEU
,
820 true, mode
, NULL_RTX
, NULL
, done_label
,
821 profile_probability::very_likely ());
826 if (!uns0_p
&& uns1_p
&& !unsr_p
)
828 /* Compute the operation. On RTL level, the addition is always
830 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
831 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
832 rtx tem
= expand_binop (mode
, add_optab
,
833 code
== PLUS_EXPR
? res
: op0
, sgn
,
834 NULL_RTX
, false, OPTAB_LIB_WIDEN
);
835 do_compare_rtx_and_jump (tem
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
836 done_label
, profile_probability::very_likely ());
841 if (code
== PLUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
843 op1
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
845 /* As we've changed op1, we have to avoid using the value range
846 for the original argument. */
847 arg1
= error_mark_node
;
853 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& unsr_p
)
855 op0
= expand_binop (mode
, add_optab
, op0
, sgn
, NULL_RTX
, false,
857 /* As we've changed op0, we have to avoid using the value range
858 for the original argument. */
859 arg0
= error_mark_node
;
865 if (code
== MINUS_EXPR
&& !uns0_p
&& uns1_p
&& unsr_p
)
867 /* Compute the operation. On RTL level, the addition is always
869 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
871 int pos_neg
= get_range_pos_neg (arg0
);
873 /* If ARG0 is known to be always negative, this is always overflow. */
874 emit_jump (do_error
);
875 else if (pos_neg
== 3)
876 /* If ARG0 is not known to be always positive, check at runtime. */
877 do_compare_rtx_and_jump (op0
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
878 NULL
, do_error
, profile_probability::very_unlikely ());
879 do_compare_rtx_and_jump (op1
, op0
, LEU
, true, mode
, NULL_RTX
, NULL
,
880 done_label
, profile_probability::very_likely ());
885 if (code
== MINUS_EXPR
&& uns0_p
&& !uns1_p
&& !unsr_p
)
887 /* Compute the operation. On RTL level, the addition is always
889 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
891 rtx tem
= expand_binop (mode
, add_optab
, op1
, sgn
, NULL_RTX
, false,
893 do_compare_rtx_and_jump (op0
, tem
, LTU
, true, mode
, NULL_RTX
, NULL
,
894 done_label
, profile_probability::very_likely ());
899 if (code
== PLUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
901 /* Compute the operation. On RTL level, the addition is always
903 res
= expand_binop (mode
, add_optab
, op0
, op1
, NULL_RTX
, false,
905 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
906 NULL
, do_error
, profile_probability::very_unlikely ());
908 /* The operation is commutative, so we can pick operand to compare
909 against. For prec <= BITS_PER_WORD, I think preferring REG operand
910 is better over CONST_INT, because the CONST_INT might enlarge the
911 instruction or CSE would need to figure out we'd already loaded it
912 into a register before. For prec > BITS_PER_WORD, I think CONST_INT
913 might be more beneficial, as then the multi-word comparison can be
914 perhaps simplified. */
915 if (prec
<= BITS_PER_WORD
916 ? (CONST_SCALAR_INT_P (op1
) && REG_P (op0
))
917 : CONST_SCALAR_INT_P (op0
))
919 do_compare_rtx_and_jump (res
, tem
, GEU
, true, mode
, NULL_RTX
, NULL
,
920 done_label
, profile_probability::very_likely ());
925 if (!uns0_p
&& !uns1_p
&& unsr_p
)
927 /* Compute the operation. On RTL level, the addition is always
929 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
930 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
931 int pos_neg
= get_range_pos_neg (arg1
);
932 if (code
== PLUS_EXPR
)
934 int pos_neg0
= get_range_pos_neg (arg0
);
935 if (pos_neg0
!= 3 && pos_neg
== 3)
937 std::swap (op0
, op1
);
944 tem
= expand_binop (mode
, ((pos_neg
== 1) ^ (code
== MINUS_EXPR
))
945 ? and_optab
: ior_optab
,
946 op0
, res
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
947 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL
,
948 NULL
, done_label
, profile_probability::very_likely ());
952 rtx_code_label
*do_ior_label
= gen_label_rtx ();
953 do_compare_rtx_and_jump (op1
, const0_rtx
,
954 code
== MINUS_EXPR
? GE
: LT
, false, mode
,
955 NULL_RTX
, NULL
, do_ior_label
,
956 profile_probability::even ());
957 tem
= expand_binop (mode
, and_optab
, op0
, res
, NULL_RTX
, false,
959 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
960 NULL
, done_label
, profile_probability::very_likely ());
961 emit_jump (do_error
);
962 emit_label (do_ior_label
);
963 tem
= expand_binop (mode
, ior_optab
, op0
, res
, NULL_RTX
, false,
965 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
966 NULL
, done_label
, profile_probability::very_likely ());
972 if (code
== MINUS_EXPR
&& uns0_p
&& uns1_p
&& !unsr_p
)
974 /* Compute the operation. On RTL level, the addition is always
976 res
= expand_binop (mode
, sub_optab
, op0
, op1
, NULL_RTX
, false,
978 rtx_code_label
*op0_geu_op1
= gen_label_rtx ();
979 do_compare_rtx_and_jump (op0
, op1
, GEU
, true, mode
, NULL_RTX
, NULL
,
980 op0_geu_op1
, profile_probability::even ());
981 do_compare_rtx_and_jump (res
, const0_rtx
, LT
, false, mode
, NULL_RTX
,
982 NULL
, done_label
, profile_probability::very_likely ());
983 emit_jump (do_error
);
984 emit_label (op0_geu_op1
);
985 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
986 NULL
, done_label
, profile_probability::very_likely ());
990 gcc_assert (!uns0_p
&& !uns1_p
&& !unsr_p
);
995 insn_code icode
= optab_handler (code
== PLUS_EXPR
? addv4_optab
996 : subv4_optab
, mode
);
997 if (icode
!= CODE_FOR_nothing
)
999 class expand_operand ops
[4];
1000 rtx_insn
*last
= get_last_insn ();
1002 res
= gen_reg_rtx (mode
);
1003 create_output_operand (&ops
[0], res
, mode
);
1004 create_input_operand (&ops
[1], op0
, mode
);
1005 create_input_operand (&ops
[2], op1
, mode
);
1006 create_fixed_operand (&ops
[3], do_error
);
1007 if (maybe_expand_insn (icode
, 4, ops
))
1009 last
= get_last_insn ();
1010 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1012 && any_condjump_p (last
)
1013 && !find_reg_note (last
, REG_BR_PROB
, 0))
1014 add_reg_br_prob_note (last
,
1015 profile_probability::very_unlikely ());
1016 emit_jump (done_label
);
1017 goto do_error_label
;
1020 delete_insns_since (last
);
1023 /* Compute the operation. On RTL level, the addition is always
1025 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
1026 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
1028 /* If we can prove that one of the arguments (for MINUS_EXPR only
1029 the second operand, as subtraction is not commutative) is always
1030 non-negative or always negative, we can do just one comparison
1031 and conditional jump. */
1032 int pos_neg
= get_range_pos_neg (arg1
);
1033 if (code
== PLUS_EXPR
)
1035 int pos_neg0
= get_range_pos_neg (arg0
);
1036 if (pos_neg0
!= 3 && pos_neg
== 3)
1038 std::swap (op0
, op1
);
1043 /* Addition overflows if and only if the two operands have the same sign,
1044 and the result has the opposite sign. Subtraction overflows if and
1045 only if the two operands have opposite sign, and the subtrahend has
1046 the same sign as the result. Here 0 is counted as positive. */
1049 /* Compute op0 ^ op1 (operands have opposite sign). */
1050 rtx op_xor
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1053 /* Compute res ^ op1 (result and 2nd operand have opposite sign). */
1054 rtx res_xor
= expand_binop (mode
, xor_optab
, res
, op1
, NULL_RTX
, false,
1058 if (code
== PLUS_EXPR
)
1060 /* Compute (res ^ op1) & ~(op0 ^ op1). */
1061 tem
= expand_unop (mode
, one_cmpl_optab
, op_xor
, NULL_RTX
, false);
1062 tem
= expand_binop (mode
, and_optab
, res_xor
, tem
, NULL_RTX
, false,
1067 /* Compute (op0 ^ op1) & ~(res ^ op1). */
1068 tem
= expand_unop (mode
, one_cmpl_optab
, res_xor
, NULL_RTX
, false);
1069 tem
= expand_binop (mode
, and_optab
, op_xor
, tem
, NULL_RTX
, false,
1073 /* No overflow if the result has bit sign cleared. */
1074 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1075 NULL
, done_label
, profile_probability::very_likely ());
1078 /* Compare the result of the operation with the first operand.
1079 No overflow for addition if second operand is positive and result
1080 is larger or second operand is negative and result is smaller.
1081 Likewise for subtraction with sign of second operand flipped. */
1083 do_compare_rtx_and_jump (res
, op0
,
1084 (pos_neg
== 1) ^ (code
== MINUS_EXPR
) ? GE
: LE
,
1085 false, mode
, NULL_RTX
, NULL
, done_label
,
1086 profile_probability::very_likely ());
1090 emit_label (do_error
);
1093 /* Expand the ubsan builtin call. */
1095 fn
= ubsan_build_overflow_builtin (code
, loc
, TREE_TYPE (arg0
),
1099 do_pending_stack_adjust ();
1102 expand_arith_set_overflow (lhs
, target
);
1105 emit_label (done_label
);
1110 expand_ubsan_result_store (target
, res
);
1114 res
= expand_binop (mode
, add_optab
, res
, sgn
, NULL_RTX
, false,
1117 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1122 /* Add negate overflow checking to the statement STMT. */
1125 expand_neg_overflow (location_t loc
, tree lhs
, tree arg1
, bool is_ubsan
,
1130 rtx_code_label
*done_label
, *do_error
;
1131 rtx target
= NULL_RTX
;
1133 done_label
= gen_label_rtx ();
1134 do_error
= gen_label_rtx ();
1136 do_pending_stack_adjust ();
1137 op1
= expand_normal (arg1
);
1139 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1
));
1142 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1144 write_complex_part (target
, const0_rtx
, true);
1147 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
1148 if (icode
!= CODE_FOR_nothing
)
1150 class expand_operand ops
[3];
1151 rtx_insn
*last
= get_last_insn ();
1153 res
= gen_reg_rtx (mode
);
1154 create_output_operand (&ops
[0], res
, mode
);
1155 create_input_operand (&ops
[1], op1
, mode
);
1156 create_fixed_operand (&ops
[2], do_error
);
1157 if (maybe_expand_insn (icode
, 3, ops
))
1159 last
= get_last_insn ();
1160 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1162 && any_condjump_p (last
)
1163 && !find_reg_note (last
, REG_BR_PROB
, 0))
1164 add_reg_br_prob_note (last
,
1165 profile_probability::very_unlikely ());
1166 emit_jump (done_label
);
1170 delete_insns_since (last
);
1171 icode
= CODE_FOR_nothing
;
1175 if (icode
== CODE_FOR_nothing
)
1177 /* Compute the operation. On RTL level, the addition is always
1179 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1181 /* Compare the operand with the most negative value. */
1182 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
1183 do_compare_rtx_and_jump (op1
, minv
, NE
, true, mode
, NULL_RTX
, NULL
,
1184 done_label
, profile_probability::very_likely ());
1187 emit_label (do_error
);
1190 /* Expand the ubsan builtin call. */
1192 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, loc
, TREE_TYPE (arg1
),
1193 arg1
, NULL_TREE
, datap
);
1196 do_pending_stack_adjust ();
1199 expand_arith_set_overflow (lhs
, target
);
1202 emit_label (done_label
);
1207 expand_ubsan_result_store (target
, res
);
1209 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
1213 /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
1214 mode MODE can be expanded without using a libcall. */
1217 can_widen_mult_without_libcall (scalar_int_mode wmode
, scalar_int_mode mode
,
1218 rtx op0
, rtx op1
, bool uns
)
1220 if (find_widening_optab_handler (umul_widen_optab
, wmode
, mode
)
1221 != CODE_FOR_nothing
)
1224 if (find_widening_optab_handler (smul_widen_optab
, wmode
, mode
)
1225 != CODE_FOR_nothing
)
1228 rtx_insn
*last
= get_last_insn ();
1229 if (CONSTANT_P (op0
))
1230 op0
= convert_modes (wmode
, mode
, op0
, uns
);
1232 op0
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 1);
1233 if (CONSTANT_P (op1
))
1234 op1
= convert_modes (wmode
, mode
, op1
, uns
);
1236 op1
= gen_raw_REG (wmode
, LAST_VIRTUAL_REGISTER
+ 2);
1237 rtx ret
= expand_mult (wmode
, op0
, op1
, NULL_RTX
, uns
, true);
1238 delete_insns_since (last
);
1239 return ret
!= NULL_RTX
;
1242 /* Add mul overflow checking to the statement STMT. */
1245 expand_mul_overflow (location_t loc
, tree lhs
, tree arg0
, tree arg1
,
1246 bool unsr_p
, bool uns0_p
, bool uns1_p
, bool is_ubsan
,
1251 rtx_code_label
*done_label
, *do_error
;
1252 rtx target
= NULL_RTX
;
1254 enum insn_code icode
;
1256 done_label
= gen_label_rtx ();
1257 do_error
= gen_label_rtx ();
1259 do_pending_stack_adjust ();
1260 op0
= expand_normal (arg0
);
1261 op1
= expand_normal (arg1
);
1263 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0
));
1267 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
1269 write_complex_part (target
, const0_rtx
, true);
1273 gcc_assert (!unsr_p
&& !uns0_p
&& !uns1_p
);
1275 /* We assume both operands and result have the same precision
1276 here (GET_MODE_BITSIZE (mode)), S stands for signed type
1277 with that precision, U for unsigned type with that precision,
1278 sgn for unsigned most significant bit in that precision.
1279 s1 is signed first operand, u1 is unsigned first operand,
1280 s2 is signed second operand, u2 is unsigned second operand,
1281 sr is signed result, ur is unsigned result and the following
1282 rules say how to compute result (which is always result of
1283 the operands as if both were unsigned, cast to the right
1284 signedness) and how to compute whether operation overflowed.
1285 main_ovf (false) stands for jump on signed multiplication
1286 overflow or the main algorithm with uns == false.
1287 main_ovf (true) stands for jump on unsigned multiplication
1288 overflow or the main algorithm with uns == true.
1291 res = (S) ((U) s1 * (U) s2)
1292 ovf = main_ovf (false)
1295 ovf = main_ovf (true)
1298 ovf = (s1 < 0 && u2) || main_ovf (true)
1301 ovf = res < 0 || main_ovf (true)
1303 res = (S) ((U) s1 * u2)
1304 ovf = (S) u2 >= 0 ? main_ovf (false)
1305 : (s1 != 0 && (s1 != -1 || u2 != (U) res))
1307 t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
1308 t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
1310 ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true) */
1312 if (uns0_p
&& !uns1_p
)
1314 /* Multiplication is commutative, if operand signedness differs,
1315 canonicalize to the first operand being signed and second
1316 unsigned to simplify following code. */
1317 std::swap (op0
, op1
);
1318 std::swap (arg0
, arg1
);
1323 int pos_neg0
= get_range_pos_neg (arg0
);
1324 int pos_neg1
= get_range_pos_neg (arg1
);
1327 if (!uns0_p
&& uns1_p
&& unsr_p
)
1332 /* If s1 is non-negative, just perform normal u1 * u2 -> ur. */
1335 /* If s1 is negative, avoid the main code, just multiply and
1336 signal overflow if op1 is not 0. */
1337 struct separate_ops ops
;
1338 ops
.code
= MULT_EXPR
;
1339 ops
.type
= TREE_TYPE (arg1
);
1340 ops
.op0
= make_tree (ops
.type
, op0
);
1341 ops
.op1
= make_tree (ops
.type
, op1
);
1342 ops
.op2
= NULL_TREE
;
1344 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1345 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1346 NULL
, done_label
, profile_probability::very_likely ());
1347 goto do_error_label
;
1349 rtx_code_label
*do_main_label
;
1350 do_main_label
= gen_label_rtx ();
1351 do_compare_rtx_and_jump (op0
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1352 NULL
, do_main_label
, profile_probability::very_likely ());
1353 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1354 NULL
, do_main_label
, profile_probability::very_likely ());
1355 expand_arith_set_overflow (lhs
, target
);
1356 emit_label (do_main_label
);
1364 if (uns0_p
&& uns1_p
&& !unsr_p
)
1367 /* Rest of handling of this case after res is computed. */
1372 if (!uns0_p
&& uns1_p
&& !unsr_p
)
1379 /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
1380 avoid the main code, just multiply and signal overflow
1381 unless 0 * u2 or -1 * ((U) Smin). */
1382 struct separate_ops ops
;
1383 ops
.code
= MULT_EXPR
;
1384 ops
.type
= TREE_TYPE (arg1
);
1385 ops
.op0
= make_tree (ops
.type
, op0
);
1386 ops
.op1
= make_tree (ops
.type
, op1
);
1387 ops
.op2
= NULL_TREE
;
1389 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1390 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1391 NULL
, done_label
, profile_probability::very_likely ());
1392 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
1393 NULL
, do_error
, profile_probability::very_unlikely ());
1395 prec
= GET_MODE_PRECISION (mode
);
1397 sgn
= immed_wide_int_const (wi::min_value (prec
, SIGNED
), mode
);
1398 do_compare_rtx_and_jump (op1
, sgn
, EQ
, true, mode
, NULL_RTX
,
1399 NULL
, done_label
, profile_probability::very_likely ());
1400 goto do_error_label
;
1402 /* Rest of handling of this case after res is computed. */
1410 if (!uns0_p
&& !uns1_p
&& unsr_p
)
1413 switch (pos_neg0
| pos_neg1
)
1415 case 1: /* Both operands known to be non-negative. */
1417 case 2: /* Both operands known to be negative. */
1418 op0
= expand_unop (mode
, neg_optab
, op0
, NULL_RTX
, false);
1419 op1
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
1420 /* Avoid looking at arg0/arg1 ranges, as we've changed
1422 arg0
= error_mark_node
;
1423 arg1
= error_mark_node
;
1426 if ((pos_neg0
^ pos_neg1
) == 3)
1428 /* If one operand is known to be negative and the other
1429 non-negative, this overflows always, unless the non-negative
1430 one is 0. Just do normal multiply and set overflow
1431 unless one of the operands is 0. */
1432 struct separate_ops ops
;
1433 ops
.code
= MULT_EXPR
;
1435 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode
),
1437 ops
.op0
= make_tree (ops
.type
, op0
);
1438 ops
.op1
= make_tree (ops
.type
, op1
);
1439 ops
.op2
= NULL_TREE
;
1441 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1442 do_compare_rtx_and_jump (pos_neg0
== 1 ? op0
: op1
, const0_rtx
, EQ
,
1443 true, mode
, NULL_RTX
, NULL
, done_label
,
1444 profile_probability::very_likely ());
1445 goto do_error_label
;
1447 /* The general case, do all the needed comparisons at runtime. */
1448 rtx_code_label
*do_main_label
, *after_negate_label
;
1450 rop0
= gen_reg_rtx (mode
);
1451 rop1
= gen_reg_rtx (mode
);
1452 emit_move_insn (rop0
, op0
);
1453 emit_move_insn (rop1
, op1
);
1456 do_main_label
= gen_label_rtx ();
1457 after_negate_label
= gen_label_rtx ();
1458 tem
= expand_binop (mode
, and_optab
, op0
, op1
, NULL_RTX
, false,
1460 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1461 NULL
, after_negate_label
, profile_probability::very_likely ());
1462 /* Both arguments negative here, negate them and continue with
1463 normal unsigned overflow checking multiplication. */
1464 emit_move_insn (op0
, expand_unop (mode
, neg_optab
, op0
,
1466 emit_move_insn (op1
, expand_unop (mode
, neg_optab
, op1
,
1468 /* Avoid looking at arg0/arg1 ranges, as we might have changed
1470 arg0
= error_mark_node
;
1471 arg1
= error_mark_node
;
1472 emit_jump (do_main_label
);
1473 emit_label (after_negate_label
);
1474 tem
= expand_binop (mode
, xor_optab
, op0
, op1
, NULL_RTX
, false,
1476 do_compare_rtx_and_jump (tem
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1477 NULL
, do_main_label
,
1478 profile_probability::very_likely ());
1479 /* One argument is negative here, the other positive. This
1480 overflows always, unless one of the arguments is 0. But
1481 if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
1482 is, thus we can keep do_main code oring in overflow as is. */
1484 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1485 NULL
, do_main_label
,
1486 profile_probability::very_unlikely ());
1488 do_compare_rtx_and_jump (op1
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
1489 NULL
, do_main_label
,
1490 profile_probability::very_unlikely ());
1491 expand_arith_set_overflow (lhs
, target
);
1492 emit_label (do_main_label
);
1500 type
= build_nonstandard_integer_type (GET_MODE_PRECISION (mode
), uns
);
1501 sign
= uns
? UNSIGNED
: SIGNED
;
1502 icode
= optab_handler (uns
? umulv4_optab
: mulv4_optab
, mode
);
1504 && (integer_pow2p (arg0
) || integer_pow2p (arg1
))
1505 && (optimize_insn_for_speed_p () || icode
== CODE_FOR_nothing
))
1507 /* Optimize unsigned multiplication by power of 2 constant
1508 using 2 shifts, one for result, one to extract the shifted
1509 out bits to see if they are all zero.
1510 Don't do this if optimizing for size and we have umulv4_optab,
1511 in that case assume multiplication will be shorter.
1512 This is heuristics based on the single target that provides
1513 umulv4 right now (i?86/x86_64), if further targets add it, this
1514 might need to be revisited.
1515 Cases where both operands are constant should be folded already
1516 during GIMPLE, and cases where one operand is constant but not
1517 power of 2 are questionable, either the WIDEN_MULT_EXPR case
1518 below can be done without multiplication, just by shifts and adds,
1519 or we'd need to divide the result (and hope it actually doesn't
1520 really divide nor multiply) and compare the result of the division
1521 with the original operand. */
1526 if (integer_pow2p (arg0
))
1528 std::swap (opn0
, opn1
);
1529 std::swap (argn0
, argn1
);
1531 int cnt
= tree_log2 (argn1
);
1532 if (cnt
>= 0 && cnt
< GET_MODE_PRECISION (mode
))
1534 rtx upper
= const0_rtx
;
1535 res
= expand_shift (LSHIFT_EXPR
, mode
, opn0
, cnt
, NULL_RTX
, uns
);
1537 upper
= expand_shift (RSHIFT_EXPR
, mode
, opn0
,
1538 GET_MODE_PRECISION (mode
) - cnt
,
1540 do_compare_rtx_and_jump (upper
, const0_rtx
, EQ
, true, mode
,
1541 NULL_RTX
, NULL
, done_label
,
1542 profile_probability::very_likely ());
1543 goto do_error_label
;
1546 if (icode
!= CODE_FOR_nothing
)
1548 class expand_operand ops
[4];
1549 rtx_insn
*last
= get_last_insn ();
1551 res
= gen_reg_rtx (mode
);
1552 create_output_operand (&ops
[0], res
, mode
);
1553 create_input_operand (&ops
[1], op0
, mode
);
1554 create_input_operand (&ops
[2], op1
, mode
);
1555 create_fixed_operand (&ops
[3], do_error
);
1556 if (maybe_expand_insn (icode
, 4, ops
))
1558 last
= get_last_insn ();
1559 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
1561 && any_condjump_p (last
)
1562 && !find_reg_note (last
, REG_BR_PROB
, 0))
1563 add_reg_br_prob_note (last
,
1564 profile_probability::very_unlikely ());
1565 emit_jump (done_label
);
1569 delete_insns_since (last
);
1570 icode
= CODE_FOR_nothing
;
1574 if (icode
== CODE_FOR_nothing
)
1576 struct separate_ops ops
;
1577 int prec
= GET_MODE_PRECISION (mode
);
1578 scalar_int_mode hmode
, wmode
;
1579 ops
.op0
= make_tree (type
, op0
);
1580 ops
.op1
= make_tree (type
, op1
);
1581 ops
.op2
= NULL_TREE
;
1584 /* Optimize unsigned overflow check where we don't use the
1585 multiplication result, just whether overflow happened.
1586 If we can do MULT_HIGHPART_EXPR, that followed by
1587 comparison of the result against zero is cheapest.
1588 We'll still compute res, but it should be DCEd later. */
1594 && !(uns0_p
&& uns1_p
&& !unsr_p
)
1595 && can_mult_highpart_p (mode
, uns
) == 1
1596 && single_imm_use (lhs
, &use
, &use_stmt
)
1597 && is_gimple_assign (use_stmt
)
1598 && gimple_assign_rhs_code (use_stmt
) == IMAGPART_EXPR
)
1601 if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
1602 && targetm
.scalar_mode_supported_p (wmode
)
1603 && can_widen_mult_without_libcall (wmode
, mode
, op0
, op1
, uns
))
1606 ops
.code
= WIDEN_MULT_EXPR
;
1608 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), uns
);
1610 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
1611 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
, prec
,
1613 hipart
= convert_modes (mode
, wmode
, hipart
, uns
);
1614 res
= convert_modes (mode
, wmode
, res
, uns
);
1616 /* For the unsigned multiplication, there was overflow if
1617 HIPART is non-zero. */
1618 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1619 NULL_RTX
, NULL
, done_label
,
1620 profile_probability::very_likely ());
1623 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1625 /* RES is low half of the double width result, HIPART
1626 the high half. There was overflow if
1627 HIPART is different from RES < 0 ? -1 : 0. */
1628 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1629 NULL_RTX
, NULL
, done_label
,
1630 profile_probability::very_likely ());
1633 else if (can_mult_highpart_p (mode
, uns
) == 1)
1636 ops
.code
= MULT_HIGHPART_EXPR
;
1639 rtx hipart
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
,
1641 ops
.code
= MULT_EXPR
;
1642 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1644 /* For the unsigned multiplication, there was overflow if
1645 HIPART is non-zero. */
1646 do_compare_rtx_and_jump (hipart
, const0_rtx
, EQ
, true, mode
,
1647 NULL_RTX
, NULL
, done_label
,
1648 profile_probability::very_likely ());
1651 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
, prec
- 1,
1653 /* RES is low half of the double width result, HIPART
1654 the high half. There was overflow if
1655 HIPART is different from RES < 0 ? -1 : 0. */
1656 do_compare_rtx_and_jump (signbit
, hipart
, EQ
, true, mode
,
1657 NULL_RTX
, NULL
, done_label
,
1658 profile_probability::very_likely ());
1662 else if (int_mode_for_size (prec
/ 2, 1).exists (&hmode
)
1663 && 2 * GET_MODE_PRECISION (hmode
) == prec
)
1665 rtx_code_label
*large_op0
= gen_label_rtx ();
1666 rtx_code_label
*small_op0_large_op1
= gen_label_rtx ();
1667 rtx_code_label
*one_small_one_large
= gen_label_rtx ();
1668 rtx_code_label
*both_ops_large
= gen_label_rtx ();
1669 rtx_code_label
*after_hipart_neg
= uns
? NULL
: gen_label_rtx ();
1670 rtx_code_label
*after_lopart_neg
= uns
? NULL
: gen_label_rtx ();
1671 rtx_code_label
*do_overflow
= gen_label_rtx ();
1672 rtx_code_label
*hipart_different
= uns
? NULL
: gen_label_rtx ();
1674 unsigned int hprec
= GET_MODE_PRECISION (hmode
);
1675 rtx hipart0
= expand_shift (RSHIFT_EXPR
, mode
, op0
, hprec
,
1677 hipart0
= convert_modes (hmode
, mode
, hipart0
, uns
);
1678 rtx lopart0
= convert_modes (hmode
, mode
, op0
, uns
);
1679 rtx signbit0
= const0_rtx
;
1681 signbit0
= expand_shift (RSHIFT_EXPR
, hmode
, lopart0
, hprec
- 1,
1683 rtx hipart1
= expand_shift (RSHIFT_EXPR
, mode
, op1
, hprec
,
1685 hipart1
= convert_modes (hmode
, mode
, hipart1
, uns
);
1686 rtx lopart1
= convert_modes (hmode
, mode
, op1
, uns
);
1687 rtx signbit1
= const0_rtx
;
1689 signbit1
= expand_shift (RSHIFT_EXPR
, hmode
, lopart1
, hprec
- 1,
1692 res
= gen_reg_rtx (mode
);
1694 /* True if op0 resp. op1 are known to be in the range of
1696 bool op0_small_p
= false;
1697 bool op1_small_p
= false;
1698 /* True if op0 resp. op1 are known to have all zeros or all ones
1699 in the upper half of bits, but are not known to be
1701 bool op0_medium_p
= false;
1702 bool op1_medium_p
= false;
1703 /* -1 if op{0,1} is known to be negative, 0 if it is known to be
1704 nonnegative, 1 if unknown. */
1710 else if (pos_neg0
== 2)
1714 else if (pos_neg1
== 2)
1717 unsigned int mprec0
= prec
;
1718 if (arg0
!= error_mark_node
)
1719 mprec0
= get_min_precision (arg0
, sign
);
1720 if (mprec0
<= hprec
)
1722 else if (!uns
&& mprec0
<= hprec
+ 1)
1723 op0_medium_p
= true;
1724 unsigned int mprec1
= prec
;
1725 if (arg1
!= error_mark_node
)
1726 mprec1
= get_min_precision (arg1
, sign
);
1727 if (mprec1
<= hprec
)
1729 else if (!uns
&& mprec1
<= hprec
+ 1)
1730 op1_medium_p
= true;
1732 int smaller_sign
= 1;
1733 int larger_sign
= 1;
1736 smaller_sign
= op0_sign
;
1737 larger_sign
= op1_sign
;
1739 else if (op1_small_p
)
1741 smaller_sign
= op1_sign
;
1742 larger_sign
= op0_sign
;
1744 else if (op0_sign
== op1_sign
)
1746 smaller_sign
= op0_sign
;
1747 larger_sign
= op0_sign
;
1751 do_compare_rtx_and_jump (signbit0
, hipart0
, NE
, true, hmode
,
1752 NULL_RTX
, NULL
, large_op0
,
1753 profile_probability::unlikely ());
1756 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1757 NULL_RTX
, NULL
, small_op0_large_op1
,
1758 profile_probability::unlikely ());
1760 /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
1761 hmode to mode, the multiplication will never overflow. We can
1762 do just one hmode x hmode => mode widening multiplication. */
1763 tree halfstype
= build_nonstandard_integer_type (hprec
, uns
);
1764 ops
.op0
= make_tree (halfstype
, lopart0
);
1765 ops
.op1
= make_tree (halfstype
, lopart1
);
1766 ops
.code
= WIDEN_MULT_EXPR
;
1769 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1770 emit_move_insn (res
, thisres
);
1771 emit_jump (done_label
);
1773 emit_label (small_op0_large_op1
);
1775 /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
1776 but op1 is not, just swap the arguments and handle it as op1
1777 sign/zero extended, op0 not. */
1778 rtx larger
= gen_reg_rtx (mode
);
1779 rtx hipart
= gen_reg_rtx (hmode
);
1780 rtx lopart
= gen_reg_rtx (hmode
);
1781 emit_move_insn (larger
, op1
);
1782 emit_move_insn (hipart
, hipart1
);
1783 emit_move_insn (lopart
, lopart0
);
1784 emit_jump (one_small_one_large
);
1786 emit_label (large_op0
);
1789 do_compare_rtx_and_jump (signbit1
, hipart1
, NE
, true, hmode
,
1790 NULL_RTX
, NULL
, both_ops_large
,
1791 profile_probability::unlikely ());
1793 /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
1794 but op0 is not, prepare larger, hipart and lopart pseudos and
1795 handle it together with small_op0_large_op1. */
1796 emit_move_insn (larger
, op0
);
1797 emit_move_insn (hipart
, hipart0
);
1798 emit_move_insn (lopart
, lopart1
);
1800 emit_label (one_small_one_large
);
1802 /* lopart is the low part of the operand that is sign extended
1803 to mode, larger is the other operand, hipart is the
1804 high part of larger and lopart0 and lopart1 are the low parts
1806 We perform lopart0 * lopart1 and lopart * hipart widening
1808 tree halfutype
= build_nonstandard_integer_type (hprec
, 1);
1809 ops
.op0
= make_tree (halfutype
, lopart0
);
1810 ops
.op1
= make_tree (halfutype
, lopart1
);
1812 = expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1814 ops
.op0
= make_tree (halfutype
, lopart
);
1815 ops
.op1
= make_tree (halfutype
, hipart
);
1816 rtx loxhi
= gen_reg_rtx (mode
);
1817 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1818 emit_move_insn (loxhi
, tem
);
1822 /* if (hipart < 0) loxhi -= lopart << (bitsize / 2); */
1823 if (larger_sign
== 0)
1824 emit_jump (after_hipart_neg
);
1825 else if (larger_sign
!= -1)
1826 do_compare_rtx_and_jump (hipart
, const0_rtx
, GE
, false, hmode
,
1827 NULL_RTX
, NULL
, after_hipart_neg
,
1828 profile_probability::even ());
1830 tem
= convert_modes (mode
, hmode
, lopart
, 1);
1831 tem
= expand_shift (LSHIFT_EXPR
, mode
, tem
, hprec
, NULL_RTX
, 1);
1832 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, tem
, NULL_RTX
,
1834 emit_move_insn (loxhi
, tem
);
1836 emit_label (after_hipart_neg
);
1838 /* if (lopart < 0) loxhi -= larger; */
1839 if (smaller_sign
== 0)
1840 emit_jump (after_lopart_neg
);
1841 else if (smaller_sign
!= -1)
1842 do_compare_rtx_and_jump (lopart
, const0_rtx
, GE
, false, hmode
,
1843 NULL_RTX
, NULL
, after_lopart_neg
,
1844 profile_probability::even ());
1846 tem
= expand_simple_binop (mode
, MINUS
, loxhi
, larger
, NULL_RTX
,
1848 emit_move_insn (loxhi
, tem
);
1850 emit_label (after_lopart_neg
);
1853 /* loxhi += (uns) lo0xlo1 >> (bitsize / 2); */
1854 tem
= expand_shift (RSHIFT_EXPR
, mode
, lo0xlo1
, hprec
, NULL_RTX
, 1);
1855 tem
= expand_simple_binop (mode
, PLUS
, loxhi
, tem
, NULL_RTX
,
1857 emit_move_insn (loxhi
, tem
);
1859 /* if (loxhi >> (bitsize / 2)
1860 == (hmode) loxhi >> (bitsize / 2 - 1)) (if !uns)
1861 if (loxhi >> (bitsize / 2) == 0 (if uns). */
1862 rtx hipartloxhi
= expand_shift (RSHIFT_EXPR
, mode
, loxhi
, hprec
,
1864 hipartloxhi
= convert_modes (hmode
, mode
, hipartloxhi
, 0);
1865 rtx signbitloxhi
= const0_rtx
;
1867 signbitloxhi
= expand_shift (RSHIFT_EXPR
, hmode
,
1868 convert_modes (hmode
, mode
,
1870 hprec
- 1, NULL_RTX
, 0);
1872 do_compare_rtx_and_jump (signbitloxhi
, hipartloxhi
, NE
, true, hmode
,
1873 NULL_RTX
, NULL
, do_overflow
,
1874 profile_probability::very_unlikely ());
1876 /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
1877 rtx loxhishifted
= expand_shift (LSHIFT_EXPR
, mode
, loxhi
, hprec
,
1879 tem
= convert_modes (mode
, hmode
,
1880 convert_modes (hmode
, mode
, lo0xlo1
, 1), 1);
1882 tem
= expand_simple_binop (mode
, IOR
, loxhishifted
, tem
, res
,
1885 emit_move_insn (res
, tem
);
1886 emit_jump (done_label
);
1888 emit_label (both_ops_large
);
1890 /* If both operands are large (not sign (!uns) or zero (uns)
1891 extended from hmode), then perform the full multiplication
1892 which will be the result of the operation.
1893 The only cases which don't overflow are for signed multiplication
1894 some cases where both hipart0 and highpart1 are 0 or -1.
1895 For unsigned multiplication when high parts are both non-zero
1896 this overflows always. */
1897 ops
.code
= MULT_EXPR
;
1898 ops
.op0
= make_tree (type
, op0
);
1899 ops
.op1
= make_tree (type
, op1
);
1900 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1901 emit_move_insn (res
, tem
);
1907 tem
= expand_simple_binop (hmode
, PLUS
, hipart0
, const1_rtx
,
1908 NULL_RTX
, 1, OPTAB_WIDEN
);
1909 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1910 NULL_RTX
, NULL
, do_error
,
1911 profile_probability::very_unlikely ());
1916 tem
= expand_simple_binop (hmode
, PLUS
, hipart1
, const1_rtx
,
1917 NULL_RTX
, 1, OPTAB_WIDEN
);
1918 do_compare_rtx_and_jump (tem
, const1_rtx
, GTU
, true, hmode
,
1919 NULL_RTX
, NULL
, do_error
,
1920 profile_probability::very_unlikely ());
1923 /* At this point hipart{0,1} are both in [-1, 0]. If they are
1924 the same, overflow happened if res is non-positive, if they
1925 are different, overflow happened if res is positive. */
1926 if (op0_sign
!= 1 && op1_sign
!= 1 && op0_sign
!= op1_sign
)
1927 emit_jump (hipart_different
);
1928 else if (op0_sign
== 1 || op1_sign
== 1)
1929 do_compare_rtx_and_jump (hipart0
, hipart1
, NE
, true, hmode
,
1930 NULL_RTX
, NULL
, hipart_different
,
1931 profile_probability::even ());
1933 do_compare_rtx_and_jump (res
, const0_rtx
, LE
, false, mode
,
1934 NULL_RTX
, NULL
, do_error
,
1935 profile_probability::very_unlikely ());
1936 emit_jump (done_label
);
1938 emit_label (hipart_different
);
1940 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
,
1941 NULL_RTX
, NULL
, do_error
,
1942 profile_probability::very_unlikely ());
1943 emit_jump (done_label
);
1946 emit_label (do_overflow
);
1948 /* Overflow, do full multiplication and fallthru into do_error. */
1949 ops
.op0
= make_tree (type
, op0
);
1950 ops
.op1
= make_tree (type
, op1
);
1951 tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1952 emit_move_insn (res
, tem
);
1954 else if (GET_MODE_2XWIDER_MODE (mode
).exists (&wmode
)
1955 && targetm
.scalar_mode_supported_p (wmode
))
1956 /* Even emitting a libcall is better than not detecting overflow
1961 gcc_assert (!is_ubsan
);
1962 ops
.code
= MULT_EXPR
;
1964 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
1965 emit_jump (done_label
);
1970 emit_label (do_error
);
1973 /* Expand the ubsan builtin call. */
1975 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, loc
, TREE_TYPE (arg0
),
1979 do_pending_stack_adjust ();
1982 expand_arith_set_overflow (lhs
, target
);
1985 emit_label (done_label
);
1988 if (uns0_p
&& uns1_p
&& !unsr_p
)
1990 rtx_code_label
*all_done_label
= gen_label_rtx ();
1991 do_compare_rtx_and_jump (res
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
1992 NULL
, all_done_label
, profile_probability::very_likely ());
1993 expand_arith_set_overflow (lhs
, target
);
1994 emit_label (all_done_label
);
1998 if (!uns0_p
&& uns1_p
&& !unsr_p
&& pos_neg1
== 3)
2000 rtx_code_label
*all_done_label
= gen_label_rtx ();
2001 rtx_code_label
*set_noovf
= gen_label_rtx ();
2002 do_compare_rtx_and_jump (op1
, const0_rtx
, GE
, false, mode
, NULL_RTX
,
2003 NULL
, all_done_label
, profile_probability::very_likely ());
2004 expand_arith_set_overflow (lhs
, target
);
2005 do_compare_rtx_and_jump (op0
, const0_rtx
, EQ
, true, mode
, NULL_RTX
,
2006 NULL
, set_noovf
, profile_probability::very_likely ());
2007 do_compare_rtx_and_jump (op0
, constm1_rtx
, NE
, true, mode
, NULL_RTX
,
2008 NULL
, all_done_label
, profile_probability::very_unlikely ());
2009 do_compare_rtx_and_jump (op1
, res
, NE
, true, mode
, NULL_RTX
, NULL
,
2010 all_done_label
, profile_probability::very_unlikely ());
2011 emit_label (set_noovf
);
2012 write_complex_part (target
, const0_rtx
, true);
2013 emit_label (all_done_label
);
2019 expand_ubsan_result_store (target
, res
);
2021 expand_arith_overflow_result_store (lhs
, target
, mode
, res
);
2025 /* Expand UBSAN_CHECK_* internal function if it has vector operands. */
2028 expand_vector_ubsan_overflow (location_t loc
, enum tree_code code
, tree lhs
,
2029 tree arg0
, tree arg1
)
2031 poly_uint64 cnt
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0
));
2032 rtx_code_label
*loop_lab
= NULL
;
2033 rtx cntvar
= NULL_RTX
;
2034 tree cntv
= NULL_TREE
;
2035 tree eltype
= TREE_TYPE (TREE_TYPE (arg0
));
2036 tree sz
= TYPE_SIZE (eltype
);
2037 tree data
= NULL_TREE
;
2038 tree resv
= NULL_TREE
;
2039 rtx lhsr
= NULL_RTX
;
2040 rtx resvr
= NULL_RTX
;
2041 unsigned HOST_WIDE_INT const_cnt
= 0;
2042 bool use_loop_p
= (!cnt
.is_constant (&const_cnt
) || const_cnt
> 4);
2047 lhsr
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2048 if (!VECTOR_MODE_P (GET_MODE (lhsr
))
2049 || (op
= optab_for_tree_code (code
, TREE_TYPE (arg0
),
2050 optab_default
)) == unknown_optab
2051 || (optab_handler (op
, TYPE_MODE (TREE_TYPE (arg0
)))
2052 == CODE_FOR_nothing
))
2055 resv
= make_tree (TREE_TYPE (lhs
), lhsr
);
2058 resvr
= assign_temp (TREE_TYPE (lhs
), 1, 1);
2059 resv
= make_tree (TREE_TYPE (lhs
), resvr
);
2065 do_pending_stack_adjust ();
2066 loop_lab
= gen_label_rtx ();
2067 cntvar
= gen_reg_rtx (TYPE_MODE (sizetype
));
2068 cntv
= make_tree (sizetype
, cntvar
);
2069 emit_move_insn (cntvar
, const0_rtx
);
2070 emit_label (loop_lab
);
2072 if (TREE_CODE (arg0
) != VECTOR_CST
)
2074 rtx arg0r
= expand_normal (arg0
);
2075 arg0
= make_tree (TREE_TYPE (arg0
), arg0r
);
2077 if (TREE_CODE (arg1
) != VECTOR_CST
)
2079 rtx arg1r
= expand_normal (arg1
);
2080 arg1
= make_tree (TREE_TYPE (arg1
), arg1r
);
2082 for (unsigned int i
= 0; i
< (use_loop_p
? 1 : const_cnt
); i
++)
2084 tree op0
, op1
, res
= NULL_TREE
;
2087 tree atype
= build_array_type_nelts (eltype
, cnt
);
2088 op0
= uniform_vector_p (arg0
);
2089 if (op0
== NULL_TREE
)
2091 op0
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg0
);
2092 op0
= build4_loc (loc
, ARRAY_REF
, eltype
, op0
, cntv
,
2093 NULL_TREE
, NULL_TREE
);
2095 op1
= uniform_vector_p (arg1
);
2096 if (op1
== NULL_TREE
)
2098 op1
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, arg1
);
2099 op1
= build4_loc (loc
, ARRAY_REF
, eltype
, op1
, cntv
,
2100 NULL_TREE
, NULL_TREE
);
2104 res
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, atype
, resv
);
2105 res
= build4_loc (loc
, ARRAY_REF
, eltype
, res
, cntv
,
2106 NULL_TREE
, NULL_TREE
);
2111 tree bitpos
= bitsize_int (tree_to_uhwi (sz
) * i
);
2112 op0
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg0
, sz
, bitpos
);
2113 op1
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, arg1
, sz
, bitpos
);
2115 res
= fold_build3_loc (loc
, BIT_FIELD_REF
, eltype
, resv
, sz
,
2121 expand_addsub_overflow (loc
, PLUS_EXPR
, res
, op0
, op1
,
2122 false, false, false, true, &data
);
2125 if (use_loop_p
? integer_zerop (arg0
) : integer_zerop (op0
))
2126 expand_neg_overflow (loc
, res
, op1
, true, &data
);
2128 expand_addsub_overflow (loc
, MINUS_EXPR
, res
, op0
, op1
,
2129 false, false, false, true, &data
);
2132 expand_mul_overflow (loc
, res
, op0
, op1
, false, false, false,
2141 struct separate_ops ops
;
2142 ops
.code
= PLUS_EXPR
;
2143 ops
.type
= TREE_TYPE (cntv
);
2145 ops
.op1
= build_int_cst (TREE_TYPE (cntv
), 1);
2146 ops
.op2
= NULL_TREE
;
2148 rtx ret
= expand_expr_real_2 (&ops
, cntvar
, TYPE_MODE (sizetype
),
2151 emit_move_insn (cntvar
, ret
);
2152 rtx cntrtx
= gen_int_mode (cnt
, TYPE_MODE (sizetype
));
2153 do_compare_rtx_and_jump (cntvar
, cntrtx
, NE
, false,
2154 TYPE_MODE (sizetype
), NULL_RTX
, NULL
, loop_lab
,
2155 profile_probability::very_likely ());
2157 if (lhs
&& resv
== NULL_TREE
)
2159 struct separate_ops ops
;
2161 ops
.type
= TREE_TYPE (arg0
);
2164 ops
.op2
= NULL_TREE
;
2166 rtx ret
= expand_expr_real_2 (&ops
, lhsr
, TYPE_MODE (TREE_TYPE (arg0
)),
2169 emit_move_insn (lhsr
, ret
);
2172 emit_move_insn (lhsr
, resvr
);
2175 /* Expand UBSAN_CHECK_ADD call STMT. */
2178 expand_UBSAN_CHECK_ADD (internal_fn
, gcall
*stmt
)
2180 location_t loc
= gimple_location (stmt
);
2181 tree lhs
= gimple_call_lhs (stmt
);
2182 tree arg0
= gimple_call_arg (stmt
, 0);
2183 tree arg1
= gimple_call_arg (stmt
, 1);
2184 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2185 expand_vector_ubsan_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
);
2187 expand_addsub_overflow (loc
, PLUS_EXPR
, lhs
, arg0
, arg1
,
2188 false, false, false, true, NULL
);
2191 /* Expand UBSAN_CHECK_SUB call STMT. */
2194 expand_UBSAN_CHECK_SUB (internal_fn
, gcall
*stmt
)
2196 location_t loc
= gimple_location (stmt
);
2197 tree lhs
= gimple_call_lhs (stmt
);
2198 tree arg0
= gimple_call_arg (stmt
, 0);
2199 tree arg1
= gimple_call_arg (stmt
, 1);
2200 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2201 expand_vector_ubsan_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
);
2202 else if (integer_zerop (arg0
))
2203 expand_neg_overflow (loc
, lhs
, arg1
, true, NULL
);
2205 expand_addsub_overflow (loc
, MINUS_EXPR
, lhs
, arg0
, arg1
,
2206 false, false, false, true, NULL
);
2209 /* Expand UBSAN_CHECK_MUL call STMT. */
2212 expand_UBSAN_CHECK_MUL (internal_fn
, gcall
*stmt
)
2214 location_t loc
= gimple_location (stmt
);
2215 tree lhs
= gimple_call_lhs (stmt
);
2216 tree arg0
= gimple_call_arg (stmt
, 0);
2217 tree arg1
= gimple_call_arg (stmt
, 1);
2218 if (VECTOR_TYPE_P (TREE_TYPE (arg0
)))
2219 expand_vector_ubsan_overflow (loc
, MULT_EXPR
, lhs
, arg0
, arg1
);
2221 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, false, false, false, true,
2225 /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion. */
2228 expand_arith_overflow (enum tree_code code
, gimple
*stmt
)
2230 tree lhs
= gimple_call_lhs (stmt
);
2231 if (lhs
== NULL_TREE
)
2233 tree arg0
= gimple_call_arg (stmt
, 0);
2234 tree arg1
= gimple_call_arg (stmt
, 1);
2235 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2236 int uns0_p
= TYPE_UNSIGNED (TREE_TYPE (arg0
));
2237 int uns1_p
= TYPE_UNSIGNED (TREE_TYPE (arg1
));
2238 int unsr_p
= TYPE_UNSIGNED (type
);
2239 int prec0
= TYPE_PRECISION (TREE_TYPE (arg0
));
2240 int prec1
= TYPE_PRECISION (TREE_TYPE (arg1
));
2241 int precres
= TYPE_PRECISION (type
);
2242 location_t loc
= gimple_location (stmt
);
2243 if (!uns0_p
&& get_range_pos_neg (arg0
) == 1)
2245 if (!uns1_p
&& get_range_pos_neg (arg1
) == 1)
2247 int pr
= get_min_precision (arg0
, uns0_p
? UNSIGNED
: SIGNED
);
2248 prec0
= MIN (prec0
, pr
);
2249 pr
= get_min_precision (arg1
, uns1_p
? UNSIGNED
: SIGNED
);
2250 prec1
= MIN (prec1
, pr
);
2252 /* If uns0_p && uns1_p, precop is minimum needed precision
2253 of unsigned type to hold the exact result, otherwise
2254 precop is minimum needed precision of signed type to
2255 hold the exact result. */
2257 if (code
== MULT_EXPR
)
2258 precop
= prec0
+ prec1
+ (uns0_p
!= uns1_p
);
2261 if (uns0_p
== uns1_p
)
2262 precop
= MAX (prec0
, prec1
) + 1;
2264 precop
= MAX (prec0
+ 1, prec1
) + 1;
2266 precop
= MAX (prec0
, prec1
+ 1) + 1;
2268 int orig_precres
= precres
;
2272 if ((uns0_p
&& uns1_p
)
2273 ? ((precop
+ !unsr_p
) <= precres
2274 /* u1 - u2 -> ur can overflow, no matter what precision
2276 && (code
!= MINUS_EXPR
|| !unsr_p
))
2277 : (!unsr_p
&& precop
<= precres
))
2279 /* The infinity precision result will always fit into result. */
2280 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2281 write_complex_part (target
, const0_rtx
, true);
2282 scalar_int_mode mode
= SCALAR_INT_TYPE_MODE (type
);
2283 struct separate_ops ops
;
2286 ops
.op0
= fold_convert_loc (loc
, type
, arg0
);
2287 ops
.op1
= fold_convert_loc (loc
, type
, arg1
);
2288 ops
.op2
= NULL_TREE
;
2290 rtx tem
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
2291 expand_arith_overflow_result_store (lhs
, target
, mode
, tem
);
2295 /* For operations with low precision, if target doesn't have them, start
2296 with precres widening right away, otherwise do it only if the most
2297 simple cases can't be used. */
2298 const int min_precision
= targetm
.min_arithmetic_precision ();
2299 if (orig_precres
== precres
&& precres
< min_precision
)
2301 else if ((uns0_p
&& uns1_p
&& unsr_p
&& prec0
<= precres
2302 && prec1
<= precres
)
2303 || ((!uns0_p
|| !uns1_p
) && !unsr_p
2304 && prec0
+ uns0_p
<= precres
2305 && prec1
+ uns1_p
<= precres
))
2307 arg0
= fold_convert_loc (loc
, type
, arg0
);
2308 arg1
= fold_convert_loc (loc
, type
, arg1
);
2312 if (integer_zerop (arg0
) && !unsr_p
)
2314 expand_neg_overflow (loc
, lhs
, arg1
, false, NULL
);
2319 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2320 unsr_p
, unsr_p
, false, NULL
);
2323 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2324 unsr_p
, unsr_p
, false, NULL
);
2331 /* For sub-word operations, retry with a wider type first. */
2332 if (orig_precres
== precres
&& precop
<= BITS_PER_WORD
)
2334 int p
= MAX (min_precision
, precop
);
2335 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2336 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2339 p
= TYPE_PRECISION (optype
);
2343 unsr_p
= TYPE_UNSIGNED (optype
);
2349 if (prec0
<= precres
&& prec1
<= precres
)
2354 types
[0] = build_nonstandard_integer_type (precres
, 0);
2360 types
[1] = build_nonstandard_integer_type (precres
, 1);
2362 arg0
= fold_convert_loc (loc
, types
[uns0_p
], arg0
);
2363 arg1
= fold_convert_loc (loc
, types
[uns1_p
], arg1
);
2364 if (code
!= MULT_EXPR
)
2365 expand_addsub_overflow (loc
, code
, lhs
, arg0
, arg1
, unsr_p
,
2366 uns0_p
, uns1_p
, false, NULL
);
2368 expand_mul_overflow (loc
, lhs
, arg0
, arg1
, unsr_p
,
2369 uns0_p
, uns1_p
, false, NULL
);
2373 /* Retry with a wider type. */
2374 if (orig_precres
== precres
)
2376 int p
= MAX (prec0
, prec1
);
2377 scalar_int_mode m
= smallest_int_mode_for_size (p
);
2378 tree optype
= build_nonstandard_integer_type (GET_MODE_PRECISION (m
),
2381 p
= TYPE_PRECISION (optype
);
2385 unsr_p
= TYPE_UNSIGNED (optype
);
2396 /* Expand ADD_OVERFLOW STMT. */
2399 expand_ADD_OVERFLOW (internal_fn
, gcall
*stmt
)
2401 expand_arith_overflow (PLUS_EXPR
, stmt
);
2404 /* Expand SUB_OVERFLOW STMT. */
2407 expand_SUB_OVERFLOW (internal_fn
, gcall
*stmt
)
2409 expand_arith_overflow (MINUS_EXPR
, stmt
);
2412 /* Expand MUL_OVERFLOW STMT. */
2415 expand_MUL_OVERFLOW (internal_fn
, gcall
*stmt
)
2417 expand_arith_overflow (MULT_EXPR
, stmt
);
2420 /* This should get folded in tree-vectorizer.c. */
2423 expand_LOOP_VECTORIZED (internal_fn
, gcall
*)
2428 /* This should get folded in tree-vectorizer.c. */
2431 expand_LOOP_DIST_ALIAS (internal_fn
, gcall
*)
2436 /* Return a memory reference of type TYPE for argument INDEX of STMT.
2437 Use argument INDEX + 1 to derive the second (TBAA) operand. */
2440 expand_call_mem_ref (tree type
, gcall
*stmt
, int index
)
2442 tree addr
= gimple_call_arg (stmt
, index
);
2443 tree alias_ptr_type
= TREE_TYPE (gimple_call_arg (stmt
, index
+ 1));
2444 unsigned int align
= tree_to_shwi (gimple_call_arg (stmt
, index
+ 1));
2445 if (TYPE_ALIGN (type
) != align
)
2446 type
= build_aligned_type (type
, align
);
2449 if (TREE_CODE (tmp
) == SSA_NAME
)
2451 gimple
*def
= SSA_NAME_DEF_STMT (tmp
);
2452 if (gimple_assign_single_p (def
))
2453 tmp
= gimple_assign_rhs1 (def
);
2456 if (TREE_CODE (tmp
) == ADDR_EXPR
)
2458 tree mem
= TREE_OPERAND (tmp
, 0);
2459 if (TREE_CODE (mem
) == TARGET_MEM_REF
2460 && types_compatible_p (TREE_TYPE (mem
), type
))
2462 tree offset
= TMR_OFFSET (mem
);
2463 if (type
!= TREE_TYPE (mem
)
2464 || alias_ptr_type
!= TREE_TYPE (offset
)
2465 || !integer_zerop (offset
))
2467 mem
= copy_node (mem
);
2468 TMR_OFFSET (mem
) = wide_int_to_tree (alias_ptr_type
,
2469 wi::to_poly_wide (offset
));
2470 TREE_TYPE (mem
) = type
;
2476 return fold_build2 (MEM_REF
, type
, addr
, build_int_cst (alias_ptr_type
, 0));
2479 /* Expand MASK_LOAD{,_LANES} call STMT using optab OPTAB. */
2482 expand_mask_load_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2484 class expand_operand ops
[3];
2485 tree type
, lhs
, rhs
, maskt
;
2486 rtx mem
, target
, mask
;
2489 maskt
= gimple_call_arg (stmt
, 2);
2490 lhs
= gimple_call_lhs (stmt
);
2491 if (lhs
== NULL_TREE
)
2493 type
= TREE_TYPE (lhs
);
2494 rhs
= expand_call_mem_ref (type
, stmt
, 0);
2496 if (optab
== vec_mask_load_lanes_optab
)
2497 icode
= get_multi_vector_move (type
, optab
);
2499 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2500 TYPE_MODE (TREE_TYPE (maskt
)));
2502 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2503 gcc_assert (MEM_P (mem
));
2504 mask
= expand_normal (maskt
);
2505 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2506 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
2507 create_fixed_operand (&ops
[1], mem
);
2508 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2509 expand_insn (icode
, 3, ops
);
2512 #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
2514 /* Expand MASK_STORE{,_LANES} call STMT using optab OPTAB. */
2517 expand_mask_store_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2519 class expand_operand ops
[3];
2520 tree type
, lhs
, rhs
, maskt
;
2524 maskt
= gimple_call_arg (stmt
, 2);
2525 rhs
= gimple_call_arg (stmt
, 3);
2526 type
= TREE_TYPE (rhs
);
2527 lhs
= expand_call_mem_ref (type
, stmt
, 0);
2529 if (optab
== vec_mask_store_lanes_optab
)
2530 icode
= get_multi_vector_move (type
, optab
);
2532 icode
= convert_optab_handler (optab
, TYPE_MODE (type
),
2533 TYPE_MODE (TREE_TYPE (maskt
)));
2535 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2536 gcc_assert (MEM_P (mem
));
2537 mask
= expand_normal (maskt
);
2538 reg
= expand_normal (rhs
);
2539 create_fixed_operand (&ops
[0], mem
);
2540 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
2541 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
2542 expand_insn (icode
, 3, ops
);
2545 #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
2548 expand_ABNORMAL_DISPATCHER (internal_fn
, gcall
*)
2553 expand_BUILTIN_EXPECT (internal_fn
, gcall
*stmt
)
2555 /* When guessing was done, the hints should be already stripped away. */
2556 gcc_assert (!flag_guess_branch_prob
|| optimize
== 0 || seen_error ());
2559 tree lhs
= gimple_call_lhs (stmt
);
2561 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2563 target
= const0_rtx
;
2564 rtx val
= expand_expr (gimple_call_arg (stmt
, 0), target
, VOIDmode
, EXPAND_NORMAL
);
2565 if (lhs
&& val
!= target
)
2566 emit_move_insn (target
, val
);
2569 /* IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function
2570 should never be called. */
2573 expand_VA_ARG (internal_fn
, gcall
*)
2578 /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this
2579 dummy function should never be called. */
2582 expand_VEC_CONVERT (internal_fn
, gcall
*)
2587 /* Expand the IFN_UNIQUE function according to its first argument. */
2590 expand_UNIQUE (internal_fn
, gcall
*stmt
)
2592 rtx pattern
= NULL_RTX
;
2593 enum ifn_unique_kind kind
2594 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0));
2601 case IFN_UNIQUE_UNSPEC
:
2602 if (targetm
.have_unique ())
2603 pattern
= targetm
.gen_unique ();
2606 case IFN_UNIQUE_OACC_FORK
:
2607 case IFN_UNIQUE_OACC_JOIN
:
2608 if (targetm
.have_oacc_fork () && targetm
.have_oacc_join ())
2610 tree lhs
= gimple_call_lhs (stmt
);
2611 rtx target
= const0_rtx
;
2614 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2616 rtx data_dep
= expand_normal (gimple_call_arg (stmt
, 1));
2617 rtx axis
= expand_normal (gimple_call_arg (stmt
, 2));
2619 if (kind
== IFN_UNIQUE_OACC_FORK
)
2620 pattern
= targetm
.gen_oacc_fork (target
, data_dep
, axis
);
2622 pattern
= targetm
.gen_oacc_join (target
, data_dep
, axis
);
2630 emit_insn (pattern
);
2633 /* The size of an OpenACC compute dimension. */
2636 expand_GOACC_DIM_SIZE (internal_fn
, gcall
*stmt
)
2638 tree lhs
= gimple_call_lhs (stmt
);
2643 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2644 if (targetm
.have_oacc_dim_size ())
2646 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2647 VOIDmode
, EXPAND_NORMAL
);
2648 emit_insn (targetm
.gen_oacc_dim_size (target
, dim
));
2651 emit_move_insn (target
, GEN_INT (1));
2654 /* The position of an OpenACC execution engine along one compute axis. */
2657 expand_GOACC_DIM_POS (internal_fn
, gcall
*stmt
)
2659 tree lhs
= gimple_call_lhs (stmt
);
2664 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2665 if (targetm
.have_oacc_dim_pos ())
2667 rtx dim
= expand_expr (gimple_call_arg (stmt
, 0), NULL_RTX
,
2668 VOIDmode
, EXPAND_NORMAL
);
2669 emit_insn (targetm
.gen_oacc_dim_pos (target
, dim
));
2672 emit_move_insn (target
, const0_rtx
);
2675 /* This is expanded by oacc_device_lower pass. */
2678 expand_GOACC_LOOP (internal_fn
, gcall
*)
2683 /* This is expanded by oacc_device_lower pass. */
2686 expand_GOACC_REDUCTION (internal_fn
, gcall
*)
2691 /* This is expanded by oacc_device_lower pass. */
2694 expand_GOACC_TILE (internal_fn
, gcall
*)
2699 /* Set errno to EDOM. */
2702 expand_SET_EDOM (internal_fn
, gcall
*)
2705 #ifdef GEN_ERRNO_RTX
2706 rtx errno_rtx
= GEN_ERRNO_RTX
;
2708 rtx errno_rtx
= gen_rtx_MEM (word_mode
, gen_rtx_SYMBOL_REF (Pmode
, "errno"));
2710 emit_move_insn (errno_rtx
,
2711 gen_int_mode (TARGET_EDOM
, GET_MODE (errno_rtx
)));
2717 /* Expand atomic bit test and set. */
2720 expand_ATOMIC_BIT_TEST_AND_SET (internal_fn
, gcall
*call
)
2722 expand_ifn_atomic_bit_test_and (call
);
2725 /* Expand atomic bit test and complement. */
2728 expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn
, gcall
*call
)
2730 expand_ifn_atomic_bit_test_and (call
);
2733 /* Expand atomic bit test and reset. */
2736 expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn
, gcall
*call
)
2738 expand_ifn_atomic_bit_test_and (call
);
2741 /* Expand atomic bit test and set. */
2744 expand_ATOMIC_COMPARE_EXCHANGE (internal_fn
, gcall
*call
)
2746 expand_ifn_atomic_compare_exchange (call
);
2749 /* Expand LAUNDER to assignment, lhs = arg0. */
2752 expand_LAUNDER (internal_fn
, gcall
*call
)
2754 tree lhs
= gimple_call_lhs (call
);
2759 expand_assignment (lhs
, gimple_call_arg (call
, 0), false);
2762 /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB. */
2765 expand_scatter_store_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
2767 internal_fn ifn
= gimple_call_internal_fn (stmt
);
2768 int rhs_index
= internal_fn_stored_value_index (ifn
);
2769 int mask_index
= internal_fn_mask_index (ifn
);
2770 tree base
= gimple_call_arg (stmt
, 0);
2771 tree offset
= gimple_call_arg (stmt
, 1);
2772 tree scale
= gimple_call_arg (stmt
, 2);
2773 tree rhs
= gimple_call_arg (stmt
, rhs_index
);
2775 rtx base_rtx
= expand_normal (base
);
2776 rtx offset_rtx
= expand_normal (offset
);
2777 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
2778 rtx rhs_rtx
= expand_normal (rhs
);
2780 class expand_operand ops
[6];
2782 create_address_operand (&ops
[i
++], base_rtx
);
2783 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
2784 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
2785 create_integer_operand (&ops
[i
++], scale_int
);
2786 create_input_operand (&ops
[i
++], rhs_rtx
, TYPE_MODE (TREE_TYPE (rhs
)));
2787 if (mask_index
>= 0)
2789 tree mask
= gimple_call_arg (stmt
, mask_index
);
2790 rtx mask_rtx
= expand_normal (mask
);
2791 create_input_operand (&ops
[i
++], mask_rtx
, TYPE_MODE (TREE_TYPE (mask
)));
2794 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (rhs
)),
2795 TYPE_MODE (TREE_TYPE (offset
)));
2796 expand_insn (icode
, i
, ops
);
2799 /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB. */
2802 expand_gather_load_optab_fn (internal_fn
, gcall
*stmt
, direct_optab optab
)
2804 tree lhs
= gimple_call_lhs (stmt
);
2805 tree base
= gimple_call_arg (stmt
, 0);
2806 tree offset
= gimple_call_arg (stmt
, 1);
2807 tree scale
= gimple_call_arg (stmt
, 2);
2809 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2810 rtx base_rtx
= expand_normal (base
);
2811 rtx offset_rtx
= expand_normal (offset
);
2812 HOST_WIDE_INT scale_int
= tree_to_shwi (scale
);
2815 class expand_operand ops
[6];
2816 create_output_operand (&ops
[i
++], lhs_rtx
, TYPE_MODE (TREE_TYPE (lhs
)));
2817 create_address_operand (&ops
[i
++], base_rtx
);
2818 create_input_operand (&ops
[i
++], offset_rtx
, TYPE_MODE (TREE_TYPE (offset
)));
2819 create_integer_operand (&ops
[i
++], TYPE_UNSIGNED (TREE_TYPE (offset
)));
2820 create_integer_operand (&ops
[i
++], scale_int
);
2821 if (optab
== mask_gather_load_optab
)
2823 tree mask
= gimple_call_arg (stmt
, 4);
2824 rtx mask_rtx
= expand_normal (mask
);
2825 create_input_operand (&ops
[i
++], mask_rtx
, TYPE_MODE (TREE_TYPE (mask
)));
2827 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (TREE_TYPE (lhs
)),
2828 TYPE_MODE (TREE_TYPE (offset
)));
2829 expand_insn (icode
, i
, ops
);
2832 /* Expand DIVMOD() using:
2833 a) optab handler for udivmod/sdivmod if it is available.
2834 b) If optab_handler doesn't exist, generate call to
2835 target-specific divmod libfunc. */
2838 expand_DIVMOD (internal_fn
, gcall
*call_stmt
)
2840 tree lhs
= gimple_call_lhs (call_stmt
);
2841 tree arg0
= gimple_call_arg (call_stmt
, 0);
2842 tree arg1
= gimple_call_arg (call_stmt
, 1);
2844 gcc_assert (TREE_CODE (TREE_TYPE (lhs
)) == COMPLEX_TYPE
);
2845 tree type
= TREE_TYPE (TREE_TYPE (lhs
));
2846 machine_mode mode
= TYPE_MODE (type
);
2847 bool unsignedp
= TYPE_UNSIGNED (type
);
2848 optab tab
= (unsignedp
) ? udivmod_optab
: sdivmod_optab
;
2850 rtx op0
= expand_normal (arg0
);
2851 rtx op1
= expand_normal (arg1
);
2852 rtx target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2854 rtx quotient
, remainder
, libfunc
;
2856 /* Check if optab_handler exists for divmod_optab for given mode. */
2857 if (optab_handler (tab
, mode
) != CODE_FOR_nothing
)
2859 quotient
= gen_reg_rtx (mode
);
2860 remainder
= gen_reg_rtx (mode
);
2861 expand_twoval_binop (tab
, op0
, op1
, quotient
, remainder
, unsignedp
);
2864 /* Generate call to divmod libfunc if it exists. */
2865 else if ((libfunc
= optab_libfunc (tab
, mode
)) != NULL_RTX
)
2866 targetm
.expand_divmod_libfunc (libfunc
, mode
, op0
, op1
,
2867 "ient
, &remainder
);
2872 /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR. */
2873 expand_expr (build2 (COMPLEX_EXPR
, TREE_TYPE (lhs
),
2874 make_tree (TREE_TYPE (arg0
), quotient
),
2875 make_tree (TREE_TYPE (arg1
), remainder
)),
2876 target
, VOIDmode
, EXPAND_NORMAL
);
2882 expand_NOP (internal_fn
, gcall
*)
2884 /* Nothing. But it shouldn't really prevail. */
2887 /* Coroutines, all should have been processed at this stage. */
2890 expand_CO_FRAME (internal_fn
, gcall
*)
2896 expand_CO_YIELD (internal_fn
, gcall
*)
2902 expand_CO_SUSPN (internal_fn
, gcall
*)
2908 expand_CO_ACTOR (internal_fn
, gcall
*)
2913 /* Expand a call to FN using the operands in STMT. FN has a single
2914 output operand and NARGS input operands. */
2917 expand_direct_optab_fn (internal_fn fn
, gcall
*stmt
, direct_optab optab
,
2920 expand_operand
*ops
= XALLOCAVEC (expand_operand
, nargs
+ 1);
2922 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
2923 insn_code icode
= direct_optab_handler (optab
, TYPE_MODE (types
.first
));
2924 gcc_assert (icode
!= CODE_FOR_nothing
);
2926 tree lhs
= gimple_call_lhs (stmt
);
2927 rtx lhs_rtx
= NULL_RTX
;
2929 lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2931 /* Do not assign directly to a promoted subreg, since there is no
2932 guarantee that the instruction will leave the upper bits of the
2933 register in the state required by SUBREG_PROMOTED_SIGN. */
2935 if (dest
&& GET_CODE (dest
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (dest
))
2938 create_output_operand (&ops
[0], dest
, insn_data
[icode
].operand
[0].mode
);
2940 for (unsigned int i
= 0; i
< nargs
; ++i
)
2942 tree rhs
= gimple_call_arg (stmt
, i
);
2943 tree rhs_type
= TREE_TYPE (rhs
);
2944 rtx rhs_rtx
= expand_normal (rhs
);
2945 if (INTEGRAL_TYPE_P (rhs_type
))
2946 create_convert_operand_from (&ops
[i
+ 1], rhs_rtx
,
2947 TYPE_MODE (rhs_type
),
2948 TYPE_UNSIGNED (rhs_type
));
2950 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
));
2953 expand_insn (icode
, nargs
+ 1, ops
);
2954 if (lhs_rtx
&& !rtx_equal_p (lhs_rtx
, ops
[0].value
))
2956 /* If the return value has an integral type, convert the instruction
2957 result to that type. This is useful for things that return an
2958 int regardless of the size of the input. If the instruction result
2959 is smaller than required, assume that it is signed.
2961 If the return value has a nonintegral type, its mode must match
2962 the instruction result. */
2963 if (GET_CODE (lhs_rtx
) == SUBREG
&& SUBREG_PROMOTED_VAR_P (lhs_rtx
))
2965 /* If this is a scalar in a register that is stored in a wider
2966 mode than the declared mode, compute the result into its
2967 declared mode and then convert to the wider mode. */
2968 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
2969 rtx tmp
= convert_to_mode (GET_MODE (lhs_rtx
), ops
[0].value
, 0);
2970 convert_move (SUBREG_REG (lhs_rtx
), tmp
,
2971 SUBREG_PROMOTED_SIGN (lhs_rtx
));
2973 else if (GET_MODE (lhs_rtx
) == GET_MODE (ops
[0].value
))
2974 emit_move_insn (lhs_rtx
, ops
[0].value
);
2977 gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs
)));
2978 convert_move (lhs_rtx
, ops
[0].value
, 0);
2983 /* Expand WHILE_ULT call STMT using optab OPTAB. */
2986 expand_while_optab_fn (internal_fn
, gcall
*stmt
, convert_optab optab
)
2988 expand_operand ops
[3];
2991 tree lhs
= gimple_call_lhs (stmt
);
2992 tree lhs_type
= TREE_TYPE (lhs
);
2993 rtx lhs_rtx
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
2994 create_output_operand (&ops
[0], lhs_rtx
, TYPE_MODE (lhs_type
));
2996 for (unsigned int i
= 0; i
< 2; ++i
)
2998 tree rhs
= gimple_call_arg (stmt
, i
);
2999 rhs_type
[i
] = TREE_TYPE (rhs
);
3000 rtx rhs_rtx
= expand_normal (rhs
);
3001 create_input_operand (&ops
[i
+ 1], rhs_rtx
, TYPE_MODE (rhs_type
[i
]));
3004 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (rhs_type
[0]),
3005 TYPE_MODE (lhs_type
));
3007 expand_insn (icode
, 3, ops
);
3008 if (!rtx_equal_p (lhs_rtx
, ops
[0].value
))
3009 emit_move_insn (lhs_rtx
, ops
[0].value
);
3012 /* Expanders for optabs that can use expand_direct_optab_fn. */
3014 #define expand_unary_optab_fn(FN, STMT, OPTAB) \
3015 expand_direct_optab_fn (FN, STMT, OPTAB, 1)
3017 #define expand_binary_optab_fn(FN, STMT, OPTAB) \
3018 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3020 #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
3021 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3023 #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
3024 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3026 #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
3027 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3029 #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
3030 expand_direct_optab_fn (FN, STMT, OPTAB, 5)
3032 #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
3033 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3035 #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
3036 expand_direct_optab_fn (FN, STMT, OPTAB, 2)
3038 #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
3039 expand_direct_optab_fn (FN, STMT, OPTAB, 3)
3041 #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
3042 expand_direct_optab_fn (FN, STMT, OPTAB, 4)
3044 /* RETURN_TYPE and ARGS are a return type and argument list that are
3045 in principle compatible with FN (which satisfies direct_internal_fn_p).
3046 Return the types that should be used to determine whether the
3047 target supports FN. */
3050 direct_internal_fn_types (internal_fn fn
, tree return_type
, tree
*args
)
3052 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3053 tree type0
= (info
.type0
< 0 ? return_type
: TREE_TYPE (args
[info
.type0
]));
3054 tree type1
= (info
.type1
< 0 ? return_type
: TREE_TYPE (args
[info
.type1
]));
3055 return tree_pair (type0
, type1
);
3058 /* CALL is a call whose return type and arguments are in principle
3059 compatible with FN (which satisfies direct_internal_fn_p). Return the
3060 types that should be used to determine whether the target supports FN. */
3063 direct_internal_fn_types (internal_fn fn
, gcall
*call
)
3065 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3066 tree op0
= (info
.type0
< 0
3067 ? gimple_call_lhs (call
)
3068 : gimple_call_arg (call
, info
.type0
));
3069 tree op1
= (info
.type1
< 0
3070 ? gimple_call_lhs (call
)
3071 : gimple_call_arg (call
, info
.type1
));
3072 return tree_pair (TREE_TYPE (op0
), TREE_TYPE (op1
));
3075 /* Return true if OPTAB is supported for TYPES (whose modes should be
3076 the same) when the optimization type is OPT_TYPE. Used for simple
3080 direct_optab_supported_p (direct_optab optab
, tree_pair types
,
3081 optimization_type opt_type
)
3083 machine_mode mode
= TYPE_MODE (types
.first
);
3084 gcc_checking_assert (mode
== TYPE_MODE (types
.second
));
3085 return direct_optab_handler (optab
, mode
, opt_type
) != CODE_FOR_nothing
;
3088 /* Return true if OPTAB is supported for TYPES, where the first type
3089 is the destination and the second type is the source. Used for
3093 convert_optab_supported_p (convert_optab optab
, tree_pair types
,
3094 optimization_type opt_type
)
3096 return (convert_optab_handler (optab
, TYPE_MODE (types
.first
),
3097 TYPE_MODE (types
.second
), opt_type
)
3098 != CODE_FOR_nothing
);
3101 /* Return true if load/store lanes optab OPTAB is supported for
3102 array type TYPES.first when the optimization type is OPT_TYPE. */
3105 multi_vector_optab_supported_p (convert_optab optab
, tree_pair types
,
3106 optimization_type opt_type
)
3108 gcc_assert (TREE_CODE (types
.first
) == ARRAY_TYPE
);
3109 machine_mode imode
= TYPE_MODE (types
.first
);
3110 machine_mode vmode
= TYPE_MODE (TREE_TYPE (types
.first
));
3111 return (convert_optab_handler (optab
, imode
, vmode
, opt_type
)
3112 != CODE_FOR_nothing
);
3115 #define direct_unary_optab_supported_p direct_optab_supported_p
3116 #define direct_binary_optab_supported_p direct_optab_supported_p
3117 #define direct_ternary_optab_supported_p direct_optab_supported_p
3118 #define direct_cond_unary_optab_supported_p direct_optab_supported_p
3119 #define direct_cond_binary_optab_supported_p direct_optab_supported_p
3120 #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
3121 #define direct_mask_load_optab_supported_p direct_optab_supported_p
3122 #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
3123 #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
3124 #define direct_gather_load_optab_supported_p convert_optab_supported_p
3125 #define direct_mask_store_optab_supported_p direct_optab_supported_p
3126 #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
3127 #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
3128 #define direct_scatter_store_optab_supported_p convert_optab_supported_p
3129 #define direct_while_optab_supported_p convert_optab_supported_p
3130 #define direct_fold_extract_optab_supported_p direct_optab_supported_p
3131 #define direct_fold_left_optab_supported_p direct_optab_supported_p
3132 #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
3133 #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
3135 /* Return the optab used by internal function FN. */
3138 direct_internal_fn_optab (internal_fn fn
, tree_pair types
)
3142 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3143 case IFN_##CODE: break;
3144 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3145 case IFN_##CODE: return OPTAB##_optab;
3146 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3147 UNSIGNED_OPTAB, TYPE) \
3148 case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR) \
3149 ? UNSIGNED_OPTAB ## _optab \
3150 : SIGNED_OPTAB ## _optab);
3151 #include "internal-fn.def"
3159 /* Return the optab used by internal function FN. */
3162 direct_internal_fn_optab (internal_fn fn
)
3166 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3167 case IFN_##CODE: break;
3168 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3169 case IFN_##CODE: return OPTAB##_optab;
3170 #include "internal-fn.def"
3178 /* Return true if FN is supported for the types in TYPES when the
3179 optimization type is OPT_TYPE. The types are those associated with
3180 the "type0" and "type1" fields of FN's direct_internal_fn_info
3184 direct_internal_fn_supported_p (internal_fn fn
, tree_pair types
,
3185 optimization_type opt_type
)
3189 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
3190 case IFN_##CODE: break;
3191 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3193 return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
3195 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3196 UNSIGNED_OPTAB, TYPE) \
3199 optab which_optab = (TYPE_UNSIGNED (types.SELECTOR) \
3200 ? UNSIGNED_OPTAB ## _optab \
3201 : SIGNED_OPTAB ## _optab); \
3202 return direct_##TYPE##_optab_supported_p (which_optab, types, \
3205 #include "internal-fn.def"
3213 /* Return true if FN is supported for type TYPE when the optimization
3214 type is OPT_TYPE. The caller knows that the "type0" and "type1"
3215 fields of FN's direct_internal_fn_info structure are the same. */
3218 direct_internal_fn_supported_p (internal_fn fn
, tree type
,
3219 optimization_type opt_type
)
3221 const direct_internal_fn_info
&info
= direct_internal_fn (fn
);
3222 gcc_checking_assert (info
.type0
== info
.type1
);
3223 return direct_internal_fn_supported_p (fn
, tree_pair (type
, type
), opt_type
);
3226 /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
3227 given that STMT is a call to a direct internal function. */
3230 direct_internal_fn_supported_p (gcall
*stmt
, optimization_type opt_type
)
3232 internal_fn fn
= gimple_call_internal_fn (stmt
);
3233 tree_pair types
= direct_internal_fn_types (fn
, stmt
);
3234 return direct_internal_fn_supported_p (fn
, types
, opt_type
);
3237 /* If FN is commutative in two consecutive arguments, return the
3238 index of the first, otherwise return -1. */
3241 first_commutative_argument (internal_fn fn
)
3275 /* Return true if IFN_SET_EDOM is supported. */
3278 set_edom_supported_p (void)
3287 #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
3289 expand_##CODE (internal_fn fn, gcall *stmt) \
3291 expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
3293 #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
3294 UNSIGNED_OPTAB, TYPE) \
3296 expand_##CODE (internal_fn fn, gcall *stmt) \
3298 tree_pair types = direct_internal_fn_types (fn, stmt); \
3299 optab which_optab = direct_internal_fn_optab (fn, types); \
3300 expand_##TYPE##_optab_fn (fn, stmt, which_optab); \
3302 #include "internal-fn.def"
3304 /* Routines to expand each internal function, indexed by function number.
3305 Each routine has the prototype:
3307 expand_<NAME> (gcall *stmt)
3309 where STMT is the statement that performs the call. */
3310 static void (*const internal_fn_expanders
[]) (internal_fn
, gcall
*) = {
3311 #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
3312 #include "internal-fn.def"
3316 /* Invoke T(CODE, IFN) for each conditional function IFN that maps to a
3318 #define FOR_EACH_CODE_MAPPING(T) \
3319 T (PLUS_EXPR, IFN_COND_ADD) \
3320 T (MINUS_EXPR, IFN_COND_SUB) \
3321 T (MULT_EXPR, IFN_COND_MUL) \
3322 T (TRUNC_DIV_EXPR, IFN_COND_DIV) \
3323 T (TRUNC_MOD_EXPR, IFN_COND_MOD) \
3324 T (RDIV_EXPR, IFN_COND_RDIV) \
3325 T (MIN_EXPR, IFN_COND_MIN) \
3326 T (MAX_EXPR, IFN_COND_MAX) \
3327 T (BIT_AND_EXPR, IFN_COND_AND) \
3328 T (BIT_IOR_EXPR, IFN_COND_IOR) \
3329 T (BIT_XOR_EXPR, IFN_COND_XOR) \
3330 T (LSHIFT_EXPR, IFN_COND_SHL) \
3331 T (RSHIFT_EXPR, IFN_COND_SHR)
3333 /* Return a function that only performs CODE when a certain condition is met
3334 and that uses a given fallback value otherwise. For example, if CODE is
3335 a binary operation associated with conditional function FN:
3337 LHS = FN (COND, A, B, ELSE)
3339 is equivalent to the C expression:
3341 LHS = COND ? A CODE B : ELSE;
3343 operating elementwise if the operands are vectors.
3345 Return IFN_LAST if no such function exists. */
3348 get_conditional_internal_fn (tree_code code
)
3352 #define CASE(CODE, IFN) case CODE: return IFN;
3353 FOR_EACH_CODE_MAPPING(CASE
)
3360 /* If IFN implements the conditional form of a tree code, return that
3361 tree code, otherwise return ERROR_MARK. */
3364 conditional_internal_fn_code (internal_fn ifn
)
3368 #define CASE(CODE, IFN) case IFN: return CODE;
3369 FOR_EACH_CODE_MAPPING(CASE
)
3376 /* Invoke T(IFN) for each internal function IFN that also has an
3378 #define FOR_EACH_COND_FN_PAIR(T) \
3384 /* Return a function that only performs internal function FN when a
3385 certain condition is met and that uses a given fallback value otherwise.
3386 In other words, the returned function FN' is such that:
3388 LHS = FN' (COND, A1, ... An, ELSE)
3390 is equivalent to the C expression:
3392 LHS = COND ? FN (A1, ..., An) : ELSE;
3394 operating elementwise if the operands are vectors.
3396 Return IFN_LAST if no such function exists. */
3399 get_conditional_internal_fn (internal_fn fn
)
3403 #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
3404 FOR_EACH_COND_FN_PAIR(CASE
)
3411 /* If IFN implements the conditional form of an unconditional internal
3412 function, return that unconditional function, otherwise return IFN_LAST. */
3415 get_unconditional_internal_fn (internal_fn ifn
)
3419 #define CASE(NAME) case IFN_COND_##NAME: return IFN_##NAME;
3420 FOR_EACH_COND_FN_PAIR(CASE
)
3427 /* Return true if STMT can be interpreted as a conditional tree code
3428 operation of the form:
3430 LHS = COND ? OP (RHS1, ...) : ELSE;
3432 operating elementwise if the operands are vectors. This includes
3433 the case of an all-true COND, so that the operation always happens.
3435 When returning true, set:
3437 - *COND_OUT to the condition COND, or to NULL_TREE if the condition
3438 is known to be all-true
3439 - *CODE_OUT to the tree code
3440 - OPS[I] to operand I of *CODE_OUT
3441 - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
3442 condition is known to be all true. */
3445 can_interpret_as_conditional_op_p (gimple
*stmt
, tree
*cond_out
,
3446 tree_code
*code_out
,
3447 tree (&ops
)[3], tree
*else_out
)
3449 if (gassign
*assign
= dyn_cast
<gassign
*> (stmt
))
3451 *cond_out
= NULL_TREE
;
3452 *code_out
= gimple_assign_rhs_code (assign
);
3453 ops
[0] = gimple_assign_rhs1 (assign
);
3454 ops
[1] = gimple_assign_rhs2 (assign
);
3455 ops
[2] = gimple_assign_rhs3 (assign
);
3456 *else_out
= NULL_TREE
;
3459 if (gcall
*call
= dyn_cast
<gcall
*> (stmt
))
3460 if (gimple_call_internal_p (call
))
3462 internal_fn ifn
= gimple_call_internal_fn (call
);
3463 tree_code code
= conditional_internal_fn_code (ifn
);
3464 if (code
!= ERROR_MARK
)
3466 *cond_out
= gimple_call_arg (call
, 0);
3468 unsigned int nops
= gimple_call_num_args (call
) - 2;
3469 for (unsigned int i
= 0; i
< 3; ++i
)
3470 ops
[i
] = i
< nops
? gimple_call_arg (call
, i
+ 1) : NULL_TREE
;
3471 *else_out
= gimple_call_arg (call
, nops
+ 1);
3472 if (integer_truep (*cond_out
))
3474 *cond_out
= NULL_TREE
;
3475 *else_out
= NULL_TREE
;
3483 /* Return true if IFN is some form of load from memory. */
3486 internal_load_fn_p (internal_fn fn
)
3491 case IFN_LOAD_LANES
:
3492 case IFN_MASK_LOAD_LANES
:
3493 case IFN_GATHER_LOAD
:
3494 case IFN_MASK_GATHER_LOAD
:
3502 /* Return true if IFN is some form of store to memory. */
3505 internal_store_fn_p (internal_fn fn
)
3509 case IFN_MASK_STORE
:
3510 case IFN_STORE_LANES
:
3511 case IFN_MASK_STORE_LANES
:
3512 case IFN_SCATTER_STORE
:
3513 case IFN_MASK_SCATTER_STORE
:
3521 /* Return true if IFN is some form of gather load or scatter store. */
3524 internal_gather_scatter_fn_p (internal_fn fn
)
3528 case IFN_GATHER_LOAD
:
3529 case IFN_MASK_GATHER_LOAD
:
3530 case IFN_SCATTER_STORE
:
3531 case IFN_MASK_SCATTER_STORE
:
3539 /* If FN takes a vector mask argument, return the index of that argument,
3540 otherwise return -1. */
3543 internal_fn_mask_index (internal_fn fn
)
3548 case IFN_MASK_LOAD_LANES
:
3549 case IFN_MASK_STORE
:
3550 case IFN_MASK_STORE_LANES
:
3553 case IFN_MASK_GATHER_LOAD
:
3554 case IFN_MASK_SCATTER_STORE
:
3558 return (conditional_internal_fn_code (fn
) != ERROR_MARK
3559 || get_unconditional_internal_fn (fn
) != IFN_LAST
? 0 : -1);
3563 /* If FN takes a value that should be stored to memory, return the index
3564 of that argument, otherwise return -1. */
3567 internal_fn_stored_value_index (internal_fn fn
)
3571 case IFN_MASK_STORE
:
3572 case IFN_SCATTER_STORE
:
3573 case IFN_MASK_SCATTER_STORE
:
3581 /* Return true if the target supports gather load or scatter store function
3582 IFN. For loads, VECTOR_TYPE is the vector type of the load result,
3583 while for stores it is the vector type of the stored data argument.
3584 MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
3585 or stored. OFFSET_VECTOR_TYPE is the vector type that holds the
3586 offset from the shared base address of each loaded or stored element.
3587 SCALE is the amount by which these offsets should be multiplied
3588 *after* they have been extended to address width. */
3591 internal_gather_scatter_fn_supported_p (internal_fn ifn
, tree vector_type
,
3592 tree memory_element_type
,
3593 tree offset_vector_type
, int scale
)
3595 if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type
)),
3596 TYPE_SIZE (memory_element_type
)))
3598 if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type
),
3599 TYPE_VECTOR_SUBPARTS (offset_vector_type
)))
3601 optab optab
= direct_internal_fn_optab (ifn
);
3602 insn_code icode
= convert_optab_handler (optab
, TYPE_MODE (vector_type
),
3603 TYPE_MODE (offset_vector_type
));
3604 int output_ops
= internal_load_fn_p (ifn
) ? 1 : 0;
3605 bool unsigned_p
= TYPE_UNSIGNED (TREE_TYPE (offset_vector_type
));
3606 return (icode
!= CODE_FOR_nothing
3607 && insn_operand_matches (icode
, 2 + output_ops
, GEN_INT (unsigned_p
))
3608 && insn_operand_matches (icode
, 3 + output_ops
, GEN_INT (scale
)));
3611 /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
3612 for pointers of type TYPE when the accesses have LENGTH bytes and their
3613 common byte alignment is ALIGN. */
3616 internal_check_ptrs_fn_supported_p (internal_fn ifn
, tree type
,
3617 poly_uint64 length
, unsigned int align
)
3619 machine_mode mode
= TYPE_MODE (type
);
3620 optab optab
= direct_internal_fn_optab (ifn
);
3621 insn_code icode
= direct_optab_handler (optab
, mode
);
3622 if (icode
== CODE_FOR_nothing
)
3624 rtx length_rtx
= immed_wide_int_const (length
, mode
);
3625 return (insn_operand_matches (icode
, 3, length_rtx
)
3626 && insn_operand_matches (icode
, 4, GEN_INT (align
)));
3629 /* Expand STMT as though it were a call to internal function FN. */
3632 expand_internal_call (internal_fn fn
, gcall
*stmt
)
3634 internal_fn_expanders
[fn
] (fn
, stmt
);
3637 /* Expand STMT, which is a call to internal function FN. */
3640 expand_internal_call (gcall
*stmt
)
3642 expand_internal_call (gimple_call_internal_fn (stmt
), stmt
);
3645 /* If TYPE is a vector type, return true if IFN is a direct internal
3646 function that is supported for that type. If TYPE is a scalar type,
3647 return true if IFN is a direct internal function that is supported for
3648 the target's preferred vector version of TYPE. */
3651 vectorized_internal_fn_supported_p (internal_fn ifn
, tree type
)
3654 if (!VECTOR_TYPE_P (type
) && is_a
<scalar_mode
> (TYPE_MODE (type
), &smode
))
3656 machine_mode vmode
= targetm
.vectorize
.preferred_simd_mode (smode
);
3657 if (VECTOR_MODE_P (vmode
))
3658 type
= build_vector_type_for_mode (type
, vmode
);
3661 return (VECTOR_MODE_P (TYPE_MODE (type
))
3662 && direct_internal_fn_supported_p (ifn
, type
, OPTIMIZE_FOR_SPEED
));
3666 expand_PHI (internal_fn
, gcall
*)