2 Copyright (C) 2011-2013 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
23 #include "internal-fn.h"
25 #include "stor-layout.h"
28 #include "basic-block.h"
29 #include "tree-ssa-alias.h"
30 #include "internal-fn.h"
31 #include "gimple-expr.h"
38 /* The names of each internal function, indexed by function number. */
39 const char *const internal_fn_name_array
[] = {
40 #define DEF_INTERNAL_FN(CODE, FLAGS) #CODE,
41 #include "internal-fn.def"
42 #undef DEF_INTERNAL_FN
46 /* The ECF_* flags of each internal function, indexed by function number. */
47 const int internal_fn_flags_array
[] = {
48 #define DEF_INTERNAL_FN(CODE, FLAGS) FLAGS,
49 #include "internal-fn.def"
50 #undef DEF_INTERNAL_FN
54 /* ARRAY_TYPE is an array of vector modes. Return the associated insn
55 for load-lanes-style optab OPTAB. The insn must exist. */
58 get_multi_vector_move (tree array_type
, convert_optab optab
)
61 enum machine_mode imode
;
62 enum machine_mode vmode
;
64 gcc_assert (TREE_CODE (array_type
) == ARRAY_TYPE
);
65 imode
= TYPE_MODE (array_type
);
66 vmode
= TYPE_MODE (TREE_TYPE (array_type
));
68 icode
= convert_optab_handler (optab
, imode
, vmode
);
69 gcc_assert (icode
!= CODE_FOR_nothing
);
73 /* Expand LOAD_LANES call STMT. */
76 expand_LOAD_LANES (gimple stmt
)
78 struct expand_operand ops
[2];
82 lhs
= gimple_call_lhs (stmt
);
83 rhs
= gimple_call_arg (stmt
, 0);
84 type
= TREE_TYPE (lhs
);
86 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
87 mem
= expand_normal (rhs
);
89 gcc_assert (MEM_P (mem
));
90 PUT_MODE (mem
, TYPE_MODE (type
));
92 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
93 create_fixed_operand (&ops
[1], mem
);
94 expand_insn (get_multi_vector_move (type
, vec_load_lanes_optab
), 2, ops
);
97 /* Expand STORE_LANES call STMT. */
100 expand_STORE_LANES (gimple stmt
)
102 struct expand_operand ops
[2];
106 lhs
= gimple_call_lhs (stmt
);
107 rhs
= gimple_call_arg (stmt
, 0);
108 type
= TREE_TYPE (rhs
);
110 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
111 reg
= expand_normal (rhs
);
113 gcc_assert (MEM_P (target
));
114 PUT_MODE (target
, TYPE_MODE (type
));
116 create_fixed_operand (&ops
[0], target
);
117 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
118 expand_insn (get_multi_vector_move (type
, vec_store_lanes_optab
), 2, ops
);
122 expand_ANNOTATE (gimple stmt ATTRIBUTE_UNUSED
)
127 /* This should get expanded in adjust_simduid_builtins. */
130 expand_GOMP_SIMD_LANE (gimple stmt ATTRIBUTE_UNUSED
)
135 /* This should get expanded in adjust_simduid_builtins. */
138 expand_GOMP_SIMD_VF (gimple stmt ATTRIBUTE_UNUSED
)
143 /* This should get expanded in adjust_simduid_builtins. */
146 expand_GOMP_SIMD_LAST_LANE (gimple stmt ATTRIBUTE_UNUSED
)
151 /* This should get expanded in the sanopt pass. */
154 expand_UBSAN_NULL (gimple stmt ATTRIBUTE_UNUSED
)
159 /* Add sub/add overflow checking to the statement STMT.
160 CODE says whether the operation is +, or -. */
163 ubsan_expand_si_overflow_addsub_check (tree_code code
, gimple stmt
)
166 tree lhs
, fn
, arg0
, arg1
;
167 rtx done_label
, do_error
, target
= NULL_RTX
;
169 lhs
= gimple_call_lhs (stmt
);
170 arg0
= gimple_call_arg (stmt
, 0);
171 arg1
= gimple_call_arg (stmt
, 1);
172 done_label
= gen_label_rtx ();
173 do_error
= gen_label_rtx ();
174 do_pending_stack_adjust ();
175 op0
= expand_normal (arg0
);
176 op1
= expand_normal (arg1
);
178 enum machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
180 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
183 = optab_handler (code
== PLUS_EXPR
? addv4_optab
: subv4_optab
, mode
);
184 if (icode
!= CODE_FOR_nothing
)
186 struct expand_operand ops
[4];
187 rtx last
= get_last_insn ();
189 res
= gen_reg_rtx (mode
);
190 create_output_operand (&ops
[0], res
, mode
);
191 create_input_operand (&ops
[1], op0
, mode
);
192 create_input_operand (&ops
[2], op1
, mode
);
193 create_fixed_operand (&ops
[3], do_error
);
194 if (maybe_expand_insn (icode
, 4, ops
))
196 last
= get_last_insn ();
197 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
199 && any_condjump_p (last
)
200 && !find_reg_note (last
, REG_BR_PROB
, 0))
201 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
202 emit_jump (done_label
);
206 delete_insns_since (last
);
207 icode
= CODE_FOR_nothing
;
211 if (icode
== CODE_FOR_nothing
)
213 rtx sub_check
= gen_label_rtx ();
215 /* Compute the operation. On RTL level, the addition is always
217 res
= expand_binop (mode
, code
== PLUS_EXPR
? add_optab
: sub_optab
,
218 op0
, op1
, NULL_RTX
, false, OPTAB_LIB_WIDEN
);
220 /* If the op1 is negative, we have to use a different check. */
221 emit_cmp_and_jump_insns (op1
, const0_rtx
, LT
, NULL_RTX
, mode
,
222 false, sub_check
, PROB_EVEN
);
224 /* Compare the result of the operation with one of the operands. */
225 emit_cmp_and_jump_insns (res
, op0
, code
== PLUS_EXPR
? GE
: LE
,
226 NULL_RTX
, mode
, false, done_label
,
228 /* If we get here, we have to print the error. */
229 emit_jump (do_error
);
231 emit_label (sub_check
);
232 /* We have k = a + b for b < 0 here. k <= a must hold. */
233 emit_cmp_and_jump_insns (res
, op0
, code
== PLUS_EXPR
? LE
: GE
,
234 NULL_RTX
, mode
, false, done_label
,
238 emit_label (do_error
);
239 /* Expand the ubsan builtin call. */
241 fn
= ubsan_build_overflow_builtin (code
, gimple_location (stmt
),
242 TREE_TYPE (arg0
), arg0
, arg1
);
245 do_pending_stack_adjust ();
248 emit_label (done_label
);
251 emit_move_insn (target
, res
);
254 /* Add negate overflow checking to the statement STMT. */
257 ubsan_expand_si_overflow_neg_check (gimple stmt
)
261 rtx done_label
, do_error
, target
= NULL_RTX
;
263 lhs
= gimple_call_lhs (stmt
);
264 arg1
= gimple_call_arg (stmt
, 1);
265 done_label
= gen_label_rtx ();
266 do_error
= gen_label_rtx ();
268 do_pending_stack_adjust ();
269 op1
= expand_normal (arg1
);
271 enum machine_mode mode
= TYPE_MODE (TREE_TYPE (arg1
));
273 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
275 enum insn_code icode
= optab_handler (negv3_optab
, mode
);
276 if (icode
!= CODE_FOR_nothing
)
278 struct expand_operand ops
[3];
279 rtx last
= get_last_insn ();
281 res
= gen_reg_rtx (mode
);
282 create_output_operand (&ops
[0], res
, mode
);
283 create_input_operand (&ops
[1], op1
, mode
);
284 create_fixed_operand (&ops
[2], do_error
);
285 if (maybe_expand_insn (icode
, 3, ops
))
287 last
= get_last_insn ();
288 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
290 && any_condjump_p (last
)
291 && !find_reg_note (last
, REG_BR_PROB
, 0))
292 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
293 emit_jump (done_label
);
297 delete_insns_since (last
);
298 icode
= CODE_FOR_nothing
;
302 if (icode
== CODE_FOR_nothing
)
304 /* Compute the operation. On RTL level, the addition is always
306 res
= expand_unop (mode
, neg_optab
, op1
, NULL_RTX
, false);
308 /* Compare the operand with the most negative value. */
309 rtx minv
= expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1
)));
310 emit_cmp_and_jump_insns (op1
, minv
, NE
, NULL_RTX
, mode
, false,
311 done_label
, PROB_VERY_LIKELY
);
314 emit_label (do_error
);
315 /* Expand the ubsan builtin call. */
317 fn
= ubsan_build_overflow_builtin (NEGATE_EXPR
, gimple_location (stmt
),
318 TREE_TYPE (arg1
), arg1
, NULL_TREE
);
321 do_pending_stack_adjust ();
324 emit_label (done_label
);
327 emit_move_insn (target
, res
);
330 /* Add mul overflow checking to the statement STMT. */
333 ubsan_expand_si_overflow_mul_check (gimple stmt
)
336 tree lhs
, fn
, arg0
, arg1
;
337 rtx done_label
, do_error
, target
= NULL_RTX
;
339 lhs
= gimple_call_lhs (stmt
);
340 arg0
= gimple_call_arg (stmt
, 0);
341 arg1
= gimple_call_arg (stmt
, 1);
342 done_label
= gen_label_rtx ();
343 do_error
= gen_label_rtx ();
345 do_pending_stack_adjust ();
346 op0
= expand_normal (arg0
);
347 op1
= expand_normal (arg1
);
349 enum machine_mode mode
= TYPE_MODE (TREE_TYPE (arg0
));
351 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
353 enum insn_code icode
= optab_handler (mulv4_optab
, mode
);
354 if (icode
!= CODE_FOR_nothing
)
356 struct expand_operand ops
[4];
357 rtx last
= get_last_insn ();
359 res
= gen_reg_rtx (mode
);
360 create_output_operand (&ops
[0], res
, mode
);
361 create_input_operand (&ops
[1], op0
, mode
);
362 create_input_operand (&ops
[2], op1
, mode
);
363 create_fixed_operand (&ops
[3], do_error
);
364 if (maybe_expand_insn (icode
, 4, ops
))
366 last
= get_last_insn ();
367 if (profile_status_for_fn (cfun
) != PROFILE_ABSENT
369 && any_condjump_p (last
)
370 && !find_reg_note (last
, REG_BR_PROB
, 0))
371 add_int_reg_note (last
, REG_BR_PROB
, PROB_VERY_UNLIKELY
);
372 emit_jump (done_label
);
376 delete_insns_since (last
);
377 icode
= CODE_FOR_nothing
;
381 if (icode
== CODE_FOR_nothing
)
383 struct separate_ops ops
;
387 ops
.location
= gimple_location (stmt
);
388 if (GET_MODE_2XWIDER_MODE (mode
) != VOIDmode
389 && targetm
.scalar_mode_supported_p (GET_MODE_2XWIDER_MODE (mode
)))
391 enum machine_mode wmode
= GET_MODE_2XWIDER_MODE (mode
);
392 ops
.code
= WIDEN_MULT_EXPR
;
394 = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode
), 0);
396 res
= expand_expr_real_2 (&ops
, NULL_RTX
, wmode
, EXPAND_NORMAL
);
397 rtx hipart
= expand_shift (RSHIFT_EXPR
, wmode
, res
,
398 GET_MODE_PRECISION (mode
), NULL_RTX
, 0);
399 hipart
= gen_lowpart (mode
, hipart
);
400 res
= gen_lowpart (mode
, res
);
401 rtx signbit
= expand_shift (RSHIFT_EXPR
, mode
, res
,
402 GET_MODE_PRECISION (mode
) - 1,
404 /* RES is low half of the double width result, HIPART
405 the high half. There was overflow if
406 HIPART is different from RES < 0 ? -1 : 0. */
407 emit_cmp_and_jump_insns (signbit
, hipart
, EQ
, NULL_RTX
, mode
,
408 false, done_label
, PROB_VERY_LIKELY
);
412 /* For now we don't instrument this. See __mulvDI3 in libgcc2.c
413 for what could be done. */
414 ops
.code
= MULT_EXPR
;
415 ops
.type
= TREE_TYPE (arg0
);
416 res
= expand_expr_real_2 (&ops
, NULL_RTX
, mode
, EXPAND_NORMAL
);
417 emit_jump (done_label
);
421 emit_label (do_error
);
422 /* Expand the ubsan builtin call. */
424 fn
= ubsan_build_overflow_builtin (MULT_EXPR
, gimple_location (stmt
),
425 TREE_TYPE (arg0
), arg0
, arg1
);
428 do_pending_stack_adjust ();
431 emit_label (done_label
);
434 emit_move_insn (target
, res
);
437 /* Expand UBSAN_CHECK_ADD call STMT. */
440 expand_UBSAN_CHECK_ADD (gimple stmt
)
442 ubsan_expand_si_overflow_addsub_check (PLUS_EXPR
, stmt
);
445 /* Expand UBSAN_CHECK_SUB call STMT. */
448 expand_UBSAN_CHECK_SUB (gimple stmt
)
450 if (integer_zerop (gimple_call_arg (stmt
, 0)))
451 ubsan_expand_si_overflow_neg_check (stmt
);
453 ubsan_expand_si_overflow_addsub_check (MINUS_EXPR
, stmt
);
456 /* Expand UBSAN_CHECK_MUL call STMT. */
459 expand_UBSAN_CHECK_MUL (gimple stmt
)
461 ubsan_expand_si_overflow_mul_check (stmt
);
464 /* This should get folded in tree-vectorizer.c. */
467 expand_LOOP_VECTORIZED (gimple stmt ATTRIBUTE_UNUSED
)
473 expand_MASK_LOAD (gimple stmt
)
475 struct expand_operand ops
[3];
476 tree type
, lhs
, rhs
, maskt
;
477 rtx mem
, target
, mask
;
479 maskt
= gimple_call_arg (stmt
, 2);
480 lhs
= gimple_call_lhs (stmt
);
481 type
= TREE_TYPE (lhs
);
482 rhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0),
483 gimple_call_arg (stmt
, 1));
485 mem
= expand_expr (rhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
486 gcc_assert (MEM_P (mem
));
487 mask
= expand_normal (maskt
);
488 target
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
489 create_output_operand (&ops
[0], target
, TYPE_MODE (type
));
490 create_fixed_operand (&ops
[1], mem
);
491 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
492 expand_insn (optab_handler (maskload_optab
, TYPE_MODE (type
)), 3, ops
);
496 expand_MASK_STORE (gimple stmt
)
498 struct expand_operand ops
[3];
499 tree type
, lhs
, rhs
, maskt
;
502 maskt
= gimple_call_arg (stmt
, 2);
503 rhs
= gimple_call_arg (stmt
, 3);
504 type
= TREE_TYPE (rhs
);
505 lhs
= fold_build2 (MEM_REF
, type
, gimple_call_arg (stmt
, 0),
506 gimple_call_arg (stmt
, 1));
508 mem
= expand_expr (lhs
, NULL_RTX
, VOIDmode
, EXPAND_WRITE
);
509 gcc_assert (MEM_P (mem
));
510 mask
= expand_normal (maskt
);
511 reg
= expand_normal (rhs
);
512 create_fixed_operand (&ops
[0], mem
);
513 create_input_operand (&ops
[1], reg
, TYPE_MODE (type
));
514 create_input_operand (&ops
[2], mask
, TYPE_MODE (TREE_TYPE (maskt
)));
515 expand_insn (optab_handler (maskstore_optab
, TYPE_MODE (type
)), 3, ops
);
518 /* Routines to expand each internal function, indexed by function number.
519 Each routine has the prototype:
521 expand_<NAME> (gimple stmt)
523 where STMT is the statement that performs the call. */
524 static void (*const internal_fn_expanders
[]) (gimple
) = {
525 #define DEF_INTERNAL_FN(CODE, FLAGS) expand_##CODE,
526 #include "internal-fn.def"
527 #undef DEF_INTERNAL_FN
531 /* Expand STMT, which is a call to internal function FN. */
534 expand_internal_call (gimple stmt
)
536 internal_fn_expanders
[(int) gimple_call_internal_fn (stmt
)] (stmt
);