re PR sanitizer/64265 (r217669 broke tsan)
[gcc.git] / gcc / tree-inline.c
1 /* Tree inlining.
2 Copyright (C) 2001-2015 Free Software Foundation, Inc.
3 Contributed by Alexandre Oliva <aoliva@redhat.com>
4
5 This file is part of GCC.
6
7 GCC is free software; you can redistribute it and/or modify
8 it under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
10 any later version.
11
12 GCC is distributed in the hope that it will be useful,
13 but WITHOUT ANY WARRANTY; without even the implied warranty of
14 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 GNU General Public License for more details.
16
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
20
21 #include "config.h"
22 #include "system.h"
23 #include "coretypes.h"
24 #include "tm.h"
25 #include "diagnostic-core.h"
26 #include "tree.h"
27 #include "stor-layout.h"
28 #include "calls.h"
29 #include "tree-inline.h"
30 #include "flags.h"
31 #include "params.h"
32 #include "input.h"
33 #include "insn-config.h"
34 #include "hashtab.h"
35 #include "langhooks.h"
36 #include "predict.h"
37 #include "vec.h"
38 #include "hash-set.h"
39 #include "machmode.h"
40 #include "hard-reg-set.h"
41 #include "function.h"
42 #include "dominance.h"
43 #include "cfg.h"
44 #include "cfganal.h"
45 #include "basic-block.h"
46 #include "tree-iterator.h"
47 #include "intl.h"
48 #include "tree-ssa-alias.h"
49 #include "internal-fn.h"
50 #include "gimple-fold.h"
51 #include "tree-eh.h"
52 #include "gimple-expr.h"
53 #include "is-a.h"
54 #include "gimple.h"
55 #include "gimplify.h"
56 #include "gimple-iterator.h"
57 #include "gimplify-me.h"
58 #include "gimple-walk.h"
59 #include "gimple-ssa.h"
60 #include "tree-cfg.h"
61 #include "tree-phinodes.h"
62 #include "ssa-iterators.h"
63 #include "stringpool.h"
64 #include "tree-ssanames.h"
65 #include "tree-into-ssa.h"
66 #include "expr.h"
67 #include "tree-dfa.h"
68 #include "tree-ssa.h"
69 #include "tree-pretty-print.h"
70 #include "except.h"
71 #include "debug.h"
72 #include "hash-map.h"
73 #include "plugin-api.h"
74 #include "ipa-ref.h"
75 #include "cgraph.h"
76 #include "alloc-pool.h"
77 #include "symbol-summary.h"
78 #include "ipa-prop.h"
79 #include "value-prof.h"
80 #include "tree-pass.h"
81 #include "target.h"
82 #include "cfgloop.h"
83 #include "builtins.h"
84 #include "tree-chkp.h"
85
86 #include "rtl.h" /* FIXME: For asm_str_count. */
87
88 /* I'm not real happy about this, but we need to handle gimple and
89 non-gimple trees. */
90
91 /* Inlining, Cloning, Versioning, Parallelization
92
93 Inlining: a function body is duplicated, but the PARM_DECLs are
94 remapped into VAR_DECLs, and non-void RETURN_EXPRs become
95 MODIFY_EXPRs that store to a dedicated returned-value variable.
96 The duplicated eh_region info of the copy will later be appended
97 to the info for the caller; the eh_region info in copied throwing
98 statements and RESX statements are adjusted accordingly.
99
100 Cloning: (only in C++) We have one body for a con/de/structor, and
101 multiple function decls, each with a unique parameter list.
102 Duplicate the body, using the given splay tree; some parameters
103 will become constants (like 0 or 1).
104
105 Versioning: a function body is duplicated and the result is a new
106 function rather than into blocks of an existing function as with
107 inlining. Some parameters will become constants.
108
109 Parallelization: a region of a function is duplicated resulting in
110 a new function. Variables may be replaced with complex expressions
111 to enable shared variable semantics.
112
113 All of these will simultaneously lookup any callgraph edges. If
114 we're going to inline the duplicated function body, and the given
115 function has some cloned callgraph nodes (one for each place this
116 function will be inlined) those callgraph edges will be duplicated.
117 If we're cloning the body, those callgraph edges will be
118 updated to point into the new body. (Note that the original
119 callgraph node and edge list will not be altered.)
120
121 See the CALL_EXPR handling case in copy_tree_body_r (). */
122
123 /* To Do:
124
125 o In order to make inlining-on-trees work, we pessimized
126 function-local static constants. In particular, they are now
127 always output, even when not addressed. Fix this by treating
128 function-local static constants just like global static
129 constants; the back-end already knows not to output them if they
130 are not needed.
131
132 o Provide heuristics to clamp inlining of recursive template
133 calls? */
134
135
136 /* Weights that estimate_num_insns uses to estimate the size of the
137 produced code. */
138
139 eni_weights eni_size_weights;
140
141 /* Weights that estimate_num_insns uses to estimate the time necessary
142 to execute the produced code. */
143
144 eni_weights eni_time_weights;
145
146 /* Prototypes. */
147
148 static tree declare_return_variable (copy_body_data *, tree, tree, tree,
149 basic_block);
150 static void remap_block (tree *, copy_body_data *);
151 static void copy_bind_expr (tree *, int *, copy_body_data *);
152 static void declare_inline_vars (tree, tree);
153 static void remap_save_expr (tree *, hash_map<tree, tree> *, int *);
154 static void prepend_lexical_block (tree current_block, tree new_block);
155 static tree copy_decl_to_var (tree, copy_body_data *);
156 static tree copy_result_decl_to_var (tree, copy_body_data *);
157 static tree copy_decl_maybe_to_var (tree, copy_body_data *);
158 static gimple_seq remap_gimple_stmt (gimple, copy_body_data *);
159 static bool delete_unreachable_blocks_update_callgraph (copy_body_data *id);
160 static void insert_init_stmt (copy_body_data *, basic_block, gimple);
161
162 /* Insert a tree->tree mapping for ID. Despite the name suggests
163 that the trees should be variables, it is used for more than that. */
164
165 void
166 insert_decl_map (copy_body_data *id, tree key, tree value)
167 {
168 id->decl_map->put (key, value);
169
170 /* Always insert an identity map as well. If we see this same new
171 node again, we won't want to duplicate it a second time. */
172 if (key != value)
173 id->decl_map->put (value, value);
174 }
175
176 /* Insert a tree->tree mapping for ID. This is only used for
177 variables. */
178
179 static void
180 insert_debug_decl_map (copy_body_data *id, tree key, tree value)
181 {
182 if (!gimple_in_ssa_p (id->src_cfun))
183 return;
184
185 if (!MAY_HAVE_DEBUG_STMTS)
186 return;
187
188 if (!target_for_debug_bind (key))
189 return;
190
191 gcc_assert (TREE_CODE (key) == PARM_DECL);
192 gcc_assert (TREE_CODE (value) == VAR_DECL);
193
194 if (!id->debug_map)
195 id->debug_map = new hash_map<tree, tree>;
196
197 id->debug_map->put (key, value);
198 }
199
200 /* If nonzero, we're remapping the contents of inlined debug
201 statements. If negative, an error has occurred, such as a
202 reference to a variable that isn't available in the inlined
203 context. */
204 static int processing_debug_stmt = 0;
205
206 /* Construct new SSA name for old NAME. ID is the inline context. */
207
208 static tree
209 remap_ssa_name (tree name, copy_body_data *id)
210 {
211 tree new_tree, var;
212 tree *n;
213
214 gcc_assert (TREE_CODE (name) == SSA_NAME);
215
216 n = id->decl_map->get (name);
217 if (n)
218 return unshare_expr (*n);
219
220 if (processing_debug_stmt)
221 {
222 if (SSA_NAME_IS_DEFAULT_DEF (name)
223 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL
224 && id->entry_bb == NULL
225 && single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (cfun)))
226 {
227 tree vexpr = make_node (DEBUG_EXPR_DECL);
228 gimple def_temp;
229 gimple_stmt_iterator gsi;
230 tree val = SSA_NAME_VAR (name);
231
232 n = id->decl_map->get (val);
233 if (n != NULL)
234 val = *n;
235 if (TREE_CODE (val) != PARM_DECL)
236 {
237 processing_debug_stmt = -1;
238 return name;
239 }
240 def_temp = gimple_build_debug_source_bind (vexpr, val, NULL);
241 DECL_ARTIFICIAL (vexpr) = 1;
242 TREE_TYPE (vexpr) = TREE_TYPE (name);
243 DECL_MODE (vexpr) = DECL_MODE (SSA_NAME_VAR (name));
244 gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
245 gsi_insert_before (&gsi, def_temp, GSI_SAME_STMT);
246 return vexpr;
247 }
248
249 processing_debug_stmt = -1;
250 return name;
251 }
252
253 /* Remap anonymous SSA names or SSA names of anonymous decls. */
254 var = SSA_NAME_VAR (name);
255 if (!var
256 || (!SSA_NAME_IS_DEFAULT_DEF (name)
257 && TREE_CODE (var) == VAR_DECL
258 && !VAR_DECL_IS_VIRTUAL_OPERAND (var)
259 && DECL_ARTIFICIAL (var)
260 && DECL_IGNORED_P (var)
261 && !DECL_NAME (var)))
262 {
263 struct ptr_info_def *pi;
264 new_tree = make_ssa_name (remap_type (TREE_TYPE (name), id));
265 if (!var && SSA_NAME_IDENTIFIER (name))
266 SET_SSA_NAME_VAR_OR_IDENTIFIER (new_tree, SSA_NAME_IDENTIFIER (name));
267 insert_decl_map (id, name, new_tree);
268 SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree)
269 = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name);
270 /* At least IPA points-to info can be directly transferred. */
271 if (id->src_cfun->gimple_df
272 && id->src_cfun->gimple_df->ipa_pta
273 && (pi = SSA_NAME_PTR_INFO (name))
274 && !pi->pt.anything)
275 {
276 struct ptr_info_def *new_pi = get_ptr_info (new_tree);
277 new_pi->pt = pi->pt;
278 }
279 return new_tree;
280 }
281
282 /* Do not set DEF_STMT yet as statement is not copied yet. We do that
283 in copy_bb. */
284 new_tree = remap_decl (var, id);
285
286 /* We might've substituted constant or another SSA_NAME for
287 the variable.
288
289 Replace the SSA name representing RESULT_DECL by variable during
290 inlining: this saves us from need to introduce PHI node in a case
291 return value is just partly initialized. */
292 if ((TREE_CODE (new_tree) == VAR_DECL || TREE_CODE (new_tree) == PARM_DECL)
293 && (!SSA_NAME_VAR (name)
294 || TREE_CODE (SSA_NAME_VAR (name)) != RESULT_DECL
295 || !id->transform_return_to_modify))
296 {
297 struct ptr_info_def *pi;
298 new_tree = make_ssa_name (new_tree);
299 insert_decl_map (id, name, new_tree);
300 SSA_NAME_OCCURS_IN_ABNORMAL_PHI (new_tree)
301 = SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name);
302 /* At least IPA points-to info can be directly transferred. */
303 if (id->src_cfun->gimple_df
304 && id->src_cfun->gimple_df->ipa_pta
305 && (pi = SSA_NAME_PTR_INFO (name))
306 && !pi->pt.anything)
307 {
308 struct ptr_info_def *new_pi = get_ptr_info (new_tree);
309 new_pi->pt = pi->pt;
310 }
311 if (SSA_NAME_IS_DEFAULT_DEF (name))
312 {
313 /* By inlining function having uninitialized variable, we might
314 extend the lifetime (variable might get reused). This cause
315 ICE in the case we end up extending lifetime of SSA name across
316 abnormal edge, but also increase register pressure.
317
318 We simply initialize all uninitialized vars by 0 except
319 for case we are inlining to very first BB. We can avoid
320 this for all BBs that are not inside strongly connected
321 regions of the CFG, but this is expensive to test. */
322 if (id->entry_bb
323 && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (name)
324 && (!SSA_NAME_VAR (name)
325 || TREE_CODE (SSA_NAME_VAR (name)) != PARM_DECL)
326 && (id->entry_bb != EDGE_SUCC (ENTRY_BLOCK_PTR_FOR_FN (cfun),
327 0)->dest
328 || EDGE_COUNT (id->entry_bb->preds) != 1))
329 {
330 gimple_stmt_iterator gsi = gsi_last_bb (id->entry_bb);
331 gimple init_stmt;
332 tree zero = build_zero_cst (TREE_TYPE (new_tree));
333
334 init_stmt = gimple_build_assign (new_tree, zero);
335 gsi_insert_after (&gsi, init_stmt, GSI_NEW_STMT);
336 SSA_NAME_IS_DEFAULT_DEF (new_tree) = 0;
337 }
338 else
339 {
340 SSA_NAME_DEF_STMT (new_tree) = gimple_build_nop ();
341 set_ssa_default_def (cfun, SSA_NAME_VAR (new_tree), new_tree);
342 }
343 }
344 }
345 else
346 insert_decl_map (id, name, new_tree);
347 return new_tree;
348 }
349
350 /* Remap DECL during the copying of the BLOCK tree for the function. */
351
352 tree
353 remap_decl (tree decl, copy_body_data *id)
354 {
355 tree *n;
356
357 /* We only remap local variables in the current function. */
358
359 /* See if we have remapped this declaration. */
360
361 n = id->decl_map->get (decl);
362
363 if (!n && processing_debug_stmt)
364 {
365 processing_debug_stmt = -1;
366 return decl;
367 }
368
369 /* If we didn't already have an equivalent for this declaration,
370 create one now. */
371 if (!n)
372 {
373 /* Make a copy of the variable or label. */
374 tree t = id->copy_decl (decl, id);
375
376 /* Remember it, so that if we encounter this local entity again
377 we can reuse this copy. Do this early because remap_type may
378 need this decl for TYPE_STUB_DECL. */
379 insert_decl_map (id, decl, t);
380
381 if (!DECL_P (t))
382 return t;
383
384 /* Remap types, if necessary. */
385 TREE_TYPE (t) = remap_type (TREE_TYPE (t), id);
386 if (TREE_CODE (t) == TYPE_DECL)
387 DECL_ORIGINAL_TYPE (t) = remap_type (DECL_ORIGINAL_TYPE (t), id);
388
389 /* Remap sizes as necessary. */
390 walk_tree (&DECL_SIZE (t), copy_tree_body_r, id, NULL);
391 walk_tree (&DECL_SIZE_UNIT (t), copy_tree_body_r, id, NULL);
392
393 /* If fields, do likewise for offset and qualifier. */
394 if (TREE_CODE (t) == FIELD_DECL)
395 {
396 walk_tree (&DECL_FIELD_OFFSET (t), copy_tree_body_r, id, NULL);
397 if (TREE_CODE (DECL_CONTEXT (t)) == QUAL_UNION_TYPE)
398 walk_tree (&DECL_QUALIFIER (t), copy_tree_body_r, id, NULL);
399 }
400
401 return t;
402 }
403
404 if (id->do_not_unshare)
405 return *n;
406 else
407 return unshare_expr (*n);
408 }
409
410 static tree
411 remap_type_1 (tree type, copy_body_data *id)
412 {
413 tree new_tree, t;
414
415 /* We do need a copy. build and register it now. If this is a pointer or
416 reference type, remap the designated type and make a new pointer or
417 reference type. */
418 if (TREE_CODE (type) == POINTER_TYPE)
419 {
420 new_tree = build_pointer_type_for_mode (remap_type (TREE_TYPE (type), id),
421 TYPE_MODE (type),
422 TYPE_REF_CAN_ALIAS_ALL (type));
423 if (TYPE_ATTRIBUTES (type) || TYPE_QUALS (type))
424 new_tree = build_type_attribute_qual_variant (new_tree,
425 TYPE_ATTRIBUTES (type),
426 TYPE_QUALS (type));
427 insert_decl_map (id, type, new_tree);
428 return new_tree;
429 }
430 else if (TREE_CODE (type) == REFERENCE_TYPE)
431 {
432 new_tree = build_reference_type_for_mode (remap_type (TREE_TYPE (type), id),
433 TYPE_MODE (type),
434 TYPE_REF_CAN_ALIAS_ALL (type));
435 if (TYPE_ATTRIBUTES (type) || TYPE_QUALS (type))
436 new_tree = build_type_attribute_qual_variant (new_tree,
437 TYPE_ATTRIBUTES (type),
438 TYPE_QUALS (type));
439 insert_decl_map (id, type, new_tree);
440 return new_tree;
441 }
442 else
443 new_tree = copy_node (type);
444
445 insert_decl_map (id, type, new_tree);
446
447 /* This is a new type, not a copy of an old type. Need to reassociate
448 variants. We can handle everything except the main variant lazily. */
449 t = TYPE_MAIN_VARIANT (type);
450 if (type != t)
451 {
452 t = remap_type (t, id);
453 TYPE_MAIN_VARIANT (new_tree) = t;
454 TYPE_NEXT_VARIANT (new_tree) = TYPE_NEXT_VARIANT (t);
455 TYPE_NEXT_VARIANT (t) = new_tree;
456 }
457 else
458 {
459 TYPE_MAIN_VARIANT (new_tree) = new_tree;
460 TYPE_NEXT_VARIANT (new_tree) = NULL;
461 }
462
463 if (TYPE_STUB_DECL (type))
464 TYPE_STUB_DECL (new_tree) = remap_decl (TYPE_STUB_DECL (type), id);
465
466 /* Lazily create pointer and reference types. */
467 TYPE_POINTER_TO (new_tree) = NULL;
468 TYPE_REFERENCE_TO (new_tree) = NULL;
469
470 /* Copy all types that may contain references to local variables; be sure to
471 preserve sharing in between type and its main variant when possible. */
472 switch (TREE_CODE (new_tree))
473 {
474 case INTEGER_TYPE:
475 case REAL_TYPE:
476 case FIXED_POINT_TYPE:
477 case ENUMERAL_TYPE:
478 case BOOLEAN_TYPE:
479 if (TYPE_MAIN_VARIANT (new_tree) != new_tree)
480 {
481 gcc_checking_assert (TYPE_MIN_VALUE (type) == TYPE_MIN_VALUE (TYPE_MAIN_VARIANT (type)));
482 gcc_checking_assert (TYPE_MAX_VALUE (type) == TYPE_MAX_VALUE (TYPE_MAIN_VARIANT (type)));
483
484 TYPE_MIN_VALUE (new_tree) = TYPE_MIN_VALUE (TYPE_MAIN_VARIANT (new_tree));
485 TYPE_MAX_VALUE (new_tree) = TYPE_MAX_VALUE (TYPE_MAIN_VARIANT (new_tree));
486 }
487 else
488 {
489 t = TYPE_MIN_VALUE (new_tree);
490 if (t && TREE_CODE (t) != INTEGER_CST)
491 walk_tree (&TYPE_MIN_VALUE (new_tree), copy_tree_body_r, id, NULL);
492
493 t = TYPE_MAX_VALUE (new_tree);
494 if (t && TREE_CODE (t) != INTEGER_CST)
495 walk_tree (&TYPE_MAX_VALUE (new_tree), copy_tree_body_r, id, NULL);
496 }
497 return new_tree;
498
499 case FUNCTION_TYPE:
500 if (TYPE_MAIN_VARIANT (new_tree) != new_tree
501 && TREE_TYPE (type) == TREE_TYPE (TYPE_MAIN_VARIANT (type)))
502 TREE_TYPE (new_tree) = TREE_TYPE (TYPE_MAIN_VARIANT (new_tree));
503 else
504 TREE_TYPE (new_tree) = remap_type (TREE_TYPE (new_tree), id);
505 if (TYPE_MAIN_VARIANT (new_tree) != new_tree
506 && TYPE_ARG_TYPES (type) == TYPE_ARG_TYPES (TYPE_MAIN_VARIANT (type)))
507 TYPE_ARG_TYPES (new_tree) = TYPE_ARG_TYPES (TYPE_MAIN_VARIANT (new_tree));
508 else
509 walk_tree (&TYPE_ARG_TYPES (new_tree), copy_tree_body_r, id, NULL);
510 return new_tree;
511
512 case ARRAY_TYPE:
513 if (TYPE_MAIN_VARIANT (new_tree) != new_tree
514 && TREE_TYPE (type) == TREE_TYPE (TYPE_MAIN_VARIANT (type)))
515 TREE_TYPE (new_tree) = TREE_TYPE (TYPE_MAIN_VARIANT (new_tree));
516 else
517 TREE_TYPE (new_tree) = remap_type (TREE_TYPE (new_tree), id);
518
519 if (TYPE_MAIN_VARIANT (new_tree) != new_tree)
520 {
521 gcc_checking_assert (TYPE_DOMAIN (type) == TYPE_DOMAIN (TYPE_MAIN_VARIANT (type)));
522 TYPE_DOMAIN (new_tree) = TYPE_DOMAIN (TYPE_MAIN_VARIANT (new_tree));
523 }
524 else
525 TYPE_DOMAIN (new_tree) = remap_type (TYPE_DOMAIN (new_tree), id);
526 break;
527
528 case RECORD_TYPE:
529 case UNION_TYPE:
530 case QUAL_UNION_TYPE:
531 if (TYPE_MAIN_VARIANT (type) != type
532 && TYPE_FIELDS (type) == TYPE_FIELDS (TYPE_MAIN_VARIANT (type)))
533 TYPE_FIELDS (new_tree) = TYPE_FIELDS (TYPE_MAIN_VARIANT (new_tree));
534 else
535 {
536 tree f, nf = NULL;
537
538 for (f = TYPE_FIELDS (new_tree); f ; f = DECL_CHAIN (f))
539 {
540 t = remap_decl (f, id);
541 DECL_CONTEXT (t) = new_tree;
542 DECL_CHAIN (t) = nf;
543 nf = t;
544 }
545 TYPE_FIELDS (new_tree) = nreverse (nf);
546 }
547 break;
548
549 case OFFSET_TYPE:
550 default:
551 /* Shouldn't have been thought variable sized. */
552 gcc_unreachable ();
553 }
554
555 /* All variants of type share the same size, so use the already remaped data. */
556 if (TYPE_MAIN_VARIANT (new_tree) != new_tree)
557 {
558 gcc_checking_assert (TYPE_SIZE (type) == TYPE_SIZE (TYPE_MAIN_VARIANT (type)));
559 gcc_checking_assert (TYPE_SIZE_UNIT (type) == TYPE_SIZE_UNIT (TYPE_MAIN_VARIANT (type)));
560
561 TYPE_SIZE (new_tree) = TYPE_SIZE (TYPE_MAIN_VARIANT (new_tree));
562 TYPE_SIZE_UNIT (new_tree) = TYPE_SIZE_UNIT (TYPE_MAIN_VARIANT (new_tree));
563 }
564 else
565 {
566 walk_tree (&TYPE_SIZE (new_tree), copy_tree_body_r, id, NULL);
567 walk_tree (&TYPE_SIZE_UNIT (new_tree), copy_tree_body_r, id, NULL);
568 }
569
570 return new_tree;
571 }
572
573 tree
574 remap_type (tree type, copy_body_data *id)
575 {
576 tree *node;
577 tree tmp;
578
579 if (type == NULL)
580 return type;
581
582 /* See if we have remapped this type. */
583 node = id->decl_map->get (type);
584 if (node)
585 return *node;
586
587 /* The type only needs remapping if it's variably modified. */
588 if (! variably_modified_type_p (type, id->src_fn))
589 {
590 insert_decl_map (id, type, type);
591 return type;
592 }
593
594 id->remapping_type_depth++;
595 tmp = remap_type_1 (type, id);
596 id->remapping_type_depth--;
597
598 return tmp;
599 }
600
601 /* Decide if DECL can be put into BLOCK_NONLOCAL_VARs. */
602
603 static bool
604 can_be_nonlocal (tree decl, copy_body_data *id)
605 {
606 /* We can not duplicate function decls. */
607 if (TREE_CODE (decl) == FUNCTION_DECL)
608 return true;
609
610 /* Local static vars must be non-local or we get multiple declaration
611 problems. */
612 if (TREE_CODE (decl) == VAR_DECL
613 && !auto_var_in_fn_p (decl, id->src_fn))
614 return true;
615
616 return false;
617 }
618
619 static tree
620 remap_decls (tree decls, vec<tree, va_gc> **nonlocalized_list,
621 copy_body_data *id)
622 {
623 tree old_var;
624 tree new_decls = NULL_TREE;
625
626 /* Remap its variables. */
627 for (old_var = decls; old_var; old_var = DECL_CHAIN (old_var))
628 {
629 tree new_var;
630
631 if (can_be_nonlocal (old_var, id))
632 {
633 /* We need to add this variable to the local decls as otherwise
634 nothing else will do so. */
635 if (TREE_CODE (old_var) == VAR_DECL
636 && ! DECL_EXTERNAL (old_var))
637 add_local_decl (cfun, old_var);
638 if ((!optimize || debug_info_level > DINFO_LEVEL_TERSE)
639 && !DECL_IGNORED_P (old_var)
640 && nonlocalized_list)
641 vec_safe_push (*nonlocalized_list, old_var);
642 continue;
643 }
644
645 /* Remap the variable. */
646 new_var = remap_decl (old_var, id);
647
648 /* If we didn't remap this variable, we can't mess with its
649 TREE_CHAIN. If we remapped this variable to the return slot, it's
650 already declared somewhere else, so don't declare it here. */
651
652 if (new_var == id->retvar)
653 ;
654 else if (!new_var)
655 {
656 if ((!optimize || debug_info_level > DINFO_LEVEL_TERSE)
657 && !DECL_IGNORED_P (old_var)
658 && nonlocalized_list)
659 vec_safe_push (*nonlocalized_list, old_var);
660 }
661 else
662 {
663 gcc_assert (DECL_P (new_var));
664 DECL_CHAIN (new_var) = new_decls;
665 new_decls = new_var;
666
667 /* Also copy value-expressions. */
668 if (TREE_CODE (new_var) == VAR_DECL
669 && DECL_HAS_VALUE_EXPR_P (new_var))
670 {
671 tree tem = DECL_VALUE_EXPR (new_var);
672 bool old_regimplify = id->regimplify;
673 id->remapping_type_depth++;
674 walk_tree (&tem, copy_tree_body_r, id, NULL);
675 id->remapping_type_depth--;
676 id->regimplify = old_regimplify;
677 SET_DECL_VALUE_EXPR (new_var, tem);
678 }
679 }
680 }
681
682 return nreverse (new_decls);
683 }
684
685 /* Copy the BLOCK to contain remapped versions of the variables
686 therein. And hook the new block into the block-tree. */
687
688 static void
689 remap_block (tree *block, copy_body_data *id)
690 {
691 tree old_block;
692 tree new_block;
693
694 /* Make the new block. */
695 old_block = *block;
696 new_block = make_node (BLOCK);
697 TREE_USED (new_block) = TREE_USED (old_block);
698 BLOCK_ABSTRACT_ORIGIN (new_block) = old_block;
699 BLOCK_SOURCE_LOCATION (new_block) = BLOCK_SOURCE_LOCATION (old_block);
700 BLOCK_NONLOCALIZED_VARS (new_block)
701 = vec_safe_copy (BLOCK_NONLOCALIZED_VARS (old_block));
702 *block = new_block;
703
704 /* Remap its variables. */
705 BLOCK_VARS (new_block) = remap_decls (BLOCK_VARS (old_block),
706 &BLOCK_NONLOCALIZED_VARS (new_block),
707 id);
708
709 if (id->transform_lang_insert_block)
710 id->transform_lang_insert_block (new_block);
711
712 /* Remember the remapped block. */
713 insert_decl_map (id, old_block, new_block);
714 }
715
716 /* Copy the whole block tree and root it in id->block. */
717 static tree
718 remap_blocks (tree block, copy_body_data *id)
719 {
720 tree t;
721 tree new_tree = block;
722
723 if (!block)
724 return NULL;
725
726 remap_block (&new_tree, id);
727 gcc_assert (new_tree != block);
728 for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
729 prepend_lexical_block (new_tree, remap_blocks (t, id));
730 /* Blocks are in arbitrary order, but make things slightly prettier and do
731 not swap order when producing a copy. */
732 BLOCK_SUBBLOCKS (new_tree) = blocks_nreverse (BLOCK_SUBBLOCKS (new_tree));
733 return new_tree;
734 }
735
736 /* Remap the block tree rooted at BLOCK to nothing. */
737 static void
738 remap_blocks_to_null (tree block, copy_body_data *id)
739 {
740 tree t;
741 insert_decl_map (id, block, NULL_TREE);
742 for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
743 remap_blocks_to_null (t, id);
744 }
745
746 static void
747 copy_statement_list (tree *tp)
748 {
749 tree_stmt_iterator oi, ni;
750 tree new_tree;
751
752 new_tree = alloc_stmt_list ();
753 ni = tsi_start (new_tree);
754 oi = tsi_start (*tp);
755 TREE_TYPE (new_tree) = TREE_TYPE (*tp);
756 *tp = new_tree;
757
758 for (; !tsi_end_p (oi); tsi_next (&oi))
759 {
760 tree stmt = tsi_stmt (oi);
761 if (TREE_CODE (stmt) == STATEMENT_LIST)
762 /* This copy is not redundant; tsi_link_after will smash this
763 STATEMENT_LIST into the end of the one we're building, and we
764 don't want to do that with the original. */
765 copy_statement_list (&stmt);
766 tsi_link_after (&ni, stmt, TSI_CONTINUE_LINKING);
767 }
768 }
769
770 static void
771 copy_bind_expr (tree *tp, int *walk_subtrees, copy_body_data *id)
772 {
773 tree block = BIND_EXPR_BLOCK (*tp);
774 /* Copy (and replace) the statement. */
775 copy_tree_r (tp, walk_subtrees, NULL);
776 if (block)
777 {
778 remap_block (&block, id);
779 BIND_EXPR_BLOCK (*tp) = block;
780 }
781
782 if (BIND_EXPR_VARS (*tp))
783 /* This will remap a lot of the same decls again, but this should be
784 harmless. */
785 BIND_EXPR_VARS (*tp) = remap_decls (BIND_EXPR_VARS (*tp), NULL, id);
786 }
787
788
789 /* Create a new gimple_seq by remapping all the statements in BODY
790 using the inlining information in ID. */
791
792 static gimple_seq
793 remap_gimple_seq (gimple_seq body, copy_body_data *id)
794 {
795 gimple_stmt_iterator si;
796 gimple_seq new_body = NULL;
797
798 for (si = gsi_start (body); !gsi_end_p (si); gsi_next (&si))
799 {
800 gimple_seq new_stmts = remap_gimple_stmt (gsi_stmt (si), id);
801 gimple_seq_add_seq (&new_body, new_stmts);
802 }
803
804 return new_body;
805 }
806
807
808 /* Copy a GIMPLE_BIND statement STMT, remapping all the symbols in its
809 block using the mapping information in ID. */
810
811 static gimple
812 copy_gimple_bind (gbind *stmt, copy_body_data *id)
813 {
814 gimple new_bind;
815 tree new_block, new_vars;
816 gimple_seq body, new_body;
817
818 /* Copy the statement. Note that we purposely don't use copy_stmt
819 here because we need to remap statements as we copy. */
820 body = gimple_bind_body (stmt);
821 new_body = remap_gimple_seq (body, id);
822
823 new_block = gimple_bind_block (stmt);
824 if (new_block)
825 remap_block (&new_block, id);
826
827 /* This will remap a lot of the same decls again, but this should be
828 harmless. */
829 new_vars = gimple_bind_vars (stmt);
830 if (new_vars)
831 new_vars = remap_decls (new_vars, NULL, id);
832
833 new_bind = gimple_build_bind (new_vars, new_body, new_block);
834
835 return new_bind;
836 }
837
838 /* Return true if DECL is a parameter or a SSA_NAME for a parameter. */
839
840 static bool
841 is_parm (tree decl)
842 {
843 if (TREE_CODE (decl) == SSA_NAME)
844 {
845 decl = SSA_NAME_VAR (decl);
846 if (!decl)
847 return false;
848 }
849
850 return (TREE_CODE (decl) == PARM_DECL);
851 }
852
853 /* Remap the dependence CLIQUE from the source to the destination function
854 as specified in ID. */
855
856 static unsigned short
857 remap_dependence_clique (copy_body_data *id, unsigned short clique)
858 {
859 if (clique == 0)
860 return 0;
861 if (!id->dependence_map)
862 id->dependence_map
863 = new hash_map<unsigned short, unsigned short, dependence_hasher>;
864 bool existed;
865 unsigned short &newc = id->dependence_map->get_or_insert (clique, &existed);
866 if (!existed)
867 newc = ++cfun->last_clique;
868 return newc;
869 }
870
871 /* Remap the GIMPLE operand pointed to by *TP. DATA is really a
872 'struct walk_stmt_info *'. DATA->INFO is a 'copy_body_data *'.
873 WALK_SUBTREES is used to indicate walk_gimple_op whether to keep
874 recursing into the children nodes of *TP. */
875
876 static tree
877 remap_gimple_op_r (tree *tp, int *walk_subtrees, void *data)
878 {
879 struct walk_stmt_info *wi_p = (struct walk_stmt_info *) data;
880 copy_body_data *id = (copy_body_data *) wi_p->info;
881 tree fn = id->src_fn;
882
883 if (TREE_CODE (*tp) == SSA_NAME)
884 {
885 *tp = remap_ssa_name (*tp, id);
886 *walk_subtrees = 0;
887 return NULL;
888 }
889 else if (auto_var_in_fn_p (*tp, fn))
890 {
891 /* Local variables and labels need to be replaced by equivalent
892 variables. We don't want to copy static variables; there's
893 only one of those, no matter how many times we inline the
894 containing function. Similarly for globals from an outer
895 function. */
896 tree new_decl;
897
898 /* Remap the declaration. */
899 new_decl = remap_decl (*tp, id);
900 gcc_assert (new_decl);
901 /* Replace this variable with the copy. */
902 STRIP_TYPE_NOPS (new_decl);
903 /* ??? The C++ frontend uses void * pointer zero to initialize
904 any other type. This confuses the middle-end type verification.
905 As cloned bodies do not go through gimplification again the fixup
906 there doesn't trigger. */
907 if (TREE_CODE (new_decl) == INTEGER_CST
908 && !useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (new_decl)))
909 new_decl = fold_convert (TREE_TYPE (*tp), new_decl);
910 *tp = new_decl;
911 *walk_subtrees = 0;
912 }
913 else if (TREE_CODE (*tp) == STATEMENT_LIST)
914 gcc_unreachable ();
915 else if (TREE_CODE (*tp) == SAVE_EXPR)
916 gcc_unreachable ();
917 else if (TREE_CODE (*tp) == LABEL_DECL
918 && (!DECL_CONTEXT (*tp)
919 || decl_function_context (*tp) == id->src_fn))
920 /* These may need to be remapped for EH handling. */
921 *tp = remap_decl (*tp, id);
922 else if (TREE_CODE (*tp) == FIELD_DECL)
923 {
924 /* If the enclosing record type is variably_modified_type_p, the field
925 has already been remapped. Otherwise, it need not be. */
926 tree *n = id->decl_map->get (*tp);
927 if (n)
928 *tp = *n;
929 *walk_subtrees = 0;
930 }
931 else if (TYPE_P (*tp))
932 /* Types may need remapping as well. */
933 *tp = remap_type (*tp, id);
934 else if (CONSTANT_CLASS_P (*tp))
935 {
936 /* If this is a constant, we have to copy the node iff the type
937 will be remapped. copy_tree_r will not copy a constant. */
938 tree new_type = remap_type (TREE_TYPE (*tp), id);
939
940 if (new_type == TREE_TYPE (*tp))
941 *walk_subtrees = 0;
942
943 else if (TREE_CODE (*tp) == INTEGER_CST)
944 *tp = wide_int_to_tree (new_type, *tp);
945 else
946 {
947 *tp = copy_node (*tp);
948 TREE_TYPE (*tp) = new_type;
949 }
950 }
951 else
952 {
953 /* Otherwise, just copy the node. Note that copy_tree_r already
954 knows not to copy VAR_DECLs, etc., so this is safe. */
955
956 if (TREE_CODE (*tp) == MEM_REF)
957 {
958 /* We need to re-canonicalize MEM_REFs from inline substitutions
959 that can happen when a pointer argument is an ADDR_EXPR.
960 Recurse here manually to allow that. */
961 tree ptr = TREE_OPERAND (*tp, 0);
962 tree type = remap_type (TREE_TYPE (*tp), id);
963 tree old = *tp;
964 walk_tree (&ptr, remap_gimple_op_r, data, NULL);
965 *tp = fold_build2 (MEM_REF, type, ptr, TREE_OPERAND (*tp, 1));
966 TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
967 TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
968 TREE_NO_WARNING (*tp) = TREE_NO_WARNING (old);
969 if (MR_DEPENDENCE_CLIQUE (old) != 0)
970 {
971 MR_DEPENDENCE_CLIQUE (*tp)
972 = remap_dependence_clique (id, MR_DEPENDENCE_CLIQUE (old));
973 MR_DEPENDENCE_BASE (*tp) = MR_DEPENDENCE_BASE (old);
974 }
975 /* We cannot propagate the TREE_THIS_NOTRAP flag if we have
976 remapped a parameter as the property might be valid only
977 for the parameter itself. */
978 if (TREE_THIS_NOTRAP (old)
979 && (!is_parm (TREE_OPERAND (old, 0))
980 || (!id->transform_parameter && is_parm (ptr))))
981 TREE_THIS_NOTRAP (*tp) = 1;
982 *walk_subtrees = 0;
983 return NULL;
984 }
985
986 /* Here is the "usual case". Copy this tree node, and then
987 tweak some special cases. */
988 copy_tree_r (tp, walk_subtrees, NULL);
989
990 if (TREE_CODE (*tp) != OMP_CLAUSE)
991 TREE_TYPE (*tp) = remap_type (TREE_TYPE (*tp), id);
992
993 if (TREE_CODE (*tp) == TARGET_EXPR && TREE_OPERAND (*tp, 3))
994 {
995 /* The copied TARGET_EXPR has never been expanded, even if the
996 original node was expanded already. */
997 TREE_OPERAND (*tp, 1) = TREE_OPERAND (*tp, 3);
998 TREE_OPERAND (*tp, 3) = NULL_TREE;
999 }
1000 else if (TREE_CODE (*tp) == ADDR_EXPR)
1001 {
1002 /* Variable substitution need not be simple. In particular,
1003 the MEM_REF substitution above. Make sure that
1004 TREE_CONSTANT and friends are up-to-date. */
1005 int invariant = is_gimple_min_invariant (*tp);
1006 walk_tree (&TREE_OPERAND (*tp, 0), remap_gimple_op_r, data, NULL);
1007 recompute_tree_invariant_for_addr_expr (*tp);
1008
1009 /* If this used to be invariant, but is not any longer,
1010 then regimplification is probably needed. */
1011 if (invariant && !is_gimple_min_invariant (*tp))
1012 id->regimplify = true;
1013
1014 *walk_subtrees = 0;
1015 }
1016 }
1017
1018 /* Update the TREE_BLOCK for the cloned expr. */
1019 if (EXPR_P (*tp))
1020 {
1021 tree new_block = id->remapping_type_depth == 0 ? id->block : NULL;
1022 tree old_block = TREE_BLOCK (*tp);
1023 if (old_block)
1024 {
1025 tree *n;
1026 n = id->decl_map->get (TREE_BLOCK (*tp));
1027 if (n)
1028 new_block = *n;
1029 }
1030 TREE_SET_BLOCK (*tp, new_block);
1031 }
1032
1033 /* Keep iterating. */
1034 return NULL_TREE;
1035 }
1036
1037
1038 /* Called from copy_body_id via walk_tree. DATA is really a
1039 `copy_body_data *'. */
1040
1041 tree
1042 copy_tree_body_r (tree *tp, int *walk_subtrees, void *data)
1043 {
1044 copy_body_data *id = (copy_body_data *) data;
1045 tree fn = id->src_fn;
1046 tree new_block;
1047
1048 /* Begin by recognizing trees that we'll completely rewrite for the
1049 inlining context. Our output for these trees is completely
1050 different from out input (e.g. RETURN_EXPR is deleted, and morphs
1051 into an edge). Further down, we'll handle trees that get
1052 duplicated and/or tweaked. */
1053
1054 /* When requested, RETURN_EXPRs should be transformed to just the
1055 contained MODIFY_EXPR. The branch semantics of the return will
1056 be handled elsewhere by manipulating the CFG rather than a statement. */
1057 if (TREE_CODE (*tp) == RETURN_EXPR && id->transform_return_to_modify)
1058 {
1059 tree assignment = TREE_OPERAND (*tp, 0);
1060
1061 /* If we're returning something, just turn that into an
1062 assignment into the equivalent of the original RESULT_DECL.
1063 If the "assignment" is just the result decl, the result
1064 decl has already been set (e.g. a recent "foo (&result_decl,
1065 ...)"); just toss the entire RETURN_EXPR. */
1066 if (assignment && TREE_CODE (assignment) == MODIFY_EXPR)
1067 {
1068 /* Replace the RETURN_EXPR with (a copy of) the
1069 MODIFY_EXPR hanging underneath. */
1070 *tp = copy_node (assignment);
1071 }
1072 else /* Else the RETURN_EXPR returns no value. */
1073 {
1074 *tp = NULL;
1075 return (tree) (void *)1;
1076 }
1077 }
1078 else if (TREE_CODE (*tp) == SSA_NAME)
1079 {
1080 *tp = remap_ssa_name (*tp, id);
1081 *walk_subtrees = 0;
1082 return NULL;
1083 }
1084
1085 /* Local variables and labels need to be replaced by equivalent
1086 variables. We don't want to copy static variables; there's only
1087 one of those, no matter how many times we inline the containing
1088 function. Similarly for globals from an outer function. */
1089 else if (auto_var_in_fn_p (*tp, fn))
1090 {
1091 tree new_decl;
1092
1093 /* Remap the declaration. */
1094 new_decl = remap_decl (*tp, id);
1095 gcc_assert (new_decl);
1096 /* Replace this variable with the copy. */
1097 STRIP_TYPE_NOPS (new_decl);
1098 *tp = new_decl;
1099 *walk_subtrees = 0;
1100 }
1101 else if (TREE_CODE (*tp) == STATEMENT_LIST)
1102 copy_statement_list (tp);
1103 else if (TREE_CODE (*tp) == SAVE_EXPR
1104 || TREE_CODE (*tp) == TARGET_EXPR)
1105 remap_save_expr (tp, id->decl_map, walk_subtrees);
1106 else if (TREE_CODE (*tp) == LABEL_DECL
1107 && (! DECL_CONTEXT (*tp)
1108 || decl_function_context (*tp) == id->src_fn))
1109 /* These may need to be remapped for EH handling. */
1110 *tp = remap_decl (*tp, id);
1111 else if (TREE_CODE (*tp) == BIND_EXPR)
1112 copy_bind_expr (tp, walk_subtrees, id);
1113 /* Types may need remapping as well. */
1114 else if (TYPE_P (*tp))
1115 *tp = remap_type (*tp, id);
1116
1117 /* If this is a constant, we have to copy the node iff the type will be
1118 remapped. copy_tree_r will not copy a constant. */
1119 else if (CONSTANT_CLASS_P (*tp))
1120 {
1121 tree new_type = remap_type (TREE_TYPE (*tp), id);
1122
1123 if (new_type == TREE_TYPE (*tp))
1124 *walk_subtrees = 0;
1125
1126 else if (TREE_CODE (*tp) == INTEGER_CST)
1127 *tp = wide_int_to_tree (new_type, *tp);
1128 else
1129 {
1130 *tp = copy_node (*tp);
1131 TREE_TYPE (*tp) = new_type;
1132 }
1133 }
1134
1135 /* Otherwise, just copy the node. Note that copy_tree_r already
1136 knows not to copy VAR_DECLs, etc., so this is safe. */
1137 else
1138 {
1139 /* Here we handle trees that are not completely rewritten.
1140 First we detect some inlining-induced bogosities for
1141 discarding. */
1142 if (TREE_CODE (*tp) == MODIFY_EXPR
1143 && TREE_OPERAND (*tp, 0) == TREE_OPERAND (*tp, 1)
1144 && (auto_var_in_fn_p (TREE_OPERAND (*tp, 0), fn)))
1145 {
1146 /* Some assignments VAR = VAR; don't generate any rtl code
1147 and thus don't count as variable modification. Avoid
1148 keeping bogosities like 0 = 0. */
1149 tree decl = TREE_OPERAND (*tp, 0), value;
1150 tree *n;
1151
1152 n = id->decl_map->get (decl);
1153 if (n)
1154 {
1155 value = *n;
1156 STRIP_TYPE_NOPS (value);
1157 if (TREE_CONSTANT (value) || TREE_READONLY (value))
1158 {
1159 *tp = build_empty_stmt (EXPR_LOCATION (*tp));
1160 return copy_tree_body_r (tp, walk_subtrees, data);
1161 }
1162 }
1163 }
1164 else if (TREE_CODE (*tp) == INDIRECT_REF)
1165 {
1166 /* Get rid of *& from inline substitutions that can happen when a
1167 pointer argument is an ADDR_EXPR. */
1168 tree decl = TREE_OPERAND (*tp, 0);
1169 tree *n = id->decl_map->get (decl);
1170 if (n)
1171 {
1172 /* If we happen to get an ADDR_EXPR in n->value, strip
1173 it manually here as we'll eventually get ADDR_EXPRs
1174 which lie about their types pointed to. In this case
1175 build_fold_indirect_ref wouldn't strip the INDIRECT_REF,
1176 but we absolutely rely on that. As fold_indirect_ref
1177 does other useful transformations, try that first, though. */
1178 tree type = TREE_TYPE (*tp);
1179 tree ptr = id->do_not_unshare ? *n : unshare_expr (*n);
1180 tree old = *tp;
1181 *tp = gimple_fold_indirect_ref (ptr);
1182 if (! *tp)
1183 {
1184 if (TREE_CODE (ptr) == ADDR_EXPR)
1185 {
1186 *tp
1187 = fold_indirect_ref_1 (EXPR_LOCATION (ptr), type, ptr);
1188 /* ??? We should either assert here or build
1189 a VIEW_CONVERT_EXPR instead of blindly leaking
1190 incompatible types to our IL. */
1191 if (! *tp)
1192 *tp = TREE_OPERAND (ptr, 0);
1193 }
1194 else
1195 {
1196 *tp = build1 (INDIRECT_REF, type, ptr);
1197 TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
1198 TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
1199 TREE_READONLY (*tp) = TREE_READONLY (old);
1200 /* We cannot propagate the TREE_THIS_NOTRAP flag if we
1201 have remapped a parameter as the property might be
1202 valid only for the parameter itself. */
1203 if (TREE_THIS_NOTRAP (old)
1204 && (!is_parm (TREE_OPERAND (old, 0))
1205 || (!id->transform_parameter && is_parm (ptr))))
1206 TREE_THIS_NOTRAP (*tp) = 1;
1207 }
1208 }
1209 *walk_subtrees = 0;
1210 return NULL;
1211 }
1212 }
1213 else if (TREE_CODE (*tp) == MEM_REF)
1214 {
1215 /* We need to re-canonicalize MEM_REFs from inline substitutions
1216 that can happen when a pointer argument is an ADDR_EXPR.
1217 Recurse here manually to allow that. */
1218 tree ptr = TREE_OPERAND (*tp, 0);
1219 tree type = remap_type (TREE_TYPE (*tp), id);
1220 tree old = *tp;
1221 walk_tree (&ptr, copy_tree_body_r, data, NULL);
1222 *tp = fold_build2 (MEM_REF, type, ptr, TREE_OPERAND (*tp, 1));
1223 TREE_THIS_VOLATILE (*tp) = TREE_THIS_VOLATILE (old);
1224 TREE_SIDE_EFFECTS (*tp) = TREE_SIDE_EFFECTS (old);
1225 TREE_NO_WARNING (*tp) = TREE_NO_WARNING (old);
1226 if (MR_DEPENDENCE_CLIQUE (old) != 0)
1227 {
1228 MR_DEPENDENCE_CLIQUE (*tp)
1229 = remap_dependence_clique (id, MR_DEPENDENCE_CLIQUE (old));
1230 MR_DEPENDENCE_BASE (*tp) = MR_DEPENDENCE_BASE (old);
1231 }
1232 /* We cannot propagate the TREE_THIS_NOTRAP flag if we have
1233 remapped a parameter as the property might be valid only
1234 for the parameter itself. */
1235 if (TREE_THIS_NOTRAP (old)
1236 && (!is_parm (TREE_OPERAND (old, 0))
1237 || (!id->transform_parameter && is_parm (ptr))))
1238 TREE_THIS_NOTRAP (*tp) = 1;
1239 *walk_subtrees = 0;
1240 return NULL;
1241 }
1242
1243 /* Here is the "usual case". Copy this tree node, and then
1244 tweak some special cases. */
1245 copy_tree_r (tp, walk_subtrees, NULL);
1246
1247 /* If EXPR has block defined, map it to newly constructed block.
1248 When inlining we want EXPRs without block appear in the block
1249 of function call if we are not remapping a type. */
1250 if (EXPR_P (*tp))
1251 {
1252 new_block = id->remapping_type_depth == 0 ? id->block : NULL;
1253 if (TREE_BLOCK (*tp))
1254 {
1255 tree *n;
1256 n = id->decl_map->get (TREE_BLOCK (*tp));
1257 if (n)
1258 new_block = *n;
1259 }
1260 TREE_SET_BLOCK (*tp, new_block);
1261 }
1262
1263 if (TREE_CODE (*tp) != OMP_CLAUSE)
1264 TREE_TYPE (*tp) = remap_type (TREE_TYPE (*tp), id);
1265
1266 /* The copied TARGET_EXPR has never been expanded, even if the
1267 original node was expanded already. */
1268 if (TREE_CODE (*tp) == TARGET_EXPR && TREE_OPERAND (*tp, 3))
1269 {
1270 TREE_OPERAND (*tp, 1) = TREE_OPERAND (*tp, 3);
1271 TREE_OPERAND (*tp, 3) = NULL_TREE;
1272 }
1273
1274 /* Variable substitution need not be simple. In particular, the
1275 INDIRECT_REF substitution above. Make sure that TREE_CONSTANT
1276 and friends are up-to-date. */
1277 else if (TREE_CODE (*tp) == ADDR_EXPR)
1278 {
1279 int invariant = is_gimple_min_invariant (*tp);
1280 walk_tree (&TREE_OPERAND (*tp, 0), copy_tree_body_r, id, NULL);
1281
1282 /* Handle the case where we substituted an INDIRECT_REF
1283 into the operand of the ADDR_EXPR. */
1284 if (TREE_CODE (TREE_OPERAND (*tp, 0)) == INDIRECT_REF)
1285 *tp = TREE_OPERAND (TREE_OPERAND (*tp, 0), 0);
1286 else
1287 recompute_tree_invariant_for_addr_expr (*tp);
1288
1289 /* If this used to be invariant, but is not any longer,
1290 then regimplification is probably needed. */
1291 if (invariant && !is_gimple_min_invariant (*tp))
1292 id->regimplify = true;
1293
1294 *walk_subtrees = 0;
1295 }
1296 }
1297
1298 /* Keep iterating. */
1299 return NULL_TREE;
1300 }
1301
1302 /* Helper for remap_gimple_stmt. Given an EH region number for the
1303 source function, map that to the duplicate EH region number in
1304 the destination function. */
1305
1306 static int
1307 remap_eh_region_nr (int old_nr, copy_body_data *id)
1308 {
1309 eh_region old_r, new_r;
1310
1311 old_r = get_eh_region_from_number_fn (id->src_cfun, old_nr);
1312 new_r = static_cast<eh_region> (*id->eh_map->get (old_r));
1313
1314 return new_r->index;
1315 }
1316
1317 /* Similar, but operate on INTEGER_CSTs. */
1318
1319 static tree
1320 remap_eh_region_tree_nr (tree old_t_nr, copy_body_data *id)
1321 {
1322 int old_nr, new_nr;
1323
1324 old_nr = tree_to_shwi (old_t_nr);
1325 new_nr = remap_eh_region_nr (old_nr, id);
1326
1327 return build_int_cst (integer_type_node, new_nr);
1328 }
1329
1330 /* Helper for copy_bb. Remap statement STMT using the inlining
1331 information in ID. Return the new statement copy. */
1332
1333 static gimple_seq
1334 remap_gimple_stmt (gimple stmt, copy_body_data *id)
1335 {
1336 gimple copy = NULL;
1337 struct walk_stmt_info wi;
1338 bool skip_first = false;
1339 gimple_seq stmts = NULL;
1340
1341 /* Begin by recognizing trees that we'll completely rewrite for the
1342 inlining context. Our output for these trees is completely
1343 different from out input (e.g. RETURN_EXPR is deleted, and morphs
1344 into an edge). Further down, we'll handle trees that get
1345 duplicated and/or tweaked. */
1346
1347 /* When requested, GIMPLE_RETURNs should be transformed to just the
1348 contained GIMPLE_ASSIGN. The branch semantics of the return will
1349 be handled elsewhere by manipulating the CFG rather than the
1350 statement. */
1351 if (gimple_code (stmt) == GIMPLE_RETURN && id->transform_return_to_modify)
1352 {
1353 tree retval = gimple_return_retval (as_a <greturn *> (stmt));
1354 tree retbnd = gimple_return_retbnd (stmt);
1355 tree bndslot = id->retbnd;
1356
1357 if (retbnd && bndslot)
1358 {
1359 gimple bndcopy = gimple_build_assign (bndslot, retbnd);
1360 memset (&wi, 0, sizeof (wi));
1361 wi.info = id;
1362 walk_gimple_op (bndcopy, remap_gimple_op_r, &wi);
1363 gimple_seq_add_stmt (&stmts, bndcopy);
1364 }
1365
1366 /* If we're returning something, just turn that into an
1367 assignment into the equivalent of the original RESULT_DECL.
1368 If RETVAL is just the result decl, the result decl has
1369 already been set (e.g. a recent "foo (&result_decl, ...)");
1370 just toss the entire GIMPLE_RETURN. */
1371 if (retval
1372 && (TREE_CODE (retval) != RESULT_DECL
1373 && (TREE_CODE (retval) != SSA_NAME
1374 || ! SSA_NAME_VAR (retval)
1375 || TREE_CODE (SSA_NAME_VAR (retval)) != RESULT_DECL)))
1376 {
1377 copy = gimple_build_assign (id->do_not_unshare
1378 ? id->retvar : unshare_expr (id->retvar),
1379 retval);
1380 /* id->retvar is already substituted. Skip it on later remapping. */
1381 skip_first = true;
1382
1383 /* We need to copy bounds if return structure with pointers into
1384 instrumented function. */
1385 if (chkp_function_instrumented_p (id->dst_fn)
1386 && !bndslot
1387 && !BOUNDED_P (id->retvar)
1388 && chkp_type_has_pointer (TREE_TYPE (id->retvar)))
1389 id->assign_stmts.safe_push (copy);
1390
1391 }
1392 else
1393 return stmts;
1394 }
1395 else if (gimple_has_substatements (stmt))
1396 {
1397 gimple_seq s1, s2;
1398
1399 /* When cloning bodies from the C++ front end, we will be handed bodies
1400 in High GIMPLE form. Handle here all the High GIMPLE statements that
1401 have embedded statements. */
1402 switch (gimple_code (stmt))
1403 {
1404 case GIMPLE_BIND:
1405 copy = copy_gimple_bind (as_a <gbind *> (stmt), id);
1406 break;
1407
1408 case GIMPLE_CATCH:
1409 {
1410 gcatch *catch_stmt = as_a <gcatch *> (stmt);
1411 s1 = remap_gimple_seq (gimple_catch_handler (catch_stmt), id);
1412 copy = gimple_build_catch (gimple_catch_types (catch_stmt), s1);
1413 }
1414 break;
1415
1416 case GIMPLE_EH_FILTER:
1417 s1 = remap_gimple_seq (gimple_eh_filter_failure (stmt), id);
1418 copy = gimple_build_eh_filter (gimple_eh_filter_types (stmt), s1);
1419 break;
1420
1421 case GIMPLE_TRY:
1422 s1 = remap_gimple_seq (gimple_try_eval (stmt), id);
1423 s2 = remap_gimple_seq (gimple_try_cleanup (stmt), id);
1424 copy = gimple_build_try (s1, s2, gimple_try_kind (stmt));
1425 break;
1426
1427 case GIMPLE_WITH_CLEANUP_EXPR:
1428 s1 = remap_gimple_seq (gimple_wce_cleanup (stmt), id);
1429 copy = gimple_build_wce (s1);
1430 break;
1431
1432 case GIMPLE_OMP_PARALLEL:
1433 {
1434 gomp_parallel *omp_par_stmt = as_a <gomp_parallel *> (stmt);
1435 s1 = remap_gimple_seq (gimple_omp_body (omp_par_stmt), id);
1436 copy = gimple_build_omp_parallel
1437 (s1,
1438 gimple_omp_parallel_clauses (omp_par_stmt),
1439 gimple_omp_parallel_child_fn (omp_par_stmt),
1440 gimple_omp_parallel_data_arg (omp_par_stmt));
1441 }
1442 break;
1443
1444 case GIMPLE_OMP_TASK:
1445 s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1446 copy = gimple_build_omp_task
1447 (s1,
1448 gimple_omp_task_clauses (stmt),
1449 gimple_omp_task_child_fn (stmt),
1450 gimple_omp_task_data_arg (stmt),
1451 gimple_omp_task_copy_fn (stmt),
1452 gimple_omp_task_arg_size (stmt),
1453 gimple_omp_task_arg_align (stmt));
1454 break;
1455
1456 case GIMPLE_OMP_FOR:
1457 s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1458 s2 = remap_gimple_seq (gimple_omp_for_pre_body (stmt), id);
1459 copy = gimple_build_omp_for (s1, gimple_omp_for_kind (stmt),
1460 gimple_omp_for_clauses (stmt),
1461 gimple_omp_for_collapse (stmt), s2);
1462 {
1463 size_t i;
1464 for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
1465 {
1466 gimple_omp_for_set_index (copy, i,
1467 gimple_omp_for_index (stmt, i));
1468 gimple_omp_for_set_initial (copy, i,
1469 gimple_omp_for_initial (stmt, i));
1470 gimple_omp_for_set_final (copy, i,
1471 gimple_omp_for_final (stmt, i));
1472 gimple_omp_for_set_incr (copy, i,
1473 gimple_omp_for_incr (stmt, i));
1474 gimple_omp_for_set_cond (copy, i,
1475 gimple_omp_for_cond (stmt, i));
1476 }
1477 }
1478 break;
1479
1480 case GIMPLE_OMP_MASTER:
1481 s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1482 copy = gimple_build_omp_master (s1);
1483 break;
1484
1485 case GIMPLE_OMP_TASKGROUP:
1486 s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1487 copy = gimple_build_omp_taskgroup (s1);
1488 break;
1489
1490 case GIMPLE_OMP_ORDERED:
1491 s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1492 copy = gimple_build_omp_ordered (s1);
1493 break;
1494
1495 case GIMPLE_OMP_SECTION:
1496 s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1497 copy = gimple_build_omp_section (s1);
1498 break;
1499
1500 case GIMPLE_OMP_SECTIONS:
1501 s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1502 copy = gimple_build_omp_sections
1503 (s1, gimple_omp_sections_clauses (stmt));
1504 break;
1505
1506 case GIMPLE_OMP_SINGLE:
1507 s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1508 copy = gimple_build_omp_single
1509 (s1, gimple_omp_single_clauses (stmt));
1510 break;
1511
1512 case GIMPLE_OMP_TARGET:
1513 s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1514 copy = gimple_build_omp_target
1515 (s1, gimple_omp_target_kind (stmt),
1516 gimple_omp_target_clauses (stmt));
1517 break;
1518
1519 case GIMPLE_OMP_TEAMS:
1520 s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1521 copy = gimple_build_omp_teams
1522 (s1, gimple_omp_teams_clauses (stmt));
1523 break;
1524
1525 case GIMPLE_OMP_CRITICAL:
1526 s1 = remap_gimple_seq (gimple_omp_body (stmt), id);
1527 copy = gimple_build_omp_critical (s1,
1528 gimple_omp_critical_name (
1529 as_a <gomp_critical *> (stmt)));
1530 break;
1531
1532 case GIMPLE_TRANSACTION:
1533 {
1534 gtransaction *old_trans_stmt = as_a <gtransaction *> (stmt);
1535 gtransaction *new_trans_stmt;
1536 s1 = remap_gimple_seq (gimple_transaction_body (old_trans_stmt),
1537 id);
1538 copy = new_trans_stmt
1539 = gimple_build_transaction (
1540 s1,
1541 gimple_transaction_label (old_trans_stmt));
1542 gimple_transaction_set_subcode (
1543 new_trans_stmt,
1544 gimple_transaction_subcode (old_trans_stmt));
1545 }
1546 break;
1547
1548 default:
1549 gcc_unreachable ();
1550 }
1551 }
1552 else
1553 {
1554 if (gimple_assign_copy_p (stmt)
1555 && gimple_assign_lhs (stmt) == gimple_assign_rhs1 (stmt)
1556 && auto_var_in_fn_p (gimple_assign_lhs (stmt), id->src_fn))
1557 {
1558 /* Here we handle statements that are not completely rewritten.
1559 First we detect some inlining-induced bogosities for
1560 discarding. */
1561
1562 /* Some assignments VAR = VAR; don't generate any rtl code
1563 and thus don't count as variable modification. Avoid
1564 keeping bogosities like 0 = 0. */
1565 tree decl = gimple_assign_lhs (stmt), value;
1566 tree *n;
1567
1568 n = id->decl_map->get (decl);
1569 if (n)
1570 {
1571 value = *n;
1572 STRIP_TYPE_NOPS (value);
1573 if (TREE_CONSTANT (value) || TREE_READONLY (value))
1574 return NULL;
1575 }
1576 }
1577
1578 /* For *ptr_N ={v} {CLOBBER}, if ptr_N is SSA_NAME defined
1579 in a block that we aren't copying during tree_function_versioning,
1580 just drop the clobber stmt. */
1581 if (id->blocks_to_copy && gimple_clobber_p (stmt))
1582 {
1583 tree lhs = gimple_assign_lhs (stmt);
1584 if (TREE_CODE (lhs) == MEM_REF
1585 && TREE_CODE (TREE_OPERAND (lhs, 0)) == SSA_NAME)
1586 {
1587 gimple def_stmt = SSA_NAME_DEF_STMT (TREE_OPERAND (lhs, 0));
1588 if (gimple_bb (def_stmt)
1589 && !bitmap_bit_p (id->blocks_to_copy,
1590 gimple_bb (def_stmt)->index))
1591 return NULL;
1592 }
1593 }
1594
1595 if (gimple_debug_bind_p (stmt))
1596 {
1597 gdebug *copy
1598 = gimple_build_debug_bind (gimple_debug_bind_get_var (stmt),
1599 gimple_debug_bind_get_value (stmt),
1600 stmt);
1601 id->debug_stmts.safe_push (copy);
1602 gimple_seq_add_stmt (&stmts, copy);
1603 return stmts;
1604 }
1605 if (gimple_debug_source_bind_p (stmt))
1606 {
1607 gdebug *copy = gimple_build_debug_source_bind
1608 (gimple_debug_source_bind_get_var (stmt),
1609 gimple_debug_source_bind_get_value (stmt),
1610 stmt);
1611 id->debug_stmts.safe_push (copy);
1612 gimple_seq_add_stmt (&stmts, copy);
1613 return stmts;
1614 }
1615
1616 /* Create a new deep copy of the statement. */
1617 copy = gimple_copy (stmt);
1618
1619 /* Clear flags that need revisiting. */
1620 if (gcall *call_stmt = dyn_cast <gcall *> (copy))
1621 {
1622 if (gimple_call_tail_p (call_stmt))
1623 gimple_call_set_tail (call_stmt, false);
1624 if (gimple_call_from_thunk_p (call_stmt))
1625 gimple_call_set_from_thunk (call_stmt, false);
1626 }
1627
1628 /* Remap the region numbers for __builtin_eh_{pointer,filter},
1629 RESX and EH_DISPATCH. */
1630 if (id->eh_map)
1631 switch (gimple_code (copy))
1632 {
1633 case GIMPLE_CALL:
1634 {
1635 tree r, fndecl = gimple_call_fndecl (copy);
1636 if (fndecl && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
1637 switch (DECL_FUNCTION_CODE (fndecl))
1638 {
1639 case BUILT_IN_EH_COPY_VALUES:
1640 r = gimple_call_arg (copy, 1);
1641 r = remap_eh_region_tree_nr (r, id);
1642 gimple_call_set_arg (copy, 1, r);
1643 /* FALLTHRU */
1644
1645 case BUILT_IN_EH_POINTER:
1646 case BUILT_IN_EH_FILTER:
1647 r = gimple_call_arg (copy, 0);
1648 r = remap_eh_region_tree_nr (r, id);
1649 gimple_call_set_arg (copy, 0, r);
1650 break;
1651
1652 default:
1653 break;
1654 }
1655
1656 /* Reset alias info if we didn't apply measures to
1657 keep it valid over inlining by setting DECL_PT_UID. */
1658 if (!id->src_cfun->gimple_df
1659 || !id->src_cfun->gimple_df->ipa_pta)
1660 gimple_call_reset_alias_info (as_a <gcall *> (copy));
1661 }
1662 break;
1663
1664 case GIMPLE_RESX:
1665 {
1666 gresx *resx_stmt = as_a <gresx *> (copy);
1667 int r = gimple_resx_region (resx_stmt);
1668 r = remap_eh_region_nr (r, id);
1669 gimple_resx_set_region (resx_stmt, r);
1670 }
1671 break;
1672
1673 case GIMPLE_EH_DISPATCH:
1674 {
1675 geh_dispatch *eh_dispatch = as_a <geh_dispatch *> (copy);
1676 int r = gimple_eh_dispatch_region (eh_dispatch);
1677 r = remap_eh_region_nr (r, id);
1678 gimple_eh_dispatch_set_region (eh_dispatch, r);
1679 }
1680 break;
1681
1682 default:
1683 break;
1684 }
1685 }
1686
1687 /* If STMT has a block defined, map it to the newly constructed
1688 block. */
1689 if (gimple_block (copy))
1690 {
1691 tree *n;
1692 n = id->decl_map->get (gimple_block (copy));
1693 gcc_assert (n);
1694 gimple_set_block (copy, *n);
1695 }
1696
1697 if (gimple_debug_bind_p (copy) || gimple_debug_source_bind_p (copy))
1698 {
1699 gimple_seq_add_stmt (&stmts, copy);
1700 return stmts;
1701 }
1702
1703 /* Remap all the operands in COPY. */
1704 memset (&wi, 0, sizeof (wi));
1705 wi.info = id;
1706 if (skip_first)
1707 walk_tree (gimple_op_ptr (copy, 1), remap_gimple_op_r, &wi, NULL);
1708 else
1709 walk_gimple_op (copy, remap_gimple_op_r, &wi);
1710
1711 /* Clear the copied virtual operands. We are not remapping them here
1712 but are going to recreate them from scratch. */
1713 if (gimple_has_mem_ops (copy))
1714 {
1715 gimple_set_vdef (copy, NULL_TREE);
1716 gimple_set_vuse (copy, NULL_TREE);
1717 }
1718
1719 gimple_seq_add_stmt (&stmts, copy);
1720 return stmts;
1721 }
1722
1723
1724 /* Copy basic block, scale profile accordingly. Edges will be taken care of
1725 later */
1726
1727 static basic_block
1728 copy_bb (copy_body_data *id, basic_block bb, int frequency_scale,
1729 gcov_type count_scale)
1730 {
1731 gimple_stmt_iterator gsi, copy_gsi, seq_gsi;
1732 basic_block copy_basic_block;
1733 tree decl;
1734 gcov_type freq;
1735 basic_block prev;
1736
1737 /* Search for previous copied basic block. */
1738 prev = bb->prev_bb;
1739 while (!prev->aux)
1740 prev = prev->prev_bb;
1741
1742 /* create_basic_block() will append every new block to
1743 basic_block_info automatically. */
1744 copy_basic_block = create_basic_block (NULL, (void *) 0,
1745 (basic_block) prev->aux);
1746 copy_basic_block->count = apply_scale (bb->count, count_scale);
1747
1748 /* We are going to rebuild frequencies from scratch. These values
1749 have just small importance to drive canonicalize_loop_headers. */
1750 freq = apply_scale ((gcov_type)bb->frequency, frequency_scale);
1751
1752 /* We recompute frequencies after inlining, so this is quite safe. */
1753 if (freq > BB_FREQ_MAX)
1754 freq = BB_FREQ_MAX;
1755 copy_basic_block->frequency = freq;
1756
1757 copy_gsi = gsi_start_bb (copy_basic_block);
1758
1759 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
1760 {
1761 gimple_seq stmts;
1762 gimple stmt = gsi_stmt (gsi);
1763 gimple orig_stmt = stmt;
1764 gimple_stmt_iterator stmts_gsi;
1765 bool stmt_added = false;
1766
1767 id->regimplify = false;
1768 stmts = remap_gimple_stmt (stmt, id);
1769
1770 if (gimple_seq_empty_p (stmts))
1771 continue;
1772
1773 seq_gsi = copy_gsi;
1774
1775 for (stmts_gsi = gsi_start (stmts);
1776 !gsi_end_p (stmts_gsi); )
1777 {
1778 stmt = gsi_stmt (stmts_gsi);
1779
1780 /* Advance iterator now before stmt is moved to seq_gsi. */
1781 gsi_next (&stmts_gsi);
1782
1783 if (gimple_nop_p (stmt))
1784 continue;
1785
1786 gimple_duplicate_stmt_histograms (cfun, stmt, id->src_cfun,
1787 orig_stmt);
1788
1789 /* With return slot optimization we can end up with
1790 non-gimple (foo *)&this->m, fix that here. */
1791 if (is_gimple_assign (stmt)
1792 && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt))
1793 && !is_gimple_val (gimple_assign_rhs1 (stmt)))
1794 {
1795 tree new_rhs;
1796 new_rhs = force_gimple_operand_gsi (&seq_gsi,
1797 gimple_assign_rhs1 (stmt),
1798 true, NULL, false,
1799 GSI_CONTINUE_LINKING);
1800 gimple_assign_set_rhs1 (stmt, new_rhs);
1801 id->regimplify = false;
1802 }
1803
1804 gsi_insert_after (&seq_gsi, stmt, GSI_NEW_STMT);
1805
1806 if (id->regimplify)
1807 gimple_regimplify_operands (stmt, &seq_gsi);
1808
1809 stmt_added = true;
1810 }
1811
1812 if (!stmt_added)
1813 continue;
1814
1815 /* If copy_basic_block has been empty at the start of this iteration,
1816 call gsi_start_bb again to get at the newly added statements. */
1817 if (gsi_end_p (copy_gsi))
1818 copy_gsi = gsi_start_bb (copy_basic_block);
1819 else
1820 gsi_next (&copy_gsi);
1821
1822 /* Process the new statement. The call to gimple_regimplify_operands
1823 possibly turned the statement into multiple statements, we
1824 need to process all of them. */
1825 do
1826 {
1827 tree fn;
1828 gcall *call_stmt;
1829
1830 stmt = gsi_stmt (copy_gsi);
1831 call_stmt = dyn_cast <gcall *> (stmt);
1832 if (call_stmt
1833 && gimple_call_va_arg_pack_p (call_stmt)
1834 && id->call_stmt)
1835 {
1836 /* __builtin_va_arg_pack () should be replaced by
1837 all arguments corresponding to ... in the caller. */
1838 tree p;
1839 gcall *new_call;
1840 vec<tree> argarray;
1841 size_t nargs = gimple_call_num_args (id->call_stmt);
1842 size_t n, i, nargs_to_copy;
1843 bool remove_bounds = false;
1844
1845 for (p = DECL_ARGUMENTS (id->src_fn); p; p = DECL_CHAIN (p))
1846 nargs--;
1847
1848 /* Bounds should be removed from arg pack in case
1849 we handle not instrumented call in instrumented
1850 function. */
1851 nargs_to_copy = nargs;
1852 if (gimple_call_with_bounds_p (id->call_stmt)
1853 && !gimple_call_with_bounds_p (stmt))
1854 {
1855 for (i = gimple_call_num_args (id->call_stmt) - nargs;
1856 i < gimple_call_num_args (id->call_stmt);
1857 i++)
1858 if (POINTER_BOUNDS_P (gimple_call_arg (id->call_stmt, i)))
1859 nargs_to_copy--;
1860 remove_bounds = true;
1861 }
1862
1863 /* Create the new array of arguments. */
1864 n = nargs_to_copy + gimple_call_num_args (call_stmt);
1865 argarray.create (n);
1866 argarray.safe_grow_cleared (n);
1867
1868 /* Copy all the arguments before '...' */
1869 memcpy (argarray.address (),
1870 gimple_call_arg_ptr (call_stmt, 0),
1871 gimple_call_num_args (call_stmt) * sizeof (tree));
1872
1873 if (remove_bounds)
1874 {
1875 /* Append the rest of arguments removing bounds. */
1876 unsigned cur = gimple_call_num_args (call_stmt);
1877 i = gimple_call_num_args (id->call_stmt) - nargs;
1878 for (i = gimple_call_num_args (id->call_stmt) - nargs;
1879 i < gimple_call_num_args (id->call_stmt);
1880 i++)
1881 if (!POINTER_BOUNDS_P (gimple_call_arg (id->call_stmt, i)))
1882 argarray[cur++] = gimple_call_arg (id->call_stmt, i);
1883 gcc_assert (cur == n);
1884 }
1885 else
1886 {
1887 /* Append the arguments passed in '...' */
1888 memcpy (argarray.address () + gimple_call_num_args (call_stmt),
1889 gimple_call_arg_ptr (id->call_stmt, 0)
1890 + (gimple_call_num_args (id->call_stmt) - nargs),
1891 nargs * sizeof (tree));
1892 }
1893
1894 new_call = gimple_build_call_vec (gimple_call_fn (call_stmt),
1895 argarray);
1896
1897 argarray.release ();
1898
1899 /* Copy all GIMPLE_CALL flags, location and block, except
1900 GF_CALL_VA_ARG_PACK. */
1901 gimple_call_copy_flags (new_call, call_stmt);
1902 gimple_call_set_va_arg_pack (new_call, false);
1903 gimple_set_location (new_call, gimple_location (stmt));
1904 gimple_set_block (new_call, gimple_block (stmt));
1905 gimple_call_set_lhs (new_call, gimple_call_lhs (call_stmt));
1906
1907 gsi_replace (&copy_gsi, new_call, false);
1908 stmt = new_call;
1909 }
1910 else if (call_stmt
1911 && id->call_stmt
1912 && (decl = gimple_call_fndecl (stmt))
1913 && DECL_BUILT_IN_CLASS (decl) == BUILT_IN_NORMAL
1914 && DECL_FUNCTION_CODE (decl) == BUILT_IN_VA_ARG_PACK_LEN)
1915 {
1916 /* __builtin_va_arg_pack_len () should be replaced by
1917 the number of anonymous arguments. */
1918 size_t nargs = gimple_call_num_args (id->call_stmt), i;
1919 tree count, p;
1920 gimple new_stmt;
1921
1922 for (p = DECL_ARGUMENTS (id->src_fn); p; p = DECL_CHAIN (p))
1923 nargs--;
1924
1925 /* For instrumented calls we should ignore bounds. */
1926 for (i = gimple_call_num_args (id->call_stmt) - nargs;
1927 i < gimple_call_num_args (id->call_stmt);
1928 i++)
1929 if (POINTER_BOUNDS_P (gimple_call_arg (id->call_stmt, i)))
1930 nargs--;
1931
1932 count = build_int_cst (integer_type_node, nargs);
1933 new_stmt = gimple_build_assign (gimple_call_lhs (stmt), count);
1934 gsi_replace (&copy_gsi, new_stmt, false);
1935 stmt = new_stmt;
1936 }
1937 else if (call_stmt
1938 && id->call_stmt
1939 && gimple_call_internal_p (stmt)
1940 && gimple_call_internal_fn (stmt) == IFN_TSAN_FUNC_EXIT)
1941 {
1942 /* Drop TSAN_FUNC_EXIT () internal calls during inlining. */
1943 gsi_remove (&copy_gsi, false);
1944 continue;
1945 }
1946
1947 /* Statements produced by inlining can be unfolded, especially
1948 when we constant propagated some operands. We can't fold
1949 them right now for two reasons:
1950 1) folding require SSA_NAME_DEF_STMTs to be correct
1951 2) we can't change function calls to builtins.
1952 So we just mark statement for later folding. We mark
1953 all new statements, instead just statements that has changed
1954 by some nontrivial substitution so even statements made
1955 foldable indirectly are updated. If this turns out to be
1956 expensive, copy_body can be told to watch for nontrivial
1957 changes. */
1958 if (id->statements_to_fold)
1959 id->statements_to_fold->add (stmt);
1960
1961 /* We're duplicating a CALL_EXPR. Find any corresponding
1962 callgraph edges and update or duplicate them. */
1963 if (gcall *call_stmt = dyn_cast <gcall *> (stmt))
1964 {
1965 struct cgraph_edge *edge;
1966
1967 switch (id->transform_call_graph_edges)
1968 {
1969 case CB_CGE_DUPLICATE:
1970 edge = id->src_node->get_edge (orig_stmt);
1971 if (edge)
1972 {
1973 int edge_freq = edge->frequency;
1974 int new_freq;
1975 struct cgraph_edge *old_edge = edge;
1976 edge = edge->clone (id->dst_node, call_stmt,
1977 gimple_uid (stmt),
1978 REG_BR_PROB_BASE, CGRAPH_FREQ_BASE,
1979 true);
1980 /* We could also just rescale the frequency, but
1981 doing so would introduce roundoff errors and make
1982 verifier unhappy. */
1983 new_freq = compute_call_stmt_bb_frequency (id->dst_node->decl,
1984 copy_basic_block);
1985
1986 /* Speculative calls consist of two edges - direct and indirect.
1987 Duplicate the whole thing and distribute frequencies accordingly. */
1988 if (edge->speculative)
1989 {
1990 struct cgraph_edge *direct, *indirect;
1991 struct ipa_ref *ref;
1992
1993 gcc_assert (!edge->indirect_unknown_callee);
1994 old_edge->speculative_call_info (direct, indirect, ref);
1995 indirect = indirect->clone (id->dst_node, call_stmt,
1996 gimple_uid (stmt),
1997 REG_BR_PROB_BASE, CGRAPH_FREQ_BASE,
1998 true);
1999 if (old_edge->frequency + indirect->frequency)
2000 {
2001 edge->frequency = MIN (RDIV ((gcov_type)new_freq * old_edge->frequency,
2002 (old_edge->frequency + indirect->frequency)),
2003 CGRAPH_FREQ_MAX);
2004 indirect->frequency = MIN (RDIV ((gcov_type)new_freq * indirect->frequency,
2005 (old_edge->frequency + indirect->frequency)),
2006 CGRAPH_FREQ_MAX);
2007 }
2008 id->dst_node->clone_reference (ref, stmt);
2009 }
2010 else
2011 {
2012 edge->frequency = new_freq;
2013 if (dump_file
2014 && profile_status_for_fn (cfun) != PROFILE_ABSENT
2015 && (edge_freq > edge->frequency + 10
2016 || edge_freq < edge->frequency - 10))
2017 {
2018 fprintf (dump_file, "Edge frequency estimated by "
2019 "cgraph %i diverge from inliner's estimate %i\n",
2020 edge_freq,
2021 edge->frequency);
2022 fprintf (dump_file,
2023 "Orig bb: %i, orig bb freq %i, new bb freq %i\n",
2024 bb->index,
2025 bb->frequency,
2026 copy_basic_block->frequency);
2027 }
2028 }
2029 }
2030 break;
2031
2032 case CB_CGE_MOVE_CLONES:
2033 id->dst_node->set_call_stmt_including_clones (orig_stmt,
2034 call_stmt);
2035 edge = id->dst_node->get_edge (stmt);
2036 break;
2037
2038 case CB_CGE_MOVE:
2039 edge = id->dst_node->get_edge (orig_stmt);
2040 if (edge)
2041 edge->set_call_stmt (call_stmt);
2042 break;
2043
2044 default:
2045 gcc_unreachable ();
2046 }
2047
2048 /* Constant propagation on argument done during inlining
2049 may create new direct call. Produce an edge for it. */
2050 if ((!edge
2051 || (edge->indirect_inlining_edge
2052 && id->transform_call_graph_edges == CB_CGE_MOVE_CLONES))
2053 && id->dst_node->definition
2054 && (fn = gimple_call_fndecl (stmt)) != NULL)
2055 {
2056 struct cgraph_node *dest = cgraph_node::get (fn);
2057
2058 /* We have missing edge in the callgraph. This can happen
2059 when previous inlining turned an indirect call into a
2060 direct call by constant propagating arguments or we are
2061 producing dead clone (for further cloning). In all
2062 other cases we hit a bug (incorrect node sharing is the
2063 most common reason for missing edges). */
2064 gcc_assert (!dest->definition
2065 || dest->address_taken
2066 || !id->src_node->definition
2067 || !id->dst_node->definition);
2068 if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES)
2069 id->dst_node->create_edge_including_clones
2070 (dest, orig_stmt, call_stmt, bb->count,
2071 compute_call_stmt_bb_frequency (id->dst_node->decl,
2072 copy_basic_block),
2073 CIF_ORIGINALLY_INDIRECT_CALL);
2074 else
2075 id->dst_node->create_edge (dest, call_stmt,
2076 bb->count,
2077 compute_call_stmt_bb_frequency
2078 (id->dst_node->decl,
2079 copy_basic_block))->inline_failed
2080 = CIF_ORIGINALLY_INDIRECT_CALL;
2081 if (dump_file)
2082 {
2083 fprintf (dump_file, "Created new direct edge to %s\n",
2084 dest->name ());
2085 }
2086 }
2087
2088 notice_special_calls (as_a <gcall *> (stmt));
2089 }
2090
2091 maybe_duplicate_eh_stmt_fn (cfun, stmt, id->src_cfun, orig_stmt,
2092 id->eh_map, id->eh_lp_nr);
2093
2094 if (gimple_in_ssa_p (cfun) && !is_gimple_debug (stmt))
2095 {
2096 ssa_op_iter i;
2097 tree def;
2098
2099 FOR_EACH_SSA_TREE_OPERAND (def, stmt, i, SSA_OP_DEF)
2100 if (TREE_CODE (def) == SSA_NAME)
2101 SSA_NAME_DEF_STMT (def) = stmt;
2102 }
2103
2104 gsi_next (&copy_gsi);
2105 }
2106 while (!gsi_end_p (copy_gsi));
2107
2108 copy_gsi = gsi_last_bb (copy_basic_block);
2109 }
2110
2111 return copy_basic_block;
2112 }
2113
2114 /* Inserting Single Entry Multiple Exit region in SSA form into code in SSA
2115 form is quite easy, since dominator relationship for old basic blocks does
2116 not change.
2117
2118 There is however exception where inlining might change dominator relation
2119 across EH edges from basic block within inlined functions destinating
2120 to landing pads in function we inline into.
2121
2122 The function fills in PHI_RESULTs of such PHI nodes if they refer
2123 to gimple regs. Otherwise, the function mark PHI_RESULT of such
2124 PHI nodes for renaming. For non-gimple regs, renaming is safe: the
2125 EH edges are abnormal and SSA_NAME_OCCURS_IN_ABNORMAL_PHI must be
2126 set, and this means that there will be no overlapping live ranges
2127 for the underlying symbol.
2128
2129 This might change in future if we allow redirecting of EH edges and
2130 we might want to change way build CFG pre-inlining to include
2131 all the possible edges then. */
2132 static void
2133 update_ssa_across_abnormal_edges (basic_block bb, basic_block ret_bb,
2134 bool can_throw, bool nonlocal_goto)
2135 {
2136 edge e;
2137 edge_iterator ei;
2138
2139 FOR_EACH_EDGE (e, ei, bb->succs)
2140 if (!e->dest->aux
2141 || ((basic_block)e->dest->aux)->index == ENTRY_BLOCK)
2142 {
2143 gphi *phi;
2144 gphi_iterator si;
2145
2146 if (!nonlocal_goto)
2147 gcc_assert (e->flags & EDGE_EH);
2148
2149 if (!can_throw)
2150 gcc_assert (!(e->flags & EDGE_EH));
2151
2152 for (si = gsi_start_phis (e->dest); !gsi_end_p (si); gsi_next (&si))
2153 {
2154 edge re;
2155
2156 phi = si.phi ();
2157
2158 /* For abnormal goto/call edges the receiver can be the
2159 ENTRY_BLOCK. Do not assert this cannot happen. */
2160
2161 gcc_assert ((e->flags & EDGE_EH)
2162 || SSA_NAME_OCCURS_IN_ABNORMAL_PHI (PHI_RESULT (phi)));
2163
2164 re = find_edge (ret_bb, e->dest);
2165 gcc_checking_assert (re);
2166 gcc_assert ((re->flags & (EDGE_EH | EDGE_ABNORMAL))
2167 == (e->flags & (EDGE_EH | EDGE_ABNORMAL)));
2168
2169 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e),
2170 USE_FROM_PTR (PHI_ARG_DEF_PTR_FROM_EDGE (phi, re)));
2171 }
2172 }
2173 }
2174
2175
2176 /* Copy edges from BB into its copy constructed earlier, scale profile
2177 accordingly. Edges will be taken care of later. Assume aux
2178 pointers to point to the copies of each BB. Return true if any
2179 debug stmts are left after a statement that must end the basic block. */
2180
2181 static bool
2182 copy_edges_for_bb (basic_block bb, gcov_type count_scale, basic_block ret_bb,
2183 basic_block abnormal_goto_dest)
2184 {
2185 basic_block new_bb = (basic_block) bb->aux;
2186 edge_iterator ei;
2187 edge old_edge;
2188 gimple_stmt_iterator si;
2189 int flags;
2190 bool need_debug_cleanup = false;
2191
2192 /* Use the indices from the original blocks to create edges for the
2193 new ones. */
2194 FOR_EACH_EDGE (old_edge, ei, bb->succs)
2195 if (!(old_edge->flags & EDGE_EH))
2196 {
2197 edge new_edge;
2198
2199 flags = old_edge->flags;
2200
2201 /* Return edges do get a FALLTHRU flag when the get inlined. */
2202 if (old_edge->dest->index == EXIT_BLOCK
2203 && !(old_edge->flags & (EDGE_TRUE_VALUE|EDGE_FALSE_VALUE|EDGE_FAKE))
2204 && old_edge->dest->aux != EXIT_BLOCK_PTR_FOR_FN (cfun))
2205 flags |= EDGE_FALLTHRU;
2206 new_edge = make_edge (new_bb, (basic_block) old_edge->dest->aux, flags);
2207 new_edge->count = apply_scale (old_edge->count, count_scale);
2208 new_edge->probability = old_edge->probability;
2209 }
2210
2211 if (bb->index == ENTRY_BLOCK || bb->index == EXIT_BLOCK)
2212 return false;
2213
2214 for (si = gsi_start_bb (new_bb); !gsi_end_p (si);)
2215 {
2216 gimple copy_stmt;
2217 bool can_throw, nonlocal_goto;
2218
2219 copy_stmt = gsi_stmt (si);
2220 if (!is_gimple_debug (copy_stmt))
2221 update_stmt (copy_stmt);
2222
2223 /* Do this before the possible split_block. */
2224 gsi_next (&si);
2225
2226 /* If this tree could throw an exception, there are two
2227 cases where we need to add abnormal edge(s): the
2228 tree wasn't in a region and there is a "current
2229 region" in the caller; or the original tree had
2230 EH edges. In both cases split the block after the tree,
2231 and add abnormal edge(s) as needed; we need both
2232 those from the callee and the caller.
2233 We check whether the copy can throw, because the const
2234 propagation can change an INDIRECT_REF which throws
2235 into a COMPONENT_REF which doesn't. If the copy
2236 can throw, the original could also throw. */
2237 can_throw = stmt_can_throw_internal (copy_stmt);
2238 nonlocal_goto
2239 = (stmt_can_make_abnormal_goto (copy_stmt)
2240 && !computed_goto_p (copy_stmt));
2241
2242 if (can_throw || nonlocal_goto)
2243 {
2244 if (!gsi_end_p (si))
2245 {
2246 while (!gsi_end_p (si) && is_gimple_debug (gsi_stmt (si)))
2247 gsi_next (&si);
2248 if (gsi_end_p (si))
2249 need_debug_cleanup = true;
2250 }
2251 if (!gsi_end_p (si))
2252 /* Note that bb's predecessor edges aren't necessarily
2253 right at this point; split_block doesn't care. */
2254 {
2255 edge e = split_block (new_bb, copy_stmt);
2256
2257 new_bb = e->dest;
2258 new_bb->aux = e->src->aux;
2259 si = gsi_start_bb (new_bb);
2260 }
2261 }
2262
2263 if (gimple_code (copy_stmt) == GIMPLE_EH_DISPATCH)
2264 make_eh_dispatch_edges (as_a <geh_dispatch *> (copy_stmt));
2265 else if (can_throw)
2266 make_eh_edges (copy_stmt);
2267
2268 /* If the call we inline cannot make abnormal goto do not add
2269 additional abnormal edges but only retain those already present
2270 in the original function body. */
2271 if (abnormal_goto_dest == NULL)
2272 nonlocal_goto = false;
2273 if (nonlocal_goto)
2274 {
2275 basic_block copy_stmt_bb = gimple_bb (copy_stmt);
2276
2277 if (get_abnormal_succ_dispatcher (copy_stmt_bb))
2278 nonlocal_goto = false;
2279 /* ABNORMAL_DISPATCHER (1) is for longjmp/setjmp or nonlocal gotos
2280 in OpenMP regions which aren't allowed to be left abnormally.
2281 So, no need to add abnormal edge in that case. */
2282 else if (is_gimple_call (copy_stmt)
2283 && gimple_call_internal_p (copy_stmt)
2284 && (gimple_call_internal_fn (copy_stmt)
2285 == IFN_ABNORMAL_DISPATCHER)
2286 && gimple_call_arg (copy_stmt, 0) == boolean_true_node)
2287 nonlocal_goto = false;
2288 else
2289 make_edge (copy_stmt_bb, abnormal_goto_dest, EDGE_ABNORMAL);
2290 }
2291
2292 if ((can_throw || nonlocal_goto)
2293 && gimple_in_ssa_p (cfun))
2294 update_ssa_across_abnormal_edges (gimple_bb (copy_stmt), ret_bb,
2295 can_throw, nonlocal_goto);
2296 }
2297 return need_debug_cleanup;
2298 }
2299
2300 /* Copy the PHIs. All blocks and edges are copied, some blocks
2301 was possibly split and new outgoing EH edges inserted.
2302 BB points to the block of original function and AUX pointers links
2303 the original and newly copied blocks. */
2304
2305 static void
2306 copy_phis_for_bb (basic_block bb, copy_body_data *id)
2307 {
2308 basic_block const new_bb = (basic_block) bb->aux;
2309 edge_iterator ei;
2310 gphi *phi;
2311 gphi_iterator si;
2312 edge new_edge;
2313 bool inserted = false;
2314
2315 for (si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si))
2316 {
2317 tree res, new_res;
2318 gphi *new_phi;
2319
2320 phi = si.phi ();
2321 res = PHI_RESULT (phi);
2322 new_res = res;
2323 if (!virtual_operand_p (res))
2324 {
2325 walk_tree (&new_res, copy_tree_body_r, id, NULL);
2326 new_phi = create_phi_node (new_res, new_bb);
2327 FOR_EACH_EDGE (new_edge, ei, new_bb->preds)
2328 {
2329 edge old_edge = find_edge ((basic_block) new_edge->src->aux, bb);
2330 tree arg;
2331 tree new_arg;
2332 edge_iterator ei2;
2333 location_t locus;
2334
2335 /* When doing partial cloning, we allow PHIs on the entry block
2336 as long as all the arguments are the same. Find any input
2337 edge to see argument to copy. */
2338 if (!old_edge)
2339 FOR_EACH_EDGE (old_edge, ei2, bb->preds)
2340 if (!old_edge->src->aux)
2341 break;
2342
2343 arg = PHI_ARG_DEF_FROM_EDGE (phi, old_edge);
2344 new_arg = arg;
2345 walk_tree (&new_arg, copy_tree_body_r, id, NULL);
2346 gcc_assert (new_arg);
2347 /* With return slot optimization we can end up with
2348 non-gimple (foo *)&this->m, fix that here. */
2349 if (TREE_CODE (new_arg) != SSA_NAME
2350 && TREE_CODE (new_arg) != FUNCTION_DECL
2351 && !is_gimple_val (new_arg))
2352 {
2353 gimple_seq stmts = NULL;
2354 new_arg = force_gimple_operand (new_arg, &stmts, true, NULL);
2355 gsi_insert_seq_on_edge (new_edge, stmts);
2356 inserted = true;
2357 }
2358 locus = gimple_phi_arg_location_from_edge (phi, old_edge);
2359 if (LOCATION_BLOCK (locus))
2360 {
2361 tree *n;
2362 n = id->decl_map->get (LOCATION_BLOCK (locus));
2363 gcc_assert (n);
2364 if (*n)
2365 locus = COMBINE_LOCATION_DATA (line_table, locus, *n);
2366 else
2367 locus = LOCATION_LOCUS (locus);
2368 }
2369 else
2370 locus = LOCATION_LOCUS (locus);
2371
2372 add_phi_arg (new_phi, new_arg, new_edge, locus);
2373 }
2374 }
2375 }
2376
2377 /* Commit the delayed edge insertions. */
2378 if (inserted)
2379 FOR_EACH_EDGE (new_edge, ei, new_bb->preds)
2380 gsi_commit_one_edge_insert (new_edge, NULL);
2381 }
2382
2383
2384 /* Wrapper for remap_decl so it can be used as a callback. */
2385
2386 static tree
2387 remap_decl_1 (tree decl, void *data)
2388 {
2389 return remap_decl (decl, (copy_body_data *) data);
2390 }
2391
2392 /* Build struct function and associated datastructures for the new clone
2393 NEW_FNDECL to be build. CALLEE_FNDECL is the original. Function changes
2394 the cfun to the function of new_fndecl (and current_function_decl too). */
2395
2396 static void
2397 initialize_cfun (tree new_fndecl, tree callee_fndecl, gcov_type count)
2398 {
2399 struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
2400 gcov_type count_scale;
2401
2402 if (!DECL_ARGUMENTS (new_fndecl))
2403 DECL_ARGUMENTS (new_fndecl) = DECL_ARGUMENTS (callee_fndecl);
2404 if (!DECL_RESULT (new_fndecl))
2405 DECL_RESULT (new_fndecl) = DECL_RESULT (callee_fndecl);
2406
2407 if (ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count)
2408 count_scale
2409 = GCOV_COMPUTE_SCALE (count,
2410 ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count);
2411 else
2412 count_scale = REG_BR_PROB_BASE;
2413
2414 /* Register specific tree functions. */
2415 gimple_register_cfg_hooks ();
2416
2417 /* Get clean struct function. */
2418 push_struct_function (new_fndecl);
2419
2420 /* We will rebuild these, so just sanity check that they are empty. */
2421 gcc_assert (VALUE_HISTOGRAMS (cfun) == NULL);
2422 gcc_assert (cfun->local_decls == NULL);
2423 gcc_assert (cfun->cfg == NULL);
2424 gcc_assert (cfun->decl == new_fndecl);
2425
2426 /* Copy items we preserve during cloning. */
2427 cfun->static_chain_decl = src_cfun->static_chain_decl;
2428 cfun->nonlocal_goto_save_area = src_cfun->nonlocal_goto_save_area;
2429 cfun->function_end_locus = src_cfun->function_end_locus;
2430 cfun->curr_properties = src_cfun->curr_properties;
2431 cfun->last_verified = src_cfun->last_verified;
2432 cfun->va_list_gpr_size = src_cfun->va_list_gpr_size;
2433 cfun->va_list_fpr_size = src_cfun->va_list_fpr_size;
2434 cfun->has_nonlocal_label = src_cfun->has_nonlocal_label;
2435 cfun->stdarg = src_cfun->stdarg;
2436 cfun->after_inlining = src_cfun->after_inlining;
2437 cfun->can_throw_non_call_exceptions
2438 = src_cfun->can_throw_non_call_exceptions;
2439 cfun->can_delete_dead_exceptions = src_cfun->can_delete_dead_exceptions;
2440 cfun->returns_struct = src_cfun->returns_struct;
2441 cfun->returns_pcc_struct = src_cfun->returns_pcc_struct;
2442
2443 init_empty_tree_cfg ();
2444
2445 profile_status_for_fn (cfun) = profile_status_for_fn (src_cfun);
2446 ENTRY_BLOCK_PTR_FOR_FN (cfun)->count =
2447 (ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count * count_scale /
2448 REG_BR_PROB_BASE);
2449 ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency
2450 = ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->frequency;
2451 EXIT_BLOCK_PTR_FOR_FN (cfun)->count =
2452 (EXIT_BLOCK_PTR_FOR_FN (src_cfun)->count * count_scale /
2453 REG_BR_PROB_BASE);
2454 EXIT_BLOCK_PTR_FOR_FN (cfun)->frequency =
2455 EXIT_BLOCK_PTR_FOR_FN (src_cfun)->frequency;
2456 if (src_cfun->eh)
2457 init_eh_for_function ();
2458
2459 if (src_cfun->gimple_df)
2460 {
2461 init_tree_ssa (cfun);
2462 cfun->gimple_df->in_ssa_p = true;
2463 init_ssa_operands (cfun);
2464 }
2465 }
2466
2467 /* Helper function for copy_cfg_body. Move debug stmts from the end
2468 of NEW_BB to the beginning of successor basic blocks when needed. If the
2469 successor has multiple predecessors, reset them, otherwise keep
2470 their value. */
2471
2472 static void
2473 maybe_move_debug_stmts_to_successors (copy_body_data *id, basic_block new_bb)
2474 {
2475 edge e;
2476 edge_iterator ei;
2477 gimple_stmt_iterator si = gsi_last_nondebug_bb (new_bb);
2478
2479 if (gsi_end_p (si)
2480 || gsi_one_before_end_p (si)
2481 || !(stmt_can_throw_internal (gsi_stmt (si))
2482 || stmt_can_make_abnormal_goto (gsi_stmt (si))))
2483 return;
2484
2485 FOR_EACH_EDGE (e, ei, new_bb->succs)
2486 {
2487 gimple_stmt_iterator ssi = gsi_last_bb (new_bb);
2488 gimple_stmt_iterator dsi = gsi_after_labels (e->dest);
2489 while (is_gimple_debug (gsi_stmt (ssi)))
2490 {
2491 gimple stmt = gsi_stmt (ssi);
2492 gdebug *new_stmt;
2493 tree var;
2494 tree value;
2495
2496 /* For the last edge move the debug stmts instead of copying
2497 them. */
2498 if (ei_one_before_end_p (ei))
2499 {
2500 si = ssi;
2501 gsi_prev (&ssi);
2502 if (!single_pred_p (e->dest) && gimple_debug_bind_p (stmt))
2503 gimple_debug_bind_reset_value (stmt);
2504 gsi_remove (&si, false);
2505 gsi_insert_before (&dsi, stmt, GSI_SAME_STMT);
2506 continue;
2507 }
2508
2509 if (gimple_debug_bind_p (stmt))
2510 {
2511 var = gimple_debug_bind_get_var (stmt);
2512 if (single_pred_p (e->dest))
2513 {
2514 value = gimple_debug_bind_get_value (stmt);
2515 value = unshare_expr (value);
2516 }
2517 else
2518 value = NULL_TREE;
2519 new_stmt = gimple_build_debug_bind (var, value, stmt);
2520 }
2521 else if (gimple_debug_source_bind_p (stmt))
2522 {
2523 var = gimple_debug_source_bind_get_var (stmt);
2524 value = gimple_debug_source_bind_get_value (stmt);
2525 new_stmt = gimple_build_debug_source_bind (var, value, stmt);
2526 }
2527 else
2528 gcc_unreachable ();
2529 gsi_insert_before (&dsi, new_stmt, GSI_SAME_STMT);
2530 id->debug_stmts.safe_push (new_stmt);
2531 gsi_prev (&ssi);
2532 }
2533 }
2534 }
2535
2536 /* Make a copy of the sub-loops of SRC_PARENT and place them
2537 as siblings of DEST_PARENT. */
2538
2539 static void
2540 copy_loops (copy_body_data *id,
2541 struct loop *dest_parent, struct loop *src_parent)
2542 {
2543 struct loop *src_loop = src_parent->inner;
2544 while (src_loop)
2545 {
2546 if (!id->blocks_to_copy
2547 || bitmap_bit_p (id->blocks_to_copy, src_loop->header->index))
2548 {
2549 struct loop *dest_loop = alloc_loop ();
2550
2551 /* Assign the new loop its header and latch and associate
2552 those with the new loop. */
2553 dest_loop->header = (basic_block)src_loop->header->aux;
2554 dest_loop->header->loop_father = dest_loop;
2555 if (src_loop->latch != NULL)
2556 {
2557 dest_loop->latch = (basic_block)src_loop->latch->aux;
2558 dest_loop->latch->loop_father = dest_loop;
2559 }
2560
2561 /* Copy loop meta-data. */
2562 copy_loop_info (src_loop, dest_loop);
2563
2564 /* Finally place it into the loop array and the loop tree. */
2565 place_new_loop (cfun, dest_loop);
2566 flow_loop_tree_node_add (dest_parent, dest_loop);
2567
2568 dest_loop->safelen = src_loop->safelen;
2569 dest_loop->dont_vectorize = src_loop->dont_vectorize;
2570 if (src_loop->force_vectorize)
2571 {
2572 dest_loop->force_vectorize = true;
2573 cfun->has_force_vectorize_loops = true;
2574 }
2575 if (src_loop->simduid)
2576 {
2577 dest_loop->simduid = remap_decl (src_loop->simduid, id);
2578 cfun->has_simduid_loops = true;
2579 }
2580
2581 /* Recurse. */
2582 copy_loops (id, dest_loop, src_loop);
2583 }
2584 src_loop = src_loop->next;
2585 }
2586 }
2587
2588 /* Call cgraph_redirect_edge_call_stmt_to_callee on all calls in BB */
2589
2590 void
2591 redirect_all_calls (copy_body_data * id, basic_block bb)
2592 {
2593 gimple_stmt_iterator si;
2594 gimple last = last_stmt (bb);
2595 for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
2596 {
2597 gimple stmt = gsi_stmt (si);
2598 if (is_gimple_call (stmt))
2599 {
2600 struct cgraph_edge *edge = id->dst_node->get_edge (stmt);
2601 if (edge)
2602 {
2603 edge->redirect_call_stmt_to_callee ();
2604 if (stmt == last && id->call_stmt && maybe_clean_eh_stmt (stmt))
2605 gimple_purge_dead_eh_edges (bb);
2606 }
2607 }
2608 }
2609 }
2610
2611 /* Convert estimated frequencies into counts for NODE, scaling COUNT
2612 with each bb's frequency. Used when NODE has a 0-weight entry
2613 but we are about to inline it into a non-zero count call bb.
2614 See the comments for handle_missing_profiles() in predict.c for
2615 when this can happen for COMDATs. */
2616
2617 void
2618 freqs_to_counts (struct cgraph_node *node, gcov_type count)
2619 {
2620 basic_block bb;
2621 edge_iterator ei;
2622 edge e;
2623 struct function *fn = DECL_STRUCT_FUNCTION (node->decl);
2624
2625 FOR_ALL_BB_FN(bb, fn)
2626 {
2627 bb->count = apply_scale (count,
2628 GCOV_COMPUTE_SCALE (bb->frequency, BB_FREQ_MAX));
2629 FOR_EACH_EDGE (e, ei, bb->succs)
2630 e->count = apply_probability (e->src->count, e->probability);
2631 }
2632 }
2633
2634 /* Make a copy of the body of FN so that it can be inserted inline in
2635 another function. Walks FN via CFG, returns new fndecl. */
2636
2637 static tree
2638 copy_cfg_body (copy_body_data * id, gcov_type count, int frequency_scale,
2639 basic_block entry_block_map, basic_block exit_block_map,
2640 basic_block new_entry)
2641 {
2642 tree callee_fndecl = id->src_fn;
2643 /* Original cfun for the callee, doesn't change. */
2644 struct function *src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
2645 struct function *cfun_to_copy;
2646 basic_block bb;
2647 tree new_fndecl = NULL;
2648 bool need_debug_cleanup = false;
2649 gcov_type count_scale;
2650 int last;
2651 int incoming_frequency = 0;
2652 gcov_type incoming_count = 0;
2653
2654 /* This can happen for COMDAT routines that end up with 0 counts
2655 despite being called (see the comments for handle_missing_profiles()
2656 in predict.c as to why). Apply counts to the blocks in the callee
2657 before inlining, using the guessed edge frequencies, so that we don't
2658 end up with a 0-count inline body which can confuse downstream
2659 optimizations such as function splitting. */
2660 if (!ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count && count)
2661 {
2662 /* Apply the larger of the call bb count and the total incoming
2663 call edge count to the callee. */
2664 gcov_type in_count = 0;
2665 struct cgraph_edge *in_edge;
2666 for (in_edge = id->src_node->callers; in_edge;
2667 in_edge = in_edge->next_caller)
2668 in_count += in_edge->count;
2669 freqs_to_counts (id->src_node, count > in_count ? count : in_count);
2670 }
2671
2672 if (ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count)
2673 count_scale
2674 = GCOV_COMPUTE_SCALE (count,
2675 ENTRY_BLOCK_PTR_FOR_FN (src_cfun)->count);
2676 else
2677 count_scale = REG_BR_PROB_BASE;
2678
2679 /* Register specific tree functions. */
2680 gimple_register_cfg_hooks ();
2681
2682 /* If we are inlining just region of the function, make sure to connect
2683 new entry to ENTRY_BLOCK_PTR_FOR_FN (cfun). Since new entry can be
2684 part of loop, we must compute frequency and probability of
2685 ENTRY_BLOCK_PTR_FOR_FN (cfun) based on the frequencies and
2686 probabilities of edges incoming from nonduplicated region. */
2687 if (new_entry)
2688 {
2689 edge e;
2690 edge_iterator ei;
2691
2692 FOR_EACH_EDGE (e, ei, new_entry->preds)
2693 if (!e->src->aux)
2694 {
2695 incoming_frequency += EDGE_FREQUENCY (e);
2696 incoming_count += e->count;
2697 }
2698 incoming_count = apply_scale (incoming_count, count_scale);
2699 incoming_frequency
2700 = apply_scale ((gcov_type)incoming_frequency, frequency_scale);
2701 ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = incoming_count;
2702 ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency = incoming_frequency;
2703 }
2704
2705 /* Must have a CFG here at this point. */
2706 gcc_assert (ENTRY_BLOCK_PTR_FOR_FN
2707 (DECL_STRUCT_FUNCTION (callee_fndecl)));
2708
2709 cfun_to_copy = id->src_cfun = DECL_STRUCT_FUNCTION (callee_fndecl);
2710
2711 ENTRY_BLOCK_PTR_FOR_FN (cfun_to_copy)->aux = entry_block_map;
2712 EXIT_BLOCK_PTR_FOR_FN (cfun_to_copy)->aux = exit_block_map;
2713 entry_block_map->aux = ENTRY_BLOCK_PTR_FOR_FN (cfun_to_copy);
2714 exit_block_map->aux = EXIT_BLOCK_PTR_FOR_FN (cfun_to_copy);
2715
2716 /* Duplicate any exception-handling regions. */
2717 if (cfun->eh)
2718 id->eh_map = duplicate_eh_regions (cfun_to_copy, NULL, id->eh_lp_nr,
2719 remap_decl_1, id);
2720
2721 /* Use aux pointers to map the original blocks to copy. */
2722 FOR_EACH_BB_FN (bb, cfun_to_copy)
2723 if (!id->blocks_to_copy || bitmap_bit_p (id->blocks_to_copy, bb->index))
2724 {
2725 basic_block new_bb = copy_bb (id, bb, frequency_scale, count_scale);
2726 bb->aux = new_bb;
2727 new_bb->aux = bb;
2728 new_bb->loop_father = entry_block_map->loop_father;
2729 }
2730
2731 last = last_basic_block_for_fn (cfun);
2732
2733 /* Now that we've duplicated the blocks, duplicate their edges. */
2734 basic_block abnormal_goto_dest = NULL;
2735 if (id->call_stmt
2736 && stmt_can_make_abnormal_goto (id->call_stmt))
2737 {
2738 gimple_stmt_iterator gsi = gsi_for_stmt (id->call_stmt);
2739
2740 bb = gimple_bb (id->call_stmt);
2741 gsi_next (&gsi);
2742 if (gsi_end_p (gsi))
2743 abnormal_goto_dest = get_abnormal_succ_dispatcher (bb);
2744 }
2745 FOR_ALL_BB_FN (bb, cfun_to_copy)
2746 if (!id->blocks_to_copy
2747 || (bb->index > 0 && bitmap_bit_p (id->blocks_to_copy, bb->index)))
2748 need_debug_cleanup |= copy_edges_for_bb (bb, count_scale, exit_block_map,
2749 abnormal_goto_dest);
2750
2751 if (new_entry)
2752 {
2753 edge e = make_edge (entry_block_map, (basic_block)new_entry->aux, EDGE_FALLTHRU);
2754 e->probability = REG_BR_PROB_BASE;
2755 e->count = incoming_count;
2756 }
2757
2758 /* Duplicate the loop tree, if available and wanted. */
2759 if (loops_for_fn (src_cfun) != NULL
2760 && current_loops != NULL)
2761 {
2762 copy_loops (id, entry_block_map->loop_father,
2763 get_loop (src_cfun, 0));
2764 /* Defer to cfgcleanup to update loop-father fields of basic-blocks. */
2765 loops_state_set (LOOPS_NEED_FIXUP);
2766 }
2767
2768 /* If the loop tree in the source function needed fixup, mark the
2769 destination loop tree for fixup, too. */
2770 if (loops_for_fn (src_cfun)->state & LOOPS_NEED_FIXUP)
2771 loops_state_set (LOOPS_NEED_FIXUP);
2772
2773 if (gimple_in_ssa_p (cfun))
2774 FOR_ALL_BB_FN (bb, cfun_to_copy)
2775 if (!id->blocks_to_copy
2776 || (bb->index > 0 && bitmap_bit_p (id->blocks_to_copy, bb->index)))
2777 copy_phis_for_bb (bb, id);
2778
2779 FOR_ALL_BB_FN (bb, cfun_to_copy)
2780 if (bb->aux)
2781 {
2782 if (need_debug_cleanup
2783 && bb->index != ENTRY_BLOCK
2784 && bb->index != EXIT_BLOCK)
2785 maybe_move_debug_stmts_to_successors (id, (basic_block) bb->aux);
2786 /* Update call edge destinations. This can not be done before loop
2787 info is updated, because we may split basic blocks. */
2788 if (id->transform_call_graph_edges == CB_CGE_DUPLICATE)
2789 redirect_all_calls (id, (basic_block)bb->aux);
2790 ((basic_block)bb->aux)->aux = NULL;
2791 bb->aux = NULL;
2792 }
2793
2794 /* Zero out AUX fields of newly created block during EH edge
2795 insertion. */
2796 for (; last < last_basic_block_for_fn (cfun); last++)
2797 {
2798 if (need_debug_cleanup)
2799 maybe_move_debug_stmts_to_successors (id,
2800 BASIC_BLOCK_FOR_FN (cfun, last));
2801 BASIC_BLOCK_FOR_FN (cfun, last)->aux = NULL;
2802 /* Update call edge destinations. This can not be done before loop
2803 info is updated, because we may split basic blocks. */
2804 if (id->transform_call_graph_edges == CB_CGE_DUPLICATE)
2805 redirect_all_calls (id, BASIC_BLOCK_FOR_FN (cfun, last));
2806 }
2807 entry_block_map->aux = NULL;
2808 exit_block_map->aux = NULL;
2809
2810 if (id->eh_map)
2811 {
2812 delete id->eh_map;
2813 id->eh_map = NULL;
2814 }
2815 if (id->dependence_map)
2816 {
2817 delete id->dependence_map;
2818 id->dependence_map = NULL;
2819 }
2820
2821 return new_fndecl;
2822 }
2823
2824 /* Copy the debug STMT using ID. We deal with these statements in a
2825 special way: if any variable in their VALUE expression wasn't
2826 remapped yet, we won't remap it, because that would get decl uids
2827 out of sync, causing codegen differences between -g and -g0. If
2828 this arises, we drop the VALUE expression altogether. */
2829
2830 static void
2831 copy_debug_stmt (gdebug *stmt, copy_body_data *id)
2832 {
2833 tree t, *n;
2834 struct walk_stmt_info wi;
2835
2836 if (gimple_block (stmt))
2837 {
2838 n = id->decl_map->get (gimple_block (stmt));
2839 gimple_set_block (stmt, n ? *n : id->block);
2840 }
2841
2842 /* Remap all the operands in COPY. */
2843 memset (&wi, 0, sizeof (wi));
2844 wi.info = id;
2845
2846 processing_debug_stmt = 1;
2847
2848 if (gimple_debug_source_bind_p (stmt))
2849 t = gimple_debug_source_bind_get_var (stmt);
2850 else
2851 t = gimple_debug_bind_get_var (stmt);
2852
2853 if (TREE_CODE (t) == PARM_DECL && id->debug_map
2854 && (n = id->debug_map->get (t)))
2855 {
2856 gcc_assert (TREE_CODE (*n) == VAR_DECL);
2857 t = *n;
2858 }
2859 else if (TREE_CODE (t) == VAR_DECL
2860 && !is_global_var (t)
2861 && !id->decl_map->get (t))
2862 /* T is a non-localized variable. */;
2863 else
2864 walk_tree (&t, remap_gimple_op_r, &wi, NULL);
2865
2866 if (gimple_debug_bind_p (stmt))
2867 {
2868 gimple_debug_bind_set_var (stmt, t);
2869
2870 if (gimple_debug_bind_has_value_p (stmt))
2871 walk_tree (gimple_debug_bind_get_value_ptr (stmt),
2872 remap_gimple_op_r, &wi, NULL);
2873
2874 /* Punt if any decl couldn't be remapped. */
2875 if (processing_debug_stmt < 0)
2876 gimple_debug_bind_reset_value (stmt);
2877 }
2878 else if (gimple_debug_source_bind_p (stmt))
2879 {
2880 gimple_debug_source_bind_set_var (stmt, t);
2881 walk_tree (gimple_debug_source_bind_get_value_ptr (stmt),
2882 remap_gimple_op_r, &wi, NULL);
2883 /* When inlining and source bind refers to one of the optimized
2884 away parameters, change the source bind into normal debug bind
2885 referring to the corresponding DEBUG_EXPR_DECL that should have
2886 been bound before the call stmt. */
2887 t = gimple_debug_source_bind_get_value (stmt);
2888 if (t != NULL_TREE
2889 && TREE_CODE (t) == PARM_DECL
2890 && id->call_stmt)
2891 {
2892 vec<tree, va_gc> **debug_args = decl_debug_args_lookup (id->src_fn);
2893 unsigned int i;
2894 if (debug_args != NULL)
2895 {
2896 for (i = 0; i < vec_safe_length (*debug_args); i += 2)
2897 if ((**debug_args)[i] == DECL_ORIGIN (t)
2898 && TREE_CODE ((**debug_args)[i + 1]) == DEBUG_EXPR_DECL)
2899 {
2900 t = (**debug_args)[i + 1];
2901 stmt->subcode = GIMPLE_DEBUG_BIND;
2902 gimple_debug_bind_set_value (stmt, t);
2903 break;
2904 }
2905 }
2906 }
2907 }
2908
2909 processing_debug_stmt = 0;
2910
2911 update_stmt (stmt);
2912 }
2913
2914 /* Process deferred debug stmts. In order to give values better odds
2915 of being successfully remapped, we delay the processing of debug
2916 stmts until all other stmts that might require remapping are
2917 processed. */
2918
2919 static void
2920 copy_debug_stmts (copy_body_data *id)
2921 {
2922 size_t i;
2923 gdebug *stmt;
2924
2925 if (!id->debug_stmts.exists ())
2926 return;
2927
2928 FOR_EACH_VEC_ELT (id->debug_stmts, i, stmt)
2929 copy_debug_stmt (stmt, id);
2930
2931 id->debug_stmts.release ();
2932 }
2933
2934 /* Make a copy of the body of SRC_FN so that it can be inserted inline in
2935 another function. */
2936
2937 static tree
2938 copy_tree_body (copy_body_data *id)
2939 {
2940 tree fndecl = id->src_fn;
2941 tree body = DECL_SAVED_TREE (fndecl);
2942
2943 walk_tree (&body, copy_tree_body_r, id, NULL);
2944
2945 return body;
2946 }
2947
2948 /* Make a copy of the body of FN so that it can be inserted inline in
2949 another function. */
2950
2951 static tree
2952 copy_body (copy_body_data *id, gcov_type count, int frequency_scale,
2953 basic_block entry_block_map, basic_block exit_block_map,
2954 basic_block new_entry)
2955 {
2956 tree fndecl = id->src_fn;
2957 tree body;
2958
2959 /* If this body has a CFG, walk CFG and copy. */
2960 gcc_assert (ENTRY_BLOCK_PTR_FOR_FN (DECL_STRUCT_FUNCTION (fndecl)));
2961 body = copy_cfg_body (id, count, frequency_scale, entry_block_map, exit_block_map,
2962 new_entry);
2963 copy_debug_stmts (id);
2964
2965 return body;
2966 }
2967
2968 /* Return true if VALUE is an ADDR_EXPR of an automatic variable
2969 defined in function FN, or of a data member thereof. */
2970
2971 static bool
2972 self_inlining_addr_expr (tree value, tree fn)
2973 {
2974 tree var;
2975
2976 if (TREE_CODE (value) != ADDR_EXPR)
2977 return false;
2978
2979 var = get_base_address (TREE_OPERAND (value, 0));
2980
2981 return var && auto_var_in_fn_p (var, fn);
2982 }
2983
2984 /* Append to BB a debug annotation that binds VAR to VALUE, inheriting
2985 lexical block and line number information from base_stmt, if given,
2986 or from the last stmt of the block otherwise. */
2987
2988 static gimple
2989 insert_init_debug_bind (copy_body_data *id,
2990 basic_block bb, tree var, tree value,
2991 gimple base_stmt)
2992 {
2993 gimple note;
2994 gimple_stmt_iterator gsi;
2995 tree tracked_var;
2996
2997 if (!gimple_in_ssa_p (id->src_cfun))
2998 return NULL;
2999
3000 if (!MAY_HAVE_DEBUG_STMTS)
3001 return NULL;
3002
3003 tracked_var = target_for_debug_bind (var);
3004 if (!tracked_var)
3005 return NULL;
3006
3007 if (bb)
3008 {
3009 gsi = gsi_last_bb (bb);
3010 if (!base_stmt && !gsi_end_p (gsi))
3011 base_stmt = gsi_stmt (gsi);
3012 }
3013
3014 note = gimple_build_debug_bind (tracked_var, value, base_stmt);
3015
3016 if (bb)
3017 {
3018 if (!gsi_end_p (gsi))
3019 gsi_insert_after (&gsi, note, GSI_SAME_STMT);
3020 else
3021 gsi_insert_before (&gsi, note, GSI_SAME_STMT);
3022 }
3023
3024 return note;
3025 }
3026
3027 static void
3028 insert_init_stmt (copy_body_data *id, basic_block bb, gimple init_stmt)
3029 {
3030 /* If VAR represents a zero-sized variable, it's possible that the
3031 assignment statement may result in no gimple statements. */
3032 if (init_stmt)
3033 {
3034 gimple_stmt_iterator si = gsi_last_bb (bb);
3035
3036 /* We can end up with init statements that store to a non-register
3037 from a rhs with a conversion. Handle that here by forcing the
3038 rhs into a temporary. gimple_regimplify_operands is not
3039 prepared to do this for us. */
3040 if (!is_gimple_debug (init_stmt)
3041 && !is_gimple_reg (gimple_assign_lhs (init_stmt))
3042 && is_gimple_reg_type (TREE_TYPE (gimple_assign_lhs (init_stmt)))
3043 && gimple_assign_rhs_class (init_stmt) == GIMPLE_UNARY_RHS)
3044 {
3045 tree rhs = build1 (gimple_assign_rhs_code (init_stmt),
3046 gimple_expr_type (init_stmt),
3047 gimple_assign_rhs1 (init_stmt));
3048 rhs = force_gimple_operand_gsi (&si, rhs, true, NULL_TREE, false,
3049 GSI_NEW_STMT);
3050 gimple_assign_set_rhs_code (init_stmt, TREE_CODE (rhs));
3051 gimple_assign_set_rhs1 (init_stmt, rhs);
3052 }
3053 gsi_insert_after (&si, init_stmt, GSI_NEW_STMT);
3054 gimple_regimplify_operands (init_stmt, &si);
3055
3056 if (!is_gimple_debug (init_stmt) && MAY_HAVE_DEBUG_STMTS)
3057 {
3058 tree def = gimple_assign_lhs (init_stmt);
3059 insert_init_debug_bind (id, bb, def, def, init_stmt);
3060 }
3061 }
3062 }
3063
3064 /* Initialize parameter P with VALUE. If needed, produce init statement
3065 at the end of BB. When BB is NULL, we return init statement to be
3066 output later. */
3067 static gimple
3068 setup_one_parameter (copy_body_data *id, tree p, tree value, tree fn,
3069 basic_block bb, tree *vars)
3070 {
3071 gimple init_stmt = NULL;
3072 tree var;
3073 tree rhs = value;
3074 tree def = (gimple_in_ssa_p (cfun)
3075 ? ssa_default_def (id->src_cfun, p) : NULL);
3076
3077 if (value
3078 && value != error_mark_node
3079 && !useless_type_conversion_p (TREE_TYPE (p), TREE_TYPE (value)))
3080 {
3081 /* If we can match up types by promotion/demotion do so. */
3082 if (fold_convertible_p (TREE_TYPE (p), value))
3083 rhs = fold_convert (TREE_TYPE (p), value);
3084 else
3085 {
3086 /* ??? For valid programs we should not end up here.
3087 Still if we end up with truly mismatched types here, fall back
3088 to using a VIEW_CONVERT_EXPR or a literal zero to not leak invalid
3089 GIMPLE to the following passes. */
3090 if (!is_gimple_reg_type (TREE_TYPE (value))
3091 || TYPE_SIZE (TREE_TYPE (p)) == TYPE_SIZE (TREE_TYPE (value)))
3092 rhs = fold_build1 (VIEW_CONVERT_EXPR, TREE_TYPE (p), value);
3093 else
3094 rhs = build_zero_cst (TREE_TYPE (p));
3095 }
3096 }
3097
3098 /* Make an equivalent VAR_DECL. Note that we must NOT remap the type
3099 here since the type of this decl must be visible to the calling
3100 function. */
3101 var = copy_decl_to_var (p, id);
3102
3103 /* Declare this new variable. */
3104 DECL_CHAIN (var) = *vars;
3105 *vars = var;
3106
3107 /* Make gimplifier happy about this variable. */
3108 DECL_SEEN_IN_BIND_EXPR_P (var) = 1;
3109
3110 /* If the parameter is never assigned to, has no SSA_NAMEs created,
3111 we would not need to create a new variable here at all, if it
3112 weren't for debug info. Still, we can just use the argument
3113 value. */
3114 if (TREE_READONLY (p)
3115 && !TREE_ADDRESSABLE (p)
3116 && value && !TREE_SIDE_EFFECTS (value)
3117 && !def)
3118 {
3119 /* We may produce non-gimple trees by adding NOPs or introduce
3120 invalid sharing when operand is not really constant.
3121 It is not big deal to prohibit constant propagation here as
3122 we will constant propagate in DOM1 pass anyway. */
3123 if (is_gimple_min_invariant (value)
3124 && useless_type_conversion_p (TREE_TYPE (p),
3125 TREE_TYPE (value))
3126 /* We have to be very careful about ADDR_EXPR. Make sure
3127 the base variable isn't a local variable of the inlined
3128 function, e.g., when doing recursive inlining, direct or
3129 mutually-recursive or whatever, which is why we don't
3130 just test whether fn == current_function_decl. */
3131 && ! self_inlining_addr_expr (value, fn))
3132 {
3133 insert_decl_map (id, p, value);
3134 insert_debug_decl_map (id, p, var);
3135 return insert_init_debug_bind (id, bb, var, value, NULL);
3136 }
3137 }
3138
3139 /* Register the VAR_DECL as the equivalent for the PARM_DECL;
3140 that way, when the PARM_DECL is encountered, it will be
3141 automatically replaced by the VAR_DECL. */
3142 insert_decl_map (id, p, var);
3143
3144 /* Even if P was TREE_READONLY, the new VAR should not be.
3145 In the original code, we would have constructed a
3146 temporary, and then the function body would have never
3147 changed the value of P. However, now, we will be
3148 constructing VAR directly. The constructor body may
3149 change its value multiple times as it is being
3150 constructed. Therefore, it must not be TREE_READONLY;
3151 the back-end assumes that TREE_READONLY variable is
3152 assigned to only once. */
3153 if (TYPE_NEEDS_CONSTRUCTING (TREE_TYPE (p)))
3154 TREE_READONLY (var) = 0;
3155
3156 /* If there is no setup required and we are in SSA, take the easy route
3157 replacing all SSA names representing the function parameter by the
3158 SSA name passed to function.
3159
3160 We need to construct map for the variable anyway as it might be used
3161 in different SSA names when parameter is set in function.
3162
3163 Do replacement at -O0 for const arguments replaced by constant.
3164 This is important for builtin_constant_p and other construct requiring
3165 constant argument to be visible in inlined function body. */
3166 if (gimple_in_ssa_p (cfun) && rhs && def && is_gimple_reg (p)
3167 && (optimize
3168 || (TREE_READONLY (p)
3169 && is_gimple_min_invariant (rhs)))
3170 && (TREE_CODE (rhs) == SSA_NAME
3171 || is_gimple_min_invariant (rhs))
3172 && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (def))
3173 {
3174 insert_decl_map (id, def, rhs);
3175 return insert_init_debug_bind (id, bb, var, rhs, NULL);
3176 }
3177
3178 /* If the value of argument is never used, don't care about initializing
3179 it. */
3180 if (optimize && gimple_in_ssa_p (cfun) && !def && is_gimple_reg (p))
3181 {
3182 gcc_assert (!value || !TREE_SIDE_EFFECTS (value));
3183 return insert_init_debug_bind (id, bb, var, rhs, NULL);
3184 }
3185
3186 /* Initialize this VAR_DECL from the equivalent argument. Convert
3187 the argument to the proper type in case it was promoted. */
3188 if (value)
3189 {
3190 if (rhs == error_mark_node)
3191 {
3192 insert_decl_map (id, p, var);
3193 return insert_init_debug_bind (id, bb, var, rhs, NULL);
3194 }
3195
3196 STRIP_USELESS_TYPE_CONVERSION (rhs);
3197
3198 /* If we are in SSA form properly remap the default definition
3199 or assign to a dummy SSA name if the parameter is unused and
3200 we are not optimizing. */
3201 if (gimple_in_ssa_p (cfun) && is_gimple_reg (p))
3202 {
3203 if (def)
3204 {
3205 def = remap_ssa_name (def, id);
3206 init_stmt = gimple_build_assign (def, rhs);
3207 SSA_NAME_IS_DEFAULT_DEF (def) = 0;
3208 set_ssa_default_def (cfun, var, NULL);
3209 }
3210 else if (!optimize)
3211 {
3212 def = make_ssa_name (var);
3213 init_stmt = gimple_build_assign (def, rhs);
3214 }
3215 }
3216 else
3217 init_stmt = gimple_build_assign (var, rhs);
3218
3219 if (bb && init_stmt)
3220 insert_init_stmt (id, bb, init_stmt);
3221 }
3222 return init_stmt;
3223 }
3224
3225 /* Generate code to initialize the parameters of the function at the
3226 top of the stack in ID from the GIMPLE_CALL STMT. */
3227
3228 static void
3229 initialize_inlined_parameters (copy_body_data *id, gimple stmt,
3230 tree fn, basic_block bb)
3231 {
3232 tree parms;
3233 size_t i;
3234 tree p;
3235 tree vars = NULL_TREE;
3236 tree static_chain = gimple_call_chain (stmt);
3237
3238 /* Figure out what the parameters are. */
3239 parms = DECL_ARGUMENTS (fn);
3240
3241 /* Loop through the parameter declarations, replacing each with an
3242 equivalent VAR_DECL, appropriately initialized. */
3243 for (p = parms, i = 0; p; p = DECL_CHAIN (p), i++)
3244 {
3245 tree val;
3246 val = i < gimple_call_num_args (stmt) ? gimple_call_arg (stmt, i) : NULL;
3247 setup_one_parameter (id, p, val, fn, bb, &vars);
3248 }
3249 /* After remapping parameters remap their types. This has to be done
3250 in a second loop over all parameters to appropriately remap
3251 variable sized arrays when the size is specified in a
3252 parameter following the array. */
3253 for (p = parms, i = 0; p; p = DECL_CHAIN (p), i++)
3254 {
3255 tree *varp = id->decl_map->get (p);
3256 if (varp
3257 && TREE_CODE (*varp) == VAR_DECL)
3258 {
3259 tree def = (gimple_in_ssa_p (cfun) && is_gimple_reg (p)
3260 ? ssa_default_def (id->src_cfun, p) : NULL);
3261 tree var = *varp;
3262 TREE_TYPE (var) = remap_type (TREE_TYPE (var), id);
3263 /* Also remap the default definition if it was remapped
3264 to the default definition of the parameter replacement
3265 by the parameter setup. */
3266 if (def)
3267 {
3268 tree *defp = id->decl_map->get (def);
3269 if (defp
3270 && TREE_CODE (*defp) == SSA_NAME
3271 && SSA_NAME_VAR (*defp) == var)
3272 TREE_TYPE (*defp) = TREE_TYPE (var);
3273 }
3274 }
3275 }
3276
3277 /* Initialize the static chain. */
3278 p = DECL_STRUCT_FUNCTION (fn)->static_chain_decl;
3279 gcc_assert (fn != current_function_decl);
3280 if (p)
3281 {
3282 /* No static chain? Seems like a bug in tree-nested.c. */
3283 gcc_assert (static_chain);
3284
3285 setup_one_parameter (id, p, static_chain, fn, bb, &vars);
3286 }
3287
3288 declare_inline_vars (id->block, vars);
3289 }
3290
3291
3292 /* Declare a return variable to replace the RESULT_DECL for the
3293 function we are calling. An appropriate DECL_STMT is returned.
3294 The USE_STMT is filled to contain a use of the declaration to
3295 indicate the return value of the function.
3296
3297 RETURN_SLOT, if non-null is place where to store the result. It
3298 is set only for CALL_EXPR_RETURN_SLOT_OPT. MODIFY_DEST, if non-null,
3299 was the LHS of the MODIFY_EXPR to which this call is the RHS.
3300
3301 RETURN_BOUNDS holds a destination for returned bounds.
3302
3303 The return value is a (possibly null) value that holds the result
3304 as seen by the caller. */
3305
3306 static tree
3307 declare_return_variable (copy_body_data *id, tree return_slot, tree modify_dest,
3308 tree return_bounds, basic_block entry_bb)
3309 {
3310 tree callee = id->src_fn;
3311 tree result = DECL_RESULT (callee);
3312 tree callee_type = TREE_TYPE (result);
3313 tree caller_type;
3314 tree var, use;
3315
3316 /* Handle type-mismatches in the function declaration return type
3317 vs. the call expression. */
3318 if (modify_dest)
3319 caller_type = TREE_TYPE (modify_dest);
3320 else
3321 caller_type = TREE_TYPE (TREE_TYPE (callee));
3322
3323 /* We don't need to do anything for functions that don't return anything. */
3324 if (VOID_TYPE_P (callee_type))
3325 return NULL_TREE;
3326
3327 /* If there was a return slot, then the return value is the
3328 dereferenced address of that object. */
3329 if (return_slot)
3330 {
3331 /* The front end shouldn't have used both return_slot and
3332 a modify expression. */
3333 gcc_assert (!modify_dest);
3334 if (DECL_BY_REFERENCE (result))
3335 {
3336 tree return_slot_addr = build_fold_addr_expr (return_slot);
3337 STRIP_USELESS_TYPE_CONVERSION (return_slot_addr);
3338
3339 /* We are going to construct *&return_slot and we can't do that
3340 for variables believed to be not addressable.
3341
3342 FIXME: This check possibly can match, because values returned
3343 via return slot optimization are not believed to have address
3344 taken by alias analysis. */
3345 gcc_assert (TREE_CODE (return_slot) != SSA_NAME);
3346 var = return_slot_addr;
3347 }
3348 else
3349 {
3350 var = return_slot;
3351 gcc_assert (TREE_CODE (var) != SSA_NAME);
3352 if (TREE_ADDRESSABLE (result))
3353 mark_addressable (var);
3354 }
3355 if ((TREE_CODE (TREE_TYPE (result)) == COMPLEX_TYPE
3356 || TREE_CODE (TREE_TYPE (result)) == VECTOR_TYPE)
3357 && !DECL_GIMPLE_REG_P (result)
3358 && DECL_P (var))
3359 DECL_GIMPLE_REG_P (var) = 0;
3360 use = NULL;
3361 goto done;
3362 }
3363
3364 /* All types requiring non-trivial constructors should have been handled. */
3365 gcc_assert (!TREE_ADDRESSABLE (callee_type));
3366
3367 /* Attempt to avoid creating a new temporary variable. */
3368 if (modify_dest
3369 && TREE_CODE (modify_dest) != SSA_NAME)
3370 {
3371 bool use_it = false;
3372
3373 /* We can't use MODIFY_DEST if there's type promotion involved. */
3374 if (!useless_type_conversion_p (callee_type, caller_type))
3375 use_it = false;
3376
3377 /* ??? If we're assigning to a variable sized type, then we must
3378 reuse the destination variable, because we've no good way to
3379 create variable sized temporaries at this point. */
3380 else if (TREE_CODE (TYPE_SIZE_UNIT (caller_type)) != INTEGER_CST)
3381 use_it = true;
3382
3383 /* If the callee cannot possibly modify MODIFY_DEST, then we can
3384 reuse it as the result of the call directly. Don't do this if
3385 it would promote MODIFY_DEST to addressable. */
3386 else if (TREE_ADDRESSABLE (result))
3387 use_it = false;
3388 else
3389 {
3390 tree base_m = get_base_address (modify_dest);
3391
3392 /* If the base isn't a decl, then it's a pointer, and we don't
3393 know where that's going to go. */
3394 if (!DECL_P (base_m))
3395 use_it = false;
3396 else if (is_global_var (base_m))
3397 use_it = false;
3398 else if ((TREE_CODE (TREE_TYPE (result)) == COMPLEX_TYPE
3399 || TREE_CODE (TREE_TYPE (result)) == VECTOR_TYPE)
3400 && !DECL_GIMPLE_REG_P (result)
3401 && DECL_GIMPLE_REG_P (base_m))
3402 use_it = false;
3403 else if (!TREE_ADDRESSABLE (base_m))
3404 use_it = true;
3405 }
3406
3407 if (use_it)
3408 {
3409 var = modify_dest;
3410 use = NULL;
3411 goto done;
3412 }
3413 }
3414
3415 gcc_assert (TREE_CODE (TYPE_SIZE_UNIT (callee_type)) == INTEGER_CST);
3416
3417 var = copy_result_decl_to_var (result, id);
3418 DECL_SEEN_IN_BIND_EXPR_P (var) = 1;
3419
3420 /* Do not have the rest of GCC warn about this variable as it should
3421 not be visible to the user. */
3422 TREE_NO_WARNING (var) = 1;
3423
3424 declare_inline_vars (id->block, var);
3425
3426 /* Build the use expr. If the return type of the function was
3427 promoted, convert it back to the expected type. */
3428 use = var;
3429 if (!useless_type_conversion_p (caller_type, TREE_TYPE (var)))
3430 {
3431 /* If we can match up types by promotion/demotion do so. */
3432 if (fold_convertible_p (caller_type, var))
3433 use = fold_convert (caller_type, var);
3434 else
3435 {
3436 /* ??? For valid programs we should not end up here.
3437 Still if we end up with truly mismatched types here, fall back
3438 to using a MEM_REF to not leak invalid GIMPLE to the following
3439 passes. */
3440 /* Prevent var from being written into SSA form. */
3441 if (TREE_CODE (TREE_TYPE (var)) == VECTOR_TYPE
3442 || TREE_CODE (TREE_TYPE (var)) == COMPLEX_TYPE)
3443 DECL_GIMPLE_REG_P (var) = false;
3444 else if (is_gimple_reg_type (TREE_TYPE (var)))
3445 TREE_ADDRESSABLE (var) = true;
3446 use = fold_build2 (MEM_REF, caller_type,
3447 build_fold_addr_expr (var),
3448 build_int_cst (ptr_type_node, 0));
3449 }
3450 }
3451
3452 STRIP_USELESS_TYPE_CONVERSION (use);
3453
3454 if (DECL_BY_REFERENCE (result))
3455 {
3456 TREE_ADDRESSABLE (var) = 1;
3457 var = build_fold_addr_expr (var);
3458 }
3459
3460 done:
3461 /* Register the VAR_DECL as the equivalent for the RESULT_DECL; that
3462 way, when the RESULT_DECL is encountered, it will be
3463 automatically replaced by the VAR_DECL.
3464
3465 When returning by reference, ensure that RESULT_DECL remaps to
3466 gimple_val. */
3467 if (DECL_BY_REFERENCE (result)
3468 && !is_gimple_val (var))
3469 {
3470 tree temp = create_tmp_var (TREE_TYPE (result), "retvalptr");
3471 insert_decl_map (id, result, temp);
3472 /* When RESULT_DECL is in SSA form, we need to remap and initialize
3473 it's default_def SSA_NAME. */
3474 if (gimple_in_ssa_p (id->src_cfun)
3475 && is_gimple_reg (result))
3476 {
3477 temp = make_ssa_name (temp);
3478 insert_decl_map (id, ssa_default_def (id->src_cfun, result), temp);
3479 }
3480 insert_init_stmt (id, entry_bb, gimple_build_assign (temp, var));
3481 }
3482 else
3483 insert_decl_map (id, result, var);
3484
3485 /* Remember this so we can ignore it in remap_decls. */
3486 id->retvar = var;
3487
3488 /* If returned bounds are used, then make var for them. */
3489 if (return_bounds)
3490 {
3491 tree bndtemp = create_tmp_var (pointer_bounds_type_node, "retbnd");
3492 DECL_SEEN_IN_BIND_EXPR_P (bndtemp) = 1;
3493 TREE_NO_WARNING (bndtemp) = 1;
3494 declare_inline_vars (id->block, bndtemp);
3495
3496 id->retbnd = bndtemp;
3497 insert_init_stmt (id, entry_bb,
3498 gimple_build_assign (bndtemp, chkp_get_zero_bounds_var ()));
3499 }
3500
3501 return use;
3502 }
3503
3504 /* Callback through walk_tree. Determine if a DECL_INITIAL makes reference
3505 to a local label. */
3506
3507 static tree
3508 has_label_address_in_static_1 (tree *nodep, int *walk_subtrees, void *fnp)
3509 {
3510 tree node = *nodep;
3511 tree fn = (tree) fnp;
3512
3513 if (TREE_CODE (node) == LABEL_DECL && DECL_CONTEXT (node) == fn)
3514 return node;
3515
3516 if (TYPE_P (node))
3517 *walk_subtrees = 0;
3518
3519 return NULL_TREE;
3520 }
3521
3522 /* Determine if the function can be copied. If so return NULL. If
3523 not return a string describng the reason for failure. */
3524
3525 static const char *
3526 copy_forbidden (struct function *fun, tree fndecl)
3527 {
3528 const char *reason = fun->cannot_be_copied_reason;
3529 tree decl;
3530 unsigned ix;
3531
3532 /* Only examine the function once. */
3533 if (fun->cannot_be_copied_set)
3534 return reason;
3535
3536 /* We cannot copy a function that receives a non-local goto
3537 because we cannot remap the destination label used in the
3538 function that is performing the non-local goto. */
3539 /* ??? Actually, this should be possible, if we work at it.
3540 No doubt there's just a handful of places that simply
3541 assume it doesn't happen and don't substitute properly. */
3542 if (fun->has_nonlocal_label)
3543 {
3544 reason = G_("function %q+F can never be copied "
3545 "because it receives a non-local goto");
3546 goto fail;
3547 }
3548
3549 FOR_EACH_LOCAL_DECL (fun, ix, decl)
3550 if (TREE_CODE (decl) == VAR_DECL
3551 && TREE_STATIC (decl)
3552 && !DECL_EXTERNAL (decl)
3553 && DECL_INITIAL (decl)
3554 && walk_tree_without_duplicates (&DECL_INITIAL (decl),
3555 has_label_address_in_static_1,
3556 fndecl))
3557 {
3558 reason = G_("function %q+F can never be copied because it saves "
3559 "address of local label in a static variable");
3560 goto fail;
3561 }
3562
3563 fail:
3564 fun->cannot_be_copied_reason = reason;
3565 fun->cannot_be_copied_set = true;
3566 return reason;
3567 }
3568
3569
3570 static const char *inline_forbidden_reason;
3571
3572 /* A callback for walk_gimple_seq to handle statements. Returns non-null
3573 iff a function can not be inlined. Also sets the reason why. */
3574
3575 static tree
3576 inline_forbidden_p_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p,
3577 struct walk_stmt_info *wip)
3578 {
3579 tree fn = (tree) wip->info;
3580 tree t;
3581 gimple stmt = gsi_stmt (*gsi);
3582
3583 switch (gimple_code (stmt))
3584 {
3585 case GIMPLE_CALL:
3586 /* Refuse to inline alloca call unless user explicitly forced so as
3587 this may change program's memory overhead drastically when the
3588 function using alloca is called in loop. In GCC present in
3589 SPEC2000 inlining into schedule_block cause it to require 2GB of
3590 RAM instead of 256MB. Don't do so for alloca calls emitted for
3591 VLA objects as those can't cause unbounded growth (they're always
3592 wrapped inside stack_save/stack_restore regions. */
3593 if (gimple_alloca_call_p (stmt)
3594 && !gimple_call_alloca_for_var_p (as_a <gcall *> (stmt))
3595 && !lookup_attribute ("always_inline", DECL_ATTRIBUTES (fn)))
3596 {
3597 inline_forbidden_reason
3598 = G_("function %q+F can never be inlined because it uses "
3599 "alloca (override using the always_inline attribute)");
3600 *handled_ops_p = true;
3601 return fn;
3602 }
3603
3604 t = gimple_call_fndecl (stmt);
3605 if (t == NULL_TREE)
3606 break;
3607
3608 /* We cannot inline functions that call setjmp. */
3609 if (setjmp_call_p (t))
3610 {
3611 inline_forbidden_reason
3612 = G_("function %q+F can never be inlined because it uses setjmp");
3613 *handled_ops_p = true;
3614 return t;
3615 }
3616
3617 if (DECL_BUILT_IN_CLASS (t) == BUILT_IN_NORMAL)
3618 switch (DECL_FUNCTION_CODE (t))
3619 {
3620 /* We cannot inline functions that take a variable number of
3621 arguments. */
3622 case BUILT_IN_VA_START:
3623 case BUILT_IN_NEXT_ARG:
3624 case BUILT_IN_VA_END:
3625 inline_forbidden_reason
3626 = G_("function %q+F can never be inlined because it "
3627 "uses variable argument lists");
3628 *handled_ops_p = true;
3629 return t;
3630
3631 case BUILT_IN_LONGJMP:
3632 /* We can't inline functions that call __builtin_longjmp at
3633 all. The non-local goto machinery really requires the
3634 destination be in a different function. If we allow the
3635 function calling __builtin_longjmp to be inlined into the
3636 function calling __builtin_setjmp, Things will Go Awry. */
3637 inline_forbidden_reason
3638 = G_("function %q+F can never be inlined because "
3639 "it uses setjmp-longjmp exception handling");
3640 *handled_ops_p = true;
3641 return t;
3642
3643 case BUILT_IN_NONLOCAL_GOTO:
3644 /* Similarly. */
3645 inline_forbidden_reason
3646 = G_("function %q+F can never be inlined because "
3647 "it uses non-local goto");
3648 *handled_ops_p = true;
3649 return t;
3650
3651 case BUILT_IN_RETURN:
3652 case BUILT_IN_APPLY_ARGS:
3653 /* If a __builtin_apply_args caller would be inlined,
3654 it would be saving arguments of the function it has
3655 been inlined into. Similarly __builtin_return would
3656 return from the function the inline has been inlined into. */
3657 inline_forbidden_reason
3658 = G_("function %q+F can never be inlined because "
3659 "it uses __builtin_return or __builtin_apply_args");
3660 *handled_ops_p = true;
3661 return t;
3662
3663 default:
3664 break;
3665 }
3666 break;
3667
3668 case GIMPLE_GOTO:
3669 t = gimple_goto_dest (stmt);
3670
3671 /* We will not inline a function which uses computed goto. The
3672 addresses of its local labels, which may be tucked into
3673 global storage, are of course not constant across
3674 instantiations, which causes unexpected behavior. */
3675 if (TREE_CODE (t) != LABEL_DECL)
3676 {
3677 inline_forbidden_reason
3678 = G_("function %q+F can never be inlined "
3679 "because it contains a computed goto");
3680 *handled_ops_p = true;
3681 return t;
3682 }
3683 break;
3684
3685 default:
3686 break;
3687 }
3688
3689 *handled_ops_p = false;
3690 return NULL_TREE;
3691 }
3692
3693 /* Return true if FNDECL is a function that cannot be inlined into
3694 another one. */
3695
3696 static bool
3697 inline_forbidden_p (tree fndecl)
3698 {
3699 struct function *fun = DECL_STRUCT_FUNCTION (fndecl);
3700 struct walk_stmt_info wi;
3701 basic_block bb;
3702 bool forbidden_p = false;
3703
3704 /* First check for shared reasons not to copy the code. */
3705 inline_forbidden_reason = copy_forbidden (fun, fndecl);
3706 if (inline_forbidden_reason != NULL)
3707 return true;
3708
3709 /* Next, walk the statements of the function looking for
3710 constraucts we can't handle, or are non-optimal for inlining. */
3711 hash_set<tree> visited_nodes;
3712 memset (&wi, 0, sizeof (wi));
3713 wi.info = (void *) fndecl;
3714 wi.pset = &visited_nodes;
3715
3716 FOR_EACH_BB_FN (bb, fun)
3717 {
3718 gimple ret;
3719 gimple_seq seq = bb_seq (bb);
3720 ret = walk_gimple_seq (seq, inline_forbidden_p_stmt, NULL, &wi);
3721 forbidden_p = (ret != NULL);
3722 if (forbidden_p)
3723 break;
3724 }
3725
3726 return forbidden_p;
3727 }
3728 \f
3729 /* Return false if the function FNDECL cannot be inlined on account of its
3730 attributes, true otherwise. */
3731 static bool
3732 function_attribute_inlinable_p (const_tree fndecl)
3733 {
3734 if (targetm.attribute_table)
3735 {
3736 const_tree a;
3737
3738 for (a = DECL_ATTRIBUTES (fndecl); a; a = TREE_CHAIN (a))
3739 {
3740 const_tree name = TREE_PURPOSE (a);
3741 int i;
3742
3743 for (i = 0; targetm.attribute_table[i].name != NULL; i++)
3744 if (is_attribute_p (targetm.attribute_table[i].name, name))
3745 return targetm.function_attribute_inlinable_p (fndecl);
3746 }
3747 }
3748
3749 return true;
3750 }
3751
3752 /* Returns nonzero if FN is a function that does not have any
3753 fundamental inline blocking properties. */
3754
3755 bool
3756 tree_inlinable_function_p (tree fn)
3757 {
3758 bool inlinable = true;
3759 bool do_warning;
3760 tree always_inline;
3761
3762 /* If we've already decided this function shouldn't be inlined,
3763 there's no need to check again. */
3764 if (DECL_UNINLINABLE (fn))
3765 return false;
3766
3767 /* We only warn for functions declared `inline' by the user. */
3768 do_warning = (warn_inline
3769 && DECL_DECLARED_INLINE_P (fn)
3770 && !DECL_NO_INLINE_WARNING_P (fn)
3771 && !DECL_IN_SYSTEM_HEADER (fn));
3772
3773 always_inline = lookup_attribute ("always_inline", DECL_ATTRIBUTES (fn));
3774
3775 if (flag_no_inline
3776 && always_inline == NULL)
3777 {
3778 if (do_warning)
3779 warning (OPT_Winline, "function %q+F can never be inlined because it "
3780 "is suppressed using -fno-inline", fn);
3781 inlinable = false;
3782 }
3783
3784 else if (!function_attribute_inlinable_p (fn))
3785 {
3786 if (do_warning)
3787 warning (OPT_Winline, "function %q+F can never be inlined because it "
3788 "uses attributes conflicting with inlining", fn);
3789 inlinable = false;
3790 }
3791
3792 else if (inline_forbidden_p (fn))
3793 {
3794 /* See if we should warn about uninlinable functions. Previously,
3795 some of these warnings would be issued while trying to expand
3796 the function inline, but that would cause multiple warnings
3797 about functions that would for example call alloca. But since
3798 this a property of the function, just one warning is enough.
3799 As a bonus we can now give more details about the reason why a
3800 function is not inlinable. */
3801 if (always_inline)
3802 error (inline_forbidden_reason, fn);
3803 else if (do_warning)
3804 warning (OPT_Winline, inline_forbidden_reason, fn);
3805
3806 inlinable = false;
3807 }
3808
3809 /* Squirrel away the result so that we don't have to check again. */
3810 DECL_UNINLINABLE (fn) = !inlinable;
3811
3812 return inlinable;
3813 }
3814
3815 /* Estimate the cost of a memory move of type TYPE. Use machine dependent
3816 word size and take possible memcpy call into account and return
3817 cost based on whether optimizing for size or speed according to SPEED_P. */
3818
3819 int
3820 estimate_move_cost (tree type, bool ARG_UNUSED (speed_p))
3821 {
3822 HOST_WIDE_INT size;
3823
3824 gcc_assert (!VOID_TYPE_P (type));
3825
3826 if (TREE_CODE (type) == VECTOR_TYPE)
3827 {
3828 machine_mode inner = TYPE_MODE (TREE_TYPE (type));
3829 machine_mode simd
3830 = targetm.vectorize.preferred_simd_mode (inner);
3831 int simd_mode_size = GET_MODE_SIZE (simd);
3832 return ((GET_MODE_SIZE (TYPE_MODE (type)) + simd_mode_size - 1)
3833 / simd_mode_size);
3834 }
3835
3836 size = int_size_in_bytes (type);
3837
3838 if (size < 0 || size > MOVE_MAX_PIECES * MOVE_RATIO (speed_p))
3839 /* Cost of a memcpy call, 3 arguments and the call. */
3840 return 4;
3841 else
3842 return ((size + MOVE_MAX_PIECES - 1) / MOVE_MAX_PIECES);
3843 }
3844
3845 /* Returns cost of operation CODE, according to WEIGHTS */
3846
3847 static int
3848 estimate_operator_cost (enum tree_code code, eni_weights *weights,
3849 tree op1 ATTRIBUTE_UNUSED, tree op2)
3850 {
3851 switch (code)
3852 {
3853 /* These are "free" conversions, or their presumed cost
3854 is folded into other operations. */
3855 case RANGE_EXPR:
3856 CASE_CONVERT:
3857 case COMPLEX_EXPR:
3858 case PAREN_EXPR:
3859 case VIEW_CONVERT_EXPR:
3860 return 0;
3861
3862 /* Assign cost of 1 to usual operations.
3863 ??? We may consider mapping RTL costs to this. */
3864 case COND_EXPR:
3865 case VEC_COND_EXPR:
3866 case VEC_PERM_EXPR:
3867
3868 case PLUS_EXPR:
3869 case POINTER_PLUS_EXPR:
3870 case MINUS_EXPR:
3871 case MULT_EXPR:
3872 case MULT_HIGHPART_EXPR:
3873 case FMA_EXPR:
3874
3875 case ADDR_SPACE_CONVERT_EXPR:
3876 case FIXED_CONVERT_EXPR:
3877 case FIX_TRUNC_EXPR:
3878
3879 case NEGATE_EXPR:
3880 case FLOAT_EXPR:
3881 case MIN_EXPR:
3882 case MAX_EXPR:
3883 case ABS_EXPR:
3884
3885 case LSHIFT_EXPR:
3886 case RSHIFT_EXPR:
3887 case LROTATE_EXPR:
3888 case RROTATE_EXPR:
3889
3890 case BIT_IOR_EXPR:
3891 case BIT_XOR_EXPR:
3892 case BIT_AND_EXPR:
3893 case BIT_NOT_EXPR:
3894
3895 case TRUTH_ANDIF_EXPR:
3896 case TRUTH_ORIF_EXPR:
3897 case TRUTH_AND_EXPR:
3898 case TRUTH_OR_EXPR:
3899 case TRUTH_XOR_EXPR:
3900 case TRUTH_NOT_EXPR:
3901
3902 case LT_EXPR:
3903 case LE_EXPR:
3904 case GT_EXPR:
3905 case GE_EXPR:
3906 case EQ_EXPR:
3907 case NE_EXPR:
3908 case ORDERED_EXPR:
3909 case UNORDERED_EXPR:
3910
3911 case UNLT_EXPR:
3912 case UNLE_EXPR:
3913 case UNGT_EXPR:
3914 case UNGE_EXPR:
3915 case UNEQ_EXPR:
3916 case LTGT_EXPR:
3917
3918 case CONJ_EXPR:
3919
3920 case PREDECREMENT_EXPR:
3921 case PREINCREMENT_EXPR:
3922 case POSTDECREMENT_EXPR:
3923 case POSTINCREMENT_EXPR:
3924
3925 case REALIGN_LOAD_EXPR:
3926
3927 case REDUC_MAX_EXPR:
3928 case REDUC_MIN_EXPR:
3929 case REDUC_PLUS_EXPR:
3930 case WIDEN_SUM_EXPR:
3931 case WIDEN_MULT_EXPR:
3932 case DOT_PROD_EXPR:
3933 case SAD_EXPR:
3934 case WIDEN_MULT_PLUS_EXPR:
3935 case WIDEN_MULT_MINUS_EXPR:
3936 case WIDEN_LSHIFT_EXPR:
3937
3938 case VEC_WIDEN_MULT_HI_EXPR:
3939 case VEC_WIDEN_MULT_LO_EXPR:
3940 case VEC_WIDEN_MULT_EVEN_EXPR:
3941 case VEC_WIDEN_MULT_ODD_EXPR:
3942 case VEC_UNPACK_HI_EXPR:
3943 case VEC_UNPACK_LO_EXPR:
3944 case VEC_UNPACK_FLOAT_HI_EXPR:
3945 case VEC_UNPACK_FLOAT_LO_EXPR:
3946 case VEC_PACK_TRUNC_EXPR:
3947 case VEC_PACK_SAT_EXPR:
3948 case VEC_PACK_FIX_TRUNC_EXPR:
3949 case VEC_WIDEN_LSHIFT_HI_EXPR:
3950 case VEC_WIDEN_LSHIFT_LO_EXPR:
3951
3952 return 1;
3953
3954 /* Few special cases of expensive operations. This is useful
3955 to avoid inlining on functions having too many of these. */
3956 case TRUNC_DIV_EXPR:
3957 case CEIL_DIV_EXPR:
3958 case FLOOR_DIV_EXPR:
3959 case ROUND_DIV_EXPR:
3960 case EXACT_DIV_EXPR:
3961 case TRUNC_MOD_EXPR:
3962 case CEIL_MOD_EXPR:
3963 case FLOOR_MOD_EXPR:
3964 case ROUND_MOD_EXPR:
3965 case RDIV_EXPR:
3966 if (TREE_CODE (op2) != INTEGER_CST)
3967 return weights->div_mod_cost;
3968 return 1;
3969
3970 default:
3971 /* We expect a copy assignment with no operator. */
3972 gcc_assert (get_gimple_rhs_class (code) == GIMPLE_SINGLE_RHS);
3973 return 0;
3974 }
3975 }
3976
3977
3978 /* Estimate number of instructions that will be created by expanding
3979 the statements in the statement sequence STMTS.
3980 WEIGHTS contains weights attributed to various constructs. */
3981
3982 static
3983 int estimate_num_insns_seq (gimple_seq stmts, eni_weights *weights)
3984 {
3985 int cost;
3986 gimple_stmt_iterator gsi;
3987
3988 cost = 0;
3989 for (gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
3990 cost += estimate_num_insns (gsi_stmt (gsi), weights);
3991
3992 return cost;
3993 }
3994
3995
3996 /* Estimate number of instructions that will be created by expanding STMT.
3997 WEIGHTS contains weights attributed to various constructs. */
3998
3999 int
4000 estimate_num_insns (gimple stmt, eni_weights *weights)
4001 {
4002 unsigned cost, i;
4003 enum gimple_code code = gimple_code (stmt);
4004 tree lhs;
4005 tree rhs;
4006
4007 switch (code)
4008 {
4009 case GIMPLE_ASSIGN:
4010 /* Try to estimate the cost of assignments. We have three cases to
4011 deal with:
4012 1) Simple assignments to registers;
4013 2) Stores to things that must live in memory. This includes
4014 "normal" stores to scalars, but also assignments of large
4015 structures, or constructors of big arrays;
4016
4017 Let us look at the first two cases, assuming we have "a = b + C":
4018 <GIMPLE_ASSIGN <var_decl "a">
4019 <plus_expr <var_decl "b"> <constant C>>
4020 If "a" is a GIMPLE register, the assignment to it is free on almost
4021 any target, because "a" usually ends up in a real register. Hence
4022 the only cost of this expression comes from the PLUS_EXPR, and we
4023 can ignore the GIMPLE_ASSIGN.
4024 If "a" is not a GIMPLE register, the assignment to "a" will most
4025 likely be a real store, so the cost of the GIMPLE_ASSIGN is the cost
4026 of moving something into "a", which we compute using the function
4027 estimate_move_cost. */
4028 if (gimple_clobber_p (stmt))
4029 return 0; /* ={v} {CLOBBER} stmt expands to nothing. */
4030
4031 lhs = gimple_assign_lhs (stmt);
4032 rhs = gimple_assign_rhs1 (stmt);
4033
4034 cost = 0;
4035
4036 /* Account for the cost of moving to / from memory. */
4037 if (gimple_store_p (stmt))
4038 cost += estimate_move_cost (TREE_TYPE (lhs), weights->time_based);
4039 if (gimple_assign_load_p (stmt))
4040 cost += estimate_move_cost (TREE_TYPE (rhs), weights->time_based);
4041
4042 cost += estimate_operator_cost (gimple_assign_rhs_code (stmt), weights,
4043 gimple_assign_rhs1 (stmt),
4044 get_gimple_rhs_class (gimple_assign_rhs_code (stmt))
4045 == GIMPLE_BINARY_RHS
4046 ? gimple_assign_rhs2 (stmt) : NULL);
4047 break;
4048
4049 case GIMPLE_COND:
4050 cost = 1 + estimate_operator_cost (gimple_cond_code (stmt), weights,
4051 gimple_op (stmt, 0),
4052 gimple_op (stmt, 1));
4053 break;
4054
4055 case GIMPLE_SWITCH:
4056 {
4057 gswitch *switch_stmt = as_a <gswitch *> (stmt);
4058 /* Take into account cost of the switch + guess 2 conditional jumps for
4059 each case label.
4060
4061 TODO: once the switch expansion logic is sufficiently separated, we can
4062 do better job on estimating cost of the switch. */
4063 if (weights->time_based)
4064 cost = floor_log2 (gimple_switch_num_labels (switch_stmt)) * 2;
4065 else
4066 cost = gimple_switch_num_labels (switch_stmt) * 2;
4067 }
4068 break;
4069
4070 case GIMPLE_CALL:
4071 {
4072 tree decl;
4073
4074 if (gimple_call_internal_p (stmt))
4075 return 0;
4076 else if ((decl = gimple_call_fndecl (stmt))
4077 && DECL_BUILT_IN (decl))
4078 {
4079 /* Do not special case builtins where we see the body.
4080 This just confuse inliner. */
4081 struct cgraph_node *node;
4082 if (!(node = cgraph_node::get (decl))
4083 || node->definition)
4084 ;
4085 /* For buitins that are likely expanded to nothing or
4086 inlined do not account operand costs. */
4087 else if (is_simple_builtin (decl))
4088 return 0;
4089 else if (is_inexpensive_builtin (decl))
4090 return weights->target_builtin_call_cost;
4091 else if (DECL_BUILT_IN_CLASS (decl) == BUILT_IN_NORMAL)
4092 {
4093 /* We canonicalize x * x to pow (x, 2.0) with -ffast-math, so
4094 specialize the cheap expansion we do here.
4095 ??? This asks for a more general solution. */
4096 switch (DECL_FUNCTION_CODE (decl))
4097 {
4098 case BUILT_IN_POW:
4099 case BUILT_IN_POWF:
4100 case BUILT_IN_POWL:
4101 if (TREE_CODE (gimple_call_arg (stmt, 1)) == REAL_CST
4102 && REAL_VALUES_EQUAL
4103 (TREE_REAL_CST (gimple_call_arg (stmt, 1)), dconst2))
4104 return estimate_operator_cost
4105 (MULT_EXPR, weights, gimple_call_arg (stmt, 0),
4106 gimple_call_arg (stmt, 0));
4107 break;
4108
4109 default:
4110 break;
4111 }
4112 }
4113 }
4114
4115 cost = decl ? weights->call_cost : weights->indirect_call_cost;
4116 if (gimple_call_lhs (stmt))
4117 cost += estimate_move_cost (TREE_TYPE (gimple_call_lhs (stmt)),
4118 weights->time_based);
4119 for (i = 0; i < gimple_call_num_args (stmt); i++)
4120 {
4121 tree arg = gimple_call_arg (stmt, i);
4122 cost += estimate_move_cost (TREE_TYPE (arg),
4123 weights->time_based);
4124 }
4125 break;
4126 }
4127
4128 case GIMPLE_RETURN:
4129 return weights->return_cost;
4130
4131 case GIMPLE_GOTO:
4132 case GIMPLE_LABEL:
4133 case GIMPLE_NOP:
4134 case GIMPLE_PHI:
4135 case GIMPLE_PREDICT:
4136 case GIMPLE_DEBUG:
4137 return 0;
4138
4139 case GIMPLE_ASM:
4140 {
4141 int count = asm_str_count (gimple_asm_string (as_a <gasm *> (stmt)));
4142 /* 1000 means infinity. This avoids overflows later
4143 with very long asm statements. */
4144 if (count > 1000)
4145 count = 1000;
4146 return count;
4147 }
4148
4149 case GIMPLE_RESX:
4150 /* This is either going to be an external function call with one
4151 argument, or two register copy statements plus a goto. */
4152 return 2;
4153
4154 case GIMPLE_EH_DISPATCH:
4155 /* ??? This is going to turn into a switch statement. Ideally
4156 we'd have a look at the eh region and estimate the number of
4157 edges involved. */
4158 return 10;
4159
4160 case GIMPLE_BIND:
4161 return estimate_num_insns_seq (
4162 gimple_bind_body (as_a <gbind *> (stmt)),
4163 weights);
4164
4165 case GIMPLE_EH_FILTER:
4166 return estimate_num_insns_seq (gimple_eh_filter_failure (stmt), weights);
4167
4168 case GIMPLE_CATCH:
4169 return estimate_num_insns_seq (gimple_catch_handler (
4170 as_a <gcatch *> (stmt)),
4171 weights);
4172
4173 case GIMPLE_TRY:
4174 return (estimate_num_insns_seq (gimple_try_eval (stmt), weights)
4175 + estimate_num_insns_seq (gimple_try_cleanup (stmt), weights));
4176
4177 /* OpenMP directives are generally very expensive. */
4178
4179 case GIMPLE_OMP_RETURN:
4180 case GIMPLE_OMP_SECTIONS_SWITCH:
4181 case GIMPLE_OMP_ATOMIC_STORE:
4182 case GIMPLE_OMP_CONTINUE:
4183 /* ...except these, which are cheap. */
4184 return 0;
4185
4186 case GIMPLE_OMP_ATOMIC_LOAD:
4187 return weights->omp_cost;
4188
4189 case GIMPLE_OMP_FOR:
4190 return (weights->omp_cost
4191 + estimate_num_insns_seq (gimple_omp_body (stmt), weights)
4192 + estimate_num_insns_seq (gimple_omp_for_pre_body (stmt), weights));
4193
4194 case GIMPLE_OMP_PARALLEL:
4195 case GIMPLE_OMP_TASK:
4196 case GIMPLE_OMP_CRITICAL:
4197 case GIMPLE_OMP_MASTER:
4198 case GIMPLE_OMP_TASKGROUP:
4199 case GIMPLE_OMP_ORDERED:
4200 case GIMPLE_OMP_SECTION:
4201 case GIMPLE_OMP_SECTIONS:
4202 case GIMPLE_OMP_SINGLE:
4203 case GIMPLE_OMP_TARGET:
4204 case GIMPLE_OMP_TEAMS:
4205 return (weights->omp_cost
4206 + estimate_num_insns_seq (gimple_omp_body (stmt), weights));
4207
4208 case GIMPLE_TRANSACTION:
4209 return (weights->tm_cost
4210 + estimate_num_insns_seq (gimple_transaction_body (
4211 as_a <gtransaction *> (stmt)),
4212 weights));
4213
4214 default:
4215 gcc_unreachable ();
4216 }
4217
4218 return cost;
4219 }
4220
4221 /* Estimate number of instructions that will be created by expanding
4222 function FNDECL. WEIGHTS contains weights attributed to various
4223 constructs. */
4224
4225 int
4226 estimate_num_insns_fn (tree fndecl, eni_weights *weights)
4227 {
4228 struct function *my_function = DECL_STRUCT_FUNCTION (fndecl);
4229 gimple_stmt_iterator bsi;
4230 basic_block bb;
4231 int n = 0;
4232
4233 gcc_assert (my_function && my_function->cfg);
4234 FOR_EACH_BB_FN (bb, my_function)
4235 {
4236 for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
4237 n += estimate_num_insns (gsi_stmt (bsi), weights);
4238 }
4239
4240 return n;
4241 }
4242
4243
4244 /* Initializes weights used by estimate_num_insns. */
4245
4246 void
4247 init_inline_once (void)
4248 {
4249 eni_size_weights.call_cost = 1;
4250 eni_size_weights.indirect_call_cost = 3;
4251 eni_size_weights.target_builtin_call_cost = 1;
4252 eni_size_weights.div_mod_cost = 1;
4253 eni_size_weights.omp_cost = 40;
4254 eni_size_weights.tm_cost = 10;
4255 eni_size_weights.time_based = false;
4256 eni_size_weights.return_cost = 1;
4257
4258 /* Estimating time for call is difficult, since we have no idea what the
4259 called function does. In the current uses of eni_time_weights,
4260 underestimating the cost does less harm than overestimating it, so
4261 we choose a rather small value here. */
4262 eni_time_weights.call_cost = 10;
4263 eni_time_weights.indirect_call_cost = 15;
4264 eni_time_weights.target_builtin_call_cost = 1;
4265 eni_time_weights.div_mod_cost = 10;
4266 eni_time_weights.omp_cost = 40;
4267 eni_time_weights.tm_cost = 40;
4268 eni_time_weights.time_based = true;
4269 eni_time_weights.return_cost = 2;
4270 }
4271
4272 /* Estimate the number of instructions in a gimple_seq. */
4273
4274 int
4275 count_insns_seq (gimple_seq seq, eni_weights *weights)
4276 {
4277 gimple_stmt_iterator gsi;
4278 int n = 0;
4279 for (gsi = gsi_start (seq); !gsi_end_p (gsi); gsi_next (&gsi))
4280 n += estimate_num_insns (gsi_stmt (gsi), weights);
4281
4282 return n;
4283 }
4284
4285
4286 /* Install new lexical TREE_BLOCK underneath 'current_block'. */
4287
4288 static void
4289 prepend_lexical_block (tree current_block, tree new_block)
4290 {
4291 BLOCK_CHAIN (new_block) = BLOCK_SUBBLOCKS (current_block);
4292 BLOCK_SUBBLOCKS (current_block) = new_block;
4293 BLOCK_SUPERCONTEXT (new_block) = current_block;
4294 }
4295
4296 /* Add local variables from CALLEE to CALLER. */
4297
4298 static inline void
4299 add_local_variables (struct function *callee, struct function *caller,
4300 copy_body_data *id)
4301 {
4302 tree var;
4303 unsigned ix;
4304
4305 FOR_EACH_LOCAL_DECL (callee, ix, var)
4306 if (!can_be_nonlocal (var, id))
4307 {
4308 tree new_var = remap_decl (var, id);
4309
4310 /* Remap debug-expressions. */
4311 if (TREE_CODE (new_var) == VAR_DECL
4312 && DECL_HAS_DEBUG_EXPR_P (var)
4313 && new_var != var)
4314 {
4315 tree tem = DECL_DEBUG_EXPR (var);
4316 bool old_regimplify = id->regimplify;
4317 id->remapping_type_depth++;
4318 walk_tree (&tem, copy_tree_body_r, id, NULL);
4319 id->remapping_type_depth--;
4320 id->regimplify = old_regimplify;
4321 SET_DECL_DEBUG_EXPR (new_var, tem);
4322 DECL_HAS_DEBUG_EXPR_P (new_var) = 1;
4323 }
4324 add_local_decl (caller, new_var);
4325 }
4326 }
4327
4328 /* If STMT is a GIMPLE_CALL, replace it with its inline expansion. */
4329
4330 static bool
4331 expand_call_inline (basic_block bb, gimple stmt, copy_body_data *id)
4332 {
4333 tree use_retvar;
4334 tree fn;
4335 hash_map<tree, tree> *dst;
4336 hash_map<tree, tree> *st = NULL;
4337 tree return_slot;
4338 tree modify_dest;
4339 tree return_bounds = NULL;
4340 location_t saved_location;
4341 struct cgraph_edge *cg_edge;
4342 cgraph_inline_failed_t reason;
4343 basic_block return_block;
4344 edge e;
4345 gimple_stmt_iterator gsi, stmt_gsi;
4346 bool successfully_inlined = FALSE;
4347 bool purge_dead_abnormal_edges;
4348 gcall *call_stmt;
4349 unsigned int i;
4350
4351 /* Set input_location here so we get the right instantiation context
4352 if we call instantiate_decl from inlinable_function_p. */
4353 /* FIXME: instantiate_decl isn't called by inlinable_function_p. */
4354 saved_location = input_location;
4355 input_location = gimple_location (stmt);
4356
4357 /* From here on, we're only interested in CALL_EXPRs. */
4358 call_stmt = dyn_cast <gcall *> (stmt);
4359 if (!call_stmt)
4360 goto egress;
4361
4362 cg_edge = id->dst_node->get_edge (stmt);
4363 gcc_checking_assert (cg_edge);
4364 /* First, see if we can figure out what function is being called.
4365 If we cannot, then there is no hope of inlining the function. */
4366 if (cg_edge->indirect_unknown_callee)
4367 goto egress;
4368 fn = cg_edge->callee->decl;
4369 gcc_checking_assert (fn);
4370
4371 /* If FN is a declaration of a function in a nested scope that was
4372 globally declared inline, we don't set its DECL_INITIAL.
4373 However, we can't blindly follow DECL_ABSTRACT_ORIGIN because the
4374 C++ front-end uses it for cdtors to refer to their internal
4375 declarations, that are not real functions. Fortunately those
4376 don't have trees to be saved, so we can tell by checking their
4377 gimple_body. */
4378 if (!DECL_INITIAL (fn)
4379 && DECL_ABSTRACT_ORIGIN (fn)
4380 && gimple_has_body_p (DECL_ABSTRACT_ORIGIN (fn)))
4381 fn = DECL_ABSTRACT_ORIGIN (fn);
4382
4383 /* Don't try to inline functions that are not well-suited to inlining. */
4384 if (cg_edge->inline_failed)
4385 {
4386 reason = cg_edge->inline_failed;
4387 /* If this call was originally indirect, we do not want to emit any
4388 inlining related warnings or sorry messages because there are no
4389 guarantees regarding those. */
4390 if (cg_edge->indirect_inlining_edge)
4391 goto egress;
4392
4393 if (lookup_attribute ("always_inline", DECL_ATTRIBUTES (fn))
4394 /* For extern inline functions that get redefined we always
4395 silently ignored always_inline flag. Better behaviour would
4396 be to be able to keep both bodies and use extern inline body
4397 for inlining, but we can't do that because frontends overwrite
4398 the body. */
4399 && !cg_edge->callee->local.redefined_extern_inline
4400 /* During early inline pass, report only when optimization is
4401 not turned on. */
4402 && (symtab->global_info_ready
4403 || !optimize
4404 || cgraph_inline_failed_type (reason) == CIF_FINAL_ERROR)
4405 /* PR 20090218-1_0.c. Body can be provided by another module. */
4406 && (reason != CIF_BODY_NOT_AVAILABLE || !flag_generate_lto))
4407 {
4408 error ("inlining failed in call to always_inline %q+F: %s", fn,
4409 cgraph_inline_failed_string (reason));
4410 error ("called from here");
4411 }
4412 else if (warn_inline
4413 && DECL_DECLARED_INLINE_P (fn)
4414 && !DECL_NO_INLINE_WARNING_P (fn)
4415 && !DECL_IN_SYSTEM_HEADER (fn)
4416 && reason != CIF_UNSPECIFIED
4417 && !lookup_attribute ("noinline", DECL_ATTRIBUTES (fn))
4418 /* Do not warn about not inlined recursive calls. */
4419 && !cg_edge->recursive_p ()
4420 /* Avoid warnings during early inline pass. */
4421 && symtab->global_info_ready)
4422 {
4423 warning (OPT_Winline, "inlining failed in call to %q+F: %s",
4424 fn, _(cgraph_inline_failed_string (reason)));
4425 warning (OPT_Winline, "called from here");
4426 }
4427 goto egress;
4428 }
4429 fn = cg_edge->callee->decl;
4430 cg_edge->callee->get_untransformed_body ();
4431
4432 #ifdef ENABLE_CHECKING
4433 if (cg_edge->callee->decl != id->dst_node->decl)
4434 cg_edge->callee->verify ();
4435 #endif
4436
4437 /* We will be inlining this callee. */
4438 id->eh_lp_nr = lookup_stmt_eh_lp (stmt);
4439 id->assign_stmts.create (0);
4440
4441 /* Update the callers EH personality. */
4442 if (DECL_FUNCTION_PERSONALITY (cg_edge->callee->decl))
4443 DECL_FUNCTION_PERSONALITY (cg_edge->caller->decl)
4444 = DECL_FUNCTION_PERSONALITY (cg_edge->callee->decl);
4445
4446 /* Split the block holding the GIMPLE_CALL. */
4447 e = split_block (bb, stmt);
4448 bb = e->src;
4449 return_block = e->dest;
4450 remove_edge (e);
4451
4452 /* split_block splits after the statement; work around this by
4453 moving the call into the second block manually. Not pretty,
4454 but seems easier than doing the CFG manipulation by hand
4455 when the GIMPLE_CALL is in the last statement of BB. */
4456 stmt_gsi = gsi_last_bb (bb);
4457 gsi_remove (&stmt_gsi, false);
4458
4459 /* If the GIMPLE_CALL was in the last statement of BB, it may have
4460 been the source of abnormal edges. In this case, schedule
4461 the removal of dead abnormal edges. */
4462 gsi = gsi_start_bb (return_block);
4463 if (gsi_end_p (gsi))
4464 {
4465 gsi_insert_after (&gsi, stmt, GSI_NEW_STMT);
4466 purge_dead_abnormal_edges = true;
4467 }
4468 else
4469 {
4470 gsi_insert_before (&gsi, stmt, GSI_NEW_STMT);
4471 purge_dead_abnormal_edges = false;
4472 }
4473
4474 stmt_gsi = gsi_start_bb (return_block);
4475
4476 /* Build a block containing code to initialize the arguments, the
4477 actual inline expansion of the body, and a label for the return
4478 statements within the function to jump to. The type of the
4479 statement expression is the return type of the function call.
4480 ??? If the call does not have an associated block then we will
4481 remap all callee blocks to NULL, effectively dropping most of
4482 its debug information. This should only happen for calls to
4483 artificial decls inserted by the compiler itself. We need to
4484 either link the inlined blocks into the caller block tree or
4485 not refer to them in any way to not break GC for locations. */
4486 if (gimple_block (stmt))
4487 {
4488 id->block = make_node (BLOCK);
4489 BLOCK_ABSTRACT_ORIGIN (id->block) = fn;
4490 BLOCK_SOURCE_LOCATION (id->block) = LOCATION_LOCUS (input_location);
4491 prepend_lexical_block (gimple_block (stmt), id->block);
4492 }
4493
4494 /* Local declarations will be replaced by their equivalents in this
4495 map. */
4496 st = id->decl_map;
4497 id->decl_map = new hash_map<tree, tree>;
4498 dst = id->debug_map;
4499 id->debug_map = NULL;
4500
4501 /* Record the function we are about to inline. */
4502 id->src_fn = fn;
4503 id->src_node = cg_edge->callee;
4504 id->src_cfun = DECL_STRUCT_FUNCTION (fn);
4505 id->call_stmt = stmt;
4506
4507 gcc_assert (!id->src_cfun->after_inlining);
4508
4509 id->entry_bb = bb;
4510 if (lookup_attribute ("cold", DECL_ATTRIBUTES (fn)))
4511 {
4512 gimple_stmt_iterator si = gsi_last_bb (bb);
4513 gsi_insert_after (&si, gimple_build_predict (PRED_COLD_FUNCTION,
4514 NOT_TAKEN),
4515 GSI_NEW_STMT);
4516 }
4517 initialize_inlined_parameters (id, stmt, fn, bb);
4518
4519 if (DECL_INITIAL (fn))
4520 {
4521 if (gimple_block (stmt))
4522 {
4523 tree *var;
4524
4525 prepend_lexical_block (id->block,
4526 remap_blocks (DECL_INITIAL (fn), id));
4527 gcc_checking_assert (BLOCK_SUBBLOCKS (id->block)
4528 && (BLOCK_CHAIN (BLOCK_SUBBLOCKS (id->block))
4529 == NULL_TREE));
4530 /* Move vars for PARM_DECLs from DECL_INITIAL block to id->block,
4531 otherwise for DWARF DW_TAG_formal_parameter will not be children of
4532 DW_TAG_inlined_subroutine, but of a DW_TAG_lexical_block
4533 under it. The parameters can be then evaluated in the debugger,
4534 but don't show in backtraces. */
4535 for (var = &BLOCK_VARS (BLOCK_SUBBLOCKS (id->block)); *var; )
4536 if (TREE_CODE (DECL_ORIGIN (*var)) == PARM_DECL)
4537 {
4538 tree v = *var;
4539 *var = TREE_CHAIN (v);
4540 TREE_CHAIN (v) = BLOCK_VARS (id->block);
4541 BLOCK_VARS (id->block) = v;
4542 }
4543 else
4544 var = &TREE_CHAIN (*var);
4545 }
4546 else
4547 remap_blocks_to_null (DECL_INITIAL (fn), id);
4548 }
4549
4550 /* Return statements in the function body will be replaced by jumps
4551 to the RET_LABEL. */
4552 gcc_assert (DECL_INITIAL (fn));
4553 gcc_assert (TREE_CODE (DECL_INITIAL (fn)) == BLOCK);
4554
4555 /* Find the LHS to which the result of this call is assigned. */
4556 return_slot = NULL;
4557 if (gimple_call_lhs (stmt))
4558 {
4559 modify_dest = gimple_call_lhs (stmt);
4560
4561 /* Remember where to copy returned bounds. */
4562 if (gimple_call_with_bounds_p (stmt)
4563 && TREE_CODE (modify_dest) == SSA_NAME)
4564 {
4565 gcall *retbnd = chkp_retbnd_call_by_val (modify_dest);
4566 if (retbnd)
4567 {
4568 return_bounds = gimple_call_lhs (retbnd);
4569 /* If returned bounds are not used then just
4570 remove unused call. */
4571 if (!return_bounds)
4572 {
4573 gimple_stmt_iterator iter = gsi_for_stmt (retbnd);
4574 gsi_remove (&iter, true);
4575 }
4576 }
4577 }
4578
4579 /* The function which we are inlining might not return a value,
4580 in which case we should issue a warning that the function
4581 does not return a value. In that case the optimizers will
4582 see that the variable to which the value is assigned was not
4583 initialized. We do not want to issue a warning about that
4584 uninitialized variable. */
4585 if (DECL_P (modify_dest))
4586 TREE_NO_WARNING (modify_dest) = 1;
4587
4588 if (gimple_call_return_slot_opt_p (call_stmt))
4589 {
4590 return_slot = modify_dest;
4591 modify_dest = NULL;
4592 }
4593 }
4594 else
4595 modify_dest = NULL;
4596
4597 /* If we are inlining a call to the C++ operator new, we don't want
4598 to use type based alias analysis on the return value. Otherwise
4599 we may get confused if the compiler sees that the inlined new
4600 function returns a pointer which was just deleted. See bug
4601 33407. */
4602 if (DECL_IS_OPERATOR_NEW (fn))
4603 {
4604 return_slot = NULL;
4605 modify_dest = NULL;
4606 }
4607
4608 /* Declare the return variable for the function. */
4609 use_retvar = declare_return_variable (id, return_slot, modify_dest,
4610 return_bounds, bb);
4611
4612 /* Add local vars in this inlined callee to caller. */
4613 add_local_variables (id->src_cfun, cfun, id);
4614
4615 if (dump_file && (dump_flags & TDF_DETAILS))
4616 {
4617 fprintf (dump_file, "Inlining ");
4618 print_generic_expr (dump_file, id->src_fn, 0);
4619 fprintf (dump_file, " to ");
4620 print_generic_expr (dump_file, id->dst_fn, 0);
4621 fprintf (dump_file, " with frequency %i\n", cg_edge->frequency);
4622 }
4623
4624 /* This is it. Duplicate the callee body. Assume callee is
4625 pre-gimplified. Note that we must not alter the caller
4626 function in any way before this point, as this CALL_EXPR may be
4627 a self-referential call; if we're calling ourselves, we need to
4628 duplicate our body before altering anything. */
4629 copy_body (id, cg_edge->callee->count,
4630 GCOV_COMPUTE_SCALE (cg_edge->frequency, CGRAPH_FREQ_BASE),
4631 bb, return_block, NULL);
4632
4633 /* Reset the escaped solution. */
4634 if (cfun->gimple_df)
4635 pt_solution_reset (&cfun->gimple_df->escaped);
4636
4637 /* Clean up. */
4638 if (id->debug_map)
4639 {
4640 delete id->debug_map;
4641 id->debug_map = dst;
4642 }
4643 delete id->decl_map;
4644 id->decl_map = st;
4645
4646 /* Unlink the calls virtual operands before replacing it. */
4647 unlink_stmt_vdef (stmt);
4648 if (gimple_vdef (stmt)
4649 && TREE_CODE (gimple_vdef (stmt)) == SSA_NAME)
4650 release_ssa_name (gimple_vdef (stmt));
4651
4652 /* If the inlined function returns a result that we care about,
4653 substitute the GIMPLE_CALL with an assignment of the return
4654 variable to the LHS of the call. That is, if STMT was
4655 'a = foo (...)', substitute the call with 'a = USE_RETVAR'. */
4656 if (use_retvar && gimple_call_lhs (stmt))
4657 {
4658 gimple old_stmt = stmt;
4659 stmt = gimple_build_assign (gimple_call_lhs (stmt), use_retvar);
4660 gsi_replace (&stmt_gsi, stmt, false);
4661 maybe_clean_or_replace_eh_stmt (old_stmt, stmt);
4662
4663 /* Copy bounds if we copy structure with bounds. */
4664 if (chkp_function_instrumented_p (id->dst_fn)
4665 && !BOUNDED_P (use_retvar)
4666 && chkp_type_has_pointer (TREE_TYPE (use_retvar)))
4667 id->assign_stmts.safe_push (stmt);
4668 }
4669 else
4670 {
4671 /* Handle the case of inlining a function with no return
4672 statement, which causes the return value to become undefined. */
4673 if (gimple_call_lhs (stmt)
4674 && TREE_CODE (gimple_call_lhs (stmt)) == SSA_NAME)
4675 {
4676 tree name = gimple_call_lhs (stmt);
4677 tree var = SSA_NAME_VAR (name);
4678 tree def = ssa_default_def (cfun, var);
4679
4680 if (def)
4681 {
4682 /* If the variable is used undefined, make this name
4683 undefined via a move. */
4684 stmt = gimple_build_assign (gimple_call_lhs (stmt), def);
4685 gsi_replace (&stmt_gsi, stmt, true);
4686 }
4687 else
4688 {
4689 /* Otherwise make this variable undefined. */
4690 gsi_remove (&stmt_gsi, true);
4691 set_ssa_default_def (cfun, var, name);
4692 SSA_NAME_DEF_STMT (name) = gimple_build_nop ();
4693 }
4694 }
4695 else
4696 gsi_remove (&stmt_gsi, true);
4697 }
4698
4699 /* Put returned bounds into the correct place if required. */
4700 if (return_bounds)
4701 {
4702 gimple old_stmt = SSA_NAME_DEF_STMT (return_bounds);
4703 gimple new_stmt = gimple_build_assign (return_bounds, id->retbnd);
4704 gimple_stmt_iterator bnd_gsi = gsi_for_stmt (old_stmt);
4705 unlink_stmt_vdef (old_stmt);
4706 gsi_replace (&bnd_gsi, new_stmt, false);
4707 maybe_clean_or_replace_eh_stmt (old_stmt, new_stmt);
4708 cgraph_update_edges_for_call_stmt (old_stmt,
4709 gimple_call_fndecl (old_stmt),
4710 new_stmt);
4711 }
4712
4713 if (purge_dead_abnormal_edges)
4714 {
4715 gimple_purge_dead_eh_edges (return_block);
4716 gimple_purge_dead_abnormal_call_edges (return_block);
4717 }
4718
4719 /* If the value of the new expression is ignored, that's OK. We
4720 don't warn about this for CALL_EXPRs, so we shouldn't warn about
4721 the equivalent inlined version either. */
4722 if (is_gimple_assign (stmt))
4723 {
4724 gcc_assert (gimple_assign_single_p (stmt)
4725 || CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt)));
4726 TREE_USED (gimple_assign_rhs1 (stmt)) = 1;
4727 }
4728
4729 /* Copy bounds for all generated assigns that need it. */
4730 for (i = 0; i < id->assign_stmts.length (); i++)
4731 chkp_copy_bounds_for_assign (id->assign_stmts[i], cg_edge);
4732 id->assign_stmts.release ();
4733
4734 /* Output the inlining info for this abstract function, since it has been
4735 inlined. If we don't do this now, we can lose the information about the
4736 variables in the function when the blocks get blown away as soon as we
4737 remove the cgraph node. */
4738 if (gimple_block (stmt))
4739 (*debug_hooks->outlining_inline_function) (cg_edge->callee->decl);
4740
4741 /* Update callgraph if needed. */
4742 cg_edge->callee->remove ();
4743
4744 id->block = NULL_TREE;
4745 successfully_inlined = TRUE;
4746
4747 egress:
4748 input_location = saved_location;
4749 return successfully_inlined;
4750 }
4751
4752 /* Expand call statements reachable from STMT_P.
4753 We can only have CALL_EXPRs as the "toplevel" tree code or nested
4754 in a MODIFY_EXPR. */
4755
4756 static bool
4757 gimple_expand_calls_inline (basic_block bb, copy_body_data *id)
4758 {
4759 gimple_stmt_iterator gsi;
4760
4761 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
4762 {
4763 gimple stmt = gsi_stmt (gsi);
4764
4765 if (is_gimple_call (stmt)
4766 && !gimple_call_internal_p (stmt)
4767 && expand_call_inline (bb, stmt, id))
4768 return true;
4769 }
4770
4771 return false;
4772 }
4773
4774
4775 /* Walk all basic blocks created after FIRST and try to fold every statement
4776 in the STATEMENTS pointer set. */
4777
4778 static void
4779 fold_marked_statements (int first, hash_set<gimple> *statements)
4780 {
4781 for (; first < n_basic_blocks_for_fn (cfun); first++)
4782 if (BASIC_BLOCK_FOR_FN (cfun, first))
4783 {
4784 gimple_stmt_iterator gsi;
4785
4786 for (gsi = gsi_start_bb (BASIC_BLOCK_FOR_FN (cfun, first));
4787 !gsi_end_p (gsi);
4788 gsi_next (&gsi))
4789 if (statements->contains (gsi_stmt (gsi)))
4790 {
4791 gimple old_stmt = gsi_stmt (gsi);
4792 tree old_decl = is_gimple_call (old_stmt) ? gimple_call_fndecl (old_stmt) : 0;
4793
4794 if (old_decl && DECL_BUILT_IN (old_decl))
4795 {
4796 /* Folding builtins can create multiple instructions,
4797 we need to look at all of them. */
4798 gimple_stmt_iterator i2 = gsi;
4799 gsi_prev (&i2);
4800 if (fold_stmt (&gsi))
4801 {
4802 gimple new_stmt;
4803 /* If a builtin at the end of a bb folded into nothing,
4804 the following loop won't work. */
4805 if (gsi_end_p (gsi))
4806 {
4807 cgraph_update_edges_for_call_stmt (old_stmt,
4808 old_decl, NULL);
4809 break;
4810 }
4811 if (gsi_end_p (i2))
4812 i2 = gsi_start_bb (BASIC_BLOCK_FOR_FN (cfun, first));
4813 else
4814 gsi_next (&i2);
4815 while (1)
4816 {
4817 new_stmt = gsi_stmt (i2);
4818 update_stmt (new_stmt);
4819 cgraph_update_edges_for_call_stmt (old_stmt, old_decl,
4820 new_stmt);
4821
4822 if (new_stmt == gsi_stmt (gsi))
4823 {
4824 /* It is okay to check only for the very last
4825 of these statements. If it is a throwing
4826 statement nothing will change. If it isn't
4827 this can remove EH edges. If that weren't
4828 correct then because some intermediate stmts
4829 throw, but not the last one. That would mean
4830 we'd have to split the block, which we can't
4831 here and we'd loose anyway. And as builtins
4832 probably never throw, this all
4833 is mood anyway. */
4834 if (maybe_clean_or_replace_eh_stmt (old_stmt,
4835 new_stmt))
4836 gimple_purge_dead_eh_edges (
4837 BASIC_BLOCK_FOR_FN (cfun, first));
4838 break;
4839 }
4840 gsi_next (&i2);
4841 }
4842 }
4843 }
4844 else if (fold_stmt (&gsi))
4845 {
4846 /* Re-read the statement from GSI as fold_stmt() may
4847 have changed it. */
4848 gimple new_stmt = gsi_stmt (gsi);
4849 update_stmt (new_stmt);
4850
4851 if (is_gimple_call (old_stmt)
4852 || is_gimple_call (new_stmt))
4853 cgraph_update_edges_for_call_stmt (old_stmt, old_decl,
4854 new_stmt);
4855
4856 if (maybe_clean_or_replace_eh_stmt (old_stmt, new_stmt))
4857 gimple_purge_dead_eh_edges (BASIC_BLOCK_FOR_FN (cfun,
4858 first));
4859 }
4860 }
4861 }
4862 }
4863
4864 /* Expand calls to inline functions in the body of FN. */
4865
4866 unsigned int
4867 optimize_inline_calls (tree fn)
4868 {
4869 copy_body_data id;
4870 basic_block bb;
4871 int last = n_basic_blocks_for_fn (cfun);
4872 bool inlined_p = false;
4873
4874 /* Clear out ID. */
4875 memset (&id, 0, sizeof (id));
4876
4877 id.src_node = id.dst_node = cgraph_node::get (fn);
4878 gcc_assert (id.dst_node->definition);
4879 id.dst_fn = fn;
4880 /* Or any functions that aren't finished yet. */
4881 if (current_function_decl)
4882 id.dst_fn = current_function_decl;
4883
4884 id.copy_decl = copy_decl_maybe_to_var;
4885 id.transform_call_graph_edges = CB_CGE_DUPLICATE;
4886 id.transform_new_cfg = false;
4887 id.transform_return_to_modify = true;
4888 id.transform_parameter = true;
4889 id.transform_lang_insert_block = NULL;
4890 id.statements_to_fold = new hash_set<gimple>;
4891
4892 push_gimplify_context ();
4893
4894 /* We make no attempts to keep dominance info up-to-date. */
4895 free_dominance_info (CDI_DOMINATORS);
4896 free_dominance_info (CDI_POST_DOMINATORS);
4897
4898 /* Register specific gimple functions. */
4899 gimple_register_cfg_hooks ();
4900
4901 /* Reach the trees by walking over the CFG, and note the
4902 enclosing basic-blocks in the call edges. */
4903 /* We walk the blocks going forward, because inlined function bodies
4904 will split id->current_basic_block, and the new blocks will
4905 follow it; we'll trudge through them, processing their CALL_EXPRs
4906 along the way. */
4907 FOR_EACH_BB_FN (bb, cfun)
4908 inlined_p |= gimple_expand_calls_inline (bb, &id);
4909
4910 pop_gimplify_context (NULL);
4911
4912 #ifdef ENABLE_CHECKING
4913 {
4914 struct cgraph_edge *e;
4915
4916 id.dst_node->verify ();
4917
4918 /* Double check that we inlined everything we are supposed to inline. */
4919 for (e = id.dst_node->callees; e; e = e->next_callee)
4920 gcc_assert (e->inline_failed);
4921 }
4922 #endif
4923
4924 /* Fold queued statements. */
4925 fold_marked_statements (last, id.statements_to_fold);
4926 delete id.statements_to_fold;
4927
4928 gcc_assert (!id.debug_stmts.exists ());
4929
4930 /* If we didn't inline into the function there is nothing to do. */
4931 if (!inlined_p)
4932 return 0;
4933
4934 /* Renumber the lexical scoping (non-code) blocks consecutively. */
4935 number_blocks (fn);
4936
4937 delete_unreachable_blocks_update_callgraph (&id);
4938 #ifdef ENABLE_CHECKING
4939 id.dst_node->verify ();
4940 #endif
4941
4942 /* It would be nice to check SSA/CFG/statement consistency here, but it is
4943 not possible yet - the IPA passes might make various functions to not
4944 throw and they don't care to proactively update local EH info. This is
4945 done later in fixup_cfg pass that also execute the verification. */
4946 return (TODO_update_ssa
4947 | TODO_cleanup_cfg
4948 | (gimple_in_ssa_p (cfun) ? TODO_remove_unused_locals : 0)
4949 | (gimple_in_ssa_p (cfun) ? TODO_update_address_taken : 0)
4950 | (profile_status_for_fn (cfun) != PROFILE_ABSENT
4951 ? TODO_rebuild_frequencies : 0));
4952 }
4953
4954 /* Passed to walk_tree. Copies the node pointed to, if appropriate. */
4955
4956 tree
4957 copy_tree_r (tree *tp, int *walk_subtrees, void *data ATTRIBUTE_UNUSED)
4958 {
4959 enum tree_code code = TREE_CODE (*tp);
4960 enum tree_code_class cl = TREE_CODE_CLASS (code);
4961
4962 /* We make copies of most nodes. */
4963 if (IS_EXPR_CODE_CLASS (cl)
4964 || code == TREE_LIST
4965 || code == TREE_VEC
4966 || code == TYPE_DECL
4967 || code == OMP_CLAUSE)
4968 {
4969 /* Because the chain gets clobbered when we make a copy, we save it
4970 here. */
4971 tree chain = NULL_TREE, new_tree;
4972
4973 if (CODE_CONTAINS_STRUCT (code, TS_COMMON))
4974 chain = TREE_CHAIN (*tp);
4975
4976 /* Copy the node. */
4977 new_tree = copy_node (*tp);
4978
4979 *tp = new_tree;
4980
4981 /* Now, restore the chain, if appropriate. That will cause
4982 walk_tree to walk into the chain as well. */
4983 if (code == PARM_DECL
4984 || code == TREE_LIST
4985 || code == OMP_CLAUSE)
4986 TREE_CHAIN (*tp) = chain;
4987
4988 /* For now, we don't update BLOCKs when we make copies. So, we
4989 have to nullify all BIND_EXPRs. */
4990 if (TREE_CODE (*tp) == BIND_EXPR)
4991 BIND_EXPR_BLOCK (*tp) = NULL_TREE;
4992 }
4993 else if (code == CONSTRUCTOR)
4994 {
4995 /* CONSTRUCTOR nodes need special handling because
4996 we need to duplicate the vector of elements. */
4997 tree new_tree;
4998
4999 new_tree = copy_node (*tp);
5000 CONSTRUCTOR_ELTS (new_tree) = vec_safe_copy (CONSTRUCTOR_ELTS (*tp));
5001 *tp = new_tree;
5002 }
5003 else if (code == STATEMENT_LIST)
5004 /* We used to just abort on STATEMENT_LIST, but we can run into them
5005 with statement-expressions (c++/40975). */
5006 copy_statement_list (tp);
5007 else if (TREE_CODE_CLASS (code) == tcc_type)
5008 *walk_subtrees = 0;
5009 else if (TREE_CODE_CLASS (code) == tcc_declaration)
5010 *walk_subtrees = 0;
5011 else if (TREE_CODE_CLASS (code) == tcc_constant)
5012 *walk_subtrees = 0;
5013 return NULL_TREE;
5014 }
5015
5016 /* The SAVE_EXPR pointed to by TP is being copied. If ST contains
5017 information indicating to what new SAVE_EXPR this one should be mapped,
5018 use that one. Otherwise, create a new node and enter it in ST. FN is
5019 the function into which the copy will be placed. */
5020
5021 static void
5022 remap_save_expr (tree *tp, hash_map<tree, tree> *st, int *walk_subtrees)
5023 {
5024 tree *n;
5025 tree t;
5026
5027 /* See if we already encountered this SAVE_EXPR. */
5028 n = st->get (*tp);
5029
5030 /* If we didn't already remap this SAVE_EXPR, do so now. */
5031 if (!n)
5032 {
5033 t = copy_node (*tp);
5034
5035 /* Remember this SAVE_EXPR. */
5036 st->put (*tp, t);
5037 /* Make sure we don't remap an already-remapped SAVE_EXPR. */
5038 st->put (t, t);
5039 }
5040 else
5041 {
5042 /* We've already walked into this SAVE_EXPR; don't do it again. */
5043 *walk_subtrees = 0;
5044 t = *n;
5045 }
5046
5047 /* Replace this SAVE_EXPR with the copy. */
5048 *tp = t;
5049 }
5050
5051 /* Called via walk_gimple_seq. If *GSIP points to a GIMPLE_LABEL for a local
5052 label, copies the declaration and enters it in the splay_tree in DATA (which
5053 is really a 'copy_body_data *'. */
5054
5055 static tree
5056 mark_local_labels_stmt (gimple_stmt_iterator *gsip,
5057 bool *handled_ops_p ATTRIBUTE_UNUSED,
5058 struct walk_stmt_info *wi)
5059 {
5060 copy_body_data *id = (copy_body_data *) wi->info;
5061 glabel *stmt = dyn_cast <glabel *> (gsi_stmt (*gsip));
5062
5063 if (stmt)
5064 {
5065 tree decl = gimple_label_label (stmt);
5066
5067 /* Copy the decl and remember the copy. */
5068 insert_decl_map (id, decl, id->copy_decl (decl, id));
5069 }
5070
5071 return NULL_TREE;
5072 }
5073
5074
5075 /* Called via walk_gimple_seq by copy_gimple_seq_and_replace_local.
5076 Using the splay_tree pointed to by ST (which is really a `splay_tree'),
5077 remaps all local declarations to appropriate replacements in gimple
5078 operands. */
5079
5080 static tree
5081 replace_locals_op (tree *tp, int *walk_subtrees, void *data)
5082 {
5083 struct walk_stmt_info *wi = (struct walk_stmt_info*) data;
5084 copy_body_data *id = (copy_body_data *) wi->info;
5085 hash_map<tree, tree> *st = id->decl_map;
5086 tree *n;
5087 tree expr = *tp;
5088
5089 /* Only a local declaration (variable or label). */
5090 if ((TREE_CODE (expr) == VAR_DECL
5091 && !TREE_STATIC (expr))
5092 || TREE_CODE (expr) == LABEL_DECL)
5093 {
5094 /* Lookup the declaration. */
5095 n = st->get (expr);
5096
5097 /* If it's there, remap it. */
5098 if (n)
5099 *tp = *n;
5100 *walk_subtrees = 0;
5101 }
5102 else if (TREE_CODE (expr) == STATEMENT_LIST
5103 || TREE_CODE (expr) == BIND_EXPR
5104 || TREE_CODE (expr) == SAVE_EXPR)
5105 gcc_unreachable ();
5106 else if (TREE_CODE (expr) == TARGET_EXPR)
5107 {
5108 /* Don't mess with a TARGET_EXPR that hasn't been expanded.
5109 It's OK for this to happen if it was part of a subtree that
5110 isn't immediately expanded, such as operand 2 of another
5111 TARGET_EXPR. */
5112 if (!TREE_OPERAND (expr, 1))
5113 {
5114 TREE_OPERAND (expr, 1) = TREE_OPERAND (expr, 3);
5115 TREE_OPERAND (expr, 3) = NULL_TREE;
5116 }
5117 }
5118
5119 /* Keep iterating. */
5120 return NULL_TREE;
5121 }
5122
5123
5124 /* Called via walk_gimple_seq by copy_gimple_seq_and_replace_local.
5125 Using the splay_tree pointed to by ST (which is really a `splay_tree'),
5126 remaps all local declarations to appropriate replacements in gimple
5127 statements. */
5128
5129 static tree
5130 replace_locals_stmt (gimple_stmt_iterator *gsip,
5131 bool *handled_ops_p ATTRIBUTE_UNUSED,
5132 struct walk_stmt_info *wi)
5133 {
5134 copy_body_data *id = (copy_body_data *) wi->info;
5135 gimple gs = gsi_stmt (*gsip);
5136
5137 if (gbind *stmt = dyn_cast <gbind *> (gs))
5138 {
5139 tree block = gimple_bind_block (stmt);
5140
5141 if (block)
5142 {
5143 remap_block (&block, id);
5144 gimple_bind_set_block (stmt, block);
5145 }
5146
5147 /* This will remap a lot of the same decls again, but this should be
5148 harmless. */
5149 if (gimple_bind_vars (stmt))
5150 gimple_bind_set_vars (stmt, remap_decls (gimple_bind_vars (stmt),
5151 NULL, id));
5152 }
5153
5154 /* Keep iterating. */
5155 return NULL_TREE;
5156 }
5157
5158
5159 /* Copies everything in SEQ and replaces variables and labels local to
5160 current_function_decl. */
5161
5162 gimple_seq
5163 copy_gimple_seq_and_replace_locals (gimple_seq seq)
5164 {
5165 copy_body_data id;
5166 struct walk_stmt_info wi;
5167 gimple_seq copy;
5168
5169 /* There's nothing to do for NULL_TREE. */
5170 if (seq == NULL)
5171 return seq;
5172
5173 /* Set up ID. */
5174 memset (&id, 0, sizeof (id));
5175 id.src_fn = current_function_decl;
5176 id.dst_fn = current_function_decl;
5177 id.decl_map = new hash_map<tree, tree>;
5178 id.debug_map = NULL;
5179
5180 id.copy_decl = copy_decl_no_change;
5181 id.transform_call_graph_edges = CB_CGE_DUPLICATE;
5182 id.transform_new_cfg = false;
5183 id.transform_return_to_modify = false;
5184 id.transform_parameter = false;
5185 id.transform_lang_insert_block = NULL;
5186
5187 /* Walk the tree once to find local labels. */
5188 memset (&wi, 0, sizeof (wi));
5189 hash_set<tree> visited;
5190 wi.info = &id;
5191 wi.pset = &visited;
5192 walk_gimple_seq (seq, mark_local_labels_stmt, NULL, &wi);
5193
5194 copy = gimple_seq_copy (seq);
5195
5196 /* Walk the copy, remapping decls. */
5197 memset (&wi, 0, sizeof (wi));
5198 wi.info = &id;
5199 walk_gimple_seq (copy, replace_locals_stmt, replace_locals_op, &wi);
5200
5201 /* Clean up. */
5202 delete id.decl_map;
5203 if (id.debug_map)
5204 delete id.debug_map;
5205 if (id.dependence_map)
5206 {
5207 delete id.dependence_map;
5208 id.dependence_map = NULL;
5209 }
5210
5211 return copy;
5212 }
5213
5214
5215 /* Allow someone to determine if SEARCH is a child of TOP from gdb. */
5216
5217 static tree
5218 debug_find_tree_1 (tree *tp, int *walk_subtrees ATTRIBUTE_UNUSED, void *data)
5219 {
5220 if (*tp == data)
5221 return (tree) data;
5222 else
5223 return NULL;
5224 }
5225
5226 DEBUG_FUNCTION bool
5227 debug_find_tree (tree top, tree search)
5228 {
5229 return walk_tree_without_duplicates (&top, debug_find_tree_1, search) != 0;
5230 }
5231
5232
5233 /* Declare the variables created by the inliner. Add all the variables in
5234 VARS to BIND_EXPR. */
5235
5236 static void
5237 declare_inline_vars (tree block, tree vars)
5238 {
5239 tree t;
5240 for (t = vars; t; t = DECL_CHAIN (t))
5241 {
5242 DECL_SEEN_IN_BIND_EXPR_P (t) = 1;
5243 gcc_assert (!TREE_STATIC (t) && !TREE_ASM_WRITTEN (t));
5244 add_local_decl (cfun, t);
5245 }
5246
5247 if (block)
5248 BLOCK_VARS (block) = chainon (BLOCK_VARS (block), vars);
5249 }
5250
5251 /* Copy NODE (which must be a DECL). The DECL originally was in the FROM_FN,
5252 but now it will be in the TO_FN. PARM_TO_VAR means enable PARM_DECL to
5253 VAR_DECL translation. */
5254
5255 static tree
5256 copy_decl_for_dup_finish (copy_body_data *id, tree decl, tree copy)
5257 {
5258 /* Don't generate debug information for the copy if we wouldn't have
5259 generated it for the copy either. */
5260 DECL_ARTIFICIAL (copy) = DECL_ARTIFICIAL (decl);
5261 DECL_IGNORED_P (copy) = DECL_IGNORED_P (decl);
5262
5263 /* Set the DECL_ABSTRACT_ORIGIN so the debugging routines know what
5264 declaration inspired this copy. */
5265 DECL_ABSTRACT_ORIGIN (copy) = DECL_ORIGIN (decl);
5266
5267 /* The new variable/label has no RTL, yet. */
5268 if (CODE_CONTAINS_STRUCT (TREE_CODE (copy), TS_DECL_WRTL)
5269 && !TREE_STATIC (copy) && !DECL_EXTERNAL (copy))
5270 SET_DECL_RTL (copy, 0);
5271
5272 /* These args would always appear unused, if not for this. */
5273 TREE_USED (copy) = 1;
5274
5275 /* Set the context for the new declaration. */
5276 if (!DECL_CONTEXT (decl))
5277 /* Globals stay global. */
5278 ;
5279 else if (DECL_CONTEXT (decl) != id->src_fn)
5280 /* Things that weren't in the scope of the function we're inlining
5281 from aren't in the scope we're inlining to, either. */
5282 ;
5283 else if (TREE_STATIC (decl))
5284 /* Function-scoped static variables should stay in the original
5285 function. */
5286 ;
5287 else
5288 /* Ordinary automatic local variables are now in the scope of the
5289 new function. */
5290 DECL_CONTEXT (copy) = id->dst_fn;
5291
5292 return copy;
5293 }
5294
5295 static tree
5296 copy_decl_to_var (tree decl, copy_body_data *id)
5297 {
5298 tree copy, type;
5299
5300 gcc_assert (TREE_CODE (decl) == PARM_DECL
5301 || TREE_CODE (decl) == RESULT_DECL);
5302
5303 type = TREE_TYPE (decl);
5304
5305 copy = build_decl (DECL_SOURCE_LOCATION (id->dst_fn),
5306 VAR_DECL, DECL_NAME (decl), type);
5307 if (DECL_PT_UID_SET_P (decl))
5308 SET_DECL_PT_UID (copy, DECL_PT_UID (decl));
5309 TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (decl);
5310 TREE_READONLY (copy) = TREE_READONLY (decl);
5311 TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (decl);
5312 DECL_GIMPLE_REG_P (copy) = DECL_GIMPLE_REG_P (decl);
5313
5314 return copy_decl_for_dup_finish (id, decl, copy);
5315 }
5316
5317 /* Like copy_decl_to_var, but create a return slot object instead of a
5318 pointer variable for return by invisible reference. */
5319
5320 static tree
5321 copy_result_decl_to_var (tree decl, copy_body_data *id)
5322 {
5323 tree copy, type;
5324
5325 gcc_assert (TREE_CODE (decl) == PARM_DECL
5326 || TREE_CODE (decl) == RESULT_DECL);
5327
5328 type = TREE_TYPE (decl);
5329 if (DECL_BY_REFERENCE (decl))
5330 type = TREE_TYPE (type);
5331
5332 copy = build_decl (DECL_SOURCE_LOCATION (id->dst_fn),
5333 VAR_DECL, DECL_NAME (decl), type);
5334 if (DECL_PT_UID_SET_P (decl))
5335 SET_DECL_PT_UID (copy, DECL_PT_UID (decl));
5336 TREE_READONLY (copy) = TREE_READONLY (decl);
5337 TREE_THIS_VOLATILE (copy) = TREE_THIS_VOLATILE (decl);
5338 if (!DECL_BY_REFERENCE (decl))
5339 {
5340 TREE_ADDRESSABLE (copy) = TREE_ADDRESSABLE (decl);
5341 DECL_GIMPLE_REG_P (copy) = DECL_GIMPLE_REG_P (decl);
5342 }
5343
5344 return copy_decl_for_dup_finish (id, decl, copy);
5345 }
5346
5347 tree
5348 copy_decl_no_change (tree decl, copy_body_data *id)
5349 {
5350 tree copy;
5351
5352 copy = copy_node (decl);
5353
5354 /* The COPY is not abstract; it will be generated in DST_FN. */
5355 DECL_ABSTRACT_P (copy) = false;
5356 lang_hooks.dup_lang_specific_decl (copy);
5357
5358 /* TREE_ADDRESSABLE isn't used to indicate that a label's address has
5359 been taken; it's for internal bookkeeping in expand_goto_internal. */
5360 if (TREE_CODE (copy) == LABEL_DECL)
5361 {
5362 TREE_ADDRESSABLE (copy) = 0;
5363 LABEL_DECL_UID (copy) = -1;
5364 }
5365
5366 return copy_decl_for_dup_finish (id, decl, copy);
5367 }
5368
5369 static tree
5370 copy_decl_maybe_to_var (tree decl, copy_body_data *id)
5371 {
5372 if (TREE_CODE (decl) == PARM_DECL || TREE_CODE (decl) == RESULT_DECL)
5373 return copy_decl_to_var (decl, id);
5374 else
5375 return copy_decl_no_change (decl, id);
5376 }
5377
5378 /* Return a copy of the function's argument tree. */
5379 static tree
5380 copy_arguments_for_versioning (tree orig_parm, copy_body_data * id,
5381 bitmap args_to_skip, tree *vars)
5382 {
5383 tree arg, *parg;
5384 tree new_parm = NULL;
5385 int i = 0;
5386
5387 parg = &new_parm;
5388
5389 for (arg = orig_parm; arg; arg = DECL_CHAIN (arg), i++)
5390 if (!args_to_skip || !bitmap_bit_p (args_to_skip, i))
5391 {
5392 tree new_tree = remap_decl (arg, id);
5393 if (TREE_CODE (new_tree) != PARM_DECL)
5394 new_tree = id->copy_decl (arg, id);
5395 lang_hooks.dup_lang_specific_decl (new_tree);
5396 *parg = new_tree;
5397 parg = &DECL_CHAIN (new_tree);
5398 }
5399 else if (!id->decl_map->get (arg))
5400 {
5401 /* Make an equivalent VAR_DECL. If the argument was used
5402 as temporary variable later in function, the uses will be
5403 replaced by local variable. */
5404 tree var = copy_decl_to_var (arg, id);
5405 insert_decl_map (id, arg, var);
5406 /* Declare this new variable. */
5407 DECL_CHAIN (var) = *vars;
5408 *vars = var;
5409 }
5410 return new_parm;
5411 }
5412
5413 /* Return a copy of the function's static chain. */
5414 static tree
5415 copy_static_chain (tree static_chain, copy_body_data * id)
5416 {
5417 tree *chain_copy, *pvar;
5418
5419 chain_copy = &static_chain;
5420 for (pvar = chain_copy; *pvar; pvar = &DECL_CHAIN (*pvar))
5421 {
5422 tree new_tree = remap_decl (*pvar, id);
5423 lang_hooks.dup_lang_specific_decl (new_tree);
5424 DECL_CHAIN (new_tree) = DECL_CHAIN (*pvar);
5425 *pvar = new_tree;
5426 }
5427 return static_chain;
5428 }
5429
5430 /* Return true if the function is allowed to be versioned.
5431 This is a guard for the versioning functionality. */
5432
5433 bool
5434 tree_versionable_function_p (tree fndecl)
5435 {
5436 return (!lookup_attribute ("noclone", DECL_ATTRIBUTES (fndecl))
5437 && copy_forbidden (DECL_STRUCT_FUNCTION (fndecl), fndecl) == NULL);
5438 }
5439
5440 /* Delete all unreachable basic blocks and update callgraph.
5441 Doing so is somewhat nontrivial because we need to update all clones and
5442 remove inline function that become unreachable. */
5443
5444 static bool
5445 delete_unreachable_blocks_update_callgraph (copy_body_data *id)
5446 {
5447 bool changed = false;
5448 basic_block b, next_bb;
5449
5450 find_unreachable_blocks ();
5451
5452 /* Delete all unreachable basic blocks. */
5453
5454 for (b = ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb; b
5455 != EXIT_BLOCK_PTR_FOR_FN (cfun); b = next_bb)
5456 {
5457 next_bb = b->next_bb;
5458
5459 if (!(b->flags & BB_REACHABLE))
5460 {
5461 gimple_stmt_iterator bsi;
5462
5463 for (bsi = gsi_start_bb (b); !gsi_end_p (bsi); gsi_next (&bsi))
5464 {
5465 struct cgraph_edge *e;
5466 struct cgraph_node *node;
5467
5468 id->dst_node->remove_stmt_references (gsi_stmt (bsi));
5469
5470 if (gimple_code (gsi_stmt (bsi)) == GIMPLE_CALL
5471 &&(e = id->dst_node->get_edge (gsi_stmt (bsi))) != NULL)
5472 {
5473 if (!e->inline_failed)
5474 e->callee->remove_symbol_and_inline_clones (id->dst_node);
5475 else
5476 e->remove ();
5477 }
5478 if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES
5479 && id->dst_node->clones)
5480 for (node = id->dst_node->clones; node != id->dst_node;)
5481 {
5482 node->remove_stmt_references (gsi_stmt (bsi));
5483 if (gimple_code (gsi_stmt (bsi)) == GIMPLE_CALL
5484 && (e = node->get_edge (gsi_stmt (bsi))) != NULL)
5485 {
5486 if (!e->inline_failed)
5487 e->callee->remove_symbol_and_inline_clones (id->dst_node);
5488 else
5489 e->remove ();
5490 }
5491
5492 if (node->clones)
5493 node = node->clones;
5494 else if (node->next_sibling_clone)
5495 node = node->next_sibling_clone;
5496 else
5497 {
5498 while (node != id->dst_node && !node->next_sibling_clone)
5499 node = node->clone_of;
5500 if (node != id->dst_node)
5501 node = node->next_sibling_clone;
5502 }
5503 }
5504 }
5505 delete_basic_block (b);
5506 changed = true;
5507 }
5508 }
5509
5510 return changed;
5511 }
5512
5513 /* Update clone info after duplication. */
5514
5515 static void
5516 update_clone_info (copy_body_data * id)
5517 {
5518 struct cgraph_node *node;
5519 if (!id->dst_node->clones)
5520 return;
5521 for (node = id->dst_node->clones; node != id->dst_node;)
5522 {
5523 /* First update replace maps to match the new body. */
5524 if (node->clone.tree_map)
5525 {
5526 unsigned int i;
5527 for (i = 0; i < vec_safe_length (node->clone.tree_map); i++)
5528 {
5529 struct ipa_replace_map *replace_info;
5530 replace_info = (*node->clone.tree_map)[i];
5531 walk_tree (&replace_info->old_tree, copy_tree_body_r, id, NULL);
5532 walk_tree (&replace_info->new_tree, copy_tree_body_r, id, NULL);
5533 }
5534 }
5535 if (node->clones)
5536 node = node->clones;
5537 else if (node->next_sibling_clone)
5538 node = node->next_sibling_clone;
5539 else
5540 {
5541 while (node != id->dst_node && !node->next_sibling_clone)
5542 node = node->clone_of;
5543 if (node != id->dst_node)
5544 node = node->next_sibling_clone;
5545 }
5546 }
5547 }
5548
5549 /* Create a copy of a function's tree.
5550 OLD_DECL and NEW_DECL are FUNCTION_DECL tree nodes
5551 of the original function and the new copied function
5552 respectively. In case we want to replace a DECL
5553 tree with another tree while duplicating the function's
5554 body, TREE_MAP represents the mapping between these
5555 trees. If UPDATE_CLONES is set, the call_stmt fields
5556 of edges of clones of the function will be updated.
5557
5558 If non-NULL ARGS_TO_SKIP determine function parameters to remove
5559 from new version.
5560 If SKIP_RETURN is true, the new version will return void.
5561 If non-NULL BLOCK_TO_COPY determine what basic blocks to copy.
5562 If non_NULL NEW_ENTRY determine new entry BB of the clone.
5563 */
5564 void
5565 tree_function_versioning (tree old_decl, tree new_decl,
5566 vec<ipa_replace_map *, va_gc> *tree_map,
5567 bool update_clones, bitmap args_to_skip,
5568 bool skip_return, bitmap blocks_to_copy,
5569 basic_block new_entry)
5570 {
5571 struct cgraph_node *old_version_node;
5572 struct cgraph_node *new_version_node;
5573 copy_body_data id;
5574 tree p;
5575 unsigned i;
5576 struct ipa_replace_map *replace_info;
5577 basic_block old_entry_block, bb;
5578 auto_vec<gimple, 10> init_stmts;
5579 tree vars = NULL_TREE;
5580
5581 gcc_assert (TREE_CODE (old_decl) == FUNCTION_DECL
5582 && TREE_CODE (new_decl) == FUNCTION_DECL);
5583 DECL_POSSIBLY_INLINED (old_decl) = 1;
5584
5585 old_version_node = cgraph_node::get (old_decl);
5586 gcc_checking_assert (old_version_node);
5587 new_version_node = cgraph_node::get (new_decl);
5588 gcc_checking_assert (new_version_node);
5589
5590 /* Copy over debug args. */
5591 if (DECL_HAS_DEBUG_ARGS_P (old_decl))
5592 {
5593 vec<tree, va_gc> **new_debug_args, **old_debug_args;
5594 gcc_checking_assert (decl_debug_args_lookup (new_decl) == NULL);
5595 DECL_HAS_DEBUG_ARGS_P (new_decl) = 0;
5596 old_debug_args = decl_debug_args_lookup (old_decl);
5597 if (old_debug_args)
5598 {
5599 new_debug_args = decl_debug_args_insert (new_decl);
5600 *new_debug_args = vec_safe_copy (*old_debug_args);
5601 }
5602 }
5603
5604 /* Output the inlining info for this abstract function, since it has been
5605 inlined. If we don't do this now, we can lose the information about the
5606 variables in the function when the blocks get blown away as soon as we
5607 remove the cgraph node. */
5608 (*debug_hooks->outlining_inline_function) (old_decl);
5609
5610 DECL_ARTIFICIAL (new_decl) = 1;
5611 DECL_ABSTRACT_ORIGIN (new_decl) = DECL_ORIGIN (old_decl);
5612 if (DECL_ORIGIN (old_decl) == old_decl)
5613 old_version_node->used_as_abstract_origin = true;
5614 DECL_FUNCTION_PERSONALITY (new_decl) = DECL_FUNCTION_PERSONALITY (old_decl);
5615
5616 /* Prepare the data structures for the tree copy. */
5617 memset (&id, 0, sizeof (id));
5618
5619 /* Generate a new name for the new version. */
5620 id.statements_to_fold = new hash_set<gimple>;
5621
5622 id.decl_map = new hash_map<tree, tree>;
5623 id.debug_map = NULL;
5624 id.src_fn = old_decl;
5625 id.dst_fn = new_decl;
5626 id.src_node = old_version_node;
5627 id.dst_node = new_version_node;
5628 id.src_cfun = DECL_STRUCT_FUNCTION (old_decl);
5629 id.blocks_to_copy = blocks_to_copy;
5630
5631 id.copy_decl = copy_decl_no_change;
5632 id.transform_call_graph_edges
5633 = update_clones ? CB_CGE_MOVE_CLONES : CB_CGE_MOVE;
5634 id.transform_new_cfg = true;
5635 id.transform_return_to_modify = false;
5636 id.transform_parameter = false;
5637 id.transform_lang_insert_block = NULL;
5638
5639 old_entry_block = ENTRY_BLOCK_PTR_FOR_FN
5640 (DECL_STRUCT_FUNCTION (old_decl));
5641 DECL_RESULT (new_decl) = DECL_RESULT (old_decl);
5642 DECL_ARGUMENTS (new_decl) = DECL_ARGUMENTS (old_decl);
5643 initialize_cfun (new_decl, old_decl,
5644 old_entry_block->count);
5645 if (DECL_STRUCT_FUNCTION (new_decl)->gimple_df)
5646 DECL_STRUCT_FUNCTION (new_decl)->gimple_df->ipa_pta
5647 = id.src_cfun->gimple_df->ipa_pta;
5648
5649 /* Copy the function's static chain. */
5650 p = DECL_STRUCT_FUNCTION (old_decl)->static_chain_decl;
5651 if (p)
5652 DECL_STRUCT_FUNCTION (new_decl)->static_chain_decl =
5653 copy_static_chain (DECL_STRUCT_FUNCTION (old_decl)->static_chain_decl,
5654 &id);
5655
5656 /* If there's a tree_map, prepare for substitution. */
5657 if (tree_map)
5658 for (i = 0; i < tree_map->length (); i++)
5659 {
5660 gimple init;
5661 replace_info = (*tree_map)[i];
5662 if (replace_info->replace_p)
5663 {
5664 if (!replace_info->old_tree)
5665 {
5666 int i = replace_info->parm_num;
5667 tree parm;
5668 tree req_type;
5669
5670 for (parm = DECL_ARGUMENTS (old_decl); i; parm = DECL_CHAIN (parm))
5671 i --;
5672 replace_info->old_tree = parm;
5673 req_type = TREE_TYPE (parm);
5674 if (!useless_type_conversion_p (req_type, TREE_TYPE (replace_info->new_tree)))
5675 {
5676 if (fold_convertible_p (req_type, replace_info->new_tree))
5677 replace_info->new_tree = fold_build1 (NOP_EXPR, req_type, replace_info->new_tree);
5678 else if (TYPE_SIZE (req_type) == TYPE_SIZE (TREE_TYPE (replace_info->new_tree)))
5679 replace_info->new_tree = fold_build1 (VIEW_CONVERT_EXPR, req_type, replace_info->new_tree);
5680 else
5681 {
5682 if (dump_file)
5683 {
5684 fprintf (dump_file, " const ");
5685 print_generic_expr (dump_file, replace_info->new_tree, 0);
5686 fprintf (dump_file, " can't be converted to param ");
5687 print_generic_expr (dump_file, parm, 0);
5688 fprintf (dump_file, "\n");
5689 }
5690 replace_info->old_tree = NULL;
5691 }
5692 }
5693 }
5694 else
5695 gcc_assert (TREE_CODE (replace_info->old_tree) == PARM_DECL);
5696 if (replace_info->old_tree)
5697 {
5698 init = setup_one_parameter (&id, replace_info->old_tree,
5699 replace_info->new_tree, id.src_fn,
5700 NULL,
5701 &vars);
5702 if (init)
5703 init_stmts.safe_push (init);
5704 }
5705 }
5706 }
5707 /* Copy the function's arguments. */
5708 if (DECL_ARGUMENTS (old_decl) != NULL_TREE)
5709 DECL_ARGUMENTS (new_decl) =
5710 copy_arguments_for_versioning (DECL_ARGUMENTS (old_decl), &id,
5711 args_to_skip, &vars);
5712
5713 DECL_INITIAL (new_decl) = remap_blocks (DECL_INITIAL (id.src_fn), &id);
5714 BLOCK_SUPERCONTEXT (DECL_INITIAL (new_decl)) = new_decl;
5715
5716 declare_inline_vars (DECL_INITIAL (new_decl), vars);
5717
5718 if (!vec_safe_is_empty (DECL_STRUCT_FUNCTION (old_decl)->local_decls))
5719 /* Add local vars. */
5720 add_local_variables (DECL_STRUCT_FUNCTION (old_decl), cfun, &id);
5721
5722 if (DECL_RESULT (old_decl) == NULL_TREE)
5723 ;
5724 else if (skip_return && !VOID_TYPE_P (TREE_TYPE (DECL_RESULT (old_decl))))
5725 {
5726 DECL_RESULT (new_decl)
5727 = build_decl (DECL_SOURCE_LOCATION (DECL_RESULT (old_decl)),
5728 RESULT_DECL, NULL_TREE, void_type_node);
5729 DECL_CONTEXT (DECL_RESULT (new_decl)) = new_decl;
5730 cfun->returns_struct = 0;
5731 cfun->returns_pcc_struct = 0;
5732 }
5733 else
5734 {
5735 tree old_name;
5736 DECL_RESULT (new_decl) = remap_decl (DECL_RESULT (old_decl), &id);
5737 lang_hooks.dup_lang_specific_decl (DECL_RESULT (new_decl));
5738 if (gimple_in_ssa_p (id.src_cfun)
5739 && DECL_BY_REFERENCE (DECL_RESULT (old_decl))
5740 && (old_name = ssa_default_def (id.src_cfun, DECL_RESULT (old_decl))))
5741 {
5742 tree new_name = make_ssa_name (DECL_RESULT (new_decl));
5743 insert_decl_map (&id, old_name, new_name);
5744 SSA_NAME_DEF_STMT (new_name) = gimple_build_nop ();
5745 set_ssa_default_def (cfun, DECL_RESULT (new_decl), new_name);
5746 }
5747 }
5748
5749 /* Set up the destination functions loop tree. */
5750 if (loops_for_fn (DECL_STRUCT_FUNCTION (old_decl)) != NULL)
5751 {
5752 cfun->curr_properties &= ~PROP_loops;
5753 loop_optimizer_init (AVOID_CFG_MODIFICATIONS);
5754 cfun->curr_properties |= PROP_loops;
5755 }
5756
5757 /* Copy the Function's body. */
5758 copy_body (&id, old_entry_block->count, REG_BR_PROB_BASE,
5759 ENTRY_BLOCK_PTR_FOR_FN (cfun), EXIT_BLOCK_PTR_FOR_FN (cfun),
5760 new_entry);
5761
5762 /* Renumber the lexical scoping (non-code) blocks consecutively. */
5763 number_blocks (new_decl);
5764
5765 /* We want to create the BB unconditionally, so that the addition of
5766 debug stmts doesn't affect BB count, which may in the end cause
5767 codegen differences. */
5768 bb = split_edge (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
5769 while (init_stmts.length ())
5770 insert_init_stmt (&id, bb, init_stmts.pop ());
5771 update_clone_info (&id);
5772
5773 /* Remap the nonlocal_goto_save_area, if any. */
5774 if (cfun->nonlocal_goto_save_area)
5775 {
5776 struct walk_stmt_info wi;
5777
5778 memset (&wi, 0, sizeof (wi));
5779 wi.info = &id;
5780 walk_tree (&cfun->nonlocal_goto_save_area, remap_gimple_op_r, &wi, NULL);
5781 }
5782
5783 /* Clean up. */
5784 delete id.decl_map;
5785 if (id.debug_map)
5786 delete id.debug_map;
5787 free_dominance_info (CDI_DOMINATORS);
5788 free_dominance_info (CDI_POST_DOMINATORS);
5789
5790 fold_marked_statements (0, id.statements_to_fold);
5791 delete id.statements_to_fold;
5792 fold_cond_expr_cond ();
5793 delete_unreachable_blocks_update_callgraph (&id);
5794 if (id.dst_node->definition)
5795 cgraph_edge::rebuild_references ();
5796 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP))
5797 {
5798 calculate_dominance_info (CDI_DOMINATORS);
5799 fix_loop_structure (NULL);
5800 }
5801 update_ssa (TODO_update_ssa);
5802
5803 /* After partial cloning we need to rescale frequencies, so they are
5804 within proper range in the cloned function. */
5805 if (new_entry)
5806 {
5807 struct cgraph_edge *e;
5808 rebuild_frequencies ();
5809
5810 new_version_node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
5811 for (e = new_version_node->callees; e; e = e->next_callee)
5812 {
5813 basic_block bb = gimple_bb (e->call_stmt);
5814 e->frequency = compute_call_stmt_bb_frequency (current_function_decl,
5815 bb);
5816 e->count = bb->count;
5817 }
5818 for (e = new_version_node->indirect_calls; e; e = e->next_callee)
5819 {
5820 basic_block bb = gimple_bb (e->call_stmt);
5821 e->frequency = compute_call_stmt_bb_frequency (current_function_decl,
5822 bb);
5823 e->count = bb->count;
5824 }
5825 }
5826
5827 free_dominance_info (CDI_DOMINATORS);
5828 free_dominance_info (CDI_POST_DOMINATORS);
5829
5830 gcc_assert (!id.debug_stmts.exists ());
5831 pop_cfun ();
5832 return;
5833 }
5834
5835 /* EXP is CALL_EXPR present in a GENERIC expression tree. Try to integrate
5836 the callee and return the inlined body on success. */
5837
5838 tree
5839 maybe_inline_call_in_expr (tree exp)
5840 {
5841 tree fn = get_callee_fndecl (exp);
5842
5843 /* We can only try to inline "const" functions. */
5844 if (fn && TREE_READONLY (fn) && DECL_SAVED_TREE (fn))
5845 {
5846 call_expr_arg_iterator iter;
5847 copy_body_data id;
5848 tree param, arg, t;
5849 hash_map<tree, tree> decl_map;
5850
5851 /* Remap the parameters. */
5852 for (param = DECL_ARGUMENTS (fn), arg = first_call_expr_arg (exp, &iter);
5853 param;
5854 param = DECL_CHAIN (param), arg = next_call_expr_arg (&iter))
5855 decl_map.put (param, arg);
5856
5857 memset (&id, 0, sizeof (id));
5858 id.src_fn = fn;
5859 id.dst_fn = current_function_decl;
5860 id.src_cfun = DECL_STRUCT_FUNCTION (fn);
5861 id.decl_map = &decl_map;
5862
5863 id.copy_decl = copy_decl_no_change;
5864 id.transform_call_graph_edges = CB_CGE_DUPLICATE;
5865 id.transform_new_cfg = false;
5866 id.transform_return_to_modify = true;
5867 id.transform_parameter = true;
5868 id.transform_lang_insert_block = NULL;
5869
5870 /* Make sure not to unshare trees behind the front-end's back
5871 since front-end specific mechanisms may rely on sharing. */
5872 id.regimplify = false;
5873 id.do_not_unshare = true;
5874
5875 /* We're not inside any EH region. */
5876 id.eh_lp_nr = 0;
5877
5878 t = copy_tree_body (&id);
5879
5880 /* We can only return something suitable for use in a GENERIC
5881 expression tree. */
5882 if (TREE_CODE (t) == MODIFY_EXPR)
5883 return TREE_OPERAND (t, 1);
5884 }
5885
5886 return NULL_TREE;
5887 }
5888
5889 /* Duplicate a type, fields and all. */
5890
5891 tree
5892 build_duplicate_type (tree type)
5893 {
5894 struct copy_body_data id;
5895
5896 memset (&id, 0, sizeof (id));
5897 id.src_fn = current_function_decl;
5898 id.dst_fn = current_function_decl;
5899 id.src_cfun = cfun;
5900 id.decl_map = new hash_map<tree, tree>;
5901 id.debug_map = NULL;
5902 id.copy_decl = copy_decl_no_change;
5903
5904 type = remap_type_1 (type, &id);
5905
5906 delete id.decl_map;
5907 if (id.debug_map)
5908 delete id.debug_map;
5909
5910 TYPE_CANONICAL (type) = type;
5911
5912 return type;
5913 }
5914
5915 /* Unshare the entire DECL_SAVED_TREE of FN and return the remapped
5916 parameters and RESULT_DECL in PARMS and RESULT. Used by C++ constexpr
5917 evaluation. */
5918
5919 tree
5920 copy_fn (tree fn, tree& parms, tree& result)
5921 {
5922 copy_body_data id;
5923 tree param;
5924 hash_map<tree, tree> decl_map;
5925
5926 tree *p = &parms;
5927 *p = NULL_TREE;
5928
5929 memset (&id, 0, sizeof (id));
5930 id.src_fn = fn;
5931 id.dst_fn = current_function_decl;
5932 id.src_cfun = DECL_STRUCT_FUNCTION (fn);
5933 id.decl_map = &decl_map;
5934
5935 id.copy_decl = copy_decl_no_change;
5936 id.transform_call_graph_edges = CB_CGE_DUPLICATE;
5937 id.transform_new_cfg = false;
5938 id.transform_return_to_modify = false;
5939 id.transform_parameter = true;
5940 id.transform_lang_insert_block = NULL;
5941
5942 /* Make sure not to unshare trees behind the front-end's back
5943 since front-end specific mechanisms may rely on sharing. */
5944 id.regimplify = false;
5945 id.do_not_unshare = true;
5946
5947 /* We're not inside any EH region. */
5948 id.eh_lp_nr = 0;
5949
5950 /* Remap the parameters and result and return them to the caller. */
5951 for (param = DECL_ARGUMENTS (fn);
5952 param;
5953 param = DECL_CHAIN (param))
5954 {
5955 *p = remap_decl (param, &id);
5956 p = &DECL_CHAIN (*p);
5957 }
5958
5959 if (DECL_RESULT (fn))
5960 result = remap_decl (DECL_RESULT (fn), &id);
5961 else
5962 result = NULL_TREE;
5963
5964 return copy_tree_body (&id);
5965 }