1 /* Inlining decision heuristics.
2 Copyright (C) 2003, 2004, 2007, 2008 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Inlining decision heuristics
23 We separate inlining decisions from the inliner itself and store it
24 inside callgraph as so called inline plan. Refer to cgraph.c
25 documentation about particular representation of inline plans in the
28 There are three major parts of this file:
30 cgraph_mark_inline implementation
32 This function allows to mark given call inline and performs necessary
33 modifications of cgraph (production of the clones and updating overall
36 inlining heuristics limits
38 These functions allow to check that particular inlining is allowed
39 by the limits specified by user (allowed function growth, overall unit
44 This is implementation of IPA pass aiming to get as much of benefit
45 from inlining obeying the limits checked above.
47 The implementation of particular heuristics is separated from
48 the rest of code to make it easier to replace it with more complicated
49 implementation in the future. The rest of inlining code acts as a
50 library aimed to modify the callgraph and verify that the parameters
51 on code size growth fits.
53 To mark given call inline, use cgraph_mark_inline function, the
54 verification is performed by cgraph_default_inline_p and
55 cgraph_check_inline_limits.
57 The heuristics implements simple knapsack style algorithm ordering
58 all functions by their "profitability" (estimated by code size growth)
59 and inlining them in priority order.
61 cgraph_decide_inlining implements heuristics taking whole callgraph
62 into account, while cgraph_decide_inlining_incrementally considers
63 only one function at a time and is used by early inliner.
65 The inliner itself is split into several passes:
67 pass_inline_parameters
69 This pass computes local properties of functions that are used by inliner:
70 estimated function body size, whether function is inlinable at all and
71 stack frame consumption.
73 Before executing any of inliner passes, this local pass has to be applied
74 to each function in the callgraph (ie run as subpass of some earlier
75 IPA pass). The results are made out of date by any optimization applied
80 Simple local inlining pass inlining callees into current function. This
81 pass makes no global whole compilation unit analysis and this when allowed
82 to do inlining expanding code size it might result in unbounded growth of
85 The pass is run during conversion into SSA form. Only functions already
86 converted into SSA form are inlined, so the conversion must happen in
87 topological order on the callgraph (that is maintained by pass manager).
88 The functions after inlining are early optimized so the early inliner sees
89 unoptimized function itself, but all considered callees are already
90 optimized allowing it to unfold abstraction penalty on C++ effectively and
93 pass_ipa_early_inlining
95 With profiling, the early inlining is also necessary to reduce
96 instrumentation costs on program with high abstraction penalty (doing
97 many redundant calls). This can't happen in parallel with early
98 optimization and profile instrumentation, because we would end up
99 re-instrumenting already instrumented function bodies we brought in via
102 To avoid this, this pass is executed as IPA pass before profiling. It is
103 simple wrapper to pass_early_inlining and ensures first inlining.
107 This is the main pass implementing simple greedy algorithm to do inlining
108 of small functions that results in overall growth of compilation unit and
109 inlining of functions called once. The pass compute just so called inline
110 plan (representation of inlining to be done in callgraph) and unlike early
111 inlining it is not performing the inlining itself.
115 This pass performs actual inlining according to pass_ipa_inline on given
116 function. Possible the function body before inlining is saved when it is
117 needed for further inlining later.
122 #include "coretypes.h"
125 #include "tree-inline.h"
126 #include "langhooks.h"
129 #include "diagnostic.h"
134 #include "tree-pass.h"
136 #include "coverage.h"
138 #include "tree-flow.h"
140 #include "ipa-prop.h"
142 /* Mode incremental inliner operate on:
144 In ALWAYS_INLINE only functions marked
145 always_inline are inlined. This mode is used after detecting cycle during
148 In SIZE mode, only functions that reduce function body size after inlining
149 are inlined, this is used during early inlining.
151 in ALL mode, everything is inlined. This is used during flattening. */
154 INLINE_ALWAYS_INLINE
,
159 cgraph_decide_inlining_incrementally (struct cgraph_node
*, enum inlining_mode
,
163 /* Statistics we collect about inlining algorithm. */
164 static int ncalls_inlined
;
165 static int nfunctions_inlined
;
166 static int overall_insns
;
167 static gcov_type max_count
;
169 /* Holders of ipa cgraph hooks: */
170 static struct cgraph_node_hook_list
*function_insertion_hook_holder
;
172 static inline struct inline_summary
*
173 inline_summary (struct cgraph_node
*node
)
175 return &node
->local
.inline_summary
;
178 /* Estimate size of the function after inlining WHAT into TO. */
181 cgraph_estimate_size_after_inlining (int times
, struct cgraph_node
*to
,
182 struct cgraph_node
*what
)
185 tree fndecl
= what
->decl
, arg
;
186 int call_insns
= PARAM_VALUE (PARAM_INLINE_CALL_COST
);
188 for (arg
= DECL_ARGUMENTS (fndecl
); arg
; arg
= TREE_CHAIN (arg
))
189 call_insns
+= estimate_move_cost (TREE_TYPE (arg
));
190 size
= (what
->global
.insns
- call_insns
) * times
+ to
->global
.insns
;
191 gcc_assert (size
>= 0);
195 /* E is expected to be an edge being inlined. Clone destination node of
196 the edge and redirect it to the new clone.
197 DUPLICATE is used for bookkeeping on whether we are actually creating new
198 clones or re-using node originally representing out-of-line function call.
201 cgraph_clone_inlined_nodes (struct cgraph_edge
*e
, bool duplicate
,
202 bool update_original
)
208 /* We may eliminate the need for out-of-line copy to be output.
209 In that case just go ahead and re-use it. */
210 if (!e
->callee
->callers
->next_caller
211 && !e
->callee
->needed
212 && !cgraph_new_nodes
)
214 gcc_assert (!e
->callee
->global
.inlined_to
);
215 if (gimple_body (e
->callee
->decl
))
216 overall_insns
-= e
->callee
->global
.insns
, nfunctions_inlined
++;
221 struct cgraph_node
*n
;
222 n
= cgraph_clone_node (e
->callee
, e
->count
, e
->frequency
, e
->loop_nest
,
224 cgraph_redirect_edge_callee (e
, n
);
228 if (e
->caller
->global
.inlined_to
)
229 e
->callee
->global
.inlined_to
= e
->caller
->global
.inlined_to
;
231 e
->callee
->global
.inlined_to
= e
->caller
;
232 e
->callee
->global
.stack_frame_offset
233 = e
->caller
->global
.stack_frame_offset
234 + inline_summary (e
->caller
)->estimated_self_stack_size
;
235 peak
= e
->callee
->global
.stack_frame_offset
236 + inline_summary (e
->callee
)->estimated_self_stack_size
;
237 if (e
->callee
->global
.inlined_to
->global
.estimated_stack_size
< peak
)
238 e
->callee
->global
.inlined_to
->global
.estimated_stack_size
= peak
;
240 /* Recursively clone all bodies. */
241 for (e
= e
->callee
->callees
; e
; e
= e
->next_callee
)
242 if (!e
->inline_failed
)
243 cgraph_clone_inlined_nodes (e
, duplicate
, update_original
);
246 /* Mark edge E as inlined and update callgraph accordingly.
247 UPDATE_ORIGINAL specify whether profile of original function should be
251 cgraph_mark_inline_edge (struct cgraph_edge
*e
, bool update_original
)
253 int old_insns
= 0, new_insns
= 0;
254 struct cgraph_node
*to
= NULL
, *what
;
256 if (e
->callee
->inline_decl
)
257 cgraph_redirect_edge_callee (e
, cgraph_node (e
->callee
->inline_decl
));
259 gcc_assert (e
->inline_failed
);
260 e
->inline_failed
= NULL
;
262 if (!e
->callee
->global
.inlined
)
263 DECL_POSSIBLY_INLINED (e
->callee
->decl
) = true;
264 e
->callee
->global
.inlined
= true;
266 cgraph_clone_inlined_nodes (e
, true, update_original
);
270 /* Now update size of caller and all functions caller is inlined into. */
271 for (;e
&& !e
->inline_failed
; e
= e
->caller
->callers
)
273 old_insns
= e
->caller
->global
.insns
;
274 new_insns
= cgraph_estimate_size_after_inlining (1, e
->caller
,
276 gcc_assert (new_insns
>= 0);
278 to
->global
.insns
= new_insns
;
280 gcc_assert (what
->global
.inlined_to
== to
);
281 if (new_insns
> old_insns
)
282 overall_insns
+= new_insns
- old_insns
;
286 /* Mark all calls of EDGE->CALLEE inlined into EDGE->CALLER.
287 Return following unredirected edge in the list of callers
290 static struct cgraph_edge
*
291 cgraph_mark_inline (struct cgraph_edge
*edge
)
293 struct cgraph_node
*to
= edge
->caller
;
294 struct cgraph_node
*what
= edge
->callee
;
295 struct cgraph_edge
*e
, *next
;
297 gcc_assert (!gimple_call_cannot_inline_p (edge
->call_stmt
));
298 /* Look for all calls, mark them inline and clone recursively
299 all inlined functions. */
300 for (e
= what
->callers
; e
; e
= next
)
302 next
= e
->next_caller
;
303 if (e
->caller
== to
&& e
->inline_failed
)
305 cgraph_mark_inline_edge (e
, true);
314 /* Estimate the growth caused by inlining NODE into all callees. */
317 cgraph_estimate_growth (struct cgraph_node
*node
)
320 struct cgraph_edge
*e
;
321 if (node
->global
.estimated_growth
!= INT_MIN
)
322 return node
->global
.estimated_growth
;
324 for (e
= node
->callers
; e
; e
= e
->next_caller
)
325 if (e
->inline_failed
)
326 growth
+= (cgraph_estimate_size_after_inlining (1, e
->caller
, node
)
327 - e
->caller
->global
.insns
);
329 /* ??? Wrong for self recursive functions or cases where we decide to not
330 inline for different reasons, but it is not big deal as in that case
331 we will keep the body around, but we will also avoid some inlining. */
332 if (!node
->needed
&& !DECL_EXTERNAL (node
->decl
))
333 growth
-= node
->global
.insns
;
335 node
->global
.estimated_growth
= growth
;
339 /* Return false when inlining WHAT into TO is not good idea
340 as it would cause too large growth of function bodies.
341 When ONE_ONLY is true, assume that only one call site is going
342 to be inlined, otherwise figure out how many call sites in
343 TO calls WHAT and verify that all can be inlined.
347 cgraph_check_inline_limits (struct cgraph_node
*to
, struct cgraph_node
*what
,
348 const char **reason
, bool one_only
)
351 struct cgraph_edge
*e
;
354 HOST_WIDE_INT stack_size_limit
, inlined_stack
;
359 for (e
= to
->callees
; e
; e
= e
->next_callee
)
360 if (e
->callee
== what
)
363 if (to
->global
.inlined_to
)
364 to
= to
->global
.inlined_to
;
366 /* When inlining large function body called once into small function,
367 take the inlined function as base for limiting the growth. */
368 if (inline_summary (to
)->self_insns
> inline_summary(what
)->self_insns
)
369 limit
= inline_summary (to
)->self_insns
;
371 limit
= inline_summary (what
)->self_insns
;
373 limit
+= limit
* PARAM_VALUE (PARAM_LARGE_FUNCTION_GROWTH
) / 100;
375 /* Check the size after inlining against the function limits. But allow
376 the function to shrink if it went over the limits by forced inlining. */
377 newsize
= cgraph_estimate_size_after_inlining (times
, to
, what
);
378 if (newsize
>= to
->global
.insns
379 && newsize
> PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS
)
383 *reason
= N_("--param large-function-growth limit reached");
387 stack_size_limit
= inline_summary (to
)->estimated_self_stack_size
;
389 stack_size_limit
+= stack_size_limit
* PARAM_VALUE (PARAM_STACK_FRAME_GROWTH
) / 100;
391 inlined_stack
= (to
->global
.stack_frame_offset
392 + inline_summary (to
)->estimated_self_stack_size
393 + what
->global
.estimated_stack_size
);
394 if (inlined_stack
> stack_size_limit
395 && inlined_stack
> PARAM_VALUE (PARAM_LARGE_STACK_FRAME
))
398 *reason
= N_("--param large-stack-frame-growth limit reached");
404 /* Return true when function N is small enough to be inlined. */
407 cgraph_default_inline_p (struct cgraph_node
*n
, const char **reason
)
412 decl
= n
->inline_decl
;
413 if (!flag_inline_small_functions
&& !DECL_DECLARED_INLINE_P (decl
))
416 *reason
= N_("function not inline candidate");
420 if (!DECL_STRUCT_FUNCTION (decl
)->cfg
)
423 *reason
= N_("function body not available");
427 if (DECL_DECLARED_INLINE_P (decl
))
429 if (n
->global
.insns
>= MAX_INLINE_INSNS_SINGLE
)
432 *reason
= N_("--param max-inline-insns-single limit reached");
438 if (n
->global
.insns
>= MAX_INLINE_INSNS_AUTO
)
441 *reason
= N_("--param max-inline-insns-auto limit reached");
449 /* Return true when inlining WHAT would create recursive inlining.
450 We call recursive inlining all cases where same function appears more than
451 once in the single recursion nest path in the inline graph. */
454 cgraph_recursive_inlining_p (struct cgraph_node
*to
,
455 struct cgraph_node
*what
,
459 if (to
->global
.inlined_to
)
460 recursive
= what
->decl
== to
->global
.inlined_to
->decl
;
462 recursive
= what
->decl
== to
->decl
;
463 /* Marking recursive function inline has sane semantic and thus we should
465 if (recursive
&& reason
)
466 *reason
= (what
->local
.disregard_inline_limits
467 ? N_("recursive inlining") : "");
471 /* Return true if the call can be hot. */
473 cgraph_maybe_hot_edge_p (struct cgraph_edge
*edge
)
475 if (profile_info
&& flag_branch_probabilities
477 <= profile_info
->sum_max
/ PARAM_VALUE (HOT_BB_COUNT_FRACTION
)))
479 if (lookup_attribute ("cold", DECL_ATTRIBUTES (edge
->callee
->decl
))
480 || lookup_attribute ("cold", DECL_ATTRIBUTES (edge
->caller
->decl
)))
482 if (lookup_attribute ("hot", DECL_ATTRIBUTES (edge
->caller
->decl
)))
484 if (flag_guess_branch_prob
485 && edge
->frequency
< (CGRAPH_FREQ_MAX
486 / PARAM_VALUE (HOT_BB_FREQUENCY_FRACTION
)))
491 /* A cost model driving the inlining heuristics in a way so the edges with
492 smallest badness are inlined first. After each inlining is performed
493 the costs of all caller edges of nodes affected are recomputed so the
494 metrics may accurately depend on values such as number of inlinable callers
495 of the function or function body size. */
498 cgraph_edge_badness (struct cgraph_edge
*edge
)
502 cgraph_estimate_size_after_inlining (1, edge
->caller
, edge
->callee
);
504 growth
-= edge
->caller
->global
.insns
;
506 /* Always prefer inlining saving code size. */
508 badness
= INT_MIN
- growth
;
510 /* When profiling is available, base priorities -(#calls / growth).
511 So we optimize for overall number of "executed" inlined calls. */
513 badness
= ((int)((double)edge
->count
* INT_MIN
/ max_count
)) / growth
;
515 /* When function local profile is available, base priorities on
516 growth / frequency, so we optimize for overall frequency of inlined
517 calls. This is not too accurate since while the call might be frequent
518 within function, the function itself is infrequent.
520 Other objective to optimize for is number of different calls inlined.
521 We add the estimated growth after inlining all functions to bias the
522 priorities slightly in this direction (so fewer times called functions
523 of the same size gets priority). */
524 else if (flag_guess_branch_prob
)
526 int div
= edge
->frequency
* 100 / CGRAPH_FREQ_BASE
;
528 cgraph_estimate_size_after_inlining (1, edge
->caller
, edge
->callee
);
529 growth
-= edge
->caller
->global
.insns
;
530 badness
= growth
* 256;
532 /* Decrease badness if call is nested. */
533 /* Compress the range so we don't overflow. */
535 div
= 256 + ceil_log2 (div
) - 8;
540 badness
+= cgraph_estimate_growth (edge
->callee
);
542 /* When function local profile is not available or it does not give
543 useful information (ie frequency is zero), base the cost on
544 loop nest and overall size growth, so we optimize for overall number
545 of functions fully inlined in program. */
548 int nest
= MIN (edge
->loop_nest
, 8);
549 badness
= cgraph_estimate_growth (edge
->callee
) * 256;
551 /* Decrease badness if call is nested. */
559 /* Make recursive inlining happen always after other inlining is done. */
560 if (cgraph_recursive_inlining_p (edge
->caller
, edge
->callee
, NULL
))
566 /* Recompute heap nodes for each of caller edge. */
569 update_caller_keys (fibheap_t heap
, struct cgraph_node
*node
,
570 bitmap updated_nodes
)
572 struct cgraph_edge
*edge
;
573 const char *failed_reason
;
575 if (!node
->local
.inlinable
|| node
->local
.disregard_inline_limits
576 || node
->global
.inlined_to
)
578 if (bitmap_bit_p (updated_nodes
, node
->uid
))
580 bitmap_set_bit (updated_nodes
, node
->uid
);
581 node
->global
.estimated_growth
= INT_MIN
;
583 if (!node
->local
.inlinable
)
585 /* Prune out edges we won't inline into anymore. */
586 if (!cgraph_default_inline_p (node
, &failed_reason
))
588 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
591 fibheap_delete_node (heap
, (fibnode_t
) edge
->aux
);
593 if (edge
->inline_failed
)
594 edge
->inline_failed
= failed_reason
;
599 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
600 if (edge
->inline_failed
)
602 int badness
= cgraph_edge_badness (edge
);
605 fibnode_t n
= (fibnode_t
) edge
->aux
;
606 gcc_assert (n
->data
== edge
);
607 if (n
->key
== badness
)
610 /* fibheap_replace_key only increase the keys. */
611 if (fibheap_replace_key (heap
, n
, badness
))
613 fibheap_delete_node (heap
, (fibnode_t
) edge
->aux
);
615 edge
->aux
= fibheap_insert (heap
, badness
, edge
);
619 /* Recompute heap nodes for each of caller edges of each of callees. */
622 update_callee_keys (fibheap_t heap
, struct cgraph_node
*node
,
623 bitmap updated_nodes
)
625 struct cgraph_edge
*e
;
626 node
->global
.estimated_growth
= INT_MIN
;
628 for (e
= node
->callees
; e
; e
= e
->next_callee
)
629 if (e
->inline_failed
)
630 update_caller_keys (heap
, e
->callee
, updated_nodes
);
631 else if (!e
->inline_failed
)
632 update_callee_keys (heap
, e
->callee
, updated_nodes
);
635 /* Enqueue all recursive calls from NODE into priority queue depending on
636 how likely we want to recursively inline the call. */
639 lookup_recursive_calls (struct cgraph_node
*node
, struct cgraph_node
*where
,
643 struct cgraph_edge
*e
;
644 for (e
= where
->callees
; e
; e
= e
->next_callee
)
645 if (e
->callee
== node
)
647 /* When profile feedback is available, prioritize by expected number
648 of calls. Without profile feedback we maintain simple queue
649 to order candidates via recursive depths. */
650 fibheap_insert (heap
,
651 !max_count
? priority
++
652 : -(e
->count
/ ((max_count
+ (1<<24) - 1) / (1<<24))),
655 for (e
= where
->callees
; e
; e
= e
->next_callee
)
656 if (!e
->inline_failed
)
657 lookup_recursive_calls (node
, e
->callee
, heap
);
660 /* Decide on recursive inlining: in the case function has recursive calls,
661 inline until body size reaches given argument. If any new indirect edges
662 are discovered in the process, add them to NEW_EDGES, unless it is NULL. */
665 cgraph_decide_recursive_inlining (struct cgraph_node
*node
,
666 VEC (cgraph_edge_p
, heap
) *new_edges
)
668 int limit
= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE_AUTO
);
669 int max_depth
= PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH_AUTO
);
670 int probability
= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY
);
672 struct cgraph_edge
*e
;
673 struct cgraph_node
*master_clone
, *next
;
677 if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION (node
->decl
))
678 || (!flag_inline_functions
&& !DECL_DECLARED_INLINE_P (node
->decl
)))
681 if (DECL_DECLARED_INLINE_P (node
->decl
))
683 limit
= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE
);
684 max_depth
= PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH
);
687 /* Make sure that function is small enough to be considered for inlining. */
689 || cgraph_estimate_size_after_inlining (1, node
, node
) >= limit
)
691 heap
= fibheap_new ();
692 lookup_recursive_calls (node
, node
, heap
);
693 if (fibheap_empty (heap
))
695 fibheap_delete (heap
);
701 " Performing recursive inlining on %s\n",
702 cgraph_node_name (node
));
704 /* We need original clone to copy around. */
705 master_clone
= cgraph_clone_node (node
, node
->count
, CGRAPH_FREQ_BASE
, 1, false);
706 master_clone
->needed
= true;
707 for (e
= master_clone
->callees
; e
; e
= e
->next_callee
)
708 if (!e
->inline_failed
)
709 cgraph_clone_inlined_nodes (e
, true, false);
711 /* Do the inlining and update list of recursive call during process. */
712 while (!fibheap_empty (heap
)
713 && (cgraph_estimate_size_after_inlining (1, node
, master_clone
)
716 struct cgraph_edge
*curr
717 = (struct cgraph_edge
*) fibheap_extract_min (heap
);
718 struct cgraph_node
*cnode
;
721 for (cnode
= curr
->caller
;
722 cnode
->global
.inlined_to
; cnode
= cnode
->callers
->caller
)
723 if (node
->decl
== curr
->callee
->decl
)
725 if (depth
> max_depth
)
729 " maximal depth reached\n");
735 if (!cgraph_maybe_hot_edge_p (curr
))
738 fprintf (dump_file
, " Not inlining cold call\n");
741 if (curr
->count
* 100 / node
->count
< probability
)
745 " Probability of edge is too small\n");
753 " Inlining call of depth %i", depth
);
756 fprintf (dump_file
, " called approx. %.2f times per call",
757 (double)curr
->count
/ node
->count
);
759 fprintf (dump_file
, "\n");
761 cgraph_redirect_edge_callee (curr
, master_clone
);
762 cgraph_mark_inline_edge (curr
, false);
763 if (flag_indirect_inlining
)
764 ipa_propagate_indirect_call_infos (curr
, new_edges
);
765 lookup_recursive_calls (node
, curr
->callee
, heap
);
768 if (!fibheap_empty (heap
) && dump_file
)
769 fprintf (dump_file
, " Recursive inlining growth limit met.\n");
771 fibheap_delete (heap
);
774 "\n Inlined %i times, body grown from %i to %i insns\n", n
,
775 master_clone
->global
.insns
, node
->global
.insns
);
777 /* Remove master clone we used for inlining. We rely that clones inlined
778 into master clone gets queued just before master clone so we don't
780 for (node
= cgraph_nodes
; node
!= master_clone
;
784 if (node
->global
.inlined_to
== master_clone
)
785 cgraph_remove_node (node
);
787 cgraph_remove_node (master_clone
);
788 /* FIXME: Recursive inlining actually reduces number of calls of the
789 function. At this place we should probably walk the function and
790 inline clones and compensate the counts accordingly. This probably
791 doesn't matter much in practice. */
795 /* Set inline_failed for all callers of given function to REASON. */
798 cgraph_set_inline_failed (struct cgraph_node
*node
, const char *reason
)
800 struct cgraph_edge
*e
;
803 fprintf (dump_file
, "Inlining failed: %s\n", reason
);
804 for (e
= node
->callers
; e
; e
= e
->next_caller
)
805 if (e
->inline_failed
)
806 e
->inline_failed
= reason
;
809 /* Given whole compilation unit estimate of INSNS, compute how large we can
810 allow the unit to grow. */
812 compute_max_insns (int insns
)
814 int max_insns
= insns
;
815 if (max_insns
< PARAM_VALUE (PARAM_LARGE_UNIT_INSNS
))
816 max_insns
= PARAM_VALUE (PARAM_LARGE_UNIT_INSNS
);
818 return ((HOST_WIDEST_INT
) max_insns
819 * (100 + PARAM_VALUE (PARAM_INLINE_UNIT_GROWTH
)) / 100);
822 /* Compute badness of all edges in NEW_EDGES and add them to the HEAP. */
824 add_new_edges_to_heap (fibheap_t heap
, VEC (cgraph_edge_p
, heap
) *new_edges
)
826 while (VEC_length (cgraph_edge_p
, new_edges
) > 0)
828 struct cgraph_edge
*edge
= VEC_pop (cgraph_edge_p
, new_edges
);
830 gcc_assert (!edge
->aux
);
831 edge
->aux
= fibheap_insert (heap
, cgraph_edge_badness (edge
), edge
);
836 /* We use greedy algorithm for inlining of small functions:
837 All inline candidates are put into prioritized heap based on estimated
838 growth of the overall number of instructions and then update the estimates.
840 INLINED and INLINED_CALEES are just pointers to arrays large enough
841 to be passed to cgraph_inlined_into and cgraph_inlined_callees. */
844 cgraph_decide_inlining_of_small_functions (void)
846 struct cgraph_node
*node
;
847 struct cgraph_edge
*edge
;
848 const char *failed_reason
;
849 fibheap_t heap
= fibheap_new ();
850 bitmap updated_nodes
= BITMAP_ALLOC (NULL
);
851 int min_insns
, max_insns
;
852 VEC (cgraph_edge_p
, heap
) *new_indirect_edges
= NULL
;
854 if (flag_indirect_inlining
)
855 new_indirect_edges
= VEC_alloc (cgraph_edge_p
, heap
, 8);
858 fprintf (dump_file
, "\nDeciding on smaller functions:\n");
860 /* Put all inline candidates into the heap. */
862 for (node
= cgraph_nodes
; node
; node
= node
->next
)
864 if (!node
->local
.inlinable
|| !node
->callers
865 || node
->local
.disregard_inline_limits
)
868 fprintf (dump_file
, "Considering inline candidate %s.\n", cgraph_node_name (node
));
870 node
->global
.estimated_growth
= INT_MIN
;
871 if (!cgraph_default_inline_p (node
, &failed_reason
))
873 cgraph_set_inline_failed (node
, failed_reason
);
877 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
878 if (edge
->inline_failed
)
880 gcc_assert (!edge
->aux
);
881 edge
->aux
= fibheap_insert (heap
, cgraph_edge_badness (edge
), edge
);
885 max_insns
= compute_max_insns (overall_insns
);
886 min_insns
= overall_insns
;
888 while (overall_insns
<= max_insns
889 && (edge
= (struct cgraph_edge
*) fibheap_extract_min (heap
)))
891 int old_insns
= overall_insns
;
892 struct cgraph_node
*where
;
894 cgraph_estimate_size_after_inlining (1, edge
->caller
, edge
->callee
);
895 const char *not_good
= NULL
;
897 growth
-= edge
->caller
->global
.insns
;
902 "\nConsidering %s with %i insns\n",
903 cgraph_node_name (edge
->callee
),
904 edge
->callee
->global
.insns
);
906 " to be inlined into %s\n"
907 " Estimated growth after inlined into all callees is %+i insns.\n"
908 " Estimated badness is %i, frequency %.2f.\n",
909 cgraph_node_name (edge
->caller
),
910 cgraph_estimate_growth (edge
->callee
),
911 cgraph_edge_badness (edge
),
912 edge
->frequency
/ (double)CGRAPH_FREQ_BASE
);
914 fprintf (dump_file
," Called "HOST_WIDEST_INT_PRINT_DEC
"x\n", edge
->count
);
916 gcc_assert (edge
->aux
);
918 if (!edge
->inline_failed
)
921 /* When not having profile info ready we don't weight by any way the
922 position of call in procedure itself. This means if call of
923 function A from function B seems profitable to inline, the recursive
924 call of function A in inline copy of A in B will look profitable too
925 and we end up inlining until reaching maximal function growth. This
926 is not good idea so prohibit the recursive inlining.
928 ??? When the frequencies are taken into account we might not need this
932 where
= edge
->caller
;
933 while (where
->global
.inlined_to
)
935 if (where
->decl
== edge
->callee
->decl
)
937 where
= where
->callers
->caller
;
939 if (where
->global
.inlined_to
)
942 = (edge
->callee
->local
.disregard_inline_limits
? N_("recursive inlining") : "");
944 fprintf (dump_file
, " inline_failed:Recursive inlining performed only for function itself.\n");
949 if (!cgraph_maybe_hot_edge_p (edge
))
950 not_good
= N_("call is unlikely and code size would grow");
951 if (!flag_inline_functions
952 && !DECL_DECLARED_INLINE_P (edge
->callee
->decl
))
953 not_good
= N_("function not declared inline and code size would grow");
954 if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION(edge
->caller
->decl
)))
955 not_good
= N_("optimizing for size and code size would grow");
956 if (not_good
&& growth
> 0 && cgraph_estimate_growth (edge
->callee
) > 0)
958 if (!cgraph_recursive_inlining_p (edge
->caller
, edge
->callee
,
959 &edge
->inline_failed
))
961 edge
->inline_failed
= not_good
;
963 fprintf (dump_file
, " inline_failed:%s.\n", edge
->inline_failed
);
967 if (!cgraph_default_inline_p (edge
->callee
, &edge
->inline_failed
))
969 if (!cgraph_recursive_inlining_p (edge
->caller
, edge
->callee
,
970 &edge
->inline_failed
))
973 fprintf (dump_file
, " inline_failed:%s.\n", edge
->inline_failed
);
977 if (!tree_can_inline_p (edge
->caller
->decl
, edge
->callee
->decl
))
979 gimple_call_set_cannot_inline (edge
->call_stmt
, true);
980 edge
->inline_failed
= N_("target specific option mismatch");
982 fprintf (dump_file
, " inline_failed:%s.\n", edge
->inline_failed
);
985 if (cgraph_recursive_inlining_p (edge
->caller
, edge
->callee
,
986 &edge
->inline_failed
))
988 where
= edge
->caller
;
989 if (where
->global
.inlined_to
)
990 where
= where
->global
.inlined_to
;
991 if (!cgraph_decide_recursive_inlining (where
, new_indirect_edges
))
993 if (flag_indirect_inlining
)
994 add_new_edges_to_heap (heap
, new_indirect_edges
);
995 update_callee_keys (heap
, where
, updated_nodes
);
999 struct cgraph_node
*callee
;
1000 if (gimple_call_cannot_inline_p (edge
->call_stmt
)
1001 || !cgraph_check_inline_limits (edge
->caller
, edge
->callee
,
1002 &edge
->inline_failed
, true))
1005 fprintf (dump_file
, " Not inlining into %s:%s.\n",
1006 cgraph_node_name (edge
->caller
), edge
->inline_failed
);
1009 callee
= edge
->callee
;
1010 cgraph_mark_inline_edge (edge
, true);
1011 if (flag_indirect_inlining
)
1013 ipa_propagate_indirect_call_infos (edge
, new_indirect_edges
);
1014 add_new_edges_to_heap (heap
, new_indirect_edges
);
1016 update_callee_keys (heap
, callee
, updated_nodes
);
1018 where
= edge
->caller
;
1019 if (where
->global
.inlined_to
)
1020 where
= where
->global
.inlined_to
;
1022 /* Our profitability metric can depend on local properties
1023 such as number of inlinable calls and size of the function body.
1024 After inlining these properties might change for the function we
1025 inlined into (since it's body size changed) and for the functions
1026 called by function we inlined (since number of it inlinable callers
1028 update_caller_keys (heap
, where
, updated_nodes
);
1029 bitmap_clear (updated_nodes
);
1034 " Inlined into %s which now has %i insns,"
1035 "net change of %+i insns.\n",
1036 cgraph_node_name (edge
->caller
),
1037 edge
->caller
->global
.insns
,
1038 overall_insns
- old_insns
);
1040 if (min_insns
> overall_insns
)
1042 min_insns
= overall_insns
;
1043 max_insns
= compute_max_insns (min_insns
);
1046 fprintf (dump_file
, "New minimal insns reached: %i\n", min_insns
);
1049 while ((edge
= (struct cgraph_edge
*) fibheap_extract_min (heap
)) != NULL
)
1051 gcc_assert (edge
->aux
);
1053 if (!edge
->callee
->local
.disregard_inline_limits
&& edge
->inline_failed
1054 && !cgraph_recursive_inlining_p (edge
->caller
, edge
->callee
,
1055 &edge
->inline_failed
))
1056 edge
->inline_failed
= N_("--param inline-unit-growth limit reached");
1059 if (new_indirect_edges
)
1060 VEC_free (cgraph_edge_p
, heap
, new_indirect_edges
);
1061 fibheap_delete (heap
);
1062 BITMAP_FREE (updated_nodes
);
1065 /* Decide on the inlining. We do so in the topological order to avoid
1066 expenses on updating data structures. */
1069 cgraph_decide_inlining (void)
1071 struct cgraph_node
*node
;
1073 struct cgraph_node
**order
=
1074 XCNEWVEC (struct cgraph_node
*, cgraph_n_nodes
);
1077 int initial_insns
= 0;
1079 cgraph_remove_function_insertion_hook (function_insertion_hook_holder
);
1082 for (node
= cgraph_nodes
; node
; node
= node
->next
)
1083 if (node
->analyzed
&& (node
->needed
|| node
->reachable
))
1085 struct cgraph_edge
*e
;
1087 initial_insns
+= inline_summary (node
)->self_insns
;
1088 gcc_assert (inline_summary (node
)->self_insns
== node
->global
.insns
);
1089 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1090 if (max_count
< e
->count
)
1091 max_count
= e
->count
;
1093 overall_insns
= initial_insns
;
1094 gcc_assert (!max_count
|| (profile_info
&& flag_branch_probabilities
));
1096 nnodes
= cgraph_postorder (order
);
1100 "\nDeciding on inlining. Starting with %i insns.\n",
1103 for (node
= cgraph_nodes
; node
; node
= node
->next
)
1107 fprintf (dump_file
, "\nInlining always_inline functions:\n");
1109 /* In the first pass mark all always_inline edges. Do this with a priority
1110 so none of our later choices will make this impossible. */
1111 for (i
= nnodes
- 1; i
>= 0; i
--)
1113 struct cgraph_edge
*e
, *next
;
1117 /* Handle nodes to be flattened, but don't update overall unit size. */
1118 if (lookup_attribute ("flatten", DECL_ATTRIBUTES (node
->decl
)) != NULL
)
1122 "Flattening %s\n", cgraph_node_name (node
));
1123 cgraph_decide_inlining_incrementally (node
, INLINE_ALL
, 0);
1126 if (!node
->local
.disregard_inline_limits
)
1130 "\nConsidering %s %i insns (always inline)\n",
1131 cgraph_node_name (node
), node
->global
.insns
);
1132 old_insns
= overall_insns
;
1133 for (e
= node
->callers
; e
; e
= next
)
1135 next
= e
->next_caller
;
1136 if (!e
->inline_failed
|| gimple_call_cannot_inline_p (e
->call_stmt
))
1138 if (cgraph_recursive_inlining_p (e
->caller
, e
->callee
,
1141 if (!tree_can_inline_p (e
->caller
->decl
, e
->callee
->decl
))
1143 gimple_call_set_cannot_inline (e
->call_stmt
, true);
1146 cgraph_mark_inline_edge (e
, true);
1147 if (flag_indirect_inlining
)
1148 ipa_propagate_indirect_call_infos (e
, NULL
);
1151 " Inlined into %s which now has %i insns.\n",
1152 cgraph_node_name (e
->caller
),
1153 e
->caller
->global
.insns
);
1155 /* Inlining self recursive function might introduce new calls to
1156 themselves we didn't see in the loop above. Fill in the proper
1157 reason why inline failed. */
1158 for (e
= node
->callers
; e
; e
= e
->next_caller
)
1159 if (e
->inline_failed
)
1160 e
->inline_failed
= N_("recursive inlining");
1163 " Inlined for a net change of %+i insns.\n",
1164 overall_insns
- old_insns
);
1167 cgraph_decide_inlining_of_small_functions ();
1169 /* After this point, any edge discovery performed by indirect inlining is no
1170 good so let's give up. */
1171 if (flag_indirect_inlining
)
1172 free_all_ipa_structures_after_iinln ();
1174 if (flag_inline_functions_called_once
)
1177 fprintf (dump_file
, "\nDeciding on functions called once:\n");
1179 /* And finally decide what functions are called once. */
1180 for (i
= nnodes
- 1; i
>= 0; i
--)
1185 && !node
->callers
->next_caller
1187 && node
->local
.inlinable
1188 && node
->callers
->inline_failed
1189 && !gimple_call_cannot_inline_p (node
->callers
->call_stmt
)
1190 && !DECL_EXTERNAL (node
->decl
)
1191 && !DECL_COMDAT (node
->decl
))
1196 "\nConsidering %s %i insns.\n",
1197 cgraph_node_name (node
), node
->global
.insns
);
1199 " Called once from %s %i insns.\n",
1200 cgraph_node_name (node
->callers
->caller
),
1201 node
->callers
->caller
->global
.insns
);
1204 old_insns
= overall_insns
;
1206 if (cgraph_check_inline_limits (node
->callers
->caller
, node
,
1209 cgraph_mark_inline (node
->callers
);
1212 " Inlined into %s which now has %i insns"
1213 " for a net change of %+i insns.\n",
1214 cgraph_node_name (node
->callers
->caller
),
1215 node
->callers
->caller
->global
.insns
,
1216 overall_insns
- old_insns
);
1222 " Inline limit reached, not inlined.\n");
1230 "\nInlined %i calls, eliminated %i functions, "
1231 "%i insns turned to %i insns.\n\n",
1232 ncalls_inlined
, nfunctions_inlined
, initial_insns
,
1238 /* Try to inline edge E from incremental inliner. MODE specifies mode
1241 We are detecting cycles by storing mode of inliner into cgraph_node last
1242 time we visited it in the recursion. In general when mode is set, we have
1243 recursive inlining, but as an special case, we want to try harder inline
1244 ALWAYS_INLINE functions: consider callgraph a->b->c->b, with a being
1245 flatten, b being always inline. Flattening 'a' will collapse
1246 a->b->c before hitting cycle. To accommodate always inline, we however
1247 need to inline a->b->c->b.
1249 So after hitting cycle first time, we switch into ALWAYS_INLINE mode and
1250 stop inlining only after hitting ALWAYS_INLINE in ALWAY_INLINE mode. */
1252 try_inline (struct cgraph_edge
*e
, enum inlining_mode mode
, int depth
)
1254 struct cgraph_node
*callee
= e
->callee
;
1255 enum inlining_mode callee_mode
= (enum inlining_mode
) (size_t) callee
->aux
;
1256 bool always_inline
= e
->callee
->local
.disregard_inline_limits
;
1258 /* We've hit cycle? */
1261 /* It is first time we see it and we are not in ALWAY_INLINE only
1262 mode yet. and the function in question is always_inline. */
1263 if (always_inline
&& mode
!= INLINE_ALWAYS_INLINE
)
1267 indent_to (dump_file
, depth
);
1269 "Hit cycle in %s, switching to always inline only.\n",
1270 cgraph_node_name (callee
));
1272 mode
= INLINE_ALWAYS_INLINE
;
1274 /* Otherwise it is time to give up. */
1279 indent_to (dump_file
, depth
);
1281 "Not inlining %s into %s to avoid cycle.\n",
1282 cgraph_node_name (callee
),
1283 cgraph_node_name (e
->caller
));
1285 e
->inline_failed
= (e
->callee
->local
.disregard_inline_limits
1286 ? N_("recursive inlining") : "");
1291 callee
->aux
= (void *)(size_t) mode
;
1294 indent_to (dump_file
, depth
);
1295 fprintf (dump_file
, " Inlining %s into %s.\n",
1296 cgraph_node_name (e
->callee
),
1297 cgraph_node_name (e
->caller
));
1299 if (e
->inline_failed
)
1300 cgraph_mark_inline (e
);
1302 /* In order to fully inline always_inline functions, we need to
1303 recurse here, since the inlined functions might not be processed by
1304 incremental inlining at all yet.
1306 Also flattening needs to be done recursively. */
1308 if (mode
== INLINE_ALL
|| always_inline
)
1309 cgraph_decide_inlining_incrementally (e
->callee
, mode
, depth
+ 1);
1310 callee
->aux
= (void *)(size_t) callee_mode
;
1314 /* Decide on the inlining. We do so in the topological order to avoid
1315 expenses on updating data structures.
1316 DEPTH is depth of recursion, used only for debug output. */
1319 cgraph_decide_inlining_incrementally (struct cgraph_node
*node
,
1320 enum inlining_mode mode
,
1323 struct cgraph_edge
*e
;
1324 bool inlined
= false;
1325 const char *failed_reason
;
1326 enum inlining_mode old_mode
;
1328 #ifdef ENABLE_CHECKING
1329 verify_cgraph_node (node
);
1332 old_mode
= (enum inlining_mode
) (size_t)node
->aux
;
1334 if (mode
!= INLINE_ALWAYS_INLINE
1335 && lookup_attribute ("flatten", DECL_ATTRIBUTES (node
->decl
)) != NULL
)
1339 indent_to (dump_file
, depth
);
1340 fprintf (dump_file
, "Flattening %s\n", cgraph_node_name (node
));
1345 node
->aux
= (void *)(size_t) mode
;
1347 /* First of all look for always inline functions. */
1348 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1350 if (!e
->callee
->local
.disregard_inline_limits
1351 && (mode
!= INLINE_ALL
|| !e
->callee
->local
.inlinable
))
1353 if (gimple_call_cannot_inline_p (e
->call_stmt
))
1355 /* When the edge is already inlined, we just need to recurse into
1356 it in order to fully flatten the leaves. */
1357 if (!e
->inline_failed
&& mode
== INLINE_ALL
)
1359 inlined
|= try_inline (e
, mode
, depth
);
1364 indent_to (dump_file
, depth
);
1366 "Considering to always inline inline candidate %s.\n",
1367 cgraph_node_name (e
->callee
));
1369 if (cgraph_recursive_inlining_p (node
, e
->callee
, &e
->inline_failed
))
1373 indent_to (dump_file
, depth
);
1374 fprintf (dump_file
, "Not inlining: recursive call.\n");
1378 if (!tree_can_inline_p (node
->decl
, e
->callee
->decl
))
1380 gimple_call_set_cannot_inline (e
->call_stmt
, true);
1383 indent_to (dump_file
, depth
);
1385 "Not inlining: Target specific option mismatch.\n");
1389 if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node
->decl
))
1390 != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e
->callee
->decl
)))
1394 indent_to (dump_file
, depth
);
1395 fprintf (dump_file
, "Not inlining: SSA form does not match.\n");
1399 if (!gimple_body (e
->callee
->decl
) && !e
->callee
->inline_decl
)
1403 indent_to (dump_file
, depth
);
1405 "Not inlining: Function body no longer available.\n");
1409 inlined
|= try_inline (e
, mode
, depth
);
1412 /* Now do the automatic inlining. */
1413 if (mode
!= INLINE_ALL
&& mode
!= INLINE_ALWAYS_INLINE
)
1414 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1416 if (!e
->callee
->local
.inlinable
1417 || !e
->inline_failed
1418 || e
->callee
->local
.disregard_inline_limits
)
1421 fprintf (dump_file
, "Considering inline candidate %s.\n",
1422 cgraph_node_name (e
->callee
));
1423 if (cgraph_recursive_inlining_p (node
, e
->callee
, &e
->inline_failed
))
1427 indent_to (dump_file
, depth
);
1428 fprintf (dump_file
, "Not inlining: recursive call.\n");
1432 if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node
->decl
))
1433 != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e
->callee
->decl
)))
1437 indent_to (dump_file
, depth
);
1438 fprintf (dump_file
, "Not inlining: SSA form does not match.\n");
1442 /* When the function body would grow and inlining the function won't
1443 eliminate the need for offline copy of the function, don't inline.
1445 if ((mode
== INLINE_SIZE
1446 || (!flag_inline_functions
1447 && !DECL_DECLARED_INLINE_P (e
->callee
->decl
)))
1448 && (cgraph_estimate_size_after_inlining (1, e
->caller
, e
->callee
)
1449 > e
->caller
->global
.insns
)
1450 && cgraph_estimate_growth (e
->callee
) > 0)
1454 indent_to (dump_file
, depth
);
1456 "Not inlining: code size would grow by %i insns.\n",
1457 cgraph_estimate_size_after_inlining (1, e
->caller
,
1459 - e
->caller
->global
.insns
);
1463 if (!cgraph_check_inline_limits (node
, e
->callee
, &e
->inline_failed
,
1465 || gimple_call_cannot_inline_p (e
->call_stmt
))
1469 indent_to (dump_file
, depth
);
1470 fprintf (dump_file
, "Not inlining: %s.\n", e
->inline_failed
);
1474 if (!gimple_body (e
->callee
->decl
) && !e
->callee
->inline_decl
)
1478 indent_to (dump_file
, depth
);
1480 "Not inlining: Function body no longer available.\n");
1484 if (!tree_can_inline_p (node
->decl
, e
->callee
->decl
))
1486 gimple_call_set_cannot_inline (e
->call_stmt
, true);
1489 indent_to (dump_file
, depth
);
1491 "Not inlining: Target specific option mismatch.\n");
1495 if (cgraph_default_inline_p (e
->callee
, &failed_reason
))
1496 inlined
|= try_inline (e
, mode
, depth
);
1498 node
->aux
= (void *)(size_t) old_mode
;
1502 /* Because inlining might remove no-longer reachable nodes, we need to
1503 keep the array visible to garbage collector to avoid reading collected
1506 static GTY ((length ("nnodes"))) struct cgraph_node
**order
;
1508 /* Do inlining of small functions. Doing so early helps profiling and other
1509 passes to be somewhat more effective and avoids some code duplication in
1510 later real inlining pass for testcases with very many function calls. */
1512 cgraph_early_inlining (void)
1514 struct cgraph_node
*node
= cgraph_node (current_function_decl
);
1515 unsigned int todo
= 0;
1517 if (sorrycount
|| errorcount
)
1519 if (cgraph_decide_inlining_incrementally (node
, INLINE_SIZE
, 0))
1521 timevar_push (TV_INTEGRATION
);
1522 todo
= optimize_inline_calls (current_function_decl
);
1523 timevar_pop (TV_INTEGRATION
);
1528 /* When inlining shall be performed. */
1530 cgraph_gate_early_inlining (void)
1532 return flag_early_inlining
;
1535 struct gimple_opt_pass pass_early_inline
=
1539 "einline", /* name */
1540 cgraph_gate_early_inlining
, /* gate */
1541 cgraph_early_inlining
, /* execute */
1544 0, /* static_pass_number */
1545 TV_INLINE_HEURISTICS
, /* tv_id */
1546 0, /* properties_required */
1547 PROP_cfg
, /* properties_provided */
1548 0, /* properties_destroyed */
1549 0, /* todo_flags_start */
1550 TODO_dump_func
/* todo_flags_finish */
1554 /* When inlining shall be performed. */
1556 cgraph_gate_ipa_early_inlining (void)
1558 return (flag_early_inlining
1559 && (flag_branch_probabilities
|| flag_test_coverage
1560 || profile_arc_flag
));
1563 /* IPA pass wrapper for early inlining pass. We need to run early inlining
1564 before tree profiling so we have stand alone IPA pass for doing so. */
1565 struct simple_ipa_opt_pass pass_ipa_early_inline
=
1569 "einline_ipa", /* name */
1570 cgraph_gate_ipa_early_inlining
, /* gate */
1574 0, /* static_pass_number */
1575 TV_INLINE_HEURISTICS
, /* tv_id */
1576 0, /* properties_required */
1577 PROP_cfg
, /* properties_provided */
1578 0, /* properties_destroyed */
1579 0, /* todo_flags_start */
1580 TODO_dump_cgraph
/* todo_flags_finish */
1584 /* Compute parameters of functions used by inliner. */
1586 compute_inline_parameters (struct cgraph_node
*node
)
1588 gcc_assert (!node
->global
.inlined_to
);
1589 inline_summary (node
)->estimated_self_stack_size
1590 = estimated_stack_frame_size ();
1591 node
->global
.estimated_stack_size
1592 = inline_summary (node
)->estimated_self_stack_size
;
1593 node
->global
.stack_frame_offset
= 0;
1594 node
->local
.inlinable
= tree_inlinable_function_p (current_function_decl
);
1595 inline_summary (node
)->self_insns
1596 = estimate_num_insns_fn (current_function_decl
, &eni_inlining_weights
);
1597 if (node
->local
.inlinable
&& !node
->local
.disregard_inline_limits
)
1598 node
->local
.disregard_inline_limits
1599 = DECL_DISREGARD_INLINE_LIMITS (current_function_decl
);
1600 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
1601 node
->global
.insns
= inline_summary (node
)->self_insns
;
1606 /* Compute parameters of functions used by inliner using
1607 current_function_decl. */
1609 compute_inline_parameters_for_current (void)
1611 compute_inline_parameters (cgraph_node (current_function_decl
));
1615 struct gimple_opt_pass pass_inline_parameters
=
1621 compute_inline_parameters_for_current
,/* execute */
1624 0, /* static_pass_number */
1625 TV_INLINE_HEURISTICS
, /* tv_id */
1626 0, /* properties_required */
1627 PROP_cfg
, /* properties_provided */
1628 0, /* properties_destroyed */
1629 0, /* todo_flags_start */
1630 0 /* todo_flags_finish */
1634 /* This function performs intraprocedural analyzis in NODE that is required to
1635 inline indirect calls. */
1637 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
1639 struct cgraph_edge
*cs
;
1643 ipa_count_formal_params (node
);
1644 ipa_create_param_decls_array (node
);
1645 ipa_detect_param_modifications (node
);
1647 ipa_analyze_params_uses (node
);
1650 ipa_print_node_param_flags (dump_file
, node
);
1653 for (cs
= node
->callees
; cs
; cs
= cs
->next_callee
)
1655 ipa_count_arguments (cs
);
1656 ipa_compute_jump_functions (cs
);
1660 ipa_print_node_jump_functions (dump_file
, node
);
1663 /* Note function body size. */
1665 analyze_function (struct cgraph_node
*node
)
1667 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
1668 current_function_decl
= node
->decl
;
1670 compute_inline_parameters (node
);
1671 if (flag_indirect_inlining
)
1672 inline_indirect_intraprocedural_analysis (node
);
1674 current_function_decl
= NULL
;
1678 /* Called when new function is inserted to callgraph late. */
1680 add_new_function (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
1682 analyze_function (node
);
1685 /* Note function body size. */
1687 inline_generate_summary (void)
1689 struct cgraph_node
*node
;
1691 function_insertion_hook_holder
=
1692 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
1694 if (flag_indirect_inlining
)
1696 ipa_register_cgraph_hooks ();
1697 ipa_check_create_node_params ();
1698 ipa_check_create_edge_args ();
1701 for (node
= cgraph_nodes
; node
; node
= node
->next
)
1703 analyze_function (node
);
1708 /* Apply inline plan to function. */
1710 inline_transform (struct cgraph_node
*node
)
1712 unsigned int todo
= 0;
1713 struct cgraph_edge
*e
;
1715 /* We might need the body of this function so that we can expand
1716 it inline somewhere else. */
1717 if (cgraph_preserve_function_body_p (current_function_decl
))
1718 save_inline_function_body (node
);
1720 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1721 if (!e
->inline_failed
|| warn_inline
)
1726 timevar_push (TV_INTEGRATION
);
1727 todo
= optimize_inline_calls (current_function_decl
);
1728 timevar_pop (TV_INTEGRATION
);
1730 return todo
| execute_fixup_cfg ();
1733 struct ipa_opt_pass pass_ipa_inline
=
1737 "inline", /* name */
1739 cgraph_decide_inlining
, /* execute */
1742 0, /* static_pass_number */
1743 TV_INLINE_HEURISTICS
, /* tv_id */
1744 0, /* properties_required */
1745 PROP_cfg
, /* properties_provided */
1746 0, /* properties_destroyed */
1747 TODO_remove_functions
, /* todo_flags_finish */
1748 TODO_dump_cgraph
| TODO_dump_func
1749 | TODO_remove_functions
/* todo_flags_finish */
1751 inline_generate_summary
, /* generate_summary */
1752 NULL
, /* write_summary */
1753 NULL
, /* read_summary */
1754 NULL
, /* function_read_summary */
1756 inline_transform
, /* function_transform */
1757 NULL
, /* variable_transform */
1761 #include "gt-ipa-inline.h"