1 /* Inlining decision heuristics.
2 Copyright (C) 2003, 2004, 2007, 2008 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Inlining decision heuristics
23 We separate inlining decisions from the inliner itself and store it
24 inside callgraph as so called inline plan. Refer to cgraph.c
25 documentation about particular representation of inline plans in the
28 There are three major parts of this file:
30 cgraph_mark_inline implementation
32 This function allows to mark given call inline and performs necessary
33 modifications of cgraph (production of the clones and updating overall
36 inlining heuristics limits
38 These functions allow to check that particular inlining is allowed
39 by the limits specified by user (allowed function growth, overall unit
44 This is implementation of IPA pass aiming to get as much of benefit
45 from inlining obeying the limits checked above.
47 The implementation of particular heuristics is separated from
48 the rest of code to make it easier to replace it with more complicated
49 implementation in the future. The rest of inlining code acts as a
50 library aimed to modify the callgraph and verify that the parameters
51 on code size growth fits.
53 To mark given call inline, use cgraph_mark_inline function, the
54 verification is performed by cgraph_default_inline_p and
55 cgraph_check_inline_limits.
57 The heuristics implements simple knapsack style algorithm ordering
58 all functions by their "profitability" (estimated by code size growth)
59 and inlining them in priority order.
61 cgraph_decide_inlining implements heuristics taking whole callgraph
62 into account, while cgraph_decide_inlining_incrementally considers
63 only one function at a time and is used by early inliner.
65 The inliner itself is split into several passes:
67 pass_inline_parameters
69 This pass computes local properties of functions that are used by inliner:
70 estimated function body size, whether function is inlinable at all and
71 stack frame consumption.
73 Before executing any of inliner passes, this local pass has to be applied
74 to each function in the callgraph (ie run as subpass of some earlier
75 IPA pass). The results are made out of date by any optimization applied
80 Simple local inlining pass inlining callees into current function. This
81 pass makes no global whole compilation unit analysis and this when allowed
82 to do inlining expanding code size it might result in unbounded growth of
85 The pass is run during conversion into SSA form. Only functions already
86 converted into SSA form are inlined, so the conversion must happen in
87 topological order on the callgraph (that is maintained by pass manager).
88 The functions after inlining are early optimized so the early inliner sees
89 unoptimized function itself, but all considered callees are already
90 optimized allowing it to unfold abstraction penalty on C++ effectively and
93 pass_ipa_early_inlining
95 With profiling, the early inlining is also necessary to reduce
96 instrumentation costs on program with high abstraction penalty (doing
97 many redundant calls). This can't happen in parallel with early
98 optimization and profile instrumentation, because we would end up
99 re-instrumenting already instrumented function bodies we brought in via
102 To avoid this, this pass is executed as IPA pass before profiling. It is
103 simple wrapper to pass_early_inlining and ensures first inlining.
107 This is the main pass implementing simple greedy algorithm to do inlining
108 of small functions that results in overall growth of compilation unit and
109 inlining of functions called once. The pass compute just so called inline
110 plan (representation of inlining to be done in callgraph) and unlike early
111 inlining it is not performing the inlining itself.
115 This pass performs actual inlining according to pass_ipa_inline on given
116 function. Possible the function body before inlining is saved when it is
117 needed for further inlining later.
122 #include "coretypes.h"
125 #include "tree-inline.h"
126 #include "langhooks.h"
129 #include "diagnostic.h"
134 #include "tree-pass.h"
136 #include "coverage.h"
138 #include "tree-flow.h"
140 #include "ipa-prop.h"
142 /* Mode incremental inliner operate on:
144 In ALWAYS_INLINE only functions marked
145 always_inline are inlined. This mode is used after detecting cycle during
148 In SIZE mode, only functions that reduce function body size after inlining
149 are inlined, this is used during early inlining.
151 in ALL mode, everything is inlined. This is used during flattening. */
154 INLINE_ALWAYS_INLINE
,
159 cgraph_decide_inlining_incrementally (struct cgraph_node
*, enum inlining_mode
,
163 /* Statistics we collect about inlining algorithm. */
164 static int ncalls_inlined
;
165 static int nfunctions_inlined
;
166 static int overall_insns
;
167 static gcov_type max_count
;
169 /* Holders of ipa cgraph hooks: */
170 static struct cgraph_node_hook_list
*function_insertion_hook_holder
;
172 static inline struct inline_summary
*
173 inline_summary (struct cgraph_node
*node
)
175 return &node
->local
.inline_summary
;
178 /* Estimate size of the function after inlining WHAT into TO. */
181 cgraph_estimate_size_after_inlining (int times
, struct cgraph_node
*to
,
182 struct cgraph_node
*what
)
185 tree fndecl
= what
->decl
, arg
;
186 int call_insns
= PARAM_VALUE (PARAM_INLINE_CALL_COST
);
188 for (arg
= DECL_ARGUMENTS (fndecl
); arg
; arg
= TREE_CHAIN (arg
))
189 call_insns
+= estimate_move_cost (TREE_TYPE (arg
));
190 size
= (what
->global
.insns
- call_insns
) * times
+ to
->global
.insns
;
191 gcc_assert (size
>= 0);
195 /* E is expected to be an edge being inlined. Clone destination node of
196 the edge and redirect it to the new clone.
197 DUPLICATE is used for bookkeeping on whether we are actually creating new
198 clones or re-using node originally representing out-of-line function call.
201 cgraph_clone_inlined_nodes (struct cgraph_edge
*e
, bool duplicate
,
202 bool update_original
)
208 /* We may eliminate the need for out-of-line copy to be output.
209 In that case just go ahead and re-use it. */
210 if (!e
->callee
->callers
->next_caller
211 && !e
->callee
->needed
212 && !cgraph_new_nodes
)
214 gcc_assert (!e
->callee
->global
.inlined_to
);
215 if (e
->callee
->analyzed
)
216 overall_insns
-= e
->callee
->global
.insns
, nfunctions_inlined
++;
221 struct cgraph_node
*n
;
222 n
= cgraph_clone_node (e
->callee
, e
->count
, e
->frequency
, e
->loop_nest
,
224 cgraph_redirect_edge_callee (e
, n
);
228 if (e
->caller
->global
.inlined_to
)
229 e
->callee
->global
.inlined_to
= e
->caller
->global
.inlined_to
;
231 e
->callee
->global
.inlined_to
= e
->caller
;
232 e
->callee
->global
.stack_frame_offset
233 = e
->caller
->global
.stack_frame_offset
234 + inline_summary (e
->caller
)->estimated_self_stack_size
;
235 peak
= e
->callee
->global
.stack_frame_offset
236 + inline_summary (e
->callee
)->estimated_self_stack_size
;
237 if (e
->callee
->global
.inlined_to
->global
.estimated_stack_size
< peak
)
238 e
->callee
->global
.inlined_to
->global
.estimated_stack_size
= peak
;
240 /* Recursively clone all bodies. */
241 for (e
= e
->callee
->callees
; e
; e
= e
->next_callee
)
242 if (!e
->inline_failed
)
243 cgraph_clone_inlined_nodes (e
, duplicate
, update_original
);
246 /* Mark edge E as inlined and update callgraph accordingly.
247 UPDATE_ORIGINAL specify whether profile of original function should be
251 cgraph_mark_inline_edge (struct cgraph_edge
*e
, bool update_original
)
253 int old_insns
= 0, new_insns
= 0;
254 struct cgraph_node
*to
= NULL
, *what
;
256 if (e
->callee
->inline_decl
)
257 cgraph_redirect_edge_callee (e
, cgraph_node (e
->callee
->inline_decl
));
259 gcc_assert (e
->inline_failed
);
260 e
->inline_failed
= NULL
;
262 if (!e
->callee
->global
.inlined
)
263 DECL_POSSIBLY_INLINED (e
->callee
->decl
) = true;
264 e
->callee
->global
.inlined
= true;
266 cgraph_clone_inlined_nodes (e
, true, update_original
);
270 /* Now update size of caller and all functions caller is inlined into. */
271 for (;e
&& !e
->inline_failed
; e
= e
->caller
->callers
)
273 old_insns
= e
->caller
->global
.insns
;
274 new_insns
= cgraph_estimate_size_after_inlining (1, e
->caller
,
276 gcc_assert (new_insns
>= 0);
278 to
->global
.insns
= new_insns
;
280 gcc_assert (what
->global
.inlined_to
== to
);
281 if (new_insns
> old_insns
)
282 overall_insns
+= new_insns
- old_insns
;
286 /* Mark all calls of EDGE->CALLEE inlined into EDGE->CALLER.
287 Return following unredirected edge in the list of callers
290 static struct cgraph_edge
*
291 cgraph_mark_inline (struct cgraph_edge
*edge
)
293 struct cgraph_node
*to
= edge
->caller
;
294 struct cgraph_node
*what
= edge
->callee
;
295 struct cgraph_edge
*e
, *next
;
297 gcc_assert (!gimple_call_cannot_inline_p (edge
->call_stmt
));
298 /* Look for all calls, mark them inline and clone recursively
299 all inlined functions. */
300 for (e
= what
->callers
; e
; e
= next
)
302 next
= e
->next_caller
;
303 if (e
->caller
== to
&& e
->inline_failed
)
305 cgraph_mark_inline_edge (e
, true);
314 /* Estimate the growth caused by inlining NODE into all callees. */
317 cgraph_estimate_growth (struct cgraph_node
*node
)
320 struct cgraph_edge
*e
;
321 bool self_recursive
= false;
323 if (node
->global
.estimated_growth
!= INT_MIN
)
324 return node
->global
.estimated_growth
;
326 for (e
= node
->callers
; e
; e
= e
->next_caller
)
328 if (e
->caller
== node
)
329 self_recursive
= true;
330 if (e
->inline_failed
)
331 growth
+= (cgraph_estimate_size_after_inlining (1, e
->caller
, node
)
332 - e
->caller
->global
.insns
);
335 /* ??? Wrong for non-trivially self recursive functions or cases where
336 we decide to not inline for different reasons, but it is not big deal
337 as in that case we will keep the body around, but we will also avoid
339 if (!node
->needed
&& !DECL_EXTERNAL (node
->decl
) && !self_recursive
)
340 growth
-= node
->global
.insns
;
342 node
->global
.estimated_growth
= growth
;
346 /* Return false when inlining WHAT into TO is not good idea
347 as it would cause too large growth of function bodies.
348 When ONE_ONLY is true, assume that only one call site is going
349 to be inlined, otherwise figure out how many call sites in
350 TO calls WHAT and verify that all can be inlined.
354 cgraph_check_inline_limits (struct cgraph_node
*to
, struct cgraph_node
*what
,
355 const char **reason
, bool one_only
)
358 struct cgraph_edge
*e
;
361 HOST_WIDE_INT stack_size_limit
, inlined_stack
;
366 for (e
= to
->callees
; e
; e
= e
->next_callee
)
367 if (e
->callee
== what
)
370 if (to
->global
.inlined_to
)
371 to
= to
->global
.inlined_to
;
373 /* When inlining large function body called once into small function,
374 take the inlined function as base for limiting the growth. */
375 if (inline_summary (to
)->self_insns
> inline_summary(what
)->self_insns
)
376 limit
= inline_summary (to
)->self_insns
;
378 limit
= inline_summary (what
)->self_insns
;
380 limit
+= limit
* PARAM_VALUE (PARAM_LARGE_FUNCTION_GROWTH
) / 100;
382 /* Check the size after inlining against the function limits. But allow
383 the function to shrink if it went over the limits by forced inlining. */
384 newsize
= cgraph_estimate_size_after_inlining (times
, to
, what
);
385 if (newsize
>= to
->global
.insns
386 && newsize
> PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS
)
390 *reason
= N_("--param large-function-growth limit reached");
394 stack_size_limit
= inline_summary (to
)->estimated_self_stack_size
;
396 stack_size_limit
+= stack_size_limit
* PARAM_VALUE (PARAM_STACK_FRAME_GROWTH
) / 100;
398 inlined_stack
= (to
->global
.stack_frame_offset
399 + inline_summary (to
)->estimated_self_stack_size
400 + what
->global
.estimated_stack_size
);
401 if (inlined_stack
> stack_size_limit
402 && inlined_stack
> PARAM_VALUE (PARAM_LARGE_STACK_FRAME
))
405 *reason
= N_("--param large-stack-frame-growth limit reached");
411 /* Return true when function N is small enough to be inlined. */
414 cgraph_default_inline_p (struct cgraph_node
*n
, const char **reason
)
419 decl
= n
->inline_decl
;
420 if (!flag_inline_small_functions
&& !DECL_DECLARED_INLINE_P (decl
))
423 *reason
= N_("function not inline candidate");
427 if (!DECL_STRUCT_FUNCTION (decl
)->cfg
)
430 *reason
= N_("function body not available");
434 if (DECL_DECLARED_INLINE_P (decl
))
436 if (n
->global
.insns
>= MAX_INLINE_INSNS_SINGLE
)
439 *reason
= N_("--param max-inline-insns-single limit reached");
445 if (n
->global
.insns
>= MAX_INLINE_INSNS_AUTO
)
448 *reason
= N_("--param max-inline-insns-auto limit reached");
456 /* Return true when inlining WHAT would create recursive inlining.
457 We call recursive inlining all cases where same function appears more than
458 once in the single recursion nest path in the inline graph. */
461 cgraph_recursive_inlining_p (struct cgraph_node
*to
,
462 struct cgraph_node
*what
,
466 if (to
->global
.inlined_to
)
467 recursive
= what
->decl
== to
->global
.inlined_to
->decl
;
469 recursive
= what
->decl
== to
->decl
;
470 /* Marking recursive function inline has sane semantic and thus we should
472 if (recursive
&& reason
)
473 *reason
= (what
->local
.disregard_inline_limits
474 ? N_("recursive inlining") : "");
478 /* A cost model driving the inlining heuristics in a way so the edges with
479 smallest badness are inlined first. After each inlining is performed
480 the costs of all caller edges of nodes affected are recomputed so the
481 metrics may accurately depend on values such as number of inlinable callers
482 of the function or function body size. */
485 cgraph_edge_badness (struct cgraph_edge
*edge
)
489 cgraph_estimate_size_after_inlining (1, edge
->caller
, edge
->callee
);
491 growth
-= edge
->caller
->global
.insns
;
493 /* Always prefer inlining saving code size. */
495 badness
= INT_MIN
- growth
;
497 /* When profiling is available, base priorities -(#calls / growth).
498 So we optimize for overall number of "executed" inlined calls. */
500 badness
= ((int)((double)edge
->count
* INT_MIN
/ max_count
)) / growth
;
502 /* When function local profile is available, base priorities on
503 growth / frequency, so we optimize for overall frequency of inlined
504 calls. This is not too accurate since while the call might be frequent
505 within function, the function itself is infrequent.
507 Other objective to optimize for is number of different calls inlined.
508 We add the estimated growth after inlining all functions to bias the
509 priorities slightly in this direction (so fewer times called functions
510 of the same size gets priority). */
511 else if (flag_guess_branch_prob
)
513 int div
= edge
->frequency
* 100 / CGRAPH_FREQ_BASE
;
515 cgraph_estimate_size_after_inlining (1, edge
->caller
, edge
->callee
);
516 growth
-= edge
->caller
->global
.insns
;
517 badness
= growth
* 256;
519 /* Decrease badness if call is nested. */
520 /* Compress the range so we don't overflow. */
522 div
= 256 + ceil_log2 (div
) - 8;
527 badness
+= cgraph_estimate_growth (edge
->callee
);
529 /* When function local profile is not available or it does not give
530 useful information (ie frequency is zero), base the cost on
531 loop nest and overall size growth, so we optimize for overall number
532 of functions fully inlined in program. */
535 int nest
= MIN (edge
->loop_nest
, 8);
536 badness
= cgraph_estimate_growth (edge
->callee
) * 256;
538 /* Decrease badness if call is nested. */
546 /* Make recursive inlining happen always after other inlining is done. */
547 if (cgraph_recursive_inlining_p (edge
->caller
, edge
->callee
, NULL
))
553 /* Recompute heap nodes for each of caller edge. */
556 update_caller_keys (fibheap_t heap
, struct cgraph_node
*node
,
557 bitmap updated_nodes
)
559 struct cgraph_edge
*edge
;
560 const char *failed_reason
;
562 if (!node
->local
.inlinable
|| node
->local
.disregard_inline_limits
563 || node
->global
.inlined_to
)
565 if (bitmap_bit_p (updated_nodes
, node
->uid
))
567 bitmap_set_bit (updated_nodes
, node
->uid
);
568 node
->global
.estimated_growth
= INT_MIN
;
570 if (!node
->local
.inlinable
)
572 /* Prune out edges we won't inline into anymore. */
573 if (!cgraph_default_inline_p (node
, &failed_reason
))
575 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
578 fibheap_delete_node (heap
, (fibnode_t
) edge
->aux
);
580 if (edge
->inline_failed
)
581 edge
->inline_failed
= failed_reason
;
586 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
587 if (edge
->inline_failed
)
589 int badness
= cgraph_edge_badness (edge
);
592 fibnode_t n
= (fibnode_t
) edge
->aux
;
593 gcc_assert (n
->data
== edge
);
594 if (n
->key
== badness
)
597 /* fibheap_replace_key only increase the keys. */
598 if (fibheap_replace_key (heap
, n
, badness
))
600 fibheap_delete_node (heap
, (fibnode_t
) edge
->aux
);
602 edge
->aux
= fibheap_insert (heap
, badness
, edge
);
606 /* Recompute heap nodes for each of caller edges of each of callees. */
609 update_callee_keys (fibheap_t heap
, struct cgraph_node
*node
,
610 bitmap updated_nodes
)
612 struct cgraph_edge
*e
;
613 node
->global
.estimated_growth
= INT_MIN
;
615 for (e
= node
->callees
; e
; e
= e
->next_callee
)
616 if (e
->inline_failed
)
617 update_caller_keys (heap
, e
->callee
, updated_nodes
);
618 else if (!e
->inline_failed
)
619 update_callee_keys (heap
, e
->callee
, updated_nodes
);
622 /* Enqueue all recursive calls from NODE into priority queue depending on
623 how likely we want to recursively inline the call. */
626 lookup_recursive_calls (struct cgraph_node
*node
, struct cgraph_node
*where
,
630 struct cgraph_edge
*e
;
631 for (e
= where
->callees
; e
; e
= e
->next_callee
)
632 if (e
->callee
== node
)
634 /* When profile feedback is available, prioritize by expected number
635 of calls. Without profile feedback we maintain simple queue
636 to order candidates via recursive depths. */
637 fibheap_insert (heap
,
638 !max_count
? priority
++
639 : -(e
->count
/ ((max_count
+ (1<<24) - 1) / (1<<24))),
642 for (e
= where
->callees
; e
; e
= e
->next_callee
)
643 if (!e
->inline_failed
)
644 lookup_recursive_calls (node
, e
->callee
, heap
);
647 /* Decide on recursive inlining: in the case function has recursive calls,
648 inline until body size reaches given argument. If any new indirect edges
649 are discovered in the process, add them to *NEW_EDGES, unless NEW_EDGES
653 cgraph_decide_recursive_inlining (struct cgraph_node
*node
,
654 VEC (cgraph_edge_p
, heap
) **new_edges
)
656 int limit
= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE_AUTO
);
657 int max_depth
= PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH_AUTO
);
658 int probability
= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY
);
660 struct cgraph_edge
*e
;
661 struct cgraph_node
*master_clone
, *next
;
665 if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION (node
->decl
))
666 || (!flag_inline_functions
&& !DECL_DECLARED_INLINE_P (node
->decl
)))
669 if (DECL_DECLARED_INLINE_P (node
->decl
))
671 limit
= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE
);
672 max_depth
= PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH
);
675 /* Make sure that function is small enough to be considered for inlining. */
677 || cgraph_estimate_size_after_inlining (1, node
, node
) >= limit
)
679 heap
= fibheap_new ();
680 lookup_recursive_calls (node
, node
, heap
);
681 if (fibheap_empty (heap
))
683 fibheap_delete (heap
);
689 " Performing recursive inlining on %s\n",
690 cgraph_node_name (node
));
692 /* We need original clone to copy around. */
693 master_clone
= cgraph_clone_node (node
, node
->count
, CGRAPH_FREQ_BASE
, 1, false);
694 master_clone
->needed
= true;
695 for (e
= master_clone
->callees
; e
; e
= e
->next_callee
)
696 if (!e
->inline_failed
)
697 cgraph_clone_inlined_nodes (e
, true, false);
699 /* Do the inlining and update list of recursive call during process. */
700 while (!fibheap_empty (heap
)
701 && (cgraph_estimate_size_after_inlining (1, node
, master_clone
)
704 struct cgraph_edge
*curr
705 = (struct cgraph_edge
*) fibheap_extract_min (heap
);
706 struct cgraph_node
*cnode
;
709 for (cnode
= curr
->caller
;
710 cnode
->global
.inlined_to
; cnode
= cnode
->callers
->caller
)
711 if (node
->decl
== curr
->callee
->decl
)
713 if (depth
> max_depth
)
717 " maximal depth reached\n");
723 if (!cgraph_maybe_hot_edge_p (curr
))
726 fprintf (dump_file
, " Not inlining cold call\n");
729 if (curr
->count
* 100 / node
->count
< probability
)
733 " Probability of edge is too small\n");
741 " Inlining call of depth %i", depth
);
744 fprintf (dump_file
, " called approx. %.2f times per call",
745 (double)curr
->count
/ node
->count
);
747 fprintf (dump_file
, "\n");
749 cgraph_redirect_edge_callee (curr
, master_clone
);
750 cgraph_mark_inline_edge (curr
, false);
751 if (flag_indirect_inlining
)
752 ipa_propagate_indirect_call_infos (curr
, new_edges
);
753 lookup_recursive_calls (node
, curr
->callee
, heap
);
756 if (!fibheap_empty (heap
) && dump_file
)
757 fprintf (dump_file
, " Recursive inlining growth limit met.\n");
759 fibheap_delete (heap
);
762 "\n Inlined %i times, body grown from %i to %i insns\n", n
,
763 master_clone
->global
.insns
, node
->global
.insns
);
765 /* Remove master clone we used for inlining. We rely that clones inlined
766 into master clone gets queued just before master clone so we don't
768 for (node
= cgraph_nodes
; node
!= master_clone
;
772 if (node
->global
.inlined_to
== master_clone
)
773 cgraph_remove_node (node
);
775 cgraph_remove_node (master_clone
);
776 /* FIXME: Recursive inlining actually reduces number of calls of the
777 function. At this place we should probably walk the function and
778 inline clones and compensate the counts accordingly. This probably
779 doesn't matter much in practice. */
783 /* Set inline_failed for all callers of given function to REASON. */
786 cgraph_set_inline_failed (struct cgraph_node
*node
, const char *reason
)
788 struct cgraph_edge
*e
;
791 fprintf (dump_file
, "Inlining failed: %s\n", reason
);
792 for (e
= node
->callers
; e
; e
= e
->next_caller
)
793 if (e
->inline_failed
)
794 e
->inline_failed
= reason
;
797 /* Given whole compilation unit estimate of INSNS, compute how large we can
798 allow the unit to grow. */
800 compute_max_insns (int insns
)
802 int max_insns
= insns
;
803 if (max_insns
< PARAM_VALUE (PARAM_LARGE_UNIT_INSNS
))
804 max_insns
= PARAM_VALUE (PARAM_LARGE_UNIT_INSNS
);
806 return ((HOST_WIDEST_INT
) max_insns
807 * (100 + PARAM_VALUE (PARAM_INLINE_UNIT_GROWTH
)) / 100);
810 /* Compute badness of all edges in NEW_EDGES and add them to the HEAP. */
812 add_new_edges_to_heap (fibheap_t heap
, VEC (cgraph_edge_p
, heap
) *new_edges
)
814 while (VEC_length (cgraph_edge_p
, new_edges
) > 0)
816 struct cgraph_edge
*edge
= VEC_pop (cgraph_edge_p
, new_edges
);
818 gcc_assert (!edge
->aux
);
819 edge
->aux
= fibheap_insert (heap
, cgraph_edge_badness (edge
), edge
);
824 /* We use greedy algorithm for inlining of small functions:
825 All inline candidates are put into prioritized heap based on estimated
826 growth of the overall number of instructions and then update the estimates.
828 INLINED and INLINED_CALEES are just pointers to arrays large enough
829 to be passed to cgraph_inlined_into and cgraph_inlined_callees. */
832 cgraph_decide_inlining_of_small_functions (void)
834 struct cgraph_node
*node
;
835 struct cgraph_edge
*edge
;
836 const char *failed_reason
;
837 fibheap_t heap
= fibheap_new ();
838 bitmap updated_nodes
= BITMAP_ALLOC (NULL
);
839 int min_insns
, max_insns
;
840 VEC (cgraph_edge_p
, heap
) *new_indirect_edges
= NULL
;
842 if (flag_indirect_inlining
)
843 new_indirect_edges
= VEC_alloc (cgraph_edge_p
, heap
, 8);
846 fprintf (dump_file
, "\nDeciding on smaller functions:\n");
848 /* Put all inline candidates into the heap. */
850 for (node
= cgraph_nodes
; node
; node
= node
->next
)
852 if (!node
->local
.inlinable
|| !node
->callers
853 || node
->local
.disregard_inline_limits
)
856 fprintf (dump_file
, "Considering inline candidate %s.\n", cgraph_node_name (node
));
858 node
->global
.estimated_growth
= INT_MIN
;
859 if (!cgraph_default_inline_p (node
, &failed_reason
))
861 cgraph_set_inline_failed (node
, failed_reason
);
865 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
866 if (edge
->inline_failed
)
868 gcc_assert (!edge
->aux
);
869 edge
->aux
= fibheap_insert (heap
, cgraph_edge_badness (edge
), edge
);
873 max_insns
= compute_max_insns (overall_insns
);
874 min_insns
= overall_insns
;
876 while (overall_insns
<= max_insns
877 && (edge
= (struct cgraph_edge
*) fibheap_extract_min (heap
)))
879 int old_insns
= overall_insns
;
880 struct cgraph_node
*where
;
882 cgraph_estimate_size_after_inlining (1, edge
->caller
, edge
->callee
);
883 const char *not_good
= NULL
;
885 growth
-= edge
->caller
->global
.insns
;
890 "\nConsidering %s with %i insns\n",
891 cgraph_node_name (edge
->callee
),
892 edge
->callee
->global
.insns
);
894 " to be inlined into %s\n"
895 " Estimated growth after inlined into all callees is %+i insns.\n"
896 " Estimated badness is %i, frequency %.2f.\n",
897 cgraph_node_name (edge
->caller
),
898 cgraph_estimate_growth (edge
->callee
),
899 cgraph_edge_badness (edge
),
900 edge
->frequency
/ (double)CGRAPH_FREQ_BASE
);
902 fprintf (dump_file
," Called "HOST_WIDEST_INT_PRINT_DEC
"x\n", edge
->count
);
904 gcc_assert (edge
->aux
);
906 if (!edge
->inline_failed
)
909 /* When not having profile info ready we don't weight by any way the
910 position of call in procedure itself. This means if call of
911 function A from function B seems profitable to inline, the recursive
912 call of function A in inline copy of A in B will look profitable too
913 and we end up inlining until reaching maximal function growth. This
914 is not good idea so prohibit the recursive inlining.
916 ??? When the frequencies are taken into account we might not need this
919 We need to be cureful here, in some testcases, e.g. directivec.c in
920 libcpp, we can estimate self recursive function to have negative growth
921 for inlining completely.
925 where
= edge
->caller
;
926 while (where
->global
.inlined_to
)
928 if (where
->decl
== edge
->callee
->decl
)
930 where
= where
->callers
->caller
;
932 if (where
->global
.inlined_to
)
935 = (edge
->callee
->local
.disregard_inline_limits
? N_("recursive inlining") : "");
937 fprintf (dump_file
, " inline_failed:Recursive inlining performed only for function itself.\n");
942 if (!cgraph_maybe_hot_edge_p (edge
))
943 not_good
= N_("call is unlikely and code size would grow");
944 if (!flag_inline_functions
945 && !DECL_DECLARED_INLINE_P (edge
->callee
->decl
))
946 not_good
= N_("function not declared inline and code size would grow");
947 if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION(edge
->caller
->decl
)))
948 not_good
= N_("optimizing for size and code size would grow");
949 if (not_good
&& growth
> 0 && cgraph_estimate_growth (edge
->callee
) > 0)
951 if (!cgraph_recursive_inlining_p (edge
->caller
, edge
->callee
,
952 &edge
->inline_failed
))
954 edge
->inline_failed
= not_good
;
956 fprintf (dump_file
, " inline_failed:%s.\n", edge
->inline_failed
);
960 if (!cgraph_default_inline_p (edge
->callee
, &edge
->inline_failed
))
962 if (!cgraph_recursive_inlining_p (edge
->caller
, edge
->callee
,
963 &edge
->inline_failed
))
966 fprintf (dump_file
, " inline_failed:%s.\n", edge
->inline_failed
);
970 if (!tree_can_inline_p (edge
->caller
->decl
, edge
->callee
->decl
))
972 gimple_call_set_cannot_inline (edge
->call_stmt
, true);
973 edge
->inline_failed
= N_("target specific option mismatch");
975 fprintf (dump_file
, " inline_failed:%s.\n", edge
->inline_failed
);
978 if (cgraph_recursive_inlining_p (edge
->caller
, edge
->callee
,
979 &edge
->inline_failed
))
981 where
= edge
->caller
;
982 if (where
->global
.inlined_to
)
983 where
= where
->global
.inlined_to
;
984 if (!cgraph_decide_recursive_inlining (where
,
985 flag_indirect_inlining
986 ? &new_indirect_edges
: NULL
))
988 if (flag_indirect_inlining
)
989 add_new_edges_to_heap (heap
, new_indirect_edges
);
990 update_callee_keys (heap
, where
, updated_nodes
);
994 struct cgraph_node
*callee
;
995 if (gimple_call_cannot_inline_p (edge
->call_stmt
)
996 || !cgraph_check_inline_limits (edge
->caller
, edge
->callee
,
997 &edge
->inline_failed
, true))
1000 fprintf (dump_file
, " Not inlining into %s:%s.\n",
1001 cgraph_node_name (edge
->caller
), edge
->inline_failed
);
1004 callee
= edge
->callee
;
1005 cgraph_mark_inline_edge (edge
, true);
1006 if (flag_indirect_inlining
)
1008 ipa_propagate_indirect_call_infos (edge
, &new_indirect_edges
);
1009 add_new_edges_to_heap (heap
, new_indirect_edges
);
1011 update_callee_keys (heap
, callee
, updated_nodes
);
1013 where
= edge
->caller
;
1014 if (where
->global
.inlined_to
)
1015 where
= where
->global
.inlined_to
;
1017 /* Our profitability metric can depend on local properties
1018 such as number of inlinable calls and size of the function body.
1019 After inlining these properties might change for the function we
1020 inlined into (since it's body size changed) and for the functions
1021 called by function we inlined (since number of it inlinable callers
1023 update_caller_keys (heap
, where
, updated_nodes
);
1024 bitmap_clear (updated_nodes
);
1029 " Inlined into %s which now has %i insns,"
1030 "net change of %+i insns.\n",
1031 cgraph_node_name (edge
->caller
),
1032 edge
->caller
->global
.insns
,
1033 overall_insns
- old_insns
);
1035 if (min_insns
> overall_insns
)
1037 min_insns
= overall_insns
;
1038 max_insns
= compute_max_insns (min_insns
);
1041 fprintf (dump_file
, "New minimal insns reached: %i\n", min_insns
);
1044 while ((edge
= (struct cgraph_edge
*) fibheap_extract_min (heap
)) != NULL
)
1046 gcc_assert (edge
->aux
);
1048 if (!edge
->callee
->local
.disregard_inline_limits
&& edge
->inline_failed
1049 && !cgraph_recursive_inlining_p (edge
->caller
, edge
->callee
,
1050 &edge
->inline_failed
))
1051 edge
->inline_failed
= N_("--param inline-unit-growth limit reached");
1054 if (new_indirect_edges
)
1055 VEC_free (cgraph_edge_p
, heap
, new_indirect_edges
);
1056 fibheap_delete (heap
);
1057 BITMAP_FREE (updated_nodes
);
1060 /* Decide on the inlining. We do so in the topological order to avoid
1061 expenses on updating data structures. */
1064 cgraph_decide_inlining (void)
1066 struct cgraph_node
*node
;
1068 struct cgraph_node
**order
=
1069 XCNEWVEC (struct cgraph_node
*, cgraph_n_nodes
);
1072 int initial_insns
= 0;
1074 cgraph_remove_function_insertion_hook (function_insertion_hook_holder
);
1077 for (node
= cgraph_nodes
; node
; node
= node
->next
)
1078 if (node
->analyzed
&& (node
->needed
|| node
->reachable
))
1080 struct cgraph_edge
*e
;
1082 initial_insns
+= inline_summary (node
)->self_insns
;
1083 gcc_assert (inline_summary (node
)->self_insns
== node
->global
.insns
);
1084 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1085 if (max_count
< e
->count
)
1086 max_count
= e
->count
;
1088 overall_insns
= initial_insns
;
1089 gcc_assert (!max_count
|| (profile_info
&& flag_branch_probabilities
));
1091 nnodes
= cgraph_postorder (order
);
1095 "\nDeciding on inlining. Starting with %i insns.\n",
1098 for (node
= cgraph_nodes
; node
; node
= node
->next
)
1102 fprintf (dump_file
, "\nInlining always_inline functions:\n");
1104 /* In the first pass mark all always_inline edges. Do this with a priority
1105 so none of our later choices will make this impossible. */
1106 for (i
= nnodes
- 1; i
>= 0; i
--)
1108 struct cgraph_edge
*e
, *next
;
1112 /* Handle nodes to be flattened, but don't update overall unit size. */
1113 if (lookup_attribute ("flatten", DECL_ATTRIBUTES (node
->decl
)) != NULL
)
1117 "Flattening %s\n", cgraph_node_name (node
));
1118 cgraph_decide_inlining_incrementally (node
, INLINE_ALL
, 0);
1121 if (!node
->local
.disregard_inline_limits
)
1125 "\nConsidering %s %i insns (always inline)\n",
1126 cgraph_node_name (node
), node
->global
.insns
);
1127 old_insns
= overall_insns
;
1128 for (e
= node
->callers
; e
; e
= next
)
1130 next
= e
->next_caller
;
1131 if (!e
->inline_failed
|| gimple_call_cannot_inline_p (e
->call_stmt
))
1133 if (cgraph_recursive_inlining_p (e
->caller
, e
->callee
,
1136 if (!tree_can_inline_p (e
->caller
->decl
, e
->callee
->decl
))
1138 gimple_call_set_cannot_inline (e
->call_stmt
, true);
1141 cgraph_mark_inline_edge (e
, true);
1142 if (flag_indirect_inlining
)
1143 ipa_propagate_indirect_call_infos (e
, NULL
);
1146 " Inlined into %s which now has %i insns.\n",
1147 cgraph_node_name (e
->caller
),
1148 e
->caller
->global
.insns
);
1150 /* Inlining self recursive function might introduce new calls to
1151 themselves we didn't see in the loop above. Fill in the proper
1152 reason why inline failed. */
1153 for (e
= node
->callers
; e
; e
= e
->next_caller
)
1154 if (e
->inline_failed
)
1155 e
->inline_failed
= N_("recursive inlining");
1158 " Inlined for a net change of %+i insns.\n",
1159 overall_insns
- old_insns
);
1162 cgraph_decide_inlining_of_small_functions ();
1164 /* After this point, any edge discovery performed by indirect inlining is no
1165 good so let's give up. */
1166 if (flag_indirect_inlining
)
1167 free_all_ipa_structures_after_iinln ();
1169 if (flag_inline_functions_called_once
)
1172 fprintf (dump_file
, "\nDeciding on functions called once:\n");
1174 /* And finally decide what functions are called once. */
1175 for (i
= nnodes
- 1; i
>= 0; i
--)
1180 && !node
->callers
->next_caller
1182 && node
->local
.inlinable
1183 && node
->callers
->inline_failed
1184 && !gimple_call_cannot_inline_p (node
->callers
->call_stmt
)
1185 && !DECL_EXTERNAL (node
->decl
)
1186 && !DECL_COMDAT (node
->decl
))
1191 "\nConsidering %s %i insns.\n",
1192 cgraph_node_name (node
), node
->global
.insns
);
1194 " Called once from %s %i insns.\n",
1195 cgraph_node_name (node
->callers
->caller
),
1196 node
->callers
->caller
->global
.insns
);
1199 old_insns
= overall_insns
;
1201 if (cgraph_check_inline_limits (node
->callers
->caller
, node
,
1204 cgraph_mark_inline (node
->callers
);
1207 " Inlined into %s which now has %i insns"
1208 " for a net change of %+i insns.\n",
1209 cgraph_node_name (node
->callers
->caller
),
1210 node
->callers
->caller
->global
.insns
,
1211 overall_insns
- old_insns
);
1217 " Inline limit reached, not inlined.\n");
1225 "\nInlined %i calls, eliminated %i functions, "
1226 "%i insns turned to %i insns.\n\n",
1227 ncalls_inlined
, nfunctions_inlined
, initial_insns
,
1233 /* Try to inline edge E from incremental inliner. MODE specifies mode
1236 We are detecting cycles by storing mode of inliner into cgraph_node last
1237 time we visited it in the recursion. In general when mode is set, we have
1238 recursive inlining, but as an special case, we want to try harder inline
1239 ALWAYS_INLINE functions: consider callgraph a->b->c->b, with a being
1240 flatten, b being always inline. Flattening 'a' will collapse
1241 a->b->c before hitting cycle. To accommodate always inline, we however
1242 need to inline a->b->c->b.
1244 So after hitting cycle first time, we switch into ALWAYS_INLINE mode and
1245 stop inlining only after hitting ALWAYS_INLINE in ALWAY_INLINE mode. */
1247 try_inline (struct cgraph_edge
*e
, enum inlining_mode mode
, int depth
)
1249 struct cgraph_node
*callee
= e
->callee
;
1250 enum inlining_mode callee_mode
= (enum inlining_mode
) (size_t) callee
->aux
;
1251 bool always_inline
= e
->callee
->local
.disregard_inline_limits
;
1253 /* We've hit cycle? */
1256 /* It is first time we see it and we are not in ALWAY_INLINE only
1257 mode yet. and the function in question is always_inline. */
1258 if (always_inline
&& mode
!= INLINE_ALWAYS_INLINE
)
1262 indent_to (dump_file
, depth
);
1264 "Hit cycle in %s, switching to always inline only.\n",
1265 cgraph_node_name (callee
));
1267 mode
= INLINE_ALWAYS_INLINE
;
1269 /* Otherwise it is time to give up. */
1274 indent_to (dump_file
, depth
);
1276 "Not inlining %s into %s to avoid cycle.\n",
1277 cgraph_node_name (callee
),
1278 cgraph_node_name (e
->caller
));
1280 e
->inline_failed
= (e
->callee
->local
.disregard_inline_limits
1281 ? N_("recursive inlining") : "");
1286 callee
->aux
= (void *)(size_t) mode
;
1289 indent_to (dump_file
, depth
);
1290 fprintf (dump_file
, " Inlining %s into %s.\n",
1291 cgraph_node_name (e
->callee
),
1292 cgraph_node_name (e
->caller
));
1294 if (e
->inline_failed
)
1295 cgraph_mark_inline (e
);
1297 /* In order to fully inline always_inline functions, we need to
1298 recurse here, since the inlined functions might not be processed by
1299 incremental inlining at all yet.
1301 Also flattening needs to be done recursively. */
1303 if (mode
== INLINE_ALL
|| always_inline
)
1304 cgraph_decide_inlining_incrementally (e
->callee
, mode
, depth
+ 1);
1305 callee
->aux
= (void *)(size_t) callee_mode
;
1309 /* Decide on the inlining. We do so in the topological order to avoid
1310 expenses on updating data structures.
1311 DEPTH is depth of recursion, used only for debug output. */
1314 cgraph_decide_inlining_incrementally (struct cgraph_node
*node
,
1315 enum inlining_mode mode
,
1318 struct cgraph_edge
*e
;
1319 bool inlined
= false;
1320 const char *failed_reason
;
1321 enum inlining_mode old_mode
;
1323 #ifdef ENABLE_CHECKING
1324 verify_cgraph_node (node
);
1327 old_mode
= (enum inlining_mode
) (size_t)node
->aux
;
1329 if (mode
!= INLINE_ALWAYS_INLINE
1330 && lookup_attribute ("flatten", DECL_ATTRIBUTES (node
->decl
)) != NULL
)
1334 indent_to (dump_file
, depth
);
1335 fprintf (dump_file
, "Flattening %s\n", cgraph_node_name (node
));
1340 node
->aux
= (void *)(size_t) mode
;
1342 /* First of all look for always inline functions. */
1343 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1345 if (!e
->callee
->local
.disregard_inline_limits
1346 && (mode
!= INLINE_ALL
|| !e
->callee
->local
.inlinable
))
1348 if (gimple_call_cannot_inline_p (e
->call_stmt
))
1350 /* When the edge is already inlined, we just need to recurse into
1351 it in order to fully flatten the leaves. */
1352 if (!e
->inline_failed
&& mode
== INLINE_ALL
)
1354 inlined
|= try_inline (e
, mode
, depth
);
1359 indent_to (dump_file
, depth
);
1361 "Considering to always inline inline candidate %s.\n",
1362 cgraph_node_name (e
->callee
));
1364 if (cgraph_recursive_inlining_p (node
, e
->callee
, &e
->inline_failed
))
1368 indent_to (dump_file
, depth
);
1369 fprintf (dump_file
, "Not inlining: recursive call.\n");
1373 if (!tree_can_inline_p (node
->decl
, e
->callee
->decl
))
1375 gimple_call_set_cannot_inline (e
->call_stmt
, true);
1378 indent_to (dump_file
, depth
);
1380 "Not inlining: Target specific option mismatch.\n");
1384 if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node
->decl
))
1385 != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e
->callee
->decl
)))
1389 indent_to (dump_file
, depth
);
1390 fprintf (dump_file
, "Not inlining: SSA form does not match.\n");
1394 if (!e
->callee
->analyzed
&& !e
->callee
->inline_decl
)
1398 indent_to (dump_file
, depth
);
1400 "Not inlining: Function body no longer available.\n");
1404 inlined
|= try_inline (e
, mode
, depth
);
1407 /* Now do the automatic inlining. */
1408 if (mode
!= INLINE_ALL
&& mode
!= INLINE_ALWAYS_INLINE
)
1409 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1411 if (!e
->callee
->local
.inlinable
1412 || !e
->inline_failed
1413 || e
->callee
->local
.disregard_inline_limits
)
1416 fprintf (dump_file
, "Considering inline candidate %s.\n",
1417 cgraph_node_name (e
->callee
));
1418 if (cgraph_recursive_inlining_p (node
, e
->callee
, &e
->inline_failed
))
1422 indent_to (dump_file
, depth
);
1423 fprintf (dump_file
, "Not inlining: recursive call.\n");
1427 if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node
->decl
))
1428 != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e
->callee
->decl
)))
1432 indent_to (dump_file
, depth
);
1433 fprintf (dump_file
, "Not inlining: SSA form does not match.\n");
1437 /* When the function body would grow and inlining the function won't
1438 eliminate the need for offline copy of the function, don't inline.
1440 if ((mode
== INLINE_SIZE
1441 || (!flag_inline_functions
1442 && !DECL_DECLARED_INLINE_P (e
->callee
->decl
)))
1443 && (cgraph_estimate_size_after_inlining (1, e
->caller
, e
->callee
)
1444 > e
->caller
->global
.insns
)
1445 && cgraph_estimate_growth (e
->callee
) > 0)
1449 indent_to (dump_file
, depth
);
1451 "Not inlining: code size would grow by %i insns.\n",
1452 cgraph_estimate_size_after_inlining (1, e
->caller
,
1454 - e
->caller
->global
.insns
);
1458 if (!cgraph_check_inline_limits (node
, e
->callee
, &e
->inline_failed
,
1460 || gimple_call_cannot_inline_p (e
->call_stmt
))
1464 indent_to (dump_file
, depth
);
1465 fprintf (dump_file
, "Not inlining: %s.\n", e
->inline_failed
);
1469 if (!e
->callee
->analyzed
&& !e
->callee
->inline_decl
)
1473 indent_to (dump_file
, depth
);
1475 "Not inlining: Function body no longer available.\n");
1479 if (!tree_can_inline_p (node
->decl
, e
->callee
->decl
))
1481 gimple_call_set_cannot_inline (e
->call_stmt
, true);
1484 indent_to (dump_file
, depth
);
1486 "Not inlining: Target specific option mismatch.\n");
1490 if (cgraph_default_inline_p (e
->callee
, &failed_reason
))
1491 inlined
|= try_inline (e
, mode
, depth
);
1493 node
->aux
= (void *)(size_t) old_mode
;
1497 /* Because inlining might remove no-longer reachable nodes, we need to
1498 keep the array visible to garbage collector to avoid reading collected
1501 static GTY ((length ("nnodes"))) struct cgraph_node
**order
;
1503 /* Do inlining of small functions. Doing so early helps profiling and other
1504 passes to be somewhat more effective and avoids some code duplication in
1505 later real inlining pass for testcases with very many function calls. */
1507 cgraph_early_inlining (void)
1509 struct cgraph_node
*node
= cgraph_node (current_function_decl
);
1510 unsigned int todo
= 0;
1512 if (sorrycount
|| errorcount
)
1514 if (cgraph_decide_inlining_incrementally (node
, INLINE_SIZE
, 0))
1516 timevar_push (TV_INTEGRATION
);
1517 todo
= optimize_inline_calls (current_function_decl
);
1518 timevar_pop (TV_INTEGRATION
);
1523 /* When inlining shall be performed. */
1525 cgraph_gate_early_inlining (void)
1527 return flag_early_inlining
;
1530 struct gimple_opt_pass pass_early_inline
=
1534 "einline", /* name */
1535 cgraph_gate_early_inlining
, /* gate */
1536 cgraph_early_inlining
, /* execute */
1539 0, /* static_pass_number */
1540 TV_INLINE_HEURISTICS
, /* tv_id */
1541 0, /* properties_required */
1542 PROP_cfg
, /* properties_provided */
1543 0, /* properties_destroyed */
1544 0, /* todo_flags_start */
1545 TODO_dump_func
/* todo_flags_finish */
1549 /* When inlining shall be performed. */
1551 cgraph_gate_ipa_early_inlining (void)
1553 return (flag_early_inlining
1554 && (flag_branch_probabilities
|| flag_test_coverage
1555 || profile_arc_flag
));
1558 /* IPA pass wrapper for early inlining pass. We need to run early inlining
1559 before tree profiling so we have stand alone IPA pass for doing so. */
1560 struct simple_ipa_opt_pass pass_ipa_early_inline
=
1564 "einline_ipa", /* name */
1565 cgraph_gate_ipa_early_inlining
, /* gate */
1569 0, /* static_pass_number */
1570 TV_INLINE_HEURISTICS
, /* tv_id */
1571 0, /* properties_required */
1572 PROP_cfg
, /* properties_provided */
1573 0, /* properties_destroyed */
1574 0, /* todo_flags_start */
1575 TODO_dump_cgraph
/* todo_flags_finish */
1579 /* Compute parameters of functions used by inliner. */
1581 compute_inline_parameters (struct cgraph_node
*node
)
1583 gcc_assert (!node
->global
.inlined_to
);
1584 inline_summary (node
)->estimated_self_stack_size
1585 = estimated_stack_frame_size ();
1586 node
->global
.estimated_stack_size
1587 = inline_summary (node
)->estimated_self_stack_size
;
1588 node
->global
.stack_frame_offset
= 0;
1589 node
->local
.inlinable
= tree_inlinable_function_p (current_function_decl
);
1590 inline_summary (node
)->self_insns
1591 = estimate_num_insns_fn (current_function_decl
, &eni_inlining_weights
);
1592 if (node
->local
.inlinable
&& !node
->local
.disregard_inline_limits
)
1593 node
->local
.disregard_inline_limits
1594 = DECL_DISREGARD_INLINE_LIMITS (current_function_decl
);
1595 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
1596 node
->global
.insns
= inline_summary (node
)->self_insns
;
1601 /* Compute parameters of functions used by inliner using
1602 current_function_decl. */
1604 compute_inline_parameters_for_current (void)
1606 compute_inline_parameters (cgraph_node (current_function_decl
));
1610 struct gimple_opt_pass pass_inline_parameters
=
1616 compute_inline_parameters_for_current
,/* execute */
1619 0, /* static_pass_number */
1620 TV_INLINE_HEURISTICS
, /* tv_id */
1621 0, /* properties_required */
1622 PROP_cfg
, /* properties_provided */
1623 0, /* properties_destroyed */
1624 0, /* todo_flags_start */
1625 0 /* todo_flags_finish */
1629 /* This function performs intraprocedural analyzis in NODE that is required to
1630 inline indirect calls. */
1632 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
1634 struct cgraph_edge
*cs
;
1638 ipa_count_formal_params (node
);
1639 ipa_create_param_decls_array (node
);
1640 ipa_detect_param_modifications (node
);
1642 ipa_analyze_params_uses (node
);
1645 for (cs
= node
->callees
; cs
; cs
= cs
->next_callee
)
1647 ipa_count_arguments (cs
);
1648 ipa_compute_jump_functions (cs
);
1653 ipa_print_node_params (dump_file
, node
);
1654 ipa_print_node_jump_functions (dump_file
, node
);
1658 /* Note function body size. */
1660 analyze_function (struct cgraph_node
*node
)
1662 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
1663 current_function_decl
= node
->decl
;
1665 compute_inline_parameters (node
);
1666 if (flag_indirect_inlining
)
1667 inline_indirect_intraprocedural_analysis (node
);
1669 current_function_decl
= NULL
;
1673 /* Called when new function is inserted to callgraph late. */
1675 add_new_function (struct cgraph_node
*node
, void *data ATTRIBUTE_UNUSED
)
1677 analyze_function (node
);
1680 /* Note function body size. */
1682 inline_generate_summary (void)
1684 struct cgraph_node
*node
;
1686 function_insertion_hook_holder
=
1687 cgraph_add_function_insertion_hook (&add_new_function
, NULL
);
1689 if (flag_indirect_inlining
)
1691 ipa_register_cgraph_hooks ();
1692 ipa_check_create_node_params ();
1693 ipa_check_create_edge_args ();
1696 for (node
= cgraph_nodes
; node
; node
= node
->next
)
1698 analyze_function (node
);
1703 /* Apply inline plan to function. */
1705 inline_transform (struct cgraph_node
*node
)
1707 unsigned int todo
= 0;
1708 struct cgraph_edge
*e
;
1710 /* We might need the body of this function so that we can expand
1711 it inline somewhere else. */
1712 if (cgraph_preserve_function_body_p (node
->decl
))
1713 save_inline_function_body (node
);
1715 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1716 if (!e
->inline_failed
|| warn_inline
)
1721 timevar_push (TV_INTEGRATION
);
1722 todo
= optimize_inline_calls (current_function_decl
);
1723 timevar_pop (TV_INTEGRATION
);
1725 return todo
| execute_fixup_cfg ();
1728 struct ipa_opt_pass pass_ipa_inline
=
1732 "inline", /* name */
1734 cgraph_decide_inlining
, /* execute */
1737 0, /* static_pass_number */
1738 TV_INLINE_HEURISTICS
, /* tv_id */
1739 0, /* properties_required */
1740 PROP_cfg
, /* properties_provided */
1741 0, /* properties_destroyed */
1742 TODO_remove_functions
, /* todo_flags_finish */
1743 TODO_dump_cgraph
| TODO_dump_func
1744 | TODO_remove_functions
/* todo_flags_finish */
1746 inline_generate_summary
, /* generate_summary */
1747 NULL
, /* write_summary */
1748 NULL
, /* read_summary */
1749 NULL
, /* function_read_summary */
1751 inline_transform
, /* function_transform */
1752 NULL
, /* variable_transform */
1756 #include "gt-ipa-inline.h"