1 /* Inlining decision heuristics.
2 Copyright (C) 2003, 2004, 2007, 2008 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Inlining decision heuristics
23 We separate inlining decisions from the inliner itself and store it
24 inside callgraph as so called inline plan. Refer to cgraph.c
25 documentation about particular representation of inline plans in the
28 There are three major parts of this file:
30 cgraph_mark_inline implementation
32 This function allows to mark given call inline and performs necessary
33 modifications of cgraph (production of the clones and updating overall
36 inlining heuristics limits
38 These functions allow to check that particular inlining is allowed
39 by the limits specified by user (allowed function growth, overall unit
44 This is implementation of IPA pass aiming to get as much of benefit
45 from inlining obeying the limits checked above.
47 The implementation of particular heuristics is separated from
48 the rest of code to make it easier to replace it with more complicated
49 implementation in the future. The rest of inlining code acts as a
50 library aimed to modify the callgraph and verify that the parameters
51 on code size growth fits.
53 To mark given call inline, use cgraph_mark_inline function, the
54 verification is performed by cgraph_default_inline_p and
55 cgraph_check_inline_limits.
57 The heuristics implements simple knapsack style algorithm ordering
58 all functions by their "profitability" (estimated by code size growth)
59 and inlining them in priority order.
61 cgraph_decide_inlining implements heuristics taking whole callgraph
62 into account, while cgraph_decide_inlining_incrementally considers
63 only one function at a time and is used by early inliner.
65 The inliner itself is split into several passes:
67 pass_inline_parameters
69 This pass computes local properties of functions that are used by inliner:
70 estimated function body size, whether function is inlinable at all and
71 stack frame consumption.
73 Before executing any of inliner passes, this local pass has to be applied
74 to each function in the callgraph (ie run as subpass of some earlier
75 IPA pass). The results are made out of date by any optimization applied
80 Simple local inlining pass inlining callees into current function. This
81 pass makes no global whole compilation unit analysis and this when allowed
82 to do inlining expanding code size it might result in unbounded growth of
85 The pass is run during conversion into SSA form. Only functions already
86 converted into SSA form are inlined, so the conversion must happen in
87 topological order on the callgraph (that is maintained by pass manager).
88 The functions after inlining are early optimized so the early inliner sees
89 unoptimized function itself, but all considered callees are already
90 optimized allowing it to unfold abstraction penalty on C++ effectively and
93 pass_ipa_early_inlining
95 With profiling, the early inlining is also necessary to reduce
96 instrumentation costs on program with high abstraction penalty (doing
97 many redundant calls). This can't happen in parallel with early
98 optimization and profile instrumentation, because we would end up
99 re-instrumenting already instrumented function bodies we brought in via
102 To avoid this, this pass is executed as IPA pass before profiling. It is
103 simple wrapper to pass_early_inlining and ensures first inlining.
107 This is the main pass implementing simple greedy algorithm to do inlining
108 of small functions that results in overall growth of compilation unit and
109 inlining of functions called once. The pass compute just so called inline
110 plan (representation of inlining to be done in callgraph) and unlike early
111 inlining it is not performing the inlining itself.
115 This pass performs actual inlining according to pass_ipa_inline on given
116 function. Possible the function body before inlining is saved when it is
117 needed for further inlining later.
122 #include "coretypes.h"
125 #include "tree-inline.h"
126 #include "langhooks.h"
129 #include "diagnostic.h"
134 #include "tree-pass.h"
136 #include "coverage.h"
138 #include "tree-flow.h"
140 #include "ipa-prop.h"
142 /* Mode incremental inliner operate on:
144 In ALWAYS_INLINE only functions marked
145 always_inline are inlined. This mode is used after detecting cycle during
148 In SIZE mode, only functions that reduce function body size after inlining
149 are inlined, this is used during early inlining.
151 in ALL mode, everything is inlined. This is used during flattening. */
154 INLINE_ALWAYS_INLINE
,
159 cgraph_decide_inlining_incrementally (struct cgraph_node
*, enum inlining_mode
,
163 /* Statistics we collect about inlining algorithm. */
164 static int ncalls_inlined
;
165 static int nfunctions_inlined
;
166 static int overall_insns
;
167 static gcov_type max_count
;
169 static inline struct inline_summary
*
170 inline_summary (struct cgraph_node
*node
)
172 return &node
->local
.inline_summary
;
175 /* Estimate size of the function after inlining WHAT into TO. */
178 cgraph_estimate_size_after_inlining (int times
, struct cgraph_node
*to
,
179 struct cgraph_node
*what
)
182 tree fndecl
= what
->decl
, arg
;
183 int call_insns
= PARAM_VALUE (PARAM_INLINE_CALL_COST
);
185 for (arg
= DECL_ARGUMENTS (fndecl
); arg
; arg
= TREE_CHAIN (arg
))
186 call_insns
+= estimate_move_cost (TREE_TYPE (arg
));
187 size
= (what
->global
.insns
- call_insns
) * times
+ to
->global
.insns
;
188 gcc_assert (size
>= 0);
192 /* E is expected to be an edge being inlined. Clone destination node of
193 the edge and redirect it to the new clone.
194 DUPLICATE is used for bookkeeping on whether we are actually creating new
195 clones or re-using node originally representing out-of-line function call.
198 cgraph_clone_inlined_nodes (struct cgraph_edge
*e
, bool duplicate
, bool update_original
)
203 /* We may eliminate the need for out-of-line copy to be output.
204 In that case just go ahead and re-use it. */
205 if (!e
->callee
->callers
->next_caller
206 && !e
->callee
->needed
207 && !cgraph_new_nodes
)
209 gcc_assert (!e
->callee
->global
.inlined_to
);
210 if (DECL_SAVED_TREE (e
->callee
->decl
))
211 overall_insns
-= e
->callee
->global
.insns
, nfunctions_inlined
++;
216 struct cgraph_node
*n
;
217 n
= cgraph_clone_node (e
->callee
, e
->count
, e
->frequency
, e
->loop_nest
,
219 cgraph_redirect_edge_callee (e
, n
);
223 if (e
->caller
->global
.inlined_to
)
224 e
->callee
->global
.inlined_to
= e
->caller
->global
.inlined_to
;
226 e
->callee
->global
.inlined_to
= e
->caller
;
227 e
->callee
->global
.stack_frame_offset
228 = e
->caller
->global
.stack_frame_offset
229 + inline_summary (e
->caller
)->estimated_self_stack_size
;
230 peak
= e
->callee
->global
.stack_frame_offset
231 + inline_summary (e
->callee
)->estimated_self_stack_size
;
232 if (e
->callee
->global
.inlined_to
->global
.estimated_stack_size
< peak
)
233 e
->callee
->global
.inlined_to
->global
.estimated_stack_size
= peak
;
235 /* Recursively clone all bodies. */
236 for (e
= e
->callee
->callees
; e
; e
= e
->next_callee
)
237 if (!e
->inline_failed
)
238 cgraph_clone_inlined_nodes (e
, duplicate
, update_original
);
241 /* Mark edge E as inlined and update callgraph accordingly.
242 UPDATE_ORIGINAL specify whether profile of original function should be
246 cgraph_mark_inline_edge (struct cgraph_edge
*e
, bool update_original
)
248 int old_insns
= 0, new_insns
= 0;
249 struct cgraph_node
*to
= NULL
, *what
;
251 if (e
->callee
->inline_decl
)
252 cgraph_redirect_edge_callee (e
, cgraph_node (e
->callee
->inline_decl
));
254 gcc_assert (e
->inline_failed
);
255 e
->inline_failed
= NULL
;
257 if (!e
->callee
->global
.inlined
)
258 DECL_POSSIBLY_INLINED (e
->callee
->decl
) = true;
259 e
->callee
->global
.inlined
= true;
261 cgraph_clone_inlined_nodes (e
, true, update_original
);
265 /* Now update size of caller and all functions caller is inlined into. */
266 for (;e
&& !e
->inline_failed
; e
= e
->caller
->callers
)
268 old_insns
= e
->caller
->global
.insns
;
269 new_insns
= cgraph_estimate_size_after_inlining (1, e
->caller
,
271 gcc_assert (new_insns
>= 0);
273 to
->global
.insns
= new_insns
;
275 gcc_assert (what
->global
.inlined_to
== to
);
276 if (new_insns
> old_insns
)
277 overall_insns
+= new_insns
- old_insns
;
281 /* Mark all calls of EDGE->CALLEE inlined into EDGE->CALLER.
282 Return following unredirected edge in the list of callers
285 static struct cgraph_edge
*
286 cgraph_mark_inline (struct cgraph_edge
*edge
)
288 struct cgraph_node
*to
= edge
->caller
;
289 struct cgraph_node
*what
= edge
->callee
;
290 struct cgraph_edge
*e
, *next
;
292 gcc_assert (!CALL_STMT_CANNOT_INLINE_P (edge
->call_stmt
));
293 /* Look for all calls, mark them inline and clone recursively
294 all inlined functions. */
295 for (e
= what
->callers
; e
; e
= next
)
297 next
= e
->next_caller
;
298 if (e
->caller
== to
&& e
->inline_failed
)
300 cgraph_mark_inline_edge (e
, true);
309 /* Estimate the growth caused by inlining NODE into all callees. */
312 cgraph_estimate_growth (struct cgraph_node
*node
)
315 struct cgraph_edge
*e
;
316 if (node
->global
.estimated_growth
!= INT_MIN
)
317 return node
->global
.estimated_growth
;
319 for (e
= node
->callers
; e
; e
= e
->next_caller
)
320 if (e
->inline_failed
)
321 growth
+= (cgraph_estimate_size_after_inlining (1, e
->caller
, node
)
322 - e
->caller
->global
.insns
);
324 /* ??? Wrong for self recursive functions or cases where we decide to not
325 inline for different reasons, but it is not big deal as in that case
326 we will keep the body around, but we will also avoid some inlining. */
327 if (!node
->needed
&& !DECL_EXTERNAL (node
->decl
))
328 growth
-= node
->global
.insns
;
330 node
->global
.estimated_growth
= growth
;
334 /* Return false when inlining WHAT into TO is not good idea
335 as it would cause too large growth of function bodies.
336 When ONE_ONLY is true, assume that only one call site is going
337 to be inlined, otherwise figure out how many call sites in
338 TO calls WHAT and verify that all can be inlined.
342 cgraph_check_inline_limits (struct cgraph_node
*to
, struct cgraph_node
*what
,
343 const char **reason
, bool one_only
)
346 struct cgraph_edge
*e
;
349 HOST_WIDE_INT stack_size_limit
, inlined_stack
;
354 for (e
= to
->callees
; e
; e
= e
->next_callee
)
355 if (e
->callee
== what
)
358 if (to
->global
.inlined_to
)
359 to
= to
->global
.inlined_to
;
361 /* When inlining large function body called once into small function,
362 take the inlined function as base for limiting the growth. */
363 if (inline_summary (to
)->self_insns
> inline_summary(what
)->self_insns
)
364 limit
= inline_summary (to
)->self_insns
;
366 limit
= inline_summary (what
)->self_insns
;
368 limit
+= limit
* PARAM_VALUE (PARAM_LARGE_FUNCTION_GROWTH
) / 100;
370 /* Check the size after inlining against the function limits. But allow
371 the function to shrink if it went over the limits by forced inlining. */
372 newsize
= cgraph_estimate_size_after_inlining (times
, to
, what
);
373 if (newsize
>= to
->global
.insns
374 && newsize
> PARAM_VALUE (PARAM_LARGE_FUNCTION_INSNS
)
378 *reason
= N_("--param large-function-growth limit reached");
382 stack_size_limit
= inline_summary (to
)->estimated_self_stack_size
;
384 stack_size_limit
+= stack_size_limit
* PARAM_VALUE (PARAM_STACK_FRAME_GROWTH
) / 100;
386 inlined_stack
= (to
->global
.stack_frame_offset
387 + inline_summary (to
)->estimated_self_stack_size
388 + what
->global
.estimated_stack_size
);
389 if (inlined_stack
> stack_size_limit
390 && inlined_stack
> PARAM_VALUE (PARAM_LARGE_STACK_FRAME
))
393 *reason
= N_("--param large-stack-frame-growth limit reached");
399 /* Return true when function N is small enough to be inlined. */
402 cgraph_default_inline_p (struct cgraph_node
*n
, const char **reason
)
407 decl
= n
->inline_decl
;
408 if (!flag_inline_small_functions
&& !DECL_DECLARED_INLINE_P (decl
))
411 *reason
= N_("function not inline candidate");
415 if (!DECL_STRUCT_FUNCTION (decl
)->cfg
)
418 *reason
= N_("function body not available");
422 if (DECL_DECLARED_INLINE_P (decl
))
424 if (n
->global
.insns
>= MAX_INLINE_INSNS_SINGLE
)
427 *reason
= N_("--param max-inline-insns-single limit reached");
433 if (n
->global
.insns
>= MAX_INLINE_INSNS_AUTO
)
436 *reason
= N_("--param max-inline-insns-auto limit reached");
444 /* Return true when inlining WHAT would create recursive inlining.
445 We call recursive inlining all cases where same function appears more than
446 once in the single recursion nest path in the inline graph. */
449 cgraph_recursive_inlining_p (struct cgraph_node
*to
,
450 struct cgraph_node
*what
,
454 if (to
->global
.inlined_to
)
455 recursive
= what
->decl
== to
->global
.inlined_to
->decl
;
457 recursive
= what
->decl
== to
->decl
;
458 /* Marking recursive function inline has sane semantic and thus we should
460 if (recursive
&& reason
)
461 *reason
= (what
->local
.disregard_inline_limits
462 ? N_("recursive inlining") : "");
466 /* Return true if the call can be hot. */
468 cgraph_maybe_hot_edge_p (struct cgraph_edge
*edge
)
470 if (profile_info
&& flag_branch_probabilities
472 <= profile_info
->sum_max
/ PARAM_VALUE (HOT_BB_COUNT_FRACTION
)))
474 if (lookup_attribute ("cold", DECL_ATTRIBUTES (edge
->callee
->decl
))
475 || lookup_attribute ("cold", DECL_ATTRIBUTES (edge
->caller
->decl
)))
477 if (lookup_attribute ("hot", DECL_ATTRIBUTES (edge
->caller
->decl
)))
479 if (flag_guess_branch_prob
480 && edge
->frequency
< (CGRAPH_FREQ_MAX
481 / PARAM_VALUE (HOT_BB_FREQUENCY_FRACTION
)))
486 /* A cost model driving the inlining heuristics in a way so the edges with
487 smallest badness are inlined first. After each inlining is performed
488 the costs of all caller edges of nodes affected are recomputed so the
489 metrics may accurately depend on values such as number of inlinable callers
490 of the function or function body size. */
493 cgraph_edge_badness (struct cgraph_edge
*edge
)
497 cgraph_estimate_size_after_inlining (1, edge
->caller
, edge
->callee
);
499 growth
-= edge
->caller
->global
.insns
;
501 /* Always prefer inlining saving code size. */
503 badness
= INT_MIN
- growth
;
505 /* When profiling is available, base priorities -(#calls / growth).
506 So we optimize for overall number of "executed" inlined calls. */
508 badness
= ((int)((double)edge
->count
* INT_MIN
/ max_count
)) / growth
;
510 /* When function local profile is available, base priorities on
511 growth / frequency, so we optimize for overall frequency of inlined
512 calls. This is not too accurate since while the call might be frequent
513 within function, the function itself is infrequent.
515 Other objective to optimize for is number of different calls inlined.
516 We add the estimated growth after inlining all functions to bias the
517 priorities slightly in this direction (so fewer times called functions
518 of the same size gets priority). */
519 else if (flag_guess_branch_prob
)
521 int div
= edge
->frequency
* 100 / CGRAPH_FREQ_BASE
;
523 cgraph_estimate_size_after_inlining (1, edge
->caller
, edge
->callee
);
524 growth
-= edge
->caller
->global
.insns
;
525 badness
= growth
* 256;
527 /* Decrease badness if call is nested. */
528 /* Compress the range so we don't overflow. */
530 div
= 256 + ceil_log2 (div
) - 8;
535 badness
+= cgraph_estimate_growth (edge
->callee
);
537 /* When function local profile is not available or it does not give
538 useful information (ie frequency is zero), base the cost on
539 loop nest and overall size growth, so we optimize for overall number
540 of functions fully inlined in program. */
543 int nest
= MIN (edge
->loop_nest
, 8);
544 badness
= cgraph_estimate_growth (edge
->callee
) * 256;
546 /* Decrease badness if call is nested. */
554 /* Make recursive inlining happen always after other inlining is done. */
555 if (cgraph_recursive_inlining_p (edge
->caller
, edge
->callee
, NULL
))
561 /* Recompute heap nodes for each of caller edge. */
564 update_caller_keys (fibheap_t heap
, struct cgraph_node
*node
,
565 bitmap updated_nodes
)
567 struct cgraph_edge
*edge
;
568 const char *failed_reason
;
570 if (!node
->local
.inlinable
|| node
->local
.disregard_inline_limits
571 || node
->global
.inlined_to
)
573 if (bitmap_bit_p (updated_nodes
, node
->uid
))
575 bitmap_set_bit (updated_nodes
, node
->uid
);
576 node
->global
.estimated_growth
= INT_MIN
;
578 if (!node
->local
.inlinable
)
580 /* Prune out edges we won't inline into anymore. */
581 if (!cgraph_default_inline_p (node
, &failed_reason
))
583 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
586 fibheap_delete_node (heap
, (fibnode_t
) edge
->aux
);
588 if (edge
->inline_failed
)
589 edge
->inline_failed
= failed_reason
;
594 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
595 if (edge
->inline_failed
)
597 int badness
= cgraph_edge_badness (edge
);
600 fibnode_t n
= (fibnode_t
) edge
->aux
;
601 gcc_assert (n
->data
== edge
);
602 if (n
->key
== badness
)
605 /* fibheap_replace_key only increase the keys. */
606 if (fibheap_replace_key (heap
, n
, badness
))
608 fibheap_delete_node (heap
, (fibnode_t
) edge
->aux
);
610 edge
->aux
= fibheap_insert (heap
, badness
, edge
);
614 /* Recompute heap nodes for each of caller edges of each of callees. */
617 update_callee_keys (fibheap_t heap
, struct cgraph_node
*node
,
618 bitmap updated_nodes
)
620 struct cgraph_edge
*e
;
621 node
->global
.estimated_growth
= INT_MIN
;
623 for (e
= node
->callees
; e
; e
= e
->next_callee
)
624 if (e
->inline_failed
)
625 update_caller_keys (heap
, e
->callee
, updated_nodes
);
626 else if (!e
->inline_failed
)
627 update_callee_keys (heap
, e
->callee
, updated_nodes
);
630 /* Enqueue all recursive calls from NODE into priority queue depending on
631 how likely we want to recursively inline the call. */
634 lookup_recursive_calls (struct cgraph_node
*node
, struct cgraph_node
*where
,
638 struct cgraph_edge
*e
;
639 for (e
= where
->callees
; e
; e
= e
->next_callee
)
640 if (e
->callee
== node
)
642 /* When profile feedback is available, prioritize by expected number
643 of calls. Without profile feedback we maintain simple queue
644 to order candidates via recursive depths. */
645 fibheap_insert (heap
,
646 !max_count
? priority
++
647 : -(e
->count
/ ((max_count
+ (1<<24) - 1) / (1<<24))),
650 for (e
= where
->callees
; e
; e
= e
->next_callee
)
651 if (!e
->inline_failed
)
652 lookup_recursive_calls (node
, e
->callee
, heap
);
655 /* Decide on recursive inlining: in the case function has recursive calls,
656 inline until body size reaches given argument. If any new indirect edges
657 are discovered in the process, add them to NEW_EDGES, unless it is NULL. */
660 cgraph_decide_recursive_inlining (struct cgraph_node
*node
,
661 VEC (cgraph_edge_p
, heap
) *new_edges
)
663 int limit
= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE_AUTO
);
664 int max_depth
= PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH_AUTO
);
665 int probability
= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY
);
667 struct cgraph_edge
*e
;
668 struct cgraph_node
*master_clone
, *next
;
673 || (!flag_inline_functions
&& !DECL_DECLARED_INLINE_P (node
->decl
)))
676 if (DECL_DECLARED_INLINE_P (node
->decl
))
678 limit
= PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE
);
679 max_depth
= PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH
);
682 /* Make sure that function is small enough to be considered for inlining. */
684 || cgraph_estimate_size_after_inlining (1, node
, node
) >= limit
)
686 heap
= fibheap_new ();
687 lookup_recursive_calls (node
, node
, heap
);
688 if (fibheap_empty (heap
))
690 fibheap_delete (heap
);
696 " Performing recursive inlining on %s\n",
697 cgraph_node_name (node
));
699 /* We need original clone to copy around. */
700 master_clone
= cgraph_clone_node (node
, node
->count
, CGRAPH_FREQ_BASE
, 1, false);
701 master_clone
->needed
= true;
702 for (e
= master_clone
->callees
; e
; e
= e
->next_callee
)
703 if (!e
->inline_failed
)
704 cgraph_clone_inlined_nodes (e
, true, false);
706 /* Do the inlining and update list of recursive call during process. */
707 while (!fibheap_empty (heap
)
708 && (cgraph_estimate_size_after_inlining (1, node
, master_clone
)
711 struct cgraph_edge
*curr
712 = (struct cgraph_edge
*) fibheap_extract_min (heap
);
713 struct cgraph_node
*cnode
;
716 for (cnode
= curr
->caller
;
717 cnode
->global
.inlined_to
; cnode
= cnode
->callers
->caller
)
718 if (node
->decl
== curr
->callee
->decl
)
720 if (depth
> max_depth
)
724 " maximal depth reached\n");
730 if (!cgraph_maybe_hot_edge_p (curr
))
733 fprintf (dump_file
, " Not inlining cold call\n");
736 if (curr
->count
* 100 / node
->count
< probability
)
740 " Probability of edge is too small\n");
748 " Inlining call of depth %i", depth
);
751 fprintf (dump_file
, " called approx. %.2f times per call",
752 (double)curr
->count
/ node
->count
);
754 fprintf (dump_file
, "\n");
756 cgraph_redirect_edge_callee (curr
, master_clone
);
757 cgraph_mark_inline_edge (curr
, false);
758 if (flag_indirect_inlining
)
759 ipa_propagate_indirect_call_infos (curr
, new_edges
);
760 lookup_recursive_calls (node
, curr
->callee
, heap
);
763 if (!fibheap_empty (heap
) && dump_file
)
764 fprintf (dump_file
, " Recursive inlining growth limit met.\n");
766 fibheap_delete (heap
);
769 "\n Inlined %i times, body grown from %i to %i insns\n", n
,
770 master_clone
->global
.insns
, node
->global
.insns
);
772 /* Remove master clone we used for inlining. We rely that clones inlined
773 into master clone gets queued just before master clone so we don't
775 for (node
= cgraph_nodes
; node
!= master_clone
;
779 if (node
->global
.inlined_to
== master_clone
)
780 cgraph_remove_node (node
);
782 cgraph_remove_node (master_clone
);
783 /* FIXME: Recursive inlining actually reduces number of calls of the
784 function. At this place we should probably walk the function and
785 inline clones and compensate the counts accordingly. This probably
786 doesn't matter much in practice. */
790 /* Set inline_failed for all callers of given function to REASON. */
793 cgraph_set_inline_failed (struct cgraph_node
*node
, const char *reason
)
795 struct cgraph_edge
*e
;
798 fprintf (dump_file
, "Inlining failed: %s\n", reason
);
799 for (e
= node
->callers
; e
; e
= e
->next_caller
)
800 if (e
->inline_failed
)
801 e
->inline_failed
= reason
;
804 /* Given whole compilation unit estimate of INSNS, compute how large we can
805 allow the unit to grow. */
807 compute_max_insns (int insns
)
809 int max_insns
= insns
;
810 if (max_insns
< PARAM_VALUE (PARAM_LARGE_UNIT_INSNS
))
811 max_insns
= PARAM_VALUE (PARAM_LARGE_UNIT_INSNS
);
813 return ((HOST_WIDEST_INT
) max_insns
814 * (100 + PARAM_VALUE (PARAM_INLINE_UNIT_GROWTH
)) / 100);
817 /* Compute badness of all edges in NEW_EDGES and add them to the HEAP. */
819 add_new_edges_to_heap (fibheap_t heap
, VEC (cgraph_edge_p
, heap
) *new_edges
)
821 while (VEC_length (cgraph_edge_p
, new_edges
) > 0)
823 struct cgraph_edge
*edge
= VEC_pop (cgraph_edge_p
, new_edges
);
825 gcc_assert (!edge
->aux
);
826 edge
->aux
= fibheap_insert (heap
, cgraph_edge_badness (edge
), edge
);
831 /* We use greedy algorithm for inlining of small functions:
832 All inline candidates are put into prioritized heap based on estimated
833 growth of the overall number of instructions and then update the estimates.
835 INLINED and INLINED_CALEES are just pointers to arrays large enough
836 to be passed to cgraph_inlined_into and cgraph_inlined_callees. */
839 cgraph_decide_inlining_of_small_functions (void)
841 struct cgraph_node
*node
;
842 struct cgraph_edge
*edge
;
843 const char *failed_reason
;
844 fibheap_t heap
= fibheap_new ();
845 bitmap updated_nodes
= BITMAP_ALLOC (NULL
);
846 int min_insns
, max_insns
;
847 VEC (cgraph_edge_p
, heap
) *new_indirect_edges
= NULL
;
849 if (flag_indirect_inlining
)
850 new_indirect_edges
= VEC_alloc (cgraph_edge_p
, heap
, 8);
853 fprintf (dump_file
, "\nDeciding on smaller functions:\n");
855 /* Put all inline candidates into the heap. */
857 for (node
= cgraph_nodes
; node
; node
= node
->next
)
859 if (!node
->local
.inlinable
|| !node
->callers
860 || node
->local
.disregard_inline_limits
)
863 fprintf (dump_file
, "Considering inline candidate %s.\n", cgraph_node_name (node
));
865 node
->global
.estimated_growth
= INT_MIN
;
866 if (!cgraph_default_inline_p (node
, &failed_reason
))
868 cgraph_set_inline_failed (node
, failed_reason
);
872 for (edge
= node
->callers
; edge
; edge
= edge
->next_caller
)
873 if (edge
->inline_failed
)
875 gcc_assert (!edge
->aux
);
876 edge
->aux
= fibheap_insert (heap
, cgraph_edge_badness (edge
), edge
);
880 max_insns
= compute_max_insns (overall_insns
);
881 min_insns
= overall_insns
;
883 while (overall_insns
<= max_insns
884 && (edge
= (struct cgraph_edge
*) fibheap_extract_min (heap
)))
886 int old_insns
= overall_insns
;
887 struct cgraph_node
*where
;
889 cgraph_estimate_size_after_inlining (1, edge
->caller
, edge
->callee
);
890 const char *not_good
= NULL
;
892 growth
-= edge
->caller
->global
.insns
;
897 "\nConsidering %s with %i insns\n",
898 cgraph_node_name (edge
->callee
),
899 edge
->callee
->global
.insns
);
901 " to be inlined into %s\n"
902 " Estimated growth after inlined into all callees is %+i insns.\n"
903 " Estimated badness is %i, frequency %.2f.\n",
904 cgraph_node_name (edge
->caller
),
905 cgraph_estimate_growth (edge
->callee
),
906 cgraph_edge_badness (edge
),
907 edge
->frequency
/ (double)CGRAPH_FREQ_BASE
);
909 fprintf (dump_file
," Called "HOST_WIDEST_INT_PRINT_DEC
"x\n", edge
->count
);
911 gcc_assert (edge
->aux
);
913 if (!edge
->inline_failed
)
916 /* When not having profile info ready we don't weight by any way the
917 position of call in procedure itself. This means if call of
918 function A from function B seems profitable to inline, the recursive
919 call of function A in inline copy of A in B will look profitable too
920 and we end up inlining until reaching maximal function growth. This
921 is not good idea so prohibit the recursive inlining.
923 ??? When the frequencies are taken into account we might not need this
927 where
= edge
->caller
;
928 while (where
->global
.inlined_to
)
930 if (where
->decl
== edge
->callee
->decl
)
932 where
= where
->callers
->caller
;
934 if (where
->global
.inlined_to
)
937 = (edge
->callee
->local
.disregard_inline_limits
? N_("recursive inlining") : "");
939 fprintf (dump_file
, " inline_failed:Recursive inlining performed only for function itself.\n");
944 if (!cgraph_maybe_hot_edge_p (edge
))
945 not_good
= N_("call is unlikely and code size would grow");
946 if (!flag_inline_functions
947 && !DECL_DECLARED_INLINE_P (edge
->callee
->decl
))
948 not_good
= N_("function not declared inline and code size would grow");
950 not_good
= N_("optimizing for size and code size would grow");
951 if (not_good
&& growth
> 0 && cgraph_estimate_growth (edge
->callee
) > 0)
953 if (!cgraph_recursive_inlining_p (edge
->caller
, edge
->callee
,
954 &edge
->inline_failed
))
956 edge
->inline_failed
= not_good
;
958 fprintf (dump_file
, " inline_failed:%s.\n", edge
->inline_failed
);
962 if (!cgraph_default_inline_p (edge
->callee
, &edge
->inline_failed
))
964 if (!cgraph_recursive_inlining_p (edge
->caller
, edge
->callee
,
965 &edge
->inline_failed
))
968 fprintf (dump_file
, " inline_failed:%s.\n", edge
->inline_failed
);
972 if (!tree_can_inline_p (edge
->caller
->decl
, edge
->callee
->decl
))
974 CALL_STMT_CANNOT_INLINE_P (edge
->call_stmt
) = true;
975 edge
->inline_failed
= N_("target specific option mismatch");
977 fprintf (dump_file
, " inline_failed:%s.\n", edge
->inline_failed
);
980 if (cgraph_recursive_inlining_p (edge
->caller
, edge
->callee
,
981 &edge
->inline_failed
))
983 where
= edge
->caller
;
984 if (where
->global
.inlined_to
)
985 where
= where
->global
.inlined_to
;
986 if (!cgraph_decide_recursive_inlining (where
, new_indirect_edges
))
988 if (flag_indirect_inlining
)
989 add_new_edges_to_heap (heap
, new_indirect_edges
);
990 update_callee_keys (heap
, where
, updated_nodes
);
994 struct cgraph_node
*callee
;
995 if (CALL_STMT_CANNOT_INLINE_P (edge
->call_stmt
)
996 || !cgraph_check_inline_limits (edge
->caller
, edge
->callee
,
997 &edge
->inline_failed
, true))
1000 fprintf (dump_file
, " Not inlining into %s:%s.\n",
1001 cgraph_node_name (edge
->caller
), edge
->inline_failed
);
1004 callee
= edge
->callee
;
1005 cgraph_mark_inline_edge (edge
, true);
1006 if (flag_indirect_inlining
)
1008 ipa_propagate_indirect_call_infos (edge
, new_indirect_edges
);
1009 add_new_edges_to_heap (heap
, new_indirect_edges
);
1011 update_callee_keys (heap
, callee
, updated_nodes
);
1013 where
= edge
->caller
;
1014 if (where
->global
.inlined_to
)
1015 where
= where
->global
.inlined_to
;
1017 /* Our profitability metric can depend on local properties
1018 such as number of inlinable calls and size of the function body.
1019 After inlining these properties might change for the function we
1020 inlined into (since it's body size changed) and for the functions
1021 called by function we inlined (since number of it inlinable callers
1023 update_caller_keys (heap
, where
, updated_nodes
);
1024 bitmap_clear (updated_nodes
);
1029 " Inlined into %s which now has %i insns,"
1030 "net change of %+i insns.\n",
1031 cgraph_node_name (edge
->caller
),
1032 edge
->caller
->global
.insns
,
1033 overall_insns
- old_insns
);
1035 if (min_insns
> overall_insns
)
1037 min_insns
= overall_insns
;
1038 max_insns
= compute_max_insns (min_insns
);
1041 fprintf (dump_file
, "New minimal insns reached: %i\n", min_insns
);
1044 while ((edge
= (struct cgraph_edge
*) fibheap_extract_min (heap
)) != NULL
)
1046 gcc_assert (edge
->aux
);
1048 if (!edge
->callee
->local
.disregard_inline_limits
&& edge
->inline_failed
1049 && !cgraph_recursive_inlining_p (edge
->caller
, edge
->callee
,
1050 &edge
->inline_failed
))
1051 edge
->inline_failed
= N_("--param inline-unit-growth limit reached");
1054 if (new_indirect_edges
)
1055 VEC_free (cgraph_edge_p
, heap
, new_indirect_edges
);
1056 fibheap_delete (heap
);
1057 BITMAP_FREE (updated_nodes
);
1060 /* Decide on the inlining. We do so in the topological order to avoid
1061 expenses on updating data structures. */
1064 cgraph_decide_inlining (void)
1066 struct cgraph_node
*node
;
1068 struct cgraph_node
**order
=
1069 XCNEWVEC (struct cgraph_node
*, cgraph_n_nodes
);
1072 int initial_insns
= 0;
1075 for (node
= cgraph_nodes
; node
; node
= node
->next
)
1076 if (node
->analyzed
&& (node
->needed
|| node
->reachable
))
1078 struct cgraph_edge
*e
;
1080 initial_insns
+= inline_summary (node
)->self_insns
;
1081 gcc_assert (inline_summary (node
)->self_insns
== node
->global
.insns
);
1082 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1083 if (max_count
< e
->count
)
1084 max_count
= e
->count
;
1086 overall_insns
= initial_insns
;
1087 gcc_assert (!max_count
|| (profile_info
&& flag_branch_probabilities
));
1089 nnodes
= cgraph_postorder (order
);
1093 "\nDeciding on inlining. Starting with %i insns.\n",
1096 for (node
= cgraph_nodes
; node
; node
= node
->next
)
1100 fprintf (dump_file
, "\nInlining always_inline functions:\n");
1102 /* In the first pass mark all always_inline edges. Do this with a priority
1103 so none of our later choices will make this impossible. */
1104 for (i
= nnodes
- 1; i
>= 0; i
--)
1106 struct cgraph_edge
*e
, *next
;
1110 /* Handle nodes to be flattened, but don't update overall unit size. */
1111 if (lookup_attribute ("flatten", DECL_ATTRIBUTES (node
->decl
)) != NULL
)
1115 "Flattening %s\n", cgraph_node_name (node
));
1116 cgraph_decide_inlining_incrementally (node
, INLINE_ALL
, 0);
1119 if (!node
->local
.disregard_inline_limits
)
1123 "\nConsidering %s %i insns (always inline)\n",
1124 cgraph_node_name (node
), node
->global
.insns
);
1125 old_insns
= overall_insns
;
1126 for (e
= node
->callers
; e
; e
= next
)
1128 next
= e
->next_caller
;
1129 if (!e
->inline_failed
|| CALL_STMT_CANNOT_INLINE_P (e
->call_stmt
))
1131 if (cgraph_recursive_inlining_p (e
->caller
, e
->callee
,
1134 if (!tree_can_inline_p (e
->caller
->decl
, e
->callee
->decl
))
1136 CALL_STMT_CANNOT_INLINE_P (e
->call_stmt
) = true;
1139 cgraph_mark_inline_edge (e
, true);
1140 if (flag_indirect_inlining
)
1141 ipa_propagate_indirect_call_infos (e
, NULL
);
1144 " Inlined into %s which now has %i insns.\n",
1145 cgraph_node_name (e
->caller
),
1146 e
->caller
->global
.insns
);
1148 /* Inlining self recursive function might introduce new calls to
1149 themselves we didn't see in the loop above. Fill in the proper
1150 reason why inline failed. */
1151 for (e
= node
->callers
; e
; e
= e
->next_caller
)
1152 if (e
->inline_failed
)
1153 e
->inline_failed
= N_("recursive inlining");
1156 " Inlined for a net change of %+i insns.\n",
1157 overall_insns
- old_insns
);
1160 if (!flag_really_no_inline
)
1161 cgraph_decide_inlining_of_small_functions ();
1163 /* After this point, any edge discovery performed by indirect inlining is no
1164 good so let's give up. */
1165 if (flag_indirect_inlining
)
1166 free_all_ipa_structures_after_iinln ();
1168 if (!flag_really_no_inline
1169 && flag_inline_functions_called_once
)
1172 fprintf (dump_file
, "\nDeciding on functions called once:\n");
1174 /* And finally decide what functions are called once. */
1176 for (i
= nnodes
- 1; i
>= 0; i
--)
1180 if (node
->callers
&& !node
->callers
->next_caller
&& !node
->needed
1181 && node
->local
.inlinable
&& node
->callers
->inline_failed
1182 && !CALL_STMT_CANNOT_INLINE_P (node
->callers
->call_stmt
)
1183 && !DECL_EXTERNAL (node
->decl
) && !DECL_COMDAT (node
->decl
))
1188 "\nConsidering %s %i insns.\n",
1189 cgraph_node_name (node
), node
->global
.insns
);
1191 " Called once from %s %i insns.\n",
1192 cgraph_node_name (node
->callers
->caller
),
1193 node
->callers
->caller
->global
.insns
);
1196 old_insns
= overall_insns
;
1198 if (cgraph_check_inline_limits (node
->callers
->caller
, node
,
1201 cgraph_mark_inline (node
->callers
);
1204 " Inlined into %s which now has %i insns"
1205 " for a net change of %+i insns.\n",
1206 cgraph_node_name (node
->callers
->caller
),
1207 node
->callers
->caller
->global
.insns
,
1208 overall_insns
- old_insns
);
1214 " Inline limit reached, not inlined.\n");
1222 "\nInlined %i calls, eliminated %i functions, "
1223 "%i insns turned to %i insns.\n\n",
1224 ncalls_inlined
, nfunctions_inlined
, initial_insns
,
1230 /* Try to inline edge E from incremental inliner. MODE specifies mode
1233 We are detecting cycles by storing mode of inliner into cgraph_node last
1234 time we visited it in the recursion. In general when mode is set, we have
1235 recursive inlining, but as an special case, we want to try harder inline
1236 ALWAYS_INLINE functions: consider callgraph a->b->c->b, with a being
1237 flatten, b being always inline. Flattening 'a' will collapse
1238 a->b->c before hitting cycle. To accommodate always inline, we however
1239 need to inline a->b->c->b.
1241 So after hitting cycle first time, we switch into ALWAYS_INLINE mode and
1242 stop inlining only after hitting ALWAYS_INLINE in ALWAY_INLINE mode. */
1244 try_inline (struct cgraph_edge
*e
, enum inlining_mode mode
, int depth
)
1246 struct cgraph_node
*callee
= e
->callee
;
1247 enum inlining_mode callee_mode
= (enum inlining_mode
) (size_t) callee
->aux
;
1248 bool always_inline
= e
->callee
->local
.disregard_inline_limits
;
1250 /* We've hit cycle? */
1253 /* It is first time we see it and we are not in ALWAY_INLINE only
1254 mode yet. and the function in question is always_inline. */
1255 if (always_inline
&& mode
!= INLINE_ALWAYS_INLINE
)
1259 indent_to (dump_file
, depth
);
1261 "Hit cycle in %s, switching to always inline only.\n",
1262 cgraph_node_name (callee
));
1264 mode
= INLINE_ALWAYS_INLINE
;
1266 /* Otherwise it is time to give up. */
1271 indent_to (dump_file
, depth
);
1273 "Not inlining %s into %s to avoid cycle.\n",
1274 cgraph_node_name (callee
),
1275 cgraph_node_name (e
->caller
));
1277 e
->inline_failed
= (e
->callee
->local
.disregard_inline_limits
1278 ? N_("recursive inlining") : "");
1283 callee
->aux
= (void *)(size_t) mode
;
1286 indent_to (dump_file
, depth
);
1287 fprintf (dump_file
, " Inlining %s into %s.\n",
1288 cgraph_node_name (e
->callee
),
1289 cgraph_node_name (e
->caller
));
1291 if (e
->inline_failed
)
1292 cgraph_mark_inline (e
);
1294 /* In order to fully inline always_inline functions, we need to
1295 recurse here, since the inlined functions might not be processed by
1296 incremental inlining at all yet.
1298 Also flattening needs to be done recursively. */
1300 if (mode
== INLINE_ALL
|| always_inline
)
1301 cgraph_decide_inlining_incrementally (e
->callee
, mode
, depth
+ 1);
1302 callee
->aux
= (void *)(size_t) callee_mode
;
1306 /* Decide on the inlining. We do so in the topological order to avoid
1307 expenses on updating data structures.
1308 DEPTH is depth of recursion, used only for debug output. */
1311 cgraph_decide_inlining_incrementally (struct cgraph_node
*node
,
1312 enum inlining_mode mode
,
1315 struct cgraph_edge
*e
;
1316 bool inlined
= false;
1317 const char *failed_reason
;
1318 enum inlining_mode old_mode
;
1320 #ifdef ENABLE_CHECKING
1321 verify_cgraph_node (node
);
1324 old_mode
= (enum inlining_mode
) (size_t)node
->aux
;
1326 if (mode
!= INLINE_ALWAYS_INLINE
1327 && lookup_attribute ("flatten", DECL_ATTRIBUTES (node
->decl
)) != NULL
)
1331 indent_to (dump_file
, depth
);
1332 fprintf (dump_file
, "Flattening %s\n", cgraph_node_name (node
));
1337 node
->aux
= (void *)(size_t) mode
;
1339 /* First of all look for always inline functions. */
1340 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1342 if (!e
->callee
->local
.disregard_inline_limits
1343 && (mode
!= INLINE_ALL
|| !e
->callee
->local
.inlinable
))
1345 if (CALL_STMT_CANNOT_INLINE_P (e
->call_stmt
))
1347 /* When the edge is already inlined, we just need to recurse into
1348 it in order to fully flatten the leaves. */
1349 if (!e
->inline_failed
&& mode
== INLINE_ALL
)
1351 inlined
|= try_inline (e
, mode
, depth
);
1356 indent_to (dump_file
, depth
);
1358 "Considering to always inline inline candidate %s.\n",
1359 cgraph_node_name (e
->callee
));
1361 if (cgraph_recursive_inlining_p (node
, e
->callee
, &e
->inline_failed
))
1365 indent_to (dump_file
, depth
);
1366 fprintf (dump_file
, "Not inlining: recursive call.\n");
1370 if (!tree_can_inline_p (node
->decl
, e
->callee
->decl
))
1372 CALL_STMT_CANNOT_INLINE_P (e
->call_stmt
) = true;
1375 indent_to (dump_file
, depth
);
1377 "Not inlining: Target specific option mismatch.\n");
1381 if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node
->decl
))
1382 != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e
->callee
->decl
)))
1386 indent_to (dump_file
, depth
);
1387 fprintf (dump_file
, "Not inlining: SSA form does not match.\n");
1391 if (!DECL_SAVED_TREE (e
->callee
->decl
) && !e
->callee
->inline_decl
)
1395 indent_to (dump_file
, depth
);
1397 "Not inlining: Function body no longer available.\n");
1401 inlined
|= try_inline (e
, mode
, depth
);
1404 /* Now do the automatic inlining. */
1405 if (!flag_really_no_inline
&& mode
!= INLINE_ALL
1406 && mode
!= INLINE_ALWAYS_INLINE
)
1407 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1409 if (!e
->callee
->local
.inlinable
1410 || !e
->inline_failed
1411 || e
->callee
->local
.disregard_inline_limits
)
1414 fprintf (dump_file
, "Considering inline candidate %s.\n",
1415 cgraph_node_name (e
->callee
));
1416 if (cgraph_recursive_inlining_p (node
, e
->callee
, &e
->inline_failed
))
1420 indent_to (dump_file
, depth
);
1421 fprintf (dump_file
, "Not inlining: recursive call.\n");
1425 if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node
->decl
))
1426 != gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e
->callee
->decl
)))
1430 indent_to (dump_file
, depth
);
1431 fprintf (dump_file
, "Not inlining: SSA form does not match.\n");
1435 /* When the function body would grow and inlining the function won't
1436 eliminate the need for offline copy of the function, don't inline.
1438 if ((mode
== INLINE_SIZE
1439 || (!flag_inline_functions
1440 && !DECL_DECLARED_INLINE_P (e
->callee
->decl
)))
1441 && (cgraph_estimate_size_after_inlining (1, e
->caller
, e
->callee
)
1442 > e
->caller
->global
.insns
)
1443 && cgraph_estimate_growth (e
->callee
) > 0)
1447 indent_to (dump_file
, depth
);
1449 "Not inlining: code size would grow by %i insns.\n",
1450 cgraph_estimate_size_after_inlining (1, e
->caller
,
1452 - e
->caller
->global
.insns
);
1456 if (!cgraph_check_inline_limits (node
, e
->callee
, &e
->inline_failed
,
1458 || CALL_STMT_CANNOT_INLINE_P (e
->call_stmt
))
1462 indent_to (dump_file
, depth
);
1463 fprintf (dump_file
, "Not inlining: %s.\n", e
->inline_failed
);
1467 if (!DECL_SAVED_TREE (e
->callee
->decl
) && !e
->callee
->inline_decl
)
1471 indent_to (dump_file
, depth
);
1473 "Not inlining: Function body no longer available.\n");
1477 if (!tree_can_inline_p (node
->decl
, e
->callee
->decl
))
1479 CALL_STMT_CANNOT_INLINE_P (e
->call_stmt
) = true;
1482 indent_to (dump_file
, depth
);
1484 "Not inlining: Target specific option mismatch.\n");
1488 if (cgraph_default_inline_p (e
->callee
, &failed_reason
))
1489 inlined
|= try_inline (e
, mode
, depth
);
1491 node
->aux
= (void *)(size_t) old_mode
;
1495 /* Because inlining might remove no-longer reachable nodes, we need to
1496 keep the array visible to garbage collector to avoid reading collected
1499 static GTY ((length ("nnodes"))) struct cgraph_node
**order
;
1501 /* Do inlining of small functions. Doing so early helps profiling and other
1502 passes to be somewhat more effective and avoids some code duplication in
1503 later real inlining pass for testcases with very many function calls. */
1505 cgraph_early_inlining (void)
1507 struct cgraph_node
*node
= cgraph_node (current_function_decl
);
1508 unsigned int todo
= 0;
1510 if (sorrycount
|| errorcount
)
1512 if (cgraph_decide_inlining_incrementally (node
, INLINE_SIZE
, 0))
1514 timevar_push (TV_INTEGRATION
);
1515 todo
= optimize_inline_calls (current_function_decl
);
1516 timevar_pop (TV_INTEGRATION
);
1521 /* When inlining shall be performed. */
1523 cgraph_gate_early_inlining (void)
1525 return flag_early_inlining
;
1528 struct gimple_opt_pass pass_early_inline
=
1532 "einline", /* name */
1533 cgraph_gate_early_inlining
, /* gate */
1534 cgraph_early_inlining
, /* execute */
1537 0, /* static_pass_number */
1538 TV_INLINE_HEURISTICS
, /* tv_id */
1539 0, /* properties_required */
1540 PROP_cfg
, /* properties_provided */
1541 0, /* properties_destroyed */
1542 0, /* todo_flags_start */
1543 TODO_dump_func
/* todo_flags_finish */
1547 /* When inlining shall be performed. */
1549 cgraph_gate_ipa_early_inlining (void)
1551 return (flag_early_inlining
1552 && (flag_branch_probabilities
|| flag_test_coverage
1553 || profile_arc_flag
));
1556 /* IPA pass wrapper for early inlining pass. We need to run early inlining
1557 before tree profiling so we have stand alone IPA pass for doing so. */
1558 struct simple_ipa_opt_pass pass_ipa_early_inline
=
1562 "einline_ipa", /* name */
1563 cgraph_gate_ipa_early_inlining
, /* gate */
1567 0, /* static_pass_number */
1568 TV_INLINE_HEURISTICS
, /* tv_id */
1569 0, /* properties_required */
1570 PROP_cfg
, /* properties_provided */
1571 0, /* properties_destroyed */
1572 0, /* todo_flags_start */
1573 TODO_dump_cgraph
/* todo_flags_finish */
1577 /* Compute parameters of functions used by inliner. */
1579 compute_inline_parameters (struct cgraph_node
*node
)
1581 gcc_assert (!node
->global
.inlined_to
);
1582 inline_summary (node
)->estimated_self_stack_size
1583 = estimated_stack_frame_size ();
1584 node
->global
.estimated_stack_size
1585 = inline_summary (node
)->estimated_self_stack_size
;
1586 node
->global
.stack_frame_offset
= 0;
1587 node
->local
.inlinable
= tree_inlinable_function_p (current_function_decl
);
1588 inline_summary (node
)->self_insns
= estimate_num_insns (current_function_decl
,
1589 &eni_inlining_weights
);
1590 if (node
->local
.inlinable
&& !node
->local
.disregard_inline_limits
)
1591 node
->local
.disregard_inline_limits
1592 = DECL_DISREGARD_INLINE_LIMITS (current_function_decl
);
1593 if (flag_really_no_inline
&& !node
->local
.disregard_inline_limits
)
1594 node
->local
.inlinable
= 0;
1595 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
1596 node
->global
.insns
= inline_summary (node
)->self_insns
;
1601 /* Compute parameters of functions used by inliner using
1602 current_function_decl. */
1604 compute_inline_parameters_for_current (void)
1606 compute_inline_parameters (cgraph_node (current_function_decl
));
1610 struct gimple_opt_pass pass_inline_parameters
=
1616 compute_inline_parameters_for_current
,/* execute */
1619 0, /* static_pass_number */
1620 TV_INLINE_HEURISTICS
, /* tv_id */
1621 0, /* properties_required */
1622 PROP_cfg
, /* properties_provided */
1623 0, /* properties_destroyed */
1624 0, /* todo_flags_start */
1625 0 /* todo_flags_finish */
1629 /* This function performs intraprocedural analyzis in NODE that is required to
1630 inline indirect calls. */
1632 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
1634 struct cgraph_edge
*cs
;
1636 ipa_count_formal_params (node
);
1637 ipa_create_param_decls_array (node
);
1638 ipa_detect_param_modifications (node
);
1639 ipa_analyze_params_uses (node
);
1642 ipa_print_node_param_flags (dump_file
, node
);
1644 for (cs
= node
->callees
; cs
; cs
= cs
->next_callee
)
1646 ipa_count_arguments (cs
);
1647 ipa_compute_jump_functions (cs
);
1651 ipa_print_node_jump_functions (dump_file
, node
);
1654 /* Note function body size. */
1656 inline_generate_summary (void)
1658 struct cgraph_node
**order
=
1659 XCNEWVEC (struct cgraph_node
*, cgraph_n_nodes
);
1660 int nnodes
= cgraph_postorder (order
);
1663 if (flag_indirect_inlining
)
1665 ipa_register_cgraph_hooks ();
1666 ipa_check_create_node_params ();
1667 ipa_check_create_edge_args ();
1670 for (i
= nnodes
- 1; i
>= 0; i
--)
1672 struct cgraph_node
*node
= order
[i
];
1674 /* Allow possibly removed nodes to be garbage collected. */
1676 if (node
->analyzed
&& (node
->needed
|| node
->reachable
))
1678 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
1679 current_function_decl
= node
->decl
;
1680 compute_inline_parameters (node
);
1682 if (flag_indirect_inlining
)
1683 inline_indirect_intraprocedural_analysis (node
);
1689 current_function_decl
= NULL
;
1694 /* Apply inline plan to function. */
1696 inline_transform (struct cgraph_node
*node
)
1698 unsigned int todo
= 0;
1699 struct cgraph_edge
*e
;
1701 /* We might need the body of this function so that we can expand
1702 it inline somewhere else. */
1703 if (cgraph_preserve_function_body_p (current_function_decl
))
1704 save_inline_function_body (node
);
1706 for (e
= node
->callees
; e
; e
= e
->next_callee
)
1707 if (!e
->inline_failed
|| warn_inline
)
1711 timevar_push (TV_INTEGRATION
);
1712 todo
= optimize_inline_calls (current_function_decl
);
1713 timevar_pop (TV_INTEGRATION
);
1715 return todo
| execute_fixup_cfg ();
1718 struct ipa_opt_pass pass_ipa_inline
=
1722 "inline", /* name */
1724 cgraph_decide_inlining
, /* execute */
1727 0, /* static_pass_number */
1728 TV_INLINE_HEURISTICS
, /* tv_id */
1729 0, /* properties_required */
1730 PROP_cfg
, /* properties_provided */
1731 0, /* properties_destroyed */
1732 TODO_remove_functions
, /* todo_flags_finish */
1733 TODO_dump_cgraph
| TODO_dump_func
1734 | TODO_remove_functions
/* todo_flags_finish */
1736 inline_generate_summary
, /* generate_summary */
1737 NULL
, /* write_summary */
1738 NULL
, /* read_summary */
1739 NULL
, /* function_read_summary */
1741 inline_transform
, /* function_transform */
1742 NULL
, /* variable_transform */
1746 #include "gt-ipa-inline.h"