1 /* Inlining decision heuristics.
2 Copyright (C) 2003-2017 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Analysis used by the inliner and other passes limiting code size growth.
23 We estimate for each function
25 - average function execution time
26 - inlining size benefit (that is how much of function body size
27 and its call sequence is expected to disappear by inlining)
28 - inlining time benefit
31 - call statement size and time
33 inline_summary data structures store above information locally (i.e.
34 parameters of the function itself) and globally (i.e. parameters of
35 the function created by applying all the inline decisions already
36 present in the callgraph).
38 We provide access to the inline_summary data structure and
39 basic logic updating the parameters when inlining is performed.
41 The summaries are context sensitive. Context means
42 1) partial assignment of known constant values of operands
43 2) whether function is inlined into the call or not.
44 It is easy to add more variants. To represent function size and time
45 that depends on context (i.e. it is known to be optimized away when
46 context is known either by inlining or from IP-CP and cloning),
49 estimate_edge_size and estimate_edge_growth can be used to query
50 function size/time in the given context. inline_merge_summary merges
51 properties of caller and callee after inlining.
53 Finally pass_inline_parameters is exported. This is used to drive
54 computation of function parameters used by the early inliner. IPA
55 inlined performs analysis via its analyze_function method. */
59 #include "coretypes.h"
63 #include "alloc-pool.h"
64 #include "tree-pass.h"
66 #include "tree-streamer.h"
68 #include "diagnostic.h"
69 #include "fold-const.h"
70 #include "print-tree.h"
71 #include "tree-inline.h"
72 #include "gimple-pretty-print.h"
75 #include "gimple-iterator.h"
77 #include "tree-ssa-loop-niter.h"
78 #include "tree-ssa-loop.h"
79 #include "symbol-summary.h"
81 #include "ipa-inline.h"
83 #include "tree-scalar-evolution.h"
84 #include "ipa-utils.h"
86 #include "cfgexpand.h"
89 /* Holders of ipa cgraph hooks: */
90 static struct cgraph_2edge_hook_list
*edge_duplication_hook_holder
;
91 static struct cgraph_edge_hook_list
*edge_removal_hook_holder
;
92 static void inline_edge_removal_hook (struct cgraph_edge
*, void *);
93 static void inline_edge_duplication_hook (struct cgraph_edge
*,
94 struct cgraph_edge
*, void *);
96 /* VECtor holding inline summaries.
97 In GGC memory because conditions might point to constant trees. */
98 function_summary
<inline_summary
*> *inline_summaries
;
99 vec
<inline_edge_summary_t
> inline_edge_summary_vec
;
101 /* Cached node/edge growths. */
102 vec
<edge_growth_cache_entry
> edge_growth_cache
;
104 /* Edge predicates goes here. */
105 static object_allocator
<predicate
> edge_predicate_pool ("edge predicates");
108 /* Dump inline hints. */
110 dump_inline_hints (FILE *f
, inline_hints hints
)
114 fprintf (f
, "inline hints:");
115 if (hints
& INLINE_HINT_indirect_call
)
117 hints
&= ~INLINE_HINT_indirect_call
;
118 fprintf (f
, " indirect_call");
120 if (hints
& INLINE_HINT_loop_iterations
)
122 hints
&= ~INLINE_HINT_loop_iterations
;
123 fprintf (f
, " loop_iterations");
125 if (hints
& INLINE_HINT_loop_stride
)
127 hints
&= ~INLINE_HINT_loop_stride
;
128 fprintf (f
, " loop_stride");
130 if (hints
& INLINE_HINT_same_scc
)
132 hints
&= ~INLINE_HINT_same_scc
;
133 fprintf (f
, " same_scc");
135 if (hints
& INLINE_HINT_in_scc
)
137 hints
&= ~INLINE_HINT_in_scc
;
138 fprintf (f
, " in_scc");
140 if (hints
& INLINE_HINT_cross_module
)
142 hints
&= ~INLINE_HINT_cross_module
;
143 fprintf (f
, " cross_module");
145 if (hints
& INLINE_HINT_declared_inline
)
147 hints
&= ~INLINE_HINT_declared_inline
;
148 fprintf (f
, " declared_inline");
150 if (hints
& INLINE_HINT_array_index
)
152 hints
&= ~INLINE_HINT_array_index
;
153 fprintf (f
, " array_index");
155 if (hints
& INLINE_HINT_known_hot
)
157 hints
&= ~INLINE_HINT_known_hot
;
158 fprintf (f
, " known_hot");
164 /* Record SIZE and TIME to SUMMARY.
165 The accounted code will be executed when EXEC_PRED is true.
166 When NONCONST_PRED is false the code will evaulate to constant and
167 will get optimized out in specialized clones of the function. */
170 account_size_time (struct inline_summary
*summary
, int size
, sreal time
,
171 predicate
*exec_pred
,
172 predicate
*nonconst_pred_ptr
)
177 predicate nonconst_pred
;
179 if (*exec_pred
== false)
182 nonconst_pred
= *nonconst_pred_ptr
& *exec_pred
;
184 if (nonconst_pred
== false)
187 /* We need to create initial empty unconitional clause, but otherwie
188 we don't need to account empty times and sizes. */
189 if (!size
&& time
== 0 && summary
->entry
)
192 gcc_assert (time
>= 0);
194 for (i
= 0; vec_safe_iterate (summary
->entry
, i
, &e
); i
++)
195 if (e
->exec_predicate
== *exec_pred
196 && e
->nonconst_predicate
== nonconst_pred
)
205 e
= &(*summary
->entry
)[0];
206 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
208 "\t\tReached limit on number of entries, "
209 "ignoring the predicate.");
211 if (dump_file
&& (dump_flags
& TDF_DETAILS
) && (time
!= 0 || size
))
214 "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate exec:",
215 ((double) size
) / INLINE_SIZE_SCALE
,
216 (time
.to_double ()), found
? "" : "new ");
217 exec_pred
->dump (dump_file
, summary
->conds
, 0);
218 if (*exec_pred
!= nonconst_pred
)
220 fprintf (dump_file
, " nonconst:");
221 nonconst_pred
.dump (dump_file
, summary
->conds
);
224 fprintf (dump_file
, "\n");
228 struct size_time_entry new_entry
;
229 new_entry
.size
= size
;
230 new_entry
.time
= time
;
231 new_entry
.exec_predicate
= *exec_pred
;
232 new_entry
.nonconst_predicate
= nonconst_pred
;
233 vec_safe_push (summary
->entry
, new_entry
);
242 /* We proved E to be unreachable, redirect it to __bultin_unreachable. */
244 static struct cgraph_edge
*
245 redirect_to_unreachable (struct cgraph_edge
*e
)
247 struct cgraph_node
*callee
= !e
->inline_failed
? e
->callee
: NULL
;
248 struct cgraph_node
*target
= cgraph_node::get_create
249 (builtin_decl_implicit (BUILT_IN_UNREACHABLE
));
252 e
= e
->resolve_speculation (target
->decl
);
254 e
->make_direct (target
);
256 e
->redirect_callee (target
);
257 struct inline_edge_summary
*es
= inline_edge_summary (e
);
258 e
->inline_failed
= CIF_UNREACHABLE
;
261 es
->call_stmt_size
= 0;
262 es
->call_stmt_time
= 0;
264 callee
->remove_symbol_and_inline_clones ();
268 /* Set predicate for edge E. */
271 edge_set_predicate (struct cgraph_edge
*e
, predicate
*predicate
)
273 /* If the edge is determined to be never executed, redirect it
274 to BUILTIN_UNREACHABLE to save inliner from inlining into it. */
275 if (predicate
&& *predicate
== false
276 /* When handling speculative edges, we need to do the redirection
277 just once. Do it always on the direct edge, so we do not
278 attempt to resolve speculation while duplicating the edge. */
279 && (!e
->speculative
|| e
->callee
))
280 e
= redirect_to_unreachable (e
);
282 struct inline_edge_summary
*es
= inline_edge_summary (e
);
283 if (predicate
&& *predicate
!= true)
286 es
->predicate
= edge_predicate_pool
.allocate ();
287 *es
->predicate
= *predicate
;
292 edge_predicate_pool
.remove (es
->predicate
);
293 es
->predicate
= NULL
;
297 /* Set predicate for hint *P. */
300 set_hint_predicate (predicate
**p
, predicate new_predicate
)
302 if (new_predicate
== false || new_predicate
== true)
305 edge_predicate_pool
.remove (*p
);
311 *p
= edge_predicate_pool
.allocate ();
317 /* Compute what conditions may or may not hold given invormation about
318 parameters. RET_CLAUSE returns truths that may hold in a specialized copy,
319 whie RET_NONSPEC_CLAUSE returns truths that may hold in an nonspecialized
320 copy when called in a given context. It is a bitmask of conditions. Bit
321 0 means that condition is known to be false, while bit 1 means that condition
322 may or may not be true. These differs - for example NOT_INLINED condition
323 is always false in the second and also builtin_constant_p tests can not use
324 the fact that parameter is indeed a constant.
326 KNOWN_VALS is partial mapping of parameters of NODE to constant values.
327 KNOWN_AGGS is a vector of aggreggate jump functions for each parameter.
328 Return clause of possible truths. When INLINE_P is true, assume that we are
331 ERROR_MARK means compile time invariant. */
334 evaluate_conditions_for_known_args (struct cgraph_node
*node
,
336 vec
<tree
> known_vals
,
337 vec
<ipa_agg_jump_function_p
>
339 clause_t
*ret_clause
,
340 clause_t
*ret_nonspec_clause
)
342 clause_t clause
= inline_p
? 0 : 1 << predicate::not_inlined_condition
;
343 clause_t nonspec_clause
= 1 << predicate::not_inlined_condition
;
344 struct inline_summary
*info
= inline_summaries
->get (node
);
348 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
353 /* We allow call stmt to have fewer arguments than the callee function
354 (especially for K&R style programs). So bound check here (we assume
355 known_aggs vector, if non-NULL, has the same length as
357 gcc_checking_assert (!known_aggs
.exists ()
358 || (known_vals
.length () == known_aggs
.length ()));
359 if (c
->operand_num
>= (int) known_vals
.length ())
361 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
362 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
368 struct ipa_agg_jump_function
*agg
;
370 if (c
->code
== predicate::changed
372 && (known_vals
[c
->operand_num
] == error_mark_node
))
375 if (known_aggs
.exists ())
377 agg
= known_aggs
[c
->operand_num
];
378 val
= ipa_find_agg_cst_for_param (agg
, known_vals
[c
->operand_num
],
379 c
->offset
, c
->by_ref
);
386 val
= known_vals
[c
->operand_num
];
387 if (val
== error_mark_node
&& c
->code
!= predicate::changed
)
393 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
394 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
397 if (c
->code
== predicate::changed
)
399 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
403 if (tree_to_shwi (TYPE_SIZE (TREE_TYPE (val
))) != c
->size
)
405 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
406 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
409 if (c
->code
== predicate::is_not_constant
)
411 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
415 val
= fold_unary (VIEW_CONVERT_EXPR
, TREE_TYPE (c
->val
), val
);
417 ? fold_binary_to_constant (c
->code
, boolean_type_node
, val
, c
->val
)
420 if (res
&& integer_zerop (res
))
423 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
424 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
426 *ret_clause
= clause
;
427 if (ret_nonspec_clause
)
428 *ret_nonspec_clause
= nonspec_clause
;
432 /* Work out what conditions might be true at invocation of E. */
435 evaluate_properties_for_edge (struct cgraph_edge
*e
, bool inline_p
,
436 clause_t
*clause_ptr
, clause_t
*nonspec_clause_ptr
,
437 vec
<tree
> *known_vals_ptr
,
438 vec
<ipa_polymorphic_call_context
>
440 vec
<ipa_agg_jump_function_p
> *known_aggs_ptr
)
442 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
443 struct inline_summary
*info
= inline_summaries
->get (callee
);
444 vec
<tree
> known_vals
= vNULL
;
445 vec
<ipa_agg_jump_function_p
> known_aggs
= vNULL
;
448 *clause_ptr
= inline_p
? 0 : 1 << predicate::not_inlined_condition
;
450 known_vals_ptr
->create (0);
451 if (known_contexts_ptr
)
452 known_contexts_ptr
->create (0);
454 if (ipa_node_params_sum
455 && !e
->call_stmt_cannot_inline_p
456 && ((clause_ptr
&& info
->conds
) || known_vals_ptr
|| known_contexts_ptr
))
458 struct ipa_node_params
*parms_info
;
459 struct ipa_edge_args
*args
= IPA_EDGE_REF (e
);
460 struct inline_edge_summary
*es
= inline_edge_summary (e
);
461 int i
, count
= ipa_get_cs_argument_count (args
);
463 if (e
->caller
->global
.inlined_to
)
464 parms_info
= IPA_NODE_REF (e
->caller
->global
.inlined_to
);
466 parms_info
= IPA_NODE_REF (e
->caller
);
468 if (count
&& (info
->conds
|| known_vals_ptr
))
469 known_vals
.safe_grow_cleared (count
);
470 if (count
&& (info
->conds
|| known_aggs_ptr
))
471 known_aggs
.safe_grow_cleared (count
);
472 if (count
&& known_contexts_ptr
)
473 known_contexts_ptr
->safe_grow_cleared (count
);
475 for (i
= 0; i
< count
; i
++)
477 struct ipa_jump_func
*jf
= ipa_get_ith_jump_func (args
, i
);
478 tree cst
= ipa_value_from_jfunc (parms_info
, jf
);
480 if (!cst
&& e
->call_stmt
481 && i
< (int)gimple_call_num_args (e
->call_stmt
))
483 cst
= gimple_call_arg (e
->call_stmt
, i
);
484 if (!is_gimple_min_invariant (cst
))
489 gcc_checking_assert (TREE_CODE (cst
) != TREE_BINFO
);
490 if (known_vals
.exists ())
493 else if (inline_p
&& !es
->param
[i
].change_prob
)
494 known_vals
[i
] = error_mark_node
;
496 if (known_contexts_ptr
)
497 (*known_contexts_ptr
)[i
] = ipa_context_from_jfunc (parms_info
, e
,
499 /* TODO: When IPA-CP starts propagating and merging aggregate jump
500 functions, use its knowledge of the caller too, just like the
501 scalar case above. */
502 known_aggs
[i
] = &jf
->agg
;
505 else if (e
->call_stmt
&& !e
->call_stmt_cannot_inline_p
506 && ((clause_ptr
&& info
->conds
) || known_vals_ptr
))
508 int i
, count
= (int)gimple_call_num_args (e
->call_stmt
);
510 if (count
&& (info
->conds
|| known_vals_ptr
))
511 known_vals
.safe_grow_cleared (count
);
512 for (i
= 0; i
< count
; i
++)
514 tree cst
= gimple_call_arg (e
->call_stmt
, i
);
515 if (!is_gimple_min_invariant (cst
))
522 evaluate_conditions_for_known_args (callee
, inline_p
,
523 known_vals
, known_aggs
, clause_ptr
,
527 *known_vals_ptr
= known_vals
;
529 known_vals
.release ();
532 *known_aggs_ptr
= known_aggs
;
534 known_aggs
.release ();
538 /* Allocate the inline summary vector or resize it to cover all cgraph nodes. */
541 inline_summary_alloc (void)
543 if (!edge_removal_hook_holder
)
544 edge_removal_hook_holder
=
545 symtab
->add_edge_removal_hook (&inline_edge_removal_hook
, NULL
);
546 if (!edge_duplication_hook_holder
)
547 edge_duplication_hook_holder
=
548 symtab
->add_edge_duplication_hook (&inline_edge_duplication_hook
, NULL
);
550 if (!inline_summaries
)
551 inline_summaries
= (inline_summary_t
*) inline_summary_t::create_ggc (symtab
);
553 if (inline_edge_summary_vec
.length () <= (unsigned) symtab
->edges_max_uid
)
554 inline_edge_summary_vec
.safe_grow_cleared (symtab
->edges_max_uid
+ 1);
557 /* We are called multiple time for given function; clear
558 data from previous run so they are not cumulated. */
561 reset_inline_edge_summary (struct cgraph_edge
*e
)
563 if (e
->uid
< (int) inline_edge_summary_vec
.length ())
565 struct inline_edge_summary
*es
= inline_edge_summary (e
);
567 es
->call_stmt_size
= es
->call_stmt_time
= 0;
569 edge_predicate_pool
.remove (es
->predicate
);
570 es
->predicate
= NULL
;
571 es
->param
.release ();
575 /* We are called multiple time for given function; clear
576 data from previous run so they are not cumulated. */
579 reset_inline_summary (struct cgraph_node
*node
,
580 inline_summary
*info
)
582 struct cgraph_edge
*e
;
586 info
->estimated_stack_size
= 0;
587 info
->estimated_self_stack_size
= 0;
588 info
->stack_frame_offset
= 0;
593 if (info
->loop_iterations
)
595 edge_predicate_pool
.remove (info
->loop_iterations
);
596 info
->loop_iterations
= NULL
;
598 if (info
->loop_stride
)
600 edge_predicate_pool
.remove (info
->loop_stride
);
601 info
->loop_stride
= NULL
;
603 if (info
->array_index
)
605 edge_predicate_pool
.remove (info
->array_index
);
606 info
->array_index
= NULL
;
608 vec_free (info
->conds
);
609 vec_free (info
->entry
);
610 for (e
= node
->callees
; e
; e
= e
->next_callee
)
611 reset_inline_edge_summary (e
);
612 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
613 reset_inline_edge_summary (e
);
614 info
->fp_expressions
= false;
617 /* Hook that is called by cgraph.c when a node is removed. */
620 inline_summary_t::remove (cgraph_node
*node
, inline_summary
*info
)
622 reset_inline_summary (node
, info
);
625 /* Same as remap_predicate_after_duplication but handle hint predicate *P.
626 Additionally care about allocating new memory slot for updated predicate
627 and set it to NULL when it becomes true or false (and thus uninteresting).
631 remap_hint_predicate_after_duplication (predicate
**p
,
632 clause_t possible_truths
)
634 predicate new_predicate
;
639 new_predicate
= (*p
)->remap_after_duplication (possible_truths
);
640 /* We do not want to free previous predicate; it is used by node origin. */
642 set_hint_predicate (p
, new_predicate
);
646 /* Hook that is called by cgraph.c when a node is duplicated. */
648 inline_summary_t::duplicate (cgraph_node
*src
,
651 inline_summary
*info
)
653 inline_summary_alloc ();
654 memcpy (info
, inline_summaries
->get (src
), sizeof (inline_summary
));
655 /* TODO: as an optimization, we may avoid copying conditions
656 that are known to be false or true. */
657 info
->conds
= vec_safe_copy (info
->conds
);
659 /* When there are any replacements in the function body, see if we can figure
660 out that something was optimized out. */
661 if (ipa_node_params_sum
&& dst
->clone
.tree_map
)
663 vec
<size_time_entry
, va_gc
> *entry
= info
->entry
;
664 /* Use SRC parm info since it may not be copied yet. */
665 struct ipa_node_params
*parms_info
= IPA_NODE_REF (src
);
666 vec
<tree
> known_vals
= vNULL
;
667 int count
= ipa_get_param_count (parms_info
);
669 clause_t possible_truths
;
670 predicate true_pred
= true;
672 int optimized_out_size
= 0;
673 bool inlined_to_p
= false;
674 struct cgraph_edge
*edge
, *next
;
677 known_vals
.safe_grow_cleared (count
);
678 for (i
= 0; i
< count
; i
++)
680 struct ipa_replace_map
*r
;
682 for (j
= 0; vec_safe_iterate (dst
->clone
.tree_map
, j
, &r
); j
++)
684 if (((!r
->old_tree
&& r
->parm_num
== i
)
685 || (r
->old_tree
&& r
->old_tree
== ipa_get_param (parms_info
, i
)))
686 && r
->replace_p
&& !r
->ref_p
)
688 known_vals
[i
] = r
->new_tree
;
693 evaluate_conditions_for_known_args (dst
, false,
697 /* We are going to specialize,
698 so ignore nonspec truths. */
700 known_vals
.release ();
702 account_size_time (info
, 0, 0, &true_pred
, &true_pred
);
704 /* Remap size_time vectors.
705 Simplify the predicate by prunning out alternatives that are known
707 TODO: as on optimization, we can also eliminate conditions known
709 for (i
= 0; vec_safe_iterate (entry
, i
, &e
); i
++)
711 predicate new_exec_pred
;
712 predicate new_nonconst_pred
;
713 new_exec_pred
= e
->exec_predicate
.remap_after_duplication
715 new_nonconst_pred
= e
->nonconst_predicate
.remap_after_duplication
717 if (new_exec_pred
== false || new_nonconst_pred
== false)
718 optimized_out_size
+= e
->size
;
720 account_size_time (info
, e
->size
, e
->time
, &new_exec_pred
,
724 /* Remap edge predicates with the same simplification as above.
725 Also copy constantness arrays. */
726 for (edge
= dst
->callees
; edge
; edge
= next
)
728 predicate new_predicate
;
729 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
730 next
= edge
->next_callee
;
732 if (!edge
->inline_failed
)
736 new_predicate
= es
->predicate
->remap_after_duplication
738 if (new_predicate
== false && *es
->predicate
!= false)
739 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
740 edge_set_predicate (edge
, &new_predicate
);
743 /* Remap indirect edge predicates with the same simplificaiton as above.
744 Also copy constantness arrays. */
745 for (edge
= dst
->indirect_calls
; edge
; edge
= next
)
747 predicate new_predicate
;
748 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
749 next
= edge
->next_callee
;
751 gcc_checking_assert (edge
->inline_failed
);
754 new_predicate
= es
->predicate
->remap_after_duplication
756 if (new_predicate
== false && *es
->predicate
!= false)
757 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
758 edge_set_predicate (edge
, &new_predicate
);
760 remap_hint_predicate_after_duplication (&info
->loop_iterations
,
762 remap_hint_predicate_after_duplication (&info
->loop_stride
,
764 remap_hint_predicate_after_duplication (&info
->array_index
,
767 /* If inliner or someone after inliner will ever start producing
768 non-trivial clones, we will get trouble with lack of information
769 about updating self sizes, because size vectors already contains
770 sizes of the calees. */
771 gcc_assert (!inlined_to_p
|| !optimized_out_size
);
775 info
->entry
= vec_safe_copy (info
->entry
);
776 if (info
->loop_iterations
)
778 predicate p
= *info
->loop_iterations
;
779 info
->loop_iterations
= NULL
;
780 set_hint_predicate (&info
->loop_iterations
, p
);
782 if (info
->loop_stride
)
784 predicate p
= *info
->loop_stride
;
785 info
->loop_stride
= NULL
;
786 set_hint_predicate (&info
->loop_stride
, p
);
788 if (info
->array_index
)
790 predicate p
= *info
->array_index
;
791 info
->array_index
= NULL
;
792 set_hint_predicate (&info
->array_index
, p
);
795 if (!dst
->global
.inlined_to
)
796 inline_update_overall_summary (dst
);
800 /* Hook that is called by cgraph.c when a node is duplicated. */
803 inline_edge_duplication_hook (struct cgraph_edge
*src
,
804 struct cgraph_edge
*dst
,
805 ATTRIBUTE_UNUSED
void *data
)
807 struct inline_edge_summary
*info
;
808 struct inline_edge_summary
*srcinfo
;
809 inline_summary_alloc ();
810 info
= inline_edge_summary (dst
);
811 srcinfo
= inline_edge_summary (src
);
812 memcpy (info
, srcinfo
, sizeof (struct inline_edge_summary
));
813 info
->predicate
= NULL
;
814 edge_set_predicate (dst
, srcinfo
->predicate
);
815 info
->param
= srcinfo
->param
.copy ();
816 if (!dst
->indirect_unknown_callee
&& src
->indirect_unknown_callee
)
818 info
->call_stmt_size
-= (eni_size_weights
.indirect_call_cost
819 - eni_size_weights
.call_cost
);
820 info
->call_stmt_time
-= (eni_time_weights
.indirect_call_cost
821 - eni_time_weights
.call_cost
);
826 /* Keep edge cache consistent across edge removal. */
829 inline_edge_removal_hook (struct cgraph_edge
*edge
,
830 void *data ATTRIBUTE_UNUSED
)
832 if (edge_growth_cache
.exists ())
833 reset_edge_growth_cache (edge
);
834 reset_inline_edge_summary (edge
);
838 /* Initialize growth caches. */
841 initialize_growth_caches (void)
843 if (symtab
->edges_max_uid
)
844 edge_growth_cache
.safe_grow_cleared (symtab
->edges_max_uid
);
848 /* Free growth caches. */
851 free_growth_caches (void)
853 edge_growth_cache
.release ();
857 /* Dump edge summaries associated to NODE and recursively to all clones.
861 dump_inline_edge_summary (FILE *f
, int indent
, struct cgraph_node
*node
,
862 struct inline_summary
*info
)
864 struct cgraph_edge
*edge
;
865 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
867 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
868 struct cgraph_node
*callee
= edge
->callee
->ultimate_alias_target ();
872 "%*s%s/%i %s\n%*s loop depth:%2i freq:%4i size:%2i"
873 " time: %2i callee size:%2i stack:%2i",
874 indent
, "", callee
->name (), callee
->order
,
876 ? "inlined" : cgraph_inline_failed_string (edge
-> inline_failed
),
877 indent
, "", es
->loop_depth
, edge
->frequency
,
878 es
->call_stmt_size
, es
->call_stmt_time
,
879 (int) inline_summaries
->get (callee
)->size
/ INLINE_SIZE_SCALE
,
880 (int) inline_summaries
->get (callee
)->estimated_stack_size
);
884 fprintf (f
, " predicate: ");
885 es
->predicate
->dump (f
, info
->conds
);
889 if (es
->param
.exists ())
890 for (i
= 0; i
< (int) es
->param
.length (); i
++)
892 int prob
= es
->param
[i
].change_prob
;
895 fprintf (f
, "%*s op%i is compile time invariant\n",
897 else if (prob
!= REG_BR_PROB_BASE
)
898 fprintf (f
, "%*s op%i change %f%% of time\n", indent
+ 2, "", i
,
899 prob
* 100.0 / REG_BR_PROB_BASE
);
901 if (!edge
->inline_failed
)
903 fprintf (f
, "%*sStack frame offset %i, callee self size %i,"
906 (int) inline_summaries
->get (callee
)->stack_frame_offset
,
907 (int) inline_summaries
->get (callee
)->estimated_self_stack_size
,
908 (int) inline_summaries
->get (callee
)->estimated_stack_size
);
909 dump_inline_edge_summary (f
, indent
+ 2, callee
, info
);
912 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
914 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
915 fprintf (f
, "%*sindirect call loop depth:%2i freq:%4i size:%2i"
919 edge
->frequency
, es
->call_stmt_size
, es
->call_stmt_time
);
922 fprintf (f
, "predicate: ");
923 es
->predicate
->dump (f
, info
->conds
);
932 dump_inline_summary (FILE *f
, struct cgraph_node
*node
)
934 if (node
->definition
)
936 struct inline_summary
*s
= inline_summaries
->get (node
);
939 fprintf (f
, "Inline summary for %s/%i", node
->name (),
941 if (DECL_DISREGARD_INLINE_LIMITS (node
->decl
))
942 fprintf (f
, " always_inline");
944 fprintf (f
, " inlinable");
945 if (s
->contains_cilk_spawn
)
946 fprintf (f
, " contains_cilk_spawn");
947 if (s
->fp_expressions
)
948 fprintf (f
, " fp_expression");
949 fprintf (f
, "\n self time: %f\n", s
->self_time
.to_double ());
950 fprintf (f
, " global time: %f\n", s
->time
.to_double ());
951 fprintf (f
, " self size: %i\n", s
->self_size
);
952 fprintf (f
, " global size: %i\n", s
->size
);
953 fprintf (f
, " min size: %i\n", s
->min_size
);
954 fprintf (f
, " self stack: %i\n",
955 (int) s
->estimated_self_stack_size
);
956 fprintf (f
, " global stack: %i\n", (int) s
->estimated_stack_size
);
958 fprintf (f
, " estimated growth:%i\n", (int) s
->growth
);
960 fprintf (f
, " In SCC: %i\n", (int) s
->scc_no
);
961 for (i
= 0; vec_safe_iterate (s
->entry
, i
, &e
); i
++)
963 fprintf (f
, " size:%f, time:%f",
964 (double) e
->size
/ INLINE_SIZE_SCALE
,
965 e
->time
.to_double ());
966 if (e
->exec_predicate
!= true)
968 fprintf (f
, ", executed if:");
969 e
->exec_predicate
.dump (f
, s
->conds
, 0);
971 if (e
->exec_predicate
!= e
->nonconst_predicate
)
973 fprintf (f
, ", nonconst if:");
974 e
->nonconst_predicate
.dump (f
, s
->conds
, 0);
978 if (s
->loop_iterations
)
980 fprintf (f
, " loop iterations:");
981 s
->loop_iterations
->dump (f
, s
->conds
);
985 fprintf (f
, " loop stride:");
986 s
->loop_stride
->dump (f
, s
->conds
);
990 fprintf (f
, " array index:");
991 s
->array_index
->dump (f
, s
->conds
);
993 fprintf (f
, " calls:\n");
994 dump_inline_edge_summary (f
, 4, node
, s
);
1000 debug_inline_summary (struct cgraph_node
*node
)
1002 dump_inline_summary (stderr
, node
);
1006 dump_inline_summaries (FILE *f
)
1008 struct cgraph_node
*node
;
1010 FOR_EACH_DEFINED_FUNCTION (node
)
1011 if (!node
->global
.inlined_to
)
1012 dump_inline_summary (f
, node
);
1015 /* Give initial reasons why inlining would fail on EDGE. This gets either
1016 nullified or usually overwritten by more precise reasons later. */
1019 initialize_inline_failed (struct cgraph_edge
*e
)
1021 struct cgraph_node
*callee
= e
->callee
;
1023 if (e
->inline_failed
&& e
->inline_failed
!= CIF_BODY_NOT_AVAILABLE
1024 && cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
1026 else if (e
->indirect_unknown_callee
)
1027 e
->inline_failed
= CIF_INDIRECT_UNKNOWN_CALL
;
1028 else if (!callee
->definition
)
1029 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
1030 else if (callee
->local
.redefined_extern_inline
)
1031 e
->inline_failed
= CIF_REDEFINED_EXTERN_INLINE
;
1033 e
->inline_failed
= CIF_FUNCTION_NOT_CONSIDERED
;
1034 gcc_checking_assert (!e
->call_stmt_cannot_inline_p
1035 || cgraph_inline_failed_type (e
->inline_failed
)
1036 == CIF_FINAL_ERROR
);
1039 /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
1040 boolean variable pointed to by DATA. */
1043 mark_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef ATTRIBUTE_UNUSED
,
1046 bool *b
= (bool *) data
;
1051 /* If OP refers to value of function parameter, return the corresponding
1052 parameter. If non-NULL, the size of the memory load (or the SSA_NAME of the
1053 PARM_DECL) will be stored to *SIZE_P in that case too. */
1056 unmodified_parm_1 (gimple
*stmt
, tree op
, HOST_WIDE_INT
*size_p
)
1058 /* SSA_NAME referring to parm default def? */
1059 if (TREE_CODE (op
) == SSA_NAME
1060 && SSA_NAME_IS_DEFAULT_DEF (op
)
1061 && TREE_CODE (SSA_NAME_VAR (op
)) == PARM_DECL
)
1064 *size_p
= tree_to_shwi (TYPE_SIZE (TREE_TYPE (op
)));
1065 return SSA_NAME_VAR (op
);
1067 /* Non-SSA parm reference? */
1068 if (TREE_CODE (op
) == PARM_DECL
)
1070 bool modified
= false;
1073 ao_ref_init (&refd
, op
);
1074 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), mark_modified
, &modified
,
1079 *size_p
= tree_to_shwi (TYPE_SIZE (TREE_TYPE (op
)));
1086 /* If OP refers to value of function parameter, return the corresponding
1087 parameter. Also traverse chains of SSA register assignments. If non-NULL,
1088 the size of the memory load (or the SSA_NAME of the PARM_DECL) will be
1089 stored to *SIZE_P in that case too. */
1092 unmodified_parm (gimple
*stmt
, tree op
, HOST_WIDE_INT
*size_p
)
1094 tree res
= unmodified_parm_1 (stmt
, op
, size_p
);
1098 if (TREE_CODE (op
) == SSA_NAME
1099 && !SSA_NAME_IS_DEFAULT_DEF (op
)
1100 && gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1101 return unmodified_parm (SSA_NAME_DEF_STMT (op
),
1102 gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op
)),
1107 /* If OP refers to a value of a function parameter or value loaded from an
1108 aggregate passed to a parameter (either by value or reference), return TRUE
1109 and store the number of the parameter to *INDEX_P, the access size into
1110 *SIZE_P, and information whether and how it has been loaded from an
1111 aggregate into *AGGPOS. INFO describes the function parameters, STMT is the
1112 statement in which OP is used or loaded. */
1115 unmodified_parm_or_parm_agg_item (struct ipa_func_body_info
*fbi
,
1116 gimple
*stmt
, tree op
, int *index_p
,
1117 HOST_WIDE_INT
*size_p
,
1118 struct agg_position_info
*aggpos
)
1120 tree res
= unmodified_parm_1 (stmt
, op
, size_p
);
1122 gcc_checking_assert (aggpos
);
1125 *index_p
= ipa_get_param_decl_index (fbi
->info
, res
);
1128 aggpos
->agg_contents
= false;
1129 aggpos
->by_ref
= false;
1133 if (TREE_CODE (op
) == SSA_NAME
)
1135 if (SSA_NAME_IS_DEFAULT_DEF (op
)
1136 || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1138 stmt
= SSA_NAME_DEF_STMT (op
);
1139 op
= gimple_assign_rhs1 (stmt
);
1140 if (!REFERENCE_CLASS_P (op
))
1141 return unmodified_parm_or_parm_agg_item (fbi
, stmt
, op
, index_p
, size_p
,
1145 aggpos
->agg_contents
= true;
1146 return ipa_load_from_parm_agg (fbi
, fbi
->info
->descriptors
,
1147 stmt
, op
, index_p
, &aggpos
->offset
,
1148 size_p
, &aggpos
->by_ref
);
1151 /* See if statement might disappear after inlining.
1152 0 - means not eliminated
1153 1 - half of statements goes away
1154 2 - for sure it is eliminated.
1155 We are not terribly sophisticated, basically looking for simple abstraction
1156 penalty wrappers. */
1159 eliminated_by_inlining_prob (gimple
*stmt
)
1161 enum gimple_code code
= gimple_code (stmt
);
1162 enum tree_code rhs_code
;
1172 if (gimple_num_ops (stmt
) != 2)
1175 rhs_code
= gimple_assign_rhs_code (stmt
);
1177 /* Casts of parameters, loads from parameters passed by reference
1178 and stores to return value or parameters are often free after
1179 inlining dua to SRA and further combining.
1180 Assume that half of statements goes away. */
1181 if (CONVERT_EXPR_CODE_P (rhs_code
)
1182 || rhs_code
== VIEW_CONVERT_EXPR
1183 || rhs_code
== ADDR_EXPR
1184 || gimple_assign_rhs_class (stmt
) == GIMPLE_SINGLE_RHS
)
1186 tree rhs
= gimple_assign_rhs1 (stmt
);
1187 tree lhs
= gimple_assign_lhs (stmt
);
1188 tree inner_rhs
= get_base_address (rhs
);
1189 tree inner_lhs
= get_base_address (lhs
);
1190 bool rhs_free
= false;
1191 bool lhs_free
= false;
1198 /* Reads of parameter are expected to be free. */
1199 if (unmodified_parm (stmt
, inner_rhs
, NULL
))
1201 /* Match expressions of form &this->field. Those will most likely
1202 combine with something upstream after inlining. */
1203 else if (TREE_CODE (inner_rhs
) == ADDR_EXPR
)
1205 tree op
= get_base_address (TREE_OPERAND (inner_rhs
, 0));
1206 if (TREE_CODE (op
) == PARM_DECL
)
1208 else if (TREE_CODE (op
) == MEM_REF
1209 && unmodified_parm (stmt
, TREE_OPERAND (op
, 0), NULL
))
1213 /* When parameter is not SSA register because its address is taken
1214 and it is just copied into one, the statement will be completely
1215 free after inlining (we will copy propagate backward). */
1216 if (rhs_free
&& is_gimple_reg (lhs
))
1219 /* Reads of parameters passed by reference
1220 expected to be free (i.e. optimized out after inlining). */
1221 if (TREE_CODE (inner_rhs
) == MEM_REF
1222 && unmodified_parm (stmt
, TREE_OPERAND (inner_rhs
, 0), NULL
))
1225 /* Copying parameter passed by reference into gimple register is
1226 probably also going to copy propagate, but we can't be quite
1228 if (rhs_free
&& is_gimple_reg (lhs
))
1231 /* Writes to parameters, parameters passed by value and return value
1232 (either dirrectly or passed via invisible reference) are free.
1234 TODO: We ought to handle testcase like
1235 struct a {int a,b;};
1237 retrurnsturct (void)
1243 This translate into:
1258 For that we either need to copy ipa-split logic detecting writes
1260 if (TREE_CODE (inner_lhs
) == PARM_DECL
1261 || TREE_CODE (inner_lhs
) == RESULT_DECL
1262 || (TREE_CODE (inner_lhs
) == MEM_REF
1263 && (unmodified_parm (stmt
, TREE_OPERAND (inner_lhs
, 0), NULL
)
1264 || (TREE_CODE (TREE_OPERAND (inner_lhs
, 0)) == SSA_NAME
1265 && SSA_NAME_VAR (TREE_OPERAND (inner_lhs
, 0))
1266 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1268 0))) == RESULT_DECL
))))
1271 && (is_gimple_reg (rhs
) || is_gimple_min_invariant (rhs
)))
1273 if (lhs_free
&& rhs_free
)
1283 /* If BB ends by a conditional we can turn into predicates, attach corresponding
1284 predicates to the CFG edges. */
1287 set_cond_stmt_execution_predicate (struct ipa_func_body_info
*fbi
,
1288 struct inline_summary
*summary
,
1295 struct agg_position_info aggpos
;
1296 enum tree_code code
, inverted_code
;
1302 last
= last_stmt (bb
);
1303 if (!last
|| gimple_code (last
) != GIMPLE_COND
)
1305 if (!is_gimple_ip_invariant (gimple_cond_rhs (last
)))
1307 op
= gimple_cond_lhs (last
);
1308 /* TODO: handle conditionals like
1311 if (unmodified_parm_or_parm_agg_item (fbi
, last
, op
, &index
, &size
, &aggpos
))
1313 code
= gimple_cond_code (last
);
1314 inverted_code
= invert_tree_comparison (code
, HONOR_NANS (op
));
1316 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1318 enum tree_code this_code
= (e
->flags
& EDGE_TRUE_VALUE
1319 ? code
: inverted_code
);
1320 /* invert_tree_comparison will return ERROR_MARK on FP
1321 comparsions that are not EQ/NE instead of returning proper
1322 unordered one. Be sure it is not confused with NON_CONSTANT. */
1323 if (this_code
!= ERROR_MARK
)
1326 = add_condition (summary
, index
, size
, &aggpos
, this_code
,
1327 unshare_expr_without_location
1328 (gimple_cond_rhs (last
)));
1329 e
->aux
= edge_predicate_pool
.allocate ();
1330 *(predicate
*) e
->aux
= p
;
1335 if (TREE_CODE (op
) != SSA_NAME
)
1338 if (builtin_constant_p (op))
1342 Here we can predicate nonconstant_code. We can't
1343 really handle constant_code since we have no predicate
1344 for this and also the constant code is not known to be
1345 optimized away when inliner doen't see operand is constant.
1346 Other optimizers might think otherwise. */
1347 if (gimple_cond_code (last
) != NE_EXPR
1348 || !integer_zerop (gimple_cond_rhs (last
)))
1350 set_stmt
= SSA_NAME_DEF_STMT (op
);
1351 if (!gimple_call_builtin_p (set_stmt
, BUILT_IN_CONSTANT_P
)
1352 || gimple_call_num_args (set_stmt
) != 1)
1354 op2
= gimple_call_arg (set_stmt
, 0);
1355 if (!unmodified_parm_or_parm_agg_item (fbi
, set_stmt
, op2
, &index
, &size
,
1358 FOR_EACH_EDGE (e
, ei
, bb
->succs
) if (e
->flags
& EDGE_FALSE_VALUE
)
1360 predicate p
= add_condition (summary
, index
, size
, &aggpos
,
1361 predicate::is_not_constant
, NULL_TREE
);
1362 e
->aux
= edge_predicate_pool
.allocate ();
1363 *(predicate
*) e
->aux
= p
;
1368 /* If BB ends by a switch we can turn into predicates, attach corresponding
1369 predicates to the CFG edges. */
1372 set_switch_stmt_execution_predicate (struct ipa_func_body_info
*fbi
,
1373 struct inline_summary
*summary
,
1380 struct agg_position_info aggpos
;
1386 lastg
= last_stmt (bb
);
1387 if (!lastg
|| gimple_code (lastg
) != GIMPLE_SWITCH
)
1389 gswitch
*last
= as_a
<gswitch
*> (lastg
);
1390 op
= gimple_switch_index (last
);
1391 if (!unmodified_parm_or_parm_agg_item (fbi
, last
, op
, &index
, &size
, &aggpos
))
1394 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1396 e
->aux
= edge_predicate_pool
.allocate ();
1397 *(predicate
*) e
->aux
= false;
1399 n
= gimple_switch_num_labels (last
);
1400 for (case_idx
= 0; case_idx
< n
; ++case_idx
)
1402 tree cl
= gimple_switch_label (last
, case_idx
);
1406 e
= find_edge (bb
, label_to_block (CASE_LABEL (cl
)));
1407 min
= CASE_LOW (cl
);
1408 max
= CASE_HIGH (cl
);
1410 /* For default we might want to construct predicate that none
1411 of cases is met, but it is bit hard to do not having negations
1412 of conditionals handy. */
1416 p
= add_condition (summary
, index
, size
, &aggpos
, EQ_EXPR
,
1417 unshare_expr_without_location (min
));
1421 p1
= add_condition (summary
, index
, size
, &aggpos
, GE_EXPR
,
1422 unshare_expr_without_location (min
));
1423 p2
= add_condition (summary
, index
, size
, &aggpos
, LE_EXPR
,
1424 unshare_expr_without_location (max
));
1427 *(struct predicate
*) e
->aux
1428 = p
.or_with (summary
->conds
, *(struct predicate
*) e
->aux
);
1433 /* For each BB in NODE attach to its AUX pointer predicate under
1434 which it is executable. */
1437 compute_bb_predicates (struct ipa_func_body_info
*fbi
,
1438 struct cgraph_node
*node
,
1439 struct inline_summary
*summary
)
1441 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
1445 FOR_EACH_BB_FN (bb
, my_function
)
1447 set_cond_stmt_execution_predicate (fbi
, summary
, bb
);
1448 set_switch_stmt_execution_predicate (fbi
, summary
, bb
);
1451 /* Entry block is always executable. */
1452 ENTRY_BLOCK_PTR_FOR_FN (my_function
)->aux
1453 = edge_predicate_pool
.allocate ();
1454 *(predicate
*) ENTRY_BLOCK_PTR_FOR_FN (my_function
)->aux
= true;
1456 /* A simple dataflow propagation of predicates forward in the CFG.
1457 TODO: work in reverse postorder. */
1461 FOR_EACH_BB_FN (bb
, my_function
)
1463 predicate p
= false;
1466 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1470 predicate this_bb_predicate
1471 = *(predicate
*) e
->src
->aux
;
1473 this_bb_predicate
&= (*(struct predicate
*) e
->aux
);
1474 p
= p
.or_with (summary
->conds
, this_bb_predicate
);
1480 gcc_checking_assert (!bb
->aux
);
1486 bb
->aux
= edge_predicate_pool
.allocate ();
1487 *((predicate
*) bb
->aux
) = p
;
1489 else if (p
!= *(predicate
*) bb
->aux
)
1491 /* This OR operation is needed to ensure monotonous data flow
1492 in the case we hit the limit on number of clauses and the
1493 and/or operations above give approximate answers. */
1494 p
= p
.or_with (summary
->conds
, *(predicate
*)bb
->aux
);
1495 if (p
!= *(predicate
*) bb
->aux
)
1498 *((predicate
*) bb
->aux
) = p
;
1507 /* We keep info about constantness of SSA names. */
1509 typedef predicate predicate_t
;
1510 /* Return predicate specifying when the STMT might have result that is not
1511 a compile time constant. */
1514 will_be_nonconstant_expr_predicate (struct ipa_node_params
*info
,
1515 struct inline_summary
*summary
,
1517 vec
<predicate_t
> nonconstant_names
)
1523 while (UNARY_CLASS_P (expr
))
1524 expr
= TREE_OPERAND (expr
, 0);
1526 parm
= unmodified_parm (NULL
, expr
, &size
);
1527 if (parm
&& (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
1528 return add_condition (summary
, index
, size
, NULL
, predicate::changed
,
1530 if (is_gimple_min_invariant (expr
))
1532 if (TREE_CODE (expr
) == SSA_NAME
)
1533 return nonconstant_names
[SSA_NAME_VERSION (expr
)];
1534 if (BINARY_CLASS_P (expr
) || COMPARISON_CLASS_P (expr
))
1536 predicate p1
= will_be_nonconstant_expr_predicate
1537 (info
, summary
, TREE_OPERAND (expr
, 0),
1543 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1544 TREE_OPERAND (expr
, 1),
1546 return p1
.or_with (summary
->conds
, p2
);
1548 else if (TREE_CODE (expr
) == COND_EXPR
)
1550 predicate p1
= will_be_nonconstant_expr_predicate
1551 (info
, summary
, TREE_OPERAND (expr
, 0),
1557 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1558 TREE_OPERAND (expr
, 1),
1562 p1
= p1
.or_with (summary
->conds
, p2
);
1563 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1564 TREE_OPERAND (expr
, 2),
1566 return p2
.or_with (summary
->conds
, p1
);
1577 /* Return predicate specifying when the STMT might have result that is not
1578 a compile time constant. */
1581 will_be_nonconstant_predicate (struct ipa_func_body_info
*fbi
,
1582 struct inline_summary
*summary
,
1584 vec
<predicate_t
> nonconstant_names
)
1589 predicate op_non_const
;
1593 struct agg_position_info aggpos
;
1595 /* What statments might be optimized away
1596 when their arguments are constant. */
1597 if (gimple_code (stmt
) != GIMPLE_ASSIGN
1598 && gimple_code (stmt
) != GIMPLE_COND
1599 && gimple_code (stmt
) != GIMPLE_SWITCH
1600 && (gimple_code (stmt
) != GIMPLE_CALL
1601 || !(gimple_call_flags (stmt
) & ECF_CONST
)))
1604 /* Stores will stay anyway. */
1605 if (gimple_store_p (stmt
))
1608 is_load
= gimple_assign_load_p (stmt
);
1610 /* Loads can be optimized when the value is known. */
1614 gcc_assert (gimple_assign_single_p (stmt
));
1615 op
= gimple_assign_rhs1 (stmt
);
1616 if (!unmodified_parm_or_parm_agg_item (fbi
, stmt
, op
, &base_index
, &size
,
1623 /* See if we understand all operands before we start
1624 adding conditionals. */
1625 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1627 tree parm
= unmodified_parm (stmt
, use
, NULL
);
1628 /* For arguments we can build a condition. */
1629 if (parm
&& ipa_get_param_decl_index (fbi
->info
, parm
) >= 0)
1631 if (TREE_CODE (use
) != SSA_NAME
)
1633 /* If we know when operand is constant,
1634 we still can say something useful. */
1635 if (nonconstant_names
[SSA_NAME_VERSION (use
)] != true)
1642 add_condition (summary
, base_index
, size
, &aggpos
, predicate::changed
,
1645 op_non_const
= false;
1646 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1649 tree parm
= unmodified_parm (stmt
, use
, &size
);
1652 if (parm
&& (index
= ipa_get_param_decl_index (fbi
->info
, parm
)) >= 0)
1654 if (index
!= base_index
)
1655 p
= add_condition (summary
, index
, size
, NULL
, predicate::changed
,
1661 p
= nonconstant_names
[SSA_NAME_VERSION (use
)];
1662 op_non_const
= p
.or_with (summary
->conds
, op_non_const
);
1664 if ((gimple_code (stmt
) == GIMPLE_ASSIGN
|| gimple_code (stmt
) == GIMPLE_CALL
)
1665 && gimple_op (stmt
, 0)
1666 && TREE_CODE (gimple_op (stmt
, 0)) == SSA_NAME
)
1667 nonconstant_names
[SSA_NAME_VERSION (gimple_op (stmt
, 0))]
1669 return op_non_const
;
1672 struct record_modified_bb_info
1678 /* Value is initialized in INIT_BB and used in USE_BB. We want to copute
1679 probability how often it changes between USE_BB.
1680 INIT_BB->frequency/USE_BB->frequency is an estimate, but if INIT_BB
1681 is in different loop nest, we can do better.
1682 This is all just estimate. In theory we look for minimal cut separating
1683 INIT_BB and USE_BB, but we only want to anticipate loop invariant motion
1687 get_minimal_bb (basic_block init_bb
, basic_block use_bb
)
1689 struct loop
*l
= find_common_loop (init_bb
->loop_father
, use_bb
->loop_father
);
1690 if (l
&& l
->header
->frequency
< init_bb
->frequency
)
1695 /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
1696 set except for info->stmt. */
1699 record_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef
, void *data
)
1701 struct record_modified_bb_info
*info
=
1702 (struct record_modified_bb_info
*) data
;
1703 if (SSA_NAME_DEF_STMT (vdef
) == info
->stmt
)
1705 bitmap_set_bit (info
->bb_set
,
1706 SSA_NAME_IS_DEFAULT_DEF (vdef
)
1707 ? ENTRY_BLOCK_PTR_FOR_FN (cfun
)->index
1709 (gimple_bb (SSA_NAME_DEF_STMT (vdef
)),
1710 gimple_bb (info
->stmt
))->index
);
1714 /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
1715 will change since last invocation of STMT.
1717 Value 0 is reserved for compile time invariants.
1718 For common parameters it is REG_BR_PROB_BASE. For loop invariants it
1719 ought to be REG_BR_PROB_BASE / estimated_iters. */
1722 param_change_prob (gimple
*stmt
, int i
)
1724 tree op
= gimple_call_arg (stmt
, i
);
1725 basic_block bb
= gimple_bb (stmt
);
1727 if (TREE_CODE (op
) == WITH_SIZE_EXPR
)
1728 op
= TREE_OPERAND (op
, 0);
1730 tree base
= get_base_address (op
);
1732 /* Global invariants never change. */
1733 if (is_gimple_min_invariant (base
))
1736 /* We would have to do non-trivial analysis to really work out what
1737 is the probability of value to change (i.e. when init statement
1738 is in a sibling loop of the call).
1740 We do an conservative estimate: when call is executed N times more often
1741 than the statement defining value, we take the frequency 1/N. */
1742 if (TREE_CODE (base
) == SSA_NAME
)
1747 return REG_BR_PROB_BASE
;
1749 if (SSA_NAME_IS_DEFAULT_DEF (base
))
1750 init_freq
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->frequency
;
1752 init_freq
= get_minimal_bb
1753 (gimple_bb (SSA_NAME_DEF_STMT (base
)),
1754 gimple_bb (stmt
))->frequency
;
1758 if (init_freq
< bb
->frequency
)
1759 return MAX (GCOV_COMPUTE_SCALE (init_freq
, bb
->frequency
), 1);
1761 return REG_BR_PROB_BASE
;
1767 struct record_modified_bb_info info
;
1770 tree init
= ctor_for_folding (base
);
1772 if (init
!= error_mark_node
)
1775 return REG_BR_PROB_BASE
;
1776 ao_ref_init (&refd
, op
);
1778 info
.bb_set
= BITMAP_ALLOC (NULL
);
1779 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), record_modified
, &info
,
1781 if (bitmap_bit_p (info
.bb_set
, bb
->index
))
1783 BITMAP_FREE (info
.bb_set
);
1784 return REG_BR_PROB_BASE
;
1787 /* Assume that every memory is initialized at entry.
1788 TODO: Can we easilly determine if value is always defined
1789 and thus we may skip entry block? */
1790 if (ENTRY_BLOCK_PTR_FOR_FN (cfun
)->frequency
)
1791 max
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->frequency
;
1795 EXECUTE_IF_SET_IN_BITMAP (info
.bb_set
, 0, index
, bi
)
1796 max
= MIN (max
, BASIC_BLOCK_FOR_FN (cfun
, index
)->frequency
);
1798 BITMAP_FREE (info
.bb_set
);
1799 if (max
< bb
->frequency
)
1800 return MAX (GCOV_COMPUTE_SCALE (max
, bb
->frequency
), 1);
1802 return REG_BR_PROB_BASE
;
1806 /* Find whether a basic block BB is the final block of a (half) diamond CFG
1807 sub-graph and if the predicate the condition depends on is known. If so,
1808 return true and store the pointer the predicate in *P. */
1811 phi_result_unknown_predicate (struct ipa_node_params
*info
,
1812 inline_summary
*summary
, basic_block bb
,
1814 vec
<predicate_t
> nonconstant_names
)
1818 basic_block first_bb
= NULL
;
1821 if (single_pred_p (bb
))
1827 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1829 if (single_succ_p (e
->src
))
1831 if (!single_pred_p (e
->src
))
1834 first_bb
= single_pred (e
->src
);
1835 else if (single_pred (e
->src
) != first_bb
)
1842 else if (e
->src
!= first_bb
)
1850 stmt
= last_stmt (first_bb
);
1852 || gimple_code (stmt
) != GIMPLE_COND
1853 || !is_gimple_ip_invariant (gimple_cond_rhs (stmt
)))
1856 *p
= will_be_nonconstant_expr_predicate (info
, summary
,
1857 gimple_cond_lhs (stmt
),
1865 /* Given a PHI statement in a function described by inline properties SUMMARY
1866 and *P being the predicate describing whether the selected PHI argument is
1867 known, store a predicate for the result of the PHI statement into
1868 NONCONSTANT_NAMES, if possible. */
1871 predicate_for_phi_result (struct inline_summary
*summary
, gphi
*phi
,
1873 vec
<predicate_t
> nonconstant_names
)
1877 for (i
= 0; i
< gimple_phi_num_args (phi
); i
++)
1879 tree arg
= gimple_phi_arg (phi
, i
)->def
;
1880 if (!is_gimple_min_invariant (arg
))
1882 gcc_assert (TREE_CODE (arg
) == SSA_NAME
);
1883 *p
= p
->or_with (summary
->conds
,
1884 nonconstant_names
[SSA_NAME_VERSION (arg
)]);
1890 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1892 fprintf (dump_file
, "\t\tphi predicate: ");
1893 p
->dump (dump_file
, summary
->conds
);
1895 nonconstant_names
[SSA_NAME_VERSION (gimple_phi_result (phi
))] = *p
;
1898 /* Return predicate specifying when array index in access OP becomes non-constant. */
1901 array_index_predicate (inline_summary
*info
,
1902 vec
< predicate_t
> nonconstant_names
, tree op
)
1904 predicate p
= false;
1905 while (handled_component_p (op
))
1907 if (TREE_CODE (op
) == ARRAY_REF
|| TREE_CODE (op
) == ARRAY_RANGE_REF
)
1909 if (TREE_CODE (TREE_OPERAND (op
, 1)) == SSA_NAME
)
1910 p
= p
.or_with (info
->conds
,
1911 nonconstant_names
[SSA_NAME_VERSION
1912 (TREE_OPERAND (op
, 1))]);
1914 op
= TREE_OPERAND (op
, 0);
1919 /* For a typical usage of __builtin_expect (a<b, 1), we
1920 may introduce an extra relation stmt:
1921 With the builtin, we have
1924 t3 = __builtin_expect (t2, 1);
1927 Without the builtin, we have
1930 This affects the size/time estimation and may have
1931 an impact on the earlier inlining.
1932 Here find this pattern and fix it up later. */
1935 find_foldable_builtin_expect (basic_block bb
)
1937 gimple_stmt_iterator bsi
;
1939 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
1941 gimple
*stmt
= gsi_stmt (bsi
);
1942 if (gimple_call_builtin_p (stmt
, BUILT_IN_EXPECT
)
1943 || gimple_call_internal_p (stmt
, IFN_BUILTIN_EXPECT
))
1945 tree var
= gimple_call_lhs (stmt
);
1946 tree arg
= gimple_call_arg (stmt
, 0);
1947 use_operand_p use_p
;
1954 gcc_assert (TREE_CODE (var
) == SSA_NAME
);
1956 while (TREE_CODE (arg
) == SSA_NAME
)
1958 gimple
*stmt_tmp
= SSA_NAME_DEF_STMT (arg
);
1959 if (!is_gimple_assign (stmt_tmp
))
1961 switch (gimple_assign_rhs_code (stmt_tmp
))
1980 arg
= gimple_assign_rhs1 (stmt_tmp
);
1983 if (match
&& single_imm_use (var
, &use_p
, &use_stmt
)
1984 && gimple_code (use_stmt
) == GIMPLE_COND
)
1991 /* Return true when the basic blocks contains only clobbers followed by RESX.
1992 Such BBs are kept around to make removal of dead stores possible with
1993 presence of EH and will be optimized out by optimize_clobbers later in the
1996 NEED_EH is used to recurse in case the clobber has non-EH predecestors
1997 that can be clobber only, too.. When it is false, the RESX is not necessary
1998 on the end of basic block. */
2001 clobber_only_eh_bb_p (basic_block bb
, bool need_eh
= true)
2003 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
2009 if (gsi_end_p (gsi
))
2011 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_RESX
)
2015 else if (!single_succ_p (bb
))
2018 for (; !gsi_end_p (gsi
); gsi_prev (&gsi
))
2020 gimple
*stmt
= gsi_stmt (gsi
);
2021 if (is_gimple_debug (stmt
))
2023 if (gimple_clobber_p (stmt
))
2025 if (gimple_code (stmt
) == GIMPLE_LABEL
)
2030 /* See if all predecestors are either throws or clobber only BBs. */
2031 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
2032 if (!(e
->flags
& EDGE_EH
)
2033 && !clobber_only_eh_bb_p (e
->src
, false))
2039 /* Return true if STMT compute a floating point expression that may be affected
2040 by -ffast-math and similar flags. */
2043 fp_expression_p (gimple
*stmt
)
2048 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_DEF
|SSA_OP_USE
)
2049 if (FLOAT_TYPE_P (TREE_TYPE (op
)))
2054 /* Compute function body size parameters for NODE.
2055 When EARLY is true, we compute only simple summaries without
2056 non-trivial predicates to drive the early inliner. */
2059 estimate_function_body_sizes (struct cgraph_node
*node
, bool early
)
2062 /* Estimate static overhead for function prologue/epilogue and alignment. */
2064 /* Benefits are scaled by probability of elimination that is in range
2067 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
2069 struct inline_summary
*info
= inline_summaries
->get (node
);
2070 predicate bb_predicate
;
2071 struct ipa_func_body_info fbi
;
2072 vec
<predicate_t
> nonconstant_names
= vNULL
;
2075 predicate array_index
= true;
2076 gimple
*fix_builtin_expect_stmt
;
2078 gcc_assert (my_function
&& my_function
->cfg
);
2079 gcc_assert (cfun
== my_function
);
2081 memset(&fbi
, 0, sizeof(fbi
));
2085 /* When optimizing and analyzing for IPA inliner, initialize loop optimizer
2086 so we can produce proper inline hints.
2088 When optimizing and analyzing for early inliner, initialize node params
2089 so we can produce correct BB predicates. */
2091 if (opt_for_fn (node
->decl
, optimize
))
2093 calculate_dominance_info (CDI_DOMINATORS
);
2095 loop_optimizer_init (LOOPS_NORMAL
| LOOPS_HAVE_RECORDED_EXITS
);
2098 ipa_check_create_node_params ();
2099 ipa_initialize_node_params (node
);
2102 if (ipa_node_params_sum
)
2105 fbi
.info
= IPA_NODE_REF (node
);
2106 fbi
.bb_infos
= vNULL
;
2107 fbi
.bb_infos
.safe_grow_cleared (last_basic_block_for_fn (cfun
));
2108 fbi
.param_count
= count_formal_params(node
->decl
);
2109 nonconstant_names
.safe_grow_cleared
2110 (SSANAMES (my_function
)->length ());
2115 fprintf (dump_file
, "\nAnalyzing function body size: %s\n",
2118 /* When we run into maximal number of entries, we assign everything to the
2119 constant truth case. Be sure to have it in list. */
2120 bb_predicate
= true;
2121 account_size_time (info
, 0, 0, &bb_predicate
, &bb_predicate
);
2123 bb_predicate
= predicate::not_inlined ();
2124 account_size_time (info
, 2 * INLINE_SIZE_SCALE
, 0, &bb_predicate
,
2128 compute_bb_predicates (&fbi
, node
, info
);
2129 order
= XNEWVEC (int, n_basic_blocks_for_fn (cfun
));
2130 nblocks
= pre_and_rev_post_order_compute (NULL
, order
, false);
2131 for (n
= 0; n
< nblocks
; n
++)
2133 bb
= BASIC_BLOCK_FOR_FN (cfun
, order
[n
]);
2134 freq
= compute_call_stmt_bb_frequency (node
->decl
, bb
);
2135 if (clobber_only_eh_bb_p (bb
))
2137 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2138 fprintf (dump_file
, "\n Ignoring BB %i;"
2139 " it will be optimized away by cleanup_clobbers\n",
2144 /* TODO: Obviously predicates can be propagated down across CFG. */
2148 bb_predicate
= *(predicate
*) bb
->aux
;
2150 bb_predicate
= false;
2153 bb_predicate
= true;
2155 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2157 fprintf (dump_file
, "\n BB %i predicate:", bb
->index
);
2158 bb_predicate
.dump (dump_file
, info
->conds
);
2161 if (fbi
.info
&& nonconstant_names
.exists ())
2163 predicate phi_predicate
;
2164 bool first_phi
= true;
2166 for (gphi_iterator bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
);
2170 && !phi_result_unknown_predicate (fbi
.info
, info
, bb
,
2175 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2177 fprintf (dump_file
, " ");
2178 print_gimple_stmt (dump_file
, gsi_stmt (bsi
), 0);
2180 predicate_for_phi_result (info
, bsi
.phi (), &phi_predicate
,
2185 fix_builtin_expect_stmt
= find_foldable_builtin_expect (bb
);
2187 for (gimple_stmt_iterator bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
);
2190 gimple
*stmt
= gsi_stmt (bsi
);
2191 int this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
2192 int this_time
= estimate_num_insns (stmt
, &eni_time_weights
);
2194 predicate will_be_nonconstant
;
2196 /* This relation stmt should be folded after we remove
2197 buildin_expect call. Adjust the cost here. */
2198 if (stmt
== fix_builtin_expect_stmt
)
2204 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2206 fprintf (dump_file
, " ");
2207 print_gimple_stmt (dump_file
, stmt
, 0);
2208 fprintf (dump_file
, "\t\tfreq:%3.2f size:%3i time:%3i\n",
2209 ((double) freq
) / CGRAPH_FREQ_BASE
, this_size
,
2213 if (gimple_assign_load_p (stmt
) && nonconstant_names
.exists ())
2215 predicate this_array_index
;
2217 array_index_predicate (info
, nonconstant_names
,
2218 gimple_assign_rhs1 (stmt
));
2219 if (this_array_index
!= false)
2220 array_index
&= this_array_index
;
2222 if (gimple_store_p (stmt
) && nonconstant_names
.exists ())
2224 predicate this_array_index
;
2226 array_index_predicate (info
, nonconstant_names
,
2227 gimple_get_lhs (stmt
));
2228 if (this_array_index
!= false)
2229 array_index
&= this_array_index
;
2233 if (is_gimple_call (stmt
)
2234 && !gimple_call_internal_p (stmt
))
2236 struct cgraph_edge
*edge
= node
->get_edge (stmt
);
2237 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2239 /* Special case: results of BUILT_IN_CONSTANT_P will be always
2240 resolved as constant. We however don't want to optimize
2241 out the cgraph edges. */
2242 if (nonconstant_names
.exists ()
2243 && gimple_call_builtin_p (stmt
, BUILT_IN_CONSTANT_P
)
2244 && gimple_call_lhs (stmt
)
2245 && TREE_CODE (gimple_call_lhs (stmt
)) == SSA_NAME
)
2247 predicate false_p
= false;
2248 nonconstant_names
[SSA_NAME_VERSION (gimple_call_lhs (stmt
))]
2251 if (ipa_node_params_sum
)
2253 int count
= gimple_call_num_args (stmt
);
2257 es
->param
.safe_grow_cleared (count
);
2258 for (i
= 0; i
< count
; i
++)
2260 int prob
= param_change_prob (stmt
, i
);
2261 gcc_assert (prob
>= 0 && prob
<= REG_BR_PROB_BASE
);
2262 es
->param
[i
].change_prob
= prob
;
2266 es
->call_stmt_size
= this_size
;
2267 es
->call_stmt_time
= this_time
;
2268 es
->loop_depth
= bb_loop_depth (bb
);
2269 edge_set_predicate (edge
, &bb_predicate
);
2272 /* TODO: When conditional jump or swithc is known to be constant, but
2273 we did not translate it into the predicates, we really can account
2274 just maximum of the possible paths. */
2277 = will_be_nonconstant_predicate (&fbi
, info
,
2278 stmt
, nonconstant_names
);
2280 will_be_nonconstant
= true;
2281 if (this_time
|| this_size
)
2285 prob
= eliminated_by_inlining_prob (stmt
);
2286 if (prob
== 1 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2288 "\t\t50%% will be eliminated by inlining\n");
2289 if (prob
== 2 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2290 fprintf (dump_file
, "\t\tWill be eliminated by inlining\n");
2292 struct predicate p
= bb_predicate
& will_be_nonconstant
;
2294 /* We can ignore statement when we proved it is never going
2295 to happen, but we can not do that for call statements
2296 because edges are accounted specially. */
2298 if (*(is_gimple_call (stmt
) ? &bb_predicate
: &p
) != false)
2304 /* We account everything but the calls. Calls have their own
2305 size/time info attached to cgraph edges. This is necessary
2306 in order to make the cost disappear after inlining. */
2307 if (!is_gimple_call (stmt
))
2311 predicate ip
= bb_predicate
& predicate::not_inlined ();
2312 account_size_time (info
, this_size
* prob
,
2313 (sreal
)(this_time
* prob
)
2314 / (CGRAPH_FREQ_BASE
* 2), &ip
,
2318 account_size_time (info
, this_size
* (2 - prob
),
2319 (sreal
)(this_time
* (2 - prob
))
2320 / (CGRAPH_FREQ_BASE
* 2),
2325 if (!info
->fp_expressions
&& fp_expression_p (stmt
))
2327 info
->fp_expressions
= true;
2329 fprintf (dump_file
, " fp_expression set\n");
2332 gcc_assert (time
>= 0);
2333 gcc_assert (size
>= 0);
2337 set_hint_predicate (&inline_summaries
->get (node
)->array_index
, array_index
);
2338 time
= time
/ CGRAPH_FREQ_BASE
;
2341 if (nonconstant_names
.exists () && !early
)
2344 predicate loop_iterations
= true;
2345 predicate loop_stride
= true;
2347 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2348 flow_loops_dump (dump_file
, NULL
, 0);
2350 FOR_EACH_LOOP (loop
, 0)
2355 struct tree_niter_desc niter_desc
;
2356 bb_predicate
= *(predicate
*) loop
->header
->aux
;
2358 exits
= get_loop_exit_edges (loop
);
2359 FOR_EACH_VEC_ELT (exits
, j
, ex
)
2360 if (number_of_iterations_exit (loop
, ex
, &niter_desc
, false)
2361 && !is_gimple_min_invariant (niter_desc
.niter
))
2363 predicate will_be_nonconstant
2364 = will_be_nonconstant_expr_predicate (fbi
.info
, info
,
2367 if (will_be_nonconstant
!= true)
2368 will_be_nonconstant
= bb_predicate
& will_be_nonconstant
;
2369 if (will_be_nonconstant
!= true
2370 && will_be_nonconstant
!= false)
2371 /* This is slightly inprecise. We may want to represent each
2372 loop with independent predicate. */
2373 loop_iterations
&= will_be_nonconstant
;
2378 /* To avoid quadratic behavior we analyze stride predicates only
2379 with respect to the containing loop. Thus we simply iterate
2380 over all defs in the outermost loop body. */
2381 for (loop
= loops_for_fn (cfun
)->tree_root
->inner
;
2382 loop
!= NULL
; loop
= loop
->next
)
2384 basic_block
*body
= get_loop_body (loop
);
2385 for (unsigned i
= 0; i
< loop
->num_nodes
; i
++)
2387 gimple_stmt_iterator gsi
;
2388 bb_predicate
= *(predicate
*) body
[i
]->aux
;
2389 for (gsi
= gsi_start_bb (body
[i
]); !gsi_end_p (gsi
);
2392 gimple
*stmt
= gsi_stmt (gsi
);
2394 if (!is_gimple_assign (stmt
))
2397 tree def
= gimple_assign_lhs (stmt
);
2398 if (TREE_CODE (def
) != SSA_NAME
)
2402 if (!simple_iv (loop_containing_stmt (stmt
),
2403 loop_containing_stmt (stmt
),
2405 || is_gimple_min_invariant (iv
.step
))
2408 predicate will_be_nonconstant
2409 = will_be_nonconstant_expr_predicate (fbi
.info
, info
,
2412 if (will_be_nonconstant
!= true)
2413 will_be_nonconstant
= bb_predicate
& will_be_nonconstant
;
2414 if (will_be_nonconstant
!= true
2415 && will_be_nonconstant
!= false)
2416 /* This is slightly inprecise. We may want to represent
2417 each loop with independent predicate. */
2418 loop_stride
= loop_stride
& will_be_nonconstant
;
2423 set_hint_predicate (&inline_summaries
->get (node
)->loop_iterations
,
2425 set_hint_predicate (&inline_summaries
->get (node
)->loop_stride
,
2429 FOR_ALL_BB_FN (bb
, my_function
)
2435 edge_predicate_pool
.remove ((predicate
*)bb
->aux
);
2437 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2440 edge_predicate_pool
.remove ((predicate
*) e
->aux
);
2444 inline_summaries
->get (node
)->self_time
= time
;
2445 inline_summaries
->get (node
)->self_size
= size
;
2446 nonconstant_names
.release ();
2447 ipa_release_body_info (&fbi
);
2448 if (opt_for_fn (node
->decl
, optimize
))
2451 loop_optimizer_finalize ();
2452 else if (!ipa_edge_args_sum
)
2453 ipa_free_all_node_params ();
2454 free_dominance_info (CDI_DOMINATORS
);
2458 fprintf (dump_file
, "\n");
2459 dump_inline_summary (dump_file
, node
);
2464 /* Compute parameters of functions used by inliner.
2465 EARLY is true when we compute parameters for the early inliner */
2468 compute_inline_parameters (struct cgraph_node
*node
, bool early
)
2470 HOST_WIDE_INT self_stack_size
;
2471 struct cgraph_edge
*e
;
2472 struct inline_summary
*info
;
2474 gcc_assert (!node
->global
.inlined_to
);
2476 inline_summary_alloc ();
2478 info
= inline_summaries
->get (node
);
2479 reset_inline_summary (node
, info
);
2481 /* Estimate the stack size for the function if we're optimizing. */
2482 self_stack_size
= optimize
&& !node
->thunk
.thunk_p
2483 ? estimated_stack_frame_size (node
) : 0;
2484 info
->estimated_self_stack_size
= self_stack_size
;
2485 info
->estimated_stack_size
= self_stack_size
;
2486 info
->stack_frame_offset
= 0;
2488 if (node
->thunk
.thunk_p
)
2490 struct inline_edge_summary
*es
= inline_edge_summary (node
->callees
);
2493 node
->local
.can_change_signature
= false;
2494 es
->call_stmt_size
= eni_size_weights
.call_cost
;
2495 es
->call_stmt_time
= eni_time_weights
.call_cost
;
2496 account_size_time (info
, INLINE_SIZE_SCALE
* 2,
2498 t
= predicate::not_inlined ();
2499 account_size_time (info
, 2 * INLINE_SIZE_SCALE
, 0, &t
, &t
);
2500 inline_update_overall_summary (node
);
2501 info
->self_size
= info
->size
;
2502 info
->self_time
= info
->time
;
2503 /* We can not inline instrumentation clones. */
2504 if (node
->thunk
.add_pointer_bounds_args
)
2506 info
->inlinable
= false;
2507 node
->callees
->inline_failed
= CIF_CHKP
;
2510 info
->inlinable
= true;
2514 /* Even is_gimple_min_invariant rely on current_function_decl. */
2515 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
2517 /* Can this function be inlined at all? */
2518 if (!opt_for_fn (node
->decl
, optimize
)
2519 && !lookup_attribute ("always_inline",
2520 DECL_ATTRIBUTES (node
->decl
)))
2521 info
->inlinable
= false;
2523 info
->inlinable
= tree_inlinable_function_p (node
->decl
);
2525 info
->contains_cilk_spawn
= fn_contains_cilk_spawn_p (cfun
);
2527 /* Type attributes can use parameter indices to describe them. */
2528 if (TYPE_ATTRIBUTES (TREE_TYPE (node
->decl
)))
2529 node
->local
.can_change_signature
= false;
2532 /* Otherwise, inlinable functions always can change signature. */
2533 if (info
->inlinable
)
2534 node
->local
.can_change_signature
= true;
2537 /* Functions calling builtin_apply can not change signature. */
2538 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2540 tree
cdecl = e
->callee
->decl
;
2541 if (DECL_BUILT_IN (cdecl)
2542 && DECL_BUILT_IN_CLASS (cdecl) == BUILT_IN_NORMAL
2543 && (DECL_FUNCTION_CODE (cdecl) == BUILT_IN_APPLY_ARGS
2544 || DECL_FUNCTION_CODE (cdecl) == BUILT_IN_VA_START
))
2547 node
->local
.can_change_signature
= !e
;
2550 /* Functions called by instrumentation thunk can't change signature
2551 because instrumentation thunk modification is not supported. */
2552 if (node
->local
.can_change_signature
)
2553 for (e
= node
->callers
; e
; e
= e
->next_caller
)
2554 if (e
->caller
->thunk
.thunk_p
2555 && e
->caller
->thunk
.add_pointer_bounds_args
)
2557 node
->local
.can_change_signature
= false;
2560 estimate_function_body_sizes (node
, early
);
2563 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2564 if (e
->callee
->comdat_local_p ())
2566 node
->calls_comdat_local
= (e
!= NULL
);
2568 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
2569 info
->time
= info
->self_time
;
2570 info
->size
= info
->self_size
;
2571 info
->stack_frame_offset
= 0;
2572 info
->estimated_stack_size
= info
->estimated_self_stack_size
;
2574 /* Code above should compute exactly the same result as
2575 inline_update_overall_summary but because computation happens in
2576 different order the roundoff errors result in slight changes. */
2577 inline_update_overall_summary (node
);
2578 gcc_assert (!(info
->time
- info
->self_time
).to_int ()
2579 && info
->size
== info
->self_size
);
2583 /* Compute parameters of functions used by inliner using
2584 current_function_decl. */
2587 compute_inline_parameters_for_current (void)
2589 compute_inline_parameters (cgraph_node::get (current_function_decl
), true);
2595 const pass_data pass_data_inline_parameters
=
2597 GIMPLE_PASS
, /* type */
2598 "inline_param", /* name */
2599 OPTGROUP_INLINE
, /* optinfo_flags */
2600 TV_INLINE_PARAMETERS
, /* tv_id */
2601 0, /* properties_required */
2602 0, /* properties_provided */
2603 0, /* properties_destroyed */
2604 0, /* todo_flags_start */
2605 0, /* todo_flags_finish */
2608 class pass_inline_parameters
: public gimple_opt_pass
2611 pass_inline_parameters (gcc::context
*ctxt
)
2612 : gimple_opt_pass (pass_data_inline_parameters
, ctxt
)
2615 /* opt_pass methods: */
2616 opt_pass
* clone () { return new pass_inline_parameters (m_ctxt
); }
2617 virtual unsigned int execute (function
*)
2619 return compute_inline_parameters_for_current ();
2622 }; // class pass_inline_parameters
2627 make_pass_inline_parameters (gcc::context
*ctxt
)
2629 return new pass_inline_parameters (ctxt
);
2633 /* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS,
2634 KNOWN_CONTEXTS and KNOWN_AGGS. */
2637 estimate_edge_devirt_benefit (struct cgraph_edge
*ie
,
2638 int *size
, int *time
,
2639 vec
<tree
> known_vals
,
2640 vec
<ipa_polymorphic_call_context
> known_contexts
,
2641 vec
<ipa_agg_jump_function_p
> known_aggs
)
2644 struct cgraph_node
*callee
;
2645 struct inline_summary
*isummary
;
2646 enum availability avail
;
2649 if (!known_vals
.exists () && !known_contexts
.exists ())
2651 if (!opt_for_fn (ie
->caller
->decl
, flag_indirect_inlining
))
2654 target
= ipa_get_indirect_edge_target (ie
, known_vals
, known_contexts
,
2655 known_aggs
, &speculative
);
2656 if (!target
|| speculative
)
2659 /* Account for difference in cost between indirect and direct calls. */
2660 *size
-= (eni_size_weights
.indirect_call_cost
- eni_size_weights
.call_cost
);
2661 *time
-= (eni_time_weights
.indirect_call_cost
- eni_time_weights
.call_cost
);
2662 gcc_checking_assert (*time
>= 0);
2663 gcc_checking_assert (*size
>= 0);
2665 callee
= cgraph_node::get (target
);
2666 if (!callee
|| !callee
->definition
)
2668 callee
= callee
->function_symbol (&avail
);
2669 if (avail
< AVAIL_AVAILABLE
)
2671 isummary
= inline_summaries
->get (callee
);
2672 return isummary
->inlinable
;
2675 /* Increase SIZE, MIN_SIZE (if non-NULL) and TIME for size and time needed to
2676 handle edge E with probability PROB.
2677 Set HINTS if edge may be devirtualized.
2678 KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS describe context of the call
2682 estimate_edge_size_and_time (struct cgraph_edge
*e
, int *size
, int *min_size
,
2685 vec
<tree
> known_vals
,
2686 vec
<ipa_polymorphic_call_context
> known_contexts
,
2687 vec
<ipa_agg_jump_function_p
> known_aggs
,
2688 inline_hints
*hints
)
2690 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2691 int call_size
= es
->call_stmt_size
;
2692 int call_time
= es
->call_stmt_time
;
2695 && estimate_edge_devirt_benefit (e
, &call_size
, &call_time
,
2696 known_vals
, known_contexts
, known_aggs
)
2697 && hints
&& e
->maybe_hot_p ())
2698 *hints
|= INLINE_HINT_indirect_call
;
2699 cur_size
= call_size
* INLINE_SIZE_SCALE
;
2702 *min_size
+= cur_size
;
2703 if (prob
== REG_BR_PROB_BASE
)
2704 *time
+= ((sreal
)(call_time
* e
->frequency
)) / CGRAPH_FREQ_BASE
;
2706 *time
+= ((sreal
)call_time
) * (prob
* e
->frequency
)
2707 / (CGRAPH_FREQ_BASE
* REG_BR_PROB_BASE
);
2712 /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
2713 calls in NODE. POSSIBLE_TRUTHS, KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS
2714 describe context of the call site. */
2717 estimate_calls_size_and_time (struct cgraph_node
*node
, int *size
,
2718 int *min_size
, sreal
*time
,
2719 inline_hints
*hints
,
2720 clause_t possible_truths
,
2721 vec
<tree
> known_vals
,
2722 vec
<ipa_polymorphic_call_context
> known_contexts
,
2723 vec
<ipa_agg_jump_function_p
> known_aggs
)
2725 struct cgraph_edge
*e
;
2726 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2728 if (inline_edge_summary_vec
.length () <= (unsigned) e
->uid
)
2731 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2733 /* Do not care about zero sized builtins. */
2734 if (e
->inline_failed
&& !es
->call_stmt_size
)
2736 gcc_checking_assert (!es
->call_stmt_time
);
2740 || es
->predicate
->evaluate (possible_truths
))
2742 if (e
->inline_failed
)
2744 /* Predicates of calls shall not use NOT_CHANGED codes,
2745 sowe do not need to compute probabilities. */
2746 estimate_edge_size_and_time (e
, size
,
2747 es
->predicate
? NULL
: min_size
,
2748 time
, REG_BR_PROB_BASE
,
2749 known_vals
, known_contexts
,
2753 estimate_calls_size_and_time (e
->callee
, size
, min_size
, time
,
2756 known_vals
, known_contexts
,
2760 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2762 if (inline_edge_summary_vec
.length () <= (unsigned) e
->uid
)
2765 struct inline_edge_summary
*es
= inline_edge_summary (e
);
2767 || es
->predicate
->evaluate (possible_truths
))
2768 estimate_edge_size_and_time (e
, size
,
2769 es
->predicate
? NULL
: min_size
,
2770 time
, REG_BR_PROB_BASE
,
2771 known_vals
, known_contexts
, known_aggs
,
2777 /* Estimate size and time needed to execute NODE assuming
2778 POSSIBLE_TRUTHS clause, and KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS
2779 information about NODE's arguments. If non-NULL use also probability
2780 information present in INLINE_PARAM_SUMMARY vector.
2781 Additionally detemine hints determined by the context. Finally compute
2782 minimal size needed for the call that is independent on the call context and
2783 can be used for fast estimates. Return the values in RET_SIZE,
2784 RET_MIN_SIZE, RET_TIME and RET_HINTS. */
2787 estimate_node_size_and_time (struct cgraph_node
*node
,
2788 clause_t possible_truths
,
2789 clause_t nonspec_possible_truths
,
2790 vec
<tree
> known_vals
,
2791 vec
<ipa_polymorphic_call_context
> known_contexts
,
2792 vec
<ipa_agg_jump_function_p
> known_aggs
,
2793 int *ret_size
, int *ret_min_size
,
2795 sreal
*ret_nonspecialized_time
,
2796 inline_hints
*ret_hints
,
2797 vec
<inline_param_summary
>
2798 inline_param_summary
)
2800 struct inline_summary
*info
= inline_summaries
->get (node
);
2805 inline_hints hints
= 0;
2808 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2811 fprintf (dump_file
, " Estimating body: %s/%i\n"
2812 " Known to be false: ", node
->name (),
2815 for (i
= predicate::not_inlined_condition
;
2816 i
< (predicate::first_dynamic_condition
2817 + (int) vec_safe_length (info
->conds
)); i
++)
2818 if (!(possible_truths
& (1 << i
)))
2821 fprintf (dump_file
, ", ");
2823 dump_condition (dump_file
, info
->conds
, i
);
2827 estimate_calls_size_and_time (node
, &size
, &min_size
, &time
, &hints
, possible_truths
,
2828 known_vals
, known_contexts
, known_aggs
);
2829 sreal nonspecialized_time
= time
;
2831 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
2833 bool nonconst
= e
->nonconst_predicate
.evaluate (possible_truths
);
2834 bool exec
= e
->exec_predicate
.evaluate (nonspec_possible_truths
);
2835 gcc_assert (!nonconst
|| exec
);
2838 gcc_checking_assert (e
->time
>= 0);
2839 gcc_checking_assert (time
>= 0);
2841 /* We compute specialized size only because size of nonspecialized
2842 copy is context independent.
2844 The difference between nonspecialized execution and specialized is
2845 that nonspecialized is not going to have optimized out computations
2846 known to be constant in a specialized setting. */
2849 nonspecialized_time
+= e
->time
;
2852 else if (!inline_param_summary
.exists ())
2859 int prob
= e
->nonconst_predicate
.probability
2860 (info
->conds
, possible_truths
,
2861 inline_param_summary
);
2862 gcc_checking_assert (prob
>= 0);
2863 gcc_checking_assert (prob
<= REG_BR_PROB_BASE
);
2864 time
+= e
->time
* prob
/ REG_BR_PROB_BASE
;
2866 gcc_checking_assert (time
>= 0);
2869 gcc_checking_assert ((*info
->entry
)[0].exec_predicate
== true);
2870 gcc_checking_assert ((*info
->entry
)[0].nonconst_predicate
== true);
2871 min_size
= (*info
->entry
)[0].size
;
2872 gcc_checking_assert (size
>= 0);
2873 gcc_checking_assert (time
>= 0);
2874 /* nonspecialized_time should be always bigger than specialized time.
2875 Roundoff issues however may get into the way. */
2876 gcc_checking_assert ((nonspecialized_time
- time
) >= -1);
2878 /* Roundoff issues may make specialized time bigger than nonspecialized
2879 time. We do not really want that to happen because some heurstics
2880 may get confused by seeing negative speedups. */
2881 if (time
> nonspecialized_time
)
2882 time
= nonspecialized_time
;
2884 if (info
->loop_iterations
2885 && !info
->loop_iterations
->evaluate (possible_truths
))
2886 hints
|= INLINE_HINT_loop_iterations
;
2887 if (info
->loop_stride
2888 && !info
->loop_stride
->evaluate (possible_truths
))
2889 hints
|= INLINE_HINT_loop_stride
;
2890 if (info
->array_index
2891 && !info
->array_index
->evaluate (possible_truths
))
2892 hints
|= INLINE_HINT_array_index
;
2894 hints
|= INLINE_HINT_in_scc
;
2895 if (DECL_DECLARED_INLINE_P (node
->decl
))
2896 hints
|= INLINE_HINT_declared_inline
;
2898 size
= RDIV (size
, INLINE_SIZE_SCALE
);
2899 min_size
= RDIV (min_size
, INLINE_SIZE_SCALE
);
2901 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2902 fprintf (dump_file
, "\n size:%i time:%f nonspec time:%f\n", (int) size
,
2903 time
.to_double (), nonspecialized_time
.to_double ());
2906 if (ret_nonspecialized_time
)
2907 *ret_nonspecialized_time
= nonspecialized_time
;
2911 *ret_min_size
= min_size
;
2918 /* Estimate size and time needed to execute callee of EDGE assuming that
2919 parameters known to be constant at caller of EDGE are propagated.
2920 KNOWN_VALS and KNOWN_CONTEXTS are vectors of assumed known constant values
2921 and types for parameters. */
2924 estimate_ipcp_clone_size_and_time (struct cgraph_node
*node
,
2925 vec
<tree
> known_vals
,
2926 vec
<ipa_polymorphic_call_context
>
2928 vec
<ipa_agg_jump_function_p
> known_aggs
,
2929 int *ret_size
, sreal
*ret_time
,
2930 sreal
*ret_nonspec_time
,
2931 inline_hints
*hints
)
2933 clause_t clause
, nonspec_clause
;
2935 evaluate_conditions_for_known_args (node
, false, known_vals
, known_aggs
,
2936 &clause
, &nonspec_clause
);
2937 estimate_node_size_and_time (node
, clause
, nonspec_clause
,
2938 known_vals
, known_contexts
,
2939 known_aggs
, ret_size
, NULL
, ret_time
,
2940 ret_nonspec_time
, hints
, vNULL
);
2944 /* Update summary information of inline clones after inlining.
2945 Compute peak stack usage. */
2948 inline_update_callee_summaries (struct cgraph_node
*node
, int depth
)
2950 struct cgraph_edge
*e
;
2951 struct inline_summary
*callee_info
= inline_summaries
->get (node
);
2952 struct inline_summary
*caller_info
= inline_summaries
->get (node
->callers
->caller
);
2955 callee_info
->stack_frame_offset
2956 = caller_info
->stack_frame_offset
2957 + caller_info
->estimated_self_stack_size
;
2958 peak
= callee_info
->stack_frame_offset
2959 + callee_info
->estimated_self_stack_size
;
2960 if (inline_summaries
->get (node
->global
.inlined_to
)->estimated_stack_size
< peak
)
2961 inline_summaries
->get (node
->global
.inlined_to
)->estimated_stack_size
= peak
;
2962 ipa_propagate_frequency (node
);
2963 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2965 if (!e
->inline_failed
)
2966 inline_update_callee_summaries (e
->callee
, depth
);
2967 inline_edge_summary (e
)->loop_depth
+= depth
;
2969 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2970 inline_edge_summary (e
)->loop_depth
+= depth
;
2973 /* Update change_prob of EDGE after INLINED_EDGE has been inlined.
2974 When functoin A is inlined in B and A calls C with parameter that
2975 changes with probability PROB1 and C is known to be passthroug
2976 of argument if B that change with probability PROB2, the probability
2977 of change is now PROB1*PROB2. */
2980 remap_edge_change_prob (struct cgraph_edge
*inlined_edge
,
2981 struct cgraph_edge
*edge
)
2983 if (ipa_node_params_sum
)
2986 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
2987 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
2988 struct inline_edge_summary
*inlined_es
2989 = inline_edge_summary (inlined_edge
);
2991 for (i
= 0; i
< ipa_get_cs_argument_count (args
); i
++)
2993 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
2994 if (jfunc
->type
== IPA_JF_PASS_THROUGH
2995 || jfunc
->type
== IPA_JF_ANCESTOR
)
2997 int id
= jfunc
->type
== IPA_JF_PASS_THROUGH
2998 ? ipa_get_jf_pass_through_formal_id (jfunc
)
2999 : ipa_get_jf_ancestor_formal_id (jfunc
);
3000 if (id
< (int) inlined_es
->param
.length ())
3002 int prob1
= es
->param
[i
].change_prob
;
3003 int prob2
= inlined_es
->param
[id
].change_prob
;
3004 int prob
= combine_probabilities (prob1
, prob2
);
3006 if (prob1
&& prob2
&& !prob
)
3009 es
->param
[i
].change_prob
= prob
;
3016 /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
3018 Remap predicates of callees of NODE. Rest of arguments match
3021 Also update change probabilities. */
3024 remap_edge_summaries (struct cgraph_edge
*inlined_edge
,
3025 struct cgraph_node
*node
,
3026 struct inline_summary
*info
,
3027 struct inline_summary
*callee_info
,
3028 vec
<int> operand_map
,
3029 vec
<int> offset_map
,
3030 clause_t possible_truths
,
3031 predicate
*toplev_predicate
)
3033 struct cgraph_edge
*e
, *next
;
3034 for (e
= node
->callees
; e
; e
= next
)
3036 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3038 next
= e
->next_callee
;
3040 if (e
->inline_failed
)
3042 remap_edge_change_prob (inlined_edge
, e
);
3046 p
= es
->predicate
->remap_after_inlining
3047 (info
, callee_info
, operand_map
,
3048 offset_map
, possible_truths
,
3050 edge_set_predicate (e
, &p
);
3053 edge_set_predicate (e
, toplev_predicate
);
3056 remap_edge_summaries (inlined_edge
, e
->callee
, info
, callee_info
,
3057 operand_map
, offset_map
, possible_truths
,
3060 for (e
= node
->indirect_calls
; e
; e
= next
)
3062 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3064 next
= e
->next_callee
;
3066 remap_edge_change_prob (inlined_edge
, e
);
3069 p
= es
->predicate
->remap_after_inlining
3070 (info
, callee_info
, operand_map
, offset_map
,
3071 possible_truths
, *toplev_predicate
);
3072 edge_set_predicate (e
, &p
);
3075 edge_set_predicate (e
, toplev_predicate
);
3079 /* Same as remap_predicate, but set result into hint *HINT. */
3082 remap_hint_predicate (struct inline_summary
*info
,
3083 struct inline_summary
*callee_info
,
3085 vec
<int> operand_map
,
3086 vec
<int> offset_map
,
3087 clause_t possible_truths
,
3088 predicate
*toplev_predicate
)
3094 p
= (*hint
)->remap_after_inlining
3096 operand_map
, offset_map
,
3097 possible_truths
, *toplev_predicate
);
3098 if (p
!= false && p
!= true)
3101 set_hint_predicate (hint
, p
);
3107 /* We inlined EDGE. Update summary of the function we inlined into. */
3110 inline_merge_summary (struct cgraph_edge
*edge
)
3112 struct inline_summary
*callee_info
= inline_summaries
->get (edge
->callee
);
3113 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3114 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
3115 struct inline_summary
*info
= inline_summaries
->get (to
);
3116 clause_t clause
= 0; /* not_inline is known to be false. */
3118 vec
<int> operand_map
= vNULL
;
3119 vec
<int> offset_map
= vNULL
;
3121 predicate toplev_predicate
;
3122 predicate true_p
= true;
3123 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3126 toplev_predicate
= *es
->predicate
;
3128 toplev_predicate
= true;
3130 info
->fp_expressions
|= callee_info
->fp_expressions
;
3132 if (callee_info
->conds
)
3133 evaluate_properties_for_edge (edge
, true, &clause
, NULL
, NULL
, NULL
, NULL
);
3134 if (ipa_node_params_sum
&& callee_info
->conds
)
3136 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3137 int count
= ipa_get_cs_argument_count (args
);
3142 operand_map
.safe_grow_cleared (count
);
3143 offset_map
.safe_grow_cleared (count
);
3145 for (i
= 0; i
< count
; i
++)
3147 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3150 /* TODO: handle non-NOPs when merging. */
3151 if (jfunc
->type
== IPA_JF_PASS_THROUGH
)
3153 if (ipa_get_jf_pass_through_operation (jfunc
) == NOP_EXPR
)
3154 map
= ipa_get_jf_pass_through_formal_id (jfunc
);
3155 if (!ipa_get_jf_pass_through_agg_preserved (jfunc
))
3158 else if (jfunc
->type
== IPA_JF_ANCESTOR
)
3160 HOST_WIDE_INT offset
= ipa_get_jf_ancestor_offset (jfunc
);
3161 if (offset
>= 0 && offset
< INT_MAX
)
3163 map
= ipa_get_jf_ancestor_formal_id (jfunc
);
3164 if (!ipa_get_jf_ancestor_agg_preserved (jfunc
))
3166 offset_map
[i
] = offset
;
3169 operand_map
[i
] = map
;
3170 gcc_assert (map
< ipa_get_param_count (IPA_NODE_REF (to
)));
3173 for (i
= 0; vec_safe_iterate (callee_info
->entry
, i
, &e
); i
++)
3176 p
= e
->exec_predicate
.remap_after_inlining
3177 (info
, callee_info
, operand_map
,
3180 predicate nonconstp
;
3181 nonconstp
= e
->nonconst_predicate
.remap_after_inlining
3182 (info
, callee_info
, operand_map
,
3185 if (p
!= false && nonconstp
!= false)
3187 sreal add_time
= ((sreal
)e
->time
* edge
->frequency
) / CGRAPH_FREQ_BASE
;
3188 int prob
= e
->nonconst_predicate
.probability (callee_info
->conds
,
3190 add_time
= add_time
* prob
/ REG_BR_PROB_BASE
;
3191 if (prob
!= REG_BR_PROB_BASE
3192 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3194 fprintf (dump_file
, "\t\tScaling time by probability:%f\n",
3195 (double) prob
/ REG_BR_PROB_BASE
);
3197 account_size_time (info
, e
->size
, add_time
, &p
, &nonconstp
);
3200 remap_edge_summaries (edge
, edge
->callee
, info
, callee_info
, operand_map
,
3201 offset_map
, clause
, &toplev_predicate
);
3202 remap_hint_predicate (info
, callee_info
,
3203 &callee_info
->loop_iterations
,
3204 operand_map
, offset_map
, clause
, &toplev_predicate
);
3205 remap_hint_predicate (info
, callee_info
,
3206 &callee_info
->loop_stride
,
3207 operand_map
, offset_map
, clause
, &toplev_predicate
);
3208 remap_hint_predicate (info
, callee_info
,
3209 &callee_info
->array_index
,
3210 operand_map
, offset_map
, clause
, &toplev_predicate
);
3212 inline_update_callee_summaries (edge
->callee
,
3213 inline_edge_summary (edge
)->loop_depth
);
3215 /* We do not maintain predicates of inlined edges, free it. */
3216 edge_set_predicate (edge
, &true_p
);
3217 /* Similarly remove param summaries. */
3218 es
->param
.release ();
3219 operand_map
.release ();
3220 offset_map
.release ();
3223 /* For performance reasons inline_merge_summary is not updating overall size
3224 and time. Recompute it. */
3227 inline_update_overall_summary (struct cgraph_node
*node
)
3229 struct inline_summary
*info
= inline_summaries
->get (node
);
3235 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
3237 info
->size
+= e
->size
;
3238 info
->time
+= e
->time
;
3240 estimate_calls_size_and_time (node
, &info
->size
, &info
->min_size
,
3242 ~(clause_t
) (1 << predicate::false_condition
),
3243 vNULL
, vNULL
, vNULL
);
3244 info
->size
= (info
->size
+ INLINE_SIZE_SCALE
/ 2) / INLINE_SIZE_SCALE
;
3247 /* Return hints derrived from EDGE. */
3249 simple_edge_hints (struct cgraph_edge
*edge
)
3252 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3253 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
3254 struct cgraph_node
*callee
= edge
->callee
->ultimate_alias_target ();
3255 if (inline_summaries
->get (to
)->scc_no
3256 && inline_summaries
->get (to
)->scc_no
3257 == inline_summaries
->get (callee
)->scc_no
3258 && !edge
->recursive_p ())
3259 hints
|= INLINE_HINT_same_scc
;
3261 if (callee
->lto_file_data
&& edge
->caller
->lto_file_data
3262 && edge
->caller
->lto_file_data
!= callee
->lto_file_data
3263 && !callee
->merged_comdat
&& !callee
->icf_merged
)
3264 hints
|= INLINE_HINT_cross_module
;
3269 /* Estimate the time cost for the caller when inlining EDGE.
3270 Only to be called via estimate_edge_time, that handles the
3273 When caching, also update the cache entry. Compute both time and
3274 size, since we always need both metrics eventually. */
3277 do_estimate_edge_time (struct cgraph_edge
*edge
)
3279 sreal time
, nonspec_time
;
3282 struct cgraph_node
*callee
;
3283 clause_t clause
, nonspec_clause
;
3284 vec
<tree
> known_vals
;
3285 vec
<ipa_polymorphic_call_context
> known_contexts
;
3286 vec
<ipa_agg_jump_function_p
> known_aggs
;
3287 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3290 callee
= edge
->callee
->ultimate_alias_target ();
3292 gcc_checking_assert (edge
->inline_failed
);
3293 evaluate_properties_for_edge (edge
, true,
3294 &clause
, &nonspec_clause
, &known_vals
,
3295 &known_contexts
, &known_aggs
);
3296 estimate_node_size_and_time (callee
, clause
, nonspec_clause
, known_vals
,
3297 known_contexts
, known_aggs
, &size
, &min_size
,
3298 &time
, &nonspec_time
, &hints
, es
->param
);
3300 /* When we have profile feedback, we can quite safely identify hot
3301 edges and for those we disable size limits. Don't do that when
3302 probability that caller will call the callee is low however, since it
3303 may hurt optimization of the caller's hot path. */
3304 if (edge
->count
&& edge
->maybe_hot_p ()
3306 > (edge
->caller
->global
.inlined_to
3307 ? edge
->caller
->global
.inlined_to
->count
: edge
->caller
->count
)))
3308 hints
|= INLINE_HINT_known_hot
;
3310 known_vals
.release ();
3311 known_contexts
.release ();
3312 known_aggs
.release ();
3313 gcc_checking_assert (size
>= 0);
3314 gcc_checking_assert (time
>= 0);
3316 /* When caching, update the cache entry. */
3317 if (edge_growth_cache
.exists ())
3319 inline_summaries
->get (edge
->callee
)->min_size
= min_size
;
3320 if ((int) edge_growth_cache
.length () <= edge
->uid
)
3321 edge_growth_cache
.safe_grow_cleared (symtab
->edges_max_uid
);
3322 edge_growth_cache
[edge
->uid
].time
= time
;
3323 edge_growth_cache
[edge
->uid
].nonspec_time
= nonspec_time
;
3325 edge_growth_cache
[edge
->uid
].size
= size
+ (size
>= 0);
3326 hints
|= simple_edge_hints (edge
);
3327 edge_growth_cache
[edge
->uid
].hints
= hints
+ 1;
3333 /* Return estimated callee growth after inlining EDGE.
3334 Only to be called via estimate_edge_size. */
3337 do_estimate_edge_size (struct cgraph_edge
*edge
)
3340 struct cgraph_node
*callee
;
3341 clause_t clause
, nonspec_clause
;
3342 vec
<tree
> known_vals
;
3343 vec
<ipa_polymorphic_call_context
> known_contexts
;
3344 vec
<ipa_agg_jump_function_p
> known_aggs
;
3346 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3348 if (edge_growth_cache
.exists ())
3350 do_estimate_edge_time (edge
);
3351 size
= edge_growth_cache
[edge
->uid
].size
;
3352 gcc_checking_assert (size
);
3353 return size
- (size
> 0);
3356 callee
= edge
->callee
->ultimate_alias_target ();
3358 /* Early inliner runs without caching, go ahead and do the dirty work. */
3359 gcc_checking_assert (edge
->inline_failed
);
3360 evaluate_properties_for_edge (edge
, true,
3361 &clause
, &nonspec_clause
,
3362 &known_vals
, &known_contexts
,
3364 estimate_node_size_and_time (callee
, clause
, nonspec_clause
, known_vals
,
3365 known_contexts
, known_aggs
, &size
, NULL
, NULL
,
3367 known_vals
.release ();
3368 known_contexts
.release ();
3369 known_aggs
.release ();
3374 /* Estimate the growth of the caller when inlining EDGE.
3375 Only to be called via estimate_edge_size. */
3378 do_estimate_edge_hints (struct cgraph_edge
*edge
)
3381 struct cgraph_node
*callee
;
3382 clause_t clause
, nonspec_clause
;
3383 vec
<tree
> known_vals
;
3384 vec
<ipa_polymorphic_call_context
> known_contexts
;
3385 vec
<ipa_agg_jump_function_p
> known_aggs
;
3387 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3389 if (edge_growth_cache
.exists ())
3391 do_estimate_edge_time (edge
);
3392 hints
= edge_growth_cache
[edge
->uid
].hints
;
3393 gcc_checking_assert (hints
);
3397 callee
= edge
->callee
->ultimate_alias_target ();
3399 /* Early inliner runs without caching, go ahead and do the dirty work. */
3400 gcc_checking_assert (edge
->inline_failed
);
3401 evaluate_properties_for_edge (edge
, true,
3402 &clause
, &nonspec_clause
,
3403 &known_vals
, &known_contexts
,
3405 estimate_node_size_and_time (callee
, clause
, nonspec_clause
, known_vals
,
3406 known_contexts
, known_aggs
, NULL
, NULL
,
3407 NULL
, NULL
, &hints
, vNULL
);
3408 known_vals
.release ();
3409 known_contexts
.release ();
3410 known_aggs
.release ();
3411 hints
|= simple_edge_hints (edge
);
3415 /* Estimate the size of NODE after inlining EDGE which should be an
3416 edge to either NODE or a call inlined into NODE. */
3419 estimate_size_after_inlining (struct cgraph_node
*node
,
3420 struct cgraph_edge
*edge
)
3422 struct inline_edge_summary
*es
= inline_edge_summary (edge
);
3423 if (!es
->predicate
|| *es
->predicate
!= false)
3425 int size
= inline_summaries
->get (node
)->size
+ estimate_edge_growth (edge
);
3426 gcc_assert (size
>= 0);
3429 return inline_summaries
->get (node
)->size
;
3435 struct cgraph_node
*node
;
3436 bool self_recursive
;
3442 /* Worker for do_estimate_growth. Collect growth for all callers. */
3445 do_estimate_growth_1 (struct cgraph_node
*node
, void *data
)
3447 struct cgraph_edge
*e
;
3448 struct growth_data
*d
= (struct growth_data
*) data
;
3450 for (e
= node
->callers
; e
; e
= e
->next_caller
)
3452 gcc_checking_assert (e
->inline_failed
);
3454 if (cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
3456 d
->uninlinable
= true;
3460 if (e
->recursive_p ())
3462 d
->self_recursive
= true;
3465 d
->growth
+= estimate_edge_growth (e
);
3471 /* Estimate the growth caused by inlining NODE into all callees. */
3474 estimate_growth (struct cgraph_node
*node
)
3476 struct growth_data d
= { node
, false, false, 0 };
3477 struct inline_summary
*info
= inline_summaries
->get (node
);
3479 node
->call_for_symbol_and_aliases (do_estimate_growth_1
, &d
, true);
3481 /* For self recursive functions the growth estimation really should be
3482 infinity. We don't want to return very large values because the growth
3483 plays various roles in badness computation fractions. Be sure to not
3484 return zero or negative growths. */
3485 if (d
.self_recursive
)
3486 d
.growth
= d
.growth
< info
->size
? info
->size
: d
.growth
;
3487 else if (DECL_EXTERNAL (node
->decl
) || d
.uninlinable
)
3491 if (node
->will_be_removed_from_program_if_no_direct_calls_p ())
3492 d
.growth
-= info
->size
;
3493 /* COMDAT functions are very often not shared across multiple units
3494 since they come from various template instantiations.
3495 Take this into account. */
3496 else if (DECL_COMDAT (node
->decl
)
3497 && node
->can_remove_if_no_direct_calls_p ())
3498 d
.growth
-= (info
->size
3499 * (100 - PARAM_VALUE (PARAM_COMDAT_SHARING_PROBABILITY
))
3506 /* Verify if there are fewer than MAX_CALLERS. */
3509 check_callers (cgraph_node
*node
, int *max_callers
)
3513 if (!node
->can_remove_if_no_direct_calls_and_refs_p ())
3516 for (cgraph_edge
*e
= node
->callers
; e
; e
= e
->next_caller
)
3520 || cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
3524 FOR_EACH_ALIAS (node
, ref
)
3525 if (check_callers (dyn_cast
<cgraph_node
*> (ref
->referring
), max_callers
))
3532 /* Make cheap estimation if growth of NODE is likely positive knowing
3533 EDGE_GROWTH of one particular edge.
3534 We assume that most of other edges will have similar growth
3535 and skip computation if there are too many callers. */
3538 growth_likely_positive (struct cgraph_node
*node
,
3542 struct cgraph_edge
*e
;
3543 gcc_checking_assert (edge_growth
> 0);
3545 /* First quickly check if NODE is removable at all. */
3546 if (DECL_EXTERNAL (node
->decl
))
3548 if (!node
->can_remove_if_no_direct_calls_and_refs_p ()
3549 || node
->address_taken
)
3552 max_callers
= inline_summaries
->get (node
)->size
* 4 / edge_growth
+ 2;
3554 for (e
= node
->callers
; e
; e
= e
->next_caller
)
3558 || cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
3563 FOR_EACH_ALIAS (node
, ref
)
3564 if (check_callers (dyn_cast
<cgraph_node
*> (ref
->referring
), &max_callers
))
3567 /* Unlike for functions called once, we play unsafe with
3568 COMDATs. We can allow that since we know functions
3569 in consideration are small (and thus risk is small) and
3570 moreover grow estimates already accounts that COMDAT
3571 functions may or may not disappear when eliminated from
3572 current unit. With good probability making aggressive
3573 choice in all units is going to make overall program
3575 if (DECL_COMDAT (node
->decl
))
3577 if (!node
->can_remove_if_no_direct_calls_p ())
3580 else if (!node
->will_be_removed_from_program_if_no_direct_calls_p ())
3583 return estimate_growth (node
) > 0;
3587 /* This function performs intraprocedural analysis in NODE that is required to
3588 inline indirect calls. */
3591 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
3593 ipa_analyze_node (node
);
3594 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3596 ipa_print_node_params (dump_file
, node
);
3597 ipa_print_node_jump_functions (dump_file
, node
);
3602 /* Note function body size. */
3605 inline_analyze_function (struct cgraph_node
*node
)
3607 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
3610 fprintf (dump_file
, "\nAnalyzing function: %s/%u\n",
3611 node
->name (), node
->order
);
3612 if (opt_for_fn (node
->decl
, optimize
) && !node
->thunk
.thunk_p
)
3613 inline_indirect_intraprocedural_analysis (node
);
3614 compute_inline_parameters (node
, false);
3617 struct cgraph_edge
*e
;
3618 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3619 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3620 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3621 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3628 /* Called when new function is inserted to callgraph late. */
3631 inline_summary_t::insert (struct cgraph_node
*node
, inline_summary
*)
3633 inline_analyze_function (node
);
3636 /* Note function body size. */
3639 inline_generate_summary (void)
3641 struct cgraph_node
*node
;
3643 FOR_EACH_DEFINED_FUNCTION (node
)
3644 if (DECL_STRUCT_FUNCTION (node
->decl
))
3645 node
->local
.versionable
= tree_versionable_function_p (node
->decl
);
3647 /* When not optimizing, do not bother to analyze. Inlining is still done
3648 because edge redirection needs to happen there. */
3649 if (!optimize
&& !flag_generate_lto
&& !flag_generate_offload
&& !flag_wpa
)
3652 if (!inline_summaries
)
3653 inline_summaries
= (inline_summary_t
*) inline_summary_t::create_ggc (symtab
);
3655 inline_summaries
->enable_insertion_hook ();
3657 ipa_register_cgraph_hooks ();
3658 inline_free_summary ();
3660 FOR_EACH_DEFINED_FUNCTION (node
)
3662 inline_analyze_function (node
);
3666 /* Write inline summary for edge E to OB. */
3669 read_inline_edge_summary (struct lto_input_block
*ib
, struct cgraph_edge
*e
)
3671 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3675 es
->call_stmt_size
= streamer_read_uhwi (ib
);
3676 es
->call_stmt_time
= streamer_read_uhwi (ib
);
3677 es
->loop_depth
= streamer_read_uhwi (ib
);
3679 edge_set_predicate (e
, &p
);
3680 length
= streamer_read_uhwi (ib
);
3683 es
->param
.safe_grow_cleared (length
);
3684 for (i
= 0; i
< length
; i
++)
3685 es
->param
[i
].change_prob
= streamer_read_uhwi (ib
);
3690 /* Stream in inline summaries from the section. */
3693 inline_read_section (struct lto_file_decl_data
*file_data
, const char *data
,
3696 const struct lto_function_header
*header
=
3697 (const struct lto_function_header
*) data
;
3698 const int cfg_offset
= sizeof (struct lto_function_header
);
3699 const int main_offset
= cfg_offset
+ header
->cfg_size
;
3700 const int string_offset
= main_offset
+ header
->main_size
;
3701 struct data_in
*data_in
;
3702 unsigned int i
, count2
, j
;
3703 unsigned int f_count
;
3705 lto_input_block
ib ((const char *) data
+ main_offset
, header
->main_size
,
3706 file_data
->mode_table
);
3709 lto_data_in_create (file_data
, (const char *) data
+ string_offset
,
3710 header
->string_size
, vNULL
);
3711 f_count
= streamer_read_uhwi (&ib
);
3712 for (i
= 0; i
< f_count
; i
++)
3715 struct cgraph_node
*node
;
3716 struct inline_summary
*info
;
3717 lto_symtab_encoder_t encoder
;
3718 struct bitpack_d bp
;
3719 struct cgraph_edge
*e
;
3722 index
= streamer_read_uhwi (&ib
);
3723 encoder
= file_data
->symtab_node_encoder
;
3724 node
= dyn_cast
<cgraph_node
*> (lto_symtab_encoder_deref (encoder
,
3726 info
= inline_summaries
->get (node
);
3728 info
->estimated_stack_size
3729 = info
->estimated_self_stack_size
= streamer_read_uhwi (&ib
);
3730 info
->size
= info
->self_size
= streamer_read_uhwi (&ib
);
3731 info
->time
= info
->self_time
= sreal::stream_in (&ib
);
3733 bp
= streamer_read_bitpack (&ib
);
3734 info
->inlinable
= bp_unpack_value (&bp
, 1);
3735 info
->contains_cilk_spawn
= bp_unpack_value (&bp
, 1);
3736 info
->fp_expressions
= bp_unpack_value (&bp
, 1);
3738 count2
= streamer_read_uhwi (&ib
);
3739 gcc_assert (!info
->conds
);
3740 for (j
= 0; j
< count2
; j
++)
3743 c
.operand_num
= streamer_read_uhwi (&ib
);
3744 c
.size
= streamer_read_uhwi (&ib
);
3745 c
.code
= (enum tree_code
) streamer_read_uhwi (&ib
);
3746 c
.val
= stream_read_tree (&ib
, data_in
);
3747 bp
= streamer_read_bitpack (&ib
);
3748 c
.agg_contents
= bp_unpack_value (&bp
, 1);
3749 c
.by_ref
= bp_unpack_value (&bp
, 1);
3751 c
.offset
= streamer_read_uhwi (&ib
);
3752 vec_safe_push (info
->conds
, c
);
3754 count2
= streamer_read_uhwi (&ib
);
3755 gcc_assert (!info
->entry
);
3756 for (j
= 0; j
< count2
; j
++)
3758 struct size_time_entry e
;
3760 e
.size
= streamer_read_uhwi (&ib
);
3761 e
.time
= sreal::stream_in (&ib
);
3762 e
.exec_predicate
.stream_in (&ib
);
3763 e
.nonconst_predicate
.stream_in (&ib
);
3765 vec_safe_push (info
->entry
, e
);
3769 set_hint_predicate (&info
->loop_iterations
, p
);
3771 set_hint_predicate (&info
->loop_stride
, p
);
3773 set_hint_predicate (&info
->array_index
, p
);
3774 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3775 read_inline_edge_summary (&ib
, e
);
3776 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3777 read_inline_edge_summary (&ib
, e
);
3780 lto_free_section_data (file_data
, LTO_section_inline_summary
, NULL
, data
,
3782 lto_data_in_delete (data_in
);
3786 /* Read inline summary. Jump functions are shared among ipa-cp
3787 and inliner, so when ipa-cp is active, we don't need to write them
3791 inline_read_summary (void)
3793 struct lto_file_decl_data
**file_data_vec
= lto_get_file_decl_data ();
3794 struct lto_file_decl_data
*file_data
;
3797 inline_summary_alloc ();
3799 while ((file_data
= file_data_vec
[j
++]))
3802 const char *data
= lto_get_section_data (file_data
,
3803 LTO_section_inline_summary
,
3806 inline_read_section (file_data
, data
, len
);
3808 /* Fatal error here. We do not want to support compiling ltrans units
3809 with different version of compiler or different flags than the WPA
3810 unit, so this should never happen. */
3811 fatal_error (input_location
,
3812 "ipa inline summary is missing in input file");
3816 ipa_register_cgraph_hooks ();
3818 ipa_prop_read_jump_functions ();
3821 gcc_assert (inline_summaries
);
3822 inline_summaries
->enable_insertion_hook ();
3826 /* Write inline summary for edge E to OB. */
3829 write_inline_edge_summary (struct output_block
*ob
, struct cgraph_edge
*e
)
3831 struct inline_edge_summary
*es
= inline_edge_summary (e
);
3834 streamer_write_uhwi (ob
, es
->call_stmt_size
);
3835 streamer_write_uhwi (ob
, es
->call_stmt_time
);
3836 streamer_write_uhwi (ob
, es
->loop_depth
);
3838 es
->predicate
->stream_out (ob
);
3840 streamer_write_uhwi (ob
, 0);
3841 streamer_write_uhwi (ob
, es
->param
.length ());
3842 for (i
= 0; i
< (int) es
->param
.length (); i
++)
3843 streamer_write_uhwi (ob
, es
->param
[i
].change_prob
);
3847 /* Write inline summary for node in SET.
3848 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
3849 active, we don't need to write them twice. */
3852 inline_write_summary (void)
3854 struct output_block
*ob
= create_output_block (LTO_section_inline_summary
);
3855 lto_symtab_encoder_t encoder
= ob
->decl_state
->symtab_node_encoder
;
3856 unsigned int count
= 0;
3859 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
3861 symtab_node
*snode
= lto_symtab_encoder_deref (encoder
, i
);
3862 cgraph_node
*cnode
= dyn_cast
<cgraph_node
*> (snode
);
3863 if (cnode
&& cnode
->definition
&& !cnode
->alias
)
3866 streamer_write_uhwi (ob
, count
);
3868 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
3870 symtab_node
*snode
= lto_symtab_encoder_deref (encoder
, i
);
3871 cgraph_node
*cnode
= dyn_cast
<cgraph_node
*> (snode
);
3872 if (cnode
&& cnode
->definition
&& !cnode
->alias
)
3874 struct inline_summary
*info
= inline_summaries
->get (cnode
);
3875 struct bitpack_d bp
;
3876 struct cgraph_edge
*edge
;
3879 struct condition
*c
;
3881 streamer_write_uhwi (ob
, lto_symtab_encoder_encode (encoder
, cnode
));
3882 streamer_write_hwi (ob
, info
->estimated_self_stack_size
);
3883 streamer_write_hwi (ob
, info
->self_size
);
3884 info
->self_time
.stream_out (ob
);
3885 bp
= bitpack_create (ob
->main_stream
);
3886 bp_pack_value (&bp
, info
->inlinable
, 1);
3887 bp_pack_value (&bp
, info
->contains_cilk_spawn
, 1);
3888 bp_pack_value (&bp
, info
->fp_expressions
, 1);
3889 streamer_write_bitpack (&bp
);
3890 streamer_write_uhwi (ob
, vec_safe_length (info
->conds
));
3891 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
3893 streamer_write_uhwi (ob
, c
->operand_num
);
3894 streamer_write_uhwi (ob
, c
->size
);
3895 streamer_write_uhwi (ob
, c
->code
);
3896 stream_write_tree (ob
, c
->val
, true);
3897 bp
= bitpack_create (ob
->main_stream
);
3898 bp_pack_value (&bp
, c
->agg_contents
, 1);
3899 bp_pack_value (&bp
, c
->by_ref
, 1);
3900 streamer_write_bitpack (&bp
);
3901 if (c
->agg_contents
)
3902 streamer_write_uhwi (ob
, c
->offset
);
3904 streamer_write_uhwi (ob
, vec_safe_length (info
->entry
));
3905 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
3907 streamer_write_uhwi (ob
, e
->size
);
3908 e
->time
.stream_out (ob
);
3909 e
->exec_predicate
.stream_out (ob
);
3910 e
->nonconst_predicate
.stream_out (ob
);
3912 if (info
->loop_iterations
)
3913 info
->loop_iterations
->stream_out (ob
);
3915 streamer_write_uhwi (ob
, 0);
3916 if (info
->loop_stride
)
3917 info
->loop_stride
->stream_out (ob
);
3919 streamer_write_uhwi (ob
, 0);
3920 if (info
->array_index
)
3921 info
->array_index
->stream_out (ob
);
3923 streamer_write_uhwi (ob
, 0);
3924 for (edge
= cnode
->callees
; edge
; edge
= edge
->next_callee
)
3925 write_inline_edge_summary (ob
, edge
);
3926 for (edge
= cnode
->indirect_calls
; edge
; edge
= edge
->next_callee
)
3927 write_inline_edge_summary (ob
, edge
);
3930 streamer_write_char_stream (ob
->main_stream
, 0);
3931 produce_asm (ob
, NULL
);
3932 destroy_output_block (ob
);
3934 if (optimize
&& !flag_ipa_cp
)
3935 ipa_prop_write_jump_functions ();
3939 /* Release inline summary. */
3942 inline_free_summary (void)
3944 struct cgraph_node
*node
;
3945 if (edge_removal_hook_holder
)
3946 symtab
->remove_edge_removal_hook (edge_removal_hook_holder
);
3947 edge_removal_hook_holder
= NULL
;
3948 if (edge_duplication_hook_holder
)
3949 symtab
->remove_edge_duplication_hook (edge_duplication_hook_holder
);
3950 edge_duplication_hook_holder
= NULL
;
3951 if (!inline_edge_summary_vec
.exists ())
3953 FOR_EACH_DEFINED_FUNCTION (node
)
3955 reset_inline_summary (node
, inline_summaries
->get (node
));
3956 inline_summaries
->release ();
3957 inline_summaries
= NULL
;
3958 inline_edge_summary_vec
.release ();
3959 edge_predicate_pool
.release ();