1 /* Inlining decision heuristics.
2 Copyright (C) 2003-2017 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Analysis used by the inliner and other passes limiting code size growth.
23 We estimate for each function
25 - average function execution time
26 - inlining size benefit (that is how much of function body size
27 and its call sequence is expected to disappear by inlining)
28 - inlining time benefit
31 - call statement size and time
33 inline_summary data structures store above information locally (i.e.
34 parameters of the function itself) and globally (i.e. parameters of
35 the function created by applying all the inline decisions already
36 present in the callgraph).
38 We provide access to the inline_summary data structure and
39 basic logic updating the parameters when inlining is performed.
41 The summaries are context sensitive. Context means
42 1) partial assignment of known constant values of operands
43 2) whether function is inlined into the call or not.
44 It is easy to add more variants. To represent function size and time
45 that depends on context (i.e. it is known to be optimized away when
46 context is known either by inlining or from IP-CP and cloning),
49 estimate_edge_size and estimate_edge_growth can be used to query
50 function size/time in the given context. inline_merge_summary merges
51 properties of caller and callee after inlining.
53 Finally pass_inline_parameters is exported. This is used to drive
54 computation of function parameters used by the early inliner. IPA
55 inlined performs analysis via its analyze_function method. */
59 #include "coretypes.h"
63 #include "alloc-pool.h"
64 #include "tree-pass.h"
66 #include "tree-streamer.h"
68 #include "diagnostic.h"
69 #include "fold-const.h"
70 #include "print-tree.h"
71 #include "tree-inline.h"
72 #include "gimple-pretty-print.h"
75 #include "gimple-iterator.h"
77 #include "tree-ssa-loop-niter.h"
78 #include "tree-ssa-loop.h"
79 #include "symbol-summary.h"
81 #include "ipa-inline.h"
83 #include "tree-scalar-evolution.h"
84 #include "ipa-utils.h"
86 #include "cfgexpand.h"
90 function_summary
<inline_summary
*> *inline_summaries
;
91 call_summary
<ipa_call_summary
*> *ipa_call_summaries
;
93 /* Cached node/edge growths. */
94 vec
<edge_growth_cache_entry
> edge_growth_cache
;
96 /* Edge predicates goes here. */
97 static object_allocator
<predicate
> edge_predicate_pool ("edge predicates");
100 /* Dump inline hints. */
102 dump_inline_hints (FILE *f
, inline_hints hints
)
106 fprintf (f
, "inline hints:");
107 if (hints
& INLINE_HINT_indirect_call
)
109 hints
&= ~INLINE_HINT_indirect_call
;
110 fprintf (f
, " indirect_call");
112 if (hints
& INLINE_HINT_loop_iterations
)
114 hints
&= ~INLINE_HINT_loop_iterations
;
115 fprintf (f
, " loop_iterations");
117 if (hints
& INLINE_HINT_loop_stride
)
119 hints
&= ~INLINE_HINT_loop_stride
;
120 fprintf (f
, " loop_stride");
122 if (hints
& INLINE_HINT_same_scc
)
124 hints
&= ~INLINE_HINT_same_scc
;
125 fprintf (f
, " same_scc");
127 if (hints
& INLINE_HINT_in_scc
)
129 hints
&= ~INLINE_HINT_in_scc
;
130 fprintf (f
, " in_scc");
132 if (hints
& INLINE_HINT_cross_module
)
134 hints
&= ~INLINE_HINT_cross_module
;
135 fprintf (f
, " cross_module");
137 if (hints
& INLINE_HINT_declared_inline
)
139 hints
&= ~INLINE_HINT_declared_inline
;
140 fprintf (f
, " declared_inline");
142 if (hints
& INLINE_HINT_array_index
)
144 hints
&= ~INLINE_HINT_array_index
;
145 fprintf (f
, " array_index");
147 if (hints
& INLINE_HINT_known_hot
)
149 hints
&= ~INLINE_HINT_known_hot
;
150 fprintf (f
, " known_hot");
156 /* Record SIZE and TIME to SUMMARY.
157 The accounted code will be executed when EXEC_PRED is true.
158 When NONCONST_PRED is false the code will evaulate to constant and
159 will get optimized out in specialized clones of the function. */
162 account_size_time (struct inline_summary
*summary
, int size
, sreal time
,
163 predicate
*exec_pred
,
164 predicate
*nonconst_pred_ptr
)
169 predicate nonconst_pred
;
171 if (*exec_pred
== false)
174 nonconst_pred
= *nonconst_pred_ptr
& *exec_pred
;
176 if (nonconst_pred
== false)
179 /* We need to create initial empty unconitional clause, but otherwie
180 we don't need to account empty times and sizes. */
181 if (!size
&& time
== 0 && summary
->entry
)
184 gcc_assert (time
>= 0);
186 for (i
= 0; vec_safe_iterate (summary
->entry
, i
, &e
); i
++)
187 if (e
->exec_predicate
== *exec_pred
188 && e
->nonconst_predicate
== nonconst_pred
)
197 e
= &(*summary
->entry
)[0];
198 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
200 "\t\tReached limit on number of entries, "
201 "ignoring the predicate.");
203 if (dump_file
&& (dump_flags
& TDF_DETAILS
) && (time
!= 0 || size
))
206 "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate exec:",
207 ((double) size
) / INLINE_SIZE_SCALE
,
208 (time
.to_double ()), found
? "" : "new ");
209 exec_pred
->dump (dump_file
, summary
->conds
, 0);
210 if (*exec_pred
!= nonconst_pred
)
212 fprintf (dump_file
, " nonconst:");
213 nonconst_pred
.dump (dump_file
, summary
->conds
);
216 fprintf (dump_file
, "\n");
220 struct size_time_entry new_entry
;
221 new_entry
.size
= size
;
222 new_entry
.time
= time
;
223 new_entry
.exec_predicate
= *exec_pred
;
224 new_entry
.nonconst_predicate
= nonconst_pred
;
225 vec_safe_push (summary
->entry
, new_entry
);
234 /* We proved E to be unreachable, redirect it to __bultin_unreachable. */
236 static struct cgraph_edge
*
237 redirect_to_unreachable (struct cgraph_edge
*e
)
239 struct cgraph_node
*callee
= !e
->inline_failed
? e
->callee
: NULL
;
240 struct cgraph_node
*target
= cgraph_node::get_create
241 (builtin_decl_implicit (BUILT_IN_UNREACHABLE
));
244 e
= e
->resolve_speculation (target
->decl
);
246 e
->make_direct (target
);
248 e
->redirect_callee (target
);
249 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
250 e
->inline_failed
= CIF_UNREACHABLE
;
253 es
->call_stmt_size
= 0;
254 es
->call_stmt_time
= 0;
256 callee
->remove_symbol_and_inline_clones ();
260 /* Set predicate for edge E. */
263 edge_set_predicate (struct cgraph_edge
*e
, predicate
*predicate
)
265 /* If the edge is determined to be never executed, redirect it
266 to BUILTIN_UNREACHABLE to save inliner from inlining into it. */
267 if (predicate
&& *predicate
== false
268 /* When handling speculative edges, we need to do the redirection
269 just once. Do it always on the direct edge, so we do not
270 attempt to resolve speculation while duplicating the edge. */
271 && (!e
->speculative
|| e
->callee
))
272 e
= redirect_to_unreachable (e
);
274 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
275 if (predicate
&& *predicate
!= true)
278 es
->predicate
= edge_predicate_pool
.allocate ();
279 *es
->predicate
= *predicate
;
284 edge_predicate_pool
.remove (es
->predicate
);
285 es
->predicate
= NULL
;
289 /* Set predicate for hint *P. */
292 set_hint_predicate (predicate
**p
, predicate new_predicate
)
294 if (new_predicate
== false || new_predicate
== true)
297 edge_predicate_pool
.remove (*p
);
303 *p
= edge_predicate_pool
.allocate ();
309 /* Compute what conditions may or may not hold given invormation about
310 parameters. RET_CLAUSE returns truths that may hold in a specialized copy,
311 whie RET_NONSPEC_CLAUSE returns truths that may hold in an nonspecialized
312 copy when called in a given context. It is a bitmask of conditions. Bit
313 0 means that condition is known to be false, while bit 1 means that condition
314 may or may not be true. These differs - for example NOT_INLINED condition
315 is always false in the second and also builtin_constant_p tests can not use
316 the fact that parameter is indeed a constant.
318 KNOWN_VALS is partial mapping of parameters of NODE to constant values.
319 KNOWN_AGGS is a vector of aggreggate jump functions for each parameter.
320 Return clause of possible truths. When INLINE_P is true, assume that we are
323 ERROR_MARK means compile time invariant. */
326 evaluate_conditions_for_known_args (struct cgraph_node
*node
,
328 vec
<tree
> known_vals
,
329 vec
<ipa_agg_jump_function_p
>
331 clause_t
*ret_clause
,
332 clause_t
*ret_nonspec_clause
)
334 clause_t clause
= inline_p
? 0 : 1 << predicate::not_inlined_condition
;
335 clause_t nonspec_clause
= 1 << predicate::not_inlined_condition
;
336 struct inline_summary
*info
= inline_summaries
->get (node
);
340 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
345 /* We allow call stmt to have fewer arguments than the callee function
346 (especially for K&R style programs). So bound check here (we assume
347 known_aggs vector, if non-NULL, has the same length as
349 gcc_checking_assert (!known_aggs
.exists ()
350 || (known_vals
.length () == known_aggs
.length ()));
351 if (c
->operand_num
>= (int) known_vals
.length ())
353 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
354 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
360 struct ipa_agg_jump_function
*agg
;
362 if (c
->code
== predicate::changed
364 && (known_vals
[c
->operand_num
] == error_mark_node
))
367 if (known_aggs
.exists ())
369 agg
= known_aggs
[c
->operand_num
];
370 val
= ipa_find_agg_cst_for_param (agg
, known_vals
[c
->operand_num
],
371 c
->offset
, c
->by_ref
);
378 val
= known_vals
[c
->operand_num
];
379 if (val
== error_mark_node
&& c
->code
!= predicate::changed
)
385 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
386 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
389 if (c
->code
== predicate::changed
)
391 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
395 if (tree_to_shwi (TYPE_SIZE (TREE_TYPE (val
))) != c
->size
)
397 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
398 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
401 if (c
->code
== predicate::is_not_constant
)
403 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
407 val
= fold_unary (VIEW_CONVERT_EXPR
, TREE_TYPE (c
->val
), val
);
409 ? fold_binary_to_constant (c
->code
, boolean_type_node
, val
, c
->val
)
412 if (res
&& integer_zerop (res
))
415 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
416 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
418 *ret_clause
= clause
;
419 if (ret_nonspec_clause
)
420 *ret_nonspec_clause
= nonspec_clause
;
424 /* Work out what conditions might be true at invocation of E. */
427 evaluate_properties_for_edge (struct cgraph_edge
*e
, bool inline_p
,
428 clause_t
*clause_ptr
, clause_t
*nonspec_clause_ptr
,
429 vec
<tree
> *known_vals_ptr
,
430 vec
<ipa_polymorphic_call_context
>
432 vec
<ipa_agg_jump_function_p
> *known_aggs_ptr
)
434 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
435 struct inline_summary
*info
= inline_summaries
->get (callee
);
436 vec
<tree
> known_vals
= vNULL
;
437 vec
<ipa_agg_jump_function_p
> known_aggs
= vNULL
;
440 *clause_ptr
= inline_p
? 0 : 1 << predicate::not_inlined_condition
;
442 known_vals_ptr
->create (0);
443 if (known_contexts_ptr
)
444 known_contexts_ptr
->create (0);
446 if (ipa_node_params_sum
447 && !e
->call_stmt_cannot_inline_p
448 && ((clause_ptr
&& info
->conds
) || known_vals_ptr
|| known_contexts_ptr
))
450 struct ipa_node_params
*parms_info
;
451 struct ipa_edge_args
*args
= IPA_EDGE_REF (e
);
452 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
453 int i
, count
= ipa_get_cs_argument_count (args
);
455 if (e
->caller
->global
.inlined_to
)
456 parms_info
= IPA_NODE_REF (e
->caller
->global
.inlined_to
);
458 parms_info
= IPA_NODE_REF (e
->caller
);
460 if (count
&& (info
->conds
|| known_vals_ptr
))
461 known_vals
.safe_grow_cleared (count
);
462 if (count
&& (info
->conds
|| known_aggs_ptr
))
463 known_aggs
.safe_grow_cleared (count
);
464 if (count
&& known_contexts_ptr
)
465 known_contexts_ptr
->safe_grow_cleared (count
);
467 for (i
= 0; i
< count
; i
++)
469 struct ipa_jump_func
*jf
= ipa_get_ith_jump_func (args
, i
);
470 tree cst
= ipa_value_from_jfunc (parms_info
, jf
);
472 if (!cst
&& e
->call_stmt
473 && i
< (int)gimple_call_num_args (e
->call_stmt
))
475 cst
= gimple_call_arg (e
->call_stmt
, i
);
476 if (!is_gimple_min_invariant (cst
))
481 gcc_checking_assert (TREE_CODE (cst
) != TREE_BINFO
);
482 if (known_vals
.exists ())
485 else if (inline_p
&& !es
->param
[i
].change_prob
)
486 known_vals
[i
] = error_mark_node
;
488 if (known_contexts_ptr
)
489 (*known_contexts_ptr
)[i
] = ipa_context_from_jfunc (parms_info
, e
,
491 /* TODO: When IPA-CP starts propagating and merging aggregate jump
492 functions, use its knowledge of the caller too, just like the
493 scalar case above. */
494 known_aggs
[i
] = &jf
->agg
;
497 else if (e
->call_stmt
&& !e
->call_stmt_cannot_inline_p
498 && ((clause_ptr
&& info
->conds
) || known_vals_ptr
))
500 int i
, count
= (int)gimple_call_num_args (e
->call_stmt
);
502 if (count
&& (info
->conds
|| known_vals_ptr
))
503 known_vals
.safe_grow_cleared (count
);
504 for (i
= 0; i
< count
; i
++)
506 tree cst
= gimple_call_arg (e
->call_stmt
, i
);
507 if (!is_gimple_min_invariant (cst
))
514 evaluate_conditions_for_known_args (callee
, inline_p
,
515 known_vals
, known_aggs
, clause_ptr
,
519 *known_vals_ptr
= known_vals
;
521 known_vals
.release ();
524 *known_aggs_ptr
= known_aggs
;
526 known_aggs
.release ();
530 /* Allocate the inline summary vector or resize it to cover all cgraph nodes. */
533 inline_summary_alloc (void)
535 if (!inline_summaries
)
536 inline_summaries
= inline_summary_t::create_ggc (symtab
);
537 if (!ipa_call_summaries
)
538 ipa_call_summaries
= new ipa_call_summary_t (symtab
, false);
541 /* We are called multiple time for given function; clear
542 data from previous run so they are not cumulated. */
545 reset_ipa_call_summary (struct cgraph_edge
*e
)
547 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
549 es
->call_stmt_size
= es
->call_stmt_time
= 0;
551 edge_predicate_pool
.remove (es
->predicate
);
552 es
->predicate
= NULL
;
553 es
->param
.release ();
556 /* We are called multiple time for given function; clear
557 data from previous run so they are not cumulated. */
560 reset_inline_summary (struct cgraph_node
*node
,
561 inline_summary
*info
)
563 struct cgraph_edge
*e
;
567 info
->estimated_stack_size
= 0;
568 info
->estimated_self_stack_size
= 0;
569 info
->stack_frame_offset
= 0;
574 if (info
->loop_iterations
)
576 edge_predicate_pool
.remove (info
->loop_iterations
);
577 info
->loop_iterations
= NULL
;
579 if (info
->loop_stride
)
581 edge_predicate_pool
.remove (info
->loop_stride
);
582 info
->loop_stride
= NULL
;
584 if (info
->array_index
)
586 edge_predicate_pool
.remove (info
->array_index
);
587 info
->array_index
= NULL
;
589 vec_free (info
->conds
);
590 vec_free (info
->entry
);
591 for (e
= node
->callees
; e
; e
= e
->next_callee
)
592 reset_ipa_call_summary (e
);
593 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
594 reset_ipa_call_summary (e
);
595 info
->fp_expressions
= false;
598 /* Hook that is called by cgraph.c when a node is removed. */
601 inline_summary_t::remove (cgraph_node
*node
, inline_summary
*info
)
603 reset_inline_summary (node
, info
);
606 /* Same as remap_predicate_after_duplication but handle hint predicate *P.
607 Additionally care about allocating new memory slot for updated predicate
608 and set it to NULL when it becomes true or false (and thus uninteresting).
612 remap_hint_predicate_after_duplication (predicate
**p
,
613 clause_t possible_truths
)
615 predicate new_predicate
;
620 new_predicate
= (*p
)->remap_after_duplication (possible_truths
);
621 /* We do not want to free previous predicate; it is used by node origin. */
623 set_hint_predicate (p
, new_predicate
);
627 /* Hook that is called by cgraph.c when a node is duplicated. */
629 inline_summary_t::duplicate (cgraph_node
*src
,
632 inline_summary
*info
)
634 inline_summary_alloc ();
635 memcpy (info
, inline_summaries
->get (src
), sizeof (inline_summary
));
636 /* TODO: as an optimization, we may avoid copying conditions
637 that are known to be false or true. */
638 info
->conds
= vec_safe_copy (info
->conds
);
640 /* When there are any replacements in the function body, see if we can figure
641 out that something was optimized out. */
642 if (ipa_node_params_sum
&& dst
->clone
.tree_map
)
644 vec
<size_time_entry
, va_gc
> *entry
= info
->entry
;
645 /* Use SRC parm info since it may not be copied yet. */
646 struct ipa_node_params
*parms_info
= IPA_NODE_REF (src
);
647 vec
<tree
> known_vals
= vNULL
;
648 int count
= ipa_get_param_count (parms_info
);
650 clause_t possible_truths
;
651 predicate true_pred
= true;
653 int optimized_out_size
= 0;
654 bool inlined_to_p
= false;
655 struct cgraph_edge
*edge
, *next
;
658 known_vals
.safe_grow_cleared (count
);
659 for (i
= 0; i
< count
; i
++)
661 struct ipa_replace_map
*r
;
663 for (j
= 0; vec_safe_iterate (dst
->clone
.tree_map
, j
, &r
); j
++)
665 if (((!r
->old_tree
&& r
->parm_num
== i
)
666 || (r
->old_tree
&& r
->old_tree
== ipa_get_param (parms_info
, i
)))
667 && r
->replace_p
&& !r
->ref_p
)
669 known_vals
[i
] = r
->new_tree
;
674 evaluate_conditions_for_known_args (dst
, false,
678 /* We are going to specialize,
679 so ignore nonspec truths. */
681 known_vals
.release ();
683 account_size_time (info
, 0, 0, &true_pred
, &true_pred
);
685 /* Remap size_time vectors.
686 Simplify the predicate by prunning out alternatives that are known
688 TODO: as on optimization, we can also eliminate conditions known
690 for (i
= 0; vec_safe_iterate (entry
, i
, &e
); i
++)
692 predicate new_exec_pred
;
693 predicate new_nonconst_pred
;
694 new_exec_pred
= e
->exec_predicate
.remap_after_duplication
696 new_nonconst_pred
= e
->nonconst_predicate
.remap_after_duplication
698 if (new_exec_pred
== false || new_nonconst_pred
== false)
699 optimized_out_size
+= e
->size
;
701 account_size_time (info
, e
->size
, e
->time
, &new_exec_pred
,
705 /* Remap edge predicates with the same simplification as above.
706 Also copy constantness arrays. */
707 for (edge
= dst
->callees
; edge
; edge
= next
)
709 predicate new_predicate
;
710 struct ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
711 next
= edge
->next_callee
;
713 if (!edge
->inline_failed
)
717 new_predicate
= es
->predicate
->remap_after_duplication
719 if (new_predicate
== false && *es
->predicate
!= false)
720 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
721 edge_set_predicate (edge
, &new_predicate
);
724 /* Remap indirect edge predicates with the same simplificaiton as above.
725 Also copy constantness arrays. */
726 for (edge
= dst
->indirect_calls
; edge
; edge
= next
)
728 predicate new_predicate
;
729 struct ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
730 next
= edge
->next_callee
;
732 gcc_checking_assert (edge
->inline_failed
);
735 new_predicate
= es
->predicate
->remap_after_duplication
737 if (new_predicate
== false && *es
->predicate
!= false)
738 optimized_out_size
+= es
->call_stmt_size
* INLINE_SIZE_SCALE
;
739 edge_set_predicate (edge
, &new_predicate
);
741 remap_hint_predicate_after_duplication (&info
->loop_iterations
,
743 remap_hint_predicate_after_duplication (&info
->loop_stride
,
745 remap_hint_predicate_after_duplication (&info
->array_index
,
748 /* If inliner or someone after inliner will ever start producing
749 non-trivial clones, we will get trouble with lack of information
750 about updating self sizes, because size vectors already contains
751 sizes of the calees. */
752 gcc_assert (!inlined_to_p
|| !optimized_out_size
);
756 info
->entry
= vec_safe_copy (info
->entry
);
757 if (info
->loop_iterations
)
759 predicate p
= *info
->loop_iterations
;
760 info
->loop_iterations
= NULL
;
761 set_hint_predicate (&info
->loop_iterations
, p
);
763 if (info
->loop_stride
)
765 predicate p
= *info
->loop_stride
;
766 info
->loop_stride
= NULL
;
767 set_hint_predicate (&info
->loop_stride
, p
);
769 if (info
->array_index
)
771 predicate p
= *info
->array_index
;
772 info
->array_index
= NULL
;
773 set_hint_predicate (&info
->array_index
, p
);
776 if (!dst
->global
.inlined_to
)
777 inline_update_overall_summary (dst
);
781 /* Hook that is called by cgraph.c when a node is duplicated. */
784 ipa_call_summary_t::duplicate (struct cgraph_edge
*src
,
785 struct cgraph_edge
*dst
,
786 struct ipa_call_summary
*srcinfo
,
787 struct ipa_call_summary
*info
)
790 info
->predicate
= NULL
;
791 edge_set_predicate (dst
, srcinfo
->predicate
);
792 info
->param
= srcinfo
->param
.copy ();
793 if (!dst
->indirect_unknown_callee
&& src
->indirect_unknown_callee
)
795 info
->call_stmt_size
-= (eni_size_weights
.indirect_call_cost
796 - eni_size_weights
.call_cost
);
797 info
->call_stmt_time
-= (eni_time_weights
.indirect_call_cost
798 - eni_time_weights
.call_cost
);
803 /* Keep edge cache consistent across edge removal. */
806 ipa_call_summary_t::remove (struct cgraph_edge
*edge
,
807 struct ipa_call_summary
*)
809 if (edge_growth_cache
.exists ())
810 reset_edge_growth_cache (edge
);
811 reset_ipa_call_summary (edge
);
815 /* Initialize growth caches. */
818 initialize_growth_caches (void)
820 if (symtab
->edges_max_uid
)
821 edge_growth_cache
.safe_grow_cleared (symtab
->edges_max_uid
);
825 /* Free growth caches. */
828 free_growth_caches (void)
830 edge_growth_cache
.release ();
834 /* Dump edge summaries associated to NODE and recursively to all clones.
838 dump_ipa_call_summary (FILE *f
, int indent
, struct cgraph_node
*node
,
839 struct inline_summary
*info
)
841 struct cgraph_edge
*edge
;
842 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
844 struct ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
845 struct cgraph_node
*callee
= edge
->callee
->ultimate_alias_target ();
849 "%*s%s/%i %s\n%*s loop depth:%2i freq:%4i size:%2i"
850 " time: %2i callee size:%2i stack:%2i",
851 indent
, "", callee
->name (), callee
->order
,
853 ? "inlined" : cgraph_inline_failed_string (edge
-> inline_failed
),
854 indent
, "", es
->loop_depth
, edge
->frequency
,
855 es
->call_stmt_size
, es
->call_stmt_time
,
856 (int) inline_summaries
->get (callee
)->size
/ INLINE_SIZE_SCALE
,
857 (int) inline_summaries
->get (callee
)->estimated_stack_size
);
861 fprintf (f
, " predicate: ");
862 es
->predicate
->dump (f
, info
->conds
);
866 if (es
->param
.exists ())
867 for (i
= 0; i
< (int) es
->param
.length (); i
++)
869 int prob
= es
->param
[i
].change_prob
;
872 fprintf (f
, "%*s op%i is compile time invariant\n",
874 else if (prob
!= REG_BR_PROB_BASE
)
875 fprintf (f
, "%*s op%i change %f%% of time\n", indent
+ 2, "", i
,
876 prob
* 100.0 / REG_BR_PROB_BASE
);
878 if (!edge
->inline_failed
)
880 fprintf (f
, "%*sStack frame offset %i, callee self size %i,"
883 (int) inline_summaries
->get (callee
)->stack_frame_offset
,
884 (int) inline_summaries
->get (callee
)->estimated_self_stack_size
,
885 (int) inline_summaries
->get (callee
)->estimated_stack_size
);
886 dump_ipa_call_summary (f
, indent
+ 2, callee
, info
);
889 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
891 struct ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
892 fprintf (f
, "%*sindirect call loop depth:%2i freq:%4i size:%2i"
896 edge
->frequency
, es
->call_stmt_size
, es
->call_stmt_time
);
899 fprintf (f
, "predicate: ");
900 es
->predicate
->dump (f
, info
->conds
);
909 dump_inline_summary (FILE *f
, struct cgraph_node
*node
)
911 if (node
->definition
)
913 struct inline_summary
*s
= inline_summaries
->get (node
);
916 fprintf (f
, "Inline summary for %s/%i", node
->name (),
918 if (DECL_DISREGARD_INLINE_LIMITS (node
->decl
))
919 fprintf (f
, " always_inline");
921 fprintf (f
, " inlinable");
922 if (s
->contains_cilk_spawn
)
923 fprintf (f
, " contains_cilk_spawn");
924 if (s
->fp_expressions
)
925 fprintf (f
, " fp_expression");
926 fprintf (f
, "\n self time: %f\n", s
->self_time
.to_double ());
927 fprintf (f
, " global time: %f\n", s
->time
.to_double ());
928 fprintf (f
, " self size: %i\n", s
->self_size
);
929 fprintf (f
, " global size: %i\n", s
->size
);
930 fprintf (f
, " min size: %i\n", s
->min_size
);
931 fprintf (f
, " self stack: %i\n",
932 (int) s
->estimated_self_stack_size
);
933 fprintf (f
, " global stack: %i\n", (int) s
->estimated_stack_size
);
935 fprintf (f
, " estimated growth:%i\n", (int) s
->growth
);
937 fprintf (f
, " In SCC: %i\n", (int) s
->scc_no
);
938 for (i
= 0; vec_safe_iterate (s
->entry
, i
, &e
); i
++)
940 fprintf (f
, " size:%f, time:%f",
941 (double) e
->size
/ INLINE_SIZE_SCALE
,
942 e
->time
.to_double ());
943 if (e
->exec_predicate
!= true)
945 fprintf (f
, ", executed if:");
946 e
->exec_predicate
.dump (f
, s
->conds
, 0);
948 if (e
->exec_predicate
!= e
->nonconst_predicate
)
950 fprintf (f
, ", nonconst if:");
951 e
->nonconst_predicate
.dump (f
, s
->conds
, 0);
955 if (s
->loop_iterations
)
957 fprintf (f
, " loop iterations:");
958 s
->loop_iterations
->dump (f
, s
->conds
);
962 fprintf (f
, " loop stride:");
963 s
->loop_stride
->dump (f
, s
->conds
);
967 fprintf (f
, " array index:");
968 s
->array_index
->dump (f
, s
->conds
);
970 fprintf (f
, " calls:\n");
971 dump_ipa_call_summary (f
, 4, node
, s
);
977 debug_inline_summary (struct cgraph_node
*node
)
979 dump_inline_summary (stderr
, node
);
983 dump_inline_summaries (FILE *f
)
985 struct cgraph_node
*node
;
987 FOR_EACH_DEFINED_FUNCTION (node
)
988 if (!node
->global
.inlined_to
)
989 dump_inline_summary (f
, node
);
992 /* Give initial reasons why inlining would fail on EDGE. This gets either
993 nullified or usually overwritten by more precise reasons later. */
996 initialize_inline_failed (struct cgraph_edge
*e
)
998 struct cgraph_node
*callee
= e
->callee
;
1000 if (e
->inline_failed
&& e
->inline_failed
!= CIF_BODY_NOT_AVAILABLE
1001 && cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
1003 else if (e
->indirect_unknown_callee
)
1004 e
->inline_failed
= CIF_INDIRECT_UNKNOWN_CALL
;
1005 else if (!callee
->definition
)
1006 e
->inline_failed
= CIF_BODY_NOT_AVAILABLE
;
1007 else if (callee
->local
.redefined_extern_inline
)
1008 e
->inline_failed
= CIF_REDEFINED_EXTERN_INLINE
;
1010 e
->inline_failed
= CIF_FUNCTION_NOT_CONSIDERED
;
1011 gcc_checking_assert (!e
->call_stmt_cannot_inline_p
1012 || cgraph_inline_failed_type (e
->inline_failed
)
1013 == CIF_FINAL_ERROR
);
1016 /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
1017 boolean variable pointed to by DATA. */
1020 mark_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef ATTRIBUTE_UNUSED
,
1023 bool *b
= (bool *) data
;
1028 /* If OP refers to value of function parameter, return the corresponding
1029 parameter. If non-NULL, the size of the memory load (or the SSA_NAME of the
1030 PARM_DECL) will be stored to *SIZE_P in that case too. */
1033 unmodified_parm_1 (gimple
*stmt
, tree op
, HOST_WIDE_INT
*size_p
)
1035 /* SSA_NAME referring to parm default def? */
1036 if (TREE_CODE (op
) == SSA_NAME
1037 && SSA_NAME_IS_DEFAULT_DEF (op
)
1038 && TREE_CODE (SSA_NAME_VAR (op
)) == PARM_DECL
)
1041 *size_p
= tree_to_shwi (TYPE_SIZE (TREE_TYPE (op
)));
1042 return SSA_NAME_VAR (op
);
1044 /* Non-SSA parm reference? */
1045 if (TREE_CODE (op
) == PARM_DECL
)
1047 bool modified
= false;
1050 ao_ref_init (&refd
, op
);
1051 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), mark_modified
, &modified
,
1056 *size_p
= tree_to_shwi (TYPE_SIZE (TREE_TYPE (op
)));
1063 /* If OP refers to value of function parameter, return the corresponding
1064 parameter. Also traverse chains of SSA register assignments. If non-NULL,
1065 the size of the memory load (or the SSA_NAME of the PARM_DECL) will be
1066 stored to *SIZE_P in that case too. */
1069 unmodified_parm (gimple
*stmt
, tree op
, HOST_WIDE_INT
*size_p
)
1071 tree res
= unmodified_parm_1 (stmt
, op
, size_p
);
1075 if (TREE_CODE (op
) == SSA_NAME
1076 && !SSA_NAME_IS_DEFAULT_DEF (op
)
1077 && gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1078 return unmodified_parm (SSA_NAME_DEF_STMT (op
),
1079 gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op
)),
1084 /* If OP refers to a value of a function parameter or value loaded from an
1085 aggregate passed to a parameter (either by value or reference), return TRUE
1086 and store the number of the parameter to *INDEX_P, the access size into
1087 *SIZE_P, and information whether and how it has been loaded from an
1088 aggregate into *AGGPOS. INFO describes the function parameters, STMT is the
1089 statement in which OP is used or loaded. */
1092 unmodified_parm_or_parm_agg_item (struct ipa_func_body_info
*fbi
,
1093 gimple
*stmt
, tree op
, int *index_p
,
1094 HOST_WIDE_INT
*size_p
,
1095 struct agg_position_info
*aggpos
)
1097 tree res
= unmodified_parm_1 (stmt
, op
, size_p
);
1099 gcc_checking_assert (aggpos
);
1102 *index_p
= ipa_get_param_decl_index (fbi
->info
, res
);
1105 aggpos
->agg_contents
= false;
1106 aggpos
->by_ref
= false;
1110 if (TREE_CODE (op
) == SSA_NAME
)
1112 if (SSA_NAME_IS_DEFAULT_DEF (op
)
1113 || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1115 stmt
= SSA_NAME_DEF_STMT (op
);
1116 op
= gimple_assign_rhs1 (stmt
);
1117 if (!REFERENCE_CLASS_P (op
))
1118 return unmodified_parm_or_parm_agg_item (fbi
, stmt
, op
, index_p
, size_p
,
1122 aggpos
->agg_contents
= true;
1123 return ipa_load_from_parm_agg (fbi
, fbi
->info
->descriptors
,
1124 stmt
, op
, index_p
, &aggpos
->offset
,
1125 size_p
, &aggpos
->by_ref
);
1128 /* See if statement might disappear after inlining.
1129 0 - means not eliminated
1130 1 - half of statements goes away
1131 2 - for sure it is eliminated.
1132 We are not terribly sophisticated, basically looking for simple abstraction
1133 penalty wrappers. */
1136 eliminated_by_inlining_prob (gimple
*stmt
)
1138 enum gimple_code code
= gimple_code (stmt
);
1139 enum tree_code rhs_code
;
1149 if (gimple_num_ops (stmt
) != 2)
1152 rhs_code
= gimple_assign_rhs_code (stmt
);
1154 /* Casts of parameters, loads from parameters passed by reference
1155 and stores to return value or parameters are often free after
1156 inlining dua to SRA and further combining.
1157 Assume that half of statements goes away. */
1158 if (CONVERT_EXPR_CODE_P (rhs_code
)
1159 || rhs_code
== VIEW_CONVERT_EXPR
1160 || rhs_code
== ADDR_EXPR
1161 || gimple_assign_rhs_class (stmt
) == GIMPLE_SINGLE_RHS
)
1163 tree rhs
= gimple_assign_rhs1 (stmt
);
1164 tree lhs
= gimple_assign_lhs (stmt
);
1165 tree inner_rhs
= get_base_address (rhs
);
1166 tree inner_lhs
= get_base_address (lhs
);
1167 bool rhs_free
= false;
1168 bool lhs_free
= false;
1175 /* Reads of parameter are expected to be free. */
1176 if (unmodified_parm (stmt
, inner_rhs
, NULL
))
1178 /* Match expressions of form &this->field. Those will most likely
1179 combine with something upstream after inlining. */
1180 else if (TREE_CODE (inner_rhs
) == ADDR_EXPR
)
1182 tree op
= get_base_address (TREE_OPERAND (inner_rhs
, 0));
1183 if (TREE_CODE (op
) == PARM_DECL
)
1185 else if (TREE_CODE (op
) == MEM_REF
1186 && unmodified_parm (stmt
, TREE_OPERAND (op
, 0), NULL
))
1190 /* When parameter is not SSA register because its address is taken
1191 and it is just copied into one, the statement will be completely
1192 free after inlining (we will copy propagate backward). */
1193 if (rhs_free
&& is_gimple_reg (lhs
))
1196 /* Reads of parameters passed by reference
1197 expected to be free (i.e. optimized out after inlining). */
1198 if (TREE_CODE (inner_rhs
) == MEM_REF
1199 && unmodified_parm (stmt
, TREE_OPERAND (inner_rhs
, 0), NULL
))
1202 /* Copying parameter passed by reference into gimple register is
1203 probably also going to copy propagate, but we can't be quite
1205 if (rhs_free
&& is_gimple_reg (lhs
))
1208 /* Writes to parameters, parameters passed by value and return value
1209 (either dirrectly or passed via invisible reference) are free.
1211 TODO: We ought to handle testcase like
1212 struct a {int a,b;};
1214 retrurnsturct (void)
1220 This translate into:
1235 For that we either need to copy ipa-split logic detecting writes
1237 if (TREE_CODE (inner_lhs
) == PARM_DECL
1238 || TREE_CODE (inner_lhs
) == RESULT_DECL
1239 || (TREE_CODE (inner_lhs
) == MEM_REF
1240 && (unmodified_parm (stmt
, TREE_OPERAND (inner_lhs
, 0), NULL
)
1241 || (TREE_CODE (TREE_OPERAND (inner_lhs
, 0)) == SSA_NAME
1242 && SSA_NAME_VAR (TREE_OPERAND (inner_lhs
, 0))
1243 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1245 0))) == RESULT_DECL
))))
1248 && (is_gimple_reg (rhs
) || is_gimple_min_invariant (rhs
)))
1250 if (lhs_free
&& rhs_free
)
1260 /* If BB ends by a conditional we can turn into predicates, attach corresponding
1261 predicates to the CFG edges. */
1264 set_cond_stmt_execution_predicate (struct ipa_func_body_info
*fbi
,
1265 struct inline_summary
*summary
,
1272 struct agg_position_info aggpos
;
1273 enum tree_code code
, inverted_code
;
1279 last
= last_stmt (bb
);
1280 if (!last
|| gimple_code (last
) != GIMPLE_COND
)
1282 if (!is_gimple_ip_invariant (gimple_cond_rhs (last
)))
1284 op
= gimple_cond_lhs (last
);
1285 /* TODO: handle conditionals like
1288 if (unmodified_parm_or_parm_agg_item (fbi
, last
, op
, &index
, &size
, &aggpos
))
1290 code
= gimple_cond_code (last
);
1291 inverted_code
= invert_tree_comparison (code
, HONOR_NANS (op
));
1293 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1295 enum tree_code this_code
= (e
->flags
& EDGE_TRUE_VALUE
1296 ? code
: inverted_code
);
1297 /* invert_tree_comparison will return ERROR_MARK on FP
1298 comparsions that are not EQ/NE instead of returning proper
1299 unordered one. Be sure it is not confused with NON_CONSTANT. */
1300 if (this_code
!= ERROR_MARK
)
1303 = add_condition (summary
, index
, size
, &aggpos
, this_code
,
1304 unshare_expr_without_location
1305 (gimple_cond_rhs (last
)));
1306 e
->aux
= edge_predicate_pool
.allocate ();
1307 *(predicate
*) e
->aux
= p
;
1312 if (TREE_CODE (op
) != SSA_NAME
)
1315 if (builtin_constant_p (op))
1319 Here we can predicate nonconstant_code. We can't
1320 really handle constant_code since we have no predicate
1321 for this and also the constant code is not known to be
1322 optimized away when inliner doen't see operand is constant.
1323 Other optimizers might think otherwise. */
1324 if (gimple_cond_code (last
) != NE_EXPR
1325 || !integer_zerop (gimple_cond_rhs (last
)))
1327 set_stmt
= SSA_NAME_DEF_STMT (op
);
1328 if (!gimple_call_builtin_p (set_stmt
, BUILT_IN_CONSTANT_P
)
1329 || gimple_call_num_args (set_stmt
) != 1)
1331 op2
= gimple_call_arg (set_stmt
, 0);
1332 if (!unmodified_parm_or_parm_agg_item (fbi
, set_stmt
, op2
, &index
, &size
,
1335 FOR_EACH_EDGE (e
, ei
, bb
->succs
) if (e
->flags
& EDGE_FALSE_VALUE
)
1337 predicate p
= add_condition (summary
, index
, size
, &aggpos
,
1338 predicate::is_not_constant
, NULL_TREE
);
1339 e
->aux
= edge_predicate_pool
.allocate ();
1340 *(predicate
*) e
->aux
= p
;
1345 /* If BB ends by a switch we can turn into predicates, attach corresponding
1346 predicates to the CFG edges. */
1349 set_switch_stmt_execution_predicate (struct ipa_func_body_info
*fbi
,
1350 struct inline_summary
*summary
,
1357 struct agg_position_info aggpos
;
1363 lastg
= last_stmt (bb
);
1364 if (!lastg
|| gimple_code (lastg
) != GIMPLE_SWITCH
)
1366 gswitch
*last
= as_a
<gswitch
*> (lastg
);
1367 op
= gimple_switch_index (last
);
1368 if (!unmodified_parm_or_parm_agg_item (fbi
, last
, op
, &index
, &size
, &aggpos
))
1371 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1373 e
->aux
= edge_predicate_pool
.allocate ();
1374 *(predicate
*) e
->aux
= false;
1376 n
= gimple_switch_num_labels (last
);
1377 for (case_idx
= 0; case_idx
< n
; ++case_idx
)
1379 tree cl
= gimple_switch_label (last
, case_idx
);
1383 e
= find_edge (bb
, label_to_block (CASE_LABEL (cl
)));
1384 min
= CASE_LOW (cl
);
1385 max
= CASE_HIGH (cl
);
1387 /* For default we might want to construct predicate that none
1388 of cases is met, but it is bit hard to do not having negations
1389 of conditionals handy. */
1393 p
= add_condition (summary
, index
, size
, &aggpos
, EQ_EXPR
,
1394 unshare_expr_without_location (min
));
1398 p1
= add_condition (summary
, index
, size
, &aggpos
, GE_EXPR
,
1399 unshare_expr_without_location (min
));
1400 p2
= add_condition (summary
, index
, size
, &aggpos
, LE_EXPR
,
1401 unshare_expr_without_location (max
));
1404 *(struct predicate
*) e
->aux
1405 = p
.or_with (summary
->conds
, *(struct predicate
*) e
->aux
);
1410 /* For each BB in NODE attach to its AUX pointer predicate under
1411 which it is executable. */
1414 compute_bb_predicates (struct ipa_func_body_info
*fbi
,
1415 struct cgraph_node
*node
,
1416 struct inline_summary
*summary
)
1418 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
1422 FOR_EACH_BB_FN (bb
, my_function
)
1424 set_cond_stmt_execution_predicate (fbi
, summary
, bb
);
1425 set_switch_stmt_execution_predicate (fbi
, summary
, bb
);
1428 /* Entry block is always executable. */
1429 ENTRY_BLOCK_PTR_FOR_FN (my_function
)->aux
1430 = edge_predicate_pool
.allocate ();
1431 *(predicate
*) ENTRY_BLOCK_PTR_FOR_FN (my_function
)->aux
= true;
1433 /* A simple dataflow propagation of predicates forward in the CFG.
1434 TODO: work in reverse postorder. */
1438 FOR_EACH_BB_FN (bb
, my_function
)
1440 predicate p
= false;
1443 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1447 predicate this_bb_predicate
1448 = *(predicate
*) e
->src
->aux
;
1450 this_bb_predicate
&= (*(struct predicate
*) e
->aux
);
1451 p
= p
.or_with (summary
->conds
, this_bb_predicate
);
1457 gcc_checking_assert (!bb
->aux
);
1463 bb
->aux
= edge_predicate_pool
.allocate ();
1464 *((predicate
*) bb
->aux
) = p
;
1466 else if (p
!= *(predicate
*) bb
->aux
)
1468 /* This OR operation is needed to ensure monotonous data flow
1469 in the case we hit the limit on number of clauses and the
1470 and/or operations above give approximate answers. */
1471 p
= p
.or_with (summary
->conds
, *(predicate
*)bb
->aux
);
1472 if (p
!= *(predicate
*) bb
->aux
)
1475 *((predicate
*) bb
->aux
) = p
;
1484 /* We keep info about constantness of SSA names. */
1486 typedef predicate predicate_t
;
1487 /* Return predicate specifying when the STMT might have result that is not
1488 a compile time constant. */
1491 will_be_nonconstant_expr_predicate (struct ipa_node_params
*info
,
1492 struct inline_summary
*summary
,
1494 vec
<predicate_t
> nonconstant_names
)
1500 while (UNARY_CLASS_P (expr
))
1501 expr
= TREE_OPERAND (expr
, 0);
1503 parm
= unmodified_parm (NULL
, expr
, &size
);
1504 if (parm
&& (index
= ipa_get_param_decl_index (info
, parm
)) >= 0)
1505 return add_condition (summary
, index
, size
, NULL
, predicate::changed
,
1507 if (is_gimple_min_invariant (expr
))
1509 if (TREE_CODE (expr
) == SSA_NAME
)
1510 return nonconstant_names
[SSA_NAME_VERSION (expr
)];
1511 if (BINARY_CLASS_P (expr
) || COMPARISON_CLASS_P (expr
))
1513 predicate p1
= will_be_nonconstant_expr_predicate
1514 (info
, summary
, TREE_OPERAND (expr
, 0),
1520 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1521 TREE_OPERAND (expr
, 1),
1523 return p1
.or_with (summary
->conds
, p2
);
1525 else if (TREE_CODE (expr
) == COND_EXPR
)
1527 predicate p1
= will_be_nonconstant_expr_predicate
1528 (info
, summary
, TREE_OPERAND (expr
, 0),
1534 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1535 TREE_OPERAND (expr
, 1),
1539 p1
= p1
.or_with (summary
->conds
, p2
);
1540 p2
= will_be_nonconstant_expr_predicate (info
, summary
,
1541 TREE_OPERAND (expr
, 2),
1543 return p2
.or_with (summary
->conds
, p1
);
1554 /* Return predicate specifying when the STMT might have result that is not
1555 a compile time constant. */
1558 will_be_nonconstant_predicate (struct ipa_func_body_info
*fbi
,
1559 struct inline_summary
*summary
,
1561 vec
<predicate_t
> nonconstant_names
)
1566 predicate op_non_const
;
1570 struct agg_position_info aggpos
;
1572 /* What statments might be optimized away
1573 when their arguments are constant. */
1574 if (gimple_code (stmt
) != GIMPLE_ASSIGN
1575 && gimple_code (stmt
) != GIMPLE_COND
1576 && gimple_code (stmt
) != GIMPLE_SWITCH
1577 && (gimple_code (stmt
) != GIMPLE_CALL
1578 || !(gimple_call_flags (stmt
) & ECF_CONST
)))
1581 /* Stores will stay anyway. */
1582 if (gimple_store_p (stmt
))
1585 is_load
= gimple_assign_load_p (stmt
);
1587 /* Loads can be optimized when the value is known. */
1591 gcc_assert (gimple_assign_single_p (stmt
));
1592 op
= gimple_assign_rhs1 (stmt
);
1593 if (!unmodified_parm_or_parm_agg_item (fbi
, stmt
, op
, &base_index
, &size
,
1600 /* See if we understand all operands before we start
1601 adding conditionals. */
1602 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1604 tree parm
= unmodified_parm (stmt
, use
, NULL
);
1605 /* For arguments we can build a condition. */
1606 if (parm
&& ipa_get_param_decl_index (fbi
->info
, parm
) >= 0)
1608 if (TREE_CODE (use
) != SSA_NAME
)
1610 /* If we know when operand is constant,
1611 we still can say something useful. */
1612 if (nonconstant_names
[SSA_NAME_VERSION (use
)] != true)
1619 add_condition (summary
, base_index
, size
, &aggpos
, predicate::changed
,
1622 op_non_const
= false;
1623 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1626 tree parm
= unmodified_parm (stmt
, use
, &size
);
1629 if (parm
&& (index
= ipa_get_param_decl_index (fbi
->info
, parm
)) >= 0)
1631 if (index
!= base_index
)
1632 p
= add_condition (summary
, index
, size
, NULL
, predicate::changed
,
1638 p
= nonconstant_names
[SSA_NAME_VERSION (use
)];
1639 op_non_const
= p
.or_with (summary
->conds
, op_non_const
);
1641 if ((gimple_code (stmt
) == GIMPLE_ASSIGN
|| gimple_code (stmt
) == GIMPLE_CALL
)
1642 && gimple_op (stmt
, 0)
1643 && TREE_CODE (gimple_op (stmt
, 0)) == SSA_NAME
)
1644 nonconstant_names
[SSA_NAME_VERSION (gimple_op (stmt
, 0))]
1646 return op_non_const
;
1649 struct record_modified_bb_info
1655 /* Value is initialized in INIT_BB and used in USE_BB. We want to copute
1656 probability how often it changes between USE_BB.
1657 INIT_BB->frequency/USE_BB->frequency is an estimate, but if INIT_BB
1658 is in different loop nest, we can do better.
1659 This is all just estimate. In theory we look for minimal cut separating
1660 INIT_BB and USE_BB, but we only want to anticipate loop invariant motion
1664 get_minimal_bb (basic_block init_bb
, basic_block use_bb
)
1666 struct loop
*l
= find_common_loop (init_bb
->loop_father
, use_bb
->loop_father
);
1667 if (l
&& l
->header
->frequency
< init_bb
->frequency
)
1672 /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
1673 set except for info->stmt. */
1676 record_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef
, void *data
)
1678 struct record_modified_bb_info
*info
=
1679 (struct record_modified_bb_info
*) data
;
1680 if (SSA_NAME_DEF_STMT (vdef
) == info
->stmt
)
1682 bitmap_set_bit (info
->bb_set
,
1683 SSA_NAME_IS_DEFAULT_DEF (vdef
)
1684 ? ENTRY_BLOCK_PTR_FOR_FN (cfun
)->index
1686 (gimple_bb (SSA_NAME_DEF_STMT (vdef
)),
1687 gimple_bb (info
->stmt
))->index
);
1691 /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
1692 will change since last invocation of STMT.
1694 Value 0 is reserved for compile time invariants.
1695 For common parameters it is REG_BR_PROB_BASE. For loop invariants it
1696 ought to be REG_BR_PROB_BASE / estimated_iters. */
1699 param_change_prob (gimple
*stmt
, int i
)
1701 tree op
= gimple_call_arg (stmt
, i
);
1702 basic_block bb
= gimple_bb (stmt
);
1704 if (TREE_CODE (op
) == WITH_SIZE_EXPR
)
1705 op
= TREE_OPERAND (op
, 0);
1707 tree base
= get_base_address (op
);
1709 /* Global invariants never change. */
1710 if (is_gimple_min_invariant (base
))
1713 /* We would have to do non-trivial analysis to really work out what
1714 is the probability of value to change (i.e. when init statement
1715 is in a sibling loop of the call).
1717 We do an conservative estimate: when call is executed N times more often
1718 than the statement defining value, we take the frequency 1/N. */
1719 if (TREE_CODE (base
) == SSA_NAME
)
1724 return REG_BR_PROB_BASE
;
1726 if (SSA_NAME_IS_DEFAULT_DEF (base
))
1727 init_freq
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->frequency
;
1729 init_freq
= get_minimal_bb
1730 (gimple_bb (SSA_NAME_DEF_STMT (base
)),
1731 gimple_bb (stmt
))->frequency
;
1735 if (init_freq
< bb
->frequency
)
1736 return MAX (GCOV_COMPUTE_SCALE (init_freq
, bb
->frequency
), 1);
1738 return REG_BR_PROB_BASE
;
1744 struct record_modified_bb_info info
;
1747 tree init
= ctor_for_folding (base
);
1749 if (init
!= error_mark_node
)
1752 return REG_BR_PROB_BASE
;
1753 ao_ref_init (&refd
, op
);
1755 info
.bb_set
= BITMAP_ALLOC (NULL
);
1756 walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), record_modified
, &info
,
1758 if (bitmap_bit_p (info
.bb_set
, bb
->index
))
1760 BITMAP_FREE (info
.bb_set
);
1761 return REG_BR_PROB_BASE
;
1764 /* Assume that every memory is initialized at entry.
1765 TODO: Can we easilly determine if value is always defined
1766 and thus we may skip entry block? */
1767 if (ENTRY_BLOCK_PTR_FOR_FN (cfun
)->frequency
)
1768 max
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->frequency
;
1772 EXECUTE_IF_SET_IN_BITMAP (info
.bb_set
, 0, index
, bi
)
1773 max
= MIN (max
, BASIC_BLOCK_FOR_FN (cfun
, index
)->frequency
);
1775 BITMAP_FREE (info
.bb_set
);
1776 if (max
< bb
->frequency
)
1777 return MAX (GCOV_COMPUTE_SCALE (max
, bb
->frequency
), 1);
1779 return REG_BR_PROB_BASE
;
1783 /* Find whether a basic block BB is the final block of a (half) diamond CFG
1784 sub-graph and if the predicate the condition depends on is known. If so,
1785 return true and store the pointer the predicate in *P. */
1788 phi_result_unknown_predicate (struct ipa_node_params
*info
,
1789 inline_summary
*summary
, basic_block bb
,
1791 vec
<predicate_t
> nonconstant_names
)
1795 basic_block first_bb
= NULL
;
1798 if (single_pred_p (bb
))
1804 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1806 if (single_succ_p (e
->src
))
1808 if (!single_pred_p (e
->src
))
1811 first_bb
= single_pred (e
->src
);
1812 else if (single_pred (e
->src
) != first_bb
)
1819 else if (e
->src
!= first_bb
)
1827 stmt
= last_stmt (first_bb
);
1829 || gimple_code (stmt
) != GIMPLE_COND
1830 || !is_gimple_ip_invariant (gimple_cond_rhs (stmt
)))
1833 *p
= will_be_nonconstant_expr_predicate (info
, summary
,
1834 gimple_cond_lhs (stmt
),
1842 /* Given a PHI statement in a function described by inline properties SUMMARY
1843 and *P being the predicate describing whether the selected PHI argument is
1844 known, store a predicate for the result of the PHI statement into
1845 NONCONSTANT_NAMES, if possible. */
1848 predicate_for_phi_result (struct inline_summary
*summary
, gphi
*phi
,
1850 vec
<predicate_t
> nonconstant_names
)
1854 for (i
= 0; i
< gimple_phi_num_args (phi
); i
++)
1856 tree arg
= gimple_phi_arg (phi
, i
)->def
;
1857 if (!is_gimple_min_invariant (arg
))
1859 gcc_assert (TREE_CODE (arg
) == SSA_NAME
);
1860 *p
= p
->or_with (summary
->conds
,
1861 nonconstant_names
[SSA_NAME_VERSION (arg
)]);
1867 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1869 fprintf (dump_file
, "\t\tphi predicate: ");
1870 p
->dump (dump_file
, summary
->conds
);
1872 nonconstant_names
[SSA_NAME_VERSION (gimple_phi_result (phi
))] = *p
;
1875 /* Return predicate specifying when array index in access OP becomes non-constant. */
1878 array_index_predicate (inline_summary
*info
,
1879 vec
< predicate_t
> nonconstant_names
, tree op
)
1881 predicate p
= false;
1882 while (handled_component_p (op
))
1884 if (TREE_CODE (op
) == ARRAY_REF
|| TREE_CODE (op
) == ARRAY_RANGE_REF
)
1886 if (TREE_CODE (TREE_OPERAND (op
, 1)) == SSA_NAME
)
1887 p
= p
.or_with (info
->conds
,
1888 nonconstant_names
[SSA_NAME_VERSION
1889 (TREE_OPERAND (op
, 1))]);
1891 op
= TREE_OPERAND (op
, 0);
1896 /* For a typical usage of __builtin_expect (a<b, 1), we
1897 may introduce an extra relation stmt:
1898 With the builtin, we have
1901 t3 = __builtin_expect (t2, 1);
1904 Without the builtin, we have
1907 This affects the size/time estimation and may have
1908 an impact on the earlier inlining.
1909 Here find this pattern and fix it up later. */
1912 find_foldable_builtin_expect (basic_block bb
)
1914 gimple_stmt_iterator bsi
;
1916 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
1918 gimple
*stmt
= gsi_stmt (bsi
);
1919 if (gimple_call_builtin_p (stmt
, BUILT_IN_EXPECT
)
1920 || gimple_call_internal_p (stmt
, IFN_BUILTIN_EXPECT
))
1922 tree var
= gimple_call_lhs (stmt
);
1923 tree arg
= gimple_call_arg (stmt
, 0);
1924 use_operand_p use_p
;
1931 gcc_assert (TREE_CODE (var
) == SSA_NAME
);
1933 while (TREE_CODE (arg
) == SSA_NAME
)
1935 gimple
*stmt_tmp
= SSA_NAME_DEF_STMT (arg
);
1936 if (!is_gimple_assign (stmt_tmp
))
1938 switch (gimple_assign_rhs_code (stmt_tmp
))
1957 arg
= gimple_assign_rhs1 (stmt_tmp
);
1960 if (match
&& single_imm_use (var
, &use_p
, &use_stmt
)
1961 && gimple_code (use_stmt
) == GIMPLE_COND
)
1968 /* Return true when the basic blocks contains only clobbers followed by RESX.
1969 Such BBs are kept around to make removal of dead stores possible with
1970 presence of EH and will be optimized out by optimize_clobbers later in the
1973 NEED_EH is used to recurse in case the clobber has non-EH predecestors
1974 that can be clobber only, too.. When it is false, the RESX is not necessary
1975 on the end of basic block. */
1978 clobber_only_eh_bb_p (basic_block bb
, bool need_eh
= true)
1980 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
1986 if (gsi_end_p (gsi
))
1988 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_RESX
)
1992 else if (!single_succ_p (bb
))
1995 for (; !gsi_end_p (gsi
); gsi_prev (&gsi
))
1997 gimple
*stmt
= gsi_stmt (gsi
);
1998 if (is_gimple_debug (stmt
))
2000 if (gimple_clobber_p (stmt
))
2002 if (gimple_code (stmt
) == GIMPLE_LABEL
)
2007 /* See if all predecestors are either throws or clobber only BBs. */
2008 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
2009 if (!(e
->flags
& EDGE_EH
)
2010 && !clobber_only_eh_bb_p (e
->src
, false))
2016 /* Return true if STMT compute a floating point expression that may be affected
2017 by -ffast-math and similar flags. */
2020 fp_expression_p (gimple
*stmt
)
2025 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_DEF
|SSA_OP_USE
)
2026 if (FLOAT_TYPE_P (TREE_TYPE (op
)))
2031 /* Compute function body size parameters for NODE.
2032 When EARLY is true, we compute only simple summaries without
2033 non-trivial predicates to drive the early inliner. */
2036 estimate_function_body_sizes (struct cgraph_node
*node
, bool early
)
2039 /* Estimate static overhead for function prologue/epilogue and alignment. */
2041 /* Benefits are scaled by probability of elimination that is in range
2044 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
2046 struct inline_summary
*info
= inline_summaries
->get (node
);
2047 predicate bb_predicate
;
2048 struct ipa_func_body_info fbi
;
2049 vec
<predicate_t
> nonconstant_names
= vNULL
;
2052 predicate array_index
= true;
2053 gimple
*fix_builtin_expect_stmt
;
2055 gcc_assert (my_function
&& my_function
->cfg
);
2056 gcc_assert (cfun
== my_function
);
2058 memset(&fbi
, 0, sizeof(fbi
));
2062 /* When optimizing and analyzing for IPA inliner, initialize loop optimizer
2063 so we can produce proper inline hints.
2065 When optimizing and analyzing for early inliner, initialize node params
2066 so we can produce correct BB predicates. */
2068 if (opt_for_fn (node
->decl
, optimize
))
2070 calculate_dominance_info (CDI_DOMINATORS
);
2072 loop_optimizer_init (LOOPS_NORMAL
| LOOPS_HAVE_RECORDED_EXITS
);
2075 ipa_check_create_node_params ();
2076 ipa_initialize_node_params (node
);
2079 if (ipa_node_params_sum
)
2082 fbi
.info
= IPA_NODE_REF (node
);
2083 fbi
.bb_infos
= vNULL
;
2084 fbi
.bb_infos
.safe_grow_cleared (last_basic_block_for_fn (cfun
));
2085 fbi
.param_count
= count_formal_params(node
->decl
);
2086 nonconstant_names
.safe_grow_cleared
2087 (SSANAMES (my_function
)->length ());
2092 fprintf (dump_file
, "\nAnalyzing function body size: %s\n",
2095 /* When we run into maximal number of entries, we assign everything to the
2096 constant truth case. Be sure to have it in list. */
2097 bb_predicate
= true;
2098 account_size_time (info
, 0, 0, &bb_predicate
, &bb_predicate
);
2100 bb_predicate
= predicate::not_inlined ();
2101 account_size_time (info
, 2 * INLINE_SIZE_SCALE
, 0, &bb_predicate
,
2105 compute_bb_predicates (&fbi
, node
, info
);
2106 order
= XNEWVEC (int, n_basic_blocks_for_fn (cfun
));
2107 nblocks
= pre_and_rev_post_order_compute (NULL
, order
, false);
2108 for (n
= 0; n
< nblocks
; n
++)
2110 bb
= BASIC_BLOCK_FOR_FN (cfun
, order
[n
]);
2111 freq
= compute_call_stmt_bb_frequency (node
->decl
, bb
);
2112 if (clobber_only_eh_bb_p (bb
))
2114 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2115 fprintf (dump_file
, "\n Ignoring BB %i;"
2116 " it will be optimized away by cleanup_clobbers\n",
2121 /* TODO: Obviously predicates can be propagated down across CFG. */
2125 bb_predicate
= *(predicate
*) bb
->aux
;
2127 bb_predicate
= false;
2130 bb_predicate
= true;
2132 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2134 fprintf (dump_file
, "\n BB %i predicate:", bb
->index
);
2135 bb_predicate
.dump (dump_file
, info
->conds
);
2138 if (fbi
.info
&& nonconstant_names
.exists ())
2140 predicate phi_predicate
;
2141 bool first_phi
= true;
2143 for (gphi_iterator bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
);
2147 && !phi_result_unknown_predicate (fbi
.info
, info
, bb
,
2152 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2154 fprintf (dump_file
, " ");
2155 print_gimple_stmt (dump_file
, gsi_stmt (bsi
), 0);
2157 predicate_for_phi_result (info
, bsi
.phi (), &phi_predicate
,
2162 fix_builtin_expect_stmt
= find_foldable_builtin_expect (bb
);
2164 for (gimple_stmt_iterator bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
);
2167 gimple
*stmt
= gsi_stmt (bsi
);
2168 int this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
2169 int this_time
= estimate_num_insns (stmt
, &eni_time_weights
);
2171 predicate will_be_nonconstant
;
2173 /* This relation stmt should be folded after we remove
2174 buildin_expect call. Adjust the cost here. */
2175 if (stmt
== fix_builtin_expect_stmt
)
2181 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2183 fprintf (dump_file
, " ");
2184 print_gimple_stmt (dump_file
, stmt
, 0);
2185 fprintf (dump_file
, "\t\tfreq:%3.2f size:%3i time:%3i\n",
2186 ((double) freq
) / CGRAPH_FREQ_BASE
, this_size
,
2190 if (gimple_assign_load_p (stmt
) && nonconstant_names
.exists ())
2192 predicate this_array_index
;
2194 array_index_predicate (info
, nonconstant_names
,
2195 gimple_assign_rhs1 (stmt
));
2196 if (this_array_index
!= false)
2197 array_index
&= this_array_index
;
2199 if (gimple_store_p (stmt
) && nonconstant_names
.exists ())
2201 predicate this_array_index
;
2203 array_index_predicate (info
, nonconstant_names
,
2204 gimple_get_lhs (stmt
));
2205 if (this_array_index
!= false)
2206 array_index
&= this_array_index
;
2210 if (is_gimple_call (stmt
)
2211 && !gimple_call_internal_p (stmt
))
2213 struct cgraph_edge
*edge
= node
->get_edge (stmt
);
2214 struct ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
2216 /* Special case: results of BUILT_IN_CONSTANT_P will be always
2217 resolved as constant. We however don't want to optimize
2218 out the cgraph edges. */
2219 if (nonconstant_names
.exists ()
2220 && gimple_call_builtin_p (stmt
, BUILT_IN_CONSTANT_P
)
2221 && gimple_call_lhs (stmt
)
2222 && TREE_CODE (gimple_call_lhs (stmt
)) == SSA_NAME
)
2224 predicate false_p
= false;
2225 nonconstant_names
[SSA_NAME_VERSION (gimple_call_lhs (stmt
))]
2228 if (ipa_node_params_sum
)
2230 int count
= gimple_call_num_args (stmt
);
2234 es
->param
.safe_grow_cleared (count
);
2235 for (i
= 0; i
< count
; i
++)
2237 int prob
= param_change_prob (stmt
, i
);
2238 gcc_assert (prob
>= 0 && prob
<= REG_BR_PROB_BASE
);
2239 es
->param
[i
].change_prob
= prob
;
2243 es
->call_stmt_size
= this_size
;
2244 es
->call_stmt_time
= this_time
;
2245 es
->loop_depth
= bb_loop_depth (bb
);
2246 edge_set_predicate (edge
, &bb_predicate
);
2249 /* TODO: When conditional jump or swithc is known to be constant, but
2250 we did not translate it into the predicates, we really can account
2251 just maximum of the possible paths. */
2254 = will_be_nonconstant_predicate (&fbi
, info
,
2255 stmt
, nonconstant_names
);
2257 will_be_nonconstant
= true;
2258 if (this_time
|| this_size
)
2262 prob
= eliminated_by_inlining_prob (stmt
);
2263 if (prob
== 1 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2265 "\t\t50%% will be eliminated by inlining\n");
2266 if (prob
== 2 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2267 fprintf (dump_file
, "\t\tWill be eliminated by inlining\n");
2269 struct predicate p
= bb_predicate
& will_be_nonconstant
;
2271 /* We can ignore statement when we proved it is never going
2272 to happen, but we can not do that for call statements
2273 because edges are accounted specially. */
2275 if (*(is_gimple_call (stmt
) ? &bb_predicate
: &p
) != false)
2281 /* We account everything but the calls. Calls have their own
2282 size/time info attached to cgraph edges. This is necessary
2283 in order to make the cost disappear after inlining. */
2284 if (!is_gimple_call (stmt
))
2288 predicate ip
= bb_predicate
& predicate::not_inlined ();
2289 account_size_time (info
, this_size
* prob
,
2290 (sreal
)(this_time
* prob
)
2291 / (CGRAPH_FREQ_BASE
* 2), &ip
,
2295 account_size_time (info
, this_size
* (2 - prob
),
2296 (sreal
)(this_time
* (2 - prob
))
2297 / (CGRAPH_FREQ_BASE
* 2),
2302 if (!info
->fp_expressions
&& fp_expression_p (stmt
))
2304 info
->fp_expressions
= true;
2306 fprintf (dump_file
, " fp_expression set\n");
2309 gcc_assert (time
>= 0);
2310 gcc_assert (size
>= 0);
2314 set_hint_predicate (&inline_summaries
->get (node
)->array_index
, array_index
);
2315 time
= time
/ CGRAPH_FREQ_BASE
;
2318 if (nonconstant_names
.exists () && !early
)
2321 predicate loop_iterations
= true;
2322 predicate loop_stride
= true;
2324 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2325 flow_loops_dump (dump_file
, NULL
, 0);
2327 FOR_EACH_LOOP (loop
, 0)
2332 struct tree_niter_desc niter_desc
;
2333 bb_predicate
= *(predicate
*) loop
->header
->aux
;
2335 exits
= get_loop_exit_edges (loop
);
2336 FOR_EACH_VEC_ELT (exits
, j
, ex
)
2337 if (number_of_iterations_exit (loop
, ex
, &niter_desc
, false)
2338 && !is_gimple_min_invariant (niter_desc
.niter
))
2340 predicate will_be_nonconstant
2341 = will_be_nonconstant_expr_predicate (fbi
.info
, info
,
2344 if (will_be_nonconstant
!= true)
2345 will_be_nonconstant
= bb_predicate
& will_be_nonconstant
;
2346 if (will_be_nonconstant
!= true
2347 && will_be_nonconstant
!= false)
2348 /* This is slightly inprecise. We may want to represent each
2349 loop with independent predicate. */
2350 loop_iterations
&= will_be_nonconstant
;
2355 /* To avoid quadratic behavior we analyze stride predicates only
2356 with respect to the containing loop. Thus we simply iterate
2357 over all defs in the outermost loop body. */
2358 for (loop
= loops_for_fn (cfun
)->tree_root
->inner
;
2359 loop
!= NULL
; loop
= loop
->next
)
2361 basic_block
*body
= get_loop_body (loop
);
2362 for (unsigned i
= 0; i
< loop
->num_nodes
; i
++)
2364 gimple_stmt_iterator gsi
;
2365 bb_predicate
= *(predicate
*) body
[i
]->aux
;
2366 for (gsi
= gsi_start_bb (body
[i
]); !gsi_end_p (gsi
);
2369 gimple
*stmt
= gsi_stmt (gsi
);
2371 if (!is_gimple_assign (stmt
))
2374 tree def
= gimple_assign_lhs (stmt
);
2375 if (TREE_CODE (def
) != SSA_NAME
)
2379 if (!simple_iv (loop_containing_stmt (stmt
),
2380 loop_containing_stmt (stmt
),
2382 || is_gimple_min_invariant (iv
.step
))
2385 predicate will_be_nonconstant
2386 = will_be_nonconstant_expr_predicate (fbi
.info
, info
,
2389 if (will_be_nonconstant
!= true)
2390 will_be_nonconstant
= bb_predicate
& will_be_nonconstant
;
2391 if (will_be_nonconstant
!= true
2392 && will_be_nonconstant
!= false)
2393 /* This is slightly inprecise. We may want to represent
2394 each loop with independent predicate. */
2395 loop_stride
= loop_stride
& will_be_nonconstant
;
2400 set_hint_predicate (&inline_summaries
->get (node
)->loop_iterations
,
2402 set_hint_predicate (&inline_summaries
->get (node
)->loop_stride
,
2406 FOR_ALL_BB_FN (bb
, my_function
)
2412 edge_predicate_pool
.remove ((predicate
*)bb
->aux
);
2414 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2417 edge_predicate_pool
.remove ((predicate
*) e
->aux
);
2421 inline_summaries
->get (node
)->self_time
= time
;
2422 inline_summaries
->get (node
)->self_size
= size
;
2423 nonconstant_names
.release ();
2424 ipa_release_body_info (&fbi
);
2425 if (opt_for_fn (node
->decl
, optimize
))
2428 loop_optimizer_finalize ();
2429 else if (!ipa_edge_args_sum
)
2430 ipa_free_all_node_params ();
2431 free_dominance_info (CDI_DOMINATORS
);
2435 fprintf (dump_file
, "\n");
2436 dump_inline_summary (dump_file
, node
);
2441 /* Compute parameters of functions used by inliner.
2442 EARLY is true when we compute parameters for the early inliner */
2445 compute_inline_parameters (struct cgraph_node
*node
, bool early
)
2447 HOST_WIDE_INT self_stack_size
;
2448 struct cgraph_edge
*e
;
2449 struct inline_summary
*info
;
2451 gcc_assert (!node
->global
.inlined_to
);
2453 inline_summary_alloc ();
2455 info
= inline_summaries
->get (node
);
2456 reset_inline_summary (node
, info
);
2458 /* Estimate the stack size for the function if we're optimizing. */
2459 self_stack_size
= optimize
&& !node
->thunk
.thunk_p
2460 ? estimated_stack_frame_size (node
) : 0;
2461 info
->estimated_self_stack_size
= self_stack_size
;
2462 info
->estimated_stack_size
= self_stack_size
;
2463 info
->stack_frame_offset
= 0;
2465 if (node
->thunk
.thunk_p
)
2467 struct ipa_call_summary
*es
= ipa_call_summaries
->get (node
->callees
);
2470 node
->local
.can_change_signature
= false;
2471 es
->call_stmt_size
= eni_size_weights
.call_cost
;
2472 es
->call_stmt_time
= eni_time_weights
.call_cost
;
2473 account_size_time (info
, INLINE_SIZE_SCALE
* 2,
2475 t
= predicate::not_inlined ();
2476 account_size_time (info
, 2 * INLINE_SIZE_SCALE
, 0, &t
, &t
);
2477 inline_update_overall_summary (node
);
2478 info
->self_size
= info
->size
;
2479 info
->self_time
= info
->time
;
2480 /* We can not inline instrumentation clones. */
2481 if (node
->thunk
.add_pointer_bounds_args
)
2483 info
->inlinable
= false;
2484 node
->callees
->inline_failed
= CIF_CHKP
;
2487 info
->inlinable
= true;
2491 /* Even is_gimple_min_invariant rely on current_function_decl. */
2492 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
2494 /* Can this function be inlined at all? */
2495 if (!opt_for_fn (node
->decl
, optimize
)
2496 && !lookup_attribute ("always_inline",
2497 DECL_ATTRIBUTES (node
->decl
)))
2498 info
->inlinable
= false;
2500 info
->inlinable
= tree_inlinable_function_p (node
->decl
);
2502 info
->contains_cilk_spawn
= fn_contains_cilk_spawn_p (cfun
);
2504 /* Type attributes can use parameter indices to describe them. */
2505 if (TYPE_ATTRIBUTES (TREE_TYPE (node
->decl
)))
2506 node
->local
.can_change_signature
= false;
2509 /* Otherwise, inlinable functions always can change signature. */
2510 if (info
->inlinable
)
2511 node
->local
.can_change_signature
= true;
2514 /* Functions calling builtin_apply can not change signature. */
2515 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2517 tree
cdecl = e
->callee
->decl
;
2518 if (DECL_BUILT_IN (cdecl)
2519 && DECL_BUILT_IN_CLASS (cdecl) == BUILT_IN_NORMAL
2520 && (DECL_FUNCTION_CODE (cdecl) == BUILT_IN_APPLY_ARGS
2521 || DECL_FUNCTION_CODE (cdecl) == BUILT_IN_VA_START
))
2524 node
->local
.can_change_signature
= !e
;
2527 /* Functions called by instrumentation thunk can't change signature
2528 because instrumentation thunk modification is not supported. */
2529 if (node
->local
.can_change_signature
)
2530 for (e
= node
->callers
; e
; e
= e
->next_caller
)
2531 if (e
->caller
->thunk
.thunk_p
2532 && e
->caller
->thunk
.add_pointer_bounds_args
)
2534 node
->local
.can_change_signature
= false;
2537 estimate_function_body_sizes (node
, early
);
2540 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2541 if (e
->callee
->comdat_local_p ())
2543 node
->calls_comdat_local
= (e
!= NULL
);
2545 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
2546 info
->time
= info
->self_time
;
2547 info
->size
= info
->self_size
;
2548 info
->stack_frame_offset
= 0;
2549 info
->estimated_stack_size
= info
->estimated_self_stack_size
;
2551 /* Code above should compute exactly the same result as
2552 inline_update_overall_summary but because computation happens in
2553 different order the roundoff errors result in slight changes. */
2554 inline_update_overall_summary (node
);
2555 gcc_assert (!(info
->time
- info
->self_time
).to_int ()
2556 && info
->size
== info
->self_size
);
2560 /* Compute parameters of functions used by inliner using
2561 current_function_decl. */
2564 compute_inline_parameters_for_current (void)
2566 compute_inline_parameters (cgraph_node::get (current_function_decl
), true);
2572 const pass_data pass_data_inline_parameters
=
2574 GIMPLE_PASS
, /* type */
2575 "inline_param", /* name */
2576 OPTGROUP_INLINE
, /* optinfo_flags */
2577 TV_INLINE_PARAMETERS
, /* tv_id */
2578 0, /* properties_required */
2579 0, /* properties_provided */
2580 0, /* properties_destroyed */
2581 0, /* todo_flags_start */
2582 0, /* todo_flags_finish */
2585 class pass_inline_parameters
: public gimple_opt_pass
2588 pass_inline_parameters (gcc::context
*ctxt
)
2589 : gimple_opt_pass (pass_data_inline_parameters
, ctxt
)
2592 /* opt_pass methods: */
2593 opt_pass
* clone () { return new pass_inline_parameters (m_ctxt
); }
2594 virtual unsigned int execute (function
*)
2596 return compute_inline_parameters_for_current ();
2599 }; // class pass_inline_parameters
2604 make_pass_inline_parameters (gcc::context
*ctxt
)
2606 return new pass_inline_parameters (ctxt
);
2610 /* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS,
2611 KNOWN_CONTEXTS and KNOWN_AGGS. */
2614 estimate_edge_devirt_benefit (struct cgraph_edge
*ie
,
2615 int *size
, int *time
,
2616 vec
<tree
> known_vals
,
2617 vec
<ipa_polymorphic_call_context
> known_contexts
,
2618 vec
<ipa_agg_jump_function_p
> known_aggs
)
2621 struct cgraph_node
*callee
;
2622 struct inline_summary
*isummary
;
2623 enum availability avail
;
2626 if (!known_vals
.exists () && !known_contexts
.exists ())
2628 if (!opt_for_fn (ie
->caller
->decl
, flag_indirect_inlining
))
2631 target
= ipa_get_indirect_edge_target (ie
, known_vals
, known_contexts
,
2632 known_aggs
, &speculative
);
2633 if (!target
|| speculative
)
2636 /* Account for difference in cost between indirect and direct calls. */
2637 *size
-= (eni_size_weights
.indirect_call_cost
- eni_size_weights
.call_cost
);
2638 *time
-= (eni_time_weights
.indirect_call_cost
- eni_time_weights
.call_cost
);
2639 gcc_checking_assert (*time
>= 0);
2640 gcc_checking_assert (*size
>= 0);
2642 callee
= cgraph_node::get (target
);
2643 if (!callee
|| !callee
->definition
)
2645 callee
= callee
->function_symbol (&avail
);
2646 if (avail
< AVAIL_AVAILABLE
)
2648 isummary
= inline_summaries
->get (callee
);
2649 return isummary
->inlinable
;
2652 /* Increase SIZE, MIN_SIZE (if non-NULL) and TIME for size and time needed to
2653 handle edge E with probability PROB.
2654 Set HINTS if edge may be devirtualized.
2655 KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS describe context of the call
2659 estimate_edge_size_and_time (struct cgraph_edge
*e
, int *size
, int *min_size
,
2662 vec
<tree
> known_vals
,
2663 vec
<ipa_polymorphic_call_context
> known_contexts
,
2664 vec
<ipa_agg_jump_function_p
> known_aggs
,
2665 inline_hints
*hints
)
2667 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
2668 int call_size
= es
->call_stmt_size
;
2669 int call_time
= es
->call_stmt_time
;
2672 && estimate_edge_devirt_benefit (e
, &call_size
, &call_time
,
2673 known_vals
, known_contexts
, known_aggs
)
2674 && hints
&& e
->maybe_hot_p ())
2675 *hints
|= INLINE_HINT_indirect_call
;
2676 cur_size
= call_size
* INLINE_SIZE_SCALE
;
2679 *min_size
+= cur_size
;
2680 if (prob
== REG_BR_PROB_BASE
)
2681 *time
+= ((sreal
)(call_time
* e
->frequency
)) / CGRAPH_FREQ_BASE
;
2683 *time
+= ((sreal
)call_time
) * (prob
* e
->frequency
)
2684 / (CGRAPH_FREQ_BASE
* REG_BR_PROB_BASE
);
2689 /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
2690 calls in NODE. POSSIBLE_TRUTHS, KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS
2691 describe context of the call site. */
2694 estimate_calls_size_and_time (struct cgraph_node
*node
, int *size
,
2695 int *min_size
, sreal
*time
,
2696 inline_hints
*hints
,
2697 clause_t possible_truths
,
2698 vec
<tree
> known_vals
,
2699 vec
<ipa_polymorphic_call_context
> known_contexts
,
2700 vec
<ipa_agg_jump_function_p
> known_aggs
)
2702 struct cgraph_edge
*e
;
2703 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2705 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
2707 /* Do not care about zero sized builtins. */
2708 if (e
->inline_failed
&& !es
->call_stmt_size
)
2710 gcc_checking_assert (!es
->call_stmt_time
);
2714 || es
->predicate
->evaluate (possible_truths
))
2716 if (e
->inline_failed
)
2718 /* Predicates of calls shall not use NOT_CHANGED codes,
2719 sowe do not need to compute probabilities. */
2720 estimate_edge_size_and_time (e
, size
,
2721 es
->predicate
? NULL
: min_size
,
2722 time
, REG_BR_PROB_BASE
,
2723 known_vals
, known_contexts
,
2727 estimate_calls_size_and_time (e
->callee
, size
, min_size
, time
,
2730 known_vals
, known_contexts
,
2734 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2736 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
2738 || es
->predicate
->evaluate (possible_truths
))
2739 estimate_edge_size_and_time (e
, size
,
2740 es
->predicate
? NULL
: min_size
,
2741 time
, REG_BR_PROB_BASE
,
2742 known_vals
, known_contexts
, known_aggs
,
2748 /* Estimate size and time needed to execute NODE assuming
2749 POSSIBLE_TRUTHS clause, and KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS
2750 information about NODE's arguments. If non-NULL use also probability
2751 information present in INLINE_PARAM_SUMMARY vector.
2752 Additionally detemine hints determined by the context. Finally compute
2753 minimal size needed for the call that is independent on the call context and
2754 can be used for fast estimates. Return the values in RET_SIZE,
2755 RET_MIN_SIZE, RET_TIME and RET_HINTS. */
2758 estimate_node_size_and_time (struct cgraph_node
*node
,
2759 clause_t possible_truths
,
2760 clause_t nonspec_possible_truths
,
2761 vec
<tree
> known_vals
,
2762 vec
<ipa_polymorphic_call_context
> known_contexts
,
2763 vec
<ipa_agg_jump_function_p
> known_aggs
,
2764 int *ret_size
, int *ret_min_size
,
2766 sreal
*ret_nonspecialized_time
,
2767 inline_hints
*ret_hints
,
2768 vec
<inline_param_summary
>
2769 inline_param_summary
)
2771 struct inline_summary
*info
= inline_summaries
->get (node
);
2776 inline_hints hints
= 0;
2779 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2782 fprintf (dump_file
, " Estimating body: %s/%i\n"
2783 " Known to be false: ", node
->name (),
2786 for (i
= predicate::not_inlined_condition
;
2787 i
< (predicate::first_dynamic_condition
2788 + (int) vec_safe_length (info
->conds
)); i
++)
2789 if (!(possible_truths
& (1 << i
)))
2792 fprintf (dump_file
, ", ");
2794 dump_condition (dump_file
, info
->conds
, i
);
2798 estimate_calls_size_and_time (node
, &size
, &min_size
, &time
, &hints
, possible_truths
,
2799 known_vals
, known_contexts
, known_aggs
);
2800 sreal nonspecialized_time
= time
;
2802 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
2804 bool nonconst
= e
->nonconst_predicate
.evaluate (possible_truths
);
2805 bool exec
= e
->exec_predicate
.evaluate (nonspec_possible_truths
);
2806 gcc_assert (!nonconst
|| exec
);
2809 gcc_checking_assert (e
->time
>= 0);
2810 gcc_checking_assert (time
>= 0);
2812 /* We compute specialized size only because size of nonspecialized
2813 copy is context independent.
2815 The difference between nonspecialized execution and specialized is
2816 that nonspecialized is not going to have optimized out computations
2817 known to be constant in a specialized setting. */
2820 nonspecialized_time
+= e
->time
;
2823 else if (!inline_param_summary
.exists ())
2830 int prob
= e
->nonconst_predicate
.probability
2831 (info
->conds
, possible_truths
,
2832 inline_param_summary
);
2833 gcc_checking_assert (prob
>= 0);
2834 gcc_checking_assert (prob
<= REG_BR_PROB_BASE
);
2835 time
+= e
->time
* prob
/ REG_BR_PROB_BASE
;
2837 gcc_checking_assert (time
>= 0);
2840 gcc_checking_assert ((*info
->entry
)[0].exec_predicate
== true);
2841 gcc_checking_assert ((*info
->entry
)[0].nonconst_predicate
== true);
2842 min_size
= (*info
->entry
)[0].size
;
2843 gcc_checking_assert (size
>= 0);
2844 gcc_checking_assert (time
>= 0);
2845 /* nonspecialized_time should be always bigger than specialized time.
2846 Roundoff issues however may get into the way. */
2847 gcc_checking_assert ((nonspecialized_time
- time
) >= -1);
2849 /* Roundoff issues may make specialized time bigger than nonspecialized
2850 time. We do not really want that to happen because some heurstics
2851 may get confused by seeing negative speedups. */
2852 if (time
> nonspecialized_time
)
2853 time
= nonspecialized_time
;
2855 if (info
->loop_iterations
2856 && !info
->loop_iterations
->evaluate (possible_truths
))
2857 hints
|= INLINE_HINT_loop_iterations
;
2858 if (info
->loop_stride
2859 && !info
->loop_stride
->evaluate (possible_truths
))
2860 hints
|= INLINE_HINT_loop_stride
;
2861 if (info
->array_index
2862 && !info
->array_index
->evaluate (possible_truths
))
2863 hints
|= INLINE_HINT_array_index
;
2865 hints
|= INLINE_HINT_in_scc
;
2866 if (DECL_DECLARED_INLINE_P (node
->decl
))
2867 hints
|= INLINE_HINT_declared_inline
;
2869 size
= RDIV (size
, INLINE_SIZE_SCALE
);
2870 min_size
= RDIV (min_size
, INLINE_SIZE_SCALE
);
2872 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2873 fprintf (dump_file
, "\n size:%i time:%f nonspec time:%f\n", (int) size
,
2874 time
.to_double (), nonspecialized_time
.to_double ());
2877 if (ret_nonspecialized_time
)
2878 *ret_nonspecialized_time
= nonspecialized_time
;
2882 *ret_min_size
= min_size
;
2889 /* Estimate size and time needed to execute callee of EDGE assuming that
2890 parameters known to be constant at caller of EDGE are propagated.
2891 KNOWN_VALS and KNOWN_CONTEXTS are vectors of assumed known constant values
2892 and types for parameters. */
2895 estimate_ipcp_clone_size_and_time (struct cgraph_node
*node
,
2896 vec
<tree
> known_vals
,
2897 vec
<ipa_polymorphic_call_context
>
2899 vec
<ipa_agg_jump_function_p
> known_aggs
,
2900 int *ret_size
, sreal
*ret_time
,
2901 sreal
*ret_nonspec_time
,
2902 inline_hints
*hints
)
2904 clause_t clause
, nonspec_clause
;
2906 evaluate_conditions_for_known_args (node
, false, known_vals
, known_aggs
,
2907 &clause
, &nonspec_clause
);
2908 estimate_node_size_and_time (node
, clause
, nonspec_clause
,
2909 known_vals
, known_contexts
,
2910 known_aggs
, ret_size
, NULL
, ret_time
,
2911 ret_nonspec_time
, hints
, vNULL
);
2915 /* Update summary information of inline clones after inlining.
2916 Compute peak stack usage. */
2919 inline_update_callee_summaries (struct cgraph_node
*node
, int depth
)
2921 struct cgraph_edge
*e
;
2922 struct inline_summary
*callee_info
= inline_summaries
->get (node
);
2923 struct inline_summary
*caller_info
= inline_summaries
->get (node
->callers
->caller
);
2926 callee_info
->stack_frame_offset
2927 = caller_info
->stack_frame_offset
2928 + caller_info
->estimated_self_stack_size
;
2929 peak
= callee_info
->stack_frame_offset
2930 + callee_info
->estimated_self_stack_size
;
2931 if (inline_summaries
->get (node
->global
.inlined_to
)->estimated_stack_size
< peak
)
2932 inline_summaries
->get (node
->global
.inlined_to
)->estimated_stack_size
= peak
;
2933 ipa_propagate_frequency (node
);
2934 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2936 if (!e
->inline_failed
)
2937 inline_update_callee_summaries (e
->callee
, depth
);
2938 ipa_call_summaries
->get (e
)->loop_depth
+= depth
;
2940 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2941 ipa_call_summaries
->get (e
)->loop_depth
+= depth
;
2944 /* Update change_prob of EDGE after INLINED_EDGE has been inlined.
2945 When functoin A is inlined in B and A calls C with parameter that
2946 changes with probability PROB1 and C is known to be passthroug
2947 of argument if B that change with probability PROB2, the probability
2948 of change is now PROB1*PROB2. */
2951 remap_edge_change_prob (struct cgraph_edge
*inlined_edge
,
2952 struct cgraph_edge
*edge
)
2954 if (ipa_node_params_sum
)
2957 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
2958 struct ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
2959 struct ipa_call_summary
*inlined_es
2960 = ipa_call_summaries
->get (inlined_edge
);
2962 for (i
= 0; i
< ipa_get_cs_argument_count (args
); i
++)
2964 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
2965 if (jfunc
->type
== IPA_JF_PASS_THROUGH
2966 || jfunc
->type
== IPA_JF_ANCESTOR
)
2968 int id
= jfunc
->type
== IPA_JF_PASS_THROUGH
2969 ? ipa_get_jf_pass_through_formal_id (jfunc
)
2970 : ipa_get_jf_ancestor_formal_id (jfunc
);
2971 if (id
< (int) inlined_es
->param
.length ())
2973 int prob1
= es
->param
[i
].change_prob
;
2974 int prob2
= inlined_es
->param
[id
].change_prob
;
2975 int prob
= combine_probabilities (prob1
, prob2
);
2977 if (prob1
&& prob2
&& !prob
)
2980 es
->param
[i
].change_prob
= prob
;
2987 /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
2989 Remap predicates of callees of NODE. Rest of arguments match
2992 Also update change probabilities. */
2995 remap_edge_summaries (struct cgraph_edge
*inlined_edge
,
2996 struct cgraph_node
*node
,
2997 struct inline_summary
*info
,
2998 struct inline_summary
*callee_info
,
2999 vec
<int> operand_map
,
3000 vec
<int> offset_map
,
3001 clause_t possible_truths
,
3002 predicate
*toplev_predicate
)
3004 struct cgraph_edge
*e
, *next
;
3005 for (e
= node
->callees
; e
; e
= next
)
3007 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
3009 next
= e
->next_callee
;
3011 if (e
->inline_failed
)
3013 remap_edge_change_prob (inlined_edge
, e
);
3017 p
= es
->predicate
->remap_after_inlining
3018 (info
, callee_info
, operand_map
,
3019 offset_map
, possible_truths
,
3021 edge_set_predicate (e
, &p
);
3024 edge_set_predicate (e
, toplev_predicate
);
3027 remap_edge_summaries (inlined_edge
, e
->callee
, info
, callee_info
,
3028 operand_map
, offset_map
, possible_truths
,
3031 for (e
= node
->indirect_calls
; e
; e
= next
)
3033 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
3035 next
= e
->next_callee
;
3037 remap_edge_change_prob (inlined_edge
, e
);
3040 p
= es
->predicate
->remap_after_inlining
3041 (info
, callee_info
, operand_map
, offset_map
,
3042 possible_truths
, *toplev_predicate
);
3043 edge_set_predicate (e
, &p
);
3046 edge_set_predicate (e
, toplev_predicate
);
3050 /* Same as remap_predicate, but set result into hint *HINT. */
3053 remap_hint_predicate (struct inline_summary
*info
,
3054 struct inline_summary
*callee_info
,
3056 vec
<int> operand_map
,
3057 vec
<int> offset_map
,
3058 clause_t possible_truths
,
3059 predicate
*toplev_predicate
)
3065 p
= (*hint
)->remap_after_inlining
3067 operand_map
, offset_map
,
3068 possible_truths
, *toplev_predicate
);
3069 if (p
!= false && p
!= true)
3072 set_hint_predicate (hint
, p
);
3078 /* We inlined EDGE. Update summary of the function we inlined into. */
3081 inline_merge_summary (struct cgraph_edge
*edge
)
3083 struct inline_summary
*callee_info
= inline_summaries
->get (edge
->callee
);
3084 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3085 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
3086 struct inline_summary
*info
= inline_summaries
->get (to
);
3087 clause_t clause
= 0; /* not_inline is known to be false. */
3089 vec
<int> operand_map
= vNULL
;
3090 vec
<int> offset_map
= vNULL
;
3092 predicate toplev_predicate
;
3093 predicate true_p
= true;
3094 struct ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
3097 toplev_predicate
= *es
->predicate
;
3099 toplev_predicate
= true;
3101 info
->fp_expressions
|= callee_info
->fp_expressions
;
3103 if (callee_info
->conds
)
3104 evaluate_properties_for_edge (edge
, true, &clause
, NULL
, NULL
, NULL
, NULL
);
3105 if (ipa_node_params_sum
&& callee_info
->conds
)
3107 struct ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3108 int count
= ipa_get_cs_argument_count (args
);
3113 operand_map
.safe_grow_cleared (count
);
3114 offset_map
.safe_grow_cleared (count
);
3116 for (i
= 0; i
< count
; i
++)
3118 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3121 /* TODO: handle non-NOPs when merging. */
3122 if (jfunc
->type
== IPA_JF_PASS_THROUGH
)
3124 if (ipa_get_jf_pass_through_operation (jfunc
) == NOP_EXPR
)
3125 map
= ipa_get_jf_pass_through_formal_id (jfunc
);
3126 if (!ipa_get_jf_pass_through_agg_preserved (jfunc
))
3129 else if (jfunc
->type
== IPA_JF_ANCESTOR
)
3131 HOST_WIDE_INT offset
= ipa_get_jf_ancestor_offset (jfunc
);
3132 if (offset
>= 0 && offset
< INT_MAX
)
3134 map
= ipa_get_jf_ancestor_formal_id (jfunc
);
3135 if (!ipa_get_jf_ancestor_agg_preserved (jfunc
))
3137 offset_map
[i
] = offset
;
3140 operand_map
[i
] = map
;
3141 gcc_assert (map
< ipa_get_param_count (IPA_NODE_REF (to
)));
3144 for (i
= 0; vec_safe_iterate (callee_info
->entry
, i
, &e
); i
++)
3147 p
= e
->exec_predicate
.remap_after_inlining
3148 (info
, callee_info
, operand_map
,
3151 predicate nonconstp
;
3152 nonconstp
= e
->nonconst_predicate
.remap_after_inlining
3153 (info
, callee_info
, operand_map
,
3156 if (p
!= false && nonconstp
!= false)
3158 sreal add_time
= ((sreal
)e
->time
* edge
->frequency
) / CGRAPH_FREQ_BASE
;
3159 int prob
= e
->nonconst_predicate
.probability (callee_info
->conds
,
3161 add_time
= add_time
* prob
/ REG_BR_PROB_BASE
;
3162 if (prob
!= REG_BR_PROB_BASE
3163 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3165 fprintf (dump_file
, "\t\tScaling time by probability:%f\n",
3166 (double) prob
/ REG_BR_PROB_BASE
);
3168 account_size_time (info
, e
->size
, add_time
, &p
, &nonconstp
);
3171 remap_edge_summaries (edge
, edge
->callee
, info
, callee_info
, operand_map
,
3172 offset_map
, clause
, &toplev_predicate
);
3173 remap_hint_predicate (info
, callee_info
,
3174 &callee_info
->loop_iterations
,
3175 operand_map
, offset_map
, clause
, &toplev_predicate
);
3176 remap_hint_predicate (info
, callee_info
,
3177 &callee_info
->loop_stride
,
3178 operand_map
, offset_map
, clause
, &toplev_predicate
);
3179 remap_hint_predicate (info
, callee_info
,
3180 &callee_info
->array_index
,
3181 operand_map
, offset_map
, clause
, &toplev_predicate
);
3183 inline_update_callee_summaries (edge
->callee
,
3184 ipa_call_summaries
->get (edge
)->loop_depth
);
3186 /* We do not maintain predicates of inlined edges, free it. */
3187 edge_set_predicate (edge
, &true_p
);
3188 /* Similarly remove param summaries. */
3189 es
->param
.release ();
3190 operand_map
.release ();
3191 offset_map
.release ();
3194 /* For performance reasons inline_merge_summary is not updating overall size
3195 and time. Recompute it. */
3198 inline_update_overall_summary (struct cgraph_node
*node
)
3200 struct inline_summary
*info
= inline_summaries
->get (node
);
3206 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
3208 info
->size
+= e
->size
;
3209 info
->time
+= e
->time
;
3211 estimate_calls_size_and_time (node
, &info
->size
, &info
->min_size
,
3213 ~(clause_t
) (1 << predicate::false_condition
),
3214 vNULL
, vNULL
, vNULL
);
3215 info
->size
= (info
->size
+ INLINE_SIZE_SCALE
/ 2) / INLINE_SIZE_SCALE
;
3218 /* Return hints derrived from EDGE. */
3220 simple_edge_hints (struct cgraph_edge
*edge
)
3223 struct cgraph_node
*to
= (edge
->caller
->global
.inlined_to
3224 ? edge
->caller
->global
.inlined_to
: edge
->caller
);
3225 struct cgraph_node
*callee
= edge
->callee
->ultimate_alias_target ();
3226 if (inline_summaries
->get (to
)->scc_no
3227 && inline_summaries
->get (to
)->scc_no
3228 == inline_summaries
->get (callee
)->scc_no
3229 && !edge
->recursive_p ())
3230 hints
|= INLINE_HINT_same_scc
;
3232 if (callee
->lto_file_data
&& edge
->caller
->lto_file_data
3233 && edge
->caller
->lto_file_data
!= callee
->lto_file_data
3234 && !callee
->merged_comdat
&& !callee
->icf_merged
)
3235 hints
|= INLINE_HINT_cross_module
;
3240 /* Estimate the time cost for the caller when inlining EDGE.
3241 Only to be called via estimate_edge_time, that handles the
3244 When caching, also update the cache entry. Compute both time and
3245 size, since we always need both metrics eventually. */
3248 do_estimate_edge_time (struct cgraph_edge
*edge
)
3250 sreal time
, nonspec_time
;
3253 struct cgraph_node
*callee
;
3254 clause_t clause
, nonspec_clause
;
3255 vec
<tree
> known_vals
;
3256 vec
<ipa_polymorphic_call_context
> known_contexts
;
3257 vec
<ipa_agg_jump_function_p
> known_aggs
;
3258 struct ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
3261 callee
= edge
->callee
->ultimate_alias_target ();
3263 gcc_checking_assert (edge
->inline_failed
);
3264 evaluate_properties_for_edge (edge
, true,
3265 &clause
, &nonspec_clause
, &known_vals
,
3266 &known_contexts
, &known_aggs
);
3267 estimate_node_size_and_time (callee
, clause
, nonspec_clause
, known_vals
,
3268 known_contexts
, known_aggs
, &size
, &min_size
,
3269 &time
, &nonspec_time
, &hints
, es
->param
);
3271 /* When we have profile feedback, we can quite safely identify hot
3272 edges and for those we disable size limits. Don't do that when
3273 probability that caller will call the callee is low however, since it
3274 may hurt optimization of the caller's hot path. */
3275 if (edge
->count
&& edge
->maybe_hot_p ()
3277 > (edge
->caller
->global
.inlined_to
3278 ? edge
->caller
->global
.inlined_to
->count
: edge
->caller
->count
)))
3279 hints
|= INLINE_HINT_known_hot
;
3281 known_vals
.release ();
3282 known_contexts
.release ();
3283 known_aggs
.release ();
3284 gcc_checking_assert (size
>= 0);
3285 gcc_checking_assert (time
>= 0);
3287 /* When caching, update the cache entry. */
3288 if (edge_growth_cache
.exists ())
3290 inline_summaries
->get (edge
->callee
)->min_size
= min_size
;
3291 if ((int) edge_growth_cache
.length () <= edge
->uid
)
3292 edge_growth_cache
.safe_grow_cleared (symtab
->edges_max_uid
);
3293 edge_growth_cache
[edge
->uid
].time
= time
;
3294 edge_growth_cache
[edge
->uid
].nonspec_time
= nonspec_time
;
3296 edge_growth_cache
[edge
->uid
].size
= size
+ (size
>= 0);
3297 hints
|= simple_edge_hints (edge
);
3298 edge_growth_cache
[edge
->uid
].hints
= hints
+ 1;
3304 /* Return estimated callee growth after inlining EDGE.
3305 Only to be called via estimate_edge_size. */
3308 do_estimate_edge_size (struct cgraph_edge
*edge
)
3311 struct cgraph_node
*callee
;
3312 clause_t clause
, nonspec_clause
;
3313 vec
<tree
> known_vals
;
3314 vec
<ipa_polymorphic_call_context
> known_contexts
;
3315 vec
<ipa_agg_jump_function_p
> known_aggs
;
3317 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3319 if (edge_growth_cache
.exists ())
3321 do_estimate_edge_time (edge
);
3322 size
= edge_growth_cache
[edge
->uid
].size
;
3323 gcc_checking_assert (size
);
3324 return size
- (size
> 0);
3327 callee
= edge
->callee
->ultimate_alias_target ();
3329 /* Early inliner runs without caching, go ahead and do the dirty work. */
3330 gcc_checking_assert (edge
->inline_failed
);
3331 evaluate_properties_for_edge (edge
, true,
3332 &clause
, &nonspec_clause
,
3333 &known_vals
, &known_contexts
,
3335 estimate_node_size_and_time (callee
, clause
, nonspec_clause
, known_vals
,
3336 known_contexts
, known_aggs
, &size
, NULL
, NULL
,
3338 known_vals
.release ();
3339 known_contexts
.release ();
3340 known_aggs
.release ();
3345 /* Estimate the growth of the caller when inlining EDGE.
3346 Only to be called via estimate_edge_size. */
3349 do_estimate_edge_hints (struct cgraph_edge
*edge
)
3352 struct cgraph_node
*callee
;
3353 clause_t clause
, nonspec_clause
;
3354 vec
<tree
> known_vals
;
3355 vec
<ipa_polymorphic_call_context
> known_contexts
;
3356 vec
<ipa_agg_jump_function_p
> known_aggs
;
3358 /* When we do caching, use do_estimate_edge_time to populate the entry. */
3360 if (edge_growth_cache
.exists ())
3362 do_estimate_edge_time (edge
);
3363 hints
= edge_growth_cache
[edge
->uid
].hints
;
3364 gcc_checking_assert (hints
);
3368 callee
= edge
->callee
->ultimate_alias_target ();
3370 /* Early inliner runs without caching, go ahead and do the dirty work. */
3371 gcc_checking_assert (edge
->inline_failed
);
3372 evaluate_properties_for_edge (edge
, true,
3373 &clause
, &nonspec_clause
,
3374 &known_vals
, &known_contexts
,
3376 estimate_node_size_and_time (callee
, clause
, nonspec_clause
, known_vals
,
3377 known_contexts
, known_aggs
, NULL
, NULL
,
3378 NULL
, NULL
, &hints
, vNULL
);
3379 known_vals
.release ();
3380 known_contexts
.release ();
3381 known_aggs
.release ();
3382 hints
|= simple_edge_hints (edge
);
3386 /* Estimate the size of NODE after inlining EDGE which should be an
3387 edge to either NODE or a call inlined into NODE. */
3390 estimate_size_after_inlining (struct cgraph_node
*node
,
3391 struct cgraph_edge
*edge
)
3393 struct ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
3394 if (!es
->predicate
|| *es
->predicate
!= false)
3396 int size
= inline_summaries
->get (node
)->size
+ estimate_edge_growth (edge
);
3397 gcc_assert (size
>= 0);
3400 return inline_summaries
->get (node
)->size
;
3406 struct cgraph_node
*node
;
3407 bool self_recursive
;
3413 /* Worker for do_estimate_growth. Collect growth for all callers. */
3416 do_estimate_growth_1 (struct cgraph_node
*node
, void *data
)
3418 struct cgraph_edge
*e
;
3419 struct growth_data
*d
= (struct growth_data
*) data
;
3421 for (e
= node
->callers
; e
; e
= e
->next_caller
)
3423 gcc_checking_assert (e
->inline_failed
);
3425 if (cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
3427 d
->uninlinable
= true;
3431 if (e
->recursive_p ())
3433 d
->self_recursive
= true;
3436 d
->growth
+= estimate_edge_growth (e
);
3442 /* Estimate the growth caused by inlining NODE into all callees. */
3445 estimate_growth (struct cgraph_node
*node
)
3447 struct growth_data d
= { node
, false, false, 0 };
3448 struct inline_summary
*info
= inline_summaries
->get (node
);
3450 node
->call_for_symbol_and_aliases (do_estimate_growth_1
, &d
, true);
3452 /* For self recursive functions the growth estimation really should be
3453 infinity. We don't want to return very large values because the growth
3454 plays various roles in badness computation fractions. Be sure to not
3455 return zero or negative growths. */
3456 if (d
.self_recursive
)
3457 d
.growth
= d
.growth
< info
->size
? info
->size
: d
.growth
;
3458 else if (DECL_EXTERNAL (node
->decl
) || d
.uninlinable
)
3462 if (node
->will_be_removed_from_program_if_no_direct_calls_p ())
3463 d
.growth
-= info
->size
;
3464 /* COMDAT functions are very often not shared across multiple units
3465 since they come from various template instantiations.
3466 Take this into account. */
3467 else if (DECL_COMDAT (node
->decl
)
3468 && node
->can_remove_if_no_direct_calls_p ())
3469 d
.growth
-= (info
->size
3470 * (100 - PARAM_VALUE (PARAM_COMDAT_SHARING_PROBABILITY
))
3477 /* Verify if there are fewer than MAX_CALLERS. */
3480 check_callers (cgraph_node
*node
, int *max_callers
)
3484 if (!node
->can_remove_if_no_direct_calls_and_refs_p ())
3487 for (cgraph_edge
*e
= node
->callers
; e
; e
= e
->next_caller
)
3491 || cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
3495 FOR_EACH_ALIAS (node
, ref
)
3496 if (check_callers (dyn_cast
<cgraph_node
*> (ref
->referring
), max_callers
))
3503 /* Make cheap estimation if growth of NODE is likely positive knowing
3504 EDGE_GROWTH of one particular edge.
3505 We assume that most of other edges will have similar growth
3506 and skip computation if there are too many callers. */
3509 growth_likely_positive (struct cgraph_node
*node
,
3513 struct cgraph_edge
*e
;
3514 gcc_checking_assert (edge_growth
> 0);
3516 /* First quickly check if NODE is removable at all. */
3517 if (DECL_EXTERNAL (node
->decl
))
3519 if (!node
->can_remove_if_no_direct_calls_and_refs_p ()
3520 || node
->address_taken
)
3523 max_callers
= inline_summaries
->get (node
)->size
* 4 / edge_growth
+ 2;
3525 for (e
= node
->callers
; e
; e
= e
->next_caller
)
3529 || cgraph_inline_failed_type (e
->inline_failed
) == CIF_FINAL_ERROR
)
3534 FOR_EACH_ALIAS (node
, ref
)
3535 if (check_callers (dyn_cast
<cgraph_node
*> (ref
->referring
), &max_callers
))
3538 /* Unlike for functions called once, we play unsafe with
3539 COMDATs. We can allow that since we know functions
3540 in consideration are small (and thus risk is small) and
3541 moreover grow estimates already accounts that COMDAT
3542 functions may or may not disappear when eliminated from
3543 current unit. With good probability making aggressive
3544 choice in all units is going to make overall program
3546 if (DECL_COMDAT (node
->decl
))
3548 if (!node
->can_remove_if_no_direct_calls_p ())
3551 else if (!node
->will_be_removed_from_program_if_no_direct_calls_p ())
3554 return estimate_growth (node
) > 0;
3558 /* This function performs intraprocedural analysis in NODE that is required to
3559 inline indirect calls. */
3562 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
3564 ipa_analyze_node (node
);
3565 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3567 ipa_print_node_params (dump_file
, node
);
3568 ipa_print_node_jump_functions (dump_file
, node
);
3573 /* Note function body size. */
3576 inline_analyze_function (struct cgraph_node
*node
)
3578 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
3581 fprintf (dump_file
, "\nAnalyzing function: %s/%u\n",
3582 node
->name (), node
->order
);
3583 if (opt_for_fn (node
->decl
, optimize
) && !node
->thunk
.thunk_p
)
3584 inline_indirect_intraprocedural_analysis (node
);
3585 compute_inline_parameters (node
, false);
3588 struct cgraph_edge
*e
;
3589 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3590 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3591 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3592 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3599 /* Called when new function is inserted to callgraph late. */
3602 inline_summary_t::insert (struct cgraph_node
*node
, inline_summary
*)
3604 inline_analyze_function (node
);
3607 /* Note function body size. */
3610 inline_generate_summary (void)
3612 struct cgraph_node
*node
;
3614 FOR_EACH_DEFINED_FUNCTION (node
)
3615 if (DECL_STRUCT_FUNCTION (node
->decl
))
3616 node
->local
.versionable
= tree_versionable_function_p (node
->decl
);
3618 /* When not optimizing, do not bother to analyze. Inlining is still done
3619 because edge redirection needs to happen there. */
3620 if (!optimize
&& !flag_generate_lto
&& !flag_generate_offload
&& !flag_wpa
)
3623 if (!inline_summaries
)
3624 inline_summaries
= (inline_summary_t
*) inline_summary_t::create_ggc (symtab
);
3626 inline_summaries
->enable_insertion_hook ();
3628 ipa_register_cgraph_hooks ();
3629 inline_free_summary ();
3631 FOR_EACH_DEFINED_FUNCTION (node
)
3633 inline_analyze_function (node
);
3637 /* Write inline summary for edge E to OB. */
3640 read_ipa_call_summary (struct lto_input_block
*ib
, struct cgraph_edge
*e
)
3642 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
3646 es
->call_stmt_size
= streamer_read_uhwi (ib
);
3647 es
->call_stmt_time
= streamer_read_uhwi (ib
);
3648 es
->loop_depth
= streamer_read_uhwi (ib
);
3650 edge_set_predicate (e
, &p
);
3651 length
= streamer_read_uhwi (ib
);
3654 es
->param
.safe_grow_cleared (length
);
3655 for (i
= 0; i
< length
; i
++)
3656 es
->param
[i
].change_prob
= streamer_read_uhwi (ib
);
3661 /* Stream in inline summaries from the section. */
3664 inline_read_section (struct lto_file_decl_data
*file_data
, const char *data
,
3667 const struct lto_function_header
*header
=
3668 (const struct lto_function_header
*) data
;
3669 const int cfg_offset
= sizeof (struct lto_function_header
);
3670 const int main_offset
= cfg_offset
+ header
->cfg_size
;
3671 const int string_offset
= main_offset
+ header
->main_size
;
3672 struct data_in
*data_in
;
3673 unsigned int i
, count2
, j
;
3674 unsigned int f_count
;
3676 lto_input_block
ib ((const char *) data
+ main_offset
, header
->main_size
,
3677 file_data
->mode_table
);
3680 lto_data_in_create (file_data
, (const char *) data
+ string_offset
,
3681 header
->string_size
, vNULL
);
3682 f_count
= streamer_read_uhwi (&ib
);
3683 for (i
= 0; i
< f_count
; i
++)
3686 struct cgraph_node
*node
;
3687 struct inline_summary
*info
;
3688 lto_symtab_encoder_t encoder
;
3689 struct bitpack_d bp
;
3690 struct cgraph_edge
*e
;
3693 index
= streamer_read_uhwi (&ib
);
3694 encoder
= file_data
->symtab_node_encoder
;
3695 node
= dyn_cast
<cgraph_node
*> (lto_symtab_encoder_deref (encoder
,
3697 info
= inline_summaries
->get (node
);
3699 info
->estimated_stack_size
3700 = info
->estimated_self_stack_size
= streamer_read_uhwi (&ib
);
3701 info
->size
= info
->self_size
= streamer_read_uhwi (&ib
);
3702 info
->time
= info
->self_time
= sreal::stream_in (&ib
);
3704 bp
= streamer_read_bitpack (&ib
);
3705 info
->inlinable
= bp_unpack_value (&bp
, 1);
3706 info
->contains_cilk_spawn
= bp_unpack_value (&bp
, 1);
3707 info
->fp_expressions
= bp_unpack_value (&bp
, 1);
3709 count2
= streamer_read_uhwi (&ib
);
3710 gcc_assert (!info
->conds
);
3711 for (j
= 0; j
< count2
; j
++)
3714 c
.operand_num
= streamer_read_uhwi (&ib
);
3715 c
.size
= streamer_read_uhwi (&ib
);
3716 c
.code
= (enum tree_code
) streamer_read_uhwi (&ib
);
3717 c
.val
= stream_read_tree (&ib
, data_in
);
3718 bp
= streamer_read_bitpack (&ib
);
3719 c
.agg_contents
= bp_unpack_value (&bp
, 1);
3720 c
.by_ref
= bp_unpack_value (&bp
, 1);
3722 c
.offset
= streamer_read_uhwi (&ib
);
3723 vec_safe_push (info
->conds
, c
);
3725 count2
= streamer_read_uhwi (&ib
);
3726 gcc_assert (!info
->entry
);
3727 for (j
= 0; j
< count2
; j
++)
3729 struct size_time_entry e
;
3731 e
.size
= streamer_read_uhwi (&ib
);
3732 e
.time
= sreal::stream_in (&ib
);
3733 e
.exec_predicate
.stream_in (&ib
);
3734 e
.nonconst_predicate
.stream_in (&ib
);
3736 vec_safe_push (info
->entry
, e
);
3740 set_hint_predicate (&info
->loop_iterations
, p
);
3742 set_hint_predicate (&info
->loop_stride
, p
);
3744 set_hint_predicate (&info
->array_index
, p
);
3745 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3746 read_ipa_call_summary (&ib
, e
);
3747 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3748 read_ipa_call_summary (&ib
, e
);
3751 lto_free_section_data (file_data
, LTO_section_inline_summary
, NULL
, data
,
3753 lto_data_in_delete (data_in
);
3757 /* Read inline summary. Jump functions are shared among ipa-cp
3758 and inliner, so when ipa-cp is active, we don't need to write them
3762 inline_read_summary (void)
3764 struct lto_file_decl_data
**file_data_vec
= lto_get_file_decl_data ();
3765 struct lto_file_decl_data
*file_data
;
3768 inline_summary_alloc ();
3770 while ((file_data
= file_data_vec
[j
++]))
3773 const char *data
= lto_get_section_data (file_data
,
3774 LTO_section_inline_summary
,
3777 inline_read_section (file_data
, data
, len
);
3779 /* Fatal error here. We do not want to support compiling ltrans units
3780 with different version of compiler or different flags than the WPA
3781 unit, so this should never happen. */
3782 fatal_error (input_location
,
3783 "ipa inline summary is missing in input file");
3787 ipa_register_cgraph_hooks ();
3789 ipa_prop_read_jump_functions ();
3792 gcc_assert (inline_summaries
);
3793 inline_summaries
->enable_insertion_hook ();
3797 /* Write inline summary for edge E to OB. */
3800 write_ipa_call_summary (struct output_block
*ob
, struct cgraph_edge
*e
)
3802 struct ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
3805 streamer_write_uhwi (ob
, es
->call_stmt_size
);
3806 streamer_write_uhwi (ob
, es
->call_stmt_time
);
3807 streamer_write_uhwi (ob
, es
->loop_depth
);
3809 es
->predicate
->stream_out (ob
);
3811 streamer_write_uhwi (ob
, 0);
3812 streamer_write_uhwi (ob
, es
->param
.length ());
3813 for (i
= 0; i
< (int) es
->param
.length (); i
++)
3814 streamer_write_uhwi (ob
, es
->param
[i
].change_prob
);
3818 /* Write inline summary for node in SET.
3819 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
3820 active, we don't need to write them twice. */
3823 inline_write_summary (void)
3825 struct output_block
*ob
= create_output_block (LTO_section_inline_summary
);
3826 lto_symtab_encoder_t encoder
= ob
->decl_state
->symtab_node_encoder
;
3827 unsigned int count
= 0;
3830 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
3832 symtab_node
*snode
= lto_symtab_encoder_deref (encoder
, i
);
3833 cgraph_node
*cnode
= dyn_cast
<cgraph_node
*> (snode
);
3834 if (cnode
&& cnode
->definition
&& !cnode
->alias
)
3837 streamer_write_uhwi (ob
, count
);
3839 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
3841 symtab_node
*snode
= lto_symtab_encoder_deref (encoder
, i
);
3842 cgraph_node
*cnode
= dyn_cast
<cgraph_node
*> (snode
);
3843 if (cnode
&& cnode
->definition
&& !cnode
->alias
)
3845 struct inline_summary
*info
= inline_summaries
->get (cnode
);
3846 struct bitpack_d bp
;
3847 struct cgraph_edge
*edge
;
3850 struct condition
*c
;
3852 streamer_write_uhwi (ob
, lto_symtab_encoder_encode (encoder
, cnode
));
3853 streamer_write_hwi (ob
, info
->estimated_self_stack_size
);
3854 streamer_write_hwi (ob
, info
->self_size
);
3855 info
->self_time
.stream_out (ob
);
3856 bp
= bitpack_create (ob
->main_stream
);
3857 bp_pack_value (&bp
, info
->inlinable
, 1);
3858 bp_pack_value (&bp
, info
->contains_cilk_spawn
, 1);
3859 bp_pack_value (&bp
, info
->fp_expressions
, 1);
3860 streamer_write_bitpack (&bp
);
3861 streamer_write_uhwi (ob
, vec_safe_length (info
->conds
));
3862 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
3864 streamer_write_uhwi (ob
, c
->operand_num
);
3865 streamer_write_uhwi (ob
, c
->size
);
3866 streamer_write_uhwi (ob
, c
->code
);
3867 stream_write_tree (ob
, c
->val
, true);
3868 bp
= bitpack_create (ob
->main_stream
);
3869 bp_pack_value (&bp
, c
->agg_contents
, 1);
3870 bp_pack_value (&bp
, c
->by_ref
, 1);
3871 streamer_write_bitpack (&bp
);
3872 if (c
->agg_contents
)
3873 streamer_write_uhwi (ob
, c
->offset
);
3875 streamer_write_uhwi (ob
, vec_safe_length (info
->entry
));
3876 for (i
= 0; vec_safe_iterate (info
->entry
, i
, &e
); i
++)
3878 streamer_write_uhwi (ob
, e
->size
);
3879 e
->time
.stream_out (ob
);
3880 e
->exec_predicate
.stream_out (ob
);
3881 e
->nonconst_predicate
.stream_out (ob
);
3883 if (info
->loop_iterations
)
3884 info
->loop_iterations
->stream_out (ob
);
3886 streamer_write_uhwi (ob
, 0);
3887 if (info
->loop_stride
)
3888 info
->loop_stride
->stream_out (ob
);
3890 streamer_write_uhwi (ob
, 0);
3891 if (info
->array_index
)
3892 info
->array_index
->stream_out (ob
);
3894 streamer_write_uhwi (ob
, 0);
3895 for (edge
= cnode
->callees
; edge
; edge
= edge
->next_callee
)
3896 write_ipa_call_summary (ob
, edge
);
3897 for (edge
= cnode
->indirect_calls
; edge
; edge
= edge
->next_callee
)
3898 write_ipa_call_summary (ob
, edge
);
3901 streamer_write_char_stream (ob
->main_stream
, 0);
3902 produce_asm (ob
, NULL
);
3903 destroy_output_block (ob
);
3905 if (optimize
&& !flag_ipa_cp
)
3906 ipa_prop_write_jump_functions ();
3910 /* Release inline summary. */
3913 inline_free_summary (void)
3915 struct cgraph_node
*node
;
3916 if (!ipa_call_summaries
)
3918 FOR_EACH_DEFINED_FUNCTION (node
)
3920 reset_inline_summary (node
, inline_summaries
->get (node
));
3921 inline_summaries
->release ();
3922 inline_summaries
= NULL
;
3923 ipa_call_summaries
->release ();
3924 delete ipa_call_summaries
;
3925 ipa_call_summaries
= NULL
;
3926 edge_predicate_pool
.release ();