1 /* Function summary pass.
2 Copyright (C) 2003-2019 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* Analysis of function bodies used by inter-procedural passes
23 We estimate for each function
24 - function body size and size after specializing into given context
25 - average function execution time in a given context
28 - call statement size, time and how often the parameters change
30 ipa_fn_summary data structures store above information locally (i.e.
31 parameters of the function itself) and globally (i.e. parameters of
32 the function created by applying all the inline decisions already
33 present in the callgraph).
35 We provide access to the ipa_fn_summary data structure and
36 basic logic updating the parameters when inlining is performed.
38 The summaries are context sensitive. Context means
39 1) partial assignment of known constant values of operands
40 2) whether function is inlined into the call or not.
41 It is easy to add more variants. To represent function size and time
42 that depends on context (i.e. it is known to be optimized away when
43 context is known either by inlining or from IP-CP and cloning),
46 estimate_edge_size_and_time can be used to query
47 function size/time in the given context. ipa_merge_fn_summary_after_inlining merges
48 properties of caller and callee after inlining.
50 Finally pass_inline_parameters is exported. This is used to drive
51 computation of function parameters used by the early inliner. IPA
52 inlined performs analysis via its analyze_function method. */
56 #include "coretypes.h"
60 #include "alloc-pool.h"
61 #include "tree-pass.h"
63 #include "tree-streamer.h"
65 #include "diagnostic.h"
66 #include "fold-const.h"
67 #include "print-tree.h"
68 #include "tree-inline.h"
69 #include "gimple-pretty-print.h"
71 #include "gimple-iterator.h"
73 #include "tree-ssa-loop-niter.h"
74 #include "tree-ssa-loop.h"
75 #include "symbol-summary.h"
77 #include "ipa-fnsummary.h"
79 #include "tree-scalar-evolution.h"
80 #include "ipa-utils.h"
81 #include "cfgexpand.h"
83 #include "stringpool.h"
87 fast_function_summary
<ipa_fn_summary
*, va_gc
> *ipa_fn_summaries
;
88 fast_function_summary
<ipa_size_summary
*, va_heap
> *ipa_size_summaries
;
89 fast_call_summary
<ipa_call_summary
*, va_heap
> *ipa_call_summaries
;
91 /* Edge predicates goes here. */
92 static object_allocator
<predicate
> edge_predicate_pool ("edge predicates");
97 ipa_dump_hints (FILE *f
, ipa_hints hints
)
101 fprintf (f
, "IPA hints:");
102 if (hints
& INLINE_HINT_indirect_call
)
104 hints
&= ~INLINE_HINT_indirect_call
;
105 fprintf (f
, " indirect_call");
107 if (hints
& INLINE_HINT_loop_iterations
)
109 hints
&= ~INLINE_HINT_loop_iterations
;
110 fprintf (f
, " loop_iterations");
112 if (hints
& INLINE_HINT_loop_stride
)
114 hints
&= ~INLINE_HINT_loop_stride
;
115 fprintf (f
, " loop_stride");
117 if (hints
& INLINE_HINT_same_scc
)
119 hints
&= ~INLINE_HINT_same_scc
;
120 fprintf (f
, " same_scc");
122 if (hints
& INLINE_HINT_in_scc
)
124 hints
&= ~INLINE_HINT_in_scc
;
125 fprintf (f
, " in_scc");
127 if (hints
& INLINE_HINT_cross_module
)
129 hints
&= ~INLINE_HINT_cross_module
;
130 fprintf (f
, " cross_module");
132 if (hints
& INLINE_HINT_declared_inline
)
134 hints
&= ~INLINE_HINT_declared_inline
;
135 fprintf (f
, " declared_inline");
137 if (hints
& INLINE_HINT_known_hot
)
139 hints
&= ~INLINE_HINT_known_hot
;
140 fprintf (f
, " known_hot");
146 /* Record SIZE and TIME to SUMMARY.
147 The accounted code will be executed when EXEC_PRED is true.
148 When NONCONST_PRED is false the code will evaulate to constant and
149 will get optimized out in specialized clones of the function. */
152 ipa_fn_summary::account_size_time (int size
, sreal time
,
153 const predicate
&exec_pred
,
154 const predicate
&nonconst_pred_in
)
159 predicate nonconst_pred
;
161 if (exec_pred
== false)
164 nonconst_pred
= nonconst_pred_in
& exec_pred
;
166 if (nonconst_pred
== false)
169 /* We need to create initial empty unconitional clause, but otherwie
170 we don't need to account empty times and sizes. */
171 if (!size
&& time
== 0 && size_time_table
)
174 gcc_assert (time
>= 0);
176 for (i
= 0; vec_safe_iterate (size_time_table
, i
, &e
); i
++)
177 if (e
->exec_predicate
== exec_pred
178 && e
->nonconst_predicate
== nonconst_pred
)
187 e
= &(*size_time_table
)[0];
188 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
190 "\t\tReached limit on number of entries, "
191 "ignoring the predicate.");
193 if (dump_file
&& (dump_flags
& TDF_DETAILS
) && (time
!= 0 || size
))
196 "\t\tAccounting size:%3.2f, time:%3.2f on %spredicate exec:",
197 ((double) size
) / ipa_fn_summary::size_scale
,
198 (time
.to_double ()), found
? "" : "new ");
199 exec_pred
.dump (dump_file
, conds
, 0);
200 if (exec_pred
!= nonconst_pred
)
202 fprintf (dump_file
, " nonconst:");
203 nonconst_pred
.dump (dump_file
, conds
);
206 fprintf (dump_file
, "\n");
210 class size_time_entry new_entry
;
211 new_entry
.size
= size
;
212 new_entry
.time
= time
;
213 new_entry
.exec_predicate
= exec_pred
;
214 new_entry
.nonconst_predicate
= nonconst_pred
;
215 vec_safe_push (size_time_table
, new_entry
);
224 /* We proved E to be unreachable, redirect it to __bultin_unreachable. */
226 static struct cgraph_edge
*
227 redirect_to_unreachable (struct cgraph_edge
*e
)
229 struct cgraph_node
*callee
= !e
->inline_failed
? e
->callee
: NULL
;
230 struct cgraph_node
*target
= cgraph_node::get_create
231 (builtin_decl_implicit (BUILT_IN_UNREACHABLE
));
234 e
= e
->resolve_speculation (target
->decl
);
236 e
->make_direct (target
);
238 e
->redirect_callee (target
);
239 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
240 e
->inline_failed
= CIF_UNREACHABLE
;
241 e
->count
= profile_count::zero ();
242 es
->call_stmt_size
= 0;
243 es
->call_stmt_time
= 0;
245 callee
->remove_symbol_and_inline_clones ();
249 /* Set predicate for edge E. */
252 edge_set_predicate (struct cgraph_edge
*e
, predicate
*predicate
)
254 /* If the edge is determined to be never executed, redirect it
255 to BUILTIN_UNREACHABLE to make it clear to IPA passes the call will
257 if (predicate
&& *predicate
== false
258 /* When handling speculative edges, we need to do the redirection
259 just once. Do it always on the direct edge, so we do not
260 attempt to resolve speculation while duplicating the edge. */
261 && (!e
->speculative
|| e
->callee
))
262 e
= redirect_to_unreachable (e
);
264 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
265 if (predicate
&& *predicate
!= true)
268 es
->predicate
= edge_predicate_pool
.allocate ();
269 *es
->predicate
= *predicate
;
274 edge_predicate_pool
.remove (es
->predicate
);
275 es
->predicate
= NULL
;
279 /* Set predicate for hint *P. */
282 set_hint_predicate (predicate
**p
, predicate new_predicate
)
284 if (new_predicate
== false || new_predicate
== true)
287 edge_predicate_pool
.remove (*p
);
293 *p
= edge_predicate_pool
.allocate ();
299 /* Compute what conditions may or may not hold given invormation about
300 parameters. RET_CLAUSE returns truths that may hold in a specialized copy,
301 whie RET_NONSPEC_CLAUSE returns truths that may hold in an nonspecialized
302 copy when called in a given context. It is a bitmask of conditions. Bit
303 0 means that condition is known to be false, while bit 1 means that condition
304 may or may not be true. These differs - for example NOT_INLINED condition
305 is always false in the second and also builtin_constant_p tests cannot use
306 the fact that parameter is indeed a constant.
308 KNOWN_VALS is partial mapping of parameters of NODE to constant values.
309 KNOWN_AGGS is a vector of aggreggate jump functions for each parameter.
310 Return clause of possible truths. When INLINE_P is true, assume that we are
313 ERROR_MARK means compile time invariant. */
316 evaluate_conditions_for_known_args (struct cgraph_node
*node
,
318 vec
<tree
> known_vals
,
319 vec
<ipa_agg_jump_function_p
>
321 clause_t
*ret_clause
,
322 clause_t
*ret_nonspec_clause
)
324 clause_t clause
= inline_p
? 0 : 1 << predicate::not_inlined_condition
;
325 clause_t nonspec_clause
= 1 << predicate::not_inlined_condition
;
326 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (node
);
330 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
335 struct expr_eval_op
*op
;
337 /* We allow call stmt to have fewer arguments than the callee function
338 (especially for K&R style programs). So bound check here (we assume
339 known_aggs vector, if non-NULL, has the same length as
341 gcc_checking_assert (!known_aggs
.exists ()
342 || (known_vals
.length () == known_aggs
.length ()));
343 if (c
->operand_num
>= (int) known_vals
.length ())
345 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
346 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
352 struct ipa_agg_jump_function
*agg
;
354 if (c
->code
== predicate::changed
356 && (known_vals
[c
->operand_num
] == error_mark_node
))
359 if (known_aggs
.exists ())
361 agg
= known_aggs
[c
->operand_num
];
362 val
= ipa_find_agg_cst_for_param (agg
, known_vals
[c
->operand_num
],
363 c
->offset
, c
->by_ref
);
370 val
= known_vals
[c
->operand_num
];
371 if (val
== error_mark_node
&& c
->code
!= predicate::changed
)
377 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
378 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
381 if (c
->code
== predicate::changed
)
383 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
387 if (TYPE_SIZE (c
->type
) != TYPE_SIZE (TREE_TYPE (val
)))
389 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
390 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
393 if (c
->code
== predicate::is_not_constant
)
395 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
399 val
= fold_unary (VIEW_CONVERT_EXPR
, c
->type
, val
);
400 for (j
= 0; vec_safe_iterate (c
->param_ops
, j
, &op
); j
++)
405 val
= fold_unary (op
->code
, op
->type
, val
);
406 else if (!op
->val
[1])
407 val
= fold_binary (op
->code
, op
->type
,
408 op
->index
? op
->val
[0] : val
,
409 op
->index
? val
: op
->val
[0]);
410 else if (op
->index
== 0)
411 val
= fold_ternary (op
->code
, op
->type
,
412 val
, op
->val
[0], op
->val
[1]);
413 else if (op
->index
== 1)
414 val
= fold_ternary (op
->code
, op
->type
,
415 op
->val
[0], val
, op
->val
[1]);
416 else if (op
->index
== 2)
417 val
= fold_ternary (op
->code
, op
->type
,
418 op
->val
[0], op
->val
[1], val
);
424 ? fold_binary_to_constant (c
->code
, boolean_type_node
, val
, c
->val
)
427 if (res
&& integer_zerop (res
))
430 clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
431 nonspec_clause
|= 1 << (i
+ predicate::first_dynamic_condition
);
433 *ret_clause
= clause
;
434 if (ret_nonspec_clause
)
435 *ret_nonspec_clause
= nonspec_clause
;
439 /* Work out what conditions might be true at invocation of E. */
442 evaluate_properties_for_edge (struct cgraph_edge
*e
, bool inline_p
,
443 clause_t
*clause_ptr
,
444 clause_t
*nonspec_clause_ptr
,
445 vec
<tree
> *known_vals_ptr
,
446 vec
<ipa_polymorphic_call_context
>
448 vec
<ipa_agg_jump_function_p
> *known_aggs_ptr
)
450 struct cgraph_node
*callee
= e
->callee
->ultimate_alias_target ();
451 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (callee
);
452 vec
<tree
> known_vals
= vNULL
;
453 vec
<ipa_agg_jump_function_p
> known_aggs
= vNULL
;
454 class ipa_edge_args
*args
;
457 *clause_ptr
= inline_p
? 0 : 1 << predicate::not_inlined_condition
;
459 known_vals_ptr
->create (0);
460 if (known_contexts_ptr
)
461 known_contexts_ptr
->create (0);
463 if (ipa_node_params_sum
464 && !e
->call_stmt_cannot_inline_p
465 && ((clause_ptr
&& info
->conds
) || known_vals_ptr
|| known_contexts_ptr
)
466 && (args
= IPA_EDGE_REF (e
)) != NULL
)
468 class ipa_node_params
*caller_parms_info
, *callee_pi
;
469 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
470 int i
, count
= ipa_get_cs_argument_count (args
);
472 if (e
->caller
->inlined_to
)
473 caller_parms_info
= IPA_NODE_REF (e
->caller
->inlined_to
);
475 caller_parms_info
= IPA_NODE_REF (e
->caller
);
476 callee_pi
= IPA_NODE_REF (callee
);
478 if (count
&& (info
->conds
|| known_vals_ptr
))
479 known_vals
.safe_grow_cleared (count
);
480 if (count
&& (info
->conds
|| known_aggs_ptr
))
481 known_aggs
.safe_grow_cleared (count
);
482 if (count
&& known_contexts_ptr
)
483 known_contexts_ptr
->safe_grow_cleared (count
);
486 for (i
= 0; i
< count
; i
++)
488 struct ipa_jump_func
*jf
= ipa_get_ith_jump_func (args
, i
);
489 tree cst
= ipa_value_from_jfunc (caller_parms_info
, jf
,
490 ipa_get_type (callee_pi
, i
));
492 if (!cst
&& e
->call_stmt
493 && i
< (int)gimple_call_num_args (e
->call_stmt
))
495 cst
= gimple_call_arg (e
->call_stmt
, i
);
496 if (!is_gimple_min_invariant (cst
))
501 gcc_checking_assert (TREE_CODE (cst
) != TREE_BINFO
);
502 if (known_vals
.exists ())
505 else if (inline_p
&& !es
->param
[i
].change_prob
)
506 known_vals
[i
] = error_mark_node
;
508 if (known_contexts_ptr
)
509 (*known_contexts_ptr
)[i
]
510 = ipa_context_from_jfunc (caller_parms_info
, e
, i
, jf
);
511 /* TODO: When IPA-CP starts propagating and merging aggregate jump
512 functions, use its knowledge of the caller too, just like the
513 scalar case above. */
514 known_aggs
[i
] = &jf
->agg
;
517 gcc_assert (callee
->thunk
.thunk_p
);
519 else if (e
->call_stmt
&& !e
->call_stmt_cannot_inline_p
520 && ((clause_ptr
&& info
->conds
) || known_vals_ptr
))
522 int i
, count
= (int)gimple_call_num_args (e
->call_stmt
);
524 if (count
&& (info
->conds
|| known_vals_ptr
))
525 known_vals
.safe_grow_cleared (count
);
526 for (i
= 0; i
< count
; i
++)
528 tree cst
= gimple_call_arg (e
->call_stmt
, i
);
529 if (!is_gimple_min_invariant (cst
))
536 evaluate_conditions_for_known_args (callee
, inline_p
,
537 known_vals
, known_aggs
, clause_ptr
,
541 *known_vals_ptr
= known_vals
;
543 known_vals
.release ();
546 *known_aggs_ptr
= known_aggs
;
548 known_aggs
.release ();
552 /* Allocate the function summary. */
555 ipa_fn_summary_alloc (void)
557 gcc_checking_assert (!ipa_fn_summaries
);
558 ipa_size_summaries
= new fast_function_summary
<ipa_size_summary
*, va_heap
>
560 ipa_fn_summaries
= ipa_fn_summary_t::create_ggc (symtab
);
561 ipa_call_summaries
= new ipa_call_summary_t (symtab
);
564 ipa_call_summary::~ipa_call_summary ()
567 edge_predicate_pool
.remove (predicate
);
572 ipa_fn_summary::~ipa_fn_summary ()
575 edge_predicate_pool
.remove (loop_iterations
);
577 edge_predicate_pool
.remove (loop_stride
);
579 vec_free (size_time_table
);
583 ipa_fn_summary_t::remove_callees (cgraph_node
*node
)
586 for (e
= node
->callees
; e
; e
= e
->next_callee
)
587 ipa_call_summaries
->remove (e
);
588 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
589 ipa_call_summaries
->remove (e
);
592 /* Same as remap_predicate_after_duplication but handle hint predicate *P.
593 Additionally care about allocating new memory slot for updated predicate
594 and set it to NULL when it becomes true or false (and thus uninteresting).
598 remap_hint_predicate_after_duplication (predicate
**p
,
599 clause_t possible_truths
)
601 predicate new_predicate
;
606 new_predicate
= (*p
)->remap_after_duplication (possible_truths
);
607 /* We do not want to free previous predicate; it is used by node origin. */
609 set_hint_predicate (p
, new_predicate
);
613 /* Hook that is called by cgraph.c when a node is duplicated. */
615 ipa_fn_summary_t::duplicate (cgraph_node
*src
,
618 ipa_fn_summary
*info
)
620 new (info
) ipa_fn_summary (*ipa_fn_summaries
->get (src
));
621 /* TODO: as an optimization, we may avoid copying conditions
622 that are known to be false or true. */
623 info
->conds
= vec_safe_copy (info
->conds
);
625 /* When there are any replacements in the function body, see if we can figure
626 out that something was optimized out. */
627 if (ipa_node_params_sum
&& dst
->clone
.tree_map
)
629 vec
<size_time_entry
, va_gc
> *entry
= info
->size_time_table
;
630 /* Use SRC parm info since it may not be copied yet. */
631 class ipa_node_params
*parms_info
= IPA_NODE_REF (src
);
632 vec
<tree
> known_vals
= vNULL
;
633 int count
= ipa_get_param_count (parms_info
);
635 clause_t possible_truths
;
636 predicate true_pred
= true;
638 int optimized_out_size
= 0;
639 bool inlined_to_p
= false;
640 struct cgraph_edge
*edge
, *next
;
642 info
->size_time_table
= 0;
643 known_vals
.safe_grow_cleared (count
);
644 for (i
= 0; i
< count
; i
++)
646 struct ipa_replace_map
*r
;
648 for (j
= 0; vec_safe_iterate (dst
->clone
.tree_map
, j
, &r
); j
++)
650 if (r
->parm_num
== i
)
652 known_vals
[i
] = r
->new_tree
;
657 evaluate_conditions_for_known_args (dst
, false,
661 /* We are going to specialize,
662 so ignore nonspec truths. */
664 known_vals
.release ();
666 info
->account_size_time (0, 0, true_pred
, true_pred
);
668 /* Remap size_time vectors.
669 Simplify the predicate by prunning out alternatives that are known
671 TODO: as on optimization, we can also eliminate conditions known
673 for (i
= 0; vec_safe_iterate (entry
, i
, &e
); i
++)
675 predicate new_exec_pred
;
676 predicate new_nonconst_pred
;
677 new_exec_pred
= e
->exec_predicate
.remap_after_duplication
679 new_nonconst_pred
= e
->nonconst_predicate
.remap_after_duplication
681 if (new_exec_pred
== false || new_nonconst_pred
== false)
682 optimized_out_size
+= e
->size
;
684 info
->account_size_time (e
->size
, e
->time
, new_exec_pred
,
688 /* Remap edge predicates with the same simplification as above.
689 Also copy constantness arrays. */
690 for (edge
= dst
->callees
; edge
; edge
= next
)
692 predicate new_predicate
;
693 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
694 next
= edge
->next_callee
;
696 if (!edge
->inline_failed
)
700 new_predicate
= es
->predicate
->remap_after_duplication
702 if (new_predicate
== false && *es
->predicate
!= false)
703 optimized_out_size
+= es
->call_stmt_size
* ipa_fn_summary::size_scale
;
704 edge_set_predicate (edge
, &new_predicate
);
707 /* Remap indirect edge predicates with the same simplificaiton as above.
708 Also copy constantness arrays. */
709 for (edge
= dst
->indirect_calls
; edge
; edge
= next
)
711 predicate new_predicate
;
712 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
713 next
= edge
->next_callee
;
715 gcc_checking_assert (edge
->inline_failed
);
718 new_predicate
= es
->predicate
->remap_after_duplication
720 if (new_predicate
== false && *es
->predicate
!= false)
721 optimized_out_size
+= es
->call_stmt_size
* ipa_fn_summary::size_scale
;
722 edge_set_predicate (edge
, &new_predicate
);
724 remap_hint_predicate_after_duplication (&info
->loop_iterations
,
726 remap_hint_predicate_after_duplication (&info
->loop_stride
,
729 /* If inliner or someone after inliner will ever start producing
730 non-trivial clones, we will get trouble with lack of information
731 about updating self sizes, because size vectors already contains
732 sizes of the calees. */
733 gcc_assert (!inlined_to_p
|| !optimized_out_size
);
737 info
->size_time_table
= vec_safe_copy (info
->size_time_table
);
738 if (info
->loop_iterations
)
740 predicate p
= *info
->loop_iterations
;
741 info
->loop_iterations
= NULL
;
742 set_hint_predicate (&info
->loop_iterations
, p
);
744 if (info
->loop_stride
)
746 predicate p
= *info
->loop_stride
;
747 info
->loop_stride
= NULL
;
748 set_hint_predicate (&info
->loop_stride
, p
);
751 if (!dst
->inlined_to
)
752 ipa_update_overall_fn_summary (dst
);
756 /* Hook that is called by cgraph.c when a node is duplicated. */
759 ipa_call_summary_t::duplicate (struct cgraph_edge
*src
,
760 struct cgraph_edge
*dst
,
761 class ipa_call_summary
*srcinfo
,
762 class ipa_call_summary
*info
)
764 new (info
) ipa_call_summary (*srcinfo
);
765 info
->predicate
= NULL
;
766 edge_set_predicate (dst
, srcinfo
->predicate
);
767 info
->param
= srcinfo
->param
.copy ();
768 if (!dst
->indirect_unknown_callee
&& src
->indirect_unknown_callee
)
770 info
->call_stmt_size
-= (eni_size_weights
.indirect_call_cost
771 - eni_size_weights
.call_cost
);
772 info
->call_stmt_time
-= (eni_time_weights
.indirect_call_cost
773 - eni_time_weights
.call_cost
);
777 /* Dump edge summaries associated to NODE and recursively to all clones.
781 dump_ipa_call_summary (FILE *f
, int indent
, struct cgraph_node
*node
,
782 class ipa_fn_summary
*info
)
784 struct cgraph_edge
*edge
;
785 for (edge
= node
->callees
; edge
; edge
= edge
->next_callee
)
787 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
788 struct cgraph_node
*callee
= edge
->callee
->ultimate_alias_target ();
792 "%*s%s/%i %s\n%*s freq:%4.2f",
793 indent
, "", callee
->name (), callee
->order
,
795 ? "inlined" : cgraph_inline_failed_string (edge
-> inline_failed
),
796 indent
, "", edge
->sreal_frequency ().to_double ());
799 fprintf (f
, " loop depth:%2i size:%2i time: %2i",
800 es
->loop_depth
, es
->call_stmt_size
, es
->call_stmt_time
);
802 ipa_fn_summary
*s
= ipa_fn_summaries
->get (callee
);
803 ipa_size_summary
*ss
= ipa_size_summaries
->get (callee
);
805 fprintf (f
, " callee size:%2i stack:%2i",
806 (int) (ss
->size
/ ipa_fn_summary::size_scale
),
807 (int) s
->estimated_stack_size
);
809 if (es
&& es
->predicate
)
811 fprintf (f
, " predicate: ");
812 es
->predicate
->dump (f
, info
->conds
);
816 if (es
&& es
->param
.exists ())
817 for (i
= 0; i
< (int) es
->param
.length (); i
++)
819 int prob
= es
->param
[i
].change_prob
;
822 fprintf (f
, "%*s op%i is compile time invariant\n",
824 else if (prob
!= REG_BR_PROB_BASE
)
825 fprintf (f
, "%*s op%i change %f%% of time\n", indent
+ 2, "", i
,
826 prob
* 100.0 / REG_BR_PROB_BASE
);
828 if (!edge
->inline_failed
)
830 ipa_size_summary
*ss
= ipa_size_summaries
->get (callee
);
831 fprintf (f
, "%*sStack frame offset %i, callee self size %i\n",
833 (int) ipa_get_stack_frame_offset (callee
),
834 (int) ss
->estimated_self_stack_size
);
835 dump_ipa_call_summary (f
, indent
+ 2, callee
, info
);
838 for (edge
= node
->indirect_calls
; edge
; edge
= edge
->next_callee
)
840 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
841 fprintf (f
, "%*sindirect call loop depth:%2i freq:%4.2f size:%2i"
845 edge
->sreal_frequency ().to_double (), es
->call_stmt_size
,
849 fprintf (f
, "predicate: ");
850 es
->predicate
->dump (f
, info
->conds
);
859 ipa_dump_fn_summary (FILE *f
, struct cgraph_node
*node
)
861 if (node
->definition
)
863 class ipa_fn_summary
*s
= ipa_fn_summaries
->get (node
);
864 class ipa_size_summary
*ss
= ipa_size_summaries
->get (node
);
869 fprintf (f
, "IPA function summary for %s", node
->dump_name ());
870 if (DECL_DISREGARD_INLINE_LIMITS (node
->decl
))
871 fprintf (f
, " always_inline");
873 fprintf (f
, " inlinable");
874 if (s
->fp_expressions
)
875 fprintf (f
, " fp_expression");
876 fprintf (f
, "\n global time: %f\n", s
->time
.to_double ());
877 fprintf (f
, " self size: %i\n", ss
->self_size
);
878 fprintf (f
, " global size: %i\n", ss
->size
);
879 fprintf (f
, " min size: %i\n", s
->min_size
);
880 fprintf (f
, " self stack: %i\n",
881 (int) ss
->estimated_self_stack_size
);
882 fprintf (f
, " global stack: %i\n", (int) s
->estimated_stack_size
);
884 fprintf (f
, " estimated growth:%i\n", (int) s
->growth
);
886 fprintf (f
, " In SCC: %i\n", (int) s
->scc_no
);
887 for (i
= 0; vec_safe_iterate (s
->size_time_table
, i
, &e
); i
++)
889 fprintf (f
, " size:%f, time:%f",
890 (double) e
->size
/ ipa_fn_summary::size_scale
,
891 e
->time
.to_double ());
892 if (e
->exec_predicate
!= true)
894 fprintf (f
, ", executed if:");
895 e
->exec_predicate
.dump (f
, s
->conds
, 0);
897 if (e
->exec_predicate
!= e
->nonconst_predicate
)
899 fprintf (f
, ", nonconst if:");
900 e
->nonconst_predicate
.dump (f
, s
->conds
, 0);
904 if (s
->loop_iterations
)
906 fprintf (f
, " loop iterations:");
907 s
->loop_iterations
->dump (f
, s
->conds
);
911 fprintf (f
, " loop stride:");
912 s
->loop_stride
->dump (f
, s
->conds
);
914 fprintf (f
, " calls:\n");
915 dump_ipa_call_summary (f
, 4, node
, s
);
919 fprintf (f
, "IPA summary for %s is missing.\n", node
->dump_name ());
924 ipa_debug_fn_summary (struct cgraph_node
*node
)
926 ipa_dump_fn_summary (stderr
, node
);
930 ipa_dump_fn_summaries (FILE *f
)
932 struct cgraph_node
*node
;
934 FOR_EACH_DEFINED_FUNCTION (node
)
935 if (!node
->inlined_to
)
936 ipa_dump_fn_summary (f
, node
);
939 /* Callback of walk_aliased_vdefs. Flags that it has been invoked to the
940 boolean variable pointed to by DATA. */
943 mark_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef ATTRIBUTE_UNUSED
,
946 bool *b
= (bool *) data
;
951 /* If OP refers to value of function parameter, return the corresponding
952 parameter. If non-NULL, the size of the memory load (or the SSA_NAME of the
953 PARM_DECL) will be stored to *SIZE_P in that case too. */
956 unmodified_parm_1 (ipa_func_body_info
*fbi
, gimple
*stmt
, tree op
,
959 /* SSA_NAME referring to parm default def? */
960 if (TREE_CODE (op
) == SSA_NAME
961 && SSA_NAME_IS_DEFAULT_DEF (op
)
962 && TREE_CODE (SSA_NAME_VAR (op
)) == PARM_DECL
)
965 *size_p
= tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op
)));
966 return SSA_NAME_VAR (op
);
968 /* Non-SSA parm reference? */
969 if (TREE_CODE (op
) == PARM_DECL
)
971 bool modified
= false;
974 ao_ref_init (&refd
, op
);
975 int walked
= walk_aliased_vdefs (&refd
, gimple_vuse (stmt
),
976 mark_modified
, &modified
, NULL
, NULL
,
977 fbi
->aa_walk_budget
+ 1);
980 fbi
->aa_walk_budget
= 0;
986 *size_p
= tree_to_poly_int64 (TYPE_SIZE (TREE_TYPE (op
)));
993 /* If OP refers to value of function parameter, return the corresponding
994 parameter. Also traverse chains of SSA register assignments. If non-NULL,
995 the size of the memory load (or the SSA_NAME of the PARM_DECL) will be
996 stored to *SIZE_P in that case too. */
999 unmodified_parm (ipa_func_body_info
*fbi
, gimple
*stmt
, tree op
,
1002 tree res
= unmodified_parm_1 (fbi
, stmt
, op
, size_p
);
1006 if (TREE_CODE (op
) == SSA_NAME
1007 && !SSA_NAME_IS_DEFAULT_DEF (op
)
1008 && gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1009 return unmodified_parm (fbi
, SSA_NAME_DEF_STMT (op
),
1010 gimple_assign_rhs1 (SSA_NAME_DEF_STMT (op
)),
1015 /* If OP refers to a value of a function parameter or value loaded from an
1016 aggregate passed to a parameter (either by value or reference), return TRUE
1017 and store the number of the parameter to *INDEX_P, the access size into
1018 *SIZE_P, and information whether and how it has been loaded from an
1019 aggregate into *AGGPOS. INFO describes the function parameters, STMT is the
1020 statement in which OP is used or loaded. */
1023 unmodified_parm_or_parm_agg_item (struct ipa_func_body_info
*fbi
,
1024 gimple
*stmt
, tree op
, int *index_p
,
1026 struct agg_position_info
*aggpos
)
1028 tree res
= unmodified_parm_1 (fbi
, stmt
, op
, size_p
);
1030 gcc_checking_assert (aggpos
);
1033 *index_p
= ipa_get_param_decl_index (fbi
->info
, res
);
1036 aggpos
->agg_contents
= false;
1037 aggpos
->by_ref
= false;
1041 if (TREE_CODE (op
) == SSA_NAME
)
1043 if (SSA_NAME_IS_DEFAULT_DEF (op
)
1044 || !gimple_assign_single_p (SSA_NAME_DEF_STMT (op
)))
1046 stmt
= SSA_NAME_DEF_STMT (op
);
1047 op
= gimple_assign_rhs1 (stmt
);
1048 if (!REFERENCE_CLASS_P (op
))
1049 return unmodified_parm_or_parm_agg_item (fbi
, stmt
, op
, index_p
, size_p
,
1053 aggpos
->agg_contents
= true;
1054 return ipa_load_from_parm_agg (fbi
, fbi
->info
->descriptors
,
1055 stmt
, op
, index_p
, &aggpos
->offset
,
1056 size_p
, &aggpos
->by_ref
);
1059 /* See if statement might disappear after inlining.
1060 0 - means not eliminated
1061 1 - half of statements goes away
1062 2 - for sure it is eliminated.
1063 We are not terribly sophisticated, basically looking for simple abstraction
1064 penalty wrappers. */
1067 eliminated_by_inlining_prob (ipa_func_body_info
*fbi
, gimple
*stmt
)
1069 enum gimple_code code
= gimple_code (stmt
);
1070 enum tree_code rhs_code
;
1080 if (gimple_num_ops (stmt
) != 2)
1083 rhs_code
= gimple_assign_rhs_code (stmt
);
1085 /* Casts of parameters, loads from parameters passed by reference
1086 and stores to return value or parameters are often free after
1087 inlining dua to SRA and further combining.
1088 Assume that half of statements goes away. */
1089 if (CONVERT_EXPR_CODE_P (rhs_code
)
1090 || rhs_code
== VIEW_CONVERT_EXPR
1091 || rhs_code
== ADDR_EXPR
1092 || gimple_assign_rhs_class (stmt
) == GIMPLE_SINGLE_RHS
)
1094 tree rhs
= gimple_assign_rhs1 (stmt
);
1095 tree lhs
= gimple_assign_lhs (stmt
);
1096 tree inner_rhs
= get_base_address (rhs
);
1097 tree inner_lhs
= get_base_address (lhs
);
1098 bool rhs_free
= false;
1099 bool lhs_free
= false;
1106 /* Reads of parameter are expected to be free. */
1107 if (unmodified_parm (fbi
, stmt
, inner_rhs
, NULL
))
1109 /* Match expressions of form &this->field. Those will most likely
1110 combine with something upstream after inlining. */
1111 else if (TREE_CODE (inner_rhs
) == ADDR_EXPR
)
1113 tree op
= get_base_address (TREE_OPERAND (inner_rhs
, 0));
1114 if (TREE_CODE (op
) == PARM_DECL
)
1116 else if (TREE_CODE (op
) == MEM_REF
1117 && unmodified_parm (fbi
, stmt
, TREE_OPERAND (op
, 0),
1122 /* When parameter is not SSA register because its address is taken
1123 and it is just copied into one, the statement will be completely
1124 free after inlining (we will copy propagate backward). */
1125 if (rhs_free
&& is_gimple_reg (lhs
))
1128 /* Reads of parameters passed by reference
1129 expected to be free (i.e. optimized out after inlining). */
1130 if (TREE_CODE (inner_rhs
) == MEM_REF
1131 && unmodified_parm (fbi
, stmt
, TREE_OPERAND (inner_rhs
, 0), NULL
))
1134 /* Copying parameter passed by reference into gimple register is
1135 probably also going to copy propagate, but we can't be quite
1137 if (rhs_free
&& is_gimple_reg (lhs
))
1140 /* Writes to parameters, parameters passed by value and return value
1141 (either dirrectly or passed via invisible reference) are free.
1143 TODO: We ought to handle testcase like
1144 struct a {int a,b;};
1146 retrurnsturct (void)
1152 This translate into:
1167 For that we either need to copy ipa-split logic detecting writes
1169 if (TREE_CODE (inner_lhs
) == PARM_DECL
1170 || TREE_CODE (inner_lhs
) == RESULT_DECL
1171 || (TREE_CODE (inner_lhs
) == MEM_REF
1172 && (unmodified_parm (fbi
, stmt
, TREE_OPERAND (inner_lhs
, 0),
1174 || (TREE_CODE (TREE_OPERAND (inner_lhs
, 0)) == SSA_NAME
1175 && SSA_NAME_VAR (TREE_OPERAND (inner_lhs
, 0))
1176 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND
1178 0))) == RESULT_DECL
))))
1181 && (is_gimple_reg (rhs
) || is_gimple_min_invariant (rhs
)))
1183 if (lhs_free
&& rhs_free
)
1192 /* Analyze EXPR if it represents a series of simple operations performed on
1193 a function parameter and return true if so. FBI, STMT, EXPR, INDEX_P and
1194 AGGPOS have the same meaning like in unmodified_parm_or_parm_agg_item.
1195 Type of the parameter or load from an aggregate via the parameter is
1196 stored in *TYPE_P. Operations on the parameter are recorded to
1197 PARAM_OPS_P if it is not NULL. */
1200 decompose_param_expr (struct ipa_func_body_info
*fbi
,
1201 gimple
*stmt
, tree expr
,
1202 int *index_p
, tree
*type_p
,
1203 struct agg_position_info
*aggpos
,
1204 expr_eval_ops
*param_ops_p
= NULL
)
1206 int op_limit
= param_ipa_max_param_expr_ops
;
1210 *param_ops_p
= NULL
;
1214 expr_eval_op eval_op
;
1216 unsigned cst_count
= 0;
1218 if (unmodified_parm_or_parm_agg_item (fbi
, stmt
, expr
, index_p
, NULL
,
1221 tree type
= TREE_TYPE (expr
);
1223 if (aggpos
->agg_contents
)
1225 /* Stop if containing bit-field. */
1226 if (TREE_CODE (expr
) == BIT_FIELD_REF
1227 || contains_bitfld_component_ref_p (expr
))
1235 if (TREE_CODE (expr
) != SSA_NAME
|| SSA_NAME_IS_DEFAULT_DEF (expr
))
1238 if (!is_gimple_assign (stmt
= SSA_NAME_DEF_STMT (expr
)))
1241 switch (gimple_assign_rhs_class (stmt
))
1243 case GIMPLE_SINGLE_RHS
:
1244 expr
= gimple_assign_rhs1 (stmt
);
1247 case GIMPLE_UNARY_RHS
:
1251 case GIMPLE_BINARY_RHS
:
1255 case GIMPLE_TERNARY_RHS
:
1263 /* Stop if expression is too complex. */
1264 if (op_count
++ == op_limit
)
1269 eval_op
.code
= gimple_assign_rhs_code (stmt
);
1270 eval_op
.type
= TREE_TYPE (gimple_assign_lhs (stmt
));
1271 eval_op
.val
[0] = NULL_TREE
;
1272 eval_op
.val
[1] = NULL_TREE
;
1276 for (unsigned i
= 0; i
< rhs_count
; i
++)
1278 tree op
= gimple_op (stmt
, i
+ 1);
1280 gcc_assert (op
&& !TYPE_P (op
));
1281 if (is_gimple_ip_invariant (op
))
1283 if (++cst_count
== rhs_count
)
1286 eval_op
.val
[cst_count
- 1] = op
;
1290 /* Found a non-constant operand, and record its index in rhs
1297 /* Found more than one non-constant operands. */
1303 vec_safe_insert (*param_ops_p
, 0, eval_op
);
1306 /* Failed to decompose, free resource and return. */
1309 vec_free (*param_ops_p
);
1314 /* If BB ends by a conditional we can turn into predicates, attach corresponding
1315 predicates to the CFG edges. */
1318 set_cond_stmt_execution_predicate (struct ipa_func_body_info
*fbi
,
1319 class ipa_fn_summary
*summary
,
1320 class ipa_node_params
*params_summary
,
1326 struct agg_position_info aggpos
;
1327 enum tree_code code
, inverted_code
;
1332 expr_eval_ops param_ops
;
1334 last
= last_stmt (bb
);
1335 if (!last
|| gimple_code (last
) != GIMPLE_COND
)
1337 if (!is_gimple_ip_invariant (gimple_cond_rhs (last
)))
1339 op
= gimple_cond_lhs (last
);
1341 if (decompose_param_expr (fbi
, last
, op
, &index
, ¶m_type
, &aggpos
,
1344 code
= gimple_cond_code (last
);
1345 inverted_code
= invert_tree_comparison (code
, HONOR_NANS (op
));
1347 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1349 enum tree_code this_code
= (e
->flags
& EDGE_TRUE_VALUE
1350 ? code
: inverted_code
);
1351 /* invert_tree_comparison will return ERROR_MARK on FP
1352 comparsions that are not EQ/NE instead of returning proper
1353 unordered one. Be sure it is not confused with NON_CONSTANT.
1355 And if the edge's target is the final block of diamond CFG graph
1356 of this conditional statement, we do not need to compute
1357 predicate for the edge because the final block's predicate must
1358 be at least as that of the first block of the statement. */
1359 if (this_code
!= ERROR_MARK
1360 && !dominated_by_p (CDI_POST_DOMINATORS
, bb
, e
->dest
))
1363 = add_condition (summary
, params_summary
, index
,
1364 param_type
, &aggpos
,
1365 this_code
, gimple_cond_rhs (last
), param_ops
);
1366 e
->aux
= edge_predicate_pool
.allocate ();
1367 *(predicate
*) e
->aux
= p
;
1370 vec_free (param_ops
);
1373 if (TREE_CODE (op
) != SSA_NAME
)
1376 if (builtin_constant_p (op))
1380 Here we can predicate nonconstant_code. We can't
1381 really handle constant_code since we have no predicate
1382 for this and also the constant code is not known to be
1383 optimized away when inliner doen't see operand is constant.
1384 Other optimizers might think otherwise. */
1385 if (gimple_cond_code (last
) != NE_EXPR
1386 || !integer_zerop (gimple_cond_rhs (last
)))
1388 set_stmt
= SSA_NAME_DEF_STMT (op
);
1389 if (!gimple_call_builtin_p (set_stmt
, BUILT_IN_CONSTANT_P
)
1390 || gimple_call_num_args (set_stmt
) != 1)
1392 op2
= gimple_call_arg (set_stmt
, 0);
1393 if (!decompose_param_expr (fbi
, set_stmt
, op2
, &index
, ¶m_type
, &aggpos
))
1395 FOR_EACH_EDGE (e
, ei
, bb
->succs
) if (e
->flags
& EDGE_FALSE_VALUE
)
1397 predicate p
= add_condition (summary
, params_summary
, index
,
1398 param_type
, &aggpos
,
1399 predicate::is_not_constant
, NULL_TREE
);
1400 e
->aux
= edge_predicate_pool
.allocate ();
1401 *(predicate
*) e
->aux
= p
;
1406 /* If BB ends by a switch we can turn into predicates, attach corresponding
1407 predicates to the CFG edges. */
1410 set_switch_stmt_execution_predicate (struct ipa_func_body_info
*fbi
,
1411 class ipa_fn_summary
*summary
,
1412 class ipa_node_params
*params_summary
,
1418 struct agg_position_info aggpos
;
1424 expr_eval_ops param_ops
;
1426 lastg
= last_stmt (bb
);
1427 if (!lastg
|| gimple_code (lastg
) != GIMPLE_SWITCH
)
1429 gswitch
*last
= as_a
<gswitch
*> (lastg
);
1430 op
= gimple_switch_index (last
);
1431 if (!decompose_param_expr (fbi
, last
, op
, &index
, ¶m_type
, &aggpos
,
1435 auto_vec
<std::pair
<tree
, tree
> > ranges
;
1436 tree type
= TREE_TYPE (op
);
1437 int bound_limit
= param_ipa_max_switch_predicate_bounds
;
1438 int bound_count
= 0;
1439 wide_int vr_wmin
, vr_wmax
;
1440 value_range_kind vr_type
= get_range_info (op
, &vr_wmin
, &vr_wmax
);
1442 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1444 e
->aux
= edge_predicate_pool
.allocate ();
1445 *(predicate
*) e
->aux
= false;
1448 e
= gimple_switch_edge (cfun
, last
, 0);
1449 /* Set BOUND_COUNT to maximum count to bypass computing predicate for
1450 default case if its target basic block is in convergence point of all
1451 switch cases, which can be determined by checking whether it
1452 post-dominates the switch statement. */
1453 if (dominated_by_p (CDI_POST_DOMINATORS
, bb
, e
->dest
))
1454 bound_count
= INT_MAX
;
1456 n
= gimple_switch_num_labels (last
);
1457 for (case_idx
= 1; case_idx
< n
; ++case_idx
)
1459 tree cl
= gimple_switch_label (last
, case_idx
);
1460 tree min
= CASE_LOW (cl
);
1461 tree max
= CASE_HIGH (cl
);
1464 e
= gimple_switch_edge (cfun
, last
, case_idx
);
1466 /* The case value might not have same type as switch expression,
1467 extend the value based on the expression type. */
1468 if (TREE_TYPE (min
) != type
)
1469 min
= wide_int_to_tree (type
, wi::to_wide (min
));
1473 else if (TREE_TYPE (max
) != type
)
1474 max
= wide_int_to_tree (type
, wi::to_wide (max
));
1476 /* The case's target basic block is in convergence point of all switch
1477 cases, its predicate should be at least as that of the switch
1479 if (dominated_by_p (CDI_POST_DOMINATORS
, bb
, e
->dest
))
1481 else if (min
== max
)
1482 p
= add_condition (summary
, params_summary
, index
, param_type
,
1483 &aggpos
, EQ_EXPR
, min
, param_ops
);
1487 p1
= add_condition (summary
, params_summary
, index
, param_type
,
1488 &aggpos
, GE_EXPR
, min
, param_ops
);
1489 p2
= add_condition (summary
, params_summary
,index
, param_type
,
1490 &aggpos
, LE_EXPR
, max
, param_ops
);
1493 *(class predicate
*) e
->aux
1494 = p
.or_with (summary
->conds
, *(class predicate
*) e
->aux
);
1496 /* If there are too many disjoint case ranges, predicate for default
1497 case might become too complicated. So add a limit here. */
1498 if (bound_count
> bound_limit
)
1501 bool new_range
= true;
1503 if (!ranges
.is_empty ())
1505 wide_int curr_wmin
= wi::to_wide (min
);
1506 wide_int last_wmax
= wi::to_wide (ranges
.last ().second
);
1508 /* Merge case ranges if they are continuous. */
1509 if (curr_wmin
== last_wmax
+ 1)
1511 else if (vr_type
== VR_ANTI_RANGE
)
1513 /* If two disjoint case ranges can be connected by anti-range
1514 of switch index, combine them to one range. */
1515 if (wi::lt_p (vr_wmax
, curr_wmin
- 1, TYPE_SIGN (type
)))
1516 vr_type
= VR_UNDEFINED
;
1517 else if (wi::le_p (vr_wmin
, last_wmax
+ 1, TYPE_SIGN (type
)))
1522 /* Create/extend a case range. And we count endpoints of range set,
1523 this number nearly equals to number of conditions that we will create
1524 for predicate of default case. */
1527 bound_count
+= (min
== max
) ? 1 : 2;
1528 ranges
.safe_push (std::make_pair (min
, max
));
1532 bound_count
+= (ranges
.last ().first
== ranges
.last ().second
);
1533 ranges
.last ().second
= max
;
1537 e
= gimple_switch_edge (cfun
, last
, 0);
1538 if (bound_count
> bound_limit
)
1540 *(class predicate
*) e
->aux
= true;
1541 vec_free (param_ops
);
1545 predicate p_seg
= true;
1546 predicate p_all
= false;
1548 if (vr_type
!= VR_RANGE
)
1550 vr_wmin
= wi::to_wide (TYPE_MIN_VALUE (type
));
1551 vr_wmax
= wi::to_wide (TYPE_MAX_VALUE (type
));
1554 /* Construct predicate to represent default range set that is negation of
1555 all case ranges. Case range is classified as containing single/non-single
1556 values. Suppose a piece of case ranges in the following.
1558 [D1...D2] [S1] ... [Sn] [D3...D4]
1560 To represent default case's range sets between two non-single value
1561 case ranges (From D2 to D3), we construct predicate as:
1563 D2 < x < D3 && x != S1 && ... && x != Sn
1565 for (size_t i
= 0; i
< ranges
.length (); i
++)
1567 tree min
= ranges
[i
].first
;
1568 tree max
= ranges
[i
].second
;
1571 p_seg
&= add_condition (summary
, params_summary
, index
,
1572 param_type
, &aggpos
, NE_EXPR
,
1576 /* Do not create sub-predicate for range that is beyond low bound
1578 if (wi::lt_p (vr_wmin
, wi::to_wide (min
), TYPE_SIGN (type
)))
1580 p_seg
&= add_condition (summary
, params_summary
, index
,
1581 param_type
, &aggpos
,
1582 LT_EXPR
, min
, param_ops
);
1583 p_all
= p_all
.or_with (summary
->conds
, p_seg
);
1586 /* Do not create sub-predicate for range that is beyond up bound
1588 if (wi::le_p (vr_wmax
, wi::to_wide (max
), TYPE_SIGN (type
)))
1594 p_seg
= add_condition (summary
, params_summary
, index
,
1595 param_type
, &aggpos
, GT_EXPR
,
1600 p_all
= p_all
.or_with (summary
->conds
, p_seg
);
1601 *(class predicate
*) e
->aux
1602 = p_all
.or_with (summary
->conds
, *(class predicate
*) e
->aux
);
1604 vec_free (param_ops
);
1608 /* For each BB in NODE attach to its AUX pointer predicate under
1609 which it is executable. */
1612 compute_bb_predicates (struct ipa_func_body_info
*fbi
,
1613 struct cgraph_node
*node
,
1614 class ipa_fn_summary
*summary
,
1615 class ipa_node_params
*params_summary
)
1617 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
1621 FOR_EACH_BB_FN (bb
, my_function
)
1623 set_cond_stmt_execution_predicate (fbi
, summary
, params_summary
, bb
);
1624 set_switch_stmt_execution_predicate (fbi
, summary
, params_summary
, bb
);
1627 /* Entry block is always executable. */
1628 ENTRY_BLOCK_PTR_FOR_FN (my_function
)->aux
1629 = edge_predicate_pool
.allocate ();
1630 *(predicate
*) ENTRY_BLOCK_PTR_FOR_FN (my_function
)->aux
= true;
1632 /* A simple dataflow propagation of predicates forward in the CFG.
1633 TODO: work in reverse postorder. */
1637 FOR_EACH_BB_FN (bb
, my_function
)
1639 predicate p
= false;
1642 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
1646 predicate this_bb_predicate
1647 = *(predicate
*) e
->src
->aux
;
1649 this_bb_predicate
&= (*(class predicate
*) e
->aux
);
1650 p
= p
.or_with (summary
->conds
, this_bb_predicate
);
1657 basic_block pdom_bb
;
1662 bb
->aux
= edge_predicate_pool
.allocate ();
1663 *((predicate
*) bb
->aux
) = p
;
1665 else if (p
!= *(predicate
*) bb
->aux
)
1667 /* This OR operation is needed to ensure monotonous data flow
1668 in the case we hit the limit on number of clauses and the
1669 and/or operations above give approximate answers. */
1670 p
= p
.or_with (summary
->conds
, *(predicate
*)bb
->aux
);
1671 if (p
!= *(predicate
*) bb
->aux
)
1674 *((predicate
*) bb
->aux
) = p
;
1678 /* For switch/if statement, we can OR-combine predicates of all
1679 its cases/branches to get predicate for basic block in their
1680 convergence point, but sometimes this will generate very
1681 complicated predicate. Actually, we can get simplified
1682 predicate in another way by using the fact that predicate
1683 for a basic block must also hold true for its post dominators.
1684 To be specific, basic block in convergence point of
1685 conditional statement should include predicate of the
1687 pdom_bb
= get_immediate_dominator (CDI_POST_DOMINATORS
, bb
);
1688 if (pdom_bb
== EXIT_BLOCK_PTR_FOR_FN (my_function
) || !pdom_bb
)
1690 else if (!pdom_bb
->aux
)
1693 pdom_bb
->aux
= edge_predicate_pool
.allocate ();
1694 *((predicate
*) pdom_bb
->aux
) = p
;
1696 else if (p
!= *(predicate
*) pdom_bb
->aux
)
1698 p
= p
.or_with (summary
->conds
, *(predicate
*)pdom_bb
->aux
);
1699 if (p
!= *(predicate
*) pdom_bb
->aux
)
1702 *((predicate
*) pdom_bb
->aux
) = p
;
1711 /* Return predicate specifying when the STMT might have result that is not
1712 a compile time constant. */
1715 will_be_nonconstant_expr_predicate (ipa_func_body_info
*fbi
,
1716 class ipa_fn_summary
*summary
,
1717 class ipa_node_params
*params_summary
,
1719 vec
<predicate
> nonconstant_names
)
1724 while (UNARY_CLASS_P (expr
))
1725 expr
= TREE_OPERAND (expr
, 0);
1727 parm
= unmodified_parm (fbi
, NULL
, expr
, NULL
);
1728 if (parm
&& (index
= ipa_get_param_decl_index (fbi
->info
, parm
)) >= 0)
1729 return add_condition (summary
, params_summary
, index
, TREE_TYPE (parm
), NULL
,
1730 predicate::changed
, NULL_TREE
);
1731 if (is_gimple_min_invariant (expr
))
1733 if (TREE_CODE (expr
) == SSA_NAME
)
1734 return nonconstant_names
[SSA_NAME_VERSION (expr
)];
1735 if (BINARY_CLASS_P (expr
) || COMPARISON_CLASS_P (expr
))
1738 = will_be_nonconstant_expr_predicate (fbi
, summary
,
1740 TREE_OPERAND (expr
, 0),
1746 = will_be_nonconstant_expr_predicate (fbi
, summary
,
1748 TREE_OPERAND (expr
, 1),
1750 return p1
.or_with (summary
->conds
, p2
);
1752 else if (TREE_CODE (expr
) == COND_EXPR
)
1755 = will_be_nonconstant_expr_predicate (fbi
, summary
,
1757 TREE_OPERAND (expr
, 0),
1763 = will_be_nonconstant_expr_predicate (fbi
, summary
,
1765 TREE_OPERAND (expr
, 1),
1769 p1
= p1
.or_with (summary
->conds
, p2
);
1770 p2
= will_be_nonconstant_expr_predicate (fbi
, summary
,
1772 TREE_OPERAND (expr
, 2),
1774 return p2
.or_with (summary
->conds
, p1
);
1776 else if (TREE_CODE (expr
) == CALL_EXPR
)
1787 /* Return predicate specifying when the STMT might have result that is not
1788 a compile time constant. */
1791 will_be_nonconstant_predicate (struct ipa_func_body_info
*fbi
,
1792 class ipa_fn_summary
*summary
,
1793 class ipa_node_params
*params_summary
,
1795 vec
<predicate
> nonconstant_names
)
1800 tree param_type
= NULL_TREE
;
1801 predicate op_non_const
;
1804 struct agg_position_info aggpos
;
1806 /* What statments might be optimized away
1807 when their arguments are constant. */
1808 if (gimple_code (stmt
) != GIMPLE_ASSIGN
1809 && gimple_code (stmt
) != GIMPLE_COND
1810 && gimple_code (stmt
) != GIMPLE_SWITCH
1811 && (gimple_code (stmt
) != GIMPLE_CALL
1812 || !(gimple_call_flags (stmt
) & ECF_CONST
)))
1815 /* Stores will stay anyway. */
1816 if (gimple_store_p (stmt
))
1819 is_load
= gimple_assign_load_p (stmt
);
1821 /* Loads can be optimized when the value is known. */
1824 tree op
= gimple_assign_rhs1 (stmt
);
1825 if (!decompose_param_expr (fbi
, stmt
, op
, &base_index
, ¶m_type
,
1832 /* See if we understand all operands before we start
1833 adding conditionals. */
1834 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1836 tree parm
= unmodified_parm (fbi
, stmt
, use
, NULL
);
1837 /* For arguments we can build a condition. */
1838 if (parm
&& ipa_get_param_decl_index (fbi
->info
, parm
) >= 0)
1840 if (TREE_CODE (use
) != SSA_NAME
)
1842 /* If we know when operand is constant,
1843 we still can say something useful. */
1844 if (nonconstant_names
[SSA_NAME_VERSION (use
)] != true)
1851 add_condition (summary
, params_summary
,
1852 base_index
, param_type
, &aggpos
,
1853 predicate::changed
, NULL_TREE
);
1855 op_non_const
= false;
1856 FOR_EACH_SSA_TREE_OPERAND (use
, stmt
, iter
, SSA_OP_USE
)
1858 tree parm
= unmodified_parm (fbi
, stmt
, use
, NULL
);
1861 if (parm
&& (index
= ipa_get_param_decl_index (fbi
->info
, parm
)) >= 0)
1863 if (index
!= base_index
)
1864 p
= add_condition (summary
, params_summary
, index
,
1865 TREE_TYPE (parm
), NULL
,
1866 predicate::changed
, NULL_TREE
);
1871 p
= nonconstant_names
[SSA_NAME_VERSION (use
)];
1872 op_non_const
= p
.or_with (summary
->conds
, op_non_const
);
1874 if ((gimple_code (stmt
) == GIMPLE_ASSIGN
|| gimple_code (stmt
) == GIMPLE_CALL
)
1875 && gimple_op (stmt
, 0)
1876 && TREE_CODE (gimple_op (stmt
, 0)) == SSA_NAME
)
1877 nonconstant_names
[SSA_NAME_VERSION (gimple_op (stmt
, 0))]
1879 return op_non_const
;
1882 struct record_modified_bb_info
1889 /* Value is initialized in INIT_BB and used in USE_BB. We want to copute
1890 probability how often it changes between USE_BB.
1891 INIT_BB->count/USE_BB->count is an estimate, but if INIT_BB
1892 is in different loop nest, we can do better.
1893 This is all just estimate. In theory we look for minimal cut separating
1894 INIT_BB and USE_BB, but we only want to anticipate loop invariant motion
1898 get_minimal_bb (basic_block init_bb
, basic_block use_bb
)
1900 class loop
*l
= find_common_loop (init_bb
->loop_father
, use_bb
->loop_father
);
1901 if (l
&& l
->header
->count
< init_bb
->count
)
1906 /* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
1907 set except for info->stmt. */
1910 record_modified (ao_ref
*ao ATTRIBUTE_UNUSED
, tree vdef
, void *data
)
1912 struct record_modified_bb_info
*info
=
1913 (struct record_modified_bb_info
*) data
;
1914 if (SSA_NAME_DEF_STMT (vdef
) == info
->stmt
)
1916 if (gimple_clobber_p (SSA_NAME_DEF_STMT (vdef
)))
1918 bitmap_set_bit (info
->bb_set
,
1919 SSA_NAME_IS_DEFAULT_DEF (vdef
)
1920 ? ENTRY_BLOCK_PTR_FOR_FN (cfun
)->index
1922 (gimple_bb (SSA_NAME_DEF_STMT (vdef
)),
1923 gimple_bb (info
->stmt
))->index
);
1926 fprintf (dump_file
, " Param ");
1927 print_generic_expr (dump_file
, info
->op
, TDF_SLIM
);
1928 fprintf (dump_file
, " changed at bb %i, minimal: %i stmt: ",
1929 gimple_bb (SSA_NAME_DEF_STMT (vdef
))->index
,
1931 (gimple_bb (SSA_NAME_DEF_STMT (vdef
)),
1932 gimple_bb (info
->stmt
))->index
);
1933 print_gimple_stmt (dump_file
, SSA_NAME_DEF_STMT (vdef
), 0);
1938 /* Return probability (based on REG_BR_PROB_BASE) that I-th parameter of STMT
1939 will change since last invocation of STMT.
1941 Value 0 is reserved for compile time invariants.
1942 For common parameters it is REG_BR_PROB_BASE. For loop invariants it
1943 ought to be REG_BR_PROB_BASE / estimated_iters. */
1946 param_change_prob (ipa_func_body_info
*fbi
, gimple
*stmt
, int i
)
1948 tree op
= gimple_call_arg (stmt
, i
);
1949 basic_block bb
= gimple_bb (stmt
);
1951 if (TREE_CODE (op
) == WITH_SIZE_EXPR
)
1952 op
= TREE_OPERAND (op
, 0);
1954 tree base
= get_base_address (op
);
1956 /* Global invariants never change. */
1957 if (is_gimple_min_invariant (base
))
1960 /* We would have to do non-trivial analysis to really work out what
1961 is the probability of value to change (i.e. when init statement
1962 is in a sibling loop of the call).
1964 We do an conservative estimate: when call is executed N times more often
1965 than the statement defining value, we take the frequency 1/N. */
1966 if (TREE_CODE (base
) == SSA_NAME
)
1968 profile_count init_count
;
1970 if (!bb
->count
.nonzero_p ())
1971 return REG_BR_PROB_BASE
;
1973 if (SSA_NAME_IS_DEFAULT_DEF (base
))
1974 init_count
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
;
1976 init_count
= get_minimal_bb
1977 (gimple_bb (SSA_NAME_DEF_STMT (base
)),
1978 gimple_bb (stmt
))->count
;
1980 if (init_count
< bb
->count
)
1981 return MAX ((init_count
.to_sreal_scale (bb
->count
)
1982 * REG_BR_PROB_BASE
).to_int (), 1);
1983 return REG_BR_PROB_BASE
;
1988 profile_count max
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
;
1989 struct record_modified_bb_info info
;
1990 tree init
= ctor_for_folding (base
);
1992 if (init
!= error_mark_node
)
1994 if (!bb
->count
.nonzero_p ())
1995 return REG_BR_PROB_BASE
;
1998 fprintf (dump_file
, " Analyzing param change probability of ");
1999 print_generic_expr (dump_file
, op
, TDF_SLIM
);
2000 fprintf (dump_file
, "\n");
2002 ao_ref_init (&refd
, op
);
2005 info
.bb_set
= BITMAP_ALLOC (NULL
);
2007 = walk_aliased_vdefs (&refd
, gimple_vuse (stmt
), record_modified
, &info
,
2008 NULL
, NULL
, fbi
->aa_walk_budget
);
2009 if (walked
< 0 || bitmap_bit_p (info
.bb_set
, bb
->index
))
2014 fprintf (dump_file
, " Ran out of AA walking budget.\n");
2016 fprintf (dump_file
, " Set in same BB as used.\n");
2018 BITMAP_FREE (info
.bb_set
);
2019 return REG_BR_PROB_BASE
;
2024 /* Lookup the most frequent update of the value and believe that
2025 it dominates all the other; precise analysis here is difficult. */
2026 EXECUTE_IF_SET_IN_BITMAP (info
.bb_set
, 0, index
, bi
)
2027 max
= max
.max (BASIC_BLOCK_FOR_FN (cfun
, index
)->count
);
2030 fprintf (dump_file
, " Set with count ");
2031 max
.dump (dump_file
);
2032 fprintf (dump_file
, " and used with count ");
2033 bb
->count
.dump (dump_file
);
2034 fprintf (dump_file
, " freq %f\n",
2035 max
.to_sreal_scale (bb
->count
).to_double ());
2038 BITMAP_FREE (info
.bb_set
);
2039 if (max
< bb
->count
)
2040 return MAX ((max
.to_sreal_scale (bb
->count
)
2041 * REG_BR_PROB_BASE
).to_int (), 1);
2042 return REG_BR_PROB_BASE
;
2046 /* Find whether a basic block BB is the final block of a (half) diamond CFG
2047 sub-graph and if the predicate the condition depends on is known. If so,
2048 return true and store the pointer the predicate in *P. */
2051 phi_result_unknown_predicate (ipa_func_body_info
*fbi
,
2052 ipa_fn_summary
*summary
,
2053 class ipa_node_params
*params_summary
,
2056 vec
<predicate
> nonconstant_names
)
2060 basic_block first_bb
= NULL
;
2063 if (single_pred_p (bb
))
2069 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
2071 if (single_succ_p (e
->src
))
2073 if (!single_pred_p (e
->src
))
2076 first_bb
= single_pred (e
->src
);
2077 else if (single_pred (e
->src
) != first_bb
)
2084 else if (e
->src
!= first_bb
)
2092 stmt
= last_stmt (first_bb
);
2094 || gimple_code (stmt
) != GIMPLE_COND
2095 || !is_gimple_ip_invariant (gimple_cond_rhs (stmt
)))
2098 *p
= will_be_nonconstant_expr_predicate (fbi
, summary
, params_summary
,
2099 gimple_cond_lhs (stmt
),
2107 /* Given a PHI statement in a function described by inline properties SUMMARY
2108 and *P being the predicate describing whether the selected PHI argument is
2109 known, store a predicate for the result of the PHI statement into
2110 NONCONSTANT_NAMES, if possible. */
2113 predicate_for_phi_result (class ipa_fn_summary
*summary
, gphi
*phi
,
2115 vec
<predicate
> nonconstant_names
)
2119 for (i
= 0; i
< gimple_phi_num_args (phi
); i
++)
2121 tree arg
= gimple_phi_arg (phi
, i
)->def
;
2122 if (!is_gimple_min_invariant (arg
))
2124 gcc_assert (TREE_CODE (arg
) == SSA_NAME
);
2125 *p
= p
->or_with (summary
->conds
,
2126 nonconstant_names
[SSA_NAME_VERSION (arg
)]);
2132 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2134 fprintf (dump_file
, "\t\tphi predicate: ");
2135 p
->dump (dump_file
, summary
->conds
);
2137 nonconstant_names
[SSA_NAME_VERSION (gimple_phi_result (phi
))] = *p
;
2140 /* For a typical usage of __builtin_expect (a<b, 1), we
2141 may introduce an extra relation stmt:
2142 With the builtin, we have
2145 t3 = __builtin_expect (t2, 1);
2148 Without the builtin, we have
2151 This affects the size/time estimation and may have
2152 an impact on the earlier inlining.
2153 Here find this pattern and fix it up later. */
2156 find_foldable_builtin_expect (basic_block bb
)
2158 gimple_stmt_iterator bsi
;
2160 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
2162 gimple
*stmt
= gsi_stmt (bsi
);
2163 if (gimple_call_builtin_p (stmt
, BUILT_IN_EXPECT
)
2164 || gimple_call_builtin_p (stmt
, BUILT_IN_EXPECT_WITH_PROBABILITY
)
2165 || gimple_call_internal_p (stmt
, IFN_BUILTIN_EXPECT
))
2167 tree var
= gimple_call_lhs (stmt
);
2168 tree arg
= gimple_call_arg (stmt
, 0);
2169 use_operand_p use_p
;
2176 gcc_assert (TREE_CODE (var
) == SSA_NAME
);
2178 while (TREE_CODE (arg
) == SSA_NAME
)
2180 gimple
*stmt_tmp
= SSA_NAME_DEF_STMT (arg
);
2181 if (!is_gimple_assign (stmt_tmp
))
2183 switch (gimple_assign_rhs_code (stmt_tmp
))
2202 arg
= gimple_assign_rhs1 (stmt_tmp
);
2205 if (match
&& single_imm_use (var
, &use_p
, &use_stmt
)
2206 && gimple_code (use_stmt
) == GIMPLE_COND
)
2213 /* Return true when the basic blocks contains only clobbers followed by RESX.
2214 Such BBs are kept around to make removal of dead stores possible with
2215 presence of EH and will be optimized out by optimize_clobbers later in the
2218 NEED_EH is used to recurse in case the clobber has non-EH predecestors
2219 that can be clobber only, too.. When it is false, the RESX is not necessary
2220 on the end of basic block. */
2223 clobber_only_eh_bb_p (basic_block bb
, bool need_eh
= true)
2225 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
2231 if (gsi_end_p (gsi
))
2233 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_RESX
)
2237 else if (!single_succ_p (bb
))
2240 for (; !gsi_end_p (gsi
); gsi_prev (&gsi
))
2242 gimple
*stmt
= gsi_stmt (gsi
);
2243 if (is_gimple_debug (stmt
))
2245 if (gimple_clobber_p (stmt
))
2247 if (gimple_code (stmt
) == GIMPLE_LABEL
)
2252 /* See if all predecestors are either throws or clobber only BBs. */
2253 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
2254 if (!(e
->flags
& EDGE_EH
)
2255 && !clobber_only_eh_bb_p (e
->src
, false))
2261 /* Return true if STMT compute a floating point expression that may be affected
2262 by -ffast-math and similar flags. */
2265 fp_expression_p (gimple
*stmt
)
2270 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_DEF
|SSA_OP_USE
)
2271 if (FLOAT_TYPE_P (TREE_TYPE (op
)))
2276 /* Analyze function body for NODE.
2277 EARLY indicates run from early optimization pipeline. */
2280 analyze_function_body (struct cgraph_node
*node
, bool early
)
2282 sreal time
= param_uninlined_function_time
;
2283 /* Estimate static overhead for function prologue/epilogue and alignment. */
2284 int size
= param_uninlined_function_insns
;
2285 /* Benefits are scaled by probability of elimination that is in range
2288 struct function
*my_function
= DECL_STRUCT_FUNCTION (node
->decl
);
2290 class ipa_fn_summary
*info
= ipa_fn_summaries
->get_create (node
);
2291 class ipa_node_params
*params_summary
= early
? NULL
: IPA_NODE_REF (node
);
2292 predicate bb_predicate
;
2293 struct ipa_func_body_info fbi
;
2294 vec
<predicate
> nonconstant_names
= vNULL
;
2297 gimple
*fix_builtin_expect_stmt
;
2299 gcc_assert (my_function
&& my_function
->cfg
);
2300 gcc_assert (cfun
== my_function
);
2302 memset(&fbi
, 0, sizeof(fbi
));
2303 vec_free (info
->conds
);
2305 vec_free (info
->size_time_table
);
2306 info
->size_time_table
= NULL
;
2308 /* When optimizing and analyzing for IPA inliner, initialize loop optimizer
2309 so we can produce proper inline hints.
2311 When optimizing and analyzing for early inliner, initialize node params
2312 so we can produce correct BB predicates. */
2314 if (opt_for_fn (node
->decl
, optimize
))
2316 calculate_dominance_info (CDI_DOMINATORS
);
2317 calculate_dominance_info (CDI_POST_DOMINATORS
);
2319 loop_optimizer_init (LOOPS_NORMAL
| LOOPS_HAVE_RECORDED_EXITS
);
2322 ipa_check_create_node_params ();
2323 ipa_initialize_node_params (node
);
2326 if (ipa_node_params_sum
)
2329 fbi
.info
= IPA_NODE_REF (node
);
2330 fbi
.bb_infos
= vNULL
;
2331 fbi
.bb_infos
.safe_grow_cleared (last_basic_block_for_fn (cfun
));
2332 fbi
.param_count
= count_formal_params (node
->decl
);
2333 fbi
.aa_walk_budget
= param_ipa_max_aa_steps
;
2335 nonconstant_names
.safe_grow_cleared
2336 (SSANAMES (my_function
)->length ());
2341 fprintf (dump_file
, "\nAnalyzing function body size: %s\n",
2344 /* When we run into maximal number of entries, we assign everything to the
2345 constant truth case. Be sure to have it in list. */
2346 bb_predicate
= true;
2347 info
->account_size_time (0, 0, bb_predicate
, bb_predicate
);
2349 bb_predicate
= predicate::not_inlined ();
2350 info
->account_size_time (param_uninlined_function_insns
2351 * ipa_fn_summary::size_scale
,
2352 param_uninlined_function_time
,
2357 compute_bb_predicates (&fbi
, node
, info
, params_summary
);
2358 order
= XNEWVEC (int, n_basic_blocks_for_fn (cfun
));
2359 nblocks
= pre_and_rev_post_order_compute (NULL
, order
, false);
2360 for (n
= 0; n
< nblocks
; n
++)
2362 bb
= BASIC_BLOCK_FOR_FN (cfun
, order
[n
]);
2363 freq
= bb
->count
.to_sreal_scale (ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
);
2364 if (clobber_only_eh_bb_p (bb
))
2366 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2367 fprintf (dump_file
, "\n Ignoring BB %i;"
2368 " it will be optimized away by cleanup_clobbers\n",
2373 /* TODO: Obviously predicates can be propagated down across CFG. */
2377 bb_predicate
= *(predicate
*) bb
->aux
;
2379 bb_predicate
= false;
2382 bb_predicate
= true;
2384 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2386 fprintf (dump_file
, "\n BB %i predicate:", bb
->index
);
2387 bb_predicate
.dump (dump_file
, info
->conds
);
2390 if (fbi
.info
&& nonconstant_names
.exists ())
2392 predicate phi_predicate
;
2393 bool first_phi
= true;
2395 for (gphi_iterator bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
);
2399 && !phi_result_unknown_predicate (&fbi
, info
,
2406 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2408 fprintf (dump_file
, " ");
2409 print_gimple_stmt (dump_file
, gsi_stmt (bsi
), 0);
2411 predicate_for_phi_result (info
, bsi
.phi (), &phi_predicate
,
2416 fix_builtin_expect_stmt
= find_foldable_builtin_expect (bb
);
2418 for (gimple_stmt_iterator bsi
= gsi_start_nondebug_bb (bb
);
2419 !gsi_end_p (bsi
); gsi_next_nondebug (&bsi
))
2421 gimple
*stmt
= gsi_stmt (bsi
);
2422 int this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
2423 int this_time
= estimate_num_insns (stmt
, &eni_time_weights
);
2425 predicate will_be_nonconstant
;
2427 /* This relation stmt should be folded after we remove
2428 buildin_expect call. Adjust the cost here. */
2429 if (stmt
== fix_builtin_expect_stmt
)
2435 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2437 fprintf (dump_file
, " ");
2438 print_gimple_stmt (dump_file
, stmt
, 0);
2439 fprintf (dump_file
, "\t\tfreq:%3.2f size:%3i time:%3i\n",
2440 freq
.to_double (), this_size
,
2444 if (is_gimple_call (stmt
)
2445 && !gimple_call_internal_p (stmt
))
2447 struct cgraph_edge
*edge
= node
->get_edge (stmt
);
2448 ipa_call_summary
*es
= ipa_call_summaries
->get_create (edge
);
2450 /* Special case: results of BUILT_IN_CONSTANT_P will be always
2451 resolved as constant. We however don't want to optimize
2452 out the cgraph edges. */
2453 if (nonconstant_names
.exists ()
2454 && gimple_call_builtin_p (stmt
, BUILT_IN_CONSTANT_P
)
2455 && gimple_call_lhs (stmt
)
2456 && TREE_CODE (gimple_call_lhs (stmt
)) == SSA_NAME
)
2458 predicate false_p
= false;
2459 nonconstant_names
[SSA_NAME_VERSION (gimple_call_lhs (stmt
))]
2462 if (ipa_node_params_sum
)
2464 int count
= gimple_call_num_args (stmt
);
2468 es
->param
.safe_grow_cleared (count
);
2469 for (i
= 0; i
< count
; i
++)
2471 int prob
= param_change_prob (&fbi
, stmt
, i
);
2472 gcc_assert (prob
>= 0 && prob
<= REG_BR_PROB_BASE
);
2473 es
->param
[i
].change_prob
= prob
;
2477 es
->call_stmt_size
= this_size
;
2478 es
->call_stmt_time
= this_time
;
2479 es
->loop_depth
= bb_loop_depth (bb
);
2480 edge_set_predicate (edge
, &bb_predicate
);
2481 if (edge
->speculative
)
2483 cgraph_edge
*direct
, *indirect
;
2485 edge
->speculative_call_info (direct
, indirect
, ref
);
2486 gcc_assert (direct
== edge
);
2487 ipa_call_summary
*es2
2488 = ipa_call_summaries
->get_create (indirect
);
2489 ipa_call_summaries
->duplicate (edge
, indirect
,
2494 /* TODO: When conditional jump or swithc is known to be constant, but
2495 we did not translate it into the predicates, we really can account
2496 just maximum of the possible paths. */
2499 = will_be_nonconstant_predicate (&fbi
, info
, params_summary
,
2500 stmt
, nonconstant_names
);
2502 will_be_nonconstant
= true;
2503 if (this_time
|| this_size
)
2505 sreal final_time
= (sreal
)this_time
* freq
;
2507 prob
= eliminated_by_inlining_prob (&fbi
, stmt
);
2508 if (prob
== 1 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2510 "\t\t50%% will be eliminated by inlining\n");
2511 if (prob
== 2 && dump_file
&& (dump_flags
& TDF_DETAILS
))
2512 fprintf (dump_file
, "\t\tWill be eliminated by inlining\n");
2514 class predicate p
= bb_predicate
& will_be_nonconstant
;
2516 /* We can ignore statement when we proved it is never going
2517 to happen, but we cannot do that for call statements
2518 because edges are accounted specially. */
2520 if (*(is_gimple_call (stmt
) ? &bb_predicate
: &p
) != false)
2526 /* We account everything but the calls. Calls have their own
2527 size/time info attached to cgraph edges. This is necessary
2528 in order to make the cost disappear after inlining. */
2529 if (!is_gimple_call (stmt
))
2533 predicate ip
= bb_predicate
& predicate::not_inlined ();
2534 info
->account_size_time (this_size
* prob
,
2535 (final_time
* prob
) / 2, ip
,
2539 info
->account_size_time (this_size
* (2 - prob
),
2540 (final_time
* (2 - prob
) / 2),
2545 if (!info
->fp_expressions
&& fp_expression_p (stmt
))
2547 info
->fp_expressions
= true;
2549 fprintf (dump_file
, " fp_expression set\n");
2553 /* Account cost of address calculations in the statements. */
2554 for (unsigned int i
= 0; i
< gimple_num_ops (stmt
); i
++)
2556 for (tree op
= gimple_op (stmt
, i
);
2557 op
&& handled_component_p (op
);
2558 op
= TREE_OPERAND (op
, 0))
2559 if ((TREE_CODE (op
) == ARRAY_REF
2560 || TREE_CODE (op
) == ARRAY_RANGE_REF
)
2561 && TREE_CODE (TREE_OPERAND (op
, 1)) == SSA_NAME
)
2563 predicate p
= bb_predicate
;
2565 p
= p
& will_be_nonconstant_expr_predicate
2566 (&fbi
, info
, params_summary
,
2567 TREE_OPERAND (op
, 1),
2575 "\t\tAccounting address calculation.\n");
2576 info
->account_size_time (ipa_fn_summary::size_scale
,
2588 if (nonconstant_names
.exists () && !early
)
2591 predicate loop_iterations
= true;
2592 predicate loop_stride
= true;
2594 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2595 flow_loops_dump (dump_file
, NULL
, 0);
2597 FOR_EACH_LOOP (loop
, 0)
2602 class tree_niter_desc niter_desc
;
2603 bb_predicate
= *(predicate
*) loop
->header
->aux
;
2605 exits
= get_loop_exit_edges (loop
);
2606 FOR_EACH_VEC_ELT (exits
, j
, ex
)
2607 if (number_of_iterations_exit (loop
, ex
, &niter_desc
, false)
2608 && !is_gimple_min_invariant (niter_desc
.niter
))
2610 predicate will_be_nonconstant
2611 = will_be_nonconstant_expr_predicate (&fbi
, info
,
2615 if (will_be_nonconstant
!= true)
2616 will_be_nonconstant
= bb_predicate
& will_be_nonconstant
;
2617 if (will_be_nonconstant
!= true
2618 && will_be_nonconstant
!= false)
2619 /* This is slightly inprecise. We may want to represent each
2620 loop with independent predicate. */
2621 loop_iterations
&= will_be_nonconstant
;
2626 /* To avoid quadratic behavior we analyze stride predicates only
2627 with respect to the containing loop. Thus we simply iterate
2628 over all defs in the outermost loop body. */
2629 for (loop
= loops_for_fn (cfun
)->tree_root
->inner
;
2630 loop
!= NULL
; loop
= loop
->next
)
2632 basic_block
*body
= get_loop_body (loop
);
2633 for (unsigned i
= 0; i
< loop
->num_nodes
; i
++)
2635 gimple_stmt_iterator gsi
;
2636 bb_predicate
= *(predicate
*) body
[i
]->aux
;
2637 for (gsi
= gsi_start_bb (body
[i
]); !gsi_end_p (gsi
);
2640 gimple
*stmt
= gsi_stmt (gsi
);
2642 if (!is_gimple_assign (stmt
))
2645 tree def
= gimple_assign_lhs (stmt
);
2646 if (TREE_CODE (def
) != SSA_NAME
)
2650 if (!simple_iv (loop_containing_stmt (stmt
),
2651 loop_containing_stmt (stmt
),
2653 || is_gimple_min_invariant (iv
.step
))
2656 predicate will_be_nonconstant
2657 = will_be_nonconstant_expr_predicate (&fbi
, info
,
2661 if (will_be_nonconstant
!= true)
2662 will_be_nonconstant
= bb_predicate
& will_be_nonconstant
;
2663 if (will_be_nonconstant
!= true
2664 && will_be_nonconstant
!= false)
2665 /* This is slightly inprecise. We may want to represent
2666 each loop with independent predicate. */
2667 loop_stride
= loop_stride
& will_be_nonconstant
;
2672 ipa_fn_summary
*s
= ipa_fn_summaries
->get (node
);
2673 set_hint_predicate (&s
->loop_iterations
, loop_iterations
);
2674 set_hint_predicate (&s
->loop_stride
, loop_stride
);
2677 FOR_ALL_BB_FN (bb
, my_function
)
2683 edge_predicate_pool
.remove ((predicate
*)bb
->aux
);
2685 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2688 edge_predicate_pool
.remove ((predicate
*) e
->aux
);
2692 ipa_fn_summary
*s
= ipa_fn_summaries
->get (node
);
2693 ipa_size_summary
*ss
= ipa_size_summaries
->get (node
);
2695 ss
->self_size
= size
;
2696 nonconstant_names
.release ();
2697 ipa_release_body_info (&fbi
);
2698 if (opt_for_fn (node
->decl
, optimize
))
2701 loop_optimizer_finalize ();
2702 else if (!ipa_edge_args_sum
)
2703 ipa_free_all_node_params ();
2704 free_dominance_info (CDI_DOMINATORS
);
2705 free_dominance_info (CDI_POST_DOMINATORS
);
2709 fprintf (dump_file
, "\n");
2710 ipa_dump_fn_summary (dump_file
, node
);
2715 /* Compute function summary.
2716 EARLY is true when we compute parameters during early opts. */
2719 compute_fn_summary (struct cgraph_node
*node
, bool early
)
2721 HOST_WIDE_INT self_stack_size
;
2722 struct cgraph_edge
*e
;
2724 gcc_assert (!node
->inlined_to
);
2726 if (!ipa_fn_summaries
)
2727 ipa_fn_summary_alloc ();
2729 /* Create a new ipa_fn_summary. */
2730 ((ipa_fn_summary_t
*)ipa_fn_summaries
)->remove_callees (node
);
2731 ipa_fn_summaries
->remove (node
);
2732 class ipa_fn_summary
*info
= ipa_fn_summaries
->get_create (node
);
2733 class ipa_size_summary
*size_info
= ipa_size_summaries
->get_create (node
);
2735 /* Estimate the stack size for the function if we're optimizing. */
2736 self_stack_size
= optimize
&& !node
->thunk
.thunk_p
2737 ? estimated_stack_frame_size (node
) : 0;
2738 size_info
->estimated_self_stack_size
= self_stack_size
;
2739 info
->estimated_stack_size
= self_stack_size
;
2741 if (node
->thunk
.thunk_p
)
2743 ipa_call_summary
*es
= ipa_call_summaries
->get_create (node
->callees
);
2746 node
->can_change_signature
= false;
2747 es
->call_stmt_size
= eni_size_weights
.call_cost
;
2748 es
->call_stmt_time
= eni_time_weights
.call_cost
;
2749 info
->account_size_time (ipa_fn_summary::size_scale
2750 * param_uninlined_function_thunk_insns
,
2751 param_uninlined_function_thunk_time
, t
, t
);
2752 t
= predicate::not_inlined ();
2753 info
->account_size_time (2 * ipa_fn_summary::size_scale
, 0, t
, t
);
2754 ipa_update_overall_fn_summary (node
);
2755 size_info
->self_size
= size_info
->size
;
2756 if (stdarg_p (TREE_TYPE (node
->decl
)))
2758 info
->inlinable
= false;
2759 node
->callees
->inline_failed
= CIF_VARIADIC_THUNK
;
2762 info
->inlinable
= true;
2766 /* Even is_gimple_min_invariant rely on current_function_decl. */
2767 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
2769 /* Can this function be inlined at all? */
2770 if (!opt_for_fn (node
->decl
, optimize
)
2771 && !lookup_attribute ("always_inline",
2772 DECL_ATTRIBUTES (node
->decl
)))
2773 info
->inlinable
= false;
2775 info
->inlinable
= tree_inlinable_function_p (node
->decl
);
2777 /* Type attributes can use parameter indices to describe them. */
2778 if (TYPE_ATTRIBUTES (TREE_TYPE (node
->decl
))
2779 /* Likewise for #pragma omp declare simd functions or functions
2780 with simd attribute. */
2781 || lookup_attribute ("omp declare simd",
2782 DECL_ATTRIBUTES (node
->decl
)))
2783 node
->can_change_signature
= false;
2786 /* Otherwise, inlinable functions always can change signature. */
2787 if (info
->inlinable
)
2788 node
->can_change_signature
= true;
2791 /* Functions calling builtin_apply cannot change signature. */
2792 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2794 tree
cdecl = e
->callee
->decl
;
2795 if (fndecl_built_in_p (cdecl, BUILT_IN_APPLY_ARGS
)
2796 || fndecl_built_in_p (cdecl, BUILT_IN_VA_START
))
2799 node
->can_change_signature
= !e
;
2802 analyze_function_body (node
, early
);
2805 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2806 if (e
->callee
->comdat_local_p ())
2808 node
->calls_comdat_local
= (e
!= NULL
);
2810 /* Inlining characteristics are maintained by the cgraph_mark_inline. */
2811 size_info
->size
= size_info
->self_size
;
2812 info
->estimated_stack_size
= size_info
->estimated_self_stack_size
;
2814 /* Code above should compute exactly the same result as
2815 ipa_update_overall_fn_summary but because computation happens in
2816 different order the roundoff errors result in slight changes. */
2817 ipa_update_overall_fn_summary (node
);
2818 /* In LTO mode we may have speculative edges set. */
2819 gcc_assert (in_lto_p
|| size_info
->size
== size_info
->self_size
);
2823 /* Compute parameters of functions used by inliner using
2824 current_function_decl. */
2827 compute_fn_summary_for_current (void)
2829 compute_fn_summary (cgraph_node::get (current_function_decl
), true);
2833 /* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS,
2834 KNOWN_CONTEXTS and KNOWN_AGGS. */
2837 estimate_edge_devirt_benefit (struct cgraph_edge
*ie
,
2838 int *size
, int *time
,
2839 vec
<tree
> known_vals
,
2840 vec
<ipa_polymorphic_call_context
> known_contexts
,
2841 vec
<ipa_agg_jump_function_p
> known_aggs
)
2844 struct cgraph_node
*callee
;
2845 class ipa_fn_summary
*isummary
;
2846 enum availability avail
;
2849 if (!known_vals
.exists () && !known_contexts
.exists ())
2851 if (!opt_for_fn (ie
->caller
->decl
, flag_indirect_inlining
))
2854 target
= ipa_get_indirect_edge_target (ie
, known_vals
, known_contexts
,
2855 known_aggs
, &speculative
);
2856 if (!target
|| speculative
)
2859 /* Account for difference in cost between indirect and direct calls. */
2860 *size
-= (eni_size_weights
.indirect_call_cost
- eni_size_weights
.call_cost
);
2861 *time
-= (eni_time_weights
.indirect_call_cost
- eni_time_weights
.call_cost
);
2862 gcc_checking_assert (*time
>= 0);
2863 gcc_checking_assert (*size
>= 0);
2865 callee
= cgraph_node::get (target
);
2866 if (!callee
|| !callee
->definition
)
2868 callee
= callee
->function_symbol (&avail
);
2869 if (avail
< AVAIL_AVAILABLE
)
2871 isummary
= ipa_fn_summaries
->get (callee
);
2872 if (isummary
== NULL
)
2875 return isummary
->inlinable
;
2878 /* Increase SIZE, MIN_SIZE (if non-NULL) and TIME for size and time needed to
2879 handle edge E with probability PROB.
2880 Set HINTS if edge may be devirtualized.
2881 KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS describe context of the call
2885 estimate_edge_size_and_time (struct cgraph_edge
*e
, int *size
, int *min_size
,
2888 vec
<tree
> known_vals
,
2889 vec
<ipa_polymorphic_call_context
> known_contexts
,
2890 vec
<ipa_agg_jump_function_p
> known_aggs
,
2893 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
2894 int call_size
= es
->call_stmt_size
;
2895 int call_time
= es
->call_stmt_time
;
2897 if (!e
->callee
&& hints
&& e
->maybe_hot_p ()
2898 && estimate_edge_devirt_benefit (e
, &call_size
, &call_time
,
2899 known_vals
, known_contexts
, known_aggs
))
2900 *hints
|= INLINE_HINT_indirect_call
;
2901 cur_size
= call_size
* ipa_fn_summary::size_scale
;
2904 *min_size
+= cur_size
;
2907 else if (prob
== REG_BR_PROB_BASE
)
2908 *time
+= ((sreal
)call_time
) * e
->sreal_frequency ();
2910 *time
+= ((sreal
)call_time
* prob
) * e
->sreal_frequency ();
2915 /* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
2916 calls in NODE. POSSIBLE_TRUTHS, KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS
2917 describe context of the call site. */
2920 estimate_calls_size_and_time (struct cgraph_node
*node
, int *size
,
2921 int *min_size
, sreal
*time
,
2923 clause_t possible_truths
,
2924 vec
<tree
> known_vals
,
2925 vec
<ipa_polymorphic_call_context
> known_contexts
,
2926 vec
<ipa_agg_jump_function_p
> known_aggs
)
2928 struct cgraph_edge
*e
;
2929 for (e
= node
->callees
; e
; e
= e
->next_callee
)
2931 if (!e
->inline_failed
)
2933 gcc_checking_assert (!ipa_call_summaries
->get (e
));
2934 estimate_calls_size_and_time (e
->callee
, size
, min_size
, time
,
2937 known_vals
, known_contexts
,
2941 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
2943 /* Do not care about zero sized builtins. */
2944 if (!es
->call_stmt_size
)
2946 gcc_checking_assert (!es
->call_stmt_time
);
2950 || es
->predicate
->evaluate (possible_truths
))
2952 /* Predicates of calls shall not use NOT_CHANGED codes,
2953 sowe do not need to compute probabilities. */
2954 estimate_edge_size_and_time (e
, size
,
2955 es
->predicate
? NULL
: min_size
,
2956 time
, REG_BR_PROB_BASE
,
2957 known_vals
, known_contexts
,
2961 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
2963 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
2965 || es
->predicate
->evaluate (possible_truths
))
2966 estimate_edge_size_and_time (e
, size
,
2967 es
->predicate
? NULL
: min_size
,
2968 time
, REG_BR_PROB_BASE
,
2969 known_vals
, known_contexts
, known_aggs
,
2974 /* Default constructor for ipa call context.
2975 Memory alloction of known_vals, known_contexts
2976 and known_aggs vectors is owned by the caller, but can
2977 be release by ipa_call_context::release.
2979 inline_param_summary is owned by the caller. */
2980 ipa_call_context::ipa_call_context (cgraph_node
*node
,
2981 clause_t possible_truths
,
2982 clause_t nonspec_possible_truths
,
2983 vec
<tree
> known_vals
,
2984 vec
<ipa_polymorphic_call_context
>
2986 vec
<ipa_agg_jump_function_p
> known_aggs
,
2987 vec
<inline_param_summary
>
2988 inline_param_summary
)
2989 : m_node (node
), m_possible_truths (possible_truths
),
2990 m_nonspec_possible_truths (nonspec_possible_truths
),
2991 m_inline_param_summary (inline_param_summary
),
2992 m_known_vals (known_vals
),
2993 m_known_contexts (known_contexts
),
2994 m_known_aggs (known_aggs
)
2998 /* Set THIS to be a duplicate of CTX. Copy all relevant info. */
3001 ipa_call_context::duplicate_from (const ipa_call_context
&ctx
)
3003 m_node
= ctx
.m_node
;
3004 m_possible_truths
= ctx
.m_possible_truths
;
3005 m_nonspec_possible_truths
= ctx
.m_nonspec_possible_truths
;
3006 class ipa_node_params
*params_summary
= IPA_NODE_REF (m_node
);
3007 unsigned int nargs
= params_summary
3008 ? ipa_get_param_count (params_summary
) : 0;
3010 m_inline_param_summary
= vNULL
;
3011 /* Copy the info only if there is at least one useful entry. */
3012 if (ctx
.m_inline_param_summary
.exists ())
3014 unsigned int n
= MIN (ctx
.m_inline_param_summary
.length (), nargs
);
3016 for (unsigned int i
= 0; i
< n
; i
++)
3017 if (ipa_is_param_used_by_ipa_predicates (params_summary
, i
)
3018 && !ctx
.m_inline_param_summary
[i
].useless_p ())
3020 m_inline_param_summary
3021 = ctx
.m_inline_param_summary
.copy ();
3025 m_known_vals
= vNULL
;
3026 if (ctx
.m_known_vals
.exists ())
3028 unsigned int n
= MIN (ctx
.m_known_vals
.length (), nargs
);
3030 for (unsigned int i
= 0; i
< n
; i
++)
3031 if (ipa_is_param_used_by_indirect_call (params_summary
, i
)
3032 && ctx
.m_known_vals
[i
])
3034 m_known_vals
= ctx
.m_known_vals
.copy ();
3039 m_known_contexts
= vNULL
;
3040 if (ctx
.m_known_contexts
.exists ())
3042 unsigned int n
= MIN (ctx
.m_known_contexts
.length (), nargs
);
3044 for (unsigned int i
= 0; i
< n
; i
++)
3045 if (ipa_is_param_used_by_polymorphic_call (params_summary
, i
)
3046 && !ctx
.m_known_contexts
[i
].useless_p ())
3048 m_known_contexts
= ctx
.m_known_contexts
.copy ();
3053 m_known_aggs
= vNULL
;
3054 if (ctx
.m_known_aggs
.exists ())
3056 unsigned int n
= MIN (ctx
.m_known_aggs
.length (), nargs
);
3058 for (unsigned int i
= 0; i
< n
; i
++)
3059 if (ipa_is_param_used_by_indirect_call (params_summary
, i
)
3060 && ctx
.m_known_aggs
[i
])
3062 m_known_aggs
= ctx
.m_known_aggs
.copy ();
3068 /* Release memory used by known_vals/contexts/aggs vectors.
3069 If ALL is true release also inline_param_summary.
3070 This happens when context was previously duplciated to be stored
3074 ipa_call_context::release (bool all
)
3076 /* See if context is initialized at first place. */
3079 m_known_vals
.release ();
3080 m_known_contexts
.release ();
3081 m_known_aggs
.release ();
3083 m_inline_param_summary
.release ();
3086 /* Return true if CTX describes the same call context as THIS. */
3089 ipa_call_context::equal_to (const ipa_call_context
&ctx
)
3091 if (m_node
!= ctx
.m_node
3092 || m_possible_truths
!= ctx
.m_possible_truths
3093 || m_nonspec_possible_truths
!= ctx
.m_nonspec_possible_truths
)
3096 class ipa_node_params
*params_summary
= IPA_NODE_REF (m_node
);
3097 unsigned int nargs
= params_summary
3098 ? ipa_get_param_count (params_summary
) : 0;
3100 if (m_inline_param_summary
.exists () || ctx
.m_inline_param_summary
.exists ())
3102 for (unsigned int i
= 0; i
< nargs
; i
++)
3104 if (!ipa_is_param_used_by_ipa_predicates (params_summary
, i
))
3106 if (i
>= m_inline_param_summary
.length ()
3107 || m_inline_param_summary
[i
].useless_p ())
3109 if (i
< ctx
.m_inline_param_summary
.length ()
3110 && !ctx
.m_inline_param_summary
[i
].useless_p ())
3114 if (i
>= ctx
.m_inline_param_summary
.length ()
3115 || ctx
.m_inline_param_summary
[i
].useless_p ())
3117 if (i
< m_inline_param_summary
.length ()
3118 && !m_inline_param_summary
[i
].useless_p ())
3122 if (!m_inline_param_summary
[i
].equal_to
3123 (ctx
.m_inline_param_summary
[i
]))
3127 if (m_known_vals
.exists () || ctx
.m_known_vals
.exists ())
3129 for (unsigned int i
= 0; i
< nargs
; i
++)
3131 if (!ipa_is_param_used_by_indirect_call (params_summary
, i
))
3133 if (i
>= m_known_vals
.length () || !m_known_vals
[i
])
3135 if (i
< ctx
.m_known_vals
.length () && ctx
.m_known_vals
[i
])
3139 if (i
>= ctx
.m_known_vals
.length () || !ctx
.m_known_vals
[i
])
3141 if (i
< m_known_vals
.length () && m_known_vals
[i
])
3145 if (m_known_vals
[i
] != ctx
.m_known_vals
[i
])
3149 if (m_known_contexts
.exists () || ctx
.m_known_contexts
.exists ())
3151 for (unsigned int i
= 0; i
< nargs
; i
++)
3153 if (!ipa_is_param_used_by_polymorphic_call (params_summary
, i
))
3155 if (i
>= m_known_contexts
.length ()
3156 || m_known_contexts
[i
].useless_p ())
3158 if (i
< ctx
.m_known_contexts
.length ()
3159 && !ctx
.m_known_contexts
[i
].useless_p ())
3163 if (i
>= ctx
.m_known_contexts
.length ()
3164 || ctx
.m_known_contexts
[i
].useless_p ())
3166 if (i
< m_known_contexts
.length ()
3167 && !m_known_contexts
[i
].useless_p ())
3171 if (!m_known_contexts
[i
].equal_to
3172 (ctx
.m_known_contexts
[i
]))
3176 if (m_known_aggs
.exists () || ctx
.m_known_aggs
.exists ())
3178 for (unsigned int i
= 0; i
< nargs
; i
++)
3180 if (!ipa_is_param_used_by_indirect_call (params_summary
, i
))
3182 if (i
>= m_known_aggs
.length () || !m_known_aggs
[i
])
3184 if (i
< ctx
.m_known_aggs
.length () && ctx
.m_known_aggs
[i
])
3188 if (i
>= ctx
.m_known_aggs
.length () || !ctx
.m_known_aggs
[i
])
3190 if (i
< m_known_aggs
.length () && m_known_aggs
[i
])
3194 if (m_known_aggs
[i
] != ctx
.m_known_aggs
[i
])
3201 /* Estimate size and time needed to execute call in the given context.
3202 Additionally detemine hints determined by the context. Finally compute
3203 minimal size needed for the call that is independent on the call context and
3204 can be used for fast estimates. Return the values in RET_SIZE,
3205 RET_MIN_SIZE, RET_TIME and RET_HINTS. */
3208 ipa_call_context::estimate_size_and_time (int *ret_size
,
3211 sreal
*ret_nonspecialized_time
,
3212 ipa_hints
*ret_hints
)
3214 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (m_node
);
3219 ipa_hints hints
= 0;
3222 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3225 fprintf (dump_file
, " Estimating body: %s/%i\n"
3226 " Known to be false: ", m_node
->name (),
3229 for (i
= predicate::not_inlined_condition
;
3230 i
< (predicate::first_dynamic_condition
3231 + (int) vec_safe_length (info
->conds
)); i
++)
3232 if (!(m_possible_truths
& (1 << i
)))
3235 fprintf (dump_file
, ", ");
3237 dump_condition (dump_file
, info
->conds
, i
);
3241 estimate_calls_size_and_time (m_node
, &size
, &min_size
,
3242 ret_time
? &time
: NULL
,
3243 ret_hints
? &hints
: NULL
, m_possible_truths
,
3244 m_known_vals
, m_known_contexts
, m_known_aggs
);
3246 sreal nonspecialized_time
= time
;
3248 min_size
+= (*info
->size_time_table
)[0].size
;
3249 for (i
= 0; vec_safe_iterate (info
->size_time_table
, i
, &e
); i
++)
3251 bool exec
= e
->exec_predicate
.evaluate (m_nonspec_possible_truths
);
3253 /* Because predicates are conservative, it can happen that nonconst is 1
3257 bool nonconst
= e
->nonconst_predicate
.evaluate (m_possible_truths
);
3259 gcc_checking_assert (e
->time
>= 0);
3260 gcc_checking_assert (time
>= 0);
3262 /* We compute specialized size only because size of nonspecialized
3263 copy is context independent.
3265 The difference between nonspecialized execution and specialized is
3266 that nonspecialized is not going to have optimized out computations
3267 known to be constant in a specialized setting. */
3272 nonspecialized_time
+= e
->time
;
3275 else if (!m_inline_param_summary
.exists ())
3282 int prob
= e
->nonconst_predicate
.probability
3283 (info
->conds
, m_possible_truths
,
3284 m_inline_param_summary
);
3285 gcc_checking_assert (prob
>= 0);
3286 gcc_checking_assert (prob
<= REG_BR_PROB_BASE
);
3287 time
+= e
->time
* prob
/ REG_BR_PROB_BASE
;
3289 gcc_checking_assert (time
>= 0);
3292 gcc_checking_assert ((*info
->size_time_table
)[0].exec_predicate
== true);
3293 gcc_checking_assert ((*info
->size_time_table
)[0].nonconst_predicate
== true);
3294 gcc_checking_assert (min_size
>= 0);
3295 gcc_checking_assert (size
>= 0);
3296 gcc_checking_assert (time
>= 0);
3297 /* nonspecialized_time should be always bigger than specialized time.
3298 Roundoff issues however may get into the way. */
3299 gcc_checking_assert ((nonspecialized_time
- time
* 99 / 100) >= -1);
3301 /* Roundoff issues may make specialized time bigger than nonspecialized
3302 time. We do not really want that to happen because some heurstics
3303 may get confused by seeing negative speedups. */
3304 if (time
> nonspecialized_time
)
3305 time
= nonspecialized_time
;
3309 if (info
->loop_iterations
3310 && !info
->loop_iterations
->evaluate (m_possible_truths
))
3311 hints
|= INLINE_HINT_loop_iterations
;
3312 if (info
->loop_stride
3313 && !info
->loop_stride
->evaluate (m_possible_truths
))
3314 hints
|= INLINE_HINT_loop_stride
;
3316 hints
|= INLINE_HINT_in_scc
;
3317 if (DECL_DECLARED_INLINE_P (m_node
->decl
))
3318 hints
|= INLINE_HINT_declared_inline
;
3321 size
= RDIV (size
, ipa_fn_summary::size_scale
);
3322 min_size
= RDIV (min_size
, ipa_fn_summary::size_scale
);
3324 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3325 fprintf (dump_file
, "\n size:%i time:%f nonspec time:%f\n", (int) size
,
3326 time
.to_double (), nonspecialized_time
.to_double ());
3329 if (ret_nonspecialized_time
)
3330 *ret_nonspecialized_time
= nonspecialized_time
;
3334 *ret_min_size
= min_size
;
3341 /* Estimate size and time needed to execute callee of EDGE assuming that
3342 parameters known to be constant at caller of EDGE are propagated.
3343 KNOWN_VALS and KNOWN_CONTEXTS are vectors of assumed known constant values
3344 and types for parameters. */
3347 estimate_ipcp_clone_size_and_time (struct cgraph_node
*node
,
3348 vec
<tree
> known_vals
,
3349 vec
<ipa_polymorphic_call_context
>
3351 vec
<ipa_agg_jump_function_p
> known_aggs
,
3352 int *ret_size
, sreal
*ret_time
,
3353 sreal
*ret_nonspec_time
,
3356 clause_t clause
, nonspec_clause
;
3358 evaluate_conditions_for_known_args (node
, false, known_vals
, known_aggs
,
3359 &clause
, &nonspec_clause
);
3360 ipa_call_context
ctx (node
, clause
, nonspec_clause
,
3361 known_vals
, known_contexts
,
3363 ctx
.estimate_size_and_time (ret_size
, NULL
, ret_time
,
3364 ret_nonspec_time
, hints
);
3367 /* Return stack frame offset where frame of NODE is supposed to start inside
3368 of the function it is inlined to.
3369 Return 0 for functions that are not inlined. */
3372 ipa_get_stack_frame_offset (struct cgraph_node
*node
)
3374 HOST_WIDE_INT offset
= 0;
3375 if (!node
->inlined_to
)
3377 node
= node
->callers
->caller
;
3380 offset
+= ipa_size_summaries
->get (node
)->estimated_self_stack_size
;
3381 if (!node
->inlined_to
)
3383 node
= node
->callers
->caller
;
3388 /* Update summary information of inline clones after inlining.
3389 Compute peak stack usage. */
3392 inline_update_callee_summaries (struct cgraph_node
*node
, int depth
)
3394 struct cgraph_edge
*e
;
3396 ipa_propagate_frequency (node
);
3397 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3399 if (!e
->inline_failed
)
3400 inline_update_callee_summaries (e
->callee
, depth
);
3402 ipa_call_summaries
->get (e
)->loop_depth
+= depth
;
3404 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3405 ipa_call_summaries
->get (e
)->loop_depth
+= depth
;
3408 /* Update change_prob of EDGE after INLINED_EDGE has been inlined.
3409 When function A is inlined in B and A calls C with parameter that
3410 changes with probability PROB1 and C is known to be passthroug
3411 of argument if B that change with probability PROB2, the probability
3412 of change is now PROB1*PROB2. */
3415 remap_edge_change_prob (struct cgraph_edge
*inlined_edge
,
3416 struct cgraph_edge
*edge
)
3418 if (ipa_node_params_sum
)
3421 class ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3424 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
3425 class ipa_call_summary
*inlined_es
3426 = ipa_call_summaries
->get (inlined_edge
);
3428 if (es
->param
.length () == 0)
3431 for (i
= 0; i
< ipa_get_cs_argument_count (args
); i
++)
3433 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3434 if (jfunc
->type
== IPA_JF_PASS_THROUGH
3435 || jfunc
->type
== IPA_JF_ANCESTOR
)
3437 int id
= jfunc
->type
== IPA_JF_PASS_THROUGH
3438 ? ipa_get_jf_pass_through_formal_id (jfunc
)
3439 : ipa_get_jf_ancestor_formal_id (jfunc
);
3440 if (id
< (int) inlined_es
->param
.length ())
3442 int prob1
= es
->param
[i
].change_prob
;
3443 int prob2
= inlined_es
->param
[id
].change_prob
;
3444 int prob
= combine_probabilities (prob1
, prob2
);
3446 if (prob1
&& prob2
&& !prob
)
3449 es
->param
[i
].change_prob
= prob
;
3456 /* Update edge summaries of NODE after INLINED_EDGE has been inlined.
3458 Remap predicates of callees of NODE. Rest of arguments match
3461 Also update change probabilities. */
3464 remap_edge_summaries (struct cgraph_edge
*inlined_edge
,
3465 struct cgraph_node
*node
,
3466 class ipa_fn_summary
*info
,
3467 class ipa_node_params
*params_summary
,
3468 class ipa_fn_summary
*callee_info
,
3469 vec
<int> operand_map
,
3470 vec
<int> offset_map
,
3471 clause_t possible_truths
,
3472 predicate
*toplev_predicate
)
3474 struct cgraph_edge
*e
, *next
;
3475 for (e
= node
->callees
; e
; e
= next
)
3478 next
= e
->next_callee
;
3480 if (e
->inline_failed
)
3482 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
3483 remap_edge_change_prob (inlined_edge
, e
);
3487 p
= es
->predicate
->remap_after_inlining
3488 (info
, params_summary
,
3489 callee_info
, operand_map
,
3490 offset_map
, possible_truths
,
3492 edge_set_predicate (e
, &p
);
3495 edge_set_predicate (e
, toplev_predicate
);
3498 remap_edge_summaries (inlined_edge
, e
->callee
, info
,
3499 params_summary
, callee_info
,
3500 operand_map
, offset_map
, possible_truths
,
3503 for (e
= node
->indirect_calls
; e
; e
= next
)
3505 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
3507 next
= e
->next_callee
;
3509 remap_edge_change_prob (inlined_edge
, e
);
3512 p
= es
->predicate
->remap_after_inlining
3513 (info
, params_summary
,
3514 callee_info
, operand_map
, offset_map
,
3515 possible_truths
, *toplev_predicate
);
3516 edge_set_predicate (e
, &p
);
3519 edge_set_predicate (e
, toplev_predicate
);
3523 /* Same as remap_predicate, but set result into hint *HINT. */
3526 remap_hint_predicate (class ipa_fn_summary
*info
,
3527 class ipa_node_params
*params_summary
,
3528 class ipa_fn_summary
*callee_info
,
3530 vec
<int> operand_map
,
3531 vec
<int> offset_map
,
3532 clause_t possible_truths
,
3533 predicate
*toplev_predicate
)
3539 p
= (*hint
)->remap_after_inlining
3540 (info
, params_summary
, callee_info
,
3541 operand_map
, offset_map
,
3542 possible_truths
, *toplev_predicate
);
3543 if (p
!= false && p
!= true)
3546 set_hint_predicate (hint
, p
);
3552 /* We inlined EDGE. Update summary of the function we inlined into. */
3555 ipa_merge_fn_summary_after_inlining (struct cgraph_edge
*edge
)
3557 ipa_fn_summary
*callee_info
= ipa_fn_summaries
->get (edge
->callee
);
3558 struct cgraph_node
*to
= (edge
->caller
->inlined_to
3559 ? edge
->caller
->inlined_to
: edge
->caller
);
3560 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (to
);
3561 clause_t clause
= 0; /* not_inline is known to be false. */
3563 auto_vec
<int, 8> operand_map
;
3564 auto_vec
<int, 8> offset_map
;
3566 predicate toplev_predicate
;
3567 class ipa_call_summary
*es
= ipa_call_summaries
->get (edge
);
3568 class ipa_node_params
*params_summary
= (ipa_node_params_sum
3569 ? IPA_NODE_REF (to
) : NULL
);
3572 toplev_predicate
= *es
->predicate
;
3574 toplev_predicate
= true;
3576 info
->fp_expressions
|= callee_info
->fp_expressions
;
3578 if (callee_info
->conds
)
3579 evaluate_properties_for_edge (edge
, true, &clause
, NULL
, NULL
, NULL
, NULL
);
3580 if (ipa_node_params_sum
&& callee_info
->conds
)
3582 class ipa_edge_args
*args
= IPA_EDGE_REF (edge
);
3583 int count
= args
? ipa_get_cs_argument_count (args
) : 0;
3588 operand_map
.safe_grow_cleared (count
);
3589 offset_map
.safe_grow_cleared (count
);
3591 for (i
= 0; i
< count
; i
++)
3593 struct ipa_jump_func
*jfunc
= ipa_get_ith_jump_func (args
, i
);
3596 /* TODO: handle non-NOPs when merging. */
3597 if (jfunc
->type
== IPA_JF_PASS_THROUGH
)
3599 if (ipa_get_jf_pass_through_operation (jfunc
) == NOP_EXPR
)
3600 map
= ipa_get_jf_pass_through_formal_id (jfunc
);
3601 if (!ipa_get_jf_pass_through_agg_preserved (jfunc
))
3604 else if (jfunc
->type
== IPA_JF_ANCESTOR
)
3606 HOST_WIDE_INT offset
= ipa_get_jf_ancestor_offset (jfunc
);
3607 if (offset
>= 0 && offset
< INT_MAX
)
3609 map
= ipa_get_jf_ancestor_formal_id (jfunc
);
3610 if (!ipa_get_jf_ancestor_agg_preserved (jfunc
))
3612 offset_map
[i
] = offset
;
3615 operand_map
[i
] = map
;
3616 gcc_assert (map
< ipa_get_param_count (params_summary
));
3619 sreal freq
= edge
->sreal_frequency ();
3620 for (i
= 0; vec_safe_iterate (callee_info
->size_time_table
, i
, &e
); i
++)
3623 p
= e
->exec_predicate
.remap_after_inlining
3624 (info
, params_summary
,
3625 callee_info
, operand_map
,
3628 predicate nonconstp
;
3629 nonconstp
= e
->nonconst_predicate
.remap_after_inlining
3630 (info
, params_summary
,
3631 callee_info
, operand_map
,
3634 if (p
!= false && nonconstp
!= false)
3636 sreal add_time
= ((sreal
)e
->time
* freq
);
3637 int prob
= e
->nonconst_predicate
.probability (callee_info
->conds
,
3639 add_time
= add_time
* prob
/ REG_BR_PROB_BASE
;
3640 if (prob
!= REG_BR_PROB_BASE
3641 && dump_file
&& (dump_flags
& TDF_DETAILS
))
3643 fprintf (dump_file
, "\t\tScaling time by probability:%f\n",
3644 (double) prob
/ REG_BR_PROB_BASE
);
3646 info
->account_size_time (e
->size
, add_time
, p
, nonconstp
);
3649 remap_edge_summaries (edge
, edge
->callee
, info
, params_summary
,
3650 callee_info
, operand_map
,
3651 offset_map
, clause
, &toplev_predicate
);
3652 remap_hint_predicate (info
, params_summary
, callee_info
,
3653 &callee_info
->loop_iterations
,
3654 operand_map
, offset_map
, clause
, &toplev_predicate
);
3655 remap_hint_predicate (info
, params_summary
, callee_info
,
3656 &callee_info
->loop_stride
,
3657 operand_map
, offset_map
, clause
, &toplev_predicate
);
3659 HOST_WIDE_INT stack_frame_offset
= ipa_get_stack_frame_offset (edge
->callee
);
3660 HOST_WIDE_INT peak
= stack_frame_offset
+ callee_info
->estimated_stack_size
;
3662 if (info
->estimated_stack_size
< peak
)
3663 info
->estimated_stack_size
= peak
;
3665 inline_update_callee_summaries (edge
->callee
, es
->loop_depth
);
3667 /* Free summaries that are not maintained for inline clones/edges. */
3668 ipa_call_summaries
->remove (edge
);
3669 ipa_fn_summaries
->remove (edge
->callee
);
3670 ipa_remove_from_growth_caches (edge
);
3673 /* For performance reasons ipa_merge_fn_summary_after_inlining is not updating
3674 overall size and time. Recompute it. */
3677 ipa_update_overall_fn_summary (struct cgraph_node
*node
)
3679 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (node
);
3680 class ipa_size_summary
*size_info
= ipa_size_summaries
->get (node
);
3684 size_info
->size
= 0;
3686 for (i
= 0; vec_safe_iterate (info
->size_time_table
, i
, &e
); i
++)
3688 size_info
->size
+= e
->size
;
3689 info
->time
+= e
->time
;
3691 info
->min_size
= (*info
->size_time_table
)[0].size
;
3692 estimate_calls_size_and_time (node
, &size_info
->size
, &info
->min_size
,
3694 ~(clause_t
) (1 << predicate::false_condition
),
3695 vNULL
, vNULL
, vNULL
);
3696 size_info
->size
= RDIV (size_info
->size
, ipa_fn_summary::size_scale
);
3697 info
->min_size
= RDIV (info
->min_size
, ipa_fn_summary::size_scale
);
3701 /* This function performs intraprocedural analysis in NODE that is required to
3702 inline indirect calls. */
3705 inline_indirect_intraprocedural_analysis (struct cgraph_node
*node
)
3707 ipa_analyze_node (node
);
3708 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
3710 ipa_print_node_params (dump_file
, node
);
3711 ipa_print_node_jump_functions (dump_file
, node
);
3716 /* Note function body size. */
3719 inline_analyze_function (struct cgraph_node
*node
)
3721 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
3724 fprintf (dump_file
, "\nAnalyzing function: %s/%u\n",
3725 node
->name (), node
->order
);
3726 if (opt_for_fn (node
->decl
, optimize
) && !node
->thunk
.thunk_p
)
3727 inline_indirect_intraprocedural_analysis (node
);
3728 compute_fn_summary (node
, false);
3731 struct cgraph_edge
*e
;
3732 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3733 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3734 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3735 e
->inline_failed
= CIF_FUNCTION_NOT_OPTIMIZED
;
3742 /* Called when new function is inserted to callgraph late. */
3745 ipa_fn_summary_t::insert (struct cgraph_node
*node
, ipa_fn_summary
*)
3747 inline_analyze_function (node
);
3750 /* Note function body size. */
3753 ipa_fn_summary_generate (void)
3755 struct cgraph_node
*node
;
3757 FOR_EACH_DEFINED_FUNCTION (node
)
3758 if (DECL_STRUCT_FUNCTION (node
->decl
))
3759 node
->versionable
= tree_versionable_function_p (node
->decl
);
3761 ipa_fn_summary_alloc ();
3763 ipa_fn_summaries
->enable_insertion_hook ();
3765 ipa_register_cgraph_hooks ();
3767 FOR_EACH_DEFINED_FUNCTION (node
)
3769 && (flag_generate_lto
|| flag_generate_offload
|| flag_wpa
3770 || opt_for_fn (node
->decl
, optimize
)))
3771 inline_analyze_function (node
);
3775 /* Write inline summary for edge E to OB. */
3778 read_ipa_call_summary (class lto_input_block
*ib
, struct cgraph_edge
*e
,
3781 class ipa_call_summary
*es
= prevails
3782 ? ipa_call_summaries
->get_create (e
) : NULL
;
3786 int size
= streamer_read_uhwi (ib
);
3787 int time
= streamer_read_uhwi (ib
);
3788 int depth
= streamer_read_uhwi (ib
);
3792 es
->call_stmt_size
= size
;
3793 es
->call_stmt_time
= time
;
3794 es
->loop_depth
= depth
;
3797 bitpack_d bp
= streamer_read_bitpack (ib
);
3799 es
->is_return_callee_uncaptured
= bp_unpack_value (&bp
, 1);
3801 bp_unpack_value (&bp
, 1);
3805 edge_set_predicate (e
, &p
);
3806 length
= streamer_read_uhwi (ib
);
3807 if (length
&& es
&& e
->possibly_call_in_translation_unit_p ())
3809 es
->param
.safe_grow_cleared (length
);
3810 for (i
= 0; i
< length
; i
++)
3811 es
->param
[i
].change_prob
= streamer_read_uhwi (ib
);
3815 for (i
= 0; i
< length
; i
++)
3816 streamer_read_uhwi (ib
);
3821 /* Stream in inline summaries from the section. */
3824 inline_read_section (struct lto_file_decl_data
*file_data
, const char *data
,
3827 const struct lto_function_header
*header
=
3828 (const struct lto_function_header
*) data
;
3829 const int cfg_offset
= sizeof (struct lto_function_header
);
3830 const int main_offset
= cfg_offset
+ header
->cfg_size
;
3831 const int string_offset
= main_offset
+ header
->main_size
;
3832 class data_in
*data_in
;
3833 unsigned int i
, count2
, j
;
3834 unsigned int f_count
;
3836 lto_input_block
ib ((const char *) data
+ main_offset
, header
->main_size
,
3837 file_data
->mode_table
);
3840 lto_data_in_create (file_data
, (const char *) data
+ string_offset
,
3841 header
->string_size
, vNULL
);
3842 f_count
= streamer_read_uhwi (&ib
);
3843 for (i
= 0; i
< f_count
; i
++)
3846 struct cgraph_node
*node
;
3847 class ipa_fn_summary
*info
;
3848 class ipa_node_params
*params_summary
;
3849 class ipa_size_summary
*size_info
;
3850 lto_symtab_encoder_t encoder
;
3851 struct bitpack_d bp
;
3852 struct cgraph_edge
*e
;
3855 index
= streamer_read_uhwi (&ib
);
3856 encoder
= file_data
->symtab_node_encoder
;
3857 node
= dyn_cast
<cgraph_node
*> (lto_symtab_encoder_deref (encoder
,
3859 info
= node
->prevailing_p () ? ipa_fn_summaries
->get_create (node
) : NULL
;
3860 params_summary
= node
->prevailing_p () ? IPA_NODE_REF (node
) : NULL
;
3861 size_info
= node
->prevailing_p ()
3862 ? ipa_size_summaries
->get_create (node
) : NULL
;
3864 int stack_size
= streamer_read_uhwi (&ib
);
3865 int size
= streamer_read_uhwi (&ib
);
3866 sreal time
= sreal::stream_in (&ib
);
3870 info
->estimated_stack_size
3871 = size_info
->estimated_self_stack_size
= stack_size
;
3872 size_info
->size
= size_info
->self_size
= size
;
3876 bp
= streamer_read_bitpack (&ib
);
3879 info
->inlinable
= bp_unpack_value (&bp
, 1);
3880 info
->fp_expressions
= bp_unpack_value (&bp
, 1);
3884 bp_unpack_value (&bp
, 1);
3885 bp_unpack_value (&bp
, 1);
3888 count2
= streamer_read_uhwi (&ib
);
3889 gcc_assert (!info
|| !info
->conds
);
3891 vec_safe_reserve_exact (info
->conds
, count2
);
3892 for (j
= 0; j
< count2
; j
++)
3895 unsigned int k
, count3
;
3896 c
.operand_num
= streamer_read_uhwi (&ib
);
3897 c
.code
= (enum tree_code
) streamer_read_uhwi (&ib
);
3898 c
.type
= stream_read_tree (&ib
, data_in
);
3899 c
.val
= stream_read_tree (&ib
, data_in
);
3900 bp
= streamer_read_bitpack (&ib
);
3901 c
.agg_contents
= bp_unpack_value (&bp
, 1);
3902 c
.by_ref
= bp_unpack_value (&bp
, 1);
3904 c
.offset
= streamer_read_uhwi (&ib
);
3905 count3
= streamer_read_uhwi (&ib
);
3908 vec_safe_reserve_exact (c
.param_ops
, count3
);
3910 ipa_set_param_used_by_ipa_predicates
3911 (params_summary
, c
.operand_num
, true);
3912 for (k
= 0; k
< count3
; k
++)
3914 struct expr_eval_op op
;
3915 enum gimple_rhs_class rhs_class
;
3916 op
.code
= (enum tree_code
) streamer_read_uhwi (&ib
);
3917 op
.type
= stream_read_tree (&ib
, data_in
);
3918 switch (rhs_class
= get_gimple_rhs_class (op
.code
))
3920 case GIMPLE_UNARY_RHS
:
3922 op
.val
[0] = NULL_TREE
;
3923 op
.val
[1] = NULL_TREE
;
3926 case GIMPLE_BINARY_RHS
:
3927 case GIMPLE_TERNARY_RHS
:
3928 bp
= streamer_read_bitpack (&ib
);
3929 op
.index
= bp_unpack_value (&bp
, 2);
3930 op
.val
[0] = stream_read_tree (&ib
, data_in
);
3931 if (rhs_class
== GIMPLE_BINARY_RHS
)
3932 op
.val
[1] = NULL_TREE
;
3934 op
.val
[1] = stream_read_tree (&ib
, data_in
);
3938 fatal_error (UNKNOWN_LOCATION
,
3939 "invalid fnsummary in LTO stream");
3942 c
.param_ops
->quick_push (op
);
3945 info
->conds
->quick_push (c
);
3947 count2
= streamer_read_uhwi (&ib
);
3948 gcc_assert (!info
|| !info
->size_time_table
);
3950 vec_safe_reserve_exact (info
->size_time_table
, count2
);
3951 for (j
= 0; j
< count2
; j
++)
3953 class size_time_entry e
;
3955 e
.size
= streamer_read_uhwi (&ib
);
3956 e
.time
= sreal::stream_in (&ib
);
3957 e
.exec_predicate
.stream_in (&ib
);
3958 e
.nonconst_predicate
.stream_in (&ib
);
3961 info
->size_time_table
->quick_push (e
);
3966 set_hint_predicate (&info
->loop_iterations
, p
);
3969 set_hint_predicate (&info
->loop_stride
, p
);
3970 for (e
= node
->callees
; e
; e
= e
->next_callee
)
3971 read_ipa_call_summary (&ib
, e
, info
!= NULL
);
3972 for (e
= node
->indirect_calls
; e
; e
= e
->next_callee
)
3973 read_ipa_call_summary (&ib
, e
, info
!= NULL
);
3976 lto_free_section_data (file_data
, LTO_section_ipa_fn_summary
, NULL
, data
,
3978 lto_data_in_delete (data_in
);
3982 /* Read inline summary. Jump functions are shared among ipa-cp
3983 and inliner, so when ipa-cp is active, we don't need to write them
3987 ipa_fn_summary_read (void)
3989 struct lto_file_decl_data
**file_data_vec
= lto_get_file_decl_data ();
3990 struct lto_file_decl_data
*file_data
;
3993 ipa_fn_summary_alloc ();
3995 while ((file_data
= file_data_vec
[j
++]))
3999 = lto_get_summary_section_data (file_data
, LTO_section_ipa_fn_summary
,
4002 inline_read_section (file_data
, data
, len
);
4004 /* Fatal error here. We do not want to support compiling ltrans units
4005 with different version of compiler or different flags than the WPA
4006 unit, so this should never happen. */
4007 fatal_error (input_location
,
4008 "ipa inline summary is missing in input file");
4010 ipa_register_cgraph_hooks ();
4012 ipa_prop_read_jump_functions ();
4014 gcc_assert (ipa_fn_summaries
);
4015 ipa_fn_summaries
->enable_insertion_hook ();
4019 /* Write inline summary for edge E to OB. */
4022 write_ipa_call_summary (struct output_block
*ob
, struct cgraph_edge
*e
)
4024 class ipa_call_summary
*es
= ipa_call_summaries
->get (e
);
4027 streamer_write_uhwi (ob
, es
->call_stmt_size
);
4028 streamer_write_uhwi (ob
, es
->call_stmt_time
);
4029 streamer_write_uhwi (ob
, es
->loop_depth
);
4031 bitpack_d bp
= bitpack_create (ob
->main_stream
);
4032 bp_pack_value (&bp
, es
->is_return_callee_uncaptured
, 1);
4033 streamer_write_bitpack (&bp
);
4036 es
->predicate
->stream_out (ob
);
4038 streamer_write_uhwi (ob
, 0);
4039 streamer_write_uhwi (ob
, es
->param
.length ());
4040 for (i
= 0; i
< (int) es
->param
.length (); i
++)
4041 streamer_write_uhwi (ob
, es
->param
[i
].change_prob
);
4045 /* Write inline summary for node in SET.
4046 Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
4047 active, we don't need to write them twice. */
4050 ipa_fn_summary_write (void)
4052 struct output_block
*ob
= create_output_block (LTO_section_ipa_fn_summary
);
4053 lto_symtab_encoder_t encoder
= ob
->decl_state
->symtab_node_encoder
;
4054 unsigned int count
= 0;
4057 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
4059 symtab_node
*snode
= lto_symtab_encoder_deref (encoder
, i
);
4060 cgraph_node
*cnode
= dyn_cast
<cgraph_node
*> (snode
);
4061 if (cnode
&& cnode
->definition
&& !cnode
->alias
)
4064 streamer_write_uhwi (ob
, count
);
4066 for (i
= 0; i
< lto_symtab_encoder_size (encoder
); i
++)
4068 symtab_node
*snode
= lto_symtab_encoder_deref (encoder
, i
);
4069 cgraph_node
*cnode
= dyn_cast
<cgraph_node
*> (snode
);
4070 if (cnode
&& cnode
->definition
&& !cnode
->alias
)
4072 class ipa_fn_summary
*info
= ipa_fn_summaries
->get (cnode
);
4073 class ipa_size_summary
*size_info
= ipa_size_summaries
->get (cnode
);
4074 struct bitpack_d bp
;
4075 struct cgraph_edge
*edge
;
4078 struct condition
*c
;
4080 streamer_write_uhwi (ob
, lto_symtab_encoder_encode (encoder
, cnode
));
4081 streamer_write_hwi (ob
, size_info
->estimated_self_stack_size
);
4082 streamer_write_hwi (ob
, size_info
->self_size
);
4083 info
->time
.stream_out (ob
);
4084 bp
= bitpack_create (ob
->main_stream
);
4085 bp_pack_value (&bp
, info
->inlinable
, 1);
4086 bp_pack_value (&bp
, false, 1);
4087 bp_pack_value (&bp
, info
->fp_expressions
, 1);
4088 streamer_write_bitpack (&bp
);
4089 streamer_write_uhwi (ob
, vec_safe_length (info
->conds
));
4090 for (i
= 0; vec_safe_iterate (info
->conds
, i
, &c
); i
++)
4093 struct expr_eval_op
*op
;
4095 streamer_write_uhwi (ob
, c
->operand_num
);
4096 streamer_write_uhwi (ob
, c
->code
);
4097 stream_write_tree (ob
, c
->type
, true);
4098 stream_write_tree (ob
, c
->val
, true);
4099 bp
= bitpack_create (ob
->main_stream
);
4100 bp_pack_value (&bp
, c
->agg_contents
, 1);
4101 bp_pack_value (&bp
, c
->by_ref
, 1);
4102 streamer_write_bitpack (&bp
);
4103 if (c
->agg_contents
)
4104 streamer_write_uhwi (ob
, c
->offset
);
4105 streamer_write_uhwi (ob
, vec_safe_length (c
->param_ops
));
4106 for (j
= 0; vec_safe_iterate (c
->param_ops
, j
, &op
); j
++)
4108 streamer_write_uhwi (ob
, op
->code
);
4109 stream_write_tree (ob
, op
->type
, true);
4112 bp
= bitpack_create (ob
->main_stream
);
4113 bp_pack_value (&bp
, op
->index
, 2);
4114 streamer_write_bitpack (&bp
);
4115 stream_write_tree (ob
, op
->val
[0], true);
4117 stream_write_tree (ob
, op
->val
[1], true);
4121 streamer_write_uhwi (ob
, vec_safe_length (info
->size_time_table
));
4122 for (i
= 0; vec_safe_iterate (info
->size_time_table
, i
, &e
); i
++)
4124 streamer_write_uhwi (ob
, e
->size
);
4125 e
->time
.stream_out (ob
);
4126 e
->exec_predicate
.stream_out (ob
);
4127 e
->nonconst_predicate
.stream_out (ob
);
4129 if (info
->loop_iterations
)
4130 info
->loop_iterations
->stream_out (ob
);
4132 streamer_write_uhwi (ob
, 0);
4133 if (info
->loop_stride
)
4134 info
->loop_stride
->stream_out (ob
);
4136 streamer_write_uhwi (ob
, 0);
4137 for (edge
= cnode
->callees
; edge
; edge
= edge
->next_callee
)
4138 write_ipa_call_summary (ob
, edge
);
4139 for (edge
= cnode
->indirect_calls
; edge
; edge
= edge
->next_callee
)
4140 write_ipa_call_summary (ob
, edge
);
4143 streamer_write_char_stream (ob
->main_stream
, 0);
4144 produce_asm (ob
, NULL
);
4145 destroy_output_block (ob
);
4148 ipa_prop_write_jump_functions ();
4152 /* Release function summary. */
4155 ipa_free_fn_summary (void)
4157 if (!ipa_call_summaries
)
4159 ggc_delete (ipa_fn_summaries
);
4160 ipa_fn_summaries
= NULL
;
4161 delete ipa_call_summaries
;
4162 ipa_call_summaries
= NULL
;
4163 edge_predicate_pool
.release ();
4164 /* During IPA this is one of largest datastructures to release. */
4169 /* Release function summary. */
4172 ipa_free_size_summary (void)
4174 if (!ipa_size_summaries
)
4176 delete ipa_size_summaries
;
4177 ipa_size_summaries
= NULL
;
4182 const pass_data pass_data_local_fn_summary
=
4184 GIMPLE_PASS
, /* type */
4185 "local-fnsummary", /* name */
4186 OPTGROUP_INLINE
, /* optinfo_flags */
4187 TV_INLINE_PARAMETERS
, /* tv_id */
4188 0, /* properties_required */
4189 0, /* properties_provided */
4190 0, /* properties_destroyed */
4191 0, /* todo_flags_start */
4192 0, /* todo_flags_finish */
4195 class pass_local_fn_summary
: public gimple_opt_pass
4198 pass_local_fn_summary (gcc::context
*ctxt
)
4199 : gimple_opt_pass (pass_data_local_fn_summary
, ctxt
)
4202 /* opt_pass methods: */
4203 opt_pass
* clone () { return new pass_local_fn_summary (m_ctxt
); }
4204 virtual unsigned int execute (function
*)
4206 return compute_fn_summary_for_current ();
4209 }; // class pass_local_fn_summary
4214 make_pass_local_fn_summary (gcc::context
*ctxt
)
4216 return new pass_local_fn_summary (ctxt
);
4220 /* Free inline summary. */
4224 const pass_data pass_data_ipa_free_fn_summary
=
4226 SIMPLE_IPA_PASS
, /* type */
4227 "free-fnsummary", /* name */
4228 OPTGROUP_NONE
, /* optinfo_flags */
4229 TV_IPA_FREE_INLINE_SUMMARY
, /* tv_id */
4230 0, /* properties_required */
4231 0, /* properties_provided */
4232 0, /* properties_destroyed */
4233 0, /* todo_flags_start */
4234 0, /* todo_flags_finish */
4237 class pass_ipa_free_fn_summary
: public simple_ipa_opt_pass
4240 pass_ipa_free_fn_summary (gcc::context
*ctxt
)
4241 : simple_ipa_opt_pass (pass_data_ipa_free_fn_summary
, ctxt
),
4245 /* opt_pass methods: */
4246 opt_pass
*clone () { return new pass_ipa_free_fn_summary (m_ctxt
); }
4247 void set_pass_param (unsigned int n
, bool param
)
4249 gcc_assert (n
== 0);
4252 virtual bool gate (function
*) { return true; }
4253 virtual unsigned int execute (function
*)
4255 ipa_free_fn_summary ();
4257 ipa_free_size_summary ();
4263 }; // class pass_ipa_free_fn_summary
4267 simple_ipa_opt_pass
*
4268 make_pass_ipa_free_fn_summary (gcc::context
*ctxt
)
4270 return new pass_ipa_free_fn_summary (ctxt
);
4275 const pass_data pass_data_ipa_fn_summary
=
4277 IPA_PASS
, /* type */
4278 "fnsummary", /* name */
4279 OPTGROUP_INLINE
, /* optinfo_flags */
4280 TV_IPA_FNSUMMARY
, /* tv_id */
4281 0, /* properties_required */
4282 0, /* properties_provided */
4283 0, /* properties_destroyed */
4284 0, /* todo_flags_start */
4285 ( TODO_dump_symtab
), /* todo_flags_finish */
4288 class pass_ipa_fn_summary
: public ipa_opt_pass_d
4291 pass_ipa_fn_summary (gcc::context
*ctxt
)
4292 : ipa_opt_pass_d (pass_data_ipa_fn_summary
, ctxt
,
4293 ipa_fn_summary_generate
, /* generate_summary */
4294 ipa_fn_summary_write
, /* write_summary */
4295 ipa_fn_summary_read
, /* read_summary */
4296 NULL
, /* write_optimization_summary */
4297 NULL
, /* read_optimization_summary */
4298 NULL
, /* stmt_fixup */
4299 0, /* function_transform_todo_flags_start */
4300 NULL
, /* function_transform */
4301 NULL
) /* variable_transform */
4304 /* opt_pass methods: */
4305 virtual unsigned int execute (function
*) { return 0; }
4307 }; // class pass_ipa_fn_summary
4312 make_pass_ipa_fn_summary (gcc::context
*ctxt
)
4314 return new pass_ipa_fn_summary (ctxt
);
4317 /* Reset all state within ipa-fnsummary.c so that we can rerun the compiler
4318 within the same process. For use by toplev::finalize. */
4321 ipa_fnsummary_c_finalize (void)
4323 ipa_free_fn_summary ();