/* Inlining decision heuristics.
- Copyright (C) 2003, 2004, 2007, 2008, 2009 Free Software Foundation, Inc.
+ Copyright (C) 2003, 2004, 2007, 2008, 2009, 2010
+ Free Software Foundation, Inc.
Contributed by Jan Hubicka
This file is part of GCC.
INLINE_SIZE,
INLINE_ALL
};
+
static bool
-cgraph_decide_inlining_incrementally (struct cgraph_node *, enum inlining_mode,
- int);
+cgraph_decide_inlining_incrementally (struct cgraph_node *, enum inlining_mode);
+static void cgraph_flatten (struct cgraph_node *node);
/* Statistics we collect about inlining algorithm. */
+ inline_summary (e->callee)->estimated_self_stack_size;
if (e->callee->global.inlined_to->global.estimated_stack_size < peak)
e->callee->global.inlined_to->global.estimated_stack_size = peak;
+ cgraph_propagate_frequency (e->callee);
/* Recursively clone all bodies. */
for (e = e->callee->callees; e; e = e->next_callee)
struct cgraph_node *to = NULL, *what;
struct cgraph_edge *curr = e;
int freq;
- bool duplicate = false;
- int orig_size = e->callee->global.size;
gcc_assert (e->inline_failed);
e->inline_failed = CIF_OK;
DECL_POSSIBLY_INLINED (e->callee->decl) = true;
e->callee->global.inlined = true;
- if (e->callee->callers->next_caller
- || !cgraph_can_remove_if_no_direct_calls_p (e->callee)
- || e->callee->same_comdat_group)
- duplicate = true;
cgraph_clone_inlined_nodes (e, true, update_original);
what = e->callee;
gcc_assert (what->global.inlined_to == to);
if (new_size > old_size)
overall_size += new_size - old_size;
- if (!duplicate)
- overall_size -= orig_size;
ncalls_inlined++;
if (flag_indirect_inlining)
return false;
}
-/* Mark all calls of EDGE->CALLEE inlined into EDGE->CALLER.
- Return following unredirected edge in the list of callers
- of EDGE->CALLEE */
+/* Mark all calls of EDGE->CALLEE inlined into EDGE->CALLER. */
-static struct cgraph_edge *
+static void
cgraph_mark_inline (struct cgraph_edge *edge)
{
struct cgraph_node *to = edge->caller;
edge = next;
}
}
-
- return edge;
}
/* Estimate the growth caused by inlining NODE into all callees. */
{
tree decl = n->decl;
+ if (n->local.disregard_inline_limits)
+ return true;
+
if (!flag_inline_small_functions && !DECL_DECLARED_INLINE_P (decl))
{
if (reason)
of the function or function body size. */
static int
-cgraph_edge_badness (struct cgraph_edge *edge)
+cgraph_edge_badness (struct cgraph_edge *edge, bool dump)
{
gcov_type badness;
int growth =
- cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee);
+ (cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee)
+ - edge->caller->global.size);
- growth -= edge->caller->global.size;
+ if (dump)
+ {
+ fprintf (dump_file, " Badness calculcation for %s -> %s\n",
+ cgraph_node_name (edge->caller),
+ cgraph_node_name (edge->callee));
+ fprintf (dump_file, " growth %i, time %i-%i, size %i-%i\n",
+ growth,
+ edge->callee->global.time,
+ inline_summary (edge->callee)->time_inlining_benefit,
+ edge->callee->global.size,
+ inline_summary (edge->callee)->size_inlining_benefit);
+ }
/* Always prefer inlining saving code size. */
if (growth <= 0)
- badness = INT_MIN - growth;
+ {
+ badness = INT_MIN - growth;
+ if (dump)
+ fprintf (dump_file, " %i: Growth %i < 0\n", (int) badness,
+ growth);
+ }
/* When profiling is available, base priorities -(#calls / growth).
So we optimize for overall number of "executed" inlined calls. */
else if (max_count)
- badness = ((int)((double)edge->count * INT_MIN / max_count / (max_benefit + 1))
- * (inline_summary (edge->callee)->time_inlining_benefit + 1)) / growth;
+ {
+ badness =
+ ((int)
+ ((double) edge->count * INT_MIN / max_count / (max_benefit + 1)) *
+ (inline_summary (edge->callee)->time_inlining_benefit + 1)) / growth;
+ if (dump)
+ {
+ fprintf (dump_file,
+ " %i (relative %f): profile info. Relative count %f"
+ " * Relative benefit %f\n",
+ (int) badness, (double) badness / INT_MIN,
+ (double) edge->count / max_count,
+ (double) (inline_summary (edge->callee)->
+ time_inlining_benefit + 1) / (max_benefit + 1));
+ }
+ }
/* When function local profile is available, base priorities on
growth / frequency, so we optimize for overall frequency of inlined
else if (flag_guess_branch_prob)
{
int div = edge->frequency * 100 / CGRAPH_FREQ_BASE + 1;
+ int benefitperc;
+ int growth_for_all;
badness = growth * 10000;
- div *= MIN (100 * inline_summary (edge->callee)->time_inlining_benefit
- / (edge->callee->global.time + 1) + 1, 100);
+ benefitperc =
+ MIN (100 * inline_summary (edge->callee)->time_inlining_benefit /
+ (edge->callee->global.time + 1) +1, 100);
+ div *= benefitperc;
/* Decrease badness if call is nested. */
div = 1;
if (badness > 0)
badness /= div;
- badness += cgraph_estimate_growth (edge->callee);
+ growth_for_all = cgraph_estimate_growth (edge->callee);
+ badness += growth_for_all;
if (badness > INT_MAX)
- badness = INT_MAX;
+ badness = INT_MAX;
+ if (dump)
+ {
+ fprintf (dump_file,
+ " %i: guessed profile. frequency %i, overall growth %i,"
+ " benefit %i%%, divisor %i\n",
+ (int) badness, edge->frequency, growth_for_all, benefitperc, div);
+ }
}
/* When function local profile is not available or it does not give
useful information (ie frequency is zero), base the cost on
if (badness > 0)
badness >>= nest;
else
- {
+ {
badness <<= nest;
- }
+ }
+ if (dump)
+ fprintf (dump_file, " %i: no profile. nest %i\n", (int) badness,
+ nest);
}
+
+ /* Ensure that we did not overflow in all the fixed point math above. */
+ gcc_assert (badness >= INT_MIN);
+ gcc_assert (badness <= INT_MAX - 1);
/* Make recursive inlining happen always after other inlining is done. */
if (cgraph_recursive_inlining_p (edge->caller, edge->callee, NULL))
return badness + 1;
for (edge = node->callers; edge; edge = edge->next_caller)
if (edge->inline_failed)
{
- int badness = cgraph_edge_badness (edge);
+ int badness = cgraph_edge_badness (edge, false);
if (edge->aux)
{
fibnode_t n = (fibnode_t) edge->aux;
continue;
/* fibheap_replace_key only increase the keys. */
- if (fibheap_replace_key (heap, n, badness))
- continue;
+ if (badness < n->key)
+ {
+ fibheap_replace_key (heap, n, badness);
+ gcc_assert (n->key == badness);
+ continue;
+ }
fibheap_delete_node (heap, (fibnode_t) edge->aux);
}
edge->aux = fibheap_insert (heap, badness, edge);
int depth = 0;
int n = 0;
+ /* It does not make sense to recursively inline always-inline functions
+ as we are going to sorry() on the remaining calls anyway. */
+ if (node->local.disregard_inline_limits
+ && lookup_attribute ("always_inline", DECL_ATTRIBUTES (node->decl)))
+ return false;
+
if (optimize_function_for_size_p (DECL_STRUCT_FUNCTION (node->decl))
|| (!flag_inline_functions && !DECL_DECLARED_INLINE_P (node->decl)))
return false;
struct cgraph_edge *edge = VEC_pop (cgraph_edge_p, new_edges);
gcc_assert (!edge->aux);
- edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge), edge);
+ edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge, false), edge);
}
}
for (node = cgraph_nodes; node; node = node->next)
{
- if (!node->local.inlinable || !node->callers
- || node->local.disregard_inline_limits)
+ if (!node->local.inlinable || !node->callers)
continue;
if (dump_file)
fprintf (dump_file, "Considering inline candidate %s.\n", cgraph_node_name (node));
if (edge->inline_failed)
{
gcc_assert (!edge->aux);
- edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge), edge);
+ edge->aux = fibheap_insert (heap, cgraph_edge_badness (edge, false), edge);
}
}
min_size = overall_size;
while (overall_size <= max_size
- && (edge = (struct cgraph_edge *) fibheap_extract_min (heap)))
+ && !fibheap_empty (heap))
{
int old_size = overall_size;
- struct cgraph_node *where;
- int growth =
- cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee);
+ struct cgraph_node *where, *callee;
+ int badness = fibheap_min_key (heap);
+ int growth;
cgraph_inline_failed_t not_good = CIF_OK;
- growth -= edge->caller->global.size;
+ edge = (struct cgraph_edge *) fibheap_extract_min (heap);
+ gcc_assert (edge->aux);
+ edge->aux = NULL;
+ if (!edge->inline_failed)
+ continue;
+#ifdef ENABLE_CHECKING
+ gcc_assert (cgraph_edge_badness (edge, false) == badness);
+#endif
+ callee = edge->callee;
+
+ growth = (cgraph_estimate_size_after_inlining (1, edge->caller, edge->callee)
+ - edge->caller->global.size);
if (dump_file)
{
" Estimated growth after inlined into all callees is %+i insns.\n"
" Estimated badness is %i, frequency %.2f.\n",
cgraph_node_name (edge->caller),
- gimple_filename ((const_gimple) edge->call_stmt),
- gimple_lineno ((const_gimple) edge->call_stmt),
+ flag_wpa ? "unknown"
+ : gimple_filename ((const_gimple) edge->call_stmt),
+ flag_wpa ? -1 : gimple_lineno ((const_gimple) edge->call_stmt),
cgraph_estimate_growth (edge->callee),
- cgraph_edge_badness (edge),
+ badness,
edge->frequency / (double)CGRAPH_FREQ_BASE);
if (edge->count)
fprintf (dump_file," Called "HOST_WIDEST_INT_PRINT_DEC"x\n", edge->count);
+ if (dump_flags & TDF_DETAILS)
+ cgraph_edge_badness (edge, true);
}
- gcc_assert (edge->aux);
- edge->aux = NULL;
- if (!edge->inline_failed)
- continue;
/* When not having profile info ready we don't weight by any way the
position of call in procedure itself. This means if call of
called by function we inlined (since number of it inlinable callers
might change). */
update_caller_keys (heap, where, updated_nodes);
+
+ /* We removed one call of the function we just inlined. If offline
+ copy is still needed, be sure to update the keys. */
+ if (callee != where && !callee->global.inlined_to)
+ update_caller_keys (heap, callee, updated_nodes);
bitmap_clear (updated_nodes);
if (dump_file)
fprintf (dump_file, "New minimal size reached: %i\n", min_size);
}
}
- while ((edge = (struct cgraph_edge *) fibheap_extract_min (heap)) != NULL)
+ while (!fibheap_empty (heap))
{
+ int badness = fibheap_min_key (heap);
+
+ edge = (struct cgraph_edge *) fibheap_extract_min (heap);
gcc_assert (edge->aux);
edge->aux = NULL;
+ if (!edge->inline_failed)
+ continue;
+#ifdef ENABLE_CHECKING
+ gcc_assert (cgraph_edge_badness (edge, false) == badness);
+#endif
+ if (dump_file)
+ {
+ fprintf (dump_file,
+ "\nSkipping %s with %i size\n",
+ cgraph_node_name (edge->callee),
+ edge->callee->global.size);
+ fprintf (dump_file,
+ " called by %s in %s:%i\n"
+ " Estimated growth after inlined into all callees is %+i insns.\n"
+ " Estimated badness is %i, frequency %.2f.\n",
+ cgraph_node_name (edge->caller),
+ flag_wpa ? "unknown"
+ : gimple_filename ((const_gimple) edge->call_stmt),
+ flag_wpa ? -1 : gimple_lineno ((const_gimple) edge->call_stmt),
+ cgraph_estimate_growth (edge->callee),
+ badness,
+ edge->frequency / (double)CGRAPH_FREQ_BASE);
+ if (edge->count)
+ fprintf (dump_file," Called "HOST_WIDEST_INT_PRINT_DEC"x\n", edge->count);
+ if (dump_flags & TDF_DETAILS)
+ cgraph_edge_badness (edge, true);
+ }
if (!edge->callee->local.disregard_inline_limits && edge->inline_failed
&& !cgraph_recursive_inlining_p (edge->caller, edge->callee,
&edge->inline_failed))
BITMAP_FREE (updated_nodes);
}
+/* Flatten NODE from the IPA inliner. */
+
+static void
+cgraph_flatten (struct cgraph_node *node)
+{
+ struct cgraph_edge *e;
+
+ /* We shouldn't be called recursively when we are being processed. */
+ gcc_assert (node->aux == NULL);
+
+ node->aux = (void *)(size_t) INLINE_ALL;
+
+ for (e = node->callees; e; e = e->next_callee)
+ {
+ struct cgraph_node *orig_callee;
+
+ if (e->call_stmt_cannot_inline_p)
+ continue;
+
+ if (!e->callee->analyzed)
+ {
+ if (dump_file)
+ fprintf (dump_file,
+ "Not inlining: Function body not available.\n");
+ continue;
+ }
+
+ /* We've hit cycle? It is time to give up. */
+ if (e->callee->aux)
+ {
+ if (dump_file)
+ fprintf (dump_file,
+ "Not inlining %s into %s to avoid cycle.\n",
+ cgraph_node_name (e->callee),
+ cgraph_node_name (e->caller));
+ e->inline_failed = CIF_RECURSIVE_INLINING;
+ continue;
+ }
+
+ /* When the edge is already inlined, we just need to recurse into
+ it in order to fully flatten the leaves. */
+ if (!e->inline_failed)
+ {
+ cgraph_flatten (e->callee);
+ continue;
+ }
+
+ if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not inlining: recursive call.\n");
+ continue;
+ }
+
+ if (!tree_can_inline_p (e))
+ {
+ if (dump_file)
+ fprintf (dump_file, "Not inlining: %s",
+ cgraph_inline_failed_string (e->inline_failed));
+ continue;
+ }
+
+ /* Inline the edge and flatten the inline clone. Avoid
+ recursing through the original node if the node was cloned. */
+ if (dump_file)
+ fprintf (dump_file, " Inlining %s into %s.\n",
+ cgraph_node_name (e->callee),
+ cgraph_node_name (e->caller));
+ orig_callee = e->callee;
+ cgraph_mark_inline_edge (e, true, NULL);
+ if (e->callee != orig_callee)
+ orig_callee->aux = (void *)(size_t) INLINE_ALL;
+ cgraph_flatten (e->callee);
+ if (e->callee != orig_callee)
+ orig_callee->aux = NULL;
+ }
+
+ node->aux = NULL;
+}
+
/* Decide on the inlining. We do so in the topological order to avoid
expenses on updating data structures. */
XCNEWVEC (struct cgraph_node *, cgraph_n_nodes);
int old_size = 0;
int i;
- bool redo_always_inline = true;
int initial_size = 0;
cgraph_remove_function_insertion_hook (function_insertion_hook_holder);
if (in_lto_p && flag_indirect_inlining)
ipa_update_after_lto_read ();
+ if (flag_indirect_inlining)
+ ipa_create_all_structures_for_iinln ();
max_count = 0;
max_benefit = 0;
node->aux = 0;
if (dump_file)
- fprintf (dump_file, "\nInlining always_inline functions:\n");
+ fprintf (dump_file, "\nFlattening functions:\n");
- /* In the first pass mark all always_inline edges. Do this with a priority
- so none of our later choices will make this impossible. */
- while (redo_always_inline)
+ /* In the first pass handle functions to be flattened. Do this with
+ a priority so none of our later choices will make this impossible. */
+ for (i = nnodes - 1; i >= 0; i--)
{
- redo_always_inline = false;
- for (i = nnodes - 1; i >= 0; i--)
+ node = order[i];
+
+ /* Handle nodes to be flattened, but don't update overall unit
+ size. Calling the incremental inliner here is lame,
+ a simple worklist should be enough. What should be left
+ here from the early inliner (if it runs) is cyclic cases.
+ Ideally when processing callees we stop inlining at the
+ entry of cycles, possibly cloning that entry point and
+ try to flatten itself turning it into a self-recursive
+ function. */
+ if (lookup_attribute ("flatten",
+ DECL_ATTRIBUTES (node->decl)) != NULL)
{
- struct cgraph_edge *e, *next;
-
- node = order[i];
-
- /* Handle nodes to be flattened, but don't update overall unit
- size. */
- if (lookup_attribute ("flatten",
- DECL_ATTRIBUTES (node->decl)) != NULL)
- {
- if (dump_file)
- fprintf (dump_file,
- "Flattening %s\n", cgraph_node_name (node));
- cgraph_decide_inlining_incrementally (node, INLINE_ALL, 0);
- }
-
- if (!node->local.disregard_inline_limits)
- continue;
- if (dump_file)
- fprintf (dump_file,
- "\nConsidering %s size:%i (always inline)\n",
- cgraph_node_name (node), node->global.size);
- old_size = overall_size;
- for (e = node->callers; e; e = next)
- {
- next = e->next_caller;
- if (!e->inline_failed || e->call_stmt_cannot_inline_p)
- continue;
- if (cgraph_recursive_inlining_p (e->caller, e->callee,
- &e->inline_failed))
- continue;
- if (!tree_can_inline_p (e))
- continue;
- if (cgraph_mark_inline_edge (e, true, NULL))
- redo_always_inline = true;
- if (dump_file)
- fprintf (dump_file,
- " Inlined into %s which now has size %i.\n",
- cgraph_node_name (e->caller),
- e->caller->global.size);
- }
- /* Inlining self recursive function might introduce new calls to
- themselves we didn't see in the loop above. Fill in the proper
- reason why inline failed. */
- for (e = node->callers; e; e = e->next_caller)
- if (e->inline_failed)
- e->inline_failed = CIF_RECURSIVE_INLINING;
if (dump_file)
fprintf (dump_file,
- " Inlined for a net change of %+i size.\n",
- overall_size - old_size);
+ "Flattening %s\n", cgraph_node_name (node));
+ cgraph_flatten (node);
}
}
if (cgraph_check_inline_limits (node->callers->caller, node,
&reason, false))
{
+ struct cgraph_node *caller = node->callers->caller;
cgraph_mark_inline (node->callers);
if (dump_file)
fprintf (dump_file,
" Inlined into %s which now has %i size"
" for a net change of %+i size.\n",
- cgraph_node_name (node->callers->caller),
- node->callers->caller->global.size,
+ cgraph_node_name (caller),
+ caller->global.size,
overall_size - old_size);
}
else
/* Free ipa-prop structures if they are no longer needed. */
if (flag_indirect_inlining)
- free_all_ipa_structures_after_iinln ();
+ ipa_free_all_structures_after_iinln ();
if (dump_file)
fprintf (dump_file,
return 0;
}
-/* Try to inline edge E from incremental inliner. MODE specifies mode
- of inliner.
-
- We are detecting cycles by storing mode of inliner into cgraph_node last
- time we visited it in the recursion. In general when mode is set, we have
- recursive inlining, but as an special case, we want to try harder inline
- ALWAYS_INLINE functions: consider callgraph a->b->c->b, with a being
- flatten, b being always inline. Flattening 'a' will collapse
- a->b->c before hitting cycle. To accommodate always inline, we however
- need to inline a->b->c->b.
-
- So after hitting cycle first time, we switch into ALWAYS_INLINE mode and
- stop inlining only after hitting ALWAYS_INLINE in ALWAY_INLINE mode. */
-static bool
-try_inline (struct cgraph_edge *e, enum inlining_mode mode, int depth)
-{
- struct cgraph_node *callee = e->callee;
- enum inlining_mode callee_mode = (enum inlining_mode) (size_t) callee->aux;
- bool always_inline = e->callee->local.disregard_inline_limits;
- bool inlined = false;
-
- /* We've hit cycle? */
- if (callee_mode)
- {
- /* It is first time we see it and we are not in ALWAY_INLINE only
- mode yet. and the function in question is always_inline. */
- if (always_inline && mode != INLINE_ALWAYS_INLINE)
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Hit cycle in %s, switching to always inline only.\n",
- cgraph_node_name (callee));
- }
- mode = INLINE_ALWAYS_INLINE;
- }
- /* Otherwise it is time to give up. */
- else
- {
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Not inlining %s into %s to avoid cycle.\n",
- cgraph_node_name (callee),
- cgraph_node_name (e->caller));
- }
- e->inline_failed = (e->callee->local.disregard_inline_limits
- ? CIF_RECURSIVE_INLINING : CIF_UNSPECIFIED);
- return false;
- }
- }
-
- callee->aux = (void *)(size_t) mode;
- if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, " Inlining %s into %s.\n",
- cgraph_node_name (e->callee),
- cgraph_node_name (e->caller));
- }
- if (e->inline_failed)
- {
- cgraph_mark_inline (e);
-
- /* In order to fully inline always_inline functions, we need to
- recurse here, since the inlined functions might not be processed by
- incremental inlining at all yet.
-
- Also flattening needs to be done recursively. */
-
- if (mode == INLINE_ALL || always_inline)
- cgraph_decide_inlining_incrementally (e->callee, mode, depth + 1);
- inlined = true;
- }
- callee->aux = (void *)(size_t) callee_mode;
- return inlined;
-}
-
/* Return true when N is leaf function. Accept cheap (pure&const) builtins
in leaf functions. */
static bool
}
/* Decide on the inlining. We do so in the topological order to avoid
- expenses on updating data structures.
- DEPTH is depth of recursion, used only for debug output. */
+ expenses on updating data structures. */
static bool
cgraph_decide_inlining_incrementally (struct cgraph_node *node,
- enum inlining_mode mode,
- int depth)
+ enum inlining_mode mode)
{
struct cgraph_edge *e;
bool inlined = false;
cgraph_inline_failed_t failed_reason;
- enum inlining_mode old_mode;
#ifdef ENABLE_CHECKING
verify_cgraph_node (node);
#endif
- old_mode = (enum inlining_mode) (size_t)node->aux;
-
if (mode != INLINE_ALWAYS_INLINE && mode != INLINE_SIZE_NORECURSIVE
&& lookup_attribute ("flatten", DECL_ATTRIBUTES (node->decl)) != NULL)
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Flattening %s\n", cgraph_node_name (node));
- }
+ fprintf (dump_file, "Incrementally flattening %s\n",
+ cgraph_node_name (node));
mode = INLINE_ALL;
}
- node->aux = (void *)(size_t) mode;
-
/* First of all look for always inline functions. */
if (mode != INLINE_SIZE_NORECURSIVE)
for (e = node->callees; e; e = e->next_callee)
continue;
if (e->call_stmt_cannot_inline_p)
continue;
- /* When the edge is already inlined, we just need to recurse into
- it in order to fully flatten the leaves. */
- if (!e->inline_failed && mode == INLINE_ALL)
- {
- inlined |= try_inline (e, mode, depth);
- continue;
- }
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Considering to always inline inline candidate %s.\n",
- cgraph_node_name (e->callee));
- }
+ fprintf (dump_file,
+ "Considering to always inline inline candidate %s.\n",
+ cgraph_node_name (e->callee));
if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Not inlining: recursive call.\n");
- }
+ fprintf (dump_file, "Not inlining: recursive call.\n");
continue;
}
if (!tree_can_inline_p (e))
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Not inlining: %s",
- cgraph_inline_failed_string (e->inline_failed));
- }
+ fprintf (dump_file,
+ "Not inlining: %s",
+ cgraph_inline_failed_string (e->inline_failed));
continue;
}
if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
!= gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Not inlining: SSA form does not match.\n");
- }
+ fprintf (dump_file, "Not inlining: SSA form does not match.\n");
continue;
}
if (!e->callee->analyzed)
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Not inlining: Function body no longer available.\n");
- }
+ fprintf (dump_file,
+ "Not inlining: Function body no longer available.\n");
continue;
}
- inlined |= try_inline (e, mode, depth);
+
+ if (dump_file)
+ fprintf (dump_file, " Inlining %s into %s.\n",
+ cgraph_node_name (e->callee),
+ cgraph_node_name (e->caller));
+ cgraph_mark_inline (e);
+ inlined = true;
}
/* Now do the automatic inlining. */
if (cgraph_recursive_inlining_p (node, e->callee, &e->inline_failed))
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Not inlining: recursive call.\n");
- }
+ fprintf (dump_file, "Not inlining: recursive call.\n");
continue;
}
if (gimple_in_ssa_p (DECL_STRUCT_FUNCTION (node->decl))
!= gimple_in_ssa_p (DECL_STRUCT_FUNCTION (e->callee->decl)))
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Not inlining: SSA form does not match.\n");
- }
+ fprintf (dump_file,
+ "Not inlining: SSA form does not match.\n");
continue;
}
&& cgraph_estimate_growth (e->callee) > allowed_growth)
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Not inlining: code size would grow by %i.\n",
- cgraph_estimate_size_after_inlining (1, e->caller,
- e->callee)
- - e->caller->global.size);
- }
+ fprintf (dump_file,
+ "Not inlining: code size would grow by %i.\n",
+ cgraph_estimate_size_after_inlining (1, e->caller,
+ e->callee)
+ - e->caller->global.size);
continue;
}
if (!cgraph_check_inline_limits (node, e->callee, &e->inline_failed,
|| e->call_stmt_cannot_inline_p)
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file, "Not inlining: %s.\n",
- cgraph_inline_failed_string (e->inline_failed));
- }
+ fprintf (dump_file, "Not inlining: %s.\n",
+ cgraph_inline_failed_string (e->inline_failed));
continue;
}
if (!e->callee->analyzed)
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Not inlining: Function body no longer available.\n");
- }
+ fprintf (dump_file,
+ "Not inlining: Function body no longer available.\n");
continue;
}
if (!tree_can_inline_p (e))
{
if (dump_file)
- {
- indent_to (dump_file, depth);
- fprintf (dump_file,
- "Not inlining: %s.",
- cgraph_inline_failed_string (e->inline_failed));
- }
+ fprintf (dump_file,
+ "Not inlining: %s.",
+ cgraph_inline_failed_string (e->inline_failed));
continue;
}
if (cgraph_default_inline_p (e->callee, &failed_reason))
- inlined |= try_inline (e, mode, depth);
+ {
+ if (dump_file)
+ fprintf (dump_file, " Inlining %s into %s.\n",
+ cgraph_node_name (e->callee),
+ cgraph_node_name (e->caller));
+ cgraph_mark_inline (e);
+ inlined = true;
+ }
}
BITMAP_FREE (visited);
}
- node->aux = (void *)(size_t) old_mode;
return inlined;
}
if (sorrycount || errorcount)
return 0;
- while (iterations < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS)
- && cgraph_decide_inlining_incrementally (node,
- iterations
- ? INLINE_SIZE_NORECURSIVE : INLINE_SIZE, 0))
+
+ if (!optimize
+ || flag_no_inline
+ || !flag_early_inlining)
{
+ /* When not optimizing or not inlining inline only always-inline
+ functions. */
+ cgraph_decide_inlining_incrementally (node, INLINE_ALWAYS_INLINE);
timevar_push (TV_INTEGRATION);
todo |= optimize_inline_calls (current_function_decl);
- iterations++;
timevar_pop (TV_INTEGRATION);
}
- if (dump_file)
- fprintf (dump_file, "Iterations: %i\n", iterations);
+ else
+ {
+ if (lookup_attribute ("flatten",
+ DECL_ATTRIBUTES (node->decl)) != NULL)
+ {
+ if (dump_file)
+ fprintf (dump_file,
+ "Flattening %s\n", cgraph_node_name (node));
+ cgraph_flatten (node);
+ timevar_push (TV_INTEGRATION);
+ todo |= optimize_inline_calls (current_function_decl);
+ timevar_pop (TV_INTEGRATION);
+ }
+ /* We iterate incremental inlining to get trivial cases of indirect
+ inlining. */
+ while (iterations < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS)
+ && cgraph_decide_inlining_incrementally (node,
+ iterations
+ ? INLINE_SIZE_NORECURSIVE
+ : INLINE_SIZE))
+ {
+ timevar_push (TV_INTEGRATION);
+ todo |= optimize_inline_calls (current_function_decl);
+ iterations++;
+ timevar_pop (TV_INTEGRATION);
+ }
+ if (dump_file)
+ fprintf (dump_file, "Iterations: %i\n", iterations);
+ }
+
cfun->always_inline_functions_inlined = true;
- return todo;
-}
-/* When inlining shall be performed. */
-static bool
-cgraph_gate_early_inlining (void)
-{
- return flag_early_inlining;
+ return todo;
}
struct gimple_opt_pass pass_early_inline =
{
GIMPLE_PASS,
"einline", /* name */
- cgraph_gate_early_inlining, /* gate */
+ NULL, /* gate */
cgraph_early_inlining, /* execute */
NULL, /* sub */
NULL, /* next */
int freq;
tree funtype = TREE_TYPE (node->decl);
+ if (node->local.disregard_inline_limits)
+ {
+ inline_summary (node)->self_time = 0;
+ inline_summary (node)->self_size = 0;
+ inline_summary (node)->time_inlining_benefit = 0;
+ inline_summary (node)->size_inlining_benefit = 0;
+ }
+
if (dump_file)
fprintf (dump_file, "Analyzing function body size: %s\n",
cgraph_node_name (node));
active, we don't need to write them twice. */
static void
-inline_write_summary (cgraph_node_set set)
+inline_write_summary (cgraph_node_set set,
+ varpool_node_set vset ATTRIBUTE_UNUSED)
{
if (flag_indirect_inlining && !flag_ipa_cp)
ipa_prop_write_jump_functions (set);
}
+/* When to run IPA inlining. Inlining of always-inline functions
+ happens during early inlining. */
+
+static bool
+gate_cgraph_decide_inlining (void)
+{
+ /* ??? We'd like to skip this if not optimizing or not inlining as
+ all always-inline functions have been processed by early
+ inlining already. But this at least breaks EH with C++ as
+ we need to unconditionally run fixup_cfg even at -O0.
+ So leave it on unconditionally for now. */
+ return 1;
+}
+
struct ipa_opt_pass_d pass_ipa_inline =
{
{
IPA_PASS,
"inline", /* name */
- NULL, /* gate */
+ gate_cgraph_decide_inlining, /* gate */
cgraph_decide_inlining, /* execute */
NULL, /* sub */
NULL, /* next */
inline_generate_summary, /* generate_summary */
inline_write_summary, /* write_summary */
inline_read_summary, /* read_summary */
- NULL, /* function_read_summary */
- lto_ipa_fixup_call_notes, /* stmt_fixup */
+ NULL, /* write_optimization_summary */
+ NULL, /* read_optimization_summary */
+ NULL, /* stmt_fixup */
0, /* TODOs */
inline_transform, /* function_transform */
NULL, /* variable_transform */