/* Inlining decision heuristics.
- Copyright (C) 2003-2014 Free Software Foundation, Inc.
+ Copyright (C) 2003-2015 Free Software Foundation, Inc.
Contributed by Jan Hubicka
This file is part of GCC.
#include "system.h"
#include "coretypes.h"
#include "tm.h"
+#include "hash-set.h"
+#include "machmode.h"
+#include "vec.h"
+#include "double-int.h"
+#include "input.h"
+#include "alias.h"
+#include "symtab.h"
+#include "wide-int.h"
+#include "inchash.h"
#include "tree.h"
+#include "fold-const.h"
#include "trans-mem.h"
#include "calls.h"
#include "tree-inline.h"
#include "diagnostic.h"
#include "gimple-pretty-print.h"
#include "params.h"
-#include "fibheap.h"
#include "intl.h"
#include "tree-pass.h"
#include "coverage.h"
#include "rtl.h"
#include "bitmap.h"
+#include "profile.h"
+#include "predict.h"
+#include "hard-reg-set.h"
+#include "input.h"
+#include "function.h"
#include "basic-block.h"
#include "tree-ssa-alias.h"
#include "internal-fn.h"
#include "is-a.h"
#include "gimple.h"
#include "gimple-ssa.h"
+#include "hash-map.h"
+#include "plugin-api.h"
+#include "ipa-ref.h"
+#include "cgraph.h"
+#include "alloc-pool.h"
+#include "symbol-summary.h"
#include "ipa-prop.h"
#include "except.h"
#include "target.h"
#include "ipa-inline.h"
#include "ipa-utils.h"
#include "sreal.h"
-#include "cilk.h"
+#include "auto-profile.h"
+#include "builtins.h"
+#include "fibonacci_heap.h"
+#include "lto-streamer.h"
+
+typedef fibonacci_heap <sreal, cgraph_edge> edge_heap_t;
+typedef fibonacci_node <sreal, cgraph_edge> edge_heap_node_t;
/* Statistics we collect about inlining algorithm. */
static int overall_size;
static gcov_type max_count;
-static sreal max_count_real, max_relbenefit_real, half_int_min_real;
static gcov_type spec_rem;
+/* Pre-computed constants 1/CGRAPH_FREQ_BASE and 1/100. */
+static sreal cgraph_freq_base_rec, percent_rec;
+
/* Return false when inlining edge E would lead to violating
limits on function unit growth or stack usage growth.
caller_growth_limits (struct cgraph_edge *e)
{
struct cgraph_node *to = e->caller;
- struct cgraph_node *what = cgraph_function_or_thunk_node (e->callee, NULL);
+ struct cgraph_node *what = e->callee->ultimate_alias_target ();
int newsize;
int limit = 0;
HOST_WIDE_INT stack_size_limit = 0, inlined_stack;
- struct inline_summary *info, *what_info, *outer_info = inline_summary (to);
+ inline_summary *info, *what_info, *outer_info = inline_summaries->get (to);
/* Look for function e->caller is inlined to. While doing
so work out the largest function body on the way. As
too much in order to prevent compiler from exploding". */
while (true)
{
- info = inline_summary (to);
+ info = inline_summaries->get (to);
if (limit < info->self_size)
limit = info->self_size;
if (stack_size_limit < info->estimated_self_stack_size)
break;
}
- what_info = inline_summary (what);
+ what_info = inline_summaries->get (what);
if (limit < what_info->self_size)
limit = what_info->self_size;
if (dump_file)
{
fprintf (dump_file, " not inlinable: %s/%i -> %s/%i, %s\n",
- xstrdup (e->caller->name ()), e->caller->order,
- xstrdup (e->callee->name ()), e->callee->order,
+ xstrdup_for_dump (e->caller->name ()), e->caller->order,
+ xstrdup_for_dump (e->callee->name ()), e->callee->order,
cgraph_inline_failed_string (e->inline_failed));
+ if ((e->inline_failed == CIF_TARGET_OPTION_MISMATCH
+ || e->inline_failed == CIF_OPTIMIZATION_MISMATCH)
+ && e->caller->lto_file_data
+ && e->callee->function_symbol ()->lto_file_data)
+ {
+ fprintf (dump_file, " LTO objects: %s, %s\n",
+ e->caller->lto_file_data->file_name,
+ e->callee->function_symbol ()->lto_file_data->file_name);
+ }
+ if (e->inline_failed == CIF_TARGET_OPTION_MISMATCH)
+ cl_target_option_print_diff
+ (dump_file, 2, target_opts_for_fn (e->caller->decl),
+ target_opts_for_fn (e->callee->ultimate_alias_target ()->decl));
+ if (e->inline_failed == CIF_OPTIMIZATION_MISMATCH)
+ cl_optimization_print_diff
+ (dump_file, 2, opts_for_fn (e->caller->decl),
+ opts_for_fn (e->callee->ultimate_alias_target ()->decl));
}
}
DECL_ATTRIBUTES (callee));
}
+/* Used for flags where it is safe to inline when caller's value is
+ grater than callee's. */
+#define check_maybe_up(flag) \
+ (opts_for_fn (caller->decl)->x_##flag \
+ != opts_for_fn (callee->decl)->x_##flag \
+ && (!always_inline \
+ || opts_for_fn (caller->decl)->x_##flag \
+ < opts_for_fn (callee->decl)->x_##flag))
+/* Used for flags where it is safe to inline when caller's value is
+ smaller than callee's. */
+#define check_maybe_down(flag) \
+ (opts_for_fn (caller->decl)->x_##flag \
+ != opts_for_fn (callee->decl)->x_##flag \
+ && (!always_inline \
+ || opts_for_fn (caller->decl)->x_##flag \
+ > opts_for_fn (callee->decl)->x_##flag))
+/* Used for flags where exact match is needed for correctness. */
+#define check_match(flag) \
+ (opts_for_fn (caller->decl)->x_##flag \
+ != opts_for_fn (callee->decl)->x_##flag)
+
/* Decide if we can inline the edge and possibly update
inline_failed reason.
We check whether inlining is possible at all and whether
static bool
can_inline_edge_p (struct cgraph_edge *e, bool report,
- bool disregard_limits = false)
+ bool disregard_limits = false, bool early = false)
{
+ gcc_checking_assert (e->inline_failed);
+
+ if (cgraph_inline_failed_type (e->inline_failed) == CIF_FINAL_ERROR)
+ {
+ if (report)
+ report_inline_failed_reason (e);
+ return false;
+ }
+
bool inlinable = true;
enum availability avail;
- struct cgraph_node *callee
- = cgraph_function_or_thunk_node (e->callee, &avail);
- tree caller_tree = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (e->caller->decl);
+ cgraph_node *callee = e->callee->ultimate_alias_target (&avail);
+ cgraph_node *caller = e->caller->global.inlined_to
+ ? e->caller->global.inlined_to : e->caller;
+ tree caller_tree = DECL_FUNCTION_SPECIFIC_OPTIMIZATION (caller->decl);
tree callee_tree
= callee ? DECL_FUNCTION_SPECIFIC_OPTIMIZATION (callee->decl) : NULL;
- struct function *caller_cfun = DECL_STRUCT_FUNCTION (e->caller->decl);
- struct function *callee_cfun
- = callee ? DECL_STRUCT_FUNCTION (callee->decl) : NULL;
-
- if (!caller_cfun && e->caller->clone_of)
- caller_cfun = DECL_STRUCT_FUNCTION (e->caller->clone_of->decl);
-
- if (!callee_cfun && callee && callee->clone_of)
- callee_cfun = DECL_STRUCT_FUNCTION (callee->clone_of->decl);
- gcc_assert (e->inline_failed);
-
- if (!callee || !callee->definition)
+ if (!callee->definition)
{
e->inline_failed = CIF_BODY_NOT_AVAILABLE;
inlinable = false;
e->inline_failed = CIF_USES_COMDAT_LOCAL;
inlinable = false;
}
- else if (!inline_summary (callee)->inlinable
- || (caller_cfun && fn_contains_cilk_spawn_p (caller_cfun)))
- {
- e->inline_failed = CIF_FUNCTION_NOT_INLINABLE;
- inlinable = false;
- }
- else if (avail <= AVAIL_OVERWRITABLE)
+ else if (avail <= AVAIL_INTERPOSABLE)
{
e->inline_failed = CIF_OVERWRITABLE;
inlinable = false;
inlinable = false;
}
/* Don't inline if the functions have different EH personalities. */
- else if (DECL_FUNCTION_PERSONALITY (e->caller->decl)
+ else if (DECL_FUNCTION_PERSONALITY (caller->decl)
&& DECL_FUNCTION_PERSONALITY (callee->decl)
- && (DECL_FUNCTION_PERSONALITY (e->caller->decl)
+ && (DECL_FUNCTION_PERSONALITY (caller->decl)
!= DECL_FUNCTION_PERSONALITY (callee->decl)))
{
e->inline_failed = CIF_EH_PERSONALITY;
}
/* TM pure functions should not be inlined into non-TM_pure
functions. */
- else if (is_tm_pure (callee->decl)
- && !is_tm_pure (e->caller->decl))
+ else if (is_tm_pure (callee->decl) && !is_tm_pure (caller->decl))
{
e->inline_failed = CIF_UNSPECIFIED;
inlinable = false;
}
- /* Don't inline if the callee can throw non-call exceptions but the
- caller cannot.
- FIXME: this is obviously wrong for LTO where STRUCT_FUNCTION is missing.
- Move the flag into cgraph node or mirror it in the inline summary. */
- else if (callee_cfun && callee_cfun->can_throw_non_call_exceptions
- && !(caller_cfun && caller_cfun->can_throw_non_call_exceptions))
- {
- e->inline_failed = CIF_NON_CALL_EXCEPTIONS;
- inlinable = false;
- }
/* Check compatibility of target optimization options. */
- else if (!targetm.target_option.can_inline_p (e->caller->decl,
+ else if (!targetm.target_option.can_inline_p (caller->decl,
callee->decl))
{
e->inline_failed = CIF_TARGET_OPTION_MISMATCH;
inlinable = false;
}
+ else if (!inline_summaries->get (callee)->inlinable)
+ {
+ e->inline_failed = CIF_FUNCTION_NOT_INLINABLE;
+ inlinable = false;
+ }
+ else if (inline_summaries->get (caller)->contains_cilk_spawn)
+ {
+ e->inline_failed = CIF_CILK_SPAWN;
+ inlinable = false;
+ }
/* Don't inline a function with mismatched sanitization attributes. */
- else if (!sanitize_attrs_match_for_inline_p (e->caller->decl, callee->decl))
+ else if (!sanitize_attrs_match_for_inline_p (caller->decl, callee->decl))
{
e->inline_failed = CIF_ATTRIBUTE_MISMATCH;
inlinable = false;
else if (!DECL_DISREGARD_INLINE_LIMITS (callee->decl)
&& !disregard_limits
&& !lookup_attribute ("flatten",
- DECL_ATTRIBUTES
- (e->caller->global.inlined_to
- ? e->caller->global.inlined_to->decl
- : e->caller->decl))
+ DECL_ATTRIBUTES (caller->decl))
&& !caller_growth_limits (e))
inlinable = false;
/* Don't inline a function with a higher optimization level than the
optimization attribute. */
else if (caller_tree != callee_tree)
{
- struct cl_optimization *caller_opt
- = TREE_OPTIMIZATION ((caller_tree)
- ? caller_tree
- : optimization_default_node);
-
- struct cl_optimization *callee_opt
- = TREE_OPTIMIZATION ((callee_tree)
- ? callee_tree
- : optimization_default_node);
-
- if (((caller_opt->x_optimize > callee_opt->x_optimize)
- || (caller_opt->x_optimize_size != callee_opt->x_optimize_size))
- /* gcc.dg/pr43564.c. Look at forced inline even in -O0. */
- && !DECL_DISREGARD_INLINE_LIMITS (e->callee->decl))
+ bool always_inline =
+ (DECL_DISREGARD_INLINE_LIMITS (callee->decl)
+ && lookup_attribute ("always_inline",
+ DECL_ATTRIBUTES (callee->decl)));
+
+ /* There are some options that change IL semantics which means
+ we cannot inline in these cases for correctness reason.
+ Not even for always_inline declared functions. */
+ /* Strictly speaking only when the callee contains signed integer
+ math where overflow is undefined. */
+ if ((check_maybe_up (flag_strict_overflow)
+ /* this flag is set by optimize. Allow inlining across
+ optimize boundary. */
+ && (!opt_for_fn (caller->decl, optimize)
+ == !opt_for_fn (callee->decl, optimize) || !always_inline))
+ || check_match (flag_wrapv)
+ || check_match (flag_trapv)
+ /* Strictly speaking only when the callee contains memory
+ accesses that are not using alias-set zero anyway. */
+ || check_maybe_down (flag_strict_aliasing)
+ /* Strictly speaking only when the callee uses FP math. */
+ || check_maybe_up (flag_rounding_math)
+ || check_maybe_up (flag_trapping_math)
+ || check_maybe_down (flag_unsafe_math_optimizations)
+ || check_maybe_down (flag_finite_math_only)
+ || check_maybe_up (flag_signaling_nans)
+ || check_maybe_down (flag_cx_limited_range)
+ || check_maybe_up (flag_signed_zeros)
+ || check_maybe_down (flag_associative_math)
+ || check_maybe_down (flag_reciprocal_math)
+ /* We do not want to make code compiled with exceptions to be brought
+ into a non-EH function unless we know that the callee does not
+ throw. This is tracked by DECL_FUNCTION_PERSONALITY. */
+ || (check_match (flag_non_call_exceptions)
+ /* TODO: We also may allow bringing !flag_non_call_exceptions
+ to flag_non_call_exceptions function, but that may need
+ extra work in tree-inline to add the extra EH edges. */
+ && (!opt_for_fn (callee->decl, flag_non_call_exceptions)
+ || DECL_FUNCTION_PERSONALITY (callee->decl)))
+ || (check_maybe_up (flag_exceptions)
+ && DECL_FUNCTION_PERSONALITY (callee->decl))
+ /* Strictly speaking only when the callee contains function
+ calls that may end up setting errno. */
+ || check_maybe_up (flag_errno_math)
+ /* When devirtualization is diabled for callee, it is not safe
+ to inline it as we possibly mangled the type info.
+ Allow early inlining of always inlines. */
+ || (!early && check_maybe_down (flag_devirtualize)))
+ {
+ e->inline_failed = CIF_OPTIMIZATION_MISMATCH;
+ inlinable = false;
+ }
+ /* gcc.dg/pr43564.c. Apply user-forced inline even at -O0. */
+ else if (always_inline)
+ ;
+ /* When user added an attribute to the callee honor it. */
+ else if (lookup_attribute ("optimize", DECL_ATTRIBUTES (callee->decl))
+ && opts_for_fn (caller->decl) != opts_for_fn (callee->decl))
{
e->inline_failed = CIF_OPTIMIZATION_MISMATCH;
inlinable = false;
}
+ /* If mismatch is caused by merging two LTO units with different
+ optimizationflags we want to be bit nicer. However never inline
+ if one of functions is not optimized at all. */
+ else if (!opt_for_fn (callee->decl, optimize)
+ || !opt_for_fn (caller->decl, optimize))
+ {
+ e->inline_failed = CIF_OPTIMIZATION_MISMATCH;
+ inlinable = false;
+ }
+ /* If callee is optimized for size and caller is not, allow inlining if
+ code shrinks or we are in MAX_INLINE_INSNS_SINGLE limit and callee
+ is inline (and thus likely an unified comdat). This will allow caller
+ to run faster. */
+ else if (opt_for_fn (callee->decl, optimize_size)
+ > opt_for_fn (caller->decl, optimize_size))
+ {
+ int growth = estimate_edge_growth (e);
+ if (growth > 0
+ && (!DECL_DECLARED_INLINE_P (callee->decl)
+ && growth >= MAX (MAX_INLINE_INSNS_SINGLE,
+ MAX_INLINE_INSNS_AUTO)))
+ {
+ e->inline_failed = CIF_OPTIMIZATION_MISMATCH;
+ inlinable = false;
+ }
+ }
+ /* If callee is more aggressively optimized for performance than caller,
+ we generally want to inline only cheap (runtime wise) functions. */
+ else if (opt_for_fn (callee->decl, optimize_size)
+ < opt_for_fn (caller->decl, optimize_size)
+ || (opt_for_fn (callee->decl, optimize)
+ >= opt_for_fn (caller->decl, optimize)))
+ {
+ if (estimate_edge_time (e)
+ >= 20 + inline_edge_summary (e)->call_stmt_time)
+ {
+ e->inline_failed = CIF_OPTIMIZATION_MISMATCH;
+ inlinable = false;
+ }
+ }
+
}
if (!inlinable && report)
static bool
can_early_inline_edge_p (struct cgraph_edge *e)
{
- struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee,
- NULL);
+ struct cgraph_node *callee = e->callee->ultimate_alias_target ();
/* Early inliner might get called at WPA stage when IPA pass adds new
function. In this case we can not really do any of early inlining
because function bodies are missing. */
fprintf (dump_file, " edge not inlinable: not in SSA form\n");
return false;
}
- if (!can_inline_edge_p (e, true))
+ if (!can_inline_edge_p (e, true, false, true))
return false;
return true;
}
want_early_inline_function_p (struct cgraph_edge *e)
{
bool want_inline = true;
- struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, NULL);
+ struct cgraph_node *callee = e->callee->ultimate_alias_target ();
if (DECL_DISREGARD_INLINE_LIMITS (callee->decl))
;
+ /* For AutoFDO, we need to make sure that before profile summary, all
+ hot paths' IR look exactly the same as profiled binary. As a result,
+ in einliner, we will disregard size limit and inline those callsites
+ that are:
+ * inlined in the profiled binary, and
+ * the cloned callee has enough samples to be considered "hot". */
+ else if (flag_auto_profile && afdo_callsite_hot_enough_for_early_inline (e))
+ ;
else if (!DECL_DECLARED_INLINE_P (callee->decl)
- && !flag_inline_small_functions)
+ && !opt_for_fn (e->caller->decl, flag_inline_small_functions))
{
e->inline_failed = CIF_FUNCTION_NOT_INLINE_CANDIDATE;
report_inline_failed_reason (e);
if (growth <= 0)
;
- else if (!cgraph_maybe_hot_edge_p (e)
+ else if (!e->maybe_hot_p ()
&& growth > 0)
{
if (dump_file)
fprintf (dump_file, " will not early inline: %s/%i->%s/%i, "
"call is cold and code would grow by %i\n",
- xstrdup (e->caller->name ()),
+ xstrdup_for_dump (e->caller->name ()),
e->caller->order,
- xstrdup (callee->name ()), callee->order,
+ xstrdup_for_dump (callee->name ()), callee->order,
growth);
want_inline = false;
}
if (dump_file)
fprintf (dump_file, " will not early inline: %s/%i->%s/%i, "
"growth %i exceeds --param early-inlining-insns\n",
- xstrdup (e->caller->name ()),
+ xstrdup_for_dump (e->caller->name ()),
e->caller->order,
- xstrdup (callee->name ()), callee->order,
+ xstrdup_for_dump (callee->name ()), callee->order,
growth);
want_inline = false;
}
fprintf (dump_file, " will not early inline: %s/%i->%s/%i, "
"growth %i exceeds --param early-inlining-insns "
"divided by number of calls\n",
- xstrdup (e->caller->name ()),
+ xstrdup_for_dump (e->caller->name ()),
e->caller->order,
- xstrdup (callee->name ()), callee->order,
+ xstrdup_for_dump (callee->name ()), callee->order,
growth);
want_inline = false;
}
/* Compute time of the edge->caller + edge->callee execution when inlining
does not happen. */
-inline gcov_type
+inline sreal
compute_uninlined_call_time (struct inline_summary *callee_info,
struct cgraph_edge *edge)
{
- gcov_type uninlined_call_time =
- RDIV ((gcov_type)callee_info->time * MAX (edge->frequency, 1),
- CGRAPH_FREQ_BASE);
- gcov_type caller_time = inline_summary (edge->caller->global.inlined_to
- ? edge->caller->global.inlined_to
- : edge->caller)->time;
+ sreal uninlined_call_time = (sreal)callee_info->time;
+ cgraph_node *caller = (edge->caller->global.inlined_to
+ ? edge->caller->global.inlined_to
+ : edge->caller);
+
+ if (edge->count && caller->count)
+ uninlined_call_time *= (sreal)edge->count / caller->count;
+ if (edge->frequency)
+ uninlined_call_time *= cgraph_freq_base_rec * edge->frequency;
+ else
+ uninlined_call_time = uninlined_call_time >> 11;
+
+ int caller_time = inline_summaries->get (caller)->time;
return uninlined_call_time + caller_time;
}
/* Same as compute_uinlined_call_time but compute time when inlining
does happen. */
-inline gcov_type
+inline sreal
compute_inlined_call_time (struct cgraph_edge *edge,
int edge_time)
{
- gcov_type caller_time = inline_summary (edge->caller->global.inlined_to
- ? edge->caller->global.inlined_to
- : edge->caller)->time;
- gcov_type time = (caller_time
- + RDIV (((gcov_type) edge_time
- - inline_edge_summary (edge)->call_stmt_time)
- * MAX (edge->frequency, 1), CGRAPH_FREQ_BASE));
- /* Possible one roundoff error, but watch for overflows. */
- gcc_checking_assert (time >= INT_MIN / 2);
- if (time < 0)
- time = 0;
+ cgraph_node *caller = (edge->caller->global.inlined_to
+ ? edge->caller->global.inlined_to
+ : edge->caller);
+ int caller_time = inline_summaries->get (caller)->time;
+ sreal time = edge_time;
+
+ if (edge->count && caller->count)
+ time *= (sreal)edge->count / caller->count;
+ if (edge->frequency)
+ time *= cgraph_freq_base_rec * edge->frequency;
+ else
+ time = time >> 11;
+
+ /* This calculation should match one in ipa-inline-analysis.
+ FIXME: Once ipa-inline-analysis is converted to sreal this can be
+ simplified. */
+ time -= (sreal) ((gcov_type) edge->frequency
+ * inline_edge_summary (edge)->call_stmt_time
+ * (INLINE_TIME_SCALE / CGRAPH_FREQ_BASE)) / INLINE_TIME_SCALE;
+ time += caller_time;
+ if (time <= 0)
+ time = ((sreal) 1) >> 8;
+ gcc_checking_assert (time >= 0);
return time;
}
static bool
big_speedup_p (struct cgraph_edge *e)
{
- gcov_type time = compute_uninlined_call_time (inline_summary (e->callee),
- e);
- gcov_type inlined_time = compute_inlined_call_time (e,
- estimate_edge_time (e));
+ sreal time = compute_uninlined_call_time (inline_summaries->get (e->callee),
+ e);
+ sreal inlined_time = compute_inlined_call_time (e, estimate_edge_time (e));
+
if (time - inlined_time
- > RDIV (time * PARAM_VALUE (PARAM_INLINE_MIN_SPEEDUP), 100))
+ > (sreal) time * PARAM_VALUE (PARAM_INLINE_MIN_SPEEDUP)
+ * percent_rec)
return true;
return false;
}
want_inline_small_function_p (struct cgraph_edge *e, bool report)
{
bool want_inline = true;
- struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, NULL);
+ struct cgraph_node *callee = e->callee->ultimate_alias_target ();
if (DECL_DISREGARD_INLINE_LIMITS (callee->decl))
;
else if (!DECL_DECLARED_INLINE_P (callee->decl)
- && !flag_inline_small_functions)
+ && !opt_for_fn (e->caller->decl, flag_inline_small_functions))
{
e->inline_failed = CIF_FUNCTION_NOT_INLINE_CANDIDATE;
want_inline = false;
}
/* Do fast and conservative check if the function can be good
- inline cnadidate. At themoment we allow inline hints to
- promote non-inline function to inline and we increase
- MAX_INLINE_INSNS_SINGLE 16fold for inline functions. */
+ inline candidate. At the moment we allow inline hints to
+ promote non-inline functions to inline and we increase
+ MAX_INLINE_INSNS_SINGLE 16-fold for inline functions. */
else if ((!DECL_DECLARED_INLINE_P (callee->decl)
- && (!e->count || !cgraph_maybe_hot_edge_p (e)))
- && inline_summary (callee)->min_size - inline_edge_summary (e)->call_stmt_size
+ && (!e->count || !e->maybe_hot_p ()))
+ && inline_summaries->get (callee)->min_size
+ - inline_edge_summary (e)->call_stmt_size
> MAX (MAX_INLINE_INSNS_SINGLE, MAX_INLINE_INSNS_AUTO))
{
e->inline_failed = CIF_MAX_INLINE_INSNS_AUTO_LIMIT;
want_inline = false;
}
else if ((DECL_DECLARED_INLINE_P (callee->decl) || e->count)
- && inline_summary (callee)->min_size - inline_edge_summary (e)->call_stmt_size
+ && inline_summaries->get (callee)->min_size
+ - inline_edge_summary (e)->call_stmt_size
> 16 * MAX_INLINE_INSNS_SINGLE)
{
e->inline_failed = (DECL_DECLARED_INLINE_P (callee->decl)
want_inline = false;
}
else if (!DECL_DECLARED_INLINE_P (callee->decl)
- && !flag_inline_functions)
+ && !opt_for_fn (e->caller->decl, flag_inline_functions))
{
/* growth_likely_positive is expensive, always test it last. */
if (growth >= MAX_INLINE_INSNS_SINGLE
}
}
/* If call is cold, do not inline when function body would grow. */
- else if (!cgraph_maybe_hot_edge_p (e)
+ else if (!e->maybe_hot_p ()
&& (growth >= MAX_INLINE_INSNS_SINGLE
|| growth_likely_positive (callee, growth)))
{
if (DECL_DECLARED_INLINE_P (edge->caller->decl))
max_depth = PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH);
- if (!cgraph_maybe_hot_edge_p (edge))
+ if (!edge->maybe_hot_p ())
{
reason = "recursive call is cold";
want_inline = false;
struct cgraph_edge *e;
for (e = node->callers; e; e = e->next_caller)
{
+ if (!opt_for_fn (e->caller->decl, flag_inline_functions_called_once))
+ return true;
if (!can_inline_edge_p (e, true))
return true;
- if (!(*(bool *)has_hot_call) && cgraph_maybe_hot_edge_p (e))
+ if (e->recursive_p ())
+ return true;
+ if (!(*(bool *)has_hot_call) && e->maybe_hot_p ())
*(bool *)has_hot_call = true;
}
return false;
static bool
want_inline_function_to_all_callers_p (struct cgraph_node *node, bool cold)
{
- struct cgraph_node *function = cgraph_function_or_thunk_node (node, NULL);
- bool has_hot_call = false;
-
- /* Does it have callers? */
- if (!cgraph_for_node_and_aliases (node, has_caller_p, NULL, true))
- return false;
- /* Already inlined? */
- if (function->global.inlined_to)
- return false;
- if (cgraph_function_or_thunk_node (node, NULL) != node)
- return false;
- /* Inlining into all callers would increase size? */
- if (estimate_growth (node) > 0)
- return false;
- /* All inlines must be possible. */
- if (cgraph_for_node_and_aliases (node, check_callers, &has_hot_call, true))
- return false;
- if (!cold && !has_hot_call)
- return false;
- return true;
-}
-
-#define RELATIVE_TIME_BENEFIT_RANGE (INT_MAX / 64)
+ bool has_hot_call = false;
-/* Return relative time improvement for inlining EDGE in range
- 1...RELATIVE_TIME_BENEFIT_RANGE */
-
-static inline int
-relative_time_benefit (struct inline_summary *callee_info,
- struct cgraph_edge *edge,
- int edge_time)
-{
- gcov_type relbenefit;
- gcov_type uninlined_call_time = compute_uninlined_call_time (callee_info, edge);
- gcov_type inlined_call_time = compute_inlined_call_time (edge, edge_time);
-
- /* Inlining into extern inline function is not a win. */
- if (DECL_EXTERNAL (edge->caller->global.inlined_to
- ? edge->caller->global.inlined_to->decl
- : edge->caller->decl))
- return 1;
-
- /* Watch overflows. */
- gcc_checking_assert (uninlined_call_time >= 0);
- gcc_checking_assert (inlined_call_time >= 0);
- gcc_checking_assert (uninlined_call_time >= inlined_call_time);
-
- /* Compute relative time benefit, i.e. how much the call becomes faster.
- ??? perhaps computing how much the caller+calle together become faster
- would lead to more realistic results. */
- if (!uninlined_call_time)
- uninlined_call_time = 1;
- relbenefit =
- RDIV (((gcov_type)uninlined_call_time - inlined_call_time) * RELATIVE_TIME_BENEFIT_RANGE,
- uninlined_call_time);
- relbenefit = MIN (relbenefit, RELATIVE_TIME_BENEFIT_RANGE);
- gcc_checking_assert (relbenefit >= 0);
- relbenefit = MAX (relbenefit, 1);
- return relbenefit;
+ /* Aliases gets inlined along with the function they alias. */
+ if (node->alias)
+ return false;
+ /* Already inlined? */
+ if (node->global.inlined_to)
+ return false;
+ /* Does it have callers? */
+ if (!node->call_for_symbol_and_aliases (has_caller_p, NULL, true))
+ return false;
+ /* Inlining into all callers would increase size? */
+ if (estimate_growth (node) > 0)
+ return false;
+ /* All inlines must be possible. */
+ if (node->call_for_symbol_and_aliases (check_callers, &has_hot_call,
+ true))
+ return false;
+ if (!cold && !has_hot_call)
+ return false;
+ return true;
}
-
/* A cost model driving the inlining heuristics in a way so the edges with
smallest badness are inlined first. After each inlining is performed
the costs of all caller edges of nodes affected are recomputed so the
metrics may accurately depend on values such as number of inlinable callers
of the function or function body size. */
-static int
+static sreal
edge_badness (struct cgraph_edge *edge, bool dump)
{
- gcov_type badness;
+ sreal badness;
int growth, edge_time;
- struct cgraph_node *callee = cgraph_function_or_thunk_node (edge->callee,
- NULL);
- struct inline_summary *callee_info = inline_summary (callee);
+ struct cgraph_node *callee = edge->callee->ultimate_alias_target ();
+ struct inline_summary *callee_info = inline_summaries->get (callee);
inline_hints hints;
-
- if (DECL_DISREGARD_INLINE_LIMITS (callee->decl))
- return INT_MIN;
+ cgraph_node *caller = (edge->caller->global.inlined_to
+ ? edge->caller->global.inlined_to
+ : edge->caller);
growth = estimate_edge_growth (edge);
edge_time = estimate_edge_time (edge);
if (dump)
{
fprintf (dump_file, " Badness calculation for %s/%i -> %s/%i\n",
- xstrdup (edge->caller->name ()),
+ xstrdup_for_dump (edge->caller->name ()),
edge->caller->order,
- xstrdup (callee->name ()),
+ xstrdup_for_dump (callee->name ()),
edge->callee->order);
fprintf (dump_file, " size growth %i, time %i ",
growth,
/* Always prefer inlining saving code size. */
if (growth <= 0)
{
- badness = INT_MIN / 2 + growth;
+ badness = (sreal) (-SREAL_MIN_SIG + growth) << (SREAL_MAX_EXP / 256);
if (dump)
- fprintf (dump_file, " %i: Growth %i <= 0\n", (int) badness,
+ fprintf (dump_file, " %f: Growth %d <= 0\n", badness.to_double (),
growth);
}
-
- /* When profiling is available, compute badness as:
-
- relative_edge_count * relative_time_benefit
- goodness = -------------------------------------------
- growth_f_caller
- badness = -goodness
-
- The fraction is upside down, because on edge counts and time beneits
- the bounds are known. Edge growth is essentially unlimited. */
-
- else if (max_count)
+ /* Inlining into EXTERNAL functions is not going to change anything unless
+ they are themselves inlined. */
+ else if (DECL_EXTERNAL (caller->decl))
{
- sreal tmp, relbenefit_real, growth_real;
- int relbenefit = relative_time_benefit (callee_info, edge, edge_time);
- /* Capping edge->count to max_count. edge->count can be larger than
- max_count if an inline adds new edges which increase max_count
- after max_count is computed. */
- gcov_type edge_count = edge->count > max_count ? max_count : edge->count;
-
- sreal_init (&relbenefit_real, relbenefit, 0);
- sreal_init (&growth_real, growth, 0);
-
- /* relative_edge_count. */
- sreal_init (&tmp, edge_count, 0);
- sreal_div (&tmp, &tmp, &max_count_real);
-
- /* relative_time_benefit. */
- sreal_mul (&tmp, &tmp, &relbenefit_real);
- sreal_div (&tmp, &tmp, &max_relbenefit_real);
-
- /* growth_f_caller. */
- sreal_mul (&tmp, &tmp, &half_int_min_real);
- sreal_div (&tmp, &tmp, &growth_real);
-
- badness = -1 * sreal_to_int (&tmp);
-
if (dump)
- {
- fprintf (dump_file,
- " %i (relative %f): profile info. Relative count %f%s"
- " * Relative benefit %f\n",
- (int) badness, (double) badness / INT_MIN,
- (double) edge_count / max_count,
- edge->count > max_count ? " (capped to max_count)" : "",
- relbenefit * 100.0 / RELATIVE_TIME_BENEFIT_RANGE);
- }
+ fprintf (dump_file, " max: function is external\n");
+ return sreal::max ();
}
-
- /* When function local profile is available. Compute badness as:
+ /* When profile is available. Compute badness as:
- relative_time_benefit
+ time_saved * caller_count
goodness = ---------------------------------
growth_of_caller * overall_growth
badness = - goodness
- compensated by the inline hints.
+ Again use negative value to make calls with profile appear hotter
+ then calls without.
*/
- else if (flag_guess_branch_prob)
+ else if (opt_for_fn (caller->decl, flag_guess_branch_prob) || caller->count)
{
- badness = (relative_time_benefit (callee_info, edge, edge_time)
- * (INT_MIN / 16 / RELATIVE_TIME_BENEFIT_RANGE));
- badness /= (MIN (65536/2, growth) * MIN (65536/2, MAX (1, callee_info->growth)));
- gcc_checking_assert (badness <=0 && badness >= INT_MIN / 16);
- if ((hints & (INLINE_HINT_indirect_call
- | INLINE_HINT_loop_iterations
- | INLINE_HINT_array_index
- | INLINE_HINT_loop_stride))
- || callee_info->growth <= 0)
- badness *= 8;
- if (hints & (INLINE_HINT_same_scc))
- badness /= 16;
- else if (hints & (INLINE_HINT_in_scc))
- badness /= 8;
- else if (hints & (INLINE_HINT_cross_module))
- badness /= 2;
- gcc_checking_assert (badness <= 0 && badness >= INT_MIN / 2);
- if ((hints & INLINE_HINT_declared_inline) && badness >= INT_MIN / 32)
- badness *= 16;
+ sreal numerator, denominator;
+ int overall_growth;
+
+ numerator = (compute_uninlined_call_time (callee_info, edge)
+ - compute_inlined_call_time (edge, edge_time));
+ if (numerator == 0)
+ numerator = ((sreal) 1 >> 8);
+ if (caller->count)
+ numerator *= caller->count;
+ else if (opt_for_fn (caller->decl, flag_branch_probabilities))
+ numerator = numerator >> 11;
+ denominator = growth;
+
+ overall_growth = callee_info->growth;
+
+ /* Look for inliner wrappers of the form:
+
+ inline_caller ()
+ {
+ do_fast_job...
+ if (need_more_work)
+ noninline_callee ();
+ }
+ Withhout panilizing this case, we usually inline noninline_callee
+ into the inline_caller because overall_growth is small preventing
+ further inlining of inline_caller.
+
+ Penalize only callgraph edges to functions with small overall
+ growth ...
+ */
+ if (growth > overall_growth
+ /* ... and having only one caller which is not inlined ... */
+ && callee_info->single_caller
+ && !edge->caller->global.inlined_to
+ /* ... and edges executed only conditionally ... */
+ && edge->frequency < CGRAPH_FREQ_BASE
+ /* ... consider case where callee is not inline but caller is ... */
+ && ((!DECL_DECLARED_INLINE_P (edge->callee->decl)
+ && DECL_DECLARED_INLINE_P (caller->decl))
+ /* ... or when early optimizers decided to split and edge
+ frequency still indicates splitting is a win ... */
+ || (callee->split_part && !caller->split_part
+ && edge->frequency
+ < CGRAPH_FREQ_BASE
+ * PARAM_VALUE
+ (PARAM_PARTIAL_INLINING_ENTRY_PROBABILITY) / 100
+ /* ... and do not overwrite user specified hints. */
+ && (!DECL_DECLARED_INLINE_P (edge->callee->decl)
+ || DECL_DECLARED_INLINE_P (caller->decl)))))
+ {
+ struct inline_summary *caller_info = inline_summaries->get (caller);
+ int caller_growth = caller_info->growth;
+
+ /* Only apply the penalty when caller looks like inline candidate,
+ and it is not called once and. */
+ if (!caller_info->single_caller && overall_growth < caller_growth
+ && caller_info->inlinable
+ && caller_info->size
+ < (DECL_DECLARED_INLINE_P (caller->decl)
+ ? MAX_INLINE_INSNS_SINGLE : MAX_INLINE_INSNS_AUTO))
+ {
+ if (dump)
+ fprintf (dump_file,
+ " Wrapper penalty. Increasing growth %i to %i\n",
+ overall_growth, caller_growth);
+ overall_growth = caller_growth;
+ }
+ }
+ if (overall_growth > 0)
+ {
+ /* Strongly preffer functions with few callers that can be inlined
+ fully. The square root here leads to smaller binaries at average.
+ Watch however for extreme cases and return to linear function
+ when growth is large. */
+ if (overall_growth < 256)
+ overall_growth *= overall_growth;
+ else
+ overall_growth += 256 * 256 - 256;
+ denominator *= overall_growth;
+ }
+
+ badness = - numerator / denominator;
+
if (dump)
{
fprintf (dump_file,
- " %i: guessed profile. frequency %f,"
- " benefit %f%%, time w/o inlining %i, time w inlining %i"
- " overall growth %i (current) %i (original)\n",
- (int) badness, (double)edge->frequency / CGRAPH_FREQ_BASE,
- relative_time_benefit (callee_info, edge, edge_time) * 100.0
- / RELATIVE_TIME_BENEFIT_RANGE,
- (int)compute_uninlined_call_time (callee_info, edge),
- (int)compute_inlined_call_time (edge, edge_time),
+ " %f: guessed profile. frequency %f, count %"PRId64
+ " caller count %"PRId64
+ " time w/o inlining %f, time w inlining %f"
+ " overall growth %i (current) %i (original)"
+ " %i (compensated)\n",
+ badness.to_double (),
+ (double)edge->frequency / CGRAPH_FREQ_BASE,
+ edge->count, caller->count,
+ compute_uninlined_call_time (callee_info, edge).to_double (),
+ compute_inlined_call_time (edge, edge_time).to_double (),
estimate_growth (callee),
- callee_info->growth);
+ callee_info->growth, overall_growth);
}
}
/* When function local profile is not available or it does not give
else
{
int nest = MIN (inline_edge_summary (edge)->loop_depth, 8);
- badness = growth * 256;
+ badness = growth;
/* Decrease badness if call is nested. */
if (badness > 0)
- badness >>= nest;
+ badness = badness >> nest;
else
- {
- badness <<= nest;
- }
+ badness = badness << nest;
if (dump)
- fprintf (dump_file, " %i: no profile. nest %i\n", (int) badness,
- nest);
+ fprintf (dump_file, " %f: no profile. nest %i\n",
+ badness.to_double (), nest);
}
-
- /* Ensure that we did not overflow in all the fixed point math above. */
- gcc_assert (badness >= INT_MIN);
- gcc_assert (badness <= INT_MAX - 1);
- /* Make recursive inlining happen always after other inlining is done. */
- if (cgraph_edge_recursive_p (edge))
- return badness + 1;
- else
- return badness;
+ gcc_checking_assert (badness != 0);
+
+ if (edge->recursive_p ())
+ badness = badness.shift (badness > 0 ? 4 : -4);
+ if ((hints & (INLINE_HINT_indirect_call
+ | INLINE_HINT_loop_iterations
+ | INLINE_HINT_array_index
+ | INLINE_HINT_loop_stride))
+ || callee_info->growth <= 0)
+ badness = badness.shift (badness > 0 ? -2 : 2);
+ if (hints & (INLINE_HINT_same_scc))
+ badness = badness.shift (badness > 0 ? 3 : -3);
+ else if (hints & (INLINE_HINT_in_scc))
+ badness = badness.shift (badness > 0 ? 2 : -2);
+ else if (hints & (INLINE_HINT_cross_module))
+ badness = badness.shift (badness > 0 ? 1 : -1);
+ if (DECL_DISREGARD_INLINE_LIMITS (callee->decl))
+ badness = badness.shift (badness > 0 ? -4 : 4);
+ else if ((hints & INLINE_HINT_declared_inline))
+ badness = badness.shift (badness > 0 ? -3 : 3);
+ if (dump)
+ fprintf (dump_file, " Adjusted by hints %f\n", badness.to_double ());
+ return badness;
}
/* Recompute badness of EDGE and update its key in HEAP if needed. */
static inline void
-update_edge_key (fibheap_t heap, struct cgraph_edge *edge)
+update_edge_key (edge_heap_t *heap, struct cgraph_edge *edge)
{
- int badness = edge_badness (edge, false);
+ sreal badness = edge_badness (edge, false);
if (edge->aux)
{
- fibnode_t n = (fibnode_t) edge->aux;
- gcc_checking_assert (n->data == edge);
-
- /* fibheap_replace_key only decrease the keys.
- When we increase the key we do not update heap
- and instead re-insert the element once it becomes
- a minimum of heap. */
- if (badness < n->key)
+ edge_heap_node_t *n = (edge_heap_node_t *) edge->aux;
+ gcc_checking_assert (n->get_data () == edge);
+
+ /* fibonacci_heap::replace_key does busy updating of the
+ heap that is unnecesarily expensive.
+ We do lazy increases: after extracting minimum if the key
+ turns out to be out of date, it is re-inserted into heap
+ with correct value. */
+ if (badness < n->get_key ())
{
if (dump_file && (dump_flags & TDF_DETAILS))
{
fprintf (dump_file,
- " decreasing badness %s/%i -> %s/%i, %i to %i\n",
- xstrdup (edge->caller->name ()),
+ " decreasing badness %s/%i -> %s/%i, %f"
+ " to %f\n",
+ xstrdup_for_dump (edge->caller->name ()),
edge->caller->order,
- xstrdup (edge->callee->name ()),
+ xstrdup_for_dump (edge->callee->name ()),
edge->callee->order,
- (int)n->key,
- badness);
+ n->get_key ().to_double (),
+ badness.to_double ());
}
- fibheap_replace_key (heap, n, badness);
- gcc_checking_assert (n->key == badness);
+ heap->decrease_key (n, badness);
}
}
else
if (dump_file && (dump_flags & TDF_DETAILS))
{
fprintf (dump_file,
- " enqueuing call %s/%i -> %s/%i, badness %i\n",
- xstrdup (edge->caller->name ()),
+ " enqueuing call %s/%i -> %s/%i, badness %f\n",
+ xstrdup_for_dump (edge->caller->name ()),
edge->caller->order,
- xstrdup (edge->callee->name ()),
+ xstrdup_for_dump (edge->callee->name ()),
edge->callee->order,
- badness);
+ badness.to_double ());
}
- edge->aux = fibheap_insert (heap, badness, edge);
+ edge->aux = heap->insert (badness, edge);
}
}
struct cgraph_edge *edge;
struct cgraph_edge *e = node->callees;
struct cgraph_node *where = node;
- int i;
struct ipa_ref *ref;
if (where->global.inlined_to)
where = where->global.inlined_to;
- /* WHERE body size has changed, the cached growth is invalid. */
- reset_node_growth_cache (where);
-
for (edge = where->callers; edge; edge = edge->next_caller)
if (edge->inline_failed)
reset_edge_growth_cache (edge);
- for (i = 0; ipa_ref_list_referring_iterate (&where->ref_list,
- i, ref); i++)
- if (ref->use == IPA_REF_ALIAS)
- reset_edge_caches (ipa_ref_referring_node (ref));
+
+ FOR_EACH_ALIAS (where, ref)
+ reset_edge_caches (dyn_cast <cgraph_node *> (ref->referring));
if (!e)
return;
it is inlinable. Otherwise check all edges. */
static void
-update_caller_keys (fibheap_t heap, struct cgraph_node *node,
+update_caller_keys (edge_heap_t *heap, struct cgraph_node *node,
bitmap updated_nodes,
struct cgraph_edge *check_inlinablity_for)
{
struct cgraph_edge *edge;
- int i;
struct ipa_ref *ref;
- if ((!node->alias && !inline_summary (node)->inlinable)
+ if ((!node->alias && !inline_summaries->get (node)->inlinable)
|| node->global.inlined_to)
return;
if (!bitmap_set_bit (updated_nodes, node->uid))
return;
- for (i = 0; ipa_ref_list_referring_iterate (&node->ref_list,
- i, ref); i++)
- if (ref->use == IPA_REF_ALIAS)
- {
- struct cgraph_node *alias = ipa_ref_referring_node (ref);
- update_caller_keys (heap, alias, updated_nodes, check_inlinablity_for);
- }
+ FOR_EACH_ALIAS (node, ref)
+ {
+ struct cgraph_node *alias = dyn_cast <cgraph_node *> (ref->referring);
+ update_caller_keys (heap, alias, updated_nodes, check_inlinablity_for);
+ }
for (edge = node->callers; edge; edge = edge->next_caller)
if (edge->inline_failed)
else if (edge->aux)
{
report_inline_failed_reason (edge);
- fibheap_delete_node (heap, (fibnode_t) edge->aux);
+ heap->delete_node ((edge_heap_node_t *) edge->aux);
edge->aux = NULL;
}
}
created edges into heap. */
static void
-update_callee_keys (fibheap_t heap, struct cgraph_node *node,
+update_callee_keys (edge_heap_t *heap, struct cgraph_node *node,
bitmap updated_nodes)
{
struct cgraph_edge *e = node->callees;
growth chould have just increased and consequentely badness metric
don't need updating. */
if (e->inline_failed
- && (callee = cgraph_function_or_thunk_node (e->callee, &avail))
- && inline_summary (callee)->inlinable
+ && (callee = e->callee->ultimate_alias_target (&avail))
+ && inline_summaries->get (callee)->inlinable
&& avail >= AVAIL_AVAILABLE
&& !bitmap_bit_p (updated_nodes, callee->uid))
{
else if (e->aux)
{
report_inline_failed_reason (e);
- fibheap_delete_node (heap, (fibnode_t) e->aux);
+ heap->delete_node ((edge_heap_node_t *) e->aux);
e->aux = NULL;
}
}
static void
lookup_recursive_calls (struct cgraph_node *node, struct cgraph_node *where,
- fibheap_t heap)
+ edge_heap_t *heap)
{
struct cgraph_edge *e;
enum availability avail;
for (e = where->callees; e; e = e->next_callee)
if (e->callee == node
- || (cgraph_function_or_thunk_node (e->callee, &avail) == node
- && avail > AVAIL_OVERWRITABLE))
+ || (e->callee->ultimate_alias_target (&avail) == node
+ && avail > AVAIL_INTERPOSABLE))
{
/* When profile feedback is available, prioritize by expected number
of calls. */
- fibheap_insert (heap,
- !max_count ? -e->frequency
- : -(e->count / ((max_count + (1<<24) - 1) / (1<<24))),
- e);
+ heap->insert (!max_count ? -e->frequency
+ : -(e->count / ((max_count + (1<<24) - 1) / (1<<24))),
+ e);
}
for (e = where->callees; e; e = e->next_callee)
if (!e->inline_failed)
static bool
recursive_inlining (struct cgraph_edge *edge,
- vec<cgraph_edge_p> *new_edges)
+ vec<cgraph_edge *> *new_edges)
{
int limit = PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE_AUTO);
- fibheap_t heap;
+ edge_heap_t heap (sreal::min ());
struct cgraph_node *node;
struct cgraph_edge *e;
struct cgraph_node *master_clone = NULL, *next;
/* Make sure that function is small enough to be considered for inlining. */
if (estimate_size_after_inlining (node, edge) >= limit)
return false;
- heap = fibheap_new ();
- lookup_recursive_calls (node, node, heap);
- if (fibheap_empty (heap))
- {
- fibheap_delete (heap);
- return false;
- }
+ lookup_recursive_calls (node, node, &heap);
+ if (heap.empty ())
+ return false;
if (dump_file)
fprintf (dump_file,
node->name ());
/* Do the inlining and update list of recursive call during process. */
- while (!fibheap_empty (heap))
+ while (!heap.empty ())
{
- struct cgraph_edge *curr
- = (struct cgraph_edge *) fibheap_extract_min (heap);
+ struct cgraph_edge *curr = heap.extract_min ();
struct cgraph_node *cnode, *dest = curr->callee;
if (!can_inline_edge_p (curr, true))
the already modified body. */
if (master_clone)
{
- cgraph_redirect_edge_callee (curr, master_clone);
- reset_edge_growth_cache (curr);
+ curr->redirect_callee (master_clone);
+ reset_edge_growth_cache (curr);
}
if (estimate_size_after_inlining (node, curr) > limit)
{
- cgraph_redirect_edge_callee (curr, dest);
+ curr->redirect_callee (dest);
reset_edge_growth_cache (curr);
break;
}
for (cnode = curr->caller;
cnode->global.inlined_to; cnode = cnode->callers->caller)
if (node->decl
- == cgraph_function_or_thunk_node (curr->callee, NULL)->decl)
+ == curr->callee->ultimate_alias_target ()->decl)
depth++;
if (!want_inline_self_recursive_call_p (curr, node, false, depth))
{
- cgraph_redirect_edge_callee (curr, dest);
+ curr->redirect_callee (dest);
reset_edge_growth_cache (curr);
continue;
}
if (!master_clone)
{
/* We need original clone to copy around. */
- master_clone = cgraph_clone_node (node, node->decl,
- node->count, CGRAPH_FREQ_BASE,
- false, vNULL, true, NULL, NULL);
+ master_clone = node->create_clone (node->decl, node->count,
+ CGRAPH_FREQ_BASE, false, vNULL,
+ true, NULL, NULL);
for (e = master_clone->callees; e; e = e->next_callee)
if (!e->inline_failed)
clone_inlined_nodes (e, true, false, NULL, CGRAPH_FREQ_BASE);
- cgraph_redirect_edge_callee (curr, master_clone);
+ curr->redirect_callee (master_clone);
reset_edge_growth_cache (curr);
}
inline_call (curr, false, new_edges, &overall_size, true);
- lookup_recursive_calls (node, curr->callee, heap);
+ lookup_recursive_calls (node, curr->callee, &heap);
n++;
}
- if (!fibheap_empty (heap) && dump_file)
+ if (!heap.empty () && dump_file)
fprintf (dump_file, " Recursive inlining growth limit met.\n");
- fibheap_delete (heap);
if (!master_clone)
return false;
fprintf (dump_file,
"\n Inlined %i times, "
"body grown from size %i to %i, time %i to %i\n", n,
- inline_summary (master_clone)->size, inline_summary (node)->size,
- inline_summary (master_clone)->time, inline_summary (node)->time);
+ inline_summaries->get (master_clone)->size, inline_summaries->get (node)->size,
+ inline_summaries->get (master_clone)->time, inline_summaries->get (node)->time);
/* Remove master clone we used for inlining. We rely that clones inlined
into master clone gets queued just before master clone so we don't
need recursion. */
- for (node = cgraph_first_function (); node != master_clone;
+ for (node = symtab->first_function (); node != master_clone;
node = next)
{
- next = cgraph_next_function (node);
+ next = symtab->next_function (node);
if (node->global.inlined_to == master_clone)
- cgraph_remove_node (node);
+ node->remove ();
}
- cgraph_remove_node (master_clone);
+ master_clone->remove ();
return true;
}
/* Compute badness of all edges in NEW_EDGES and add them to the HEAP. */
static void
-add_new_edges_to_heap (fibheap_t heap, vec<cgraph_edge_p> new_edges)
+add_new_edges_to_heap (edge_heap_t *heap, vec<cgraph_edge *> new_edges)
{
while (new_edges.length () > 0)
{
if (edge->inline_failed
&& can_inline_edge_p (edge, true)
&& want_inline_small_function_p (edge, true))
- edge->aux = fibheap_insert (heap, edge_badness (edge, false), edge);
+ edge->aux = heap->insert (edge_badness (edge, false), edge);
}
}
static void
heap_edge_removal_hook (struct cgraph_edge *e, void *data)
{
- if (e->callee)
- reset_node_growth_cache (e->callee);
if (e->aux)
{
- fibheap_delete_node ((fibheap_t)data, (fibnode_t)e->aux);
+ ((edge_heap_t *)data)->delete_node ((edge_heap_node_t *)e->aux);
e->aux = NULL;
}
}
speculation_useful_p (struct cgraph_edge *e, bool anticipate_inlining)
{
enum availability avail;
- struct cgraph_node *target = cgraph_function_or_thunk_node (e->callee, &avail);
+ struct cgraph_node *target = e->callee->ultimate_alias_target (&avail);
struct cgraph_edge *direct, *indirect;
struct ipa_ref *ref;
gcc_assert (e->speculative && !e->indirect_unknown_callee);
- if (!cgraph_maybe_hot_edge_p (e))
+ if (!e->maybe_hot_p ())
return false;
/* See if IP optimizations found something potentially useful about the
int ecf_flags = flags_from_decl_or_type (target->decl);
if (ecf_flags & ECF_CONST)
{
- cgraph_speculative_call_info (e, direct, indirect, ref);
+ e->speculative_call_info (direct, indirect, ref);
if (!(indirect->indirect_info->ecf_flags & ECF_CONST))
return true;
}
else if (ecf_flags & ECF_PURE)
{
- cgraph_speculative_call_info (e, direct, indirect, ref);
+ e->speculative_call_info (direct, indirect, ref);
if (!(indirect->indirect_info->ecf_flags & ECF_PURE))
return true;
}
See if we can remove speculation. */
static void
-resolve_noninline_speculation (fibheap_t edge_heap, struct cgraph_edge *edge)
+resolve_noninline_speculation (edge_heap_t *edge_heap, struct cgraph_edge *edge)
{
if (edge->speculative && !speculation_useful_p (edge, false))
{
bitmap updated_nodes = BITMAP_ALLOC (NULL);
spec_rem += edge->count;
- cgraph_resolve_speculation (edge, NULL);
+ edge->resolve_speculation ();
reset_edge_caches (where);
inline_update_overall_summary (where);
update_caller_keys (edge_heap, where,
}
}
+/* Return true if NODE should be accounted for overall size estimate.
+ Skip all nodes optimized for size so we can measure the growth of hot
+ part of program no matter of the padding. */
+
+bool
+inline_account_function_p (struct cgraph_node *node)
+{
+ return (!DECL_EXTERNAL (node->decl)
+ && !opt_for_fn (node->decl, optimize_size)
+ && node->frequency != NODE_FREQUENCY_UNLIKELY_EXECUTED);
+}
+
+/* Count number of callers of NODE and store it into DATA (that
+ points to int. Worker for cgraph_for_node_and_aliases. */
+
+static bool
+sum_callers (struct cgraph_node *node, void *data)
+{
+ struct cgraph_edge *e;
+ int *num_calls = (int *)data;
+
+ for (e = node->callers; e; e = e->next_caller)
+ (*num_calls)++;
+ return false;
+}
+
/* We use greedy algorithm for inlining of small functions:
All inline candidates are put into prioritized heap ordered in
increasing badness.
{
struct cgraph_node *node;
struct cgraph_edge *edge;
- fibheap_t edge_heap = fibheap_new ();
+ edge_heap_t edge_heap (sreal::min ());
bitmap updated_nodes = BITMAP_ALLOC (NULL);
int min_size, max_size;
- auto_vec<cgraph_edge_p> new_indirect_edges;
+ auto_vec<cgraph_edge *> new_indirect_edges;
int initial_size = 0;
- struct cgraph_node **order = XCNEWVEC (struct cgraph_node *, cgraph_n_nodes);
+ struct cgraph_node **order = XCNEWVEC (cgraph_node *, symtab->cgraph_count);
struct cgraph_edge_hook_list *edge_removal_hook_holder;
-
- if (flag_indirect_inlining)
- new_indirect_edges.create (8);
+ new_indirect_edges.create (8);
edge_removal_hook_holder
- = cgraph_add_edge_removal_hook (&heap_edge_removal_hook, edge_heap);
+ = symtab->add_edge_removal_hook (&heap_edge_removal_hook, &edge_heap);
/* Compute overall unit size and other global parameters used by badness
metrics. */
FOR_EACH_DEFINED_FUNCTION (node)
if (!node->global.inlined_to)
{
- if (cgraph_function_with_gimple_body_p (node)
- || node->thunk.thunk_p)
+ if (!node->alias && node->analyzed
+ && (node->has_gimple_body_p () || node->thunk.thunk_p))
{
- struct inline_summary *info = inline_summary (node);
+ struct inline_summary *info = inline_summaries->get (node);
struct ipa_dfs_info *dfs = (struct ipa_dfs_info *) node->aux;
/* Do not account external functions, they will be optimized out
if not inlined. Also only count the non-cold portion of program. */
- if (!DECL_EXTERNAL (node->decl)
- && node->frequency != NODE_FREQUENCY_UNLIKELY_EXECUTED)
+ if (inline_account_function_p (node))
initial_size += info->size;
info->growth = estimate_growth (node);
+
+ int num_calls = 0;
+ node->call_for_symbol_and_aliases (sum_callers, &num_calls,
+ true);
+ if (num_calls == 1)
+ info->single_caller = true;
if (dfs && dfs->next_cycle)
{
struct cgraph_node *n2;
for (n2 = node; n2;
n2 = ((struct ipa_dfs_info *) node->aux)->next_cycle)
{
- struct inline_summary *info2 = inline_summary (n2);
+ struct inline_summary *info2 = inline_summaries->get (n2);
if (info2->scc_no)
break;
info2->scc_no = id;
if (max_count < edge->count)
max_count = edge->count;
}
- sreal_init (&max_count_real, max_count, 0);
- sreal_init (&max_relbenefit_real, RELATIVE_TIME_BENEFIT_RANGE, 0);
- sreal_init (&half_int_min_real, INT_MAX / 2, 0);
ipa_free_postorder_info ();
initialize_growth_caches ();
FOR_EACH_DEFINED_FUNCTION (node)
{
bool update = false;
- struct cgraph_edge *next;
+ struct cgraph_edge *next = NULL;
+ bool has_speculative = false;
if (dump_file)
fprintf (dump_file, "Enqueueing calls in %s/%i.\n",
&& edge->inline_failed)
{
gcc_assert (!edge->aux);
- update_edge_key (edge_heap, edge);
+ update_edge_key (&edge_heap, edge);
}
- if (edge->speculative && !speculation_useful_p (edge, edge->aux != NULL))
+ if (edge->speculative)
+ has_speculative = true;
+ }
+ if (has_speculative)
+ for (edge = node->callees; edge; edge = next)
+ if (edge->speculative && !speculation_useful_p (edge,
+ edge->aux != NULL))
{
- cgraph_resolve_speculation (edge, NULL);
+ edge->resolve_speculation ();
update = true;
}
- }
if (update)
{
struct cgraph_node *where = node->global.inlined_to
? node->global.inlined_to : node;
inline_update_overall_summary (where);
- reset_node_growth_cache (where);
reset_edge_caches (where);
- update_caller_keys (edge_heap, where,
+ update_caller_keys (&edge_heap, where,
updated_nodes, NULL);
+ update_callee_keys (&edge_heap, where,
+ updated_nodes);
bitmap_clear (updated_nodes);
}
}
|| !max_count
|| (profile_info && flag_branch_probabilities));
- while (!fibheap_empty (edge_heap))
+ while (!edge_heap.empty ())
{
int old_size = overall_size;
struct cgraph_node *where, *callee;
- int badness = fibheap_min_key (edge_heap);
- int current_badness;
- int cached_badness;
+ sreal badness = edge_heap.min_key ();
+ sreal current_badness;
int growth;
- edge = (struct cgraph_edge *) fibheap_extract_min (edge_heap);
+ edge = edge_heap.extract_min ();
gcc_assert (edge->aux);
edge->aux = NULL;
if (!edge->inline_failed || !edge->callee->analyzed)
continue;
- /* Be sure that caches are maintained consistent.
- We can not make this ENABLE_CHECKING only because it cause different
- updates of the fibheap queue. */
- cached_badness = edge_badness (edge, false);
+#ifdef ENABLE_CHECKING
+ /* Be sure that caches are maintained consistent. */
+ sreal cached_badness = edge_badness (edge, false);
+
+ int old_size_est = estimate_edge_size (edge);
+ int old_time_est = estimate_edge_time (edge);
+ int old_hints_est = estimate_edge_hints (edge);
+
reset_edge_growth_cache (edge);
- reset_node_growth_cache (edge->callee);
+ gcc_assert (old_size_est == estimate_edge_size (edge));
+ gcc_assert (old_time_est == estimate_edge_time (edge));
+ /* FIXME:
+
+ gcc_assert (old_hints_est == estimate_edge_hints (edge));
+
+ fails with profile feedback because some hints depends on
+ maybe_hot_edge_p predicate and because callee gets inlined to other
+ calls, the edge may become cold.
+ This ought to be fixed by computing relative probabilities
+ for given invocation but that will be better done once whole
+ code is converted to sreals. Disable for now and revert to "wrong"
+ value so enable/disable checking paths agree. */
+ edge_growth_cache[edge->uid].hints = old_hints_est + 1;
/* When updating the edge costs, we only decrease badness in the keys.
Increases of badness are handled lazilly; when we see key with out
of date value on it, we re-insert it now. */
current_badness = edge_badness (edge, false);
- gcc_assert (cached_badness == current_badness);
+ /* Disable checking for profile because roundoff errors may cause slight
+ deviations in the order. */
+ gcc_assert (max_count || cached_badness == current_badness);
gcc_assert (current_badness >= badness);
+#else
+ current_badness = edge_badness (edge, false);
+#endif
if (current_badness != badness)
{
- edge->aux = fibheap_insert (edge_heap, current_badness, edge);
- continue;
+ if (edge_heap.min () && current_badness > edge_heap.min_key ())
+ {
+ edge->aux = edge_heap.insert (current_badness, edge);
+ continue;
+ }
+ else
+ badness = current_badness;
}
if (!can_inline_edge_p (edge, true))
{
- resolve_noninline_speculation (edge_heap, edge);
+ resolve_noninline_speculation (&edge_heap, edge);
continue;
}
- callee = cgraph_function_or_thunk_node (edge->callee, NULL);
+ callee = edge->callee->ultimate_alias_target ();
growth = estimate_edge_growth (edge);
if (dump_file)
{
fprintf (dump_file,
"\nConsidering %s/%i with %i size\n",
callee->name (), callee->order,
- inline_summary (callee)->size);
+ inline_summaries->get (callee)->size);
fprintf (dump_file,
" to be inlined into %s/%i in %s:%i\n"
- " Estimated badness is %i, frequency %.2f.\n",
+ " Estimated badness is %f, frequency %.2f.\n",
edge->caller->name (), edge->caller->order,
- flag_wpa ? "unknown"
- : gimple_filename ((const_gimple) edge->call_stmt),
- flag_wpa ? -1
- : gimple_lineno ((const_gimple) edge->call_stmt),
- badness,
+ edge->call_stmt
+ && (LOCATION_LOCUS (gimple_location ((const_gimple)
+ edge->call_stmt))
+ > BUILTINS_LOCATION)
+ ? gimple_filename ((const_gimple) edge->call_stmt)
+ : "unknown",
+ edge->call_stmt
+ ? gimple_lineno ((const_gimple) edge->call_stmt)
+ : -1,
+ badness.to_double (),
edge->frequency / (double)CGRAPH_FREQ_BASE);
if (edge->count)
fprintf (dump_file," Called %"PRId64"x\n",
{
edge->inline_failed = CIF_INLINE_UNIT_GROWTH_LIMIT;
report_inline_failed_reason (edge);
- resolve_noninline_speculation (edge_heap, edge);
+ resolve_noninline_speculation (&edge_heap, edge);
continue;
}
if (!want_inline_small_function_p (edge, true))
{
- resolve_noninline_speculation (edge_heap, edge);
+ resolve_noninline_speculation (&edge_heap, edge);
continue;
}
recursive calls where we do effects similar to loop unrolling.
When inlining such edge seems profitable, leave decision on
specific inliner. */
- if (cgraph_edge_recursive_p (edge))
+ if (edge->recursive_p ())
{
where = edge->caller;
if (where->global.inlined_to)
where = where->global.inlined_to;
if (!recursive_inlining (edge,
- flag_indirect_inlining
+ opt_for_fn (edge->caller->decl,
+ flag_indirect_inlining)
? &new_indirect_edges : NULL))
{
edge->inline_failed = CIF_RECURSIVE_INLINING;
- resolve_noninline_speculation (edge_heap, edge);
+ resolve_noninline_speculation (&edge_heap, edge);
continue;
}
reset_edge_caches (where);
/* Recursive inliner inlines all recursive calls of the function
at once. Consequently we need to update all callee keys. */
- if (flag_indirect_inlining)
- add_new_edges_to_heap (edge_heap, new_indirect_edges);
- update_callee_keys (edge_heap, where, updated_nodes);
+ if (opt_for_fn (edge->caller->decl, flag_indirect_inlining))
+ add_new_edges_to_heap (&edge_heap, new_indirect_edges);
+ update_callee_keys (&edge_heap, where, updated_nodes);
bitmap_clear (updated_nodes);
}
else
edge->inline_failed
= (DECL_DISREGARD_INLINE_LIMITS (edge->callee->decl)
? CIF_RECURSIVE_INLINING : CIF_UNSPECIFIED);
- resolve_noninline_speculation (edge_heap, edge);
+ resolve_noninline_speculation (&edge_heap, edge);
continue;
}
else if (depth && dump_file)
gcc_checking_assert (!callee->global.inlined_to);
inline_call (edge, true, &new_indirect_edges, &overall_size, true);
- if (flag_indirect_inlining)
- add_new_edges_to_heap (edge_heap, new_indirect_edges);
+ add_new_edges_to_heap (&edge_heap, new_indirect_edges);
- reset_edge_caches (edge->callee);
- reset_node_growth_cache (callee);
+ reset_edge_caches (edge->callee->function_symbol ());
- update_callee_keys (edge_heap, where, updated_nodes);
+ update_callee_keys (&edge_heap, where, updated_nodes);
}
where = edge->caller;
if (where->global.inlined_to)
inlined into (since it's body size changed) and for the functions
called by function we inlined (since number of it inlinable callers
might change). */
- update_caller_keys (edge_heap, where, updated_nodes, NULL);
+ update_caller_keys (&edge_heap, where, updated_nodes, NULL);
+ /* Offline copy count has possibly changed, recompute if profile is
+ available. */
+ if (max_count)
+ {
+ struct cgraph_node *n = cgraph_node::get (edge->callee->decl);
+ if (n != edge->callee && n->analyzed)
+ update_callee_keys (&edge_heap, n, updated_nodes);
+ }
bitmap_clear (updated_nodes);
if (dump_file)
" Inlined into %s which now has time %i and size %i,"
"net change of %+i.\n",
edge->caller->name (),
- inline_summary (edge->caller)->time,
- inline_summary (edge->caller)->size,
+ inline_summaries->get (edge->caller)->time,
+ inline_summaries->get (edge->caller)->size,
overall_size - old_size);
}
if (min_size > overall_size)
}
free_growth_caches ();
- fibheap_delete (edge_heap);
if (dump_file)
fprintf (dump_file,
"Unit growth for small function inlining: %i->%i (%i%%)\n",
initial_size, overall_size,
initial_size ? overall_size * 100 / (initial_size) - 100: 0);
BITMAP_FREE (updated_nodes);
- cgraph_remove_edge_removal_hook (edge_removal_hook_holder);
+ symtab->remove_edge_removal_hook (edge_removal_hook_holder);
}
/* Flatten NODE. Performed both during early inlining and
for (e = node->callees; e; e = e->next_callee)
{
struct cgraph_node *orig_callee;
- struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, NULL);
+ struct cgraph_node *callee = e->callee->ultimate_alias_target ();
/* We've hit cycle? It is time to give up. */
if (callee->aux)
if (dump_file)
fprintf (dump_file,
"Not inlining %s into %s to avoid cycle.\n",
- xstrdup (callee->name ()),
- xstrdup (e->caller->name ()));
+ xstrdup_for_dump (callee->name ()),
+ xstrdup_for_dump (e->caller->name ()));
e->inline_failed = CIF_RECURSIVE_INLINING;
continue;
}
: !can_early_inline_edge_p (e))
continue;
- if (cgraph_edge_recursive_p (e))
+ if (e->recursive_p ())
{
if (dump_file)
fprintf (dump_file, "Not inlining: recursive call.\n");
recursing through the original node if the node was cloned. */
if (dump_file)
fprintf (dump_file, " Inlining %s into %s.\n",
- xstrdup (callee->name ()),
- xstrdup (e->caller->name ()));
+ xstrdup_for_dump (callee->name ()),
+ xstrdup_for_dump (e->caller->name ()));
orig_callee = callee;
inline_call (e, true, NULL, NULL, false);
if (e->callee != orig_callee)
inline_update_overall_summary (node);
}
-/* Count number of callers of NODE and store it into DATA (that
- points to int. Worker for cgraph_for_node_and_aliases. */
-
-static bool
-sum_callers (struct cgraph_node *node, void *data)
-{
- struct cgraph_edge *e;
- int *num_calls = (int *)data;
-
- for (e = node->callers; e; e = e->next_caller)
- (*num_calls)++;
- return false;
-}
-
/* Inline NODE to all callers. Worker for cgraph_for_node_and_aliases.
DATA points to number of calls originally found so we avoid infinite
recursion. */
{
struct cgraph_node *caller = node->callers->caller;
+ if (!can_inline_edge_p (node->callers, true)
+ || node->callers->recursive_p ())
+ {
+ if (dump_file)
+ fprintf (dump_file, "Uninlinable call found; giving up.\n");
+ *num_calls = 0;
+ return false;
+ }
+
if (dump_file)
{
fprintf (dump_file,
"\nInlining %s size %i.\n",
node->name (),
- inline_summary (node)->size);
+ inline_summaries->get (node)->size);
fprintf (dump_file,
" Called once from %s %i insns.\n",
node->callers->caller->name (),
- inline_summary (node->callers->caller)->size);
+ inline_summaries->get (node->callers->caller)->size);
}
inline_call (node->callers, true, NULL, NULL, true, &callee_removed);
fprintf (dump_file,
" Inlined into %s which now has %i size\n",
caller->name (),
- inline_summary (caller)->size);
+ inline_summaries->get (caller)->size);
if (!(*num_calls)--)
{
if (dump_file)
if (!node->global.inlined_to
&& !node->alias)
{
- int time = inline_summary (node)->time;
+ int time = inline_summaries->get (node)->time;
sum += time;
sum_weighted += time * node->count;
}
if (!optimize)
return 0;
- order = XCNEWVEC (struct cgraph_node *, cgraph_n_nodes);
+ cgraph_freq_base_rec = (sreal) 1 / (sreal) CGRAPH_FREQ_BASE;
+ percent_rec = (sreal) 1 / (sreal) 100;
+
+ order = XCNEWVEC (struct cgraph_node *, symtab->cgraph_count);
if (in_lto_p && optimize)
ipa_update_after_lto_read ();
nnodes = ipa_reverse_postorder (order);
FOR_EACH_FUNCTION (node)
- node->aux = 0;
+ {
+ node->aux = 0;
+
+ /* Recompute the default reasons for inlining because they may have
+ changed during merging. */
+ if (in_lto_p)
+ {
+ for (cgraph_edge *e = node->callees; e; e = e->next_callee)
+ {
+ gcc_assert (e->inline_failed);
+ initialize_inline_failed (e);
+ }
+ for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
+ initialize_inline_failed (e);
+ }
+ }
if (dump_file)
fprintf (dump_file, "\nFlattening functions:\n");
inline_small_functions ();
- /* Do first after-inlining removal. We want to remove all "stale" extern inline
- functions and virtual functions so we really know what is called once. */
- symtab_remove_unreachable_nodes (false, dump_file);
+ gcc_assert (symtab->state == IPA_SSA);
+ symtab->state = IPA_SSA_AFTER_INLINING;
+ /* Do first after-inlining removal. We want to remove all "stale" extern
+ inline functions and virtual functions so we really know what is called
+ once. */
+ symtab->remove_unreachable_nodes (dump_file);
free (order);
/* Inline functions with a property that after inlining into all callers the
are met. */
if (dump_file)
fprintf (dump_file,
- "\nDeciding on functions to be inlined into all callers and removing useless speculations:\n");
+ "\nDeciding on functions to be inlined into all callers and "
+ "removing useless speculations:\n");
/* Inlining one function called once has good chance of preventing
inlining other function into the same callee. Ideally we should
next = edge->next_callee;
if (edge->speculative && !speculation_useful_p (edge, false))
{
- cgraph_resolve_speculation (edge, NULL);
+ edge->resolve_speculation ();
spec_rem += edge->count;
update = true;
remove_functions = true;
{
struct cgraph_node *where = node->global.inlined_to
? node->global.inlined_to : node;
- reset_node_growth_cache (where);
reset_edge_caches (where);
inline_update_overall_summary (where);
}
- if (flag_inline_functions_called_once
- && want_inline_function_to_all_callers_p (node, cold))
+ if (want_inline_function_to_all_callers_p (node, cold))
{
int num_calls = 0;
- cgraph_for_node_and_aliases (node, sum_callers,
- &num_calls, true);
- while (cgraph_for_node_and_aliases (node, inline_to_all_callers,
- &num_calls, true))
+ node->call_for_symbol_and_aliases (sum_callers, &num_calls,
+ true);
+ while (node->call_for_symbol_and_aliases
+ (inline_to_all_callers, &num_calls, true))
;
remove_functions = true;
}
for (e = node->callees; e; e = e->next_callee)
{
- struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, NULL);
+ struct cgraph_node *callee = e->callee->ultimate_alias_target ();
if (!DECL_DISREGARD_INLINE_LIMITS (callee->decl))
continue;
- if (cgraph_edge_recursive_p (e))
+ if (e->recursive_p ())
{
if (dump_file)
fprintf (dump_file, " Not inlining recursive call to %s.\n",
if (dump_file)
fprintf (dump_file, " Inlining %s into %s (always_inline).\n",
- xstrdup (e->callee->name ()),
- xstrdup (e->caller->name ()));
+ xstrdup_for_dump (e->callee->name ()),
+ xstrdup_for_dump (e->caller->name ()));
inline_call (e, true, NULL, NULL, false);
inlined = true;
}
for (e = node->callees; e; e = e->next_callee)
{
- struct cgraph_node *callee = cgraph_function_or_thunk_node (e->callee, NULL);
- if (!inline_summary (callee)->inlinable
+ struct cgraph_node *callee = e->callee->ultimate_alias_target ();
+ if (!inline_summaries->get (callee)->inlinable
|| !e->inline_failed)
continue;
/* Do not consider functions not declared inline. */
if (!DECL_DECLARED_INLINE_P (callee->decl)
- && !flag_inline_small_functions
- && !flag_inline_functions)
+ && !opt_for_fn (node->decl, flag_inline_small_functions)
+ && !opt_for_fn (node->decl, flag_inline_functions))
continue;
if (dump_file)
if (!can_early_inline_edge_p (e))
continue;
- if (cgraph_edge_recursive_p (e))
+ if (e->recursive_p ())
{
if (dump_file)
fprintf (dump_file, " Not inlining: recursive call.\n");
if (dump_file)
fprintf (dump_file, " Inlining %s into %s.\n",
- xstrdup (callee->name ()),
- xstrdup (e->caller->name ()));
+ xstrdup_for_dump (callee->name ()),
+ xstrdup_for_dump (e->caller->name ()));
inline_call (e, true, NULL, NULL, true);
inlined = true;
}
return inlined;
}
-/* Do inlining of small functions. Doing so early helps profiling and other
- passes to be somewhat more effective and avoids some code duplication in
- later real inlining pass for testcases with very many function calls. */
-
-namespace {
-
-const pass_data pass_data_early_inline =
-{
- GIMPLE_PASS, /* type */
- "einline", /* name */
- OPTGROUP_INLINE, /* optinfo_flags */
- true, /* has_execute */
- TV_EARLY_INLINING, /* tv_id */
- PROP_ssa, /* properties_required */
- 0, /* properties_provided */
- 0, /* properties_destroyed */
- 0, /* todo_flags_start */
- 0, /* todo_flags_finish */
-};
-
-class pass_early_inline : public gimple_opt_pass
-{
-public:
- pass_early_inline (gcc::context *ctxt)
- : gimple_opt_pass (pass_data_early_inline, ctxt)
- {}
-
- /* opt_pass methods: */
- virtual unsigned int execute (function *);
-
-}; // class pass_early_inline
-
unsigned int
-pass_early_inline::execute (function *fun)
+early_inliner (function *fun)
{
- struct cgraph_node *node = cgraph_get_node (current_function_decl);
+ struct cgraph_node *node = cgraph_node::get (current_function_decl);
struct cgraph_edge *edge;
unsigned int todo = 0;
int iterations = 0;
it. This may confuse ourself when early inliner decide to inline call to
function clone, because function clones don't have parameter list in
ipa-prop matching their signature. */
- if (ipa_node_params_vector.exists ())
+ if (ipa_node_params_sum)
return 0;
#ifdef ENABLE_CHECKING
- verify_cgraph_node (node);
+ node->verify ();
#endif
- ipa_remove_all_references (&node->ref_list);
+ node->remove_all_references ();
+
+ /* Rebuild this reference because it dosn't depend on
+ function's body and it's required to pass cgraph_node
+ verification. */
+ if (node->instrumented_version
+ && !node->instrumentation_clone)
+ node->create_reference (node->instrumented_version, IPA_REF_CHKP, NULL);
/* Even when not optimizing or not inlining inline always-inline
functions. */
cycles of edges to be always inlined in the callgraph.
We might want to be smarter and just avoid this type of inlining. */
- || DECL_DISREGARD_INLINE_LIMITS (node->decl))
+ || (DECL_DISREGARD_INLINE_LIMITS (node->decl)
+ && lookup_attribute ("always_inline",
+ DECL_ATTRIBUTES (node->decl))))
;
else if (lookup_attribute ("flatten",
DECL_ATTRIBUTES (node->decl)) != NULL)
}
else
{
+ /* If some always_inline functions was inlined, apply the changes.
+ This way we will not account always inline into growth limits and
+ moreover we will inline calls from always inlines that we skipped
+ previously becuase of conditional above. */
+ if (inlined)
+ {
+ timevar_push (TV_INTEGRATION);
+ todo |= optimize_inline_calls (current_function_decl);
+ /* optimize_inline_calls call above might have introduced new
+ statements that don't have inline parameters computed. */
+ for (edge = node->callees; edge; edge = edge->next_callee)
+ {
+ if (inline_edge_summary_vec.length () > (unsigned) edge->uid)
+ {
+ struct inline_edge_summary *es = inline_edge_summary (edge);
+ es->call_stmt_size
+ = estimate_num_insns (edge->call_stmt, &eni_size_weights);
+ es->call_stmt_time
+ = estimate_num_insns (edge->call_stmt, &eni_time_weights);
+ }
+ }
+ inline_update_overall_summary (node);
+ inlined = false;
+ timevar_pop (TV_INTEGRATION);
+ }
/* We iterate incremental inlining to get trivial cases of indirect
inlining. */
while (iterations < PARAM_VALUE (PARAM_EARLY_INLINER_MAX_ITERATIONS)
info that might be cleared out for newly discovered edges. */
for (edge = node->callees; edge; edge = edge->next_callee)
{
- struct inline_edge_summary *es = inline_edge_summary (edge);
- es->call_stmt_size
- = estimate_num_insns (edge->call_stmt, &eni_size_weights);
- es->call_stmt_time
- = estimate_num_insns (edge->call_stmt, &eni_time_weights);
+ /* We have no summary for new bound store calls yet. */
+ if (inline_edge_summary_vec.length () > (unsigned)edge->uid)
+ {
+ struct inline_edge_summary *es = inline_edge_summary (edge);
+ es->call_stmt_size
+ = estimate_num_insns (edge->call_stmt, &eni_size_weights);
+ es->call_stmt_time
+ = estimate_num_insns (edge->call_stmt, &eni_time_weights);
+ }
if (edge->callee->decl
&& !gimple_check_call_matching_types (
edge->call_stmt, edge->callee->decl, false))
return todo;
}
+/* Do inlining of small functions. Doing so early helps profiling and other
+ passes to be somewhat more effective and avoids some code duplication in
+ later real inlining pass for testcases with very many function calls. */
+
+namespace {
+
+const pass_data pass_data_early_inline =
+{
+ GIMPLE_PASS, /* type */
+ "einline", /* name */
+ OPTGROUP_INLINE, /* optinfo_flags */
+ TV_EARLY_INLINING, /* tv_id */
+ PROP_ssa, /* properties_required */
+ 0, /* properties_provided */
+ 0, /* properties_destroyed */
+ 0, /* todo_flags_start */
+ 0, /* todo_flags_finish */
+};
+
+class pass_early_inline : public gimple_opt_pass
+{
+public:
+ pass_early_inline (gcc::context *ctxt)
+ : gimple_opt_pass (pass_data_early_inline, ctxt)
+ {}
+
+ /* opt_pass methods: */
+ virtual unsigned int execute (function *);
+
+}; // class pass_early_inline
+
+unsigned int
+pass_early_inline::execute (function *fun)
+{
+ return early_inliner (fun);
+}
+
} // anon namespace
gimple_opt_pass *
IPA_PASS, /* type */
"inline", /* name */
OPTGROUP_INLINE, /* optinfo_flags */
- true, /* has_execute */
TV_IPA_INLINING, /* tv_id */
0, /* properties_required */
0, /* properties_provided */
0, /* properties_destroyed */
- TODO_remove_functions, /* todo_flags_start */
+ 0, /* todo_flags_start */
( TODO_dump_symtab ), /* todo_flags_finish */
};
projects / gcc.git / blobdiff