/* Inlining decision heuristics.
- Copyright (C) 2003-2014 Free Software Foundation, Inc.
+ Copyright (C) 2003-2016 Free Software Foundation, Inc.
Contributed by Jan Hubicka
This file is part of GCC.
#include "config.h"
#include "system.h"
#include "coretypes.h"
-#include "tm.h"
+#include "backend.h"
#include "tree.h"
-#include "stor-layout.h"
-#include "stringpool.h"
+#include "gimple.h"
+#include "alloc-pool.h"
+#include "tree-pass.h"
+#include "ssa.h"
+#include "tree-streamer.h"
+#include "cgraph.h"
+#include "diagnostic.h"
+#include "fold-const.h"
#include "print-tree.h"
#include "tree-inline.h"
-#include "langhooks.h"
-#include "flags.h"
-#include "diagnostic.h"
#include "gimple-pretty-print.h"
#include "params.h"
-#include "tree-pass.h"
-#include "coverage.h"
-#include "predict.h"
-#include "vec.h"
-#include "hashtab.h"
-#include "hash-set.h"
-#include "machmode.h"
-#include "hard-reg-set.h"
-#include "input.h"
-#include "function.h"
-#include "dominance.h"
-#include "cfg.h"
#include "cfganal.h"
-#include "basic-block.h"
-#include "tree-ssa-alias.h"
-#include "internal-fn.h"
-#include "gimple-expr.h"
-#include "is-a.h"
-#include "gimple.h"
#include "gimple-iterator.h"
-#include "gimple-ssa.h"
#include "tree-cfg.h"
-#include "tree-phinodes.h"
-#include "ssa-iterators.h"
-#include "tree-ssanames.h"
#include "tree-ssa-loop-niter.h"
#include "tree-ssa-loop.h"
-#include "hash-map.h"
-#include "plugin-api.h"
-#include "ipa-ref.h"
-#include "cgraph.h"
-#include "alloc-pool.h"
+#include "symbol-summary.h"
#include "ipa-prop.h"
-#include "lto-streamer.h"
-#include "data-streamer.h"
-#include "tree-streamer.h"
#include "ipa-inline.h"
#include "cfgloop.h"
#include "tree-scalar-evolution.h"
#include "ipa-utils.h"
#include "cilk.h"
#include "cfgexpand.h"
+#include "gimplify.h"
/* Estimate runtime of function can easilly run into huge numbers with many
nested loops. Be sure we can compute time * INLINE_SIZE_SCALE * 2 in an
#define CHANGED IDENTIFIER_NODE
/* Holders of ipa cgraph hooks: */
-static struct cgraph_node_hook_list *function_insertion_hook_holder;
-static struct cgraph_node_hook_list *node_removal_hook_holder;
-static struct cgraph_2node_hook_list *node_duplication_hook_holder;
static struct cgraph_2edge_hook_list *edge_duplication_hook_holder;
static struct cgraph_edge_hook_list *edge_removal_hook_holder;
-static void inline_node_removal_hook (struct cgraph_node *, void *);
-static void inline_node_duplication_hook (struct cgraph_node *,
- struct cgraph_node *, void *);
static void inline_edge_removal_hook (struct cgraph_edge *, void *);
static void inline_edge_duplication_hook (struct cgraph_edge *,
struct cgraph_edge *, void *);
/* VECtor holding inline summaries.
In GGC memory because conditions might point to constant trees. */
-vec<inline_summary_t, va_gc> *inline_summary_vec;
+function_summary <inline_summary *> *inline_summaries;
vec<inline_edge_summary_t> inline_edge_summary_vec;
/* Cached node/edge growths. */
-vec<int> node_growth_cache;
vec<edge_growth_cache_entry> edge_growth_cache;
/* Edge predicates goes here. */
-static alloc_pool edge_predicate_pool;
+static object_allocator<predicate> edge_predicate_pool ("edge predicates");
/* Return true predicate (tautology).
We represent it by empty list of clauses. */
}
}
+/* We proved E to be unreachable, redirect it to __bultin_unreachable. */
+
+static struct cgraph_edge *
+redirect_to_unreachable (struct cgraph_edge *e)
+{
+ struct cgraph_node *callee = !e->inline_failed ? e->callee : NULL;
+ struct cgraph_node *target = cgraph_node::get_create
+ (builtin_decl_implicit (BUILT_IN_UNREACHABLE));
+
+ if (e->speculative)
+ e = e->resolve_speculation (target->decl);
+ else if (!e->callee)
+ e->make_direct (target);
+ else
+ e->redirect_callee (target);
+ struct inline_edge_summary *es = inline_edge_summary (e);
+ e->inline_failed = CIF_UNREACHABLE;
+ e->frequency = 0;
+ e->count = 0;
+ es->call_stmt_size = 0;
+ es->call_stmt_time = 0;
+ if (callee)
+ callee->remove_symbol_and_inline_clones ();
+ return e;
+}
+
/* Set predicate for edge E. */
static void
edge_set_predicate (struct cgraph_edge *e, struct predicate *predicate)
{
- struct inline_edge_summary *es = inline_edge_summary (e);
-
/* If the edge is determined to be never executed, redirect it
to BUILTIN_UNREACHABLE to save inliner from inlining into it. */
- if (predicate && false_predicate_p (predicate) && e->callee)
- {
- struct cgraph_node *callee = !e->inline_failed ? e->callee : NULL;
+ if (predicate && false_predicate_p (predicate)
+ /* When handling speculative edges, we need to do the redirection
+ just once. Do it always on the direct edge, so we do not
+ attempt to resolve speculation while duplicating the edge. */
+ && (!e->speculative || e->callee))
+ e = redirect_to_unreachable (e);
- e->redirect_callee (cgraph_node::get_create
- (builtin_decl_implicit (BUILT_IN_UNREACHABLE)));
- e->inline_failed = CIF_UNREACHABLE;
- if (callee)
- callee->remove_symbol_and_inline_clones ();
- }
+ struct inline_edge_summary *es = inline_edge_summary (e);
if (predicate && !true_predicate_p (predicate))
{
if (!es->predicate)
- es->predicate = (struct predicate *) pool_alloc (edge_predicate_pool);
+ es->predicate = edge_predicate_pool.allocate ();
*es->predicate = *predicate;
}
else
{
if (es->predicate)
- pool_free (edge_predicate_pool, es->predicate);
+ edge_predicate_pool.remove (es->predicate);
es->predicate = NULL;
}
}
if (false_predicate_p (&new_predicate) || true_predicate_p (&new_predicate))
{
if (*p)
- pool_free (edge_predicate_pool, *p);
+ edge_predicate_pool.remove (*p);
*p = NULL;
}
else
{
if (!*p)
- *p = (struct predicate *) pool_alloc (edge_predicate_pool);
+ *p = edge_predicate_pool.allocate ();
**p = new_predicate;
}
}
known_aggs)
{
clause_t clause = inline_p ? 0 : 1 << predicate_not_inlined_condition;
- struct inline_summary *info = inline_summary (node);
+ struct inline_summary *info = inline_summaries->get (node);
int i;
struct condition *c;
vec<ipa_agg_jump_function_p> *known_aggs_ptr)
{
struct cgraph_node *callee = e->callee->ultimate_alias_target ();
- struct inline_summary *info = inline_summary (callee);
+ struct inline_summary *info = inline_summaries->get (callee);
vec<tree> known_vals = vNULL;
vec<ipa_agg_jump_function_p> known_aggs = vNULL;
if (known_contexts_ptr)
known_contexts_ptr->create (0);
- if (ipa_node_params_vector.exists ()
+ if (ipa_node_params_sum
&& !e->call_stmt_cannot_inline_p
&& ((clause_ptr && info->conds) || known_vals_ptr || known_contexts_ptr))
{
{
struct ipa_jump_func *jf = ipa_get_ith_jump_func (args, i);
tree cst = ipa_value_from_jfunc (parms_info, jf);
+
+ if (!cst && e->call_stmt
+ && i < (int)gimple_call_num_args (e->call_stmt))
+ {
+ cst = gimple_call_arg (e->call_stmt, i);
+ if (!is_gimple_min_invariant (cst))
+ cst = NULL;
+ }
if (cst)
{
gcc_checking_assert (TREE_CODE (cst) != TREE_BINFO);
known_aggs[i] = &jf->agg;
}
}
+ else if (e->call_stmt && !e->call_stmt_cannot_inline_p
+ && ((clause_ptr && info->conds) || known_vals_ptr))
+ {
+ int i, count = (int)gimple_call_num_args (e->call_stmt);
+
+ if (count && (info->conds || known_vals_ptr))
+ known_vals.safe_grow_cleared (count);
+ for (i = 0; i < count; i++)
+ {
+ tree cst = gimple_call_arg (e->call_stmt, i);
+ if (!is_gimple_min_invariant (cst))
+ cst = NULL;
+ if (cst)
+ known_vals[i] = cst;
+ }
+ }
if (clause_ptr)
*clause_ptr = evaluate_conditions_for_known_args (callee, inline_p,
static void
inline_summary_alloc (void)
{
- if (!node_removal_hook_holder)
- node_removal_hook_holder =
- symtab->add_cgraph_removal_hook (&inline_node_removal_hook, NULL);
if (!edge_removal_hook_holder)
edge_removal_hook_holder =
symtab->add_edge_removal_hook (&inline_edge_removal_hook, NULL);
- if (!node_duplication_hook_holder)
- node_duplication_hook_holder =
- symtab->add_cgraph_duplication_hook (&inline_node_duplication_hook, NULL);
if (!edge_duplication_hook_holder)
edge_duplication_hook_holder =
symtab->add_edge_duplication_hook (&inline_edge_duplication_hook, NULL);
- if (vec_safe_length (inline_summary_vec) <= (unsigned) symtab->cgraph_max_uid)
- vec_safe_grow_cleared (inline_summary_vec, symtab->cgraph_max_uid + 1);
+ if (!inline_summaries)
+ inline_summaries = (inline_summary_t*) inline_summary_t::create_ggc (symtab);
+
if (inline_edge_summary_vec.length () <= (unsigned) symtab->edges_max_uid)
inline_edge_summary_vec.safe_grow_cleared (symtab->edges_max_uid + 1);
- if (!edge_predicate_pool)
- edge_predicate_pool = create_alloc_pool ("edge predicates",
- sizeof (struct predicate), 10);
}
/* We are called multiple time for given function; clear
es->call_stmt_size = es->call_stmt_time = 0;
if (es->predicate)
- pool_free (edge_predicate_pool, es->predicate);
+ edge_predicate_pool.remove (es->predicate);
es->predicate = NULL;
es->param.release ();
}
data from previous run so they are not cumulated. */
static void
-reset_inline_summary (struct cgraph_node *node)
+reset_inline_summary (struct cgraph_node *node,
+ inline_summary *info)
{
- struct inline_summary *info = inline_summary (node);
struct cgraph_edge *e;
info->self_size = info->self_time = 0;
info->scc_no = 0;
if (info->loop_iterations)
{
- pool_free (edge_predicate_pool, info->loop_iterations);
+ edge_predicate_pool.remove (info->loop_iterations);
info->loop_iterations = NULL;
}
if (info->loop_stride)
{
- pool_free (edge_predicate_pool, info->loop_stride);
+ edge_predicate_pool.remove (info->loop_stride);
info->loop_stride = NULL;
}
if (info->array_index)
{
- pool_free (edge_predicate_pool, info->array_index);
+ edge_predicate_pool.remove (info->array_index);
info->array_index = NULL;
}
vec_free (info->conds);
/* Hook that is called by cgraph.c when a node is removed. */
-static void
-inline_node_removal_hook (struct cgraph_node *node,
- void *data ATTRIBUTE_UNUSED)
+void
+inline_summary_t::remove (cgraph_node *node, inline_summary *info)
{
- struct inline_summary *info;
- if (vec_safe_length (inline_summary_vec) <= (unsigned) node->uid)
- return;
- info = inline_summary (node);
- reset_inline_summary (node);
- memset (info, 0, sizeof (inline_summary_t));
+ reset_inline_summary (node, info);
}
/* Remap predicate P of former function to be predicate of duplicated function.
/* Hook that is called by cgraph.c when a node is duplicated. */
-
-static void
-inline_node_duplication_hook (struct cgraph_node *src,
- struct cgraph_node *dst,
- ATTRIBUTE_UNUSED void *data)
+void
+inline_summary_t::duplicate (cgraph_node *src,
+ cgraph_node *dst,
+ inline_summary *,
+ inline_summary *info)
{
- struct inline_summary *info;
inline_summary_alloc ();
- info = inline_summary (dst);
- memcpy (info, inline_summary (src), sizeof (struct inline_summary));
+ memcpy (info, inline_summaries->get (src), sizeof (inline_summary));
/* TODO: as an optimization, we may avoid copying conditions
that are known to be false or true. */
info->conds = vec_safe_copy (info->conds);
/* When there are any replacements in the function body, see if we can figure
out that something was optimized out. */
- if (ipa_node_params_vector.exists () && dst->clone.tree_map)
+ if (ipa_node_params_sum && dst->clone.tree_map)
{
vec<size_time_entry, va_gc> *entry = info->entry;
/* Use SRC parm info since it may not be copied yet. */
size_time_entry *e;
int optimized_out_size = 0;
bool inlined_to_p = false;
- struct cgraph_edge *edge;
+ struct cgraph_edge *edge, *next;
info->entry = 0;
known_vals.safe_grow_cleared (count);
/* Remap edge predicates with the same simplification as above.
Also copy constantness arrays. */
- for (edge = dst->callees; edge; edge = edge->next_callee)
+ for (edge = dst->callees; edge; edge = next)
{
struct predicate new_predicate;
struct inline_edge_summary *es = inline_edge_summary (edge);
+ next = edge->next_callee;
if (!edge->inline_failed)
inlined_to_p = true;
info);
if (false_predicate_p (&new_predicate)
&& !false_predicate_p (es->predicate))
- {
- optimized_out_size += es->call_stmt_size * INLINE_SIZE_SCALE;
- edge->frequency = 0;
- }
+ optimized_out_size += es->call_stmt_size * INLINE_SIZE_SCALE;
edge_set_predicate (edge, &new_predicate);
}
/* Remap indirect edge predicates with the same simplificaiton as above.
Also copy constantness arrays. */
- for (edge = dst->indirect_calls; edge; edge = edge->next_callee)
+ for (edge = dst->indirect_calls; edge; edge = next)
{
struct predicate new_predicate;
struct inline_edge_summary *es = inline_edge_summary (edge);
+ next = edge->next_callee;
gcc_checking_assert (edge->inline_failed);
if (!es->predicate)
info);
if (false_predicate_p (&new_predicate)
&& !false_predicate_p (es->predicate))
- {
- optimized_out_size += es->call_stmt_size * INLINE_SIZE_SCALE;
- edge->frequency = 0;
- }
+ optimized_out_size += es->call_stmt_size * INLINE_SIZE_SCALE;
edge_set_predicate (edge, &new_predicate);
}
remap_hint_predicate_after_duplication (&info->loop_iterations,
set_hint_predicate (&info->array_index, p);
}
}
- inline_update_overall_summary (dst);
+ if (!dst->global.inlined_to)
+ inline_update_overall_summary (dst);
}
info->predicate = NULL;
edge_set_predicate (dst, srcinfo->predicate);
info->param = srcinfo->param.copy ();
+ if (!dst->indirect_unknown_callee && src->indirect_unknown_callee)
+ {
+ info->call_stmt_size -= (eni_size_weights.indirect_call_cost
+ - eni_size_weights.call_cost);
+ info->call_stmt_time -= (eni_time_weights.indirect_call_cost
+ - eni_time_weights.call_cost);
+ }
}
{
if (symtab->edges_max_uid)
edge_growth_cache.safe_grow_cleared (symtab->edges_max_uid);
- if (symtab->cgraph_max_uid)
- node_growth_cache.safe_grow_cleared (symtab->cgraph_max_uid);
}
free_growth_caches (void)
{
edge_growth_cache.release ();
- node_growth_cache.release ();
}
? "inlined" : cgraph_inline_failed_string (edge-> inline_failed),
indent, "", es->loop_depth, edge->frequency,
es->call_stmt_size, es->call_stmt_time,
- (int) inline_summary (callee)->size / INLINE_SIZE_SCALE,
- (int) inline_summary (callee)->estimated_stack_size);
+ (int) inline_summaries->get (callee)->size / INLINE_SIZE_SCALE,
+ (int) inline_summaries->get (callee)->estimated_stack_size);
if (es->predicate)
{
fprintf (f, "%*sStack frame offset %i, callee self size %i,"
" callee size %i\n",
indent + 2, "",
- (int) inline_summary (callee)->stack_frame_offset,
- (int) inline_summary (callee)->estimated_self_stack_size,
- (int) inline_summary (callee)->estimated_stack_size);
+ (int) inline_summaries->get (callee)->stack_frame_offset,
+ (int) inline_summaries->get (callee)->estimated_self_stack_size,
+ (int) inline_summaries->get (callee)->estimated_stack_size);
dump_inline_edge_summary (f, indent + 2, callee, info);
}
}
{
if (node->definition)
{
- struct inline_summary *s = inline_summary (node);
+ struct inline_summary *s = inline_summaries->get (node);
size_time_entry *e;
int i;
fprintf (f, "Inline summary for %s/%i", node->name (),
fprintf (f, " always_inline");
if (s->inlinable)
fprintf (f, " inlinable");
+ if (s->contains_cilk_spawn)
+ fprintf (f, " contains_cilk_spawn");
fprintf (f, "\n self time: %i\n", s->self_time);
fprintf (f, " global time: %i\n", s->time);
fprintf (f, " self size: %i\n", s->self_size);
parameter. */
static tree
-unmodified_parm_1 (gimple stmt, tree op)
+unmodified_parm_1 (gimple *stmt, tree op)
{
/* SSA_NAME referring to parm default def? */
if (TREE_CODE (op) == SSA_NAME
parameter. Also traverse chains of SSA register assignments. */
static tree
-unmodified_parm (gimple stmt, tree op)
+unmodified_parm (gimple *stmt, tree op)
{
tree res = unmodified_parm_1 (stmt, op);
if (res)
loaded. */
static bool
-unmodified_parm_or_parm_agg_item (struct ipa_node_params *info,
- gimple stmt, tree op, int *index_p,
+unmodified_parm_or_parm_agg_item (struct ipa_func_body_info *fbi,
+ gimple *stmt, tree op, int *index_p,
struct agg_position_info *aggpos)
{
tree res = unmodified_parm_1 (stmt, op);
gcc_checking_assert (aggpos);
if (res)
{
- *index_p = ipa_get_param_decl_index (info, res);
+ *index_p = ipa_get_param_decl_index (fbi->info, res);
if (*index_p < 0)
return false;
aggpos->agg_contents = false;
stmt = SSA_NAME_DEF_STMT (op);
op = gimple_assign_rhs1 (stmt);
if (!REFERENCE_CLASS_P (op))
- return unmodified_parm_or_parm_agg_item (info, stmt, op, index_p,
+ return unmodified_parm_or_parm_agg_item (fbi, stmt, op, index_p,
aggpos);
}
aggpos->agg_contents = true;
- return ipa_load_from_parm_agg (info, stmt, op, index_p, &aggpos->offset,
- &aggpos->by_ref);
+ return ipa_load_from_parm_agg (fbi, fbi->info->descriptors,
+ stmt, op, index_p, &aggpos->offset,
+ NULL, &aggpos->by_ref);
}
/* See if statement might disappear after inlining.
penalty wrappers. */
static int
-eliminated_by_inlining_prob (gimple stmt)
+eliminated_by_inlining_prob (gimple *stmt)
{
enum gimple_code code = gimple_code (stmt);
enum tree_code rhs_code;
predicates to the CFG edges. */
static void
-set_cond_stmt_execution_predicate (struct ipa_node_params *info,
+set_cond_stmt_execution_predicate (struct ipa_func_body_info *fbi,
struct inline_summary *summary,
basic_block bb)
{
- gimple last;
+ gimple *last;
tree op;
int index;
struct agg_position_info aggpos;
enum tree_code code, inverted_code;
edge e;
edge_iterator ei;
- gimple set_stmt;
+ gimple *set_stmt;
tree op2;
last = last_stmt (bb);
/* TODO: handle conditionals like
var = op0 < 4;
if (var != 0). */
- if (unmodified_parm_or_parm_agg_item (info, last, op, &index, &aggpos))
+ if (unmodified_parm_or_parm_agg_item (fbi, last, op, &index, &aggpos))
{
code = gimple_cond_code (last);
inverted_code = invert_tree_comparison (code, HONOR_NANS (op));
unordered one. Be sure it is not confused with NON_CONSTANT. */
if (this_code != ERROR_MARK)
{
- struct predicate p = add_condition (summary, index, &aggpos,
- this_code,
- gimple_cond_rhs (last));
- e->aux = pool_alloc (edge_predicate_pool);
+ struct predicate p = add_condition
+ (summary, index, &aggpos, this_code,
+ unshare_expr_without_location (gimple_cond_rhs (last)));
+ e->aux = edge_predicate_pool.allocate ();
*(struct predicate *) e->aux = p;
}
}
|| gimple_call_num_args (set_stmt) != 1)
return;
op2 = gimple_call_arg (set_stmt, 0);
- if (!unmodified_parm_or_parm_agg_item
- (info, set_stmt, op2, &index, &aggpos))
+ if (!unmodified_parm_or_parm_agg_item (fbi, set_stmt, op2, &index, &aggpos))
return;
FOR_EACH_EDGE (e, ei, bb->succs) if (e->flags & EDGE_FALSE_VALUE)
{
struct predicate p = add_condition (summary, index, &aggpos,
IS_NOT_CONSTANT, NULL_TREE);
- e->aux = pool_alloc (edge_predicate_pool);
+ e->aux = edge_predicate_pool.allocate ();
*(struct predicate *) e->aux = p;
}
}
predicates to the CFG edges. */
static void
-set_switch_stmt_execution_predicate (struct ipa_node_params *info,
+set_switch_stmt_execution_predicate (struct ipa_func_body_info *fbi,
struct inline_summary *summary,
basic_block bb)
{
- gimple lastg;
+ gimple *lastg;
tree op;
int index;
struct agg_position_info aggpos;
return;
gswitch *last = as_a <gswitch *> (lastg);
op = gimple_switch_index (last);
- if (!unmodified_parm_or_parm_agg_item (info, last, op, &index, &aggpos))
+ if (!unmodified_parm_or_parm_agg_item (fbi, last, op, &index, &aggpos))
return;
FOR_EACH_EDGE (e, ei, bb->succs)
{
- e->aux = pool_alloc (edge_predicate_pool);
+ e->aux = edge_predicate_pool.allocate ();
*(struct predicate *) e->aux = false_predicate ();
}
n = gimple_switch_num_labels (last);
if (!min && !max)
p = true_predicate ();
else if (!max)
- p = add_condition (summary, index, &aggpos, EQ_EXPR, min);
+ p = add_condition (summary, index, &aggpos, EQ_EXPR,
+ unshare_expr_without_location (min));
else
{
struct predicate p1, p2;
- p1 = add_condition (summary, index, &aggpos, GE_EXPR, min);
- p2 = add_condition (summary, index, &aggpos, LE_EXPR, max);
+ p1 = add_condition (summary, index, &aggpos, GE_EXPR,
+ unshare_expr_without_location (min));
+ p2 = add_condition (summary, index, &aggpos, LE_EXPR,
+ unshare_expr_without_location (max));
p = and_predicates (summary->conds, &p1, &p2);
}
*(struct predicate *) e->aux
which it is executable. */
static void
-compute_bb_predicates (struct cgraph_node *node,
- struct ipa_node_params *parms_info,
+compute_bb_predicates (struct ipa_func_body_info *fbi,
+ struct cgraph_node *node,
struct inline_summary *summary)
{
struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
FOR_EACH_BB_FN (bb, my_function)
{
- set_cond_stmt_execution_predicate (parms_info, summary, bb);
- set_switch_stmt_execution_predicate (parms_info, summary, bb);
+ set_cond_stmt_execution_predicate (fbi, summary, bb);
+ set_switch_stmt_execution_predicate (fbi, summary, bb);
}
/* Entry block is always executable. */
ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux
- = pool_alloc (edge_predicate_pool);
+ = edge_predicate_pool.allocate ();
*(struct predicate *) ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux
= true_predicate ();
if (!bb->aux)
{
done = false;
- bb->aux = pool_alloc (edge_predicate_pool);
+ bb->aux = edge_predicate_pool.allocate ();
*((struct predicate *) bb->aux) = p;
}
else if (!predicates_equal_p (&p, (struct predicate *) bb->aux))
a compile time constant. */
static struct predicate
-will_be_nonconstant_predicate (struct ipa_node_params *info,
+will_be_nonconstant_predicate (struct ipa_func_body_info *fbi,
struct inline_summary *summary,
- gimple stmt,
+ gimple *stmt,
vec<predicate_t> nonconstant_names)
{
struct predicate p = true_predicate ();
tree op;
gcc_assert (gimple_assign_single_p (stmt));
op = gimple_assign_rhs1 (stmt);
- if (!unmodified_parm_or_parm_agg_item (info, stmt, op, &base_index,
+ if (!unmodified_parm_or_parm_agg_item (fbi, stmt, op, &base_index,
&aggpos))
return p;
}
{
tree parm = unmodified_parm (stmt, use);
/* For arguments we can build a condition. */
- if (parm && ipa_get_param_decl_index (info, parm) >= 0)
+ if (parm && ipa_get_param_decl_index (fbi->info, parm) >= 0)
continue;
if (TREE_CODE (use) != SSA_NAME)
return p;
tree parm = unmodified_parm (stmt, use);
int index;
- if (parm && (index = ipa_get_param_decl_index (info, parm)) >= 0)
+ if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
{
if (index != base_index)
p = add_condition (summary, index, NULL, CHANGED, NULL_TREE);
struct record_modified_bb_info
{
bitmap bb_set;
- gimple stmt;
+ gimple *stmt;
};
/* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
ought to be REG_BR_PROB_BASE / estimated_iters. */
static int
-param_change_prob (gimple stmt, int i)
+param_change_prob (gimple *stmt, int i)
{
tree op = gimple_call_arg (stmt, i);
basic_block bb = gimple_bb (stmt);
static bool
phi_result_unknown_predicate (struct ipa_node_params *info,
- struct inline_summary *summary, basic_block bb,
+ inline_summary *summary, basic_block bb,
struct predicate *p,
vec<predicate_t> nonconstant_names)
{
edge e;
edge_iterator ei;
basic_block first_bb = NULL;
- gimple stmt;
+ gimple *stmt;
if (single_pred_p (bb))
{
/* Return predicate specifying when array index in access OP becomes non-constant. */
static struct predicate
-array_index_predicate (struct inline_summary *info,
+array_index_predicate (inline_summary *info,
vec< predicate_t> nonconstant_names, tree op)
{
struct predicate p = false_predicate ();
an impact on the earlier inlining.
Here find this pattern and fix it up later. */
-static gimple
+static gimple *
find_foldable_builtin_expect (basic_block bb)
{
gimple_stmt_iterator bsi;
for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
{
- gimple stmt = gsi_stmt (bsi);
+ gimple *stmt = gsi_stmt (bsi);
if (gimple_call_builtin_p (stmt, BUILT_IN_EXPECT)
|| (is_gimple_call (stmt)
&& gimple_call_internal_p (stmt)
tree var = gimple_call_lhs (stmt);
tree arg = gimple_call_arg (stmt, 0);
use_operand_p use_p;
- gimple use_stmt;
+ gimple *use_stmt;
bool match = false;
bool done = false;
while (TREE_CODE (arg) == SSA_NAME)
{
- gimple stmt_tmp = SSA_NAME_DEF_STMT (arg);
+ gimple *stmt_tmp = SSA_NAME_DEF_STMT (arg);
if (!is_gimple_assign (stmt_tmp))
break;
switch (gimple_assign_rhs_code (stmt_tmp))
for (; !gsi_end_p (gsi); gsi_prev (&gsi))
{
- gimple stmt = gsi_stmt (gsi);
+ gimple *stmt = gsi_stmt (gsi);
if (is_gimple_debug (stmt))
continue;
if (gimple_clobber_p (stmt))
basic_block bb;
struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
int freq;
- struct inline_summary *info = inline_summary (node);
+ struct inline_summary *info = inline_summaries->get (node);
struct predicate bb_predicate;
- struct ipa_node_params *parms_info = NULL;
+ struct ipa_func_body_info fbi;
vec<predicate_t> nonconstant_names = vNULL;
int nblocks, n;
int *order;
predicate array_index = true_predicate ();
- gimple fix_builtin_expect_stmt;
+ gimple *fix_builtin_expect_stmt;
+ gcc_assert (my_function && my_function->cfg);
+ gcc_assert (cfun == my_function);
+
+ memset(&fbi, 0, sizeof(fbi));
info->conds = NULL;
info->entry = NULL;
- if (opt_for_fn (node->decl, optimize) && !early)
+ /* When optimizing and analyzing for IPA inliner, initialize loop optimizer
+ so we can produce proper inline hints.
+
+ When optimizing and analyzing for early inliner, initialize node params
+ so we can produce correct BB predicates. */
+
+ if (opt_for_fn (node->decl, optimize))
{
calculate_dominance_info (CDI_DOMINATORS);
- loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS);
+ if (!early)
+ loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS);
+ else
+ {
+ ipa_check_create_node_params ();
+ ipa_initialize_node_params (node);
+ }
- if (ipa_node_params_vector.exists ())
+ if (ipa_node_params_sum)
{
- parms_info = IPA_NODE_REF (node);
+ fbi.node = node;
+ fbi.info = IPA_NODE_REF (node);
+ fbi.bb_infos = vNULL;
+ fbi.bb_infos.safe_grow_cleared (last_basic_block_for_fn (cfun));
+ fbi.param_count = count_formal_params(node->decl);
nonconstant_names.safe_grow_cleared
(SSANAMES (my_function)->length ());
}
bb_predicate = not_inlined_predicate ();
account_size_time (info, 2 * INLINE_SIZE_SCALE, 0, &bb_predicate);
- gcc_assert (my_function && my_function->cfg);
- if (parms_info)
- compute_bb_predicates (node, parms_info, info);
- gcc_assert (cfun == my_function);
+ if (fbi.info)
+ compute_bb_predicates (&fbi, node, info);
order = XNEWVEC (int, n_basic_blocks_for_fn (cfun));
nblocks = pre_and_rev_post_order_compute (NULL, order, false);
for (n = 0; n < nblocks; n++)
}
/* TODO: Obviously predicates can be propagated down across CFG. */
- if (parms_info)
+ if (fbi.info)
{
if (bb->aux)
bb_predicate = *(struct predicate *) bb->aux;
dump_predicate (dump_file, info->conds, &bb_predicate);
}
- if (parms_info && nonconstant_names.exists ())
+ if (fbi.info && nonconstant_names.exists ())
{
struct predicate phi_predicate;
bool first_phi = true;
gsi_next (&bsi))
{
if (first_phi
- && !phi_result_unknown_predicate (parms_info, info, bb,
+ && !phi_result_unknown_predicate (fbi.info, info, bb,
&phi_predicate,
nonconstant_names))
break;
for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi);
gsi_next (&bsi))
{
- gimple stmt = gsi_stmt (bsi);
+ gimple *stmt = gsi_stmt (bsi);
int this_size = estimate_num_insns (stmt, &eni_size_weights);
int this_time = estimate_num_insns (stmt, &eni_time_weights);
int prob;
nonconstant_names[SSA_NAME_VERSION (gimple_call_lhs (stmt))]
= false_p;
}
- if (ipa_node_params_vector.exists ())
+ if (ipa_node_params_sum)
{
int count = gimple_call_num_args (stmt);
int i;
/* TODO: When conditional jump or swithc is known to be constant, but
we did not translate it into the predicates, we really can account
just maximum of the possible paths. */
- if (parms_info)
+ if (fbi.info)
will_be_nonconstant
- = will_be_nonconstant_predicate (parms_info, info,
+ = will_be_nonconstant_predicate (&fbi, info,
stmt, nonconstant_names);
if (this_time || this_size)
{
if (prob == 2 && dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "\t\tWill be eliminated by inlining\n");
- if (parms_info)
+ if (fbi.info)
p = and_predicates (info->conds, &bb_predicate,
&will_be_nonconstant);
else
}
}
}
- set_hint_predicate (&inline_summary (node)->array_index, array_index);
+ set_hint_predicate (&inline_summaries->get (node)->array_index, array_index);
time = (time + CGRAPH_FREQ_BASE / 2) / CGRAPH_FREQ_BASE;
if (time > MAX_TIME)
time = MAX_TIME;
free (order);
- if (!early && nonconstant_names.exists ())
+ if (nonconstant_names.exists () && !early)
{
struct loop *loop;
predicate loop_iterations = true_predicate ();
{
vec<edge> exits;
edge ex;
- unsigned int j, i;
+ unsigned int j;
struct tree_niter_desc niter_desc;
- basic_block *body = get_loop_body (loop);
bb_predicate = *(struct predicate *) loop->header->aux;
exits = get_loop_exit_edges (loop);
&& !is_gimple_min_invariant (niter_desc.niter))
{
predicate will_be_nonconstant
- = will_be_nonconstant_expr_predicate (parms_info, info,
+ = will_be_nonconstant_expr_predicate (fbi.info, info,
niter_desc.niter,
nonconstant_names);
if (!true_predicate_p (&will_be_nonconstant))
&will_be_nonconstant);
}
exits.release ();
+ }
- for (i = 0; i < loop->num_nodes; i++)
+ /* To avoid quadratic behavior we analyze stride predicates only
+ with respect to the containing loop. Thus we simply iterate
+ over all defs in the outermost loop body. */
+ for (loop = loops_for_fn (cfun)->tree_root->inner;
+ loop != NULL; loop = loop->next)
+ {
+ basic_block *body = get_loop_body (loop);
+ for (unsigned i = 0; i < loop->num_nodes; i++)
{
gimple_stmt_iterator gsi;
bb_predicate = *(struct predicate *) body[i]->aux;
for (gsi = gsi_start_bb (body[i]); !gsi_end_p (gsi);
gsi_next (&gsi))
{
- gimple stmt = gsi_stmt (gsi);
- affine_iv iv;
- ssa_op_iter iter;
- tree use;
+ gimple *stmt = gsi_stmt (gsi);
- FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
- {
- predicate will_be_nonconstant;
+ if (!is_gimple_assign (stmt))
+ continue;
- if (!simple_iv
- (loop, loop_containing_stmt (stmt), use, &iv, true)
- || is_gimple_min_invariant (iv.step))
- continue;
+ tree def = gimple_assign_lhs (stmt);
+ if (TREE_CODE (def) != SSA_NAME)
+ continue;
+
+ affine_iv iv;
+ if (!simple_iv (loop_containing_stmt (stmt),
+ loop_containing_stmt (stmt),
+ def, &iv, true)
+ || is_gimple_min_invariant (iv.step))
+ continue;
+
+ predicate will_be_nonconstant
+ = will_be_nonconstant_expr_predicate (fbi.info, info,
+ iv.step,
+ nonconstant_names);
+ if (!true_predicate_p (&will_be_nonconstant))
will_be_nonconstant
- = will_be_nonconstant_expr_predicate (parms_info, info,
- iv.step,
- nonconstant_names);
- if (!true_predicate_p (&will_be_nonconstant))
- will_be_nonconstant
- = and_predicates (info->conds,
- &bb_predicate,
- &will_be_nonconstant);
- if (!true_predicate_p (&will_be_nonconstant)
- && !false_predicate_p (&will_be_nonconstant))
- /* This is slightly inprecise. We may want to represent
- each loop with independent predicate. */
- loop_stride =
- and_predicates (info->conds, &loop_stride,
+ = and_predicates (info->conds, &bb_predicate,
&will_be_nonconstant);
- }
+ if (!true_predicate_p (&will_be_nonconstant)
+ && !false_predicate_p (&will_be_nonconstant))
+ /* This is slightly inprecise. We may want to represent
+ each loop with independent predicate. */
+ loop_stride = and_predicates (info->conds, &loop_stride,
+ &will_be_nonconstant);
}
}
free (body);
}
- set_hint_predicate (&inline_summary (node)->loop_iterations,
+ set_hint_predicate (&inline_summaries->get (node)->loop_iterations,
loop_iterations);
- set_hint_predicate (&inline_summary (node)->loop_stride, loop_stride);
+ set_hint_predicate (&inline_summaries->get (node)->loop_stride,
+ loop_stride);
scev_finalize ();
}
FOR_ALL_BB_FN (bb, my_function)
edge_iterator ei;
if (bb->aux)
- pool_free (edge_predicate_pool, bb->aux);
+ edge_predicate_pool.remove ((predicate *)bb->aux);
bb->aux = NULL;
FOR_EACH_EDGE (e, ei, bb->succs)
{
if (e->aux)
- pool_free (edge_predicate_pool, e->aux);
+ edge_predicate_pool.remove ((predicate *) e->aux);
e->aux = NULL;
}
}
- inline_summary (node)->self_time = time;
- inline_summary (node)->self_size = size;
+ inline_summaries->get (node)->self_time = time;
+ inline_summaries->get (node)->self_size = size;
nonconstant_names.release ();
- if (opt_for_fn (node->decl, optimize) && !early)
+ ipa_release_body_info (&fbi);
+ if (opt_for_fn (node->decl, optimize))
{
- loop_optimizer_finalize ();
+ if (!early)
+ loop_optimizer_finalize ();
+ else if (!ipa_edge_args_vector)
+ ipa_free_all_node_params ();
free_dominance_info (CDI_DOMINATORS);
}
if (dump_file)
inline_summary_alloc ();
- info = inline_summary (node);
- reset_inline_summary (node);
+ info = inline_summaries->get (node);
+ reset_inline_summary (node, info);
/* FIXME: Thunks are inlinable, but tree-inline don't know how to do that.
Once this happen, we will need to more curefully predict call
else
info->inlinable = tree_inlinable_function_p (node->decl);
+ info->contains_cilk_spawn = fn_contains_cilk_spawn_p (cfun);
+
/* Type attributes can use parameter indices to describe them. */
if (TYPE_ATTRIBUTES (TREE_TYPE (node->decl)))
node->local.can_change_signature = false;
info->size = info->self_size;
info->stack_frame_offset = 0;
info->estimated_stack_size = info->estimated_self_stack_size;
-#ifdef ENABLE_CHECKING
- inline_update_overall_summary (node);
- gcc_assert (info->time == info->self_time && info->size == info->self_size);
-#endif
+ if (flag_checking)
+ {
+ inline_update_overall_summary (node);
+ gcc_assert (info->time == info->self_time
+ && info->size == info->self_size);
+ }
pop_cfun ();
}
callee = callee->function_symbol (&avail);
if (avail < AVAIL_AVAILABLE)
return false;
- isummary = inline_summary (callee);
+ isummary = inline_summaries->get (callee);
return isummary->inlinable;
}
struct cgraph_edge *e;
for (e = node->callees; e; e = e->next_callee)
{
+ if (inline_edge_summary_vec.length () <= (unsigned) e->uid)
+ continue;
+
struct inline_edge_summary *es = inline_edge_summary (e);
+
+ /* Do not care about zero sized builtins. */
+ if (e->inline_failed && !es->call_stmt_size)
+ {
+ gcc_checking_assert (!es->call_stmt_time);
+ continue;
+ }
if (!es->predicate
|| evaluate_predicate (es->predicate, possible_truths))
{
}
for (e = node->indirect_calls; e; e = e->next_callee)
{
+ if (inline_edge_summary_vec.length () <= (unsigned) e->uid)
+ continue;
+
struct inline_edge_summary *es = inline_edge_summary (e);
if (!es->predicate
|| evaluate_predicate (es->predicate, possible_truths))
vec<inline_param_summary>
inline_param_summary)
{
- struct inline_summary *info = inline_summary (node);
+ struct inline_summary *info = inline_summaries->get (node);
size_time_entry *e;
int size = 0;
int time = 0;
inline_update_callee_summaries (struct cgraph_node *node, int depth)
{
struct cgraph_edge *e;
- struct inline_summary *callee_info = inline_summary (node);
- struct inline_summary *caller_info = inline_summary (node->callers->caller);
+ struct inline_summary *callee_info = inline_summaries->get (node);
+ struct inline_summary *caller_info = inline_summaries->get (node->callers->caller);
HOST_WIDE_INT peak;
callee_info->stack_frame_offset
+ caller_info->estimated_self_stack_size;
peak = callee_info->stack_frame_offset
+ callee_info->estimated_self_stack_size;
- if (inline_summary (node->global.inlined_to)->estimated_stack_size < peak)
- inline_summary (node->global.inlined_to)->estimated_stack_size = peak;
+ if (inline_summaries->get (node->global.inlined_to)->estimated_stack_size < peak)
+ inline_summaries->get (node->global.inlined_to)->estimated_stack_size = peak;
ipa_propagate_frequency (node);
for (e = node->callees; e; e = e->next_callee)
{
remap_edge_change_prob (struct cgraph_edge *inlined_edge,
struct cgraph_edge *edge)
{
- if (ipa_node_params_vector.exists ())
+ if (ipa_node_params_sum)
{
int i;
struct ipa_edge_args *args = IPA_EDGE_REF (edge);
clause_t possible_truths,
struct predicate *toplev_predicate)
{
- struct cgraph_edge *e;
- for (e = node->callees; e; e = e->next_callee)
+ struct cgraph_edge *e, *next;
+ for (e = node->callees; e; e = next)
{
struct inline_edge_summary *es = inline_edge_summary (e);
struct predicate p;
+ next = e->next_callee;
if (e->inline_failed)
{
es->predicate, operand_map, offset_map,
possible_truths, toplev_predicate);
edge_set_predicate (e, &p);
- /* TODO: We should remove the edge for code that will be
- optimized out, but we need to keep verifiers and tree-inline
- happy. Make it cold for now. */
- if (false_predicate_p (&p))
- {
- e->count = 0;
- e->frequency = 0;
- }
}
else
edge_set_predicate (e, toplev_predicate);
operand_map, offset_map, possible_truths,
toplev_predicate);
}
- for (e = node->indirect_calls; e; e = e->next_callee)
+ for (e = node->indirect_calls; e; e = next)
{
struct inline_edge_summary *es = inline_edge_summary (e);
struct predicate p;
+ next = e->next_callee;
remap_edge_change_prob (inlined_edge, e);
if (es->predicate)
es->predicate, operand_map, offset_map,
possible_truths, toplev_predicate);
edge_set_predicate (e, &p);
- /* TODO: We should remove the edge for code that will be optimized
- out, but we need to keep verifiers and tree-inline happy.
- Make it cold for now. */
- if (false_predicate_p (&p))
- {
- e->count = 0;
- e->frequency = 0;
- }
}
else
edge_set_predicate (e, toplev_predicate);
void
inline_merge_summary (struct cgraph_edge *edge)
{
- struct inline_summary *callee_info = inline_summary (edge->callee);
+ struct inline_summary *callee_info = inline_summaries->get (edge->callee);
struct cgraph_node *to = (edge->caller->global.inlined_to
? edge->caller->global.inlined_to : edge->caller);
- struct inline_summary *info = inline_summary (to);
+ struct inline_summary *info = inline_summaries->get (to);
clause_t clause = 0; /* not_inline is known to be false. */
size_time_entry *e;
vec<int> operand_map = vNULL;
else
toplev_predicate = true_predicate ();
- if (ipa_node_params_vector.exists () && callee_info->conds)
+ if (callee_info->conds)
+ evaluate_properties_for_edge (edge, true, &clause, NULL, NULL, NULL);
+ if (ipa_node_params_sum && callee_info->conds)
{
struct ipa_edge_args *args = IPA_EDGE_REF (edge);
int count = ipa_get_cs_argument_count (args);
int i;
- evaluate_properties_for_edge (edge, true, &clause, NULL, NULL, NULL);
if (count)
{
operand_map.safe_grow_cleared (count);
void
inline_update_overall_summary (struct cgraph_node *node)
{
- struct inline_summary *info = inline_summary (node);
+ struct inline_summary *info = inline_summaries->get (node);
size_time_entry *e;
int i;
int hints = 0;
struct cgraph_node *to = (edge->caller->global.inlined_to
? edge->caller->global.inlined_to : edge->caller);
- if (inline_summary (to)->scc_no
- && inline_summary (to)->scc_no == inline_summary (edge->callee)->scc_no
+ struct cgraph_node *callee = edge->callee->ultimate_alias_target ();
+ if (inline_summaries->get (to)->scc_no
+ && inline_summaries->get (to)->scc_no
+ == inline_summaries->get (callee)->scc_no
&& !edge->recursive_p ())
hints |= INLINE_HINT_same_scc;
- if (to->lto_file_data && edge->callee->lto_file_data
- && to->lto_file_data != edge->callee->lto_file_data)
+ if (callee->lto_file_data && edge->caller->lto_file_data
+ && edge->caller->lto_file_data != callee->lto_file_data
+ && !callee->merged_comdat && !callee->icf_merged)
hints |= INLINE_HINT_cross_module;
return hints;
/* When caching, update the cache entry. */
if (edge_growth_cache.exists ())
{
- inline_summary (edge->callee)->min_size = min_size;
+ inline_summaries->get (edge->callee)->min_size = min_size;
if ((int) edge_growth_cache.length () <= edge->uid)
edge_growth_cache.safe_grow_cleared (symtab->edges_max_uid);
edge_growth_cache[edge->uid].time = time + (time >= 0);
if (!es->predicate || !false_predicate_p (es->predicate))
{
gcov_type time =
- inline_summary (node)->time + estimate_edge_time (edge);
+ inline_summaries->get (node)->time + estimate_edge_time (edge);
if (time < 0)
time = 0;
if (time > MAX_TIME)
time = MAX_TIME;
return time;
}
- return inline_summary (node)->time;
+ return inline_summaries->get (node)->time;
}
struct inline_edge_summary *es = inline_edge_summary (edge);
if (!es->predicate || !false_predicate_p (es->predicate))
{
- int size = inline_summary (node)->size + estimate_edge_growth (edge);
+ int size = inline_summaries->get (node)->size + estimate_edge_growth (edge);
gcc_assert (size >= 0);
return size;
}
- return inline_summary (node)->size;
+ return inline_summaries->get (node)->size;
}
{
struct cgraph_node *node;
bool self_recursive;
+ bool uninlinable;
int growth;
};
{
gcc_checking_assert (e->inline_failed);
- if (e->caller == d->node
- || (e->caller->global.inlined_to
- && e->caller->global.inlined_to == d->node))
- d->self_recursive = true;
+ if (cgraph_inline_failed_type (e->inline_failed) == CIF_FINAL_ERROR)
+ {
+ d->uninlinable = true;
+ continue;
+ }
+
+ if (e->recursive_p ())
+ {
+ d->self_recursive = true;
+ continue;
+ }
d->growth += estimate_edge_growth (e);
}
return false;
/* Estimate the growth caused by inlining NODE into all callees. */
int
-do_estimate_growth (struct cgraph_node *node)
+estimate_growth (struct cgraph_node *node)
{
- struct growth_data d = { node, 0, false };
- struct inline_summary *info = inline_summary (node);
+ struct growth_data d = { node, false, false, 0 };
+ struct inline_summary *info = inline_summaries->get (node);
- node->call_for_symbol_thunks_and_aliases (do_estimate_growth_1, &d, true);
+ node->call_for_symbol_and_aliases (do_estimate_growth_1, &d, true);
/* For self recursive functions the growth estimation really should be
infinity. We don't want to return very large values because the growth
return zero or negative growths. */
if (d.self_recursive)
d.growth = d.growth < info->size ? info->size : d.growth;
- else if (DECL_EXTERNAL (node->decl))
+ else if (DECL_EXTERNAL (node->decl) || d.uninlinable)
;
else
{
+ 50) / 100;
}
- if (node_growth_cache.exists ())
+ return d.growth;
+}
+
+/* Verify if there are fewer than MAX_CALLERS. */
+
+static bool
+check_callers (cgraph_node *node, int *max_callers)
+{
+ ipa_ref *ref;
+
+ if (!node->can_remove_if_no_direct_calls_and_refs_p ())
+ return true;
+
+ for (cgraph_edge *e = node->callers; e; e = e->next_caller)
{
- if ((int) node_growth_cache.length () <= node->uid)
- node_growth_cache.safe_grow_cleared (symtab->cgraph_max_uid);
- node_growth_cache[node->uid] = d.growth + (d.growth >= 0);
+ (*max_callers)--;
+ if (!*max_callers
+ || cgraph_inline_failed_type (e->inline_failed) == CIF_FINAL_ERROR)
+ return true;
}
- return d.growth;
+
+ FOR_EACH_ALIAS (node, ref)
+ if (check_callers (dyn_cast <cgraph_node *> (ref->referring), max_callers))
+ return true;
+
+ return false;
}
and skip computation if there are too many callers. */
bool
-growth_likely_positive (struct cgraph_node *node, int edge_growth ATTRIBUTE_UNUSED)
+growth_likely_positive (struct cgraph_node *node,
+ int edge_growth)
{
int max_callers;
- int ret;
struct cgraph_edge *e;
gcc_checking_assert (edge_growth > 0);
+ /* First quickly check if NODE is removable at all. */
+ if (DECL_EXTERNAL (node->decl))
+ return true;
+ if (!node->can_remove_if_no_direct_calls_and_refs_p ()
+ || node->address_taken)
+ return true;
+
+ max_callers = inline_summaries->get (node)->size * 4 / edge_growth + 2;
+
+ for (e = node->callers; e; e = e->next_caller)
+ {
+ max_callers--;
+ if (!max_callers
+ || cgraph_inline_failed_type (e->inline_failed) == CIF_FINAL_ERROR)
+ return true;
+ }
+
+ ipa_ref *ref;
+ FOR_EACH_ALIAS (node, ref)
+ if (check_callers (dyn_cast <cgraph_node *> (ref->referring), &max_callers))
+ return true;
+
/* Unlike for functions called once, we play unsafe with
COMDATs. We can allow that since we know functions
in consideration are small (and thus risk is small) and
functions may or may not disappear when eliminated from
current unit. With good probability making aggressive
choice in all units is going to make overall program
- smaller.
-
- Consequently we ask cgraph_can_remove_if_no_direct_calls_p
- instead of
- cgraph_will_be_removed_from_program_if_no_direct_calls */
- if (DECL_EXTERNAL (node->decl)
- || !node->can_remove_if_no_direct_calls_p ())
- return true;
-
- /* If there is cached value, just go ahead. */
- if ((int)node_growth_cache.length () > node->uid
- && (ret = node_growth_cache[node->uid]))
- return ret > 0;
- if (!node->will_be_removed_from_program_if_no_direct_calls_p ()
- && (!DECL_COMDAT (node->decl)
- || !node->can_remove_if_no_direct_calls_p ()))
- return true;
- max_callers = inline_summary (node)->size * 4 / edge_growth + 2;
-
- for (e = node->callers; e; e = e->next_caller)
+ smaller. */
+ if (DECL_COMDAT (node->decl))
{
- max_callers--;
- if (!max_callers)
+ if (!node->can_remove_if_no_direct_calls_p ())
return true;
}
+ else if (!node->will_be_removed_from_program_if_no_direct_calls_p ())
+ return true;
+
return estimate_growth (node) > 0;
}
/* Called when new function is inserted to callgraph late. */
-static void
-add_new_function (struct cgraph_node *node, void *data ATTRIBUTE_UNUSED)
+void
+inline_summary_t::insert (struct cgraph_node *node, inline_summary *)
{
inline_analyze_function (node);
}
-
/* Note function body size. */
void
{
struct cgraph_node *node;
+ FOR_EACH_DEFINED_FUNCTION (node)
+ if (DECL_STRUCT_FUNCTION (node->decl))
+ node->local.versionable = tree_versionable_function_p (node->decl);
+
/* When not optimizing, do not bother to analyze. Inlining is still done
because edge redirection needs to happen there. */
if (!optimize && !flag_generate_lto && !flag_generate_offload && !flag_wpa)
return;
- function_insertion_hook_holder =
- symtab->add_cgraph_insertion_hook (&add_new_function, NULL);
+ if (!inline_summaries)
+ inline_summaries = (inline_summary_t*) inline_summary_t::create_ggc (symtab);
+
+ inline_summaries->enable_insertion_hook ();
ipa_register_cgraph_hooks ();
inline_free_summary ();
unsigned int i, count2, j;
unsigned int f_count;
- lto_input_block ib ((const char *) data + main_offset, header->main_size);
+ lto_input_block ib ((const char *) data + main_offset, header->main_size,
+ file_data->mode_table);
data_in =
lto_data_in_create (file_data, (const char *) data + string_offset,
encoder = file_data->symtab_node_encoder;
node = dyn_cast<cgraph_node *> (lto_symtab_encoder_deref (encoder,
index));
- info = inline_summary (node);
+ info = inline_summaries->get (node);
info->estimated_stack_size
= info->estimated_self_stack_size = streamer_read_uhwi (&ib);
bp = streamer_read_bitpack (&ib);
info->inlinable = bp_unpack_value (&bp, 1);
+ info->contains_cilk_spawn = bp_unpack_value (&bp, 1);
count2 = streamer_read_uhwi (&ib);
gcc_assert (!info->conds);
/* Fatal error here. We do not want to support compiling ltrans units
with different version of compiler or different flags than the WPA
unit, so this should never happen. */
- fatal_error ("ipa inline summary is missing in input file");
+ fatal_error (input_location,
+ "ipa inline summary is missing in input file");
}
if (optimize)
{
if (!flag_ipa_cp)
ipa_prop_read_jump_functions ();
}
- function_insertion_hook_holder =
- symtab->add_cgraph_insertion_hook (&add_new_function, NULL);
+
+ gcc_assert (inline_summaries);
+ inline_summaries->enable_insertion_hook ();
}
cgraph_node *cnode = dyn_cast <cgraph_node *> (snode);
if (cnode && (node = cnode)->definition && !node->alias)
{
- struct inline_summary *info = inline_summary (node);
+ struct inline_summary *info = inline_summaries->get (node);
struct bitpack_d bp;
struct cgraph_edge *edge;
int i;
streamer_write_hwi (ob, info->self_time);
bp = bitpack_create (ob->main_stream);
bp_pack_value (&bp, info->inlinable, 1);
+ bp_pack_value (&bp, info->contains_cilk_spawn, 1);
streamer_write_bitpack (&bp);
streamer_write_uhwi (ob, vec_safe_length (info->conds));
for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
inline_free_summary (void)
{
struct cgraph_node *node;
- if (function_insertion_hook_holder)
- symtab->remove_cgraph_insertion_hook (function_insertion_hook_holder);
- function_insertion_hook_holder = NULL;
- if (node_removal_hook_holder)
- symtab->remove_cgraph_removal_hook (node_removal_hook_holder);
- node_removal_hook_holder = NULL;
if (edge_removal_hook_holder)
symtab->remove_edge_removal_hook (edge_removal_hook_holder);
edge_removal_hook_holder = NULL;
- if (node_duplication_hook_holder)
- symtab->remove_cgraph_duplication_hook (node_duplication_hook_holder);
- node_duplication_hook_holder = NULL;
if (edge_duplication_hook_holder)
symtab->remove_edge_duplication_hook (edge_duplication_hook_holder);
edge_duplication_hook_holder = NULL;
return;
FOR_EACH_DEFINED_FUNCTION (node)
if (!node->alias)
- reset_inline_summary (node);
- vec_free (inline_summary_vec);
+ reset_inline_summary (node, inline_summaries->get (node));
+ inline_summaries->release ();
+ inline_summaries = NULL;
inline_edge_summary_vec.release ();
- if (edge_predicate_pool)
- free_alloc_pool (edge_predicate_pool);
- edge_predicate_pool = 0;
+ edge_predicate_pool.release ();
}