/* Inlining decision heuristics.
- Copyright (C) 2003-2013 Free Software Foundation, Inc.
+ Copyright (C) 2003-2016 Free Software Foundation, Inc.
Contributed by Jan Hubicka
This file is part of GCC.
#include "config.h"
#include "system.h"
#include "coretypes.h"
-#include "tm.h"
+#include "backend.h"
#include "tree.h"
-#include "stor-layout.h"
-#include "stringpool.h"
+#include "gimple.h"
+#include "alloc-pool.h"
+#include "tree-pass.h"
+#include "ssa.h"
+#include "tree-streamer.h"
+#include "cgraph.h"
+#include "diagnostic.h"
+#include "fold-const.h"
#include "print-tree.h"
#include "tree-inline.h"
-#include "langhooks.h"
-#include "flags.h"
-#include "diagnostic.h"
#include "gimple-pretty-print.h"
#include "params.h"
-#include "tree-pass.h"
-#include "coverage.h"
-#include "basic-block.h"
-#include "tree-ssa-alias.h"
-#include "internal-fn.h"
-#include "gimple-expr.h"
-#include "is-a.h"
-#include "gimple.h"
+#include "cfganal.h"
#include "gimple-iterator.h"
-#include "gimple-ssa.h"
#include "tree-cfg.h"
-#include "tree-phinodes.h"
-#include "ssa-iterators.h"
-#include "tree-ssanames.h"
#include "tree-ssa-loop-niter.h"
#include "tree-ssa-loop.h"
+#include "symbol-summary.h"
#include "ipa-prop.h"
-#include "lto-streamer.h"
-#include "data-streamer.h"
-#include "tree-streamer.h"
#include "ipa-inline.h"
-#include "alloc-pool.h"
#include "cfgloop.h"
#include "tree-scalar-evolution.h"
#include "ipa-utils.h"
#include "cilk.h"
#include "cfgexpand.h"
+#include "gimplify.h"
/* Estimate runtime of function can easilly run into huge numbers with many
nested loops. Be sure we can compute time * INLINE_SIZE_SCALE * 2 in an
#define CHANGED IDENTIFIER_NODE
/* Holders of ipa cgraph hooks: */
-static struct cgraph_node_hook_list *function_insertion_hook_holder;
-static struct cgraph_node_hook_list *node_removal_hook_holder;
-static struct cgraph_2node_hook_list *node_duplication_hook_holder;
static struct cgraph_2edge_hook_list *edge_duplication_hook_holder;
static struct cgraph_edge_hook_list *edge_removal_hook_holder;
-static void inline_node_removal_hook (struct cgraph_node *, void *);
-static void inline_node_duplication_hook (struct cgraph_node *,
- struct cgraph_node *, void *);
static void inline_edge_removal_hook (struct cgraph_edge *, void *);
static void inline_edge_duplication_hook (struct cgraph_edge *,
struct cgraph_edge *, void *);
/* VECtor holding inline summaries.
In GGC memory because conditions might point to constant trees. */
-vec<inline_summary_t, va_gc> *inline_summary_vec;
+function_summary <inline_summary *> *inline_summaries;
vec<inline_edge_summary_t> inline_edge_summary_vec;
/* Cached node/edge growths. */
-vec<int> node_growth_cache;
vec<edge_growth_cache_entry> edge_growth_cache;
/* Edge predicates goes here. */
-static alloc_pool edge_predicate_pool;
+static object_allocator<predicate> edge_predicate_pool ("edge predicates");
/* Return true predicate (tautology).
We represent it by empty list of clauses. */
}
-/* Return true if P is (false). */
+/* Return true if P is (true). */
static inline bool
true_predicate_p (struct predicate *p)
if (false_predicate_p (p))
return;
- /* No one should be sily enough to add false into nontrivial clauses. */
+ /* No one should be silly enough to add false into nontrivial clauses. */
gcc_checking_assert (!(clause & (1 << predicate_false_condition)));
/* Look where to insert the clause. At the same time prune out
&& cc1->val == cc2->val
&& cc2->code != IS_NOT_CONSTANT
&& cc2->code != CHANGED
- && cc1->code == invert_tree_comparison
- (cc2->code,
- HONOR_NANS (TYPE_MODE (TREE_TYPE (cc1->val)))))
+ && cc1->code == invert_tree_comparison (cc2->code,
+ HONOR_NANS (cc1->val)))
return;
}
}
static int
predicate_probability (conditions conds,
struct predicate *p, clause_t possible_truths,
- vec<inline_param_summary_t> inline_param_summary)
+ vec<inline_param_summary> inline_param_summary)
{
int i;
int combined_prob = REG_BR_PROB_BASE;
hints &= ~INLINE_HINT_array_index;
fprintf (f, " array_index");
}
+ if (hints & INLINE_HINT_known_hot)
+ {
+ hints &= ~INLINE_HINT_known_hot;
+ fprintf (f, " known_hot");
+ }
gcc_assert (!hints);
}
}
}
+/* We proved E to be unreachable, redirect it to __bultin_unreachable. */
+
+static struct cgraph_edge *
+redirect_to_unreachable (struct cgraph_edge *e)
+{
+ struct cgraph_node *callee = !e->inline_failed ? e->callee : NULL;
+ struct cgraph_node *target = cgraph_node::get_create
+ (builtin_decl_implicit (BUILT_IN_UNREACHABLE));
+
+ if (e->speculative)
+ e = e->resolve_speculation (target->decl);
+ else if (!e->callee)
+ e->make_direct (target);
+ else
+ e->redirect_callee (target);
+ struct inline_edge_summary *es = inline_edge_summary (e);
+ e->inline_failed = CIF_UNREACHABLE;
+ e->frequency = 0;
+ e->count = 0;
+ es->call_stmt_size = 0;
+ es->call_stmt_time = 0;
+ if (callee)
+ callee->remove_symbol_and_inline_clones ();
+ return e;
+}
+
/* Set predicate for edge E. */
static void
edge_set_predicate (struct cgraph_edge *e, struct predicate *predicate)
{
+ /* If the edge is determined to be never executed, redirect it
+ to BUILTIN_UNREACHABLE to save inliner from inlining into it. */
+ if (predicate && false_predicate_p (predicate)
+ /* When handling speculative edges, we need to do the redirection
+ just once. Do it always on the direct edge, so we do not
+ attempt to resolve speculation while duplicating the edge. */
+ && (!e->speculative || e->callee))
+ e = redirect_to_unreachable (e);
+
struct inline_edge_summary *es = inline_edge_summary (e);
if (predicate && !true_predicate_p (predicate))
{
if (!es->predicate)
- es->predicate = (struct predicate *) pool_alloc (edge_predicate_pool);
+ es->predicate = edge_predicate_pool.allocate ();
*es->predicate = *predicate;
}
else
{
if (es->predicate)
- pool_free (edge_predicate_pool, es->predicate);
+ edge_predicate_pool.remove (es->predicate);
es->predicate = NULL;
}
}
if (false_predicate_p (&new_predicate) || true_predicate_p (&new_predicate))
{
if (*p)
- pool_free (edge_predicate_pool, *p);
+ edge_predicate_pool.remove (*p);
*p = NULL;
}
else
{
if (!*p)
- *p = (struct predicate *) pool_alloc (edge_predicate_pool);
+ *p = edge_predicate_pool.allocate ();
**p = new_predicate;
}
}
known_aggs)
{
clause_t clause = inline_p ? 0 : 1 << predicate_not_inlined_condition;
- struct inline_summary *info = inline_summary (node);
+ struct inline_summary *info = inline_summaries->get (node);
int i;
struct condition *c;
}
if (c->code == IS_NOT_CONSTANT || c->code == CHANGED)
continue;
- res = fold_binary_to_constant (c->code, boolean_type_node, val, c->val);
- if (res && integer_zerop (res))
- continue;
+
+ if (operand_equal_p (TYPE_SIZE (TREE_TYPE (c->val)),
+ TYPE_SIZE (TREE_TYPE (val)), 0))
+ {
+ val = fold_unary (VIEW_CONVERT_EXPR, TREE_TYPE (c->val), val);
+
+ res = val
+ ? fold_binary_to_constant (c->code, boolean_type_node, val, c->val)
+ : NULL;
+
+ if (res && integer_zerop (res))
+ continue;
+ }
clause |= 1 << (i + predicate_first_dynamic_condition);
}
return clause;
evaluate_properties_for_edge (struct cgraph_edge *e, bool inline_p,
clause_t *clause_ptr,
vec<tree> *known_vals_ptr,
- vec<tree> *known_binfos_ptr,
+ vec<ipa_polymorphic_call_context>
+ *known_contexts_ptr,
vec<ipa_agg_jump_function_p> *known_aggs_ptr)
{
- struct cgraph_node *callee =
- cgraph_function_or_thunk_node (e->callee, NULL);
- struct inline_summary *info = inline_summary (callee);
+ struct cgraph_node *callee = e->callee->ultimate_alias_target ();
+ struct inline_summary *info = inline_summaries->get (callee);
vec<tree> known_vals = vNULL;
vec<ipa_agg_jump_function_p> known_aggs = vNULL;
*clause_ptr = inline_p ? 0 : 1 << predicate_not_inlined_condition;
if (known_vals_ptr)
known_vals_ptr->create (0);
- if (known_binfos_ptr)
- known_binfos_ptr->create (0);
+ if (known_contexts_ptr)
+ known_contexts_ptr->create (0);
- if (ipa_node_params_vector.exists ()
+ if (ipa_node_params_sum
&& !e->call_stmt_cannot_inline_p
- && ((clause_ptr && info->conds) || known_vals_ptr || known_binfos_ptr))
+ && ((clause_ptr && info->conds) || known_vals_ptr || known_contexts_ptr))
{
struct ipa_node_params *parms_info;
struct ipa_edge_args *args = IPA_EDGE_REF (e);
known_vals.safe_grow_cleared (count);
if (count && (info->conds || known_aggs_ptr))
known_aggs.safe_grow_cleared (count);
- if (count && known_binfos_ptr)
- known_binfos_ptr->safe_grow_cleared (count);
+ if (count && known_contexts_ptr)
+ known_contexts_ptr->safe_grow_cleared (count);
for (i = 0; i < count; i++)
{
struct ipa_jump_func *jf = ipa_get_ith_jump_func (args, i);
tree cst = ipa_value_from_jfunc (parms_info, jf);
+
+ if (!cst && e->call_stmt
+ && i < (int)gimple_call_num_args (e->call_stmt))
+ {
+ cst = gimple_call_arg (e->call_stmt, i);
+ if (!is_gimple_min_invariant (cst))
+ cst = NULL;
+ }
if (cst)
{
- if (known_vals.exists () && TREE_CODE (cst) != TREE_BINFO)
+ gcc_checking_assert (TREE_CODE (cst) != TREE_BINFO);
+ if (known_vals.exists ())
known_vals[i] = cst;
- else if (known_binfos_ptr != NULL
- && TREE_CODE (cst) == TREE_BINFO)
- (*known_binfos_ptr)[i] = cst;
}
else if (inline_p && !es->param[i].change_prob)
known_vals[i] = error_mark_node;
+
+ if (known_contexts_ptr)
+ (*known_contexts_ptr)[i] = ipa_context_from_jfunc (parms_info, e,
+ i, jf);
/* TODO: When IPA-CP starts propagating and merging aggregate jump
functions, use its knowledge of the caller too, just like the
scalar case above. */
known_aggs[i] = &jf->agg;
}
}
+ else if (e->call_stmt && !e->call_stmt_cannot_inline_p
+ && ((clause_ptr && info->conds) || known_vals_ptr))
+ {
+ int i, count = (int)gimple_call_num_args (e->call_stmt);
+
+ if (count && (info->conds || known_vals_ptr))
+ known_vals.safe_grow_cleared (count);
+ for (i = 0; i < count; i++)
+ {
+ tree cst = gimple_call_arg (e->call_stmt, i);
+ if (!is_gimple_min_invariant (cst))
+ cst = NULL;
+ if (cst)
+ known_vals[i] = cst;
+ }
+ }
if (clause_ptr)
*clause_ptr = evaluate_conditions_for_known_args (callee, inline_p,
static void
inline_summary_alloc (void)
{
- if (!node_removal_hook_holder)
- node_removal_hook_holder =
- cgraph_add_node_removal_hook (&inline_node_removal_hook, NULL);
if (!edge_removal_hook_holder)
edge_removal_hook_holder =
- cgraph_add_edge_removal_hook (&inline_edge_removal_hook, NULL);
- if (!node_duplication_hook_holder)
- node_duplication_hook_holder =
- cgraph_add_node_duplication_hook (&inline_node_duplication_hook, NULL);
+ symtab->add_edge_removal_hook (&inline_edge_removal_hook, NULL);
if (!edge_duplication_hook_holder)
edge_duplication_hook_holder =
- cgraph_add_edge_duplication_hook (&inline_edge_duplication_hook, NULL);
-
- if (vec_safe_length (inline_summary_vec) <= (unsigned) cgraph_max_uid)
- vec_safe_grow_cleared (inline_summary_vec, cgraph_max_uid + 1);
- if (inline_edge_summary_vec.length () <= (unsigned) cgraph_edge_max_uid)
- inline_edge_summary_vec.safe_grow_cleared (cgraph_edge_max_uid + 1);
- if (!edge_predicate_pool)
- edge_predicate_pool = create_alloc_pool ("edge predicates",
- sizeof (struct predicate), 10);
+ symtab->add_edge_duplication_hook (&inline_edge_duplication_hook, NULL);
+
+ if (!inline_summaries)
+ inline_summaries = (inline_summary_t*) inline_summary_t::create_ggc (symtab);
+
+ if (inline_edge_summary_vec.length () <= (unsigned) symtab->edges_max_uid)
+ inline_edge_summary_vec.safe_grow_cleared (symtab->edges_max_uid + 1);
}
/* We are called multiple time for given function; clear
es->call_stmt_size = es->call_stmt_time = 0;
if (es->predicate)
- pool_free (edge_predicate_pool, es->predicate);
+ edge_predicate_pool.remove (es->predicate);
es->predicate = NULL;
es->param.release ();
}
data from previous run so they are not cumulated. */
static void
-reset_inline_summary (struct cgraph_node *node)
+reset_inline_summary (struct cgraph_node *node,
+ inline_summary *info)
{
- struct inline_summary *info = inline_summary (node);
struct cgraph_edge *e;
info->self_size = info->self_time = 0;
info->scc_no = 0;
if (info->loop_iterations)
{
- pool_free (edge_predicate_pool, info->loop_iterations);
+ edge_predicate_pool.remove (info->loop_iterations);
info->loop_iterations = NULL;
}
if (info->loop_stride)
{
- pool_free (edge_predicate_pool, info->loop_stride);
+ edge_predicate_pool.remove (info->loop_stride);
info->loop_stride = NULL;
}
if (info->array_index)
{
- pool_free (edge_predicate_pool, info->array_index);
+ edge_predicate_pool.remove (info->array_index);
info->array_index = NULL;
}
vec_free (info->conds);
/* Hook that is called by cgraph.c when a node is removed. */
-static void
-inline_node_removal_hook (struct cgraph_node *node,
- void *data ATTRIBUTE_UNUSED)
+void
+inline_summary_t::remove (cgraph_node *node, inline_summary *info)
{
- struct inline_summary *info;
- if (vec_safe_length (inline_summary_vec) <= (unsigned) node->uid)
- return;
- info = inline_summary (node);
- reset_inline_summary (node);
- memset (info, 0, sizeof (inline_summary_t));
+ reset_inline_summary (node, info);
}
-/* Remap predicate P of former function to be predicate of duplicated functoin.
+/* Remap predicate P of former function to be predicate of duplicated function.
POSSIBLE_TRUTHS is clause of possible truths in the duplicated node,
INFO is inline summary of the duplicated node. */
/* Hook that is called by cgraph.c when a node is duplicated. */
-
-static void
-inline_node_duplication_hook (struct cgraph_node *src,
- struct cgraph_node *dst,
- ATTRIBUTE_UNUSED void *data)
+void
+inline_summary_t::duplicate (cgraph_node *src,
+ cgraph_node *dst,
+ inline_summary *,
+ inline_summary *info)
{
- struct inline_summary *info;
inline_summary_alloc ();
- info = inline_summary (dst);
- memcpy (info, inline_summary (src), sizeof (struct inline_summary));
+ memcpy (info, inline_summaries->get (src), sizeof (inline_summary));
/* TODO: as an optimization, we may avoid copying conditions
that are known to be false or true. */
info->conds = vec_safe_copy (info->conds);
/* When there are any replacements in the function body, see if we can figure
out that something was optimized out. */
- if (ipa_node_params_vector.exists () && dst->clone.tree_map)
+ if (ipa_node_params_sum && dst->clone.tree_map)
{
vec<size_time_entry, va_gc> *entry = info->entry;
/* Use SRC parm info since it may not be copied yet. */
size_time_entry *e;
int optimized_out_size = 0;
bool inlined_to_p = false;
- struct cgraph_edge *edge;
+ struct cgraph_edge *edge, *next;
info->entry = 0;
known_vals.safe_grow_cleared (count);
/* Remap edge predicates with the same simplification as above.
Also copy constantness arrays. */
- for (edge = dst->callees; edge; edge = edge->next_callee)
+ for (edge = dst->callees; edge; edge = next)
{
struct predicate new_predicate;
struct inline_edge_summary *es = inline_edge_summary (edge);
+ next = edge->next_callee;
if (!edge->inline_failed)
inlined_to_p = true;
info);
if (false_predicate_p (&new_predicate)
&& !false_predicate_p (es->predicate))
- {
- optimized_out_size += es->call_stmt_size * INLINE_SIZE_SCALE;
- edge->frequency = 0;
- }
+ optimized_out_size += es->call_stmt_size * INLINE_SIZE_SCALE;
edge_set_predicate (edge, &new_predicate);
}
/* Remap indirect edge predicates with the same simplificaiton as above.
Also copy constantness arrays. */
- for (edge = dst->indirect_calls; edge; edge = edge->next_callee)
+ for (edge = dst->indirect_calls; edge; edge = next)
{
struct predicate new_predicate;
struct inline_edge_summary *es = inline_edge_summary (edge);
+ next = edge->next_callee;
gcc_checking_assert (edge->inline_failed);
if (!es->predicate)
info);
if (false_predicate_p (&new_predicate)
&& !false_predicate_p (es->predicate))
- {
- optimized_out_size += es->call_stmt_size * INLINE_SIZE_SCALE;
- edge->frequency = 0;
- }
+ optimized_out_size += es->call_stmt_size * INLINE_SIZE_SCALE;
edge_set_predicate (edge, &new_predicate);
}
remap_hint_predicate_after_duplication (&info->loop_iterations,
set_hint_predicate (&info->array_index, p);
}
}
- inline_update_overall_summary (dst);
+ if (!dst->global.inlined_to)
+ inline_update_overall_summary (dst);
}
info->predicate = NULL;
edge_set_predicate (dst, srcinfo->predicate);
info->param = srcinfo->param.copy ();
+ if (!dst->indirect_unknown_callee && src->indirect_unknown_callee)
+ {
+ info->call_stmt_size -= (eni_size_weights.indirect_call_cost
+ - eni_size_weights.call_cost);
+ info->call_stmt_time -= (eni_time_weights.indirect_call_cost
+ - eni_time_weights.call_cost);
+ }
}
void
initialize_growth_caches (void)
{
- if (cgraph_edge_max_uid)
- edge_growth_cache.safe_grow_cleared (cgraph_edge_max_uid);
- if (cgraph_max_uid)
- node_growth_cache.safe_grow_cleared (cgraph_max_uid);
+ if (symtab->edges_max_uid)
+ edge_growth_cache.safe_grow_cleared (symtab->edges_max_uid);
}
free_growth_caches (void)
{
edge_growth_cache.release ();
- node_growth_cache.release ();
}
for (edge = node->callees; edge; edge = edge->next_callee)
{
struct inline_edge_summary *es = inline_edge_summary (edge);
- struct cgraph_node *callee =
- cgraph_function_or_thunk_node (edge->callee, NULL);
+ struct cgraph_node *callee = edge->callee->ultimate_alias_target ();
int i;
fprintf (f,
? "inlined" : cgraph_inline_failed_string (edge-> inline_failed),
indent, "", es->loop_depth, edge->frequency,
es->call_stmt_size, es->call_stmt_time,
- (int) inline_summary (callee)->size / INLINE_SIZE_SCALE,
- (int) inline_summary (callee)->estimated_stack_size);
+ (int) inline_summaries->get (callee)->size / INLINE_SIZE_SCALE,
+ (int) inline_summaries->get (callee)->estimated_stack_size);
if (es->predicate)
{
fprintf (f, "%*sStack frame offset %i, callee self size %i,"
" callee size %i\n",
indent + 2, "",
- (int) inline_summary (callee)->stack_frame_offset,
- (int) inline_summary (callee)->estimated_self_stack_size,
- (int) inline_summary (callee)->estimated_stack_size);
+ (int) inline_summaries->get (callee)->stack_frame_offset,
+ (int) inline_summaries->get (callee)->estimated_self_stack_size,
+ (int) inline_summaries->get (callee)->estimated_stack_size);
dump_inline_edge_summary (f, indent + 2, callee, info);
}
}
{
if (node->definition)
{
- struct inline_summary *s = inline_summary (node);
+ struct inline_summary *s = inline_summaries->get (node);
size_time_entry *e;
int i;
fprintf (f, "Inline summary for %s/%i", node->name (),
fprintf (f, " always_inline");
if (s->inlinable)
fprintf (f, " inlinable");
+ if (s->contains_cilk_spawn)
+ fprintf (f, " contains_cilk_spawn");
fprintf (f, "\n self time: %i\n", s->self_time);
fprintf (f, " global time: %i\n", s->time);
fprintf (f, " self size: %i\n", s->self_size);
fprintf (f, " global size: %i\n", s->size);
+ fprintf (f, " min size: %i\n", s->min_size);
fprintf (f, " self stack: %i\n",
(int) s->estimated_self_stack_size);
fprintf (f, " global stack: %i\n", (int) s->estimated_stack_size);
parameter. */
static tree
-unmodified_parm_1 (gimple stmt, tree op)
+unmodified_parm_1 (gimple *stmt, tree op)
{
/* SSA_NAME referring to parm default def? */
if (TREE_CODE (op) == SSA_NAME
parameter. Also traverse chains of SSA register assignments. */
static tree
-unmodified_parm (gimple stmt, tree op)
+unmodified_parm (gimple *stmt, tree op)
{
tree res = unmodified_parm_1 (stmt, op);
if (res)
loaded. */
static bool
-unmodified_parm_or_parm_agg_item (struct ipa_node_params *info,
- gimple stmt, tree op, int *index_p,
+unmodified_parm_or_parm_agg_item (struct ipa_func_body_info *fbi,
+ gimple *stmt, tree op, int *index_p,
struct agg_position_info *aggpos)
{
tree res = unmodified_parm_1 (stmt, op);
gcc_checking_assert (aggpos);
if (res)
{
- *index_p = ipa_get_param_decl_index (info, res);
+ *index_p = ipa_get_param_decl_index (fbi->info, res);
if (*index_p < 0)
return false;
aggpos->agg_contents = false;
stmt = SSA_NAME_DEF_STMT (op);
op = gimple_assign_rhs1 (stmt);
if (!REFERENCE_CLASS_P (op))
- return unmodified_parm_or_parm_agg_item (info, stmt, op, index_p,
+ return unmodified_parm_or_parm_agg_item (fbi, stmt, op, index_p,
aggpos);
}
aggpos->agg_contents = true;
- return ipa_load_from_parm_agg (info, stmt, op, index_p, &aggpos->offset,
- &aggpos->by_ref);
+ return ipa_load_from_parm_agg (fbi, fbi->info->descriptors,
+ stmt, op, index_p, &aggpos->offset,
+ NULL, &aggpos->by_ref);
}
/* See if statement might disappear after inlining.
penalty wrappers. */
static int
-eliminated_by_inlining_prob (gimple stmt)
+eliminated_by_inlining_prob (gimple *stmt)
{
enum gimple_code code = gimple_code (stmt);
enum tree_code rhs_code;
and stores to return value or parameters are often free after
inlining dua to SRA and further combining.
Assume that half of statements goes away. */
- if (rhs_code == CONVERT_EXPR
- || rhs_code == NOP_EXPR
+ if (CONVERT_EXPR_CODE_P (rhs_code)
|| rhs_code == VIEW_CONVERT_EXPR
|| rhs_code == ADDR_EXPR
|| gimple_assign_rhs_class (stmt) == GIMPLE_SINGLE_RHS)
predicates to the CFG edges. */
static void
-set_cond_stmt_execution_predicate (struct ipa_node_params *info,
+set_cond_stmt_execution_predicate (struct ipa_func_body_info *fbi,
struct inline_summary *summary,
basic_block bb)
{
- gimple last;
+ gimple *last;
tree op;
int index;
struct agg_position_info aggpos;
enum tree_code code, inverted_code;
edge e;
edge_iterator ei;
- gimple set_stmt;
+ gimple *set_stmt;
tree op2;
last = last_stmt (bb);
/* TODO: handle conditionals like
var = op0 < 4;
if (var != 0). */
- if (unmodified_parm_or_parm_agg_item (info, last, op, &index, &aggpos))
+ if (unmodified_parm_or_parm_agg_item (fbi, last, op, &index, &aggpos))
{
code = gimple_cond_code (last);
- inverted_code
- = invert_tree_comparison (code,
- HONOR_NANS (TYPE_MODE (TREE_TYPE (op))));
+ inverted_code = invert_tree_comparison (code, HONOR_NANS (op));
FOR_EACH_EDGE (e, ei, bb->succs)
{
- struct predicate p = add_condition (summary, index, &aggpos,
- e->flags & EDGE_TRUE_VALUE
- ? code : inverted_code,
- gimple_cond_rhs (last));
- e->aux = pool_alloc (edge_predicate_pool);
- *(struct predicate *) e->aux = p;
+ enum tree_code this_code = (e->flags & EDGE_TRUE_VALUE
+ ? code : inverted_code);
+ /* invert_tree_comparison will return ERROR_MARK on FP
+ comparsions that are not EQ/NE instead of returning proper
+ unordered one. Be sure it is not confused with NON_CONSTANT. */
+ if (this_code != ERROR_MARK)
+ {
+ struct predicate p = add_condition
+ (summary, index, &aggpos, this_code,
+ unshare_expr_without_location (gimple_cond_rhs (last)));
+ e->aux = edge_predicate_pool.allocate ();
+ *(struct predicate *) e->aux = p;
+ }
}
}
|| gimple_call_num_args (set_stmt) != 1)
return;
op2 = gimple_call_arg (set_stmt, 0);
- if (!unmodified_parm_or_parm_agg_item
- (info, set_stmt, op2, &index, &aggpos))
+ if (!unmodified_parm_or_parm_agg_item (fbi, set_stmt, op2, &index, &aggpos))
return;
FOR_EACH_EDGE (e, ei, bb->succs) if (e->flags & EDGE_FALSE_VALUE)
{
struct predicate p = add_condition (summary, index, &aggpos,
IS_NOT_CONSTANT, NULL_TREE);
- e->aux = pool_alloc (edge_predicate_pool);
+ e->aux = edge_predicate_pool.allocate ();
*(struct predicate *) e->aux = p;
}
}
predicates to the CFG edges. */
static void
-set_switch_stmt_execution_predicate (struct ipa_node_params *info,
+set_switch_stmt_execution_predicate (struct ipa_func_body_info *fbi,
struct inline_summary *summary,
basic_block bb)
{
- gimple last;
+ gimple *lastg;
tree op;
int index;
struct agg_position_info aggpos;
size_t n;
size_t case_idx;
- last = last_stmt (bb);
- if (!last || gimple_code (last) != GIMPLE_SWITCH)
+ lastg = last_stmt (bb);
+ if (!lastg || gimple_code (lastg) != GIMPLE_SWITCH)
return;
+ gswitch *last = as_a <gswitch *> (lastg);
op = gimple_switch_index (last);
- if (!unmodified_parm_or_parm_agg_item (info, last, op, &index, &aggpos))
+ if (!unmodified_parm_or_parm_agg_item (fbi, last, op, &index, &aggpos))
return;
FOR_EACH_EDGE (e, ei, bb->succs)
{
- e->aux = pool_alloc (edge_predicate_pool);
+ e->aux = edge_predicate_pool.allocate ();
*(struct predicate *) e->aux = false_predicate ();
}
n = gimple_switch_num_labels (last);
if (!min && !max)
p = true_predicate ();
else if (!max)
- p = add_condition (summary, index, &aggpos, EQ_EXPR, min);
+ p = add_condition (summary, index, &aggpos, EQ_EXPR,
+ unshare_expr_without_location (min));
else
{
struct predicate p1, p2;
- p1 = add_condition (summary, index, &aggpos, GE_EXPR, min);
- p2 = add_condition (summary, index, &aggpos, LE_EXPR, max);
+ p1 = add_condition (summary, index, &aggpos, GE_EXPR,
+ unshare_expr_without_location (min));
+ p2 = add_condition (summary, index, &aggpos, LE_EXPR,
+ unshare_expr_without_location (max));
p = and_predicates (summary->conds, &p1, &p2);
}
*(struct predicate *) e->aux
which it is executable. */
static void
-compute_bb_predicates (struct cgraph_node *node,
- struct ipa_node_params *parms_info,
+compute_bb_predicates (struct ipa_func_body_info *fbi,
+ struct cgraph_node *node,
struct inline_summary *summary)
{
struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
FOR_EACH_BB_FN (bb, my_function)
{
- set_cond_stmt_execution_predicate (parms_info, summary, bb);
- set_switch_stmt_execution_predicate (parms_info, summary, bb);
+ set_cond_stmt_execution_predicate (fbi, summary, bb);
+ set_switch_stmt_execution_predicate (fbi, summary, bb);
}
/* Entry block is always executable. */
ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux
- = pool_alloc (edge_predicate_pool);
+ = edge_predicate_pool.allocate ();
*(struct predicate *) ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux
= true_predicate ();
if (!bb->aux)
{
done = false;
- bb->aux = pool_alloc (edge_predicate_pool);
+ bb->aux = edge_predicate_pool.allocate ();
*((struct predicate *) bb->aux) = p;
}
else if (!predicates_equal_p (&p, (struct predicate *) bb->aux))
{
- done = false;
- *((struct predicate *) bb->aux) = p;
+ /* This OR operation is needed to ensure monotonous data flow
+ in the case we hit the limit on number of clauses and the
+ and/or operations above give approximate answers. */
+ p = or_predicates (summary->conds, &p, (struct predicate *)bb->aux);
+ if (!predicates_equal_p (&p, (struct predicate *) bb->aux))
+ {
+ done = false;
+ *((struct predicate *) bb->aux) = p;
+ }
}
}
}
a compile time constant. */
static struct predicate
-will_be_nonconstant_predicate (struct ipa_node_params *info,
+will_be_nonconstant_predicate (struct ipa_func_body_info *fbi,
struct inline_summary *summary,
- gimple stmt,
+ gimple *stmt,
vec<predicate_t> nonconstant_names)
{
struct predicate p = true_predicate ();
struct agg_position_info aggpos;
/* What statments might be optimized away
- when their arguments are constant
- TODO: also trivial builtins.
- builtin_constant_p is already handled later. */
+ when their arguments are constant. */
if (gimple_code (stmt) != GIMPLE_ASSIGN
&& gimple_code (stmt) != GIMPLE_COND
- && gimple_code (stmt) != GIMPLE_SWITCH)
+ && gimple_code (stmt) != GIMPLE_SWITCH
+ && (gimple_code (stmt) != GIMPLE_CALL
+ || !(gimple_call_flags (stmt) & ECF_CONST)))
return p;
/* Stores will stay anyway. */
tree op;
gcc_assert (gimple_assign_single_p (stmt));
op = gimple_assign_rhs1 (stmt);
- if (!unmodified_parm_or_parm_agg_item (info, stmt, op, &base_index,
+ if (!unmodified_parm_or_parm_agg_item (fbi, stmt, op, &base_index,
&aggpos))
return p;
}
{
tree parm = unmodified_parm (stmt, use);
/* For arguments we can build a condition. */
- if (parm && ipa_get_param_decl_index (info, parm) >= 0)
+ if (parm && ipa_get_param_decl_index (fbi->info, parm) >= 0)
continue;
if (TREE_CODE (use) != SSA_NAME)
return p;
tree parm = unmodified_parm (stmt, use);
int index;
- if (parm && (index = ipa_get_param_decl_index (info, parm)) >= 0)
+ if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
{
if (index != base_index)
p = add_condition (summary, index, NULL, CHANGED, NULL_TREE);
p = nonconstant_names[SSA_NAME_VERSION (use)];
op_non_const = or_predicates (summary->conds, &p, &op_non_const);
}
- if (gimple_code (stmt) == GIMPLE_ASSIGN
- && TREE_CODE (gimple_assign_lhs (stmt)) == SSA_NAME)
- nonconstant_names[SSA_NAME_VERSION (gimple_assign_lhs (stmt))]
+ if ((gimple_code (stmt) == GIMPLE_ASSIGN || gimple_code (stmt) == GIMPLE_CALL)
+ && gimple_op (stmt, 0)
+ && TREE_CODE (gimple_op (stmt, 0)) == SSA_NAME)
+ nonconstant_names[SSA_NAME_VERSION (gimple_op (stmt, 0))]
= op_non_const;
return op_non_const;
}
struct record_modified_bb_info
{
bitmap bb_set;
- gimple stmt;
+ gimple *stmt;
};
/* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
ought to be REG_BR_PROB_BASE / estimated_iters. */
static int
-param_change_prob (gimple stmt, int i)
+param_change_prob (gimple *stmt, int i)
{
tree op = gimple_call_arg (stmt, i);
basic_block bb = gimple_bb (stmt);
max = 1;
EXECUTE_IF_SET_IN_BITMAP (info.bb_set, 0, index, bi)
- max = MIN (max, BASIC_BLOCK (index)->frequency);
+ max = MIN (max, BASIC_BLOCK_FOR_FN (cfun, index)->frequency);
BITMAP_FREE (info.bb_set);
if (max < bb->frequency)
static bool
phi_result_unknown_predicate (struct ipa_node_params *info,
- struct inline_summary *summary, basic_block bb,
+ inline_summary *summary, basic_block bb,
struct predicate *p,
vec<predicate_t> nonconstant_names)
{
edge e;
edge_iterator ei;
basic_block first_bb = NULL;
- gimple stmt;
+ gimple *stmt;
if (single_pred_p (bb))
{
NONCONSTANT_NAMES, if possible. */
static void
-predicate_for_phi_result (struct inline_summary *summary, gimple phi,
+predicate_for_phi_result (struct inline_summary *summary, gphi *phi,
struct predicate *p,
vec<predicate_t> nonconstant_names)
{
/* Return predicate specifying when array index in access OP becomes non-constant. */
static struct predicate
-array_index_predicate (struct inline_summary *info,
+array_index_predicate (inline_summary *info,
vec< predicate_t> nonconstant_names, tree op)
{
struct predicate p = false_predicate ();
an impact on the earlier inlining.
Here find this pattern and fix it up later. */
-static gimple
+static gimple *
find_foldable_builtin_expect (basic_block bb)
{
gimple_stmt_iterator bsi;
for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
{
- gimple stmt = gsi_stmt (bsi);
- if (gimple_call_builtin_p (stmt, BUILT_IN_EXPECT))
+ gimple *stmt = gsi_stmt (bsi);
+ if (gimple_call_builtin_p (stmt, BUILT_IN_EXPECT)
+ || (is_gimple_call (stmt)
+ && gimple_call_internal_p (stmt)
+ && gimple_call_internal_fn (stmt) == IFN_BUILTIN_EXPECT))
{
tree var = gimple_call_lhs (stmt);
tree arg = gimple_call_arg (stmt, 0);
use_operand_p use_p;
- gimple use_stmt;
+ gimple *use_stmt;
bool match = false;
bool done = false;
while (TREE_CODE (arg) == SSA_NAME)
{
- gimple stmt_tmp = SSA_NAME_DEF_STMT (arg);
+ gimple *stmt_tmp = SSA_NAME_DEF_STMT (arg);
if (!is_gimple_assign (stmt_tmp))
break;
switch (gimple_assign_rhs_code (stmt_tmp))
match = true;
done = true;
break;
- case NOP_EXPR:
+ CASE_CONVERT:
break;
default:
done = true;
return NULL;
}
+/* Return true when the basic blocks contains only clobbers followed by RESX.
+ Such BBs are kept around to make removal of dead stores possible with
+ presence of EH and will be optimized out by optimize_clobbers later in the
+ game.
+
+ NEED_EH is used to recurse in case the clobber has non-EH predecestors
+ that can be clobber only, too.. When it is false, the RESX is not necessary
+ on the end of basic block. */
+
+static bool
+clobber_only_eh_bb_p (basic_block bb, bool need_eh = true)
+{
+ gimple_stmt_iterator gsi = gsi_last_bb (bb);
+ edge_iterator ei;
+ edge e;
+
+ if (need_eh)
+ {
+ if (gsi_end_p (gsi))
+ return false;
+ if (gimple_code (gsi_stmt (gsi)) != GIMPLE_RESX)
+ return false;
+ gsi_prev (&gsi);
+ }
+ else if (!single_succ_p (bb))
+ return false;
+
+ for (; !gsi_end_p (gsi); gsi_prev (&gsi))
+ {
+ gimple *stmt = gsi_stmt (gsi);
+ if (is_gimple_debug (stmt))
+ continue;
+ if (gimple_clobber_p (stmt))
+ continue;
+ if (gimple_code (stmt) == GIMPLE_LABEL)
+ break;
+ return false;
+ }
+
+ /* See if all predecestors are either throws or clobber only BBs. */
+ FOR_EACH_EDGE (e, ei, bb->preds)
+ if (!(e->flags & EDGE_EH)
+ && !clobber_only_eh_bb_p (e->src, false))
+ return false;
+
+ return true;
+}
+
/* Compute function body size parameters for NODE.
When EARLY is true, we compute only simple summaries without
non-trivial predicates to drive the early inliner. */
/* Benefits are scaled by probability of elimination that is in range
<0,2>. */
basic_block bb;
- gimple_stmt_iterator bsi;
struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
int freq;
- struct inline_summary *info = inline_summary (node);
+ struct inline_summary *info = inline_summaries->get (node);
struct predicate bb_predicate;
- struct ipa_node_params *parms_info = NULL;
+ struct ipa_func_body_info fbi;
vec<predicate_t> nonconstant_names = vNULL;
int nblocks, n;
int *order;
predicate array_index = true_predicate ();
- gimple fix_builtin_expect_stmt;
+ gimple *fix_builtin_expect_stmt;
+
+ gcc_assert (my_function && my_function->cfg);
+ gcc_assert (cfun == my_function);
+ memset(&fbi, 0, sizeof(fbi));
info->conds = NULL;
info->entry = NULL;
- if (optimize && !early)
+ /* When optimizing and analyzing for IPA inliner, initialize loop optimizer
+ so we can produce proper inline hints.
+
+ When optimizing and analyzing for early inliner, initialize node params
+ so we can produce correct BB predicates. */
+
+ if (opt_for_fn (node->decl, optimize))
{
calculate_dominance_info (CDI_DOMINATORS);
- loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS);
+ if (!early)
+ loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS);
+ else
+ {
+ ipa_check_create_node_params ();
+ ipa_initialize_node_params (node);
+ }
- if (ipa_node_params_vector.exists ())
+ if (ipa_node_params_sum)
{
- parms_info = IPA_NODE_REF (node);
+ fbi.node = node;
+ fbi.info = IPA_NODE_REF (node);
+ fbi.bb_infos = vNULL;
+ fbi.bb_infos.safe_grow_cleared (last_basic_block_for_fn (cfun));
+ fbi.param_count = count_formal_params(node->decl);
nonconstant_names.safe_grow_cleared
(SSANAMES (my_function)->length ());
}
bb_predicate = not_inlined_predicate ();
account_size_time (info, 2 * INLINE_SIZE_SCALE, 0, &bb_predicate);
- gcc_assert (my_function && my_function->cfg);
- if (parms_info)
- compute_bb_predicates (node, parms_info, info);
- gcc_assert (cfun == my_function);
+ if (fbi.info)
+ compute_bb_predicates (&fbi, node, info);
order = XNEWVEC (int, n_basic_blocks_for_fn (cfun));
nblocks = pre_and_rev_post_order_compute (NULL, order, false);
for (n = 0; n < nblocks; n++)
{
- bb = BASIC_BLOCK (order[n]);
+ bb = BASIC_BLOCK_FOR_FN (cfun, order[n]);
freq = compute_call_stmt_bb_frequency (node->decl, bb);
+ if (clobber_only_eh_bb_p (bb))
+ {
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ fprintf (dump_file, "\n Ignoring BB %i;"
+ " it will be optimized away by cleanup_clobbers\n",
+ bb->index);
+ continue;
+ }
/* TODO: Obviously predicates can be propagated down across CFG. */
- if (parms_info)
+ if (fbi.info)
{
if (bb->aux)
bb_predicate = *(struct predicate *) bb->aux;
dump_predicate (dump_file, info->conds, &bb_predicate);
}
- if (parms_info && nonconstant_names.exists ())
+ if (fbi.info && nonconstant_names.exists ())
{
struct predicate phi_predicate;
bool first_phi = true;
- for (bsi = gsi_start_phis (bb); !gsi_end_p (bsi); gsi_next (&bsi))
+ for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi);
+ gsi_next (&bsi))
{
if (first_phi
- && !phi_result_unknown_predicate (parms_info, info, bb,
+ && !phi_result_unknown_predicate (fbi.info, info, bb,
&phi_predicate,
nonconstant_names))
break;
fprintf (dump_file, " ");
print_gimple_stmt (dump_file, gsi_stmt (bsi), 0, 0);
}
- predicate_for_phi_result (info, gsi_stmt (bsi), &phi_predicate,
+ predicate_for_phi_result (info, bsi.phi (), &phi_predicate,
nonconstant_names);
}
}
fix_builtin_expect_stmt = find_foldable_builtin_expect (bb);
- for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
+ for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi);
+ gsi_next (&bsi))
{
- gimple stmt = gsi_stmt (bsi);
+ gimple *stmt = gsi_stmt (bsi);
int this_size = estimate_num_insns (stmt, &eni_size_weights);
int this_time = estimate_num_insns (stmt, &eni_time_weights);
int prob;
if (is_gimple_call (stmt)
&& !gimple_call_internal_p (stmt))
{
- struct cgraph_edge *edge = cgraph_edge (node, stmt);
+ struct cgraph_edge *edge = node->get_edge (stmt);
struct inline_edge_summary *es = inline_edge_summary (edge);
/* Special case: results of BUILT_IN_CONSTANT_P will be always
nonconstant_names[SSA_NAME_VERSION (gimple_call_lhs (stmt))]
= false_p;
}
- if (ipa_node_params_vector.exists ())
+ if (ipa_node_params_sum)
{
int count = gimple_call_num_args (stmt);
int i;
/* TODO: When conditional jump or swithc is known to be constant, but
we did not translate it into the predicates, we really can account
just maximum of the possible paths. */
- if (parms_info)
+ if (fbi.info)
will_be_nonconstant
- = will_be_nonconstant_predicate (parms_info, info,
+ = will_be_nonconstant_predicate (&fbi, info,
stmt, nonconstant_names);
if (this_time || this_size)
{
if (prob == 2 && dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "\t\tWill be eliminated by inlining\n");
- if (parms_info)
+ if (fbi.info)
p = and_predicates (info->conds, &bb_predicate,
&will_be_nonconstant);
else
p = true_predicate ();
- if (!false_predicate_p (&p))
+ if (!false_predicate_p (&p)
+ || (is_gimple_call (stmt)
+ && !false_predicate_p (&bb_predicate)))
{
time += this_time;
size += this_size;
}
}
}
- set_hint_predicate (&inline_summary (node)->array_index, array_index);
+ set_hint_predicate (&inline_summaries->get (node)->array_index, array_index);
time = (time + CGRAPH_FREQ_BASE / 2) / CGRAPH_FREQ_BASE;
if (time > MAX_TIME)
time = MAX_TIME;
free (order);
- if (!early && nonconstant_names.exists ())
+ if (nonconstant_names.exists () && !early)
{
struct loop *loop;
predicate loop_iterations = true_predicate ();
{
vec<edge> exits;
edge ex;
- unsigned int j, i;
+ unsigned int j;
struct tree_niter_desc niter_desc;
- basic_block *body = get_loop_body (loop);
bb_predicate = *(struct predicate *) loop->header->aux;
exits = get_loop_exit_edges (loop);
&& !is_gimple_min_invariant (niter_desc.niter))
{
predicate will_be_nonconstant
- = will_be_nonconstant_expr_predicate (parms_info, info,
+ = will_be_nonconstant_expr_predicate (fbi.info, info,
niter_desc.niter,
nonconstant_names);
if (!true_predicate_p (&will_be_nonconstant))
&will_be_nonconstant);
}
exits.release ();
+ }
- for (i = 0; i < loop->num_nodes; i++)
+ /* To avoid quadratic behavior we analyze stride predicates only
+ with respect to the containing loop. Thus we simply iterate
+ over all defs in the outermost loop body. */
+ for (loop = loops_for_fn (cfun)->tree_root->inner;
+ loop != NULL; loop = loop->next)
+ {
+ basic_block *body = get_loop_body (loop);
+ for (unsigned i = 0; i < loop->num_nodes; i++)
{
gimple_stmt_iterator gsi;
bb_predicate = *(struct predicate *) body[i]->aux;
for (gsi = gsi_start_bb (body[i]); !gsi_end_p (gsi);
gsi_next (&gsi))
{
- gimple stmt = gsi_stmt (gsi);
- affine_iv iv;
- ssa_op_iter iter;
- tree use;
+ gimple *stmt = gsi_stmt (gsi);
- FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
- {
- predicate will_be_nonconstant;
+ if (!is_gimple_assign (stmt))
+ continue;
- if (!simple_iv
- (loop, loop_containing_stmt (stmt), use, &iv, true)
- || is_gimple_min_invariant (iv.step))
- continue;
+ tree def = gimple_assign_lhs (stmt);
+ if (TREE_CODE (def) != SSA_NAME)
+ continue;
+
+ affine_iv iv;
+ if (!simple_iv (loop_containing_stmt (stmt),
+ loop_containing_stmt (stmt),
+ def, &iv, true)
+ || is_gimple_min_invariant (iv.step))
+ continue;
+
+ predicate will_be_nonconstant
+ = will_be_nonconstant_expr_predicate (fbi.info, info,
+ iv.step,
+ nonconstant_names);
+ if (!true_predicate_p (&will_be_nonconstant))
will_be_nonconstant
- = will_be_nonconstant_expr_predicate (parms_info, info,
- iv.step,
- nonconstant_names);
- if (!true_predicate_p (&will_be_nonconstant))
- will_be_nonconstant
- = and_predicates (info->conds,
- &bb_predicate,
- &will_be_nonconstant);
- if (!true_predicate_p (&will_be_nonconstant)
- && !false_predicate_p (&will_be_nonconstant))
- /* This is slightly inprecise. We may want to represent
- each loop with independent predicate. */
- loop_stride =
- and_predicates (info->conds, &loop_stride,
+ = and_predicates (info->conds, &bb_predicate,
&will_be_nonconstant);
- }
+ if (!true_predicate_p (&will_be_nonconstant)
+ && !false_predicate_p (&will_be_nonconstant))
+ /* This is slightly inprecise. We may want to represent
+ each loop with independent predicate. */
+ loop_stride = and_predicates (info->conds, &loop_stride,
+ &will_be_nonconstant);
}
}
free (body);
}
- set_hint_predicate (&inline_summary (node)->loop_iterations,
+ set_hint_predicate (&inline_summaries->get (node)->loop_iterations,
loop_iterations);
- set_hint_predicate (&inline_summary (node)->loop_stride, loop_stride);
+ set_hint_predicate (&inline_summaries->get (node)->loop_stride,
+ loop_stride);
scev_finalize ();
}
FOR_ALL_BB_FN (bb, my_function)
edge_iterator ei;
if (bb->aux)
- pool_free (edge_predicate_pool, bb->aux);
+ edge_predicate_pool.remove ((predicate *)bb->aux);
bb->aux = NULL;
FOR_EACH_EDGE (e, ei, bb->succs)
{
if (e->aux)
- pool_free (edge_predicate_pool, e->aux);
+ edge_predicate_pool.remove ((predicate *) e->aux);
e->aux = NULL;
}
}
- inline_summary (node)->self_time = time;
- inline_summary (node)->self_size = size;
+ inline_summaries->get (node)->self_time = time;
+ inline_summaries->get (node)->self_size = size;
nonconstant_names.release ();
- if (optimize && !early)
+ ipa_release_body_info (&fbi);
+ if (opt_for_fn (node->decl, optimize))
{
- loop_optimizer_finalize ();
+ if (!early)
+ loop_optimizer_finalize ();
+ else if (!ipa_edge_args_vector)
+ ipa_free_all_node_params ();
free_dominance_info (CDI_DOMINATORS);
}
if (dump_file)
inline_summary_alloc ();
- info = inline_summary (node);
- reset_inline_summary (node);
+ info = inline_summaries->get (node);
+ reset_inline_summary (node, info);
/* FIXME: Thunks are inlinable, but tree-inline don't know how to do that.
Once this happen, we will need to more curefully predict call
info->stack_frame_offset = 0;
/* Can this function be inlined at all? */
- if (!optimize && !lookup_attribute ("always_inline",
- DECL_ATTRIBUTES (node->decl)))
+ if (!opt_for_fn (node->decl, optimize)
+ && !lookup_attribute ("always_inline",
+ DECL_ATTRIBUTES (node->decl)))
info->inlinable = false;
else
info->inlinable = tree_inlinable_function_p (node->decl);
+ info->contains_cilk_spawn = fn_contains_cilk_spawn_p (cfun);
+
/* Type attributes can use parameter indices to describe them. */
if (TYPE_ATTRIBUTES (TREE_TYPE (node->decl)))
node->local.can_change_signature = false;
}
estimate_function_body_sizes (node, early);
+ for (e = node->callees; e; e = e->next_callee)
+ if (e->callee->comdat_local_p ())
+ break;
+ node->calls_comdat_local = (e != NULL);
+
/* Inlining characteristics are maintained by the cgraph_mark_inline. */
info->time = info->self_time;
info->size = info->self_size;
info->stack_frame_offset = 0;
info->estimated_stack_size = info->estimated_self_stack_size;
-#ifdef ENABLE_CHECKING
- inline_update_overall_summary (node);
- gcc_assert (info->time == info->self_time && info->size == info->self_size);
-#endif
+ if (flag_checking)
+ {
+ inline_update_overall_summary (node);
+ gcc_assert (info->time == info->self_time
+ && info->size == info->self_size);
+ }
pop_cfun ();
}
static unsigned int
compute_inline_parameters_for_current (void)
{
- compute_inline_parameters (cgraph_get_node (current_function_decl), true);
+ compute_inline_parameters (cgraph_node::get (current_function_decl), true);
return 0;
}
GIMPLE_PASS, /* type */
"inline_param", /* name */
OPTGROUP_INLINE, /* optinfo_flags */
- false, /* has_gate */
- true, /* has_execute */
TV_INLINE_PARAMETERS, /* tv_id */
0, /* properties_required */
0, /* properties_provided */
/* opt_pass methods: */
opt_pass * clone () { return new pass_inline_parameters (m_ctxt); }
- unsigned int execute () {
- return compute_inline_parameters_for_current ();
- }
+ virtual unsigned int execute (function *)
+ {
+ return compute_inline_parameters_for_current ();
+ }
}; // class pass_inline_parameters
}
-/* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS and
- KNOWN_BINFOS. */
+/* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS,
+ KNOWN_CONTEXTS and KNOWN_AGGS. */
static bool
estimate_edge_devirt_benefit (struct cgraph_edge *ie,
int *size, int *time,
vec<tree> known_vals,
- vec<tree> known_binfos,
+ vec<ipa_polymorphic_call_context> known_contexts,
vec<ipa_agg_jump_function_p> known_aggs)
{
tree target;
struct cgraph_node *callee;
struct inline_summary *isummary;
+ enum availability avail;
+ bool speculative;
- if (!known_vals.exists () && !known_binfos.exists ())
+ if (!known_vals.exists () && !known_contexts.exists ())
return false;
- if (!flag_indirect_inlining)
+ if (!opt_for_fn (ie->caller->decl, flag_indirect_inlining))
return false;
- target = ipa_get_indirect_edge_target (ie, known_vals, known_binfos,
- known_aggs);
- if (!target)
+ target = ipa_get_indirect_edge_target (ie, known_vals, known_contexts,
+ known_aggs, &speculative);
+ if (!target || speculative)
return false;
/* Account for difference in cost between indirect and direct calls. */
gcc_checking_assert (*time >= 0);
gcc_checking_assert (*size >= 0);
- callee = cgraph_get_node (target);
+ callee = cgraph_node::get (target);
if (!callee || !callee->definition)
return false;
- isummary = inline_summary (callee);
+ callee = callee->function_symbol (&avail);
+ if (avail < AVAIL_AVAILABLE)
+ return false;
+ isummary = inline_summaries->get (callee);
return isummary->inlinable;
}
-/* Increase SIZE and TIME for size and time needed to handle edge E. */
+/* Increase SIZE, MIN_SIZE (if non-NULL) and TIME for size and time needed to
+ handle edge E with probability PROB.
+ Set HINTS if edge may be devirtualized.
+ KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS describe context of the call
+ site. */
static inline void
-estimate_edge_size_and_time (struct cgraph_edge *e, int *size, int *time,
+estimate_edge_size_and_time (struct cgraph_edge *e, int *size, int *min_size,
+ int *time,
int prob,
vec<tree> known_vals,
- vec<tree> known_binfos,
+ vec<ipa_polymorphic_call_context> known_contexts,
vec<ipa_agg_jump_function_p> known_aggs,
inline_hints *hints)
{
struct inline_edge_summary *es = inline_edge_summary (e);
int call_size = es->call_stmt_size;
int call_time = es->call_stmt_time;
+ int cur_size;
if (!e->callee
&& estimate_edge_devirt_benefit (e, &call_size, &call_time,
- known_vals, known_binfos, known_aggs)
- && hints && cgraph_maybe_hot_edge_p (e))
+ known_vals, known_contexts, known_aggs)
+ && hints && e->maybe_hot_p ())
*hints |= INLINE_HINT_indirect_call;
- *size += call_size * INLINE_SIZE_SCALE;
+ cur_size = call_size * INLINE_SIZE_SCALE;
+ *size += cur_size;
+ if (min_size)
+ *min_size += cur_size;
*time += apply_probability ((gcov_type) call_time, prob)
* e->frequency * (INLINE_TIME_SCALE / CGRAPH_FREQ_BASE);
if (*time > MAX_TIME * INLINE_TIME_SCALE)
-/* Increase SIZE and TIME for size and time needed to handle all calls in NODE.
- POSSIBLE_TRUTHS, KNOWN_VALS and KNOWN_BINFOS describe context of the call
- site. */
+/* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
+ calls in NODE. POSSIBLE_TRUTHS, KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS
+ describe context of the call site. */
static void
-estimate_calls_size_and_time (struct cgraph_node *node, int *size, int *time,
+estimate_calls_size_and_time (struct cgraph_node *node, int *size,
+ int *min_size, int *time,
inline_hints *hints,
clause_t possible_truths,
vec<tree> known_vals,
- vec<tree> known_binfos,
+ vec<ipa_polymorphic_call_context> known_contexts,
vec<ipa_agg_jump_function_p> known_aggs)
{
struct cgraph_edge *e;
for (e = node->callees; e; e = e->next_callee)
{
+ if (inline_edge_summary_vec.length () <= (unsigned) e->uid)
+ continue;
+
struct inline_edge_summary *es = inline_edge_summary (e);
+
+ /* Do not care about zero sized builtins. */
+ if (e->inline_failed && !es->call_stmt_size)
+ {
+ gcc_checking_assert (!es->call_stmt_time);
+ continue;
+ }
if (!es->predicate
|| evaluate_predicate (es->predicate, possible_truths))
{
{
/* Predicates of calls shall not use NOT_CHANGED codes,
sowe do not need to compute probabilities. */
- estimate_edge_size_and_time (e, size, time, REG_BR_PROB_BASE,
- known_vals, known_binfos,
+ estimate_edge_size_and_time (e, size,
+ es->predicate ? NULL : min_size,
+ time, REG_BR_PROB_BASE,
+ known_vals, known_contexts,
known_aggs, hints);
}
else
- estimate_calls_size_and_time (e->callee, size, time, hints,
+ estimate_calls_size_and_time (e->callee, size, min_size, time,
+ hints,
possible_truths,
- known_vals, known_binfos,
+ known_vals, known_contexts,
known_aggs);
}
}
for (e = node->indirect_calls; e; e = e->next_callee)
{
+ if (inline_edge_summary_vec.length () <= (unsigned) e->uid)
+ continue;
+
struct inline_edge_summary *es = inline_edge_summary (e);
if (!es->predicate
|| evaluate_predicate (es->predicate, possible_truths))
- estimate_edge_size_and_time (e, size, time, REG_BR_PROB_BASE,
- known_vals, known_binfos, known_aggs,
+ estimate_edge_size_and_time (e, size,
+ es->predicate ? NULL : min_size,
+ time, REG_BR_PROB_BASE,
+ known_vals, known_contexts, known_aggs,
hints);
}
}
/* Estimate size and time needed to execute NODE assuming
- POSSIBLE_TRUTHS clause, and KNOWN_VALS and KNOWN_BINFOS information
- about NODE's arguments. */
+ POSSIBLE_TRUTHS clause, and KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS
+ information about NODE's arguments. If non-NULL use also probability
+ information present in INLINE_PARAM_SUMMARY vector.
+ Additionally detemine hints determined by the context. Finally compute
+ minimal size needed for the call that is independent on the call context and
+ can be used for fast estimates. Return the values in RET_SIZE,
+ RET_MIN_SIZE, RET_TIME and RET_HINTS. */
static void
estimate_node_size_and_time (struct cgraph_node *node,
clause_t possible_truths,
vec<tree> known_vals,
- vec<tree> known_binfos,
+ vec<ipa_polymorphic_call_context> known_contexts,
vec<ipa_agg_jump_function_p> known_aggs,
- int *ret_size, int *ret_time,
+ int *ret_size, int *ret_min_size, int *ret_time,
inline_hints *ret_hints,
- vec<inline_param_summary_t>
+ vec<inline_param_summary>
inline_param_summary)
{
- struct inline_summary *info = inline_summary (node);
+ struct inline_summary *info = inline_summaries->get (node);
size_time_entry *e;
int size = 0;
int time = 0;
+ int min_size = 0;
inline_hints hints = 0;
int i;
gcc_checking_assert (time >= 0);
}
+ gcc_checking_assert (true_predicate_p (&(*info->entry)[0].predicate));
+ min_size = (*info->entry)[0].size;
gcc_checking_assert (size >= 0);
gcc_checking_assert (time >= 0);
if (DECL_DECLARED_INLINE_P (node->decl))
hints |= INLINE_HINT_declared_inline;
- estimate_calls_size_and_time (node, &size, &time, &hints, possible_truths,
- known_vals, known_binfos, known_aggs);
+ estimate_calls_size_and_time (node, &size, &min_size, &time, &hints, possible_truths,
+ known_vals, known_contexts, known_aggs);
gcc_checking_assert (size >= 0);
gcc_checking_assert (time >= 0);
time = RDIV (time, INLINE_TIME_SCALE);
size = RDIV (size, INLINE_SIZE_SCALE);
+ min_size = RDIV (min_size, INLINE_SIZE_SCALE);
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "\n size:%i time:%i\n", (int) size, (int) time);
*ret_time = time;
if (ret_size)
*ret_size = size;
+ if (ret_min_size)
+ *ret_min_size = min_size;
if (ret_hints)
*ret_hints = hints;
return;
/* Estimate size and time needed to execute callee of EDGE assuming that
parameters known to be constant at caller of EDGE are propagated.
- KNOWN_VALS and KNOWN_BINFOS are vectors of assumed known constant values
+ KNOWN_VALS and KNOWN_CONTEXTS are vectors of assumed known constant values
and types for parameters. */
void
estimate_ipcp_clone_size_and_time (struct cgraph_node *node,
vec<tree> known_vals,
- vec<tree> known_binfos,
+ vec<ipa_polymorphic_call_context>
+ known_contexts,
vec<ipa_agg_jump_function_p> known_aggs,
int *ret_size, int *ret_time,
inline_hints *hints)
clause = evaluate_conditions_for_known_args (node, false, known_vals,
known_aggs);
- estimate_node_size_and_time (node, clause, known_vals, known_binfos,
- known_aggs, ret_size, ret_time, hints, vNULL);
+ estimate_node_size_and_time (node, clause, known_vals, known_contexts,
+ known_aggs, ret_size, NULL, ret_time, hints, vNULL);
}
/* Translate all conditions from callee representation into caller
inline_update_callee_summaries (struct cgraph_node *node, int depth)
{
struct cgraph_edge *e;
- struct inline_summary *callee_info = inline_summary (node);
- struct inline_summary *caller_info = inline_summary (node->callers->caller);
+ struct inline_summary *callee_info = inline_summaries->get (node);
+ struct inline_summary *caller_info = inline_summaries->get (node->callers->caller);
HOST_WIDE_INT peak;
callee_info->stack_frame_offset
+ caller_info->estimated_self_stack_size;
peak = callee_info->stack_frame_offset
+ callee_info->estimated_self_stack_size;
- if (inline_summary (node->global.inlined_to)->estimated_stack_size < peak)
- inline_summary (node->global.inlined_to)->estimated_stack_size = peak;
+ if (inline_summaries->get (node->global.inlined_to)->estimated_stack_size < peak)
+ inline_summaries->get (node->global.inlined_to)->estimated_stack_size = peak;
ipa_propagate_frequency (node);
for (e = node->callees; e; e = e->next_callee)
{
remap_edge_change_prob (struct cgraph_edge *inlined_edge,
struct cgraph_edge *edge)
{
- if (ipa_node_params_vector.exists ())
+ if (ipa_node_params_sum)
{
int i;
struct ipa_edge_args *args = IPA_EDGE_REF (edge);
clause_t possible_truths,
struct predicate *toplev_predicate)
{
- struct cgraph_edge *e;
- for (e = node->callees; e; e = e->next_callee)
+ struct cgraph_edge *e, *next;
+ for (e = node->callees; e; e = next)
{
struct inline_edge_summary *es = inline_edge_summary (e);
struct predicate p;
+ next = e->next_callee;
if (e->inline_failed)
{
es->predicate, operand_map, offset_map,
possible_truths, toplev_predicate);
edge_set_predicate (e, &p);
- /* TODO: We should remove the edge for code that will be
- optimized out, but we need to keep verifiers and tree-inline
- happy. Make it cold for now. */
- if (false_predicate_p (&p))
- {
- e->count = 0;
- e->frequency = 0;
- }
}
else
edge_set_predicate (e, toplev_predicate);
operand_map, offset_map, possible_truths,
toplev_predicate);
}
- for (e = node->indirect_calls; e; e = e->next_callee)
+ for (e = node->indirect_calls; e; e = next)
{
struct inline_edge_summary *es = inline_edge_summary (e);
struct predicate p;
+ next = e->next_callee;
remap_edge_change_prob (inlined_edge, e);
if (es->predicate)
es->predicate, operand_map, offset_map,
possible_truths, toplev_predicate);
edge_set_predicate (e, &p);
- /* TODO: We should remove the edge for code that will be optimized
- out, but we need to keep verifiers and tree-inline happy.
- Make it cold for now. */
- if (false_predicate_p (&p))
- {
- e->count = 0;
- e->frequency = 0;
- }
}
else
edge_set_predicate (e, toplev_predicate);
void
inline_merge_summary (struct cgraph_edge *edge)
{
- struct inline_summary *callee_info = inline_summary (edge->callee);
+ struct inline_summary *callee_info = inline_summaries->get (edge->callee);
struct cgraph_node *to = (edge->caller->global.inlined_to
? edge->caller->global.inlined_to : edge->caller);
- struct inline_summary *info = inline_summary (to);
+ struct inline_summary *info = inline_summaries->get (to);
clause_t clause = 0; /* not_inline is known to be false. */
size_time_entry *e;
vec<int> operand_map = vNULL;
else
toplev_predicate = true_predicate ();
- if (ipa_node_params_vector.exists () && callee_info->conds)
+ if (callee_info->conds)
+ evaluate_properties_for_edge (edge, true, &clause, NULL, NULL, NULL);
+ if (ipa_node_params_sum && callee_info->conds)
{
struct ipa_edge_args *args = IPA_EDGE_REF (edge);
int count = ipa_get_cs_argument_count (args);
int i;
- evaluate_properties_for_edge (edge, true, &clause, NULL, NULL, NULL);
if (count)
{
operand_map.safe_grow_cleared (count);
void
inline_update_overall_summary (struct cgraph_node *node)
{
- struct inline_summary *info = inline_summary (node);
+ struct inline_summary *info = inline_summaries->get (node);
size_time_entry *e;
int i;
if (info->time > MAX_TIME * INLINE_TIME_SCALE)
info->time = MAX_TIME * INLINE_TIME_SCALE;
}
- estimate_calls_size_and_time (node, &info->size, &info->time, NULL,
+ estimate_calls_size_and_time (node, &info->size, &info->min_size,
+ &info->time, NULL,
~(clause_t) (1 << predicate_false_condition),
vNULL, vNULL, vNULL);
info->time = (info->time + INLINE_TIME_SCALE / 2) / INLINE_TIME_SCALE;
int hints = 0;
struct cgraph_node *to = (edge->caller->global.inlined_to
? edge->caller->global.inlined_to : edge->caller);
- if (inline_summary (to)->scc_no
- && inline_summary (to)->scc_no == inline_summary (edge->callee)->scc_no
- && !cgraph_edge_recursive_p (edge))
+ struct cgraph_node *callee = edge->callee->ultimate_alias_target ();
+ if (inline_summaries->get (to)->scc_no
+ && inline_summaries->get (to)->scc_no
+ == inline_summaries->get (callee)->scc_no
+ && !edge->recursive_p ())
hints |= INLINE_HINT_same_scc;
- if (to->lto_file_data && edge->callee->lto_file_data
- && to->lto_file_data != edge->callee->lto_file_data)
+ if (callee->lto_file_data && edge->caller->lto_file_data
+ && edge->caller->lto_file_data != callee->lto_file_data
+ && !callee->merged_comdat && !callee->icf_merged)
hints |= INLINE_HINT_cross_module;
return hints;
struct cgraph_node *callee;
clause_t clause;
vec<tree> known_vals;
- vec<tree> known_binfos;
+ vec<ipa_polymorphic_call_context> known_contexts;
vec<ipa_agg_jump_function_p> known_aggs;
struct inline_edge_summary *es = inline_edge_summary (edge);
+ int min_size;
- callee = cgraph_function_or_thunk_node (edge->callee, NULL);
+ callee = edge->callee->ultimate_alias_target ();
gcc_checking_assert (edge->inline_failed);
evaluate_properties_for_edge (edge, true,
- &clause, &known_vals, &known_binfos,
+ &clause, &known_vals, &known_contexts,
&known_aggs);
- estimate_node_size_and_time (callee, clause, known_vals, known_binfos,
- known_aggs, &size, &time, &hints, es->param);
+ estimate_node_size_and_time (callee, clause, known_vals, known_contexts,
+ known_aggs, &size, &min_size, &time, &hints, es->param);
+
+ /* When we have profile feedback, we can quite safely identify hot
+ edges and for those we disable size limits. Don't do that when
+ probability that caller will call the callee is low however, since it
+ may hurt optimization of the caller's hot path. */
+ if (edge->count && edge->maybe_hot_p ()
+ && (edge->count * 2
+ > (edge->caller->global.inlined_to
+ ? edge->caller->global.inlined_to->count : edge->caller->count)))
+ hints |= INLINE_HINT_known_hot;
+
known_vals.release ();
- known_binfos.release ();
+ known_contexts.release ();
known_aggs.release ();
gcc_checking_assert (size >= 0);
gcc_checking_assert (time >= 0);
/* When caching, update the cache entry. */
if (edge_growth_cache.exists ())
{
+ inline_summaries->get (edge->callee)->min_size = min_size;
if ((int) edge_growth_cache.length () <= edge->uid)
- edge_growth_cache.safe_grow_cleared (cgraph_edge_max_uid);
+ edge_growth_cache.safe_grow_cleared (symtab->edges_max_uid);
edge_growth_cache[edge->uid].time = time + (time >= 0);
edge_growth_cache[edge->uid].size = size + (size >= 0);
struct cgraph_node *callee;
clause_t clause;
vec<tree> known_vals;
- vec<tree> known_binfos;
+ vec<ipa_polymorphic_call_context> known_contexts;
vec<ipa_agg_jump_function_p> known_aggs;
/* When we do caching, use do_estimate_edge_time to populate the entry. */
return size - (size > 0);
}
- callee = cgraph_function_or_thunk_node (edge->callee, NULL);
+ callee = edge->callee->ultimate_alias_target ();
/* Early inliner runs without caching, go ahead and do the dirty work. */
gcc_checking_assert (edge->inline_failed);
evaluate_properties_for_edge (edge, true,
- &clause, &known_vals, &known_binfos,
+ &clause, &known_vals, &known_contexts,
&known_aggs);
- estimate_node_size_and_time (callee, clause, known_vals, known_binfos,
- known_aggs, &size, NULL, NULL, vNULL);
+ estimate_node_size_and_time (callee, clause, known_vals, known_contexts,
+ known_aggs, &size, NULL, NULL, NULL, vNULL);
known_vals.release ();
- known_binfos.release ();
+ known_contexts.release ();
known_aggs.release ();
return size;
}
struct cgraph_node *callee;
clause_t clause;
vec<tree> known_vals;
- vec<tree> known_binfos;
+ vec<ipa_polymorphic_call_context> known_contexts;
vec<ipa_agg_jump_function_p> known_aggs;
/* When we do caching, use do_estimate_edge_time to populate the entry. */
return hints - 1;
}
- callee = cgraph_function_or_thunk_node (edge->callee, NULL);
+ callee = edge->callee->ultimate_alias_target ();
/* Early inliner runs without caching, go ahead and do the dirty work. */
gcc_checking_assert (edge->inline_failed);
evaluate_properties_for_edge (edge, true,
- &clause, &known_vals, &known_binfos,
+ &clause, &known_vals, &known_contexts,
&known_aggs);
- estimate_node_size_and_time (callee, clause, known_vals, known_binfos,
- known_aggs, NULL, NULL, &hints, vNULL);
+ estimate_node_size_and_time (callee, clause, known_vals, known_contexts,
+ known_aggs, NULL, NULL, NULL, &hints, vNULL);
known_vals.release ();
- known_binfos.release ();
+ known_contexts.release ();
known_aggs.release ();
hints |= simple_edge_hints (edge);
return hints;
if (!es->predicate || !false_predicate_p (es->predicate))
{
gcov_type time =
- inline_summary (node)->time + estimate_edge_time (edge);
+ inline_summaries->get (node)->time + estimate_edge_time (edge);
if (time < 0)
time = 0;
if (time > MAX_TIME)
time = MAX_TIME;
return time;
}
- return inline_summary (node)->time;
+ return inline_summaries->get (node)->time;
}
struct inline_edge_summary *es = inline_edge_summary (edge);
if (!es->predicate || !false_predicate_p (es->predicate))
{
- int size = inline_summary (node)->size + estimate_edge_growth (edge);
+ int size = inline_summaries->get (node)->size + estimate_edge_growth (edge);
gcc_assert (size >= 0);
return size;
}
- return inline_summary (node)->size;
+ return inline_summaries->get (node)->size;
}
{
struct cgraph_node *node;
bool self_recursive;
+ bool uninlinable;
int growth;
};
{
gcc_checking_assert (e->inline_failed);
- if (e->caller == d->node
- || (e->caller->global.inlined_to
- && e->caller->global.inlined_to == d->node))
- d->self_recursive = true;
+ if (cgraph_inline_failed_type (e->inline_failed) == CIF_FINAL_ERROR)
+ {
+ d->uninlinable = true;
+ continue;
+ }
+
+ if (e->recursive_p ())
+ {
+ d->self_recursive = true;
+ continue;
+ }
d->growth += estimate_edge_growth (e);
}
return false;
/* Estimate the growth caused by inlining NODE into all callees. */
int
-do_estimate_growth (struct cgraph_node *node)
+estimate_growth (struct cgraph_node *node)
{
- struct growth_data d = { node, 0, false };
- struct inline_summary *info = inline_summary (node);
+ struct growth_data d = { node, false, false, 0 };
+ struct inline_summary *info = inline_summaries->get (node);
- cgraph_for_node_and_aliases (node, do_estimate_growth_1, &d, true);
+ node->call_for_symbol_and_aliases (do_estimate_growth_1, &d, true);
/* For self recursive functions the growth estimation really should be
infinity. We don't want to return very large values because the growth
return zero or negative growths. */
if (d.self_recursive)
d.growth = d.growth < info->size ? info->size : d.growth;
- else if (DECL_EXTERNAL (node->decl))
+ else if (DECL_EXTERNAL (node->decl) || d.uninlinable)
;
else
{
- if (cgraph_will_be_removed_from_program_if_no_direct_calls (node))
+ if (node->will_be_removed_from_program_if_no_direct_calls_p ())
d.growth -= info->size;
/* COMDAT functions are very often not shared across multiple units
since they come from various template instantiations.
Take this into account. */
else if (DECL_COMDAT (node->decl)
- && cgraph_can_remove_if_no_direct_calls_p (node))
+ && node->can_remove_if_no_direct_calls_p ())
d.growth -= (info->size
* (100 - PARAM_VALUE (PARAM_COMDAT_SHARING_PROBABILITY))
+ 50) / 100;
}
- if (node_growth_cache.exists ())
+ return d.growth;
+}
+
+/* Verify if there are fewer than MAX_CALLERS. */
+
+static bool
+check_callers (cgraph_node *node, int *max_callers)
+{
+ ipa_ref *ref;
+
+ if (!node->can_remove_if_no_direct_calls_and_refs_p ())
+ return true;
+
+ for (cgraph_edge *e = node->callers; e; e = e->next_caller)
{
- if ((int) node_growth_cache.length () <= node->uid)
- node_growth_cache.safe_grow_cleared (cgraph_max_uid);
- node_growth_cache[node->uid] = d.growth + (d.growth >= 0);
+ (*max_callers)--;
+ if (!*max_callers
+ || cgraph_inline_failed_type (e->inline_failed) == CIF_FINAL_ERROR)
+ return true;
}
- return d.growth;
+
+ FOR_EACH_ALIAS (node, ref)
+ if (check_callers (dyn_cast <cgraph_node *> (ref->referring), max_callers))
+ return true;
+
+ return false;
+}
+
+
+/* Make cheap estimation if growth of NODE is likely positive knowing
+ EDGE_GROWTH of one particular edge.
+ We assume that most of other edges will have similar growth
+ and skip computation if there are too many callers. */
+
+bool
+growth_likely_positive (struct cgraph_node *node,
+ int edge_growth)
+{
+ int max_callers;
+ struct cgraph_edge *e;
+ gcc_checking_assert (edge_growth > 0);
+
+ /* First quickly check if NODE is removable at all. */
+ if (DECL_EXTERNAL (node->decl))
+ return true;
+ if (!node->can_remove_if_no_direct_calls_and_refs_p ()
+ || node->address_taken)
+ return true;
+
+ max_callers = inline_summaries->get (node)->size * 4 / edge_growth + 2;
+
+ for (e = node->callers; e; e = e->next_caller)
+ {
+ max_callers--;
+ if (!max_callers
+ || cgraph_inline_failed_type (e->inline_failed) == CIF_FINAL_ERROR)
+ return true;
+ }
+
+ ipa_ref *ref;
+ FOR_EACH_ALIAS (node, ref)
+ if (check_callers (dyn_cast <cgraph_node *> (ref->referring), &max_callers))
+ return true;
+
+ /* Unlike for functions called once, we play unsafe with
+ COMDATs. We can allow that since we know functions
+ in consideration are small (and thus risk is small) and
+ moreover grow estimates already accounts that COMDAT
+ functions may or may not disappear when eliminated from
+ current unit. With good probability making aggressive
+ choice in all units is going to make overall program
+ smaller. */
+ if (DECL_COMDAT (node->decl))
+ {
+ if (!node->can_remove_if_no_direct_calls_p ())
+ return true;
+ }
+ else if (!node->will_be_removed_from_program_if_no_direct_calls_p ())
+ return true;
+
+ return estimate_growth (node) > 0;
}
/* Note function body size. */
-static void
+void
inline_analyze_function (struct cgraph_node *node)
{
push_cfun (DECL_STRUCT_FUNCTION (node->decl));
if (dump_file)
fprintf (dump_file, "\nAnalyzing function: %s/%u\n",
node->name (), node->order);
- if (optimize && !node->thunk.thunk_p)
+ if (opt_for_fn (node->decl, optimize) && !node->thunk.thunk_p)
inline_indirect_intraprocedural_analysis (node);
compute_inline_parameters (node, false);
if (!optimize)
/* Called when new function is inserted to callgraph late. */
-static void
-add_new_function (struct cgraph_node *node, void *data ATTRIBUTE_UNUSED)
+void
+inline_summary_t::insert (struct cgraph_node *node, inline_summary *)
{
inline_analyze_function (node);
}
-
/* Note function body size. */
void
{
struct cgraph_node *node;
+ FOR_EACH_DEFINED_FUNCTION (node)
+ if (DECL_STRUCT_FUNCTION (node->decl))
+ node->local.versionable = tree_versionable_function_p (node->decl);
+
/* When not optimizing, do not bother to analyze. Inlining is still done
because edge redirection needs to happen there. */
- if (!optimize && !flag_lto && !flag_wpa)
+ if (!optimize && !flag_generate_lto && !flag_generate_offload && !flag_wpa)
return;
- function_insertion_hook_holder =
- cgraph_add_function_insertion_hook (&add_new_function, NULL);
+ if (!inline_summaries)
+ inline_summaries = (inline_summary_t*) inline_summary_t::create_ggc (symtab);
+
+ inline_summaries->enable_insertion_hook ();
ipa_register_cgraph_hooks ();
inline_free_summary ();
const int main_offset = cfg_offset + header->cfg_size;
const int string_offset = main_offset + header->main_size;
struct data_in *data_in;
- struct lto_input_block ib;
unsigned int i, count2, j;
unsigned int f_count;
- LTO_INIT_INPUT_BLOCK (ib, (const char *) data + main_offset, 0,
- header->main_size);
+ lto_input_block ib ((const char *) data + main_offset, header->main_size,
+ file_data->mode_table);
data_in =
lto_data_in_create (file_data, (const char *) data + string_offset,
index = streamer_read_uhwi (&ib);
encoder = file_data->symtab_node_encoder;
- node = cgraph (lto_symtab_encoder_deref (encoder, index));
- info = inline_summary (node);
+ node = dyn_cast<cgraph_node *> (lto_symtab_encoder_deref (encoder,
+ index));
+ info = inline_summaries->get (node);
info->estimated_stack_size
= info->estimated_self_stack_size = streamer_read_uhwi (&ib);
bp = streamer_read_bitpack (&ib);
info->inlinable = bp_unpack_value (&bp, 1);
+ info->contains_cilk_spawn = bp_unpack_value (&bp, 1);
count2 = streamer_read_uhwi (&ib);
gcc_assert (!info->conds);
/* Fatal error here. We do not want to support compiling ltrans units
with different version of compiler or different flags than the WPA
unit, so this should never happen. */
- fatal_error ("ipa inline summary is missing in input file");
+ fatal_error (input_location,
+ "ipa inline summary is missing in input file");
}
if (optimize)
{
if (!flag_ipa_cp)
ipa_prop_read_jump_functions ();
}
- function_insertion_hook_holder =
- cgraph_add_function_insertion_hook (&add_new_function, NULL);
+
+ gcc_assert (inline_summaries);
+ inline_summaries->enable_insertion_hook ();
}
for (i = 0; i < lto_symtab_encoder_size (encoder); i++)
{
symtab_node *snode = lto_symtab_encoder_deref (encoder, i);
- cgraph_node *cnode = dyn_cast <cgraph_node> (snode);
+ cgraph_node *cnode = dyn_cast <cgraph_node *> (snode);
if (cnode && cnode->definition && !cnode->alias)
count++;
}
for (i = 0; i < lto_symtab_encoder_size (encoder); i++)
{
symtab_node *snode = lto_symtab_encoder_deref (encoder, i);
- cgraph_node *cnode = dyn_cast <cgraph_node> (snode);
+ cgraph_node *cnode = dyn_cast <cgraph_node *> (snode);
if (cnode && (node = cnode)->definition && !node->alias)
{
- struct inline_summary *info = inline_summary (node);
+ struct inline_summary *info = inline_summaries->get (node);
struct bitpack_d bp;
struct cgraph_edge *edge;
int i;
streamer_write_hwi (ob, info->self_time);
bp = bitpack_create (ob->main_stream);
bp_pack_value (&bp, info->inlinable, 1);
+ bp_pack_value (&bp, info->contains_cilk_spawn, 1);
streamer_write_bitpack (&bp);
streamer_write_uhwi (ob, vec_safe_length (info->conds));
for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
inline_free_summary (void)
{
struct cgraph_node *node;
- if (!inline_edge_summary_vec.exists ())
- return;
- FOR_EACH_DEFINED_FUNCTION (node)
- reset_inline_summary (node);
- if (function_insertion_hook_holder)
- cgraph_remove_function_insertion_hook (function_insertion_hook_holder);
- function_insertion_hook_holder = NULL;
- if (node_removal_hook_holder)
- cgraph_remove_node_removal_hook (node_removal_hook_holder);
- node_removal_hook_holder = NULL;
if (edge_removal_hook_holder)
- cgraph_remove_edge_removal_hook (edge_removal_hook_holder);
+ symtab->remove_edge_removal_hook (edge_removal_hook_holder);
edge_removal_hook_holder = NULL;
- if (node_duplication_hook_holder)
- cgraph_remove_node_duplication_hook (node_duplication_hook_holder);
- node_duplication_hook_holder = NULL;
if (edge_duplication_hook_holder)
- cgraph_remove_edge_duplication_hook (edge_duplication_hook_holder);
+ symtab->remove_edge_duplication_hook (edge_duplication_hook_holder);
edge_duplication_hook_holder = NULL;
- vec_free (inline_summary_vec);
+ if (!inline_edge_summary_vec.exists ())
+ return;
+ FOR_EACH_DEFINED_FUNCTION (node)
+ if (!node->alias)
+ reset_inline_summary (node, inline_summaries->get (node));
+ inline_summaries->release ();
+ inline_summaries = NULL;
inline_edge_summary_vec.release ();
- if (edge_predicate_pool)
- free_alloc_pool (edge_predicate_pool);
- edge_predicate_pool = 0;
+ edge_predicate_pool.release ();
}
projects / gcc.git / blobdiff