/* Inlining decision heuristics.
- Copyright (C) 2003-2014 Free Software Foundation, Inc.
+ Copyright (C) 2003-2016 Free Software Foundation, Inc.
Contributed by Jan Hubicka
This file is part of GCC.
#include "config.h"
#include "system.h"
#include "coretypes.h"
-#include "tm.h"
+#include "backend.h"
#include "tree.h"
-#include "stor-layout.h"
-#include "stringpool.h"
+#include "gimple.h"
+#include "alloc-pool.h"
+#include "tree-pass.h"
+#include "ssa.h"
+#include "tree-streamer.h"
+#include "cgraph.h"
+#include "diagnostic.h"
+#include "fold-const.h"
#include "print-tree.h"
#include "tree-inline.h"
-#include "langhooks.h"
-#include "flags.h"
-#include "diagnostic.h"
#include "gimple-pretty-print.h"
#include "params.h"
-#include "tree-pass.h"
-#include "coverage.h"
-#include "predict.h"
-#include "vec.h"
-#include "hashtab.h"
-#include "hash-set.h"
-#include "machmode.h"
-#include "hard-reg-set.h"
-#include "input.h"
-#include "function.h"
-#include "dominance.h"
-#include "cfg.h"
#include "cfganal.h"
-#include "basic-block.h"
-#include "tree-ssa-alias.h"
-#include "internal-fn.h"
-#include "gimple-expr.h"
-#include "is-a.h"
-#include "gimple.h"
#include "gimple-iterator.h"
-#include "gimple-ssa.h"
#include "tree-cfg.h"
-#include "tree-phinodes.h"
-#include "ssa-iterators.h"
-#include "tree-ssanames.h"
#include "tree-ssa-loop-niter.h"
#include "tree-ssa-loop.h"
+#include "symbol-summary.h"
#include "ipa-prop.h"
-#include "lto-streamer.h"
-#include "data-streamer.h"
-#include "tree-streamer.h"
#include "ipa-inline.h"
-#include "alloc-pool.h"
#include "cfgloop.h"
#include "tree-scalar-evolution.h"
#include "ipa-utils.h"
#include "cilk.h"
#include "cfgexpand.h"
+#include "gimplify.h"
/* Estimate runtime of function can easilly run into huge numbers with many
nested loops. Be sure we can compute time * INLINE_SIZE_SCALE * 2 in an
#define CHANGED IDENTIFIER_NODE
/* Holders of ipa cgraph hooks: */
-static struct cgraph_node_hook_list *function_insertion_hook_holder;
-static struct cgraph_node_hook_list *node_removal_hook_holder;
-static struct cgraph_2node_hook_list *node_duplication_hook_holder;
static struct cgraph_2edge_hook_list *edge_duplication_hook_holder;
static struct cgraph_edge_hook_list *edge_removal_hook_holder;
-static void inline_node_removal_hook (struct cgraph_node *, void *);
-static void inline_node_duplication_hook (struct cgraph_node *,
- struct cgraph_node *, void *);
static void inline_edge_removal_hook (struct cgraph_edge *, void *);
static void inline_edge_duplication_hook (struct cgraph_edge *,
struct cgraph_edge *, void *);
/* VECtor holding inline summaries.
In GGC memory because conditions might point to constant trees. */
-vec<inline_summary_t, va_gc> *inline_summary_vec;
+function_summary <inline_summary *> *inline_summaries;
vec<inline_edge_summary_t> inline_edge_summary_vec;
/* Cached node/edge growths. */
-vec<int> node_growth_cache;
vec<edge_growth_cache_entry> edge_growth_cache;
/* Edge predicates goes here. */
-static alloc_pool edge_predicate_pool;
+static object_allocator<predicate> edge_predicate_pool ("edge predicates");
/* Return true predicate (tautology).
We represent it by empty list of clauses. */
&& cc1->val == cc2->val
&& cc2->code != IS_NOT_CONSTANT
&& cc2->code != CHANGED
- && cc1->code == invert_tree_comparison
- (cc2->code,
- HONOR_NANS (TYPE_MODE (TREE_TYPE (cc1->val)))))
+ && cc1->code == invert_tree_comparison (cc2->code,
+ HONOR_NANS (cc1->val)))
return;
}
}
}
}
+/* We proved E to be unreachable, redirect it to __bultin_unreachable. */
+
+static struct cgraph_edge *
+redirect_to_unreachable (struct cgraph_edge *e)
+{
+ struct cgraph_node *callee = !e->inline_failed ? e->callee : NULL;
+ struct cgraph_node *target = cgraph_node::get_create
+ (builtin_decl_implicit (BUILT_IN_UNREACHABLE));
+
+ if (e->speculative)
+ e = e->resolve_speculation (target->decl);
+ else if (!e->callee)
+ e->make_direct (target);
+ else
+ e->redirect_callee (target);
+ struct inline_edge_summary *es = inline_edge_summary (e);
+ e->inline_failed = CIF_UNREACHABLE;
+ e->frequency = 0;
+ e->count = 0;
+ es->call_stmt_size = 0;
+ es->call_stmt_time = 0;
+ if (callee)
+ callee->remove_symbol_and_inline_clones ();
+ return e;
+}
+
/* Set predicate for edge E. */
static void
edge_set_predicate (struct cgraph_edge *e, struct predicate *predicate)
{
- struct inline_edge_summary *es = inline_edge_summary (e);
-
/* If the edge is determined to be never executed, redirect it
to BUILTIN_UNREACHABLE to save inliner from inlining into it. */
- if (predicate && false_predicate_p (predicate) && e->callee)
- {
- struct cgraph_node *callee = !e->inline_failed ? e->callee : NULL;
+ if (predicate && false_predicate_p (predicate)
+ /* When handling speculative edges, we need to do the redirection
+ just once. Do it always on the direct edge, so we do not
+ attempt to resolve speculation while duplicating the edge. */
+ && (!e->speculative || e->callee))
+ e = redirect_to_unreachable (e);
- e->redirect_callee (cgraph_node::get_create
- (builtin_decl_implicit (BUILT_IN_UNREACHABLE)));
- e->inline_failed = CIF_UNREACHABLE;
- if (callee)
- callee->remove_symbol_and_inline_clones ();
- }
+ struct inline_edge_summary *es = inline_edge_summary (e);
if (predicate && !true_predicate_p (predicate))
{
if (!es->predicate)
- es->predicate = (struct predicate *) pool_alloc (edge_predicate_pool);
+ es->predicate = edge_predicate_pool.allocate ();
*es->predicate = *predicate;
}
else
{
if (es->predicate)
- pool_free (edge_predicate_pool, es->predicate);
+ edge_predicate_pool.remove (es->predicate);
es->predicate = NULL;
}
}
if (false_predicate_p (&new_predicate) || true_predicate_p (&new_predicate))
{
if (*p)
- pool_free (edge_predicate_pool, *p);
+ edge_predicate_pool.remove (*p);
*p = NULL;
}
else
{
if (!*p)
- *p = (struct predicate *) pool_alloc (edge_predicate_pool);
+ *p = edge_predicate_pool.allocate ();
**p = new_predicate;
}
}
known_aggs)
{
clause_t clause = inline_p ? 0 : 1 << predicate_not_inlined_condition;
- struct inline_summary *info = inline_summary (node);
+ struct inline_summary *info = inline_summaries->get (node);
int i;
struct condition *c;
}
if (c->code == IS_NOT_CONSTANT || c->code == CHANGED)
continue;
- res = fold_binary_to_constant (c->code, boolean_type_node, val, c->val);
- if (res && integer_zerop (res))
- continue;
+
+ if (operand_equal_p (TYPE_SIZE (TREE_TYPE (c->val)),
+ TYPE_SIZE (TREE_TYPE (val)), 0))
+ {
+ val = fold_unary (VIEW_CONVERT_EXPR, TREE_TYPE (c->val), val);
+
+ res = val
+ ? fold_binary_to_constant (c->code, boolean_type_node, val, c->val)
+ : NULL;
+
+ if (res && integer_zerop (res))
+ continue;
+ }
clause |= 1 << (i + predicate_first_dynamic_condition);
}
return clause;
evaluate_properties_for_edge (struct cgraph_edge *e, bool inline_p,
clause_t *clause_ptr,
vec<tree> *known_vals_ptr,
- vec<tree> *known_binfos_ptr,
+ vec<ipa_polymorphic_call_context>
+ *known_contexts_ptr,
vec<ipa_agg_jump_function_p> *known_aggs_ptr)
{
struct cgraph_node *callee = e->callee->ultimate_alias_target ();
- struct inline_summary *info = inline_summary (callee);
+ struct inline_summary *info = inline_summaries->get (callee);
vec<tree> known_vals = vNULL;
vec<ipa_agg_jump_function_p> known_aggs = vNULL;
*clause_ptr = inline_p ? 0 : 1 << predicate_not_inlined_condition;
if (known_vals_ptr)
known_vals_ptr->create (0);
- if (known_binfos_ptr)
- known_binfos_ptr->create (0);
+ if (known_contexts_ptr)
+ known_contexts_ptr->create (0);
- if (ipa_node_params_vector.exists ()
+ if (ipa_node_params_sum
&& !e->call_stmt_cannot_inline_p
- && ((clause_ptr && info->conds) || known_vals_ptr || known_binfos_ptr))
+ && ((clause_ptr && info->conds) || known_vals_ptr || known_contexts_ptr))
{
struct ipa_node_params *parms_info;
struct ipa_edge_args *args = IPA_EDGE_REF (e);
known_vals.safe_grow_cleared (count);
if (count && (info->conds || known_aggs_ptr))
known_aggs.safe_grow_cleared (count);
- if (count && known_binfos_ptr)
- known_binfos_ptr->safe_grow_cleared (count);
+ if (count && known_contexts_ptr)
+ known_contexts_ptr->safe_grow_cleared (count);
for (i = 0; i < count; i++)
{
struct ipa_jump_func *jf = ipa_get_ith_jump_func (args, i);
tree cst = ipa_value_from_jfunc (parms_info, jf);
+
+ if (!cst && e->call_stmt
+ && i < (int)gimple_call_num_args (e->call_stmt))
+ {
+ cst = gimple_call_arg (e->call_stmt, i);
+ if (!is_gimple_min_invariant (cst))
+ cst = NULL;
+ }
if (cst)
{
- if (known_vals.exists () && TREE_CODE (cst) != TREE_BINFO)
+ gcc_checking_assert (TREE_CODE (cst) != TREE_BINFO);
+ if (known_vals.exists ())
known_vals[i] = cst;
- else if (known_binfos_ptr != NULL
- && TREE_CODE (cst) == TREE_BINFO)
- (*known_binfos_ptr)[i] = cst;
}
else if (inline_p && !es->param[i].change_prob)
known_vals[i] = error_mark_node;
+
+ if (known_contexts_ptr)
+ (*known_contexts_ptr)[i] = ipa_context_from_jfunc (parms_info, e,
+ i, jf);
/* TODO: When IPA-CP starts propagating and merging aggregate jump
functions, use its knowledge of the caller too, just like the
scalar case above. */
known_aggs[i] = &jf->agg;
}
}
+ else if (e->call_stmt && !e->call_stmt_cannot_inline_p
+ && ((clause_ptr && info->conds) || known_vals_ptr))
+ {
+ int i, count = (int)gimple_call_num_args (e->call_stmt);
+
+ if (count && (info->conds || known_vals_ptr))
+ known_vals.safe_grow_cleared (count);
+ for (i = 0; i < count; i++)
+ {
+ tree cst = gimple_call_arg (e->call_stmt, i);
+ if (!is_gimple_min_invariant (cst))
+ cst = NULL;
+ if (cst)
+ known_vals[i] = cst;
+ }
+ }
if (clause_ptr)
*clause_ptr = evaluate_conditions_for_known_args (callee, inline_p,
static void
inline_summary_alloc (void)
{
- if (!node_removal_hook_holder)
- node_removal_hook_holder =
- symtab->add_cgraph_removal_hook (&inline_node_removal_hook, NULL);
if (!edge_removal_hook_holder)
edge_removal_hook_holder =
symtab->add_edge_removal_hook (&inline_edge_removal_hook, NULL);
- if (!node_duplication_hook_holder)
- node_duplication_hook_holder =
- symtab->add_cgraph_duplication_hook (&inline_node_duplication_hook, NULL);
if (!edge_duplication_hook_holder)
edge_duplication_hook_holder =
symtab->add_edge_duplication_hook (&inline_edge_duplication_hook, NULL);
- if (vec_safe_length (inline_summary_vec) <= (unsigned) symtab->cgraph_max_uid)
- vec_safe_grow_cleared (inline_summary_vec, symtab->cgraph_max_uid + 1);
+ if (!inline_summaries)
+ inline_summaries = (inline_summary_t*) inline_summary_t::create_ggc (symtab);
+
if (inline_edge_summary_vec.length () <= (unsigned) symtab->edges_max_uid)
inline_edge_summary_vec.safe_grow_cleared (symtab->edges_max_uid + 1);
- if (!edge_predicate_pool)
- edge_predicate_pool = create_alloc_pool ("edge predicates",
- sizeof (struct predicate), 10);
}
/* We are called multiple time for given function; clear
es->call_stmt_size = es->call_stmt_time = 0;
if (es->predicate)
- pool_free (edge_predicate_pool, es->predicate);
+ edge_predicate_pool.remove (es->predicate);
es->predicate = NULL;
es->param.release ();
}
data from previous run so they are not cumulated. */
static void
-reset_inline_summary (struct cgraph_node *node)
+reset_inline_summary (struct cgraph_node *node,
+ inline_summary *info)
{
- struct inline_summary *info = inline_summary (node);
struct cgraph_edge *e;
info->self_size = info->self_time = 0;
info->scc_no = 0;
if (info->loop_iterations)
{
- pool_free (edge_predicate_pool, info->loop_iterations);
+ edge_predicate_pool.remove (info->loop_iterations);
info->loop_iterations = NULL;
}
if (info->loop_stride)
{
- pool_free (edge_predicate_pool, info->loop_stride);
+ edge_predicate_pool.remove (info->loop_stride);
info->loop_stride = NULL;
}
if (info->array_index)
{
- pool_free (edge_predicate_pool, info->array_index);
+ edge_predicate_pool.remove (info->array_index);
info->array_index = NULL;
}
vec_free (info->conds);
/* Hook that is called by cgraph.c when a node is removed. */
-static void
-inline_node_removal_hook (struct cgraph_node *node,
- void *data ATTRIBUTE_UNUSED)
+void
+inline_summary_t::remove (cgraph_node *node, inline_summary *info)
{
- struct inline_summary *info;
- if (vec_safe_length (inline_summary_vec) <= (unsigned) node->uid)
- return;
- info = inline_summary (node);
- reset_inline_summary (node);
- memset (info, 0, sizeof (inline_summary_t));
+ reset_inline_summary (node, info);
}
/* Remap predicate P of former function to be predicate of duplicated function.
/* Hook that is called by cgraph.c when a node is duplicated. */
-
-static void
-inline_node_duplication_hook (struct cgraph_node *src,
- struct cgraph_node *dst,
- ATTRIBUTE_UNUSED void *data)
+void
+inline_summary_t::duplicate (cgraph_node *src,
+ cgraph_node *dst,
+ inline_summary *,
+ inline_summary *info)
{
- struct inline_summary *info;
inline_summary_alloc ();
- info = inline_summary (dst);
- memcpy (info, inline_summary (src), sizeof (struct inline_summary));
+ memcpy (info, inline_summaries->get (src), sizeof (inline_summary));
/* TODO: as an optimization, we may avoid copying conditions
that are known to be false or true. */
info->conds = vec_safe_copy (info->conds);
/* When there are any replacements in the function body, see if we can figure
out that something was optimized out. */
- if (ipa_node_params_vector.exists () && dst->clone.tree_map)
+ if (ipa_node_params_sum && dst->clone.tree_map)
{
vec<size_time_entry, va_gc> *entry = info->entry;
/* Use SRC parm info since it may not be copied yet. */
size_time_entry *e;
int optimized_out_size = 0;
bool inlined_to_p = false;
- struct cgraph_edge *edge;
+ struct cgraph_edge *edge, *next;
info->entry = 0;
known_vals.safe_grow_cleared (count);
/* Remap edge predicates with the same simplification as above.
Also copy constantness arrays. */
- for (edge = dst->callees; edge; edge = edge->next_callee)
+ for (edge = dst->callees; edge; edge = next)
{
struct predicate new_predicate;
struct inline_edge_summary *es = inline_edge_summary (edge);
+ next = edge->next_callee;
if (!edge->inline_failed)
inlined_to_p = true;
info);
if (false_predicate_p (&new_predicate)
&& !false_predicate_p (es->predicate))
- {
- optimized_out_size += es->call_stmt_size * INLINE_SIZE_SCALE;
- edge->frequency = 0;
- }
+ optimized_out_size += es->call_stmt_size * INLINE_SIZE_SCALE;
edge_set_predicate (edge, &new_predicate);
}
/* Remap indirect edge predicates with the same simplificaiton as above.
Also copy constantness arrays. */
- for (edge = dst->indirect_calls; edge; edge = edge->next_callee)
+ for (edge = dst->indirect_calls; edge; edge = next)
{
struct predicate new_predicate;
struct inline_edge_summary *es = inline_edge_summary (edge);
+ next = edge->next_callee;
gcc_checking_assert (edge->inline_failed);
if (!es->predicate)
info);
if (false_predicate_p (&new_predicate)
&& !false_predicate_p (es->predicate))
- {
- optimized_out_size += es->call_stmt_size * INLINE_SIZE_SCALE;
- edge->frequency = 0;
- }
+ optimized_out_size += es->call_stmt_size * INLINE_SIZE_SCALE;
edge_set_predicate (edge, &new_predicate);
}
remap_hint_predicate_after_duplication (&info->loop_iterations,
set_hint_predicate (&info->array_index, p);
}
}
- inline_update_overall_summary (dst);
+ if (!dst->global.inlined_to)
+ inline_update_overall_summary (dst);
}
info->predicate = NULL;
edge_set_predicate (dst, srcinfo->predicate);
info->param = srcinfo->param.copy ();
+ if (!dst->indirect_unknown_callee && src->indirect_unknown_callee)
+ {
+ info->call_stmt_size -= (eni_size_weights.indirect_call_cost
+ - eni_size_weights.call_cost);
+ info->call_stmt_time -= (eni_time_weights.indirect_call_cost
+ - eni_time_weights.call_cost);
+ }
}
{
if (symtab->edges_max_uid)
edge_growth_cache.safe_grow_cleared (symtab->edges_max_uid);
- if (symtab->cgraph_max_uid)
- node_growth_cache.safe_grow_cleared (symtab->cgraph_max_uid);
}
free_growth_caches (void)
{
edge_growth_cache.release ();
- node_growth_cache.release ();
}
? "inlined" : cgraph_inline_failed_string (edge-> inline_failed),
indent, "", es->loop_depth, edge->frequency,
es->call_stmt_size, es->call_stmt_time,
- (int) inline_summary (callee)->size / INLINE_SIZE_SCALE,
- (int) inline_summary (callee)->estimated_stack_size);
+ (int) inline_summaries->get (callee)->size / INLINE_SIZE_SCALE,
+ (int) inline_summaries->get (callee)->estimated_stack_size);
if (es->predicate)
{
fprintf (f, "%*sStack frame offset %i, callee self size %i,"
" callee size %i\n",
indent + 2, "",
- (int) inline_summary (callee)->stack_frame_offset,
- (int) inline_summary (callee)->estimated_self_stack_size,
- (int) inline_summary (callee)->estimated_stack_size);
+ (int) inline_summaries->get (callee)->stack_frame_offset,
+ (int) inline_summaries->get (callee)->estimated_self_stack_size,
+ (int) inline_summaries->get (callee)->estimated_stack_size);
dump_inline_edge_summary (f, indent + 2, callee, info);
}
}
{
if (node->definition)
{
- struct inline_summary *s = inline_summary (node);
+ struct inline_summary *s = inline_summaries->get (node);
size_time_entry *e;
int i;
fprintf (f, "Inline summary for %s/%i", node->name (),
fprintf (f, " always_inline");
if (s->inlinable)
fprintf (f, " inlinable");
+ if (s->contains_cilk_spawn)
+ fprintf (f, " contains_cilk_spawn");
fprintf (f, "\n self time: %i\n", s->self_time);
fprintf (f, " global time: %i\n", s->time);
fprintf (f, " self size: %i\n", s->self_size);
parameter. */
static tree
-unmodified_parm_1 (gimple stmt, tree op)
+unmodified_parm_1 (gimple *stmt, tree op)
{
/* SSA_NAME referring to parm default def? */
if (TREE_CODE (op) == SSA_NAME
parameter. Also traverse chains of SSA register assignments. */
static tree
-unmodified_parm (gimple stmt, tree op)
+unmodified_parm (gimple *stmt, tree op)
{
tree res = unmodified_parm_1 (stmt, op);
if (res)
loaded. */
static bool
-unmodified_parm_or_parm_agg_item (struct ipa_node_params *info,
- gimple stmt, tree op, int *index_p,
+unmodified_parm_or_parm_agg_item (struct ipa_func_body_info *fbi,
+ gimple *stmt, tree op, int *index_p,
struct agg_position_info *aggpos)
{
tree res = unmodified_parm_1 (stmt, op);
gcc_checking_assert (aggpos);
if (res)
{
- *index_p = ipa_get_param_decl_index (info, res);
+ *index_p = ipa_get_param_decl_index (fbi->info, res);
if (*index_p < 0)
return false;
aggpos->agg_contents = false;
stmt = SSA_NAME_DEF_STMT (op);
op = gimple_assign_rhs1 (stmt);
if (!REFERENCE_CLASS_P (op))
- return unmodified_parm_or_parm_agg_item (info, stmt, op, index_p,
+ return unmodified_parm_or_parm_agg_item (fbi, stmt, op, index_p,
aggpos);
}
aggpos->agg_contents = true;
- return ipa_load_from_parm_agg (info, stmt, op, index_p, &aggpos->offset,
- &aggpos->by_ref);
+ return ipa_load_from_parm_agg (fbi, fbi->info->descriptors,
+ stmt, op, index_p, &aggpos->offset,
+ NULL, &aggpos->by_ref);
}
/* See if statement might disappear after inlining.
penalty wrappers. */
static int
-eliminated_by_inlining_prob (gimple stmt)
+eliminated_by_inlining_prob (gimple *stmt)
{
enum gimple_code code = gimple_code (stmt);
enum tree_code rhs_code;
and stores to return value or parameters are often free after
inlining dua to SRA and further combining.
Assume that half of statements goes away. */
- if (rhs_code == CONVERT_EXPR
- || rhs_code == NOP_EXPR
+ if (CONVERT_EXPR_CODE_P (rhs_code)
|| rhs_code == VIEW_CONVERT_EXPR
|| rhs_code == ADDR_EXPR
|| gimple_assign_rhs_class (stmt) == GIMPLE_SINGLE_RHS)
predicates to the CFG edges. */
static void
-set_cond_stmt_execution_predicate (struct ipa_node_params *info,
+set_cond_stmt_execution_predicate (struct ipa_func_body_info *fbi,
struct inline_summary *summary,
basic_block bb)
{
- gimple last;
+ gimple *last;
tree op;
int index;
struct agg_position_info aggpos;
enum tree_code code, inverted_code;
edge e;
edge_iterator ei;
- gimple set_stmt;
+ gimple *set_stmt;
tree op2;
last = last_stmt (bb);
/* TODO: handle conditionals like
var = op0 < 4;
if (var != 0). */
- if (unmodified_parm_or_parm_agg_item (info, last, op, &index, &aggpos))
+ if (unmodified_parm_or_parm_agg_item (fbi, last, op, &index, &aggpos))
{
code = gimple_cond_code (last);
- inverted_code
- = invert_tree_comparison (code,
- HONOR_NANS (TYPE_MODE (TREE_TYPE (op))));
+ inverted_code = invert_tree_comparison (code, HONOR_NANS (op));
FOR_EACH_EDGE (e, ei, bb->succs)
{
unordered one. Be sure it is not confused with NON_CONSTANT. */
if (this_code != ERROR_MARK)
{
- struct predicate p = add_condition (summary, index, &aggpos,
- this_code,
- gimple_cond_rhs (last));
- e->aux = pool_alloc (edge_predicate_pool);
+ struct predicate p = add_condition
+ (summary, index, &aggpos, this_code,
+ unshare_expr_without_location (gimple_cond_rhs (last)));
+ e->aux = edge_predicate_pool.allocate ();
*(struct predicate *) e->aux = p;
}
}
|| gimple_call_num_args (set_stmt) != 1)
return;
op2 = gimple_call_arg (set_stmt, 0);
- if (!unmodified_parm_or_parm_agg_item
- (info, set_stmt, op2, &index, &aggpos))
+ if (!unmodified_parm_or_parm_agg_item (fbi, set_stmt, op2, &index, &aggpos))
return;
FOR_EACH_EDGE (e, ei, bb->succs) if (e->flags & EDGE_FALSE_VALUE)
{
struct predicate p = add_condition (summary, index, &aggpos,
IS_NOT_CONSTANT, NULL_TREE);
- e->aux = pool_alloc (edge_predicate_pool);
+ e->aux = edge_predicate_pool.allocate ();
*(struct predicate *) e->aux = p;
}
}
predicates to the CFG edges. */
static void
-set_switch_stmt_execution_predicate (struct ipa_node_params *info,
+set_switch_stmt_execution_predicate (struct ipa_func_body_info *fbi,
struct inline_summary *summary,
basic_block bb)
{
- gimple last;
+ gimple *lastg;
tree op;
int index;
struct agg_position_info aggpos;
size_t n;
size_t case_idx;
- last = last_stmt (bb);
- if (!last || gimple_code (last) != GIMPLE_SWITCH)
+ lastg = last_stmt (bb);
+ if (!lastg || gimple_code (lastg) != GIMPLE_SWITCH)
return;
+ gswitch *last = as_a <gswitch *> (lastg);
op = gimple_switch_index (last);
- if (!unmodified_parm_or_parm_agg_item (info, last, op, &index, &aggpos))
+ if (!unmodified_parm_or_parm_agg_item (fbi, last, op, &index, &aggpos))
return;
FOR_EACH_EDGE (e, ei, bb->succs)
{
- e->aux = pool_alloc (edge_predicate_pool);
+ e->aux = edge_predicate_pool.allocate ();
*(struct predicate *) e->aux = false_predicate ();
}
n = gimple_switch_num_labels (last);
if (!min && !max)
p = true_predicate ();
else if (!max)
- p = add_condition (summary, index, &aggpos, EQ_EXPR, min);
+ p = add_condition (summary, index, &aggpos, EQ_EXPR,
+ unshare_expr_without_location (min));
else
{
struct predicate p1, p2;
- p1 = add_condition (summary, index, &aggpos, GE_EXPR, min);
- p2 = add_condition (summary, index, &aggpos, LE_EXPR, max);
+ p1 = add_condition (summary, index, &aggpos, GE_EXPR,
+ unshare_expr_without_location (min));
+ p2 = add_condition (summary, index, &aggpos, LE_EXPR,
+ unshare_expr_without_location (max));
p = and_predicates (summary->conds, &p1, &p2);
}
*(struct predicate *) e->aux
which it is executable. */
static void
-compute_bb_predicates (struct cgraph_node *node,
- struct ipa_node_params *parms_info,
+compute_bb_predicates (struct ipa_func_body_info *fbi,
+ struct cgraph_node *node,
struct inline_summary *summary)
{
struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
FOR_EACH_BB_FN (bb, my_function)
{
- set_cond_stmt_execution_predicate (parms_info, summary, bb);
- set_switch_stmt_execution_predicate (parms_info, summary, bb);
+ set_cond_stmt_execution_predicate (fbi, summary, bb);
+ set_switch_stmt_execution_predicate (fbi, summary, bb);
}
/* Entry block is always executable. */
ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux
- = pool_alloc (edge_predicate_pool);
+ = edge_predicate_pool.allocate ();
*(struct predicate *) ENTRY_BLOCK_PTR_FOR_FN (my_function)->aux
= true_predicate ();
if (!bb->aux)
{
done = false;
- bb->aux = pool_alloc (edge_predicate_pool);
+ bb->aux = edge_predicate_pool.allocate ();
*((struct predicate *) bb->aux) = p;
}
else if (!predicates_equal_p (&p, (struct predicate *) bb->aux))
a compile time constant. */
static struct predicate
-will_be_nonconstant_predicate (struct ipa_node_params *info,
+will_be_nonconstant_predicate (struct ipa_func_body_info *fbi,
struct inline_summary *summary,
- gimple stmt,
+ gimple *stmt,
vec<predicate_t> nonconstant_names)
{
struct predicate p = true_predicate ();
struct agg_position_info aggpos;
/* What statments might be optimized away
- when their arguments are constant
- TODO: also trivial builtins.
- builtin_constant_p is already handled later. */
+ when their arguments are constant. */
if (gimple_code (stmt) != GIMPLE_ASSIGN
&& gimple_code (stmt) != GIMPLE_COND
- && gimple_code (stmt) != GIMPLE_SWITCH)
+ && gimple_code (stmt) != GIMPLE_SWITCH
+ && (gimple_code (stmt) != GIMPLE_CALL
+ || !(gimple_call_flags (stmt) & ECF_CONST)))
return p;
/* Stores will stay anyway. */
tree op;
gcc_assert (gimple_assign_single_p (stmt));
op = gimple_assign_rhs1 (stmt);
- if (!unmodified_parm_or_parm_agg_item (info, stmt, op, &base_index,
+ if (!unmodified_parm_or_parm_agg_item (fbi, stmt, op, &base_index,
&aggpos))
return p;
}
{
tree parm = unmodified_parm (stmt, use);
/* For arguments we can build a condition. */
- if (parm && ipa_get_param_decl_index (info, parm) >= 0)
+ if (parm && ipa_get_param_decl_index (fbi->info, parm) >= 0)
continue;
if (TREE_CODE (use) != SSA_NAME)
return p;
tree parm = unmodified_parm (stmt, use);
int index;
- if (parm && (index = ipa_get_param_decl_index (info, parm)) >= 0)
+ if (parm && (index = ipa_get_param_decl_index (fbi->info, parm)) >= 0)
{
if (index != base_index)
p = add_condition (summary, index, NULL, CHANGED, NULL_TREE);
p = nonconstant_names[SSA_NAME_VERSION (use)];
op_non_const = or_predicates (summary->conds, &p, &op_non_const);
}
- if (gimple_code (stmt) == GIMPLE_ASSIGN
- && TREE_CODE (gimple_assign_lhs (stmt)) == SSA_NAME)
- nonconstant_names[SSA_NAME_VERSION (gimple_assign_lhs (stmt))]
+ if ((gimple_code (stmt) == GIMPLE_ASSIGN || gimple_code (stmt) == GIMPLE_CALL)
+ && gimple_op (stmt, 0)
+ && TREE_CODE (gimple_op (stmt, 0)) == SSA_NAME)
+ nonconstant_names[SSA_NAME_VERSION (gimple_op (stmt, 0))]
= op_non_const;
return op_non_const;
}
struct record_modified_bb_info
{
bitmap bb_set;
- gimple stmt;
+ gimple *stmt;
};
/* Callback of walk_aliased_vdefs. Records basic blocks where the value may be
ought to be REG_BR_PROB_BASE / estimated_iters. */
static int
-param_change_prob (gimple stmt, int i)
+param_change_prob (gimple *stmt, int i)
{
tree op = gimple_call_arg (stmt, i);
basic_block bb = gimple_bb (stmt);
static bool
phi_result_unknown_predicate (struct ipa_node_params *info,
- struct inline_summary *summary, basic_block bb,
+ inline_summary *summary, basic_block bb,
struct predicate *p,
vec<predicate_t> nonconstant_names)
{
edge e;
edge_iterator ei;
basic_block first_bb = NULL;
- gimple stmt;
+ gimple *stmt;
if (single_pred_p (bb))
{
NONCONSTANT_NAMES, if possible. */
static void
-predicate_for_phi_result (struct inline_summary *summary, gimple phi,
+predicate_for_phi_result (struct inline_summary *summary, gphi *phi,
struct predicate *p,
vec<predicate_t> nonconstant_names)
{
/* Return predicate specifying when array index in access OP becomes non-constant. */
static struct predicate
-array_index_predicate (struct inline_summary *info,
+array_index_predicate (inline_summary *info,
vec< predicate_t> nonconstant_names, tree op)
{
struct predicate p = false_predicate ();
an impact on the earlier inlining.
Here find this pattern and fix it up later. */
-static gimple
+static gimple *
find_foldable_builtin_expect (basic_block bb)
{
gimple_stmt_iterator bsi;
for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
{
- gimple stmt = gsi_stmt (bsi);
+ gimple *stmt = gsi_stmt (bsi);
if (gimple_call_builtin_p (stmt, BUILT_IN_EXPECT)
|| (is_gimple_call (stmt)
&& gimple_call_internal_p (stmt)
tree var = gimple_call_lhs (stmt);
tree arg = gimple_call_arg (stmt, 0);
use_operand_p use_p;
- gimple use_stmt;
+ gimple *use_stmt;
bool match = false;
bool done = false;
while (TREE_CODE (arg) == SSA_NAME)
{
- gimple stmt_tmp = SSA_NAME_DEF_STMT (arg);
+ gimple *stmt_tmp = SSA_NAME_DEF_STMT (arg);
if (!is_gimple_assign (stmt_tmp))
break;
switch (gimple_assign_rhs_code (stmt_tmp))
match = true;
done = true;
break;
- case NOP_EXPR:
+ CASE_CONVERT:
break;
default:
done = true;
for (; !gsi_end_p (gsi); gsi_prev (&gsi))
{
- gimple stmt = gsi_stmt (gsi);
+ gimple *stmt = gsi_stmt (gsi);
if (is_gimple_debug (stmt))
continue;
if (gimple_clobber_p (stmt))
/* Benefits are scaled by probability of elimination that is in range
<0,2>. */
basic_block bb;
- gimple_stmt_iterator bsi;
struct function *my_function = DECL_STRUCT_FUNCTION (node->decl);
int freq;
- struct inline_summary *info = inline_summary (node);
+ struct inline_summary *info = inline_summaries->get (node);
struct predicate bb_predicate;
- struct ipa_node_params *parms_info = NULL;
+ struct ipa_func_body_info fbi;
vec<predicate_t> nonconstant_names = vNULL;
int nblocks, n;
int *order;
predicate array_index = true_predicate ();
- gimple fix_builtin_expect_stmt;
+ gimple *fix_builtin_expect_stmt;
+
+ gcc_assert (my_function && my_function->cfg);
+ gcc_assert (cfun == my_function);
+ memset(&fbi, 0, sizeof(fbi));
info->conds = NULL;
info->entry = NULL;
- if (optimize && !early)
+ /* When optimizing and analyzing for IPA inliner, initialize loop optimizer
+ so we can produce proper inline hints.
+
+ When optimizing and analyzing for early inliner, initialize node params
+ so we can produce correct BB predicates. */
+
+ if (opt_for_fn (node->decl, optimize))
{
calculate_dominance_info (CDI_DOMINATORS);
- loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS);
+ if (!early)
+ loop_optimizer_init (LOOPS_NORMAL | LOOPS_HAVE_RECORDED_EXITS);
+ else
+ {
+ ipa_check_create_node_params ();
+ ipa_initialize_node_params (node);
+ }
- if (ipa_node_params_vector.exists ())
+ if (ipa_node_params_sum)
{
- parms_info = IPA_NODE_REF (node);
+ fbi.node = node;
+ fbi.info = IPA_NODE_REF (node);
+ fbi.bb_infos = vNULL;
+ fbi.bb_infos.safe_grow_cleared (last_basic_block_for_fn (cfun));
+ fbi.param_count = count_formal_params(node->decl);
nonconstant_names.safe_grow_cleared
(SSANAMES (my_function)->length ());
}
bb_predicate = not_inlined_predicate ();
account_size_time (info, 2 * INLINE_SIZE_SCALE, 0, &bb_predicate);
- gcc_assert (my_function && my_function->cfg);
- if (parms_info)
- compute_bb_predicates (node, parms_info, info);
- gcc_assert (cfun == my_function);
+ if (fbi.info)
+ compute_bb_predicates (&fbi, node, info);
order = XNEWVEC (int, n_basic_blocks_for_fn (cfun));
nblocks = pre_and_rev_post_order_compute (NULL, order, false);
for (n = 0; n < nblocks; n++)
}
/* TODO: Obviously predicates can be propagated down across CFG. */
- if (parms_info)
+ if (fbi.info)
{
if (bb->aux)
bb_predicate = *(struct predicate *) bb->aux;
dump_predicate (dump_file, info->conds, &bb_predicate);
}
- if (parms_info && nonconstant_names.exists ())
+ if (fbi.info && nonconstant_names.exists ())
{
struct predicate phi_predicate;
bool first_phi = true;
- for (bsi = gsi_start_phis (bb); !gsi_end_p (bsi); gsi_next (&bsi))
+ for (gphi_iterator bsi = gsi_start_phis (bb); !gsi_end_p (bsi);
+ gsi_next (&bsi))
{
if (first_phi
- && !phi_result_unknown_predicate (parms_info, info, bb,
+ && !phi_result_unknown_predicate (fbi.info, info, bb,
&phi_predicate,
nonconstant_names))
break;
fprintf (dump_file, " ");
print_gimple_stmt (dump_file, gsi_stmt (bsi), 0, 0);
}
- predicate_for_phi_result (info, gsi_stmt (bsi), &phi_predicate,
+ predicate_for_phi_result (info, bsi.phi (), &phi_predicate,
nonconstant_names);
}
}
fix_builtin_expect_stmt = find_foldable_builtin_expect (bb);
- for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
+ for (gimple_stmt_iterator bsi = gsi_start_bb (bb); !gsi_end_p (bsi);
+ gsi_next (&bsi))
{
- gimple stmt = gsi_stmt (bsi);
+ gimple *stmt = gsi_stmt (bsi);
int this_size = estimate_num_insns (stmt, &eni_size_weights);
int this_time = estimate_num_insns (stmt, &eni_time_weights);
int prob;
nonconstant_names[SSA_NAME_VERSION (gimple_call_lhs (stmt))]
= false_p;
}
- if (ipa_node_params_vector.exists ())
+ if (ipa_node_params_sum)
{
int count = gimple_call_num_args (stmt);
int i;
/* TODO: When conditional jump or swithc is known to be constant, but
we did not translate it into the predicates, we really can account
just maximum of the possible paths. */
- if (parms_info)
+ if (fbi.info)
will_be_nonconstant
- = will_be_nonconstant_predicate (parms_info, info,
+ = will_be_nonconstant_predicate (&fbi, info,
stmt, nonconstant_names);
if (this_time || this_size)
{
if (prob == 2 && dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "\t\tWill be eliminated by inlining\n");
- if (parms_info)
+ if (fbi.info)
p = and_predicates (info->conds, &bb_predicate,
&will_be_nonconstant);
else
p = true_predicate ();
- if (!false_predicate_p (&p))
+ if (!false_predicate_p (&p)
+ || (is_gimple_call (stmt)
+ && !false_predicate_p (&bb_predicate)))
{
time += this_time;
size += this_size;
}
}
}
- set_hint_predicate (&inline_summary (node)->array_index, array_index);
+ set_hint_predicate (&inline_summaries->get (node)->array_index, array_index);
time = (time + CGRAPH_FREQ_BASE / 2) / CGRAPH_FREQ_BASE;
if (time > MAX_TIME)
time = MAX_TIME;
free (order);
- if (!early && nonconstant_names.exists ())
+ if (nonconstant_names.exists () && !early)
{
struct loop *loop;
predicate loop_iterations = true_predicate ();
{
vec<edge> exits;
edge ex;
- unsigned int j, i;
+ unsigned int j;
struct tree_niter_desc niter_desc;
- basic_block *body = get_loop_body (loop);
bb_predicate = *(struct predicate *) loop->header->aux;
exits = get_loop_exit_edges (loop);
&& !is_gimple_min_invariant (niter_desc.niter))
{
predicate will_be_nonconstant
- = will_be_nonconstant_expr_predicate (parms_info, info,
+ = will_be_nonconstant_expr_predicate (fbi.info, info,
niter_desc.niter,
nonconstant_names);
if (!true_predicate_p (&will_be_nonconstant))
&will_be_nonconstant);
}
exits.release ();
+ }
- for (i = 0; i < loop->num_nodes; i++)
+ /* To avoid quadratic behavior we analyze stride predicates only
+ with respect to the containing loop. Thus we simply iterate
+ over all defs in the outermost loop body. */
+ for (loop = loops_for_fn (cfun)->tree_root->inner;
+ loop != NULL; loop = loop->next)
+ {
+ basic_block *body = get_loop_body (loop);
+ for (unsigned i = 0; i < loop->num_nodes; i++)
{
gimple_stmt_iterator gsi;
bb_predicate = *(struct predicate *) body[i]->aux;
for (gsi = gsi_start_bb (body[i]); !gsi_end_p (gsi);
gsi_next (&gsi))
{
- gimple stmt = gsi_stmt (gsi);
- affine_iv iv;
- ssa_op_iter iter;
- tree use;
+ gimple *stmt = gsi_stmt (gsi);
- FOR_EACH_SSA_TREE_OPERAND (use, stmt, iter, SSA_OP_USE)
- {
- predicate will_be_nonconstant;
+ if (!is_gimple_assign (stmt))
+ continue;
- if (!simple_iv
- (loop, loop_containing_stmt (stmt), use, &iv, true)
- || is_gimple_min_invariant (iv.step))
- continue;
+ tree def = gimple_assign_lhs (stmt);
+ if (TREE_CODE (def) != SSA_NAME)
+ continue;
+
+ affine_iv iv;
+ if (!simple_iv (loop_containing_stmt (stmt),
+ loop_containing_stmt (stmt),
+ def, &iv, true)
+ || is_gimple_min_invariant (iv.step))
+ continue;
+
+ predicate will_be_nonconstant
+ = will_be_nonconstant_expr_predicate (fbi.info, info,
+ iv.step,
+ nonconstant_names);
+ if (!true_predicate_p (&will_be_nonconstant))
will_be_nonconstant
- = will_be_nonconstant_expr_predicate (parms_info, info,
- iv.step,
- nonconstant_names);
- if (!true_predicate_p (&will_be_nonconstant))
- will_be_nonconstant
- = and_predicates (info->conds,
- &bb_predicate,
- &will_be_nonconstant);
- if (!true_predicate_p (&will_be_nonconstant)
- && !false_predicate_p (&will_be_nonconstant))
- /* This is slightly inprecise. We may want to represent
- each loop with independent predicate. */
- loop_stride =
- and_predicates (info->conds, &loop_stride,
+ = and_predicates (info->conds, &bb_predicate,
&will_be_nonconstant);
- }
+ if (!true_predicate_p (&will_be_nonconstant)
+ && !false_predicate_p (&will_be_nonconstant))
+ /* This is slightly inprecise. We may want to represent
+ each loop with independent predicate. */
+ loop_stride = and_predicates (info->conds, &loop_stride,
+ &will_be_nonconstant);
}
}
free (body);
}
- set_hint_predicate (&inline_summary (node)->loop_iterations,
+ set_hint_predicate (&inline_summaries->get (node)->loop_iterations,
loop_iterations);
- set_hint_predicate (&inline_summary (node)->loop_stride, loop_stride);
+ set_hint_predicate (&inline_summaries->get (node)->loop_stride,
+ loop_stride);
scev_finalize ();
}
FOR_ALL_BB_FN (bb, my_function)
edge_iterator ei;
if (bb->aux)
- pool_free (edge_predicate_pool, bb->aux);
+ edge_predicate_pool.remove ((predicate *)bb->aux);
bb->aux = NULL;
FOR_EACH_EDGE (e, ei, bb->succs)
{
if (e->aux)
- pool_free (edge_predicate_pool, e->aux);
+ edge_predicate_pool.remove ((predicate *) e->aux);
e->aux = NULL;
}
}
- inline_summary (node)->self_time = time;
- inline_summary (node)->self_size = size;
+ inline_summaries->get (node)->self_time = time;
+ inline_summaries->get (node)->self_size = size;
nonconstant_names.release ();
- if (optimize && !early)
+ ipa_release_body_info (&fbi);
+ if (opt_for_fn (node->decl, optimize))
{
- loop_optimizer_finalize ();
+ if (!early)
+ loop_optimizer_finalize ();
+ else if (!ipa_edge_args_vector)
+ ipa_free_all_node_params ();
free_dominance_info (CDI_DOMINATORS);
}
if (dump_file)
inline_summary_alloc ();
- info = inline_summary (node);
- reset_inline_summary (node);
+ info = inline_summaries->get (node);
+ reset_inline_summary (node, info);
/* FIXME: Thunks are inlinable, but tree-inline don't know how to do that.
Once this happen, we will need to more curefully predict call
info->stack_frame_offset = 0;
/* Can this function be inlined at all? */
- if (!optimize && !lookup_attribute ("always_inline",
- DECL_ATTRIBUTES (node->decl)))
+ if (!opt_for_fn (node->decl, optimize)
+ && !lookup_attribute ("always_inline",
+ DECL_ATTRIBUTES (node->decl)))
info->inlinable = false;
else
info->inlinable = tree_inlinable_function_p (node->decl);
+ info->contains_cilk_spawn = fn_contains_cilk_spawn_p (cfun);
+
/* Type attributes can use parameter indices to describe them. */
if (TYPE_ATTRIBUTES (TREE_TYPE (node->decl)))
node->local.can_change_signature = false;
info->size = info->self_size;
info->stack_frame_offset = 0;
info->estimated_stack_size = info->estimated_self_stack_size;
-#ifdef ENABLE_CHECKING
- inline_update_overall_summary (node);
- gcc_assert (info->time == info->self_time && info->size == info->self_size);
-#endif
+ if (flag_checking)
+ {
+ inline_update_overall_summary (node);
+ gcc_assert (info->time == info->self_time
+ && info->size == info->self_size);
+ }
pop_cfun ();
}
}
-/* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS and
- KNOWN_BINFOS. */
+/* Estimate benefit devirtualizing indirect edge IE, provided KNOWN_VALS,
+ KNOWN_CONTEXTS and KNOWN_AGGS. */
static bool
estimate_edge_devirt_benefit (struct cgraph_edge *ie,
int *size, int *time,
vec<tree> known_vals,
- vec<tree> known_binfos,
+ vec<ipa_polymorphic_call_context> known_contexts,
vec<ipa_agg_jump_function_p> known_aggs)
{
tree target;
struct cgraph_node *callee;
struct inline_summary *isummary;
enum availability avail;
+ bool speculative;
- if (!known_vals.exists () && !known_binfos.exists ())
+ if (!known_vals.exists () && !known_contexts.exists ())
return false;
- if (!flag_indirect_inlining)
+ if (!opt_for_fn (ie->caller->decl, flag_indirect_inlining))
return false;
- target = ipa_get_indirect_edge_target (ie, known_vals, known_binfos,
- known_aggs);
- if (!target)
+ target = ipa_get_indirect_edge_target (ie, known_vals, known_contexts,
+ known_aggs, &speculative);
+ if (!target || speculative)
return false;
/* Account for difference in cost between indirect and direct calls. */
callee = callee->function_symbol (&avail);
if (avail < AVAIL_AVAILABLE)
return false;
- isummary = inline_summary (callee);
+ isummary = inline_summaries->get (callee);
return isummary->inlinable;
}
/* Increase SIZE, MIN_SIZE (if non-NULL) and TIME for size and time needed to
handle edge E with probability PROB.
Set HINTS if edge may be devirtualized.
- KNOWN_VALS, KNOWN_AGGS and KNOWN_BINFOS describe context of the call
+ KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS describe context of the call
site. */
static inline void
int *time,
int prob,
vec<tree> known_vals,
- vec<tree> known_binfos,
+ vec<ipa_polymorphic_call_context> known_contexts,
vec<ipa_agg_jump_function_p> known_aggs,
inline_hints *hints)
{
int cur_size;
if (!e->callee
&& estimate_edge_devirt_benefit (e, &call_size, &call_time,
- known_vals, known_binfos, known_aggs)
+ known_vals, known_contexts, known_aggs)
&& hints && e->maybe_hot_p ())
*hints |= INLINE_HINT_indirect_call;
cur_size = call_size * INLINE_SIZE_SCALE;
/* Increase SIZE, MIN_SIZE and TIME for size and time needed to handle all
- calls in NODE.
- POSSIBLE_TRUTHS, KNOWN_VALS, KNOWN_AGGS and KNOWN_BINFOS describe context of
- the call site. */
+ calls in NODE. POSSIBLE_TRUTHS, KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS
+ describe context of the call site. */
static void
estimate_calls_size_and_time (struct cgraph_node *node, int *size,
inline_hints *hints,
clause_t possible_truths,
vec<tree> known_vals,
- vec<tree> known_binfos,
+ vec<ipa_polymorphic_call_context> known_contexts,
vec<ipa_agg_jump_function_p> known_aggs)
{
struct cgraph_edge *e;
for (e = node->callees; e; e = e->next_callee)
{
+ if (inline_edge_summary_vec.length () <= (unsigned) e->uid)
+ continue;
+
struct inline_edge_summary *es = inline_edge_summary (e);
+
+ /* Do not care about zero sized builtins. */
+ if (e->inline_failed && !es->call_stmt_size)
+ {
+ gcc_checking_assert (!es->call_stmt_time);
+ continue;
+ }
if (!es->predicate
|| evaluate_predicate (es->predicate, possible_truths))
{
estimate_edge_size_and_time (e, size,
es->predicate ? NULL : min_size,
time, REG_BR_PROB_BASE,
- known_vals, known_binfos,
+ known_vals, known_contexts,
known_aggs, hints);
}
else
estimate_calls_size_and_time (e->callee, size, min_size, time,
hints,
possible_truths,
- known_vals, known_binfos,
+ known_vals, known_contexts,
known_aggs);
}
}
for (e = node->indirect_calls; e; e = e->next_callee)
{
+ if (inline_edge_summary_vec.length () <= (unsigned) e->uid)
+ continue;
+
struct inline_edge_summary *es = inline_edge_summary (e);
if (!es->predicate
|| evaluate_predicate (es->predicate, possible_truths))
estimate_edge_size_and_time (e, size,
es->predicate ? NULL : min_size,
time, REG_BR_PROB_BASE,
- known_vals, known_binfos, known_aggs,
+ known_vals, known_contexts, known_aggs,
hints);
}
}
/* Estimate size and time needed to execute NODE assuming
- POSSIBLE_TRUTHS clause, and KNOWN_VALS, KNOWN_AGGS and KNOWN_BINFOS
+ POSSIBLE_TRUTHS clause, and KNOWN_VALS, KNOWN_AGGS and KNOWN_CONTEXTS
information about NODE's arguments. If non-NULL use also probability
information present in INLINE_PARAM_SUMMARY vector.
Additionally detemine hints determined by the context. Finally compute
estimate_node_size_and_time (struct cgraph_node *node,
clause_t possible_truths,
vec<tree> known_vals,
- vec<tree> known_binfos,
+ vec<ipa_polymorphic_call_context> known_contexts,
vec<ipa_agg_jump_function_p> known_aggs,
int *ret_size, int *ret_min_size, int *ret_time,
inline_hints *ret_hints,
vec<inline_param_summary>
inline_param_summary)
{
- struct inline_summary *info = inline_summary (node);
+ struct inline_summary *info = inline_summaries->get (node);
size_time_entry *e;
int size = 0;
int time = 0;
hints |= INLINE_HINT_declared_inline;
estimate_calls_size_and_time (node, &size, &min_size, &time, &hints, possible_truths,
- known_vals, known_binfos, known_aggs);
+ known_vals, known_contexts, known_aggs);
gcc_checking_assert (size >= 0);
gcc_checking_assert (time >= 0);
time = RDIV (time, INLINE_TIME_SCALE);
/* Estimate size and time needed to execute callee of EDGE assuming that
parameters known to be constant at caller of EDGE are propagated.
- KNOWN_VALS and KNOWN_BINFOS are vectors of assumed known constant values
+ KNOWN_VALS and KNOWN_CONTEXTS are vectors of assumed known constant values
and types for parameters. */
void
estimate_ipcp_clone_size_and_time (struct cgraph_node *node,
vec<tree> known_vals,
- vec<tree> known_binfos,
+ vec<ipa_polymorphic_call_context>
+ known_contexts,
vec<ipa_agg_jump_function_p> known_aggs,
int *ret_size, int *ret_time,
inline_hints *hints)
clause = evaluate_conditions_for_known_args (node, false, known_vals,
known_aggs);
- estimate_node_size_and_time (node, clause, known_vals, known_binfos,
+ estimate_node_size_and_time (node, clause, known_vals, known_contexts,
known_aggs, ret_size, NULL, ret_time, hints, vNULL);
}
inline_update_callee_summaries (struct cgraph_node *node, int depth)
{
struct cgraph_edge *e;
- struct inline_summary *callee_info = inline_summary (node);
- struct inline_summary *caller_info = inline_summary (node->callers->caller);
+ struct inline_summary *callee_info = inline_summaries->get (node);
+ struct inline_summary *caller_info = inline_summaries->get (node->callers->caller);
HOST_WIDE_INT peak;
callee_info->stack_frame_offset
+ caller_info->estimated_self_stack_size;
peak = callee_info->stack_frame_offset
+ callee_info->estimated_self_stack_size;
- if (inline_summary (node->global.inlined_to)->estimated_stack_size < peak)
- inline_summary (node->global.inlined_to)->estimated_stack_size = peak;
+ if (inline_summaries->get (node->global.inlined_to)->estimated_stack_size < peak)
+ inline_summaries->get (node->global.inlined_to)->estimated_stack_size = peak;
ipa_propagate_frequency (node);
for (e = node->callees; e; e = e->next_callee)
{
remap_edge_change_prob (struct cgraph_edge *inlined_edge,
struct cgraph_edge *edge)
{
- if (ipa_node_params_vector.exists ())
+ if (ipa_node_params_sum)
{
int i;
struct ipa_edge_args *args = IPA_EDGE_REF (edge);
clause_t possible_truths,
struct predicate *toplev_predicate)
{
- struct cgraph_edge *e;
- for (e = node->callees; e; e = e->next_callee)
+ struct cgraph_edge *e, *next;
+ for (e = node->callees; e; e = next)
{
struct inline_edge_summary *es = inline_edge_summary (e);
struct predicate p;
+ next = e->next_callee;
if (e->inline_failed)
{
es->predicate, operand_map, offset_map,
possible_truths, toplev_predicate);
edge_set_predicate (e, &p);
- /* TODO: We should remove the edge for code that will be
- optimized out, but we need to keep verifiers and tree-inline
- happy. Make it cold for now. */
- if (false_predicate_p (&p))
- {
- e->count = 0;
- e->frequency = 0;
- }
}
else
edge_set_predicate (e, toplev_predicate);
operand_map, offset_map, possible_truths,
toplev_predicate);
}
- for (e = node->indirect_calls; e; e = e->next_callee)
+ for (e = node->indirect_calls; e; e = next)
{
struct inline_edge_summary *es = inline_edge_summary (e);
struct predicate p;
+ next = e->next_callee;
remap_edge_change_prob (inlined_edge, e);
if (es->predicate)
es->predicate, operand_map, offset_map,
possible_truths, toplev_predicate);
edge_set_predicate (e, &p);
- /* TODO: We should remove the edge for code that will be optimized
- out, but we need to keep verifiers and tree-inline happy.
- Make it cold for now. */
- if (false_predicate_p (&p))
- {
- e->count = 0;
- e->frequency = 0;
- }
}
else
edge_set_predicate (e, toplev_predicate);
void
inline_merge_summary (struct cgraph_edge *edge)
{
- struct inline_summary *callee_info = inline_summary (edge->callee);
+ struct inline_summary *callee_info = inline_summaries->get (edge->callee);
struct cgraph_node *to = (edge->caller->global.inlined_to
? edge->caller->global.inlined_to : edge->caller);
- struct inline_summary *info = inline_summary (to);
+ struct inline_summary *info = inline_summaries->get (to);
clause_t clause = 0; /* not_inline is known to be false. */
size_time_entry *e;
vec<int> operand_map = vNULL;
else
toplev_predicate = true_predicate ();
- if (ipa_node_params_vector.exists () && callee_info->conds)
+ if (callee_info->conds)
+ evaluate_properties_for_edge (edge, true, &clause, NULL, NULL, NULL);
+ if (ipa_node_params_sum && callee_info->conds)
{
struct ipa_edge_args *args = IPA_EDGE_REF (edge);
int count = ipa_get_cs_argument_count (args);
int i;
- evaluate_properties_for_edge (edge, true, &clause, NULL, NULL, NULL);
if (count)
{
operand_map.safe_grow_cleared (count);
void
inline_update_overall_summary (struct cgraph_node *node)
{
- struct inline_summary *info = inline_summary (node);
+ struct inline_summary *info = inline_summaries->get (node);
size_time_entry *e;
int i;
int hints = 0;
struct cgraph_node *to = (edge->caller->global.inlined_to
? edge->caller->global.inlined_to : edge->caller);
- if (inline_summary (to)->scc_no
- && inline_summary (to)->scc_no == inline_summary (edge->callee)->scc_no
+ struct cgraph_node *callee = edge->callee->ultimate_alias_target ();
+ if (inline_summaries->get (to)->scc_no
+ && inline_summaries->get (to)->scc_no
+ == inline_summaries->get (callee)->scc_no
&& !edge->recursive_p ())
hints |= INLINE_HINT_same_scc;
- if (to->lto_file_data && edge->callee->lto_file_data
- && to->lto_file_data != edge->callee->lto_file_data)
+ if (callee->lto_file_data && edge->caller->lto_file_data
+ && edge->caller->lto_file_data != callee->lto_file_data
+ && !callee->merged_comdat && !callee->icf_merged)
hints |= INLINE_HINT_cross_module;
return hints;
struct cgraph_node *callee;
clause_t clause;
vec<tree> known_vals;
- vec<tree> known_binfos;
+ vec<ipa_polymorphic_call_context> known_contexts;
vec<ipa_agg_jump_function_p> known_aggs;
struct inline_edge_summary *es = inline_edge_summary (edge);
int min_size;
gcc_checking_assert (edge->inline_failed);
evaluate_properties_for_edge (edge, true,
- &clause, &known_vals, &known_binfos,
+ &clause, &known_vals, &known_contexts,
&known_aggs);
- estimate_node_size_and_time (callee, clause, known_vals, known_binfos,
+ estimate_node_size_and_time (callee, clause, known_vals, known_contexts,
known_aggs, &size, &min_size, &time, &hints, es->param);
/* When we have profile feedback, we can quite safely identify hot
hints |= INLINE_HINT_known_hot;
known_vals.release ();
- known_binfos.release ();
+ known_contexts.release ();
known_aggs.release ();
gcc_checking_assert (size >= 0);
gcc_checking_assert (time >= 0);
/* When caching, update the cache entry. */
if (edge_growth_cache.exists ())
{
- inline_summary (edge->callee)->min_size = min_size;
+ inline_summaries->get (edge->callee)->min_size = min_size;
if ((int) edge_growth_cache.length () <= edge->uid)
edge_growth_cache.safe_grow_cleared (symtab->edges_max_uid);
edge_growth_cache[edge->uid].time = time + (time >= 0);
struct cgraph_node *callee;
clause_t clause;
vec<tree> known_vals;
- vec<tree> known_binfos;
+ vec<ipa_polymorphic_call_context> known_contexts;
vec<ipa_agg_jump_function_p> known_aggs;
/* When we do caching, use do_estimate_edge_time to populate the entry. */
/* Early inliner runs without caching, go ahead and do the dirty work. */
gcc_checking_assert (edge->inline_failed);
evaluate_properties_for_edge (edge, true,
- &clause, &known_vals, &known_binfos,
+ &clause, &known_vals, &known_contexts,
&known_aggs);
- estimate_node_size_and_time (callee, clause, known_vals, known_binfos,
+ estimate_node_size_and_time (callee, clause, known_vals, known_contexts,
known_aggs, &size, NULL, NULL, NULL, vNULL);
known_vals.release ();
- known_binfos.release ();
+ known_contexts.release ();
known_aggs.release ();
return size;
}
struct cgraph_node *callee;
clause_t clause;
vec<tree> known_vals;
- vec<tree> known_binfos;
+ vec<ipa_polymorphic_call_context> known_contexts;
vec<ipa_agg_jump_function_p> known_aggs;
/* When we do caching, use do_estimate_edge_time to populate the entry. */
/* Early inliner runs without caching, go ahead and do the dirty work. */
gcc_checking_assert (edge->inline_failed);
evaluate_properties_for_edge (edge, true,
- &clause, &known_vals, &known_binfos,
+ &clause, &known_vals, &known_contexts,
&known_aggs);
- estimate_node_size_and_time (callee, clause, known_vals, known_binfos,
+ estimate_node_size_and_time (callee, clause, known_vals, known_contexts,
known_aggs, NULL, NULL, NULL, &hints, vNULL);
known_vals.release ();
- known_binfos.release ();
+ known_contexts.release ();
known_aggs.release ();
hints |= simple_edge_hints (edge);
return hints;
if (!es->predicate || !false_predicate_p (es->predicate))
{
gcov_type time =
- inline_summary (node)->time + estimate_edge_time (edge);
+ inline_summaries->get (node)->time + estimate_edge_time (edge);
if (time < 0)
time = 0;
if (time > MAX_TIME)
time = MAX_TIME;
return time;
}
- return inline_summary (node)->time;
+ return inline_summaries->get (node)->time;
}
struct inline_edge_summary *es = inline_edge_summary (edge);
if (!es->predicate || !false_predicate_p (es->predicate))
{
- int size = inline_summary (node)->size + estimate_edge_growth (edge);
+ int size = inline_summaries->get (node)->size + estimate_edge_growth (edge);
gcc_assert (size >= 0);
return size;
}
- return inline_summary (node)->size;
+ return inline_summaries->get (node)->size;
}
{
struct cgraph_node *node;
bool self_recursive;
+ bool uninlinable;
int growth;
};
{
gcc_checking_assert (e->inline_failed);
- if (e->caller == d->node
- || (e->caller->global.inlined_to
- && e->caller->global.inlined_to == d->node))
- d->self_recursive = true;
+ if (cgraph_inline_failed_type (e->inline_failed) == CIF_FINAL_ERROR)
+ {
+ d->uninlinable = true;
+ continue;
+ }
+
+ if (e->recursive_p ())
+ {
+ d->self_recursive = true;
+ continue;
+ }
d->growth += estimate_edge_growth (e);
}
return false;
/* Estimate the growth caused by inlining NODE into all callees. */
int
-do_estimate_growth (struct cgraph_node *node)
+estimate_growth (struct cgraph_node *node)
{
- struct growth_data d = { node, 0, false };
- struct inline_summary *info = inline_summary (node);
+ struct growth_data d = { node, false, false, 0 };
+ struct inline_summary *info = inline_summaries->get (node);
- node->call_for_symbol_thunks_and_aliases (do_estimate_growth_1, &d, true);
+ node->call_for_symbol_and_aliases (do_estimate_growth_1, &d, true);
/* For self recursive functions the growth estimation really should be
infinity. We don't want to return very large values because the growth
return zero or negative growths. */
if (d.self_recursive)
d.growth = d.growth < info->size ? info->size : d.growth;
- else if (DECL_EXTERNAL (node->decl))
+ else if (DECL_EXTERNAL (node->decl) || d.uninlinable)
;
else
{
+ 50) / 100;
}
- if (node_growth_cache.exists ())
+ return d.growth;
+}
+
+/* Verify if there are fewer than MAX_CALLERS. */
+
+static bool
+check_callers (cgraph_node *node, int *max_callers)
+{
+ ipa_ref *ref;
+
+ if (!node->can_remove_if_no_direct_calls_and_refs_p ())
+ return true;
+
+ for (cgraph_edge *e = node->callers; e; e = e->next_caller)
{
- if ((int) node_growth_cache.length () <= node->uid)
- node_growth_cache.safe_grow_cleared (symtab->cgraph_max_uid);
- node_growth_cache[node->uid] = d.growth + (d.growth >= 0);
+ (*max_callers)--;
+ if (!*max_callers
+ || cgraph_inline_failed_type (e->inline_failed) == CIF_FINAL_ERROR)
+ return true;
}
- return d.growth;
+
+ FOR_EACH_ALIAS (node, ref)
+ if (check_callers (dyn_cast <cgraph_node *> (ref->referring), max_callers))
+ return true;
+
+ return false;
}
and skip computation if there are too many callers. */
bool
-growth_likely_positive (struct cgraph_node *node, int edge_growth ATTRIBUTE_UNUSED)
+growth_likely_positive (struct cgraph_node *node,
+ int edge_growth)
{
int max_callers;
- int ret;
struct cgraph_edge *e;
gcc_checking_assert (edge_growth > 0);
+ /* First quickly check if NODE is removable at all. */
+ if (DECL_EXTERNAL (node->decl))
+ return true;
+ if (!node->can_remove_if_no_direct_calls_and_refs_p ()
+ || node->address_taken)
+ return true;
+
+ max_callers = inline_summaries->get (node)->size * 4 / edge_growth + 2;
+
+ for (e = node->callers; e; e = e->next_caller)
+ {
+ max_callers--;
+ if (!max_callers
+ || cgraph_inline_failed_type (e->inline_failed) == CIF_FINAL_ERROR)
+ return true;
+ }
+
+ ipa_ref *ref;
+ FOR_EACH_ALIAS (node, ref)
+ if (check_callers (dyn_cast <cgraph_node *> (ref->referring), &max_callers))
+ return true;
+
/* Unlike for functions called once, we play unsafe with
COMDATs. We can allow that since we know functions
in consideration are small (and thus risk is small) and
functions may or may not disappear when eliminated from
current unit. With good probability making aggressive
choice in all units is going to make overall program
- smaller.
-
- Consequently we ask cgraph_can_remove_if_no_direct_calls_p
- instead of
- cgraph_will_be_removed_from_program_if_no_direct_calls */
- if (DECL_EXTERNAL (node->decl)
- || !node->can_remove_if_no_direct_calls_p ())
- return true;
-
- /* If there is cached value, just go ahead. */
- if ((int)node_growth_cache.length () > node->uid
- && (ret = node_growth_cache[node->uid]))
- return ret > 0;
- if (!node->will_be_removed_from_program_if_no_direct_calls_p ()
- && (!DECL_COMDAT (node->decl)
- || !node->can_remove_if_no_direct_calls_p ()))
- return true;
- max_callers = inline_summary (node)->size * 4 / edge_growth + 2;
-
- for (e = node->callers; e; e = e->next_caller)
+ smaller. */
+ if (DECL_COMDAT (node->decl))
{
- max_callers--;
- if (!max_callers)
+ if (!node->can_remove_if_no_direct_calls_p ())
return true;
}
+ else if (!node->will_be_removed_from_program_if_no_direct_calls_p ())
+ return true;
+
return estimate_growth (node) > 0;
}
if (dump_file)
fprintf (dump_file, "\nAnalyzing function: %s/%u\n",
node->name (), node->order);
- if (optimize && !node->thunk.thunk_p)
+ if (opt_for_fn (node->decl, optimize) && !node->thunk.thunk_p)
inline_indirect_intraprocedural_analysis (node);
compute_inline_parameters (node, false);
if (!optimize)
/* Called when new function is inserted to callgraph late. */
-static void
-add_new_function (struct cgraph_node *node, void *data ATTRIBUTE_UNUSED)
+void
+inline_summary_t::insert (struct cgraph_node *node, inline_summary *)
{
inline_analyze_function (node);
}
-
/* Note function body size. */
void
{
struct cgraph_node *node;
+ FOR_EACH_DEFINED_FUNCTION (node)
+ if (DECL_STRUCT_FUNCTION (node->decl))
+ node->local.versionable = tree_versionable_function_p (node->decl);
+
/* When not optimizing, do not bother to analyze. Inlining is still done
because edge redirection needs to happen there. */
- if (!optimize && !flag_lto && !flag_wpa)
+ if (!optimize && !flag_generate_lto && !flag_generate_offload && !flag_wpa)
return;
- function_insertion_hook_holder =
- symtab->add_cgraph_insertion_hook (&add_new_function, NULL);
+ if (!inline_summaries)
+ inline_summaries = (inline_summary_t*) inline_summary_t::create_ggc (symtab);
+
+ inline_summaries->enable_insertion_hook ();
ipa_register_cgraph_hooks ();
inline_free_summary ();
unsigned int i, count2, j;
unsigned int f_count;
- lto_input_block ib ((const char *) data + main_offset, header->main_size);
+ lto_input_block ib ((const char *) data + main_offset, header->main_size,
+ file_data->mode_table);
data_in =
lto_data_in_create (file_data, (const char *) data + string_offset,
encoder = file_data->symtab_node_encoder;
node = dyn_cast<cgraph_node *> (lto_symtab_encoder_deref (encoder,
index));
- info = inline_summary (node);
+ info = inline_summaries->get (node);
info->estimated_stack_size
= info->estimated_self_stack_size = streamer_read_uhwi (&ib);
bp = streamer_read_bitpack (&ib);
info->inlinable = bp_unpack_value (&bp, 1);
+ info->contains_cilk_spawn = bp_unpack_value (&bp, 1);
count2 = streamer_read_uhwi (&ib);
gcc_assert (!info->conds);
/* Fatal error here. We do not want to support compiling ltrans units
with different version of compiler or different flags than the WPA
unit, so this should never happen. */
- fatal_error ("ipa inline summary is missing in input file");
+ fatal_error (input_location,
+ "ipa inline summary is missing in input file");
}
if (optimize)
{
if (!flag_ipa_cp)
ipa_prop_read_jump_functions ();
}
- function_insertion_hook_holder =
- symtab->add_cgraph_insertion_hook (&add_new_function, NULL);
+
+ gcc_assert (inline_summaries);
+ inline_summaries->enable_insertion_hook ();
}
cgraph_node *cnode = dyn_cast <cgraph_node *> (snode);
if (cnode && (node = cnode)->definition && !node->alias)
{
- struct inline_summary *info = inline_summary (node);
+ struct inline_summary *info = inline_summaries->get (node);
struct bitpack_d bp;
struct cgraph_edge *edge;
int i;
streamer_write_hwi (ob, info->self_time);
bp = bitpack_create (ob->main_stream);
bp_pack_value (&bp, info->inlinable, 1);
+ bp_pack_value (&bp, info->contains_cilk_spawn, 1);
streamer_write_bitpack (&bp);
streamer_write_uhwi (ob, vec_safe_length (info->conds));
for (i = 0; vec_safe_iterate (info->conds, i, &c); i++)
inline_free_summary (void)
{
struct cgraph_node *node;
- if (!inline_edge_summary_vec.exists ())
- return;
- FOR_EACH_DEFINED_FUNCTION (node)
- if (!node->alias)
- reset_inline_summary (node);
- if (function_insertion_hook_holder)
- symtab->remove_cgraph_insertion_hook (function_insertion_hook_holder);
- function_insertion_hook_holder = NULL;
- if (node_removal_hook_holder)
- symtab->remove_cgraph_removal_hook (node_removal_hook_holder);
- node_removal_hook_holder = NULL;
if (edge_removal_hook_holder)
symtab->remove_edge_removal_hook (edge_removal_hook_holder);
edge_removal_hook_holder = NULL;
- if (node_duplication_hook_holder)
- symtab->remove_cgraph_duplication_hook (node_duplication_hook_holder);
- node_duplication_hook_holder = NULL;
if (edge_duplication_hook_holder)
symtab->remove_edge_duplication_hook (edge_duplication_hook_holder);
edge_duplication_hook_holder = NULL;
- vec_free (inline_summary_vec);
+ if (!inline_edge_summary_vec.exists ())
+ return;
+ FOR_EACH_DEFINED_FUNCTION (node)
+ if (!node->alias)
+ reset_inline_summary (node, inline_summaries->get (node));
+ inline_summaries->release ();
+ inline_summaries = NULL;
inline_edge_summary_vec.release ();
- if (edge_predicate_pool)
- free_alloc_pool (edge_predicate_pool);
- edge_predicate_pool = 0;
+ edge_predicate_pool.release ();
}
projects / gcc.git / blobdiff