+2011-04-26 Jan Hubicka <jh@suse.cz>
+
+ * cgraphbuild.c (build_cgraph_edges): Update call
+ of cgraph_create_edge and cgraph_create_indirect_edge.
+ * cgraph.c (cgraph_create_edge_including_clones,
+ cgraph_create_edge_1, cgraph_allocate_init_indirect_info,
+ cgraph_update_edges_for_call_stmt_node): Do not take nest
+ argument; do not initialize call_stmt_size/time.
+ (dump_cgraph_node): Do not dump nest.
+ (cgraph_clone_edge): Do not take loop_nest argument;
+ do not propagate it; do not clone call_stmt_size/time.
+ (cgraph_clone_node): Likewise.
+ (cgraph_create_virtual_clone): Update.
+ * cgraph.h (struct cgraph_edge): Remove
+ call_stmt_size/call_stmt_time/loop_nest.
+ (cgraph_create_edge, cgraph_create_indirect_edge,
+ cgraph_create_edge_including_clones, cgraph_clone_node): Update
+ prototype.
+ * tree-emutls.c (gen_emutls_addr): Update.
+ * ipa-inline-transform.c (update_noncloned_frequencies): Do not handle
+ loop_nest; handle indirect calls, too.
+ (clone_inlined_nodes): Do not care about updating inline summaries.
+ * cgraphunit.c (cgraph_copy_node_for_versioning): Update.
+ * lto-cgraph.c (lto_output_edge, input_node, input_edge): Do not
+ stream call_stmt_size/call_stmt_time/loop_nest.
+ * ipa-inline.c (edge_badness): Update.
+ (ipa_inline): dump summaries after inlining.
+ * ipa-inline.h (struct inline_edge_summary, inline_edge_summary_t):
+ new.
+ (inline_edge_summary): New function.
+ * ipa-inline-analysis.c (edge_duplication_hook_holder): New holder.
+ (inline_edge_removal_hook): Handle edge summaries.
+ (inline_edge_duplication_hook): New hook.
+ (inline_summary_alloc): Alloc hooks.
+ (initialize_growth_caches): Do not register removal hooks.
+ (free_growth_caches); Do not free removal hook.
+ (dump_inline_edge_summary): New function.
+ (dump_inline_summary): Use it.
+ (estimate_function_body_sizes, estimate_edge_size_and_time): Update.
+ (inline_update_callee_summaries): New function.
+ (inline_merge_summary): Use it.
+ (do_estimate_edge_time, do_estimate_edge_growth): Update.
+ (read_inline_edge_summary): New function.
+ (inline_read_section): Use it.
+ (write_inline_edge_summary): New function.
+ (inline_write_summary): Use it.
+ (inline_free_summary): Free edge new holders.
+ * tree-inline.c (copy_bb): Update.
+
2011-04-26 Jason Merrill <jason@redhat.com>
* tree-eh.c (lower_try_finally_switch): Create the label along with
struct cgraph_node *callee,
gimple old_stmt,
gimple stmt, gcov_type count,
- int freq, int loop_depth,
+ int freq,
cgraph_inline_failed_t reason)
{
struct cgraph_node *node;
if (!cgraph_edge (orig, stmt))
{
- edge = cgraph_create_edge (orig, callee, stmt, count, freq, loop_depth);
+ edge = cgraph_create_edge (orig, callee, stmt, count, freq);
edge->inline_failed = reason;
}
else if (!cgraph_edge (node, stmt))
{
edge = cgraph_create_edge (node, callee, stmt, count,
- freq, loop_depth);
+ freq);
edge->inline_failed = reason;
}
static struct cgraph_edge *
cgraph_create_edge_1 (struct cgraph_node *caller, struct cgraph_node *callee,
- gimple call_stmt, gcov_type count, int freq, int nest)
+ gimple call_stmt, gcov_type count, int freq)
{
struct cgraph_edge *edge;
edge->frequency = freq;
gcc_assert (freq >= 0);
gcc_assert (freq <= CGRAPH_FREQ_MAX);
- edge->loop_nest = nest;
edge->call_stmt = call_stmt;
- edge->call_stmt_size = 0;
- edge->call_stmt_time = 0;
push_cfun (DECL_STRUCT_FUNCTION (caller->decl));
edge->can_throw_external
= call_stmt ? stmt_can_throw_external (call_stmt) : false;
struct cgraph_edge *
cgraph_create_edge (struct cgraph_node *caller, struct cgraph_node *callee,
- gimple call_stmt, gcov_type count, int freq, int nest)
+ gimple call_stmt, gcov_type count, int freq)
{
struct cgraph_edge *edge = cgraph_create_edge_1 (caller, callee, call_stmt,
- count, freq, nest);
+ count, freq);
edge->indirect_unknown_callee = 0;
initialize_inline_failed (edge);
struct cgraph_edge *
cgraph_create_indirect_edge (struct cgraph_node *caller, gimple call_stmt,
int ecf_flags,
- gcov_type count, int freq, int nest)
+ gcov_type count, int freq)
{
struct cgraph_edge *edge = cgraph_create_edge_1 (caller, NULL, call_stmt,
- count, freq, nest);
+ count, freq);
edge->indirect_unknown_callee = 1;
initialize_inline_failed (edge);
struct cgraph_edge *ne = NULL;
gcov_type count;
int frequency;
- int loop_nest;
if (e)
{
attached to edge is invalid. */
count = e->count;
frequency = e->frequency;
- loop_nest = e->loop_nest;
cgraph_remove_edge (e);
}
else
count = bb->count;
frequency = compute_call_stmt_bb_frequency (current_function_decl,
bb);
- loop_nest = bb->loop_depth;
}
if (new_call)
{
ne = cgraph_create_edge (node, cgraph_get_create_node (new_call),
- new_stmt, count, frequency,
- loop_nest);
+ new_stmt, count, frequency);
gcc_assert (ne->inline_failed);
}
}
if (edge->frequency)
fprintf (f, "(%.2f per call) ",
edge->frequency / (double)CGRAPH_FREQ_BASE);
- if (edge->loop_nest)
- fprintf (f, "(nested in %i loops) ", edge->loop_nest);
if (edge->can_throw_external)
fprintf(f, "(can throw external) ");
}
struct cgraph_edge *
cgraph_clone_edge (struct cgraph_edge *e, struct cgraph_node *n,
gimple call_stmt, unsigned stmt_uid, gcov_type count_scale,
- int freq_scale, int loop_nest, bool update_original)
+ int freq_scale, bool update_original)
{
struct cgraph_edge *new_edge;
gcov_type count = e->count * count_scale / REG_BR_PROB_BASE;
{
struct cgraph_node *callee = cgraph_get_node (decl);
gcc_checking_assert (callee);
- new_edge = cgraph_create_edge (n, callee, call_stmt, count, freq,
- e->loop_nest + loop_nest);
+ new_edge = cgraph_create_edge (n, callee, call_stmt, count, freq);
}
else
{
new_edge = cgraph_create_indirect_edge (n, call_stmt,
e->indirect_info->ecf_flags,
- count, freq,
- e->loop_nest + loop_nest);
+ count, freq);
*new_edge->indirect_info = *e->indirect_info;
}
}
else
{
- new_edge = cgraph_create_edge (n, e->callee, call_stmt, count, freq,
- e->loop_nest + loop_nest);
+ new_edge = cgraph_create_edge (n, e->callee, call_stmt, count, freq);
if (e->indirect_info)
{
new_edge->indirect_info
}
}
- new_edge->call_stmt_size = e->call_stmt_size;
- new_edge->call_stmt_time = e->call_stmt_time;
new_edge->inline_failed = e->inline_failed;
new_edge->indirect_inlining_edge = e->indirect_inlining_edge;
new_edge->lto_stmt_uid = stmt_uid;
by node. */
struct cgraph_node *
cgraph_clone_node (struct cgraph_node *n, tree decl, gcov_type count, int freq,
- int loop_nest, bool update_original,
+ bool update_original,
VEC(cgraph_edge_p,heap) *redirect_callers)
{
struct cgraph_node *new_node = cgraph_create_node_1 ();
for (e = n->callees;e; e=e->next_callee)
cgraph_clone_edge (e, new_node, e->call_stmt, e->lto_stmt_uid,
- count_scale, freq, loop_nest, update_original);
+ count_scale, freq, update_original);
for (e = n->indirect_calls; e; e = e->next_callee)
cgraph_clone_edge (e, new_node, e->call_stmt, e->lto_stmt_uid,
- count_scale, freq, loop_nest, update_original);
+ count_scale, freq, update_original);
ipa_clone_references (new_node, NULL, &n->ref_list);
new_node->next_sibling_clone = n->clones;
SET_DECL_RTL (new_decl, NULL);
new_node = cgraph_clone_node (old_node, new_decl, old_node->count,
- CGRAPH_FREQ_BASE, 0, false,
+ CGRAPH_FREQ_BASE, false,
redirect_callers);
/* Update the properties.
Make clone visible only within this translation unit. Make sure
fprintf (dump_file, " Called by %s that is executed once\n",
cgraph_node_name (edge->caller));
maybe_unlikely_executed = false;
- if (edge->loop_nest)
+ if (inline_edge_summary (edge)->loop_depth)
{
maybe_executed_once = false;
if (dump_file && (dump_flags & TDF_DETAILS))
int frequency;
/* Unique id of the edge. */
int uid;
- /* Estimated size and time of the call statement. */
- int call_stmt_size;
- int call_stmt_time;
- /* Depth of loop nest, 1 means no loop nest. */
- unsigned short int loop_nest;
/* Whether this edge was made direct by indirect inlining. */
unsigned int indirect_inlining_edge : 1;
/* Whether this edge describes an indirect call with an undetermined
void cgraph_node_remove_callees (struct cgraph_node *node);
struct cgraph_edge *cgraph_create_edge (struct cgraph_node *,
struct cgraph_node *,
- gimple, gcov_type, int, int);
+ gimple, gcov_type, int);
struct cgraph_edge *cgraph_create_indirect_edge (struct cgraph_node *, gimple,
- int, gcov_type, int, int);
+ int, gcov_type, int);
struct cgraph_indirect_call_info *cgraph_allocate_init_indirect_info (void);
struct cgraph_node * cgraph_get_node (const_tree);
struct cgraph_node * cgraph_get_node_or_alias (const_tree);
void cgraph_set_call_stmt_including_clones (struct cgraph_node *, gimple, gimple);
void cgraph_create_edge_including_clones (struct cgraph_node *,
struct cgraph_node *,
- gimple, gimple, gcov_type, int, int,
+ gimple, gimple, gcov_type, int,
cgraph_inline_failed_t);
void cgraph_update_edges_for_call_stmt (gimple, tree, gimple);
struct cgraph_local_info *cgraph_local_info (tree);
const char * cgraph_node_name (struct cgraph_node *);
struct cgraph_edge * cgraph_clone_edge (struct cgraph_edge *,
struct cgraph_node *, gimple,
- unsigned, gcov_type, int, int, bool);
-struct cgraph_node * cgraph_clone_node (struct cgraph_node *, tree, gcov_type, int,
+ unsigned, gcov_type, int, bool);
+struct cgraph_node * cgraph_clone_node (struct cgraph_node *, tree, gcov_type,
int, bool, VEC(cgraph_edge_p,heap) *);
void cgraph_redirect_edge_callee (struct cgraph_edge *, struct cgraph_node *);
decl = gimple_call_fndecl (stmt);
if (decl)
cgraph_create_edge (node, cgraph_get_create_node (decl),
- stmt, bb->count, freq, bb->loop_depth);
+ stmt, bb->count, freq);
else
cgraph_create_indirect_edge (node, stmt,
gimple_call_flags (stmt),
- bb->count, freq,
- bb->loop_depth);
+ bb->count, freq);
}
walk_stmt_load_store_addr_ops (stmt, node, mark_load,
mark_store, mark_address);
decl = gimple_call_fndecl (stmt);
if (decl)
cgraph_create_edge (node, cgraph_get_create_node (decl), stmt,
- bb->count, freq, bb->loop_depth);
+ bb->count, freq);
else
cgraph_create_indirect_edge (node, stmt,
gimple_call_flags (stmt),
- bb->count, freq,
- bb->loop_depth);
+ bb->count, freq);
}
walk_stmt_load_store_addr_ops (stmt, node, mark_load,
mark_store, mark_address);
cgraph_clone_edge (e, new_version, e->call_stmt,
e->lto_stmt_uid, REG_BR_PROB_BASE,
CGRAPH_FREQ_BASE,
- e->loop_nest, true);
+ true);
for (e = old_version->indirect_calls; e; e=e->next_callee)
if (!bbs_to_copy
|| bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index))
cgraph_clone_edge (e, new_version, e->call_stmt,
e->lto_stmt_uid, REG_BR_PROB_BASE,
CGRAPH_FREQ_BASE,
- e->loop_nest, true);
+ true);
FOR_EACH_VEC_ELT (cgraph_edge_p, redirect_callers, i, e)
{
/* Redirect calls to the old version node to point to its new
static struct cgraph_node_hook_list *function_insertion_hook_holder;
static struct cgraph_node_hook_list *node_removal_hook_holder;
static struct cgraph_2node_hook_list *node_duplication_hook_holder;
+static struct cgraph_2edge_hook_list *edge_duplication_hook_holder;
static struct cgraph_edge_hook_list *edge_removal_hook_holder;
static void inline_node_removal_hook (struct cgraph_node *, void *);
static void inline_node_duplication_hook (struct cgraph_node *,
struct cgraph_node *, void *);
+static void inline_edge_removal_hook (struct cgraph_edge *, void *);
+static void inline_edge_duplication_hook (struct cgraph_edge *,
+ struct cgraph_edge *,
+ void *);
/* VECtor holding inline summaries.
In GGC memory because conditions might point to constant trees. */
VEC(inline_summary_t,gc) *inline_summary_vec;
+VEC(inline_edge_summary_t,heap) *inline_edge_summary_vec;
/* Cached node/edge growths. */
VEC(int,heap) *node_growth_cache;
if (!node_removal_hook_holder)
node_removal_hook_holder =
cgraph_add_node_removal_hook (&inline_node_removal_hook, NULL);
+ if (!edge_removal_hook_holder)
+ edge_removal_hook_holder =
+ cgraph_add_edge_removal_hook (&inline_edge_removal_hook, NULL);
if (!node_duplication_hook_holder)
node_duplication_hook_holder =
cgraph_add_node_duplication_hook (&inline_node_duplication_hook, NULL);
+ if (!edge_duplication_hook_holder)
+ edge_duplication_hook_holder =
+ cgraph_add_edge_duplication_hook (&inline_edge_duplication_hook, NULL);
if (VEC_length (inline_summary_t, inline_summary_vec)
<= (unsigned) cgraph_max_uid)
VEC_safe_grow_cleared (inline_summary_t, gc,
inline_summary_vec, cgraph_max_uid + 1);
+ if (VEC_length (inline_edge_summary_t, inline_edge_summary_vec)
+ <= (unsigned) cgraph_edge_max_uid)
+ VEC_safe_grow_cleared (inline_edge_summary_t, heap,
+ inline_edge_summary_vec, cgraph_edge_max_uid + 1);
}
/* Hook that is called by cgraph.c when a node is removed. */
memset (info, 0, sizeof (inline_summary_t));
}
+
/* Hook that is called by cgraph.c when a node is duplicated. */
static void
}
+/* Hook that is called by cgraph.c when a node is duplicated. */
+
+static void
+inline_edge_duplication_hook (struct cgraph_edge *src, struct cgraph_edge *dst,
+ ATTRIBUTE_UNUSED void *data)
+{
+ struct inline_edge_summary *info;
+ inline_summary_alloc ();
+ info = inline_edge_summary (dst);
+ memcpy (info, inline_edge_summary (src),
+ sizeof (struct inline_edge_summary));
+}
+
+
/* Keep edge cache consistent across edge removal. */
static void
inline_edge_removal_hook (struct cgraph_edge *edge, void *data ATTRIBUTE_UNUSED)
{
- reset_edge_growth_cache (edge);
+ if (edge_growth_cache)
+ reset_edge_growth_cache (edge);
+ if (edge->uid < (int)VEC_length (inline_edge_summary_t, inline_edge_summary_vec))
+ memset (inline_edge_summary (edge), 0, sizeof (struct inline_edge_summary));
}
void
initialize_growth_caches (void)
{
- if (!edge_removal_hook_holder)
- edge_removal_hook_holder =
- cgraph_add_edge_removal_hook (&inline_edge_removal_hook, NULL);
if (cgraph_edge_max_uid)
VEC_safe_grow_cleared (edge_growth_cache_entry, heap, edge_growth_cache,
cgraph_edge_max_uid);
void
free_growth_caches (void)
{
- if (edge_removal_hook_holder)
- cgraph_remove_edge_removal_hook (edge_removal_hook_holder);
VEC_free (edge_growth_cache_entry, heap, edge_growth_cache);
edge_growth_cache = 0;
VEC_free (int, heap, node_growth_cache);
}
+/* Dump edge summaries associated to NODE and recursively to all clones.
+ Indent by INDENT. */
+
+static void
+dump_inline_edge_summary (FILE * f, int indent, struct cgraph_node *node)
+{
+ struct cgraph_edge *edge;
+ for (edge = node->callees; edge; edge = edge->next_callee)
+ {
+ struct inline_edge_summary *es = inline_edge_summary (edge);
+ fprintf (f, "%*s%s/%i %s\n%*s loop depth:%2i freq:%4i size:%2i time: %2i\n",
+ indent, "", cgraph_node_name (edge->callee),
+ edge->callee->uid,
+ edge->inline_failed ? "inlined"
+ : cgraph_inline_failed_string (edge->inline_failed),
+ indent, "",
+ es->loop_depth,
+ edge->frequency,
+ es->call_stmt_size,
+ es->call_stmt_time);
+ if (!edge->inline_failed)
+ dump_inline_edge_summary (f, indent+2, edge->callee);
+ }
+ for (edge = node->indirect_calls; edge; edge = edge->next_callee)
+ {
+ struct inline_edge_summary *es = inline_edge_summary (edge);
+ fprintf (f, "%*sindirect call loop depth:%2i freq:%4i size:%2i time: %2i\n",
+ indent, "",
+ es->loop_depth,
+ edge->frequency,
+ es->call_stmt_size,
+ es->call_stmt_time);
+ }
+}
+
+
static void
dump_inline_summary (FILE * f, struct cgraph_node *node)
{
(double) e->time / INLINE_TIME_SCALE);
dump_predicate (f, s->conds, &e->predicate);
}
+ fprintf (f, " calls:\n");
+ dump_inline_edge_summary (f, 4, node);
fprintf (f, "\n");
}
}
struct cgraph_node *node;
for (node = cgraph_nodes; node; node = node->next)
- if (node->analyzed)
+ if (node->analyzed && !node->global.inlined_to)
dump_inline_summary (f, node);
}
if (is_gimple_call (stmt))
{
struct cgraph_edge *edge = cgraph_edge (node, stmt);
- edge->call_stmt_size = this_size;
- edge->call_stmt_time = this_time;
+ struct inline_edge_summary *es = inline_edge_summary (edge);
+
+ es->call_stmt_size = this_size;
+ es->call_stmt_time = this_time;
+ es->loop_depth = bb->loop_depth;
/* Do not inline calls where we cannot triviall work around
mismatches in argument or return types. */
static void
estimate_edge_size_and_time (struct cgraph_edge *e, int *size, int *time)
{
- *size += e->call_stmt_size * INLINE_SIZE_SCALE;
- *time += (e->call_stmt_time
+ struct inline_edge_summary *es = inline_edge_summary (e);
+ *size += es->call_stmt_size * INLINE_SIZE_SCALE;
+ *time += (es->call_stmt_time
* e->frequency * (INLINE_TIME_SCALE / CGRAPH_FREQ_BASE));
if (*time > MAX_TIME * INLINE_TIME_SCALE)
*time = MAX_TIME * INLINE_TIME_SCALE;
}
+/* Update summary information of inline clones after inlining.
+ Compute peak stack usage. */
+
+static void
+inline_update_callee_summaries (struct cgraph_node *node,
+ int depth)
+{
+ struct cgraph_edge *e;
+ struct inline_summary *callee_info = inline_summary (node);
+ struct inline_summary *caller_info = inline_summary (node->callers->caller);
+ HOST_WIDE_INT peak;
+
+ callee_info->stack_frame_offset
+ = caller_info->stack_frame_offset
+ + caller_info->estimated_self_stack_size;
+ peak = callee_info->stack_frame_offset
+ + callee_info->estimated_self_stack_size;
+ if (inline_summary (node->global.inlined_to)->estimated_stack_size
+ < peak)
+ inline_summary (node->global.inlined_to)->estimated_stack_size = peak;
+ cgraph_propagate_frequency (node);
+ for (e = node->callees; e; e = e->next_callee)
+ {
+ if (!e->inline_failed)
+ inline_update_callee_summaries (e->callee, depth);
+ inline_edge_summary (e)->loop_depth += depth;
+ }
+ for (e = node->indirect_calls; e; e = e->next_callee)
+ inline_edge_summary (e)->loop_depth += depth;
+}
+
+
/* We inlined EDGE. Update summary of the function we inlined into. */
void
for (i = 0; VEC_iterate (size_time_entry, info->entry, i, e); i++)
info->size += e->size, info->time += e->time;
estimate_calls_size_and_time (to, &info->size, &info->time);
+
+ inline_update_callee_summaries (edge->callee,
+ inline_edge_summary (edge)->loop_depth);
+
info->time = (info->time + INLINE_TIME_SCALE / 2) / INLINE_TIME_SCALE;
info->size = (info->size + INLINE_SIZE_SCALE / 2) / INLINE_SIZE_SCALE;
}
int time;
int size;
gcov_type ret;
+ struct inline_edge_summary *es = inline_edge_summary (edge);
gcc_checking_assert (edge->inline_failed);
estimate_callee_size_and_time (edge, true, &size, &time);
- ret = (((gcov_type)time - edge->call_stmt_time) * edge->frequency
+ ret = (((gcov_type)time - es->call_stmt_time) * edge->frequency
+ CGRAPH_FREQ_BASE / 2) / CGRAPH_FREQ_BASE;
if (ret > MAX_TIME)
ret = MAX_TIME;
VEC_index (edge_growth_cache_entry, edge_growth_cache, edge->uid)->time
= ret + (ret >= 0);
- ret_size = size - edge->call_stmt_size;
- gcc_checking_assert (edge->call_stmt_size);
+ ret_size = size - es->call_stmt_size;
+ gcc_checking_assert (es->call_stmt_size);
VEC_index (edge_growth_cache_entry, edge_growth_cache, edge->uid)->size
= ret_size + (ret_size >= 0);
}
/* Early inliner runs without caching, go ahead and do the dirty work. */
gcc_checking_assert (edge->inline_failed);
estimate_callee_size_and_time (edge, true, &size, NULL);
- gcc_checking_assert (edge->call_stmt_size);
- return size - edge->call_stmt_size;
+ gcc_checking_assert (inline_edge_summary (edge)->call_stmt_size);
+ return size - inline_edge_summary (edge)->call_stmt_size;
}
}
+/* Write inline summary for edge E to OB. */
+
+static void
+read_inline_edge_summary (struct lto_input_block *ib, struct cgraph_edge *e)
+{
+ struct inline_edge_summary *es = inline_edge_summary (e);
+ es->call_stmt_size = lto_input_uleb128 (ib);
+ es->call_stmt_time = lto_input_uleb128 (ib);
+ es->loop_depth = lto_input_uleb128 (ib);
+}
+
+
/* Stream in inline summaries from the section. */
static void
struct inline_summary *info;
lto_cgraph_encoder_t encoder;
struct bitpack_d bp;
+ struct cgraph_edge *e;
index = lto_input_uleb128 (&ib);
encoder = file_data->cgraph_node_encoder;
VEC_safe_push (size_time_entry, gc, info->entry, &e);
}
+ for (e = node->callees; e; e = e->next_callee)
+ read_inline_edge_summary (&ib, e);
+ for (e = node->indirect_calls; e; e = e->next_callee)
+ read_inline_edge_summary (&ib, e);
}
lto_free_section_data (file_data, LTO_section_inline_summary, NULL, data,
cgraph_add_function_insertion_hook (&add_new_function, NULL);
}
+/* Write inline summary for edge E to OB. */
+
+static void
+write_inline_edge_summary (struct output_block *ob, struct cgraph_edge *e)
+{
+ struct inline_edge_summary *es = inline_edge_summary (e);
+ lto_output_uleb128_stream (ob->main_stream, es->call_stmt_size);
+ lto_output_uleb128_stream (ob->main_stream, es->call_stmt_time);
+ lto_output_uleb128_stream (ob->main_stream, es->loop_depth);
+}
+
/* Write inline summary for node in SET.
Jump functions are shared among ipa-cp and inliner, so when ipa-cp is
{
struct inline_summary *info = inline_summary (node);
struct bitpack_d bp;
+ struct cgraph_edge *edge;
int i;
size_time_entry *e;
struct condition *c;
}
lto_output_uleb128_stream (ob->main_stream, 0);
}
+ for (edge = node->callees; edge; edge = edge->next_callee)
+ write_inline_edge_summary (ob, edge);
+ for (edge = node->indirect_calls; edge; edge = edge->next_callee)
+ write_inline_edge_summary (ob, edge);
}
}
lto_output_1_stream (ob->main_stream, 0);
function_insertion_hook_holder = NULL;
if (node_removal_hook_holder)
cgraph_remove_node_removal_hook (node_removal_hook_holder);
+ if (edge_removal_hook_holder)
+ cgraph_remove_edge_removal_hook (edge_removal_hook_holder);
node_removal_hook_holder = NULL;
if (node_duplication_hook_holder)
cgraph_remove_node_duplication_hook (node_duplication_hook_holder);
+ if (edge_duplication_hook_holder)
+ cgraph_remove_edge_duplication_hook (edge_duplication_hook_holder);
node_duplication_hook_holder = NULL;
VEC_free (inline_summary_t, gc, inline_summary_vec);
inline_summary_vec = NULL;
int ncalls_inlined;
int nfunctions_inlined;
-/* Scale frequency of NODE edges by FREQ_SCALE and increase loop nest
- by NEST. */
+/* Scale frequency of NODE edges by FREQ_SCALE. */
static void
update_noncloned_frequencies (struct cgraph_node *node,
- int freq_scale, int nest)
+ int freq_scale)
{
struct cgraph_edge *e;
freq_scale = 1;
for (e = node->callees; e; e = e->next_callee)
{
- e->loop_nest += nest;
e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
if (e->frequency > CGRAPH_FREQ_MAX)
e->frequency = CGRAPH_FREQ_MAX;
if (!e->inline_failed)
- update_noncloned_frequencies (e->callee, freq_scale, nest);
+ update_noncloned_frequencies (e->callee, freq_scale);
+ }
+ for (e = node->indirect_calls; e; e = e->next_callee)
+ {
+ e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE;
+ if (e->frequency > CGRAPH_FREQ_MAX)
+ e->frequency = CGRAPH_FREQ_MAX;
}
}
clone_inlined_nodes (struct cgraph_edge *e, bool duplicate,
bool update_original, int *overall_size)
{
- HOST_WIDE_INT peak;
- struct inline_summary *caller_info, *callee_info;
-
if (duplicate)
{
/* We may eliminate the need for out-of-line copy to be output.
}
duplicate = false;
e->callee->local.externally_visible = false;
- update_noncloned_frequencies (e->callee, e->frequency, e->loop_nest);
+ update_noncloned_frequencies (e->callee, e->frequency);
}
else
{
struct cgraph_node *n;
n = cgraph_clone_node (e->callee, e->callee->decl,
- e->count, e->frequency, e->loop_nest,
+ e->count, e->frequency,
update_original, NULL);
cgraph_redirect_edge_callee (e, n);
}
}
- callee_info = inline_summary (e->callee);
- caller_info = inline_summary (e->caller);
-
if (e->caller->global.inlined_to)
e->callee->global.inlined_to = e->caller->global.inlined_to;
else
e->callee->global.inlined_to = e->caller;
- callee_info->stack_frame_offset
- = caller_info->stack_frame_offset
- + caller_info->estimated_self_stack_size;
- peak = callee_info->stack_frame_offset
- + callee_info->estimated_self_stack_size;
- if (inline_summary (e->callee->global.inlined_to)->estimated_stack_size
- < peak)
- inline_summary (e->callee->global.inlined_to)->estimated_stack_size = peak;
- cgraph_propagate_frequency (e->callee);
/* Recursively clone all bodies. */
for (e = e->callee->callees; e; e = e->next_callee)
to = e->caller;
if (to->global.inlined_to)
to = to->global.inlined_to;
- old_size = inline_summary (to)->size;
- inline_merge_summary (e);
- new_size = inline_summary (to)->size;
clone_inlined_nodes (e, true, update_original, overall_size);
gcc_assert (curr->callee->global.inlined_to == to);
+
+ old_size = inline_summary (to)->size;
+ inline_merge_summary (e);
+ new_size = inline_summary (to)->size;
if (overall_size && new_size > old_size)
*overall_size += new_size - old_size;
ncalls_inlined++;
of functions fully inlined in program. */
else
{
- int nest = MIN (edge->loop_nest, 8);
+ int nest = MIN (inline_edge_summary (edge)->loop_depth, 8);
badness = estimate_growth (edge->callee) * 256;
/* Decrease badness if call is nested. */
{
/* We need original clone to copy around. */
master_clone = cgraph_clone_node (node, node->decl,
- node->count, CGRAPH_FREQ_BASE, 1,
+ node->count, CGRAPH_FREQ_BASE,
false, NULL);
for (e = master_clone->callees; e; e = e->next_callee)
if (!e->inline_failed)
"\nInlined %i calls, eliminated %i functions\n\n",
ncalls_inlined, nfunctions_inlined);
+ if (dump_file)
+ dump_inline_summaries (dump_file);
/* In WPA we use inline summaries for partitioning process. */
if (!flag_wpa)
inline_free_summary ();
info that might be cleared out for newly discovered edges. */
for (edge = node->callees; edge; edge = edge->next_callee)
{
- edge->call_stmt_size
+ struct inline_edge_summary *es = inline_edge_summary (edge);
+ es->call_stmt_size
= estimate_num_insns (edge->call_stmt, &eni_size_weights);
- edge->call_stmt_time
+ es->call_stmt_time
= estimate_num_insns (edge->call_stmt, &eni_time_weights);
}
timevar_pop (TV_INTEGRATION);
int time;
int size;
+ /* Conditional size/time information. The summaries are being
+ merged during inlining. */
conditions conds;
VEC(size_time_entry,gc) *entry;
};
DEF_VEC_ALLOC_O(inline_summary_t,gc);
extern GTY(()) VEC(inline_summary_t,gc) *inline_summary_vec;
+/* Information kept about callgraph edges. */
+struct inline_edge_summary
+{
+ /* Estimated size and time of the call statement. */
+ int call_stmt_size;
+ int call_stmt_time;
+ /* Depth of loop nest, 0 means no nesting. */
+ unsigned short int loop_depth;
+};
+
+typedef struct inline_edge_summary inline_edge_summary_t;
+DEF_VEC_O(inline_edge_summary_t);
+DEF_VEC_ALLOC_O(inline_edge_summary_t,heap);
+extern VEC(inline_edge_summary_t,heap) *inline_edge_summary_vec;
+
typedef struct edge_growth_cache_entry
{
int time, size;
return VEC_index (inline_summary_t, inline_summary_vec, node->uid);
}
+static inline struct inline_edge_summary *
+inline_edge_summary (struct cgraph_edge *edge)
+{
+ return VEC_index (inline_edge_summary_t,
+ inline_edge_summary_vec, edge->uid);
+}
/* Return estimated unit growth after inlning all calls to NODE.
Quick accesors to the inline growth caches.
bp_pack_value (&bp, uid, HOST_BITS_PER_INT);
bp_pack_value (&bp, edge->inline_failed, HOST_BITS_PER_INT);
bp_pack_value (&bp, edge->frequency, HOST_BITS_PER_INT);
- bp_pack_value (&bp, edge->call_stmt_size, HOST_BITS_PER_INT);
- bp_pack_value (&bp, edge->call_stmt_time, HOST_BITS_PER_INT);
- bp_pack_value (&bp, edge->loop_nest, 30);
bp_pack_value (&bp, edge->indirect_inlining_edge, 1);
bp_pack_value (&bp, edge->call_stmt_cannot_inline_p, 1);
bp_pack_value (&bp, edge->can_throw_external, 1);
if (clone_ref != LCC_NOT_FOUND)
{
node = cgraph_clone_node (VEC_index (cgraph_node_ptr, nodes, clone_ref), fn_decl,
- 0, CGRAPH_FREQ_BASE, 0, false, NULL);
+ 0, CGRAPH_FREQ_BASE, false, NULL);
}
else
node = cgraph_get_create_node (fn_decl);
unsigned int stmt_id;
gcov_type count;
int freq;
- unsigned int nest;
cgraph_inline_failed_t inline_failed;
struct bitpack_d bp;
int ecf_flags = 0;
- int call_stmt_time, call_stmt_size;
caller = VEC_index (cgraph_node_ptr, nodes, lto_input_sleb128 (ib));
if (caller == NULL || caller->decl == NULL_TREE)
inline_failed = (cgraph_inline_failed_t) bp_unpack_value (&bp,
HOST_BITS_PER_INT);
freq = (int) bp_unpack_value (&bp, HOST_BITS_PER_INT);
- call_stmt_size = (int) bp_unpack_value (&bp, HOST_BITS_PER_INT);
- call_stmt_time = (int) bp_unpack_value (&bp, HOST_BITS_PER_INT);
- nest = (unsigned) bp_unpack_value (&bp, 30);
if (indirect)
- edge = cgraph_create_indirect_edge (caller, NULL, 0, count, freq, nest);
+ edge = cgraph_create_indirect_edge (caller, NULL, 0, count, freq);
else
- edge = cgraph_create_edge (caller, callee, NULL, count, freq, nest);
+ edge = cgraph_create_edge (caller, callee, NULL, count, freq);
edge->indirect_inlining_edge = bp_unpack_value (&bp, 1);
edge->lto_stmt_uid = stmt_id;
edge->inline_failed = inline_failed;
edge->call_stmt_cannot_inline_p = bp_unpack_value (&bp, 1);
edge->can_throw_external = bp_unpack_value (&bp, 1);
- edge->call_stmt_size = call_stmt_size;
- edge->call_stmt_time = call_stmt_time;
if (indirect)
{
if (bp_unpack_value (&bp, 1))
gimple_seq_add_stmt (&d->seq, x);
cgraph_create_edge (d->cfun_node, d->builtin_node, x,
- d->bb->count, d->bb_freq, d->bb->loop_depth);
+ d->bb->count, d->bb_freq);
/* We may be adding a new reference to a new variable to the function.
This means we have to play with the ipa-reference web. */
edge = cgraph_clone_edge (edge, id->dst_node, stmt,
gimple_uid (stmt),
REG_BR_PROB_BASE, CGRAPH_FREQ_BASE,
- edge->frequency, true);
+ true);
/* We could also just rescale the frequency, but
doing so would introduce roundoff errors and make
verifier unhappy. */
(id->dst_node, dest, orig_stmt, stmt, bb->count,
compute_call_stmt_bb_frequency (id->dst_node->decl,
copy_basic_block),
- bb->loop_depth, CIF_ORIGINALLY_INDIRECT_CALL);
+ CIF_ORIGINALLY_INDIRECT_CALL);
else
cgraph_create_edge (id->dst_node, dest, stmt,
bb->count,
compute_call_stmt_bb_frequency
- (id->dst_node->decl, copy_basic_block),
- bb->loop_depth)->inline_failed
+ (id->dst_node->decl, copy_basic_block))->inline_failed
= CIF_ORIGINALLY_INDIRECT_CALL;
if (dump_file)
{
projects / gcc.git / commitdiff