/* Basic IPA optimizations based on profile.
- Copyright (C) 2003-2013 Free Software Foundation, Inc.
+ Copyright (C) 2003-2015 Free Software Foundation, Inc.
This file is part of GCC.
- Count histogram construction. This is a histogram analyzing how much
time is spent executing statements with a given execution count read
- from profile feedback. This histogram is complette only with LTO,
+ from profile feedback. This histogram is complete only with LTO,
otherwise it contains information only about the current unit.
Similar histogram is also estimated by coverage runtime. This histogram
#include "system.h"
#include "coretypes.h"
#include "tm.h"
+#include "alias.h"
+#include "symtab.h"
+#include "tree.h"
+#include "fold-const.h"
+#include "predict.h"
+#include "dominance.h"
+#include "cfg.h"
+#include "basic-block.h"
+#include "hard-reg-set.h"
+#include "function.h"
#include "cgraph.h"
#include "tree-pass.h"
+#include "tree-ssa-alias.h"
+#include "internal-fn.h"
+#include "gimple-expr.h"
#include "gimple.h"
-#include "ggc.h"
+#include "gimple-iterator.h"
#include "flags.h"
#include "target.h"
#include "tree-iterator.h"
#include "ipa-utils.h"
-#include "hash-table.h"
#include "profile.h"
#include "params.h"
#include "value-prof.h"
#include "tree-inline.h"
#include "lto-streamer.h"
#include "data-streamer.h"
+#include "symbol-summary.h"
+#include "ipa-prop.h"
#include "ipa-inline.h"
/* Entry in the histogram. */
duplicate entries. */
vec<histogram_entry *> histogram;
-static alloc_pool histogram_pool;
+static pool_allocator<histogram_entry> histogram_pool
+ ("IPA histogram", 10);
/* Hashtable support for storing SSA names hashed by their SSA_NAME_VAR. */
-struct histogram_hash : typed_noop_remove <histogram_entry>
+struct histogram_hash : nofree_ptr_hash <histogram_entry>
{
- typedef histogram_entry value_type;
- typedef histogram_entry compare_type;
- static inline hashval_t hash (const value_type *);
- static inline int equal (const value_type *, const compare_type *);
+ static inline hashval_t hash (const histogram_entry *);
+ static inline int equal (const histogram_entry *, const histogram_entry *);
};
inline hashval_t
HASHTABLE is the on-side hash kept to avoid duplicates. */
static void
-account_time_size (hash_table <histogram_hash> hashtable,
+account_time_size (hash_table<histogram_hash> *hashtable,
vec<histogram_entry *> &histogram,
gcov_type count, int time, int size)
{
histogram_entry key = {count, 0, 0};
- histogram_entry **val = hashtable.find_slot (&key, INSERT);
+ histogram_entry **val = hashtable->find_slot (&key, INSERT);
if (!*val)
{
- *val = (histogram_entry *) pool_alloc (histogram_pool);
+ *val = histogram_pool.allocate ();
**val = key;
histogram.safe_push (*val);
}
{
cumulated_time += histogram[i]->count * histogram[i]->time;
cumulated_size += histogram[i]->size;
- fprintf (file, " "HOST_WIDEST_INT_PRINT_DEC": time:%i (%2.2f) size:%i (%2.2f)\n",
- (HOST_WIDEST_INT) histogram[i]->count,
+ fprintf (file, " %" PRId64": time:%i (%2.2f) size:%i (%2.2f)\n",
+ (int64_t) histogram[i]->count,
histogram[i]->time,
cumulated_time * 100.0 / overall_time,
histogram[i]->size,
{
struct cgraph_node *node;
gimple_stmt_iterator gsi;
- hash_table <histogram_hash> hashtable;
basic_block bb;
- hashtable.create (10);
- histogram_pool = create_alloc_pool ("IPA histogram", sizeof (struct histogram_entry),
- 10);
+ hash_table<histogram_hash> hashtable (10);
FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
- FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->symbol.decl))
+ FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
{
int time = 0;
int size = 0;
{
histogram_value h;
h = gimple_histogram_value_of_type
- (DECL_STRUCT_FUNCTION (node->symbol.decl),
+ (DECL_STRUCT_FUNCTION (node->decl),
stmt, HIST_TYPE_INDIR_CALL);
/* No need to do sanity check: gimple_ic_transform already
takes away bad histograms. */
counter 2 is total number of executions. */
if (h->hvalue.counters[2])
{
- struct cgraph_edge * e = cgraph_edge (node, stmt);
+ struct cgraph_edge * e = node->get_edge (stmt);
+ if (e && !e->indirect_unknown_callee)
+ continue;
e->indirect_info->common_target_id
= h->hvalue.counters [0];
e->indirect_info->common_target_probability
e->indirect_info->common_target_probability = REG_BR_PROB_BASE;
}
}
- gimple_remove_histogram_value (DECL_STRUCT_FUNCTION (node->symbol.decl),
+ gimple_remove_histogram_value (DECL_STRUCT_FUNCTION (node->decl),
stmt, h);
}
}
time += estimate_num_insns (stmt, &eni_time_weights);
size += estimate_num_insns (stmt, &eni_size_weights);
}
- account_time_size (hashtable, histogram, bb->count, time, size);
+ account_time_size (&hashtable, histogram, bb->count, time, size);
}
- hashtable.dispose ();
histogram.qsort (cmp_counts);
}
= lto_create_simple_output_block (LTO_section_ipa_profile);
unsigned int i;
- streamer_write_uhwi_stream (ob->main_stream, histogram.length());
+ streamer_write_uhwi_stream (ob->main_stream, histogram.length ());
for (i = 0; i < histogram.length (); i++)
{
streamer_write_gcov_count_stream (ob->main_stream, histogram[i]->count);
struct lto_file_decl_data ** file_data_vec
= lto_get_file_decl_data ();
struct lto_file_decl_data * file_data;
- hash_table <histogram_hash> hashtable;
int j = 0;
- hashtable.create (10);
- histogram_pool = create_alloc_pool ("IPA histogram", sizeof (struct histogram_entry),
- 10);
+ hash_table<histogram_hash> hashtable (10);
while ((file_data = file_data_vec[j++]))
{
gcov_type count = streamer_read_gcov_count (ib);
int time = streamer_read_uhwi (ib);
int size = streamer_read_uhwi (ib);
- account_time_size (hashtable, histogram,
+ account_time_size (&hashtable, histogram,
count, time, size);
}
lto_destroy_simple_input_block (file_data,
ib, data, len);
}
}
- hashtable.dispose ();
histogram.qsort (cmp_counts);
}
struct ipa_propagate_frequency_data
{
+ cgraph_node *function_symbol;
bool maybe_unlikely_executed;
bool maybe_executed_once;
bool only_called_at_startup;
|| d->only_called_at_startup || d->only_called_at_exit);
edge = edge->next_caller)
{
- if (edge->caller != node)
+ if (edge->caller != d->function_symbol)
{
d->only_called_at_startup &= edge->caller->only_called_at_startup;
/* It makes sense to put main() together with the static constructors.
It will be executed for sure, but rest of functions called from
main are definitely not at startup only. */
- if (MAIN_NAME_P (DECL_NAME (edge->caller->symbol.decl)))
+ if (MAIN_NAME_P (DECL_NAME (edge->caller->decl)))
d->only_called_at_startup = 0;
d->only_called_at_exit &= edge->caller->only_called_at_exit;
}
errors can make us to push function into unlikely section even when
it is executed by the train run. Transfer the function only if all
callers are unlikely executed. */
- if (profile_info && flag_branch_probabilities
+ if (profile_info
+ && opt_for_fn (d->function_symbol->decl, flag_branch_probabilities)
+ /* Thunks are not profiled. This is more or less implementation
+ bug. */
+ && !d->function_symbol->thunk.thunk_p
&& (edge->caller->frequency != NODE_FREQUENCY_UNLIKELY_EXECUTED
|| (edge->caller->global.inlined_to
&& edge->caller->global.inlined_to->frequency
case NODE_FREQUENCY_EXECUTED_ONCE:
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, " Called by %s that is executed once\n",
- cgraph_node_name (edge->caller));
+ edge->caller->name ());
d->maybe_unlikely_executed = false;
if (inline_edge_summary (edge)->loop_depth)
{
case NODE_FREQUENCY_NORMAL:
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, " Called by %s that is normal or hot\n",
- cgraph_node_name (edge->caller));
+ edge->caller->name ());
d->maybe_unlikely_executed = false;
d->maybe_executed_once = false;
break;
{
struct cgraph_edge *e;
for (e = node->callees; e; e = e->next_callee)
- if (cgraph_maybe_hot_edge_p (e))
+ if (e->maybe_hot_p ())
return true;
else if (!e->inline_failed
&& contains_hot_call_p (e->callee))
return true;
for (e = node->indirect_calls; e; e = e->next_callee)
- if (cgraph_maybe_hot_edge_p (e))
+ if (e->maybe_hot_p ())
return true;
return false;
}
bool
ipa_propagate_frequency (struct cgraph_node *node)
{
- struct ipa_propagate_frequency_data d = {true, true, true, true};
+ struct ipa_propagate_frequency_data d = {node, true, true, true, true};
bool changed = false;
/* We can not propagate anything useful about externally visible functions
nor about virtuals. */
if (!node->local.local
- || node->symbol.alias
- || (flag_devirtualize && DECL_VIRTUAL_P (node->symbol.decl)))
+ || node->alias
+ || (opt_for_fn (node->decl, flag_devirtualize)
+ && DECL_VIRTUAL_P (node->decl)))
return false;
- gcc_assert (node->symbol.analyzed);
+ gcc_assert (node->analyzed);
if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file, "Processing frequency %s\n", cgraph_node_name (node));
+ fprintf (dump_file, "Processing frequency %s\n", node->name ());
- cgraph_for_node_and_aliases (node, ipa_propagate_frequency_1, &d, true);
+ node->call_for_symbol_and_aliases (ipa_propagate_frequency_1, &d,
+ true);
if ((d.only_called_at_startup && !d.only_called_at_exit)
&& !node->only_called_at_startup)
node->only_called_at_startup = true;
if (dump_file)
fprintf (dump_file, "Node %s promoted to only called at startup.\n",
- cgraph_node_name (node));
+ node->name ());
changed = true;
}
if ((d.only_called_at_exit && !d.only_called_at_startup)
node->only_called_at_exit = true;
if (dump_file)
fprintf (dump_file, "Node %s promoted to only called at exit.\n",
- cgraph_node_name (node));
+ node->name ());
changed = true;
}
{
if (dump_file)
fprintf (dump_file, "Node %s promoted to hot.\n",
- cgraph_node_name (node));
+ node->name ());
node->frequency = NODE_FREQUENCY_HOT;
return true;
}
{
if (dump_file)
fprintf (dump_file, "Node %s reduced to normal.\n",
- cgraph_node_name (node));
+ node->name ());
node->frequency = NODE_FREQUENCY_NORMAL;
changed = true;
}
node->frequency = NODE_FREQUENCY_UNLIKELY_EXECUTED;
if (dump_file)
fprintf (dump_file, "Node %s promoted to unlikely executed.\n",
- cgraph_node_name (node));
+ node->name ());
changed = true;
}
else if (d.maybe_executed_once && node->frequency != NODE_FREQUENCY_EXECUTED_ONCE)
node->frequency = NODE_FREQUENCY_EXECUTED_ONCE;
if (dump_file)
fprintf (dump_file, "Node %s promoted to executed once.\n",
- cgraph_node_name (node));
+ node->name ());
changed = true;
}
return changed;
gcov_type overall_time = 0, cutoff = 0, cumulated = 0, overall_size = 0;
struct cgraph_node *n,*n2;
int nindirect = 0, ncommon = 0, nunknown = 0, nuseless = 0, nconverted = 0;
+ int nmismatch = 0, nimpossible = 0;
bool node_map_initialized = false;
if (dump_file)
{
gcov_type min, cumulated_time = 0, cumulated_size = 0;
- fprintf (dump_file, "Overall time: "HOST_WIDEST_INT_PRINT_DEC"\n",
- (HOST_WIDEST_INT)overall_time);
+ fprintf (dump_file, "Overall time: %" PRId64"\n",
+ (int64_t)overall_time);
min = get_hot_bb_threshold ();
for (i = 0; i < (int)histogram.length () && histogram[i]->count >= min;
i++)
cumulated_time += histogram[i]->count * histogram[i]->time;
cumulated_size += histogram[i]->size;
}
- fprintf (dump_file, "GCOV min count: "HOST_WIDEST_INT_PRINT_DEC
+ fprintf (dump_file, "GCOV min count: %" PRId64
" Time:%3.2f%% Size:%3.2f%%\n",
- (HOST_WIDEST_INT)min,
+ (int64_t)min,
cumulated_time * 100.0 / overall_time,
cumulated_size * 100.0 / overall_size);
}
cumulated_time += histogram[i]->count * histogram[i]->time;
cumulated_size += histogram[i]->size;
}
- fprintf (dump_file, "Determined min count: "HOST_WIDEST_INT_PRINT_DEC
+ fprintf (dump_file, "Determined min count: %" PRId64
" Time:%3.2f%% Size:%3.2f%%\n",
- (HOST_WIDEST_INT)threshold,
+ (int64_t)threshold,
cumulated_time * 100.0 / overall_time,
cumulated_size * 100.0 / overall_size);
}
set_hot_bb_threshold (threshold);
}
}
- histogram.release();
- free_alloc_pool (histogram_pool);
+ histogram.release ();
+ histogram_pool.release ();
/* Produce speculative calls: we saved common traget from porfiling into
e->common_target_id. Now, at link time, we can look up corresponding
{
bool update = false;
+ if (!opt_for_fn (n->decl, flag_ipa_profile))
+ continue;
+
for (e = n->indirect_calls; e; e = e->next_callee)
{
if (n->count)
{
fprintf (dump_file, "Indirect call -> direct call from"
" other module %s/%i => %s/%i, prob %3.2f\n",
- xstrdup (cgraph_node_name (n)), n->symbol.order,
- xstrdup (cgraph_node_name (n2)), n2->symbol.order,
+ xstrdup_for_dump (n->name ()), n->order,
+ xstrdup_for_dump (n2->name ()), n2->order,
e->indirect_info->common_target_probability
/ (float)REG_BR_PROB_BASE);
}
fprintf (dump_file,
"Not speculating: probability is too low.\n");
}
- else if (!cgraph_maybe_hot_edge_p (e))
+ else if (!e->maybe_hot_p ())
{
nuseless++;
if (dump_file)
fprintf (dump_file,
"Not speculating: call is cold.\n");
}
- else if (cgraph_function_body_availability (n2)
- <= AVAIL_OVERWRITABLE
- && symtab_can_be_discarded ((symtab_node) n2))
+ else if (n2->get_availability () <= AVAIL_INTERPOSABLE
+ && n2->can_be_discarded_p ())
{
nuseless++;
if (dump_file)
"Not speculating: target is overwritable "
"and can be discarded.\n");
}
+ else if (ipa_node_params_sum && ipa_edge_args_vector
+ && !IPA_NODE_REF (n2)->descriptors.is_empty ()
+ && ipa_get_param_count (IPA_NODE_REF (n2))
+ != ipa_get_cs_argument_count (IPA_EDGE_REF (e))
+ && (ipa_get_param_count (IPA_NODE_REF (n2))
+ >= ipa_get_cs_argument_count (IPA_EDGE_REF (e))
+ || !stdarg_p (TREE_TYPE (n2->decl))))
+ {
+ nmismatch++;
+ if (dump_file)
+ fprintf (dump_file,
+ "Not speculating: "
+ "parameter count mistmatch\n");
+ }
+ else if (e->indirect_info->polymorphic
+ && !opt_for_fn (n->decl, flag_devirtualize)
+ && !possible_polymorphic_call_target_p (e, n2))
+ {
+ nimpossible++;
+ if (dump_file)
+ fprintf (dump_file,
+ "Not speculating: "
+ "function is not in the polymorphic "
+ "call target list\n");
+ }
else
{
/* Target may be overwritable, but profile says that
control flow goes to this particular implementation
of N2. Speculate on the local alias to allow inlining.
*/
- if (!symtab_can_be_discarded ((symtab_node) n2))
+ if (!n2->can_be_discarded_p ())
{
cgraph_node *alias;
- alias = cgraph (symtab_nonoverwritable_alias
- ((symtab_node)n2));
+ alias = dyn_cast<cgraph_node *> (n2->noninterposable_alias ());
if (alias)
n2 = alias;
}
nconverted++;
- cgraph_turn_edge_to_speculative
- (e, n2,
+ e->make_speculative
+ (n2,
apply_scale (e->count,
e->indirect_info->common_target_probability),
apply_scale (e->frequency,
"%i indirect calls trained.\n"
"%i (%3.2f%%) have common target.\n"
"%i (%3.2f%%) targets was not found.\n"
+ "%i (%3.2f%%) targets had parameter count mismatch.\n"
+ "%i (%3.2f%%) targets was not in polymorphic call target list.\n"
"%i (%3.2f%%) speculations seems useless.\n"
"%i (%3.2f%%) speculations produced.\n",
nindirect,
ncommon, ncommon * 100.0 / nindirect,
nunknown, nunknown * 100.0 / nindirect,
+ nmismatch, nmismatch * 100.0 / nindirect,
+ nimpossible, nimpossible * 100.0 / nindirect,
nuseless, nuseless * 100.0 / nindirect,
nconverted, nconverted * 100.0 / nindirect);
- order = XCNEWVEC (struct cgraph_node *, cgraph_n_nodes);
+ order = XCNEWVEC (struct cgraph_node *, symtab->cgraph_count);
order_pos = ipa_reverse_postorder (order);
for (i = order_pos - 1; i >= 0; i--)
{
- if (order[i]->local.local && ipa_propagate_frequency (order[i]))
+ if (order[i]->local.local
+ && opt_for_fn (order[i]->decl, flag_ipa_profile)
+ && ipa_propagate_frequency (order[i]))
{
for (e = order[i]->callees; e; e = e->next_callee)
- if (e->callee->local.local && !e->callee->symbol.aux)
+ if (e->callee->local.local && !e->callee->aux)
{
something_changed = true;
- e->callee->symbol.aux = (void *)1;
+ e->callee->aux = (void *)1;
}
}
- order[i]->symbol.aux = NULL;
+ order[i]->aux = NULL;
}
while (something_changed)
something_changed = false;
for (i = order_pos - 1; i >= 0; i--)
{
- if (order[i]->symbol.aux && ipa_propagate_frequency (order[i]))
+ if (order[i]->aux
+ && opt_for_fn (order[i]->decl, flag_ipa_profile)
+ && ipa_propagate_frequency (order[i]))
{
for (e = order[i]->callees; e; e = e->next_callee)
- if (e->callee->local.local && !e->callee->symbol.aux)
+ if (e->callee->local.local && !e->callee->aux)
{
something_changed = true;
- e->callee->symbol.aux = (void *)1;
+ e->callee->aux = (void *)1;
}
}
- order[i]->symbol.aux = NULL;
+ order[i]->aux = NULL;
}
}
free (order);
return 0;
}
-static bool
-gate_ipa_profile (void)
-{
- return flag_ipa_profile;
-}
-
namespace {
const pass_data pass_data_ipa_profile =
IPA_PASS, /* type */
"profile_estimate", /* name */
OPTGROUP_NONE, /* optinfo_flags */
- true, /* has_gate */
- true, /* has_execute */
TV_IPA_PROFILE, /* tv_id */
0, /* properties_required */
0, /* properties_provided */
class pass_ipa_profile : public ipa_opt_pass_d
{
public:
- pass_ipa_profile(gcc::context *ctxt)
- : ipa_opt_pass_d(pass_data_ipa_profile, ctxt,
- ipa_profile_generate_summary, /* generate_summary */
- ipa_profile_write_summary, /* write_summary */
- ipa_profile_read_summary, /* read_summary */
- NULL, /* write_optimization_summary */
- NULL, /* read_optimization_summary */
- NULL, /* stmt_fixup */
- 0, /* function_transform_todo_flags_start */
- NULL, /* function_transform */
- NULL) /* variable_transform */
+ pass_ipa_profile (gcc::context *ctxt)
+ : ipa_opt_pass_d (pass_data_ipa_profile, ctxt,
+ ipa_profile_generate_summary, /* generate_summary */
+ ipa_profile_write_summary, /* write_summary */
+ ipa_profile_read_summary, /* read_summary */
+ NULL, /* write_optimization_summary */
+ NULL, /* read_optimization_summary */
+ NULL, /* stmt_fixup */
+ 0, /* function_transform_todo_flags_start */
+ NULL, /* function_transform */
+ NULL) /* variable_transform */
{}
/* opt_pass methods: */
- bool gate () { return gate_ipa_profile (); }
- unsigned int execute () { return ipa_profile (); }
+ virtual bool gate (function *) { return flag_ipa_profile || in_lto_p; }
+ virtual unsigned int execute (function *) { return ipa_profile (); }
}; // class pass_ipa_profile
projects / gcc.git / blobdiff