* cgraph.c (cgraph_clone_edge): New UPDATE_ORIGINAL argument.
(cgraph_clone_node): Likewise.
* cgraph.h (cgraph_clone_edge): Update prototype.
(cgraph_clone_node): Likewise.
* ipa-inline.c (cgraph_clone_inlined_nodes): Update call of
cgraph_clone_node.
(lookup_recursive_calls): Consider profile.
(cgraph_decide_recursive_inlining): Fix updating; use new
probability argument; use profile.
* params.def (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY): New.
* tree-inline.c (copy_bb): Update clal of clone_edge.
* tree-optimize.c (tree_rest_of_compilation): UPdate cal of clone_node.
* invoke.texi (min-inline-recursive-probability): Document.
From-SVN: r102521
+2005-07-28 Jan Hubicka <jh@suse.cz>
+
+ * cgraph.c (cgraph_clone_edge): New UPDATE_ORIGINAL argument.
+ (cgraph_clone_node): Likewise.
+ * cgraph.h (cgraph_clone_edge): Update prototype.
+ (cgraph_clone_node): Likewise.
+ * ipa-inline.c (cgraph_clone_inlined_nodes): Update call of
+ cgraph_clone_node.
+ (lookup_recursive_calls): Consider profile.
+ (cgraph_decide_recursive_inlining): Fix updating; use new
+ probability argument; use profile.
+ * params.def (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY): New.
+ * tree-inline.c (copy_bb): Update clal of clone_edge.
+ * tree-optimize.c (tree_rest_of_compilation): UPdate cal of clone_node.
+
+ * invoke.texi (min-inline-recursive-probability): Document.
+
2005-07-28 Gerald Pfeifer <gerald@pfeifer.com>
* doc/install.texi (Configuration): Update Valgrind homepage.
/* Create clone of E in the node N represented by CALL_EXPR the callgraph. */
struct cgraph_edge *
cgraph_clone_edge (struct cgraph_edge *e, struct cgraph_node *n,
- tree call_stmt, int count_scale, int loop_nest)
+ tree call_stmt, int count_scale, int loop_nest,
+ bool update_original)
{
struct cgraph_edge *new;
e->loop_nest + loop_nest);
new->inline_failed = e->inline_failed;
- e->count -= new->count;
+ if (update_original)
+ e->count -= new->count;
return new;
}
/* Create node representing clone of N executed COUNT times. Decrease
- the execution counts from original node too. */
+ the execution counts from original node too.
+
+ When UPDATE_ORIGINAL is true, the counts are subtracted from the original
+ function's profile to reflect the fact that part of execution is handled
+ by node. */
struct cgraph_node *
-cgraph_clone_node (struct cgraph_node *n, gcov_type count, int loop_nest)
+cgraph_clone_node (struct cgraph_node *n, gcov_type count, int loop_nest,
+ bool update_original)
{
struct cgraph_node *new = cgraph_create_node ();
struct cgraph_edge *e;
count_scale = new->count * REG_BR_PROB_BASE / n->count;
else
count_scale = 0;
- n->count -= count;
+ if (update_original)
+ n->count -= count;
for (e = n->callees;e; e=e->next_callee)
- cgraph_clone_edge (e, new, e->call_stmt, count_scale, loop_nest);
+ cgraph_clone_edge (e, new, e->call_stmt, count_scale, loop_nest,
+ update_original);
new->next_clone = n->next_clone;
new->prev_clone = n;
struct cgraph_global_info *cgraph_global_info (tree);
struct cgraph_rtl_info *cgraph_rtl_info (tree);
const char * cgraph_node_name (struct cgraph_node *);
-struct cgraph_edge * cgraph_clone_edge (struct cgraph_edge *, struct cgraph_node *, tree, int, int);
-struct cgraph_node * cgraph_clone_node (struct cgraph_node *, gcov_type, int);
+struct cgraph_edge * cgraph_clone_edge (struct cgraph_edge *,
+ struct cgraph_node *,
+ tree, int, int, bool);
+struct cgraph_node * cgraph_clone_node (struct cgraph_node *, gcov_type,
+ int, bool);
struct cgraph_varpool_node *cgraph_varpool_node (tree);
struct cgraph_varpool_node *cgraph_varpool_node_for_asm (tree asmname);
enabled and @option{--param max-inline-recursive-depth-auto} is used. The
default value is 450.
+@item min-inline-recursive-probability
+Recursive inlining is profitable only for function having deep recursion
+in average and can hurt for function having little recursion depth by
+increasing the prologue size or complexity of function body to other
+optimizers.
+
+When profile feedback is available (see @option{-fprofile-generate}) the actual
+recursion depth can be guessed from probability that function will recurse via
+given call expression. This parameter limits inlining only to call expression
+whose probability exceeds given threshold (in percents). The default value is
+10.
+
@item inline-call-cost
Specify cost of call instruction relative to simple arithmetics operations
(having cost of 1). Increasing this cost disqualifies inlining of non-leaf
}
else if (duplicate)
{
- n = cgraph_clone_node (e->callee, e->count, e->loop_nest);
+ n = cgraph_clone_node (e->callee, e->count, e->loop_nest, true);
cgraph_redirect_edge_callee (e, n);
}
for (e = where->callees; e; e = e->next_callee)
if (e->callee == node)
{
- /* FIXME: Once counts and frequencies are available we should drive the
- order by these. For now force the order to be simple queue since
- we get order dependent on recursion depth for free by this. */
- fibheap_insert (heap, priority++, e);
+ /* When profile feedback is available, prioritize by expected number
+ of calls. Without profile feedback we maintain simple queue
+ to order candidates via recursive depths. */
+ fibheap_insert (heap,
+ !max_count ? priority++
+ : -(e->count / ((max_count + (1<<24) - 1) / (1<<24))),
+ e);
}
for (e = where->callees; e; e = e->next_callee)
if (!e->inline_failed)
{
int limit = PARAM_VALUE (PARAM_MAX_INLINE_INSNS_RECURSIVE_AUTO);
int max_depth = PARAM_VALUE (PARAM_MAX_INLINE_RECURSIVE_DEPTH_AUTO);
+ int probability = PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY);
fibheap_t heap;
struct cgraph_edge *e;
struct cgraph_node *master_clone;
cgraph_node_name (node));
/* We need original clone to copy around. */
- master_clone = cgraph_clone_node (node, 0, 1);
+ master_clone = cgraph_clone_node (node, node->count, 1, false);
master_clone->needed = true;
for (e = master_clone->callees; e; e = e->next_callee)
if (!e->inline_failed)
/* Do the inlining and update list of recursive call during process. */
while (!fibheap_empty (heap)
- && cgraph_estimate_size_after_inlining (1, node, master_clone) <= limit)
+ && (cgraph_estimate_size_after_inlining (1, node, master_clone)
+ <= limit))
{
struct cgraph_edge *curr = fibheap_extract_min (heap);
- struct cgraph_node *node;
+ struct cgraph_node *cnode;
- depth = 0;
- for (node = curr->caller;
- node; node = node->global.inlined_to)
+ depth = 1;
+ for (cnode = curr->caller;
+ cnode->global.inlined_to; cnode = cnode->callers->caller)
if (node->decl == curr->callee->decl)
depth++;
if (depth > max_depth)
- continue;
+ {
+ if (dump_file)
+ fprintf (dump_file,
+ " maxmal depth reached\n");
+ continue;
+ }
+
+ if (max_count)
+ {
+ if (!cgraph_maybe_hot_edge_p (curr))
+ {
+ if (dump_file)
+ fprintf (dump_file, " Not inlining cold call\n");
+ continue;
+ }
+ if (curr->count * 100 / node->count < probability)
+ {
+ if (dump_file)
+ fprintf (dump_file,
+ " Probability of edge is too small\n");
+ continue;
+ }
+ }
if (dump_file)
- fprintf (dump_file,
- " Inlining call of depth %i\n", depth);
+ {
+ fprintf (dump_file,
+ " Inlining call of depth %i", depth);
+ if (node->count)
+ {
+ fprintf (dump_file, " called approx. %.2f times per call",
+ (double)curr->count / node->count);
+ }
+ fprintf (dump_file, "\n");
+ }
cgraph_redirect_edge_callee (curr, master_clone);
cgraph_mark_inline_edge (curr);
lookup_recursive_calls (node, curr->callee, heap);
n++;
}
+ if (!fibheap_empty (heap) && dump_file)
+ fprintf (dump_file, " Recursive inlining growth limit met.\n");
fibheap_delete (heap);
if (dump_file)
if (node->global.inlined_to == master_clone)
cgraph_remove_node (node);
cgraph_remove_node (master_clone);
+ /* FIXME: Recursive inlining actually reduces number of calls of the
+ function. At this place we should probably walk the function and
+ inline clones and compensate the counts accordingly. This probably
+ doesn't matter much in practice. */
return true;
}
"The maximum depth of recursive inlining for non-inline functions",
8, 0, 0)
+DEFPARAM (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY,
+ "min-inline-recursive-probability",
+ "Inline recursively only when the probability of call being executed exceeds the parameter",
+ 10, 0, 0)
+
/* Limit the number of expansions created by the variable expansion
optimization to avoid register pressure. */
DEFPARAM (PARAM_MAX_VARIABLE_EXPANSIONS,
edge = cgraph_edge (id->current_node, orig_stmt);
if (edge)
cgraph_clone_edge (edge, id->node, stmt,
- REG_BR_PROB_BASE, 1);
+ REG_BR_PROB_BASE, 1, true);
}
}
/* If you think we can abort here, you are wrong.
{
struct cgraph_edge *e;
- saved_node = cgraph_clone_node (node, node->count, 1);
+ saved_node = cgraph_clone_node (node, node->count, 1, false);
for (e = saved_node->callees; e; e = e->next_callee)
if (!e->inline_failed)
cgraph_clone_inlined_nodes (e, true);
projects / gcc.git / commitdiff