2 Copyright (C) 2003-2020 Free Software Foundation, Inc.
3 Contributed by Jan Hubicka
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
21 /* This module provide facilities for cloning functions. I.e. creating
22 new functions based on existing functions with simple modifications,
23 such as replacement of parameters.
25 To allow whole program optimization without actual presence of function
26 bodies, an additional infrastructure is provided for so-called virtual
29 A virtual clone in the callgraph is a function that has no
30 associated body, just a description of how to create its body based
31 on a different function (which itself may be a virtual clone).
33 The description of function modifications includes adjustments to
34 the function's signature (which allows, for example, removing or
35 adding function arguments), substitutions to perform on the
36 function body, and, for inlined functions, a pointer to the
37 function that it will be inlined into.
39 It is also possible to redirect any edge of the callgraph from a
40 function to its virtual clone. This implies updating of the call
41 site to adjust for the new function signature.
43 Most of the transformations performed by inter-procedural
44 optimizations can be represented via virtual clones. For
45 instance, a constant propagation pass can produce a virtual clone
46 of the function which replaces one of its arguments by a
47 constant. The inliner can represent its decisions by producing a
48 clone of a function whose body will be later integrated into
51 Using virtual clones, the program can be easily updated
52 during the Execute stage, solving most of pass interactions
53 problems that would otherwise occur during Transform.
55 Virtual clones are later materialized in the LTRANS stage and
56 turned into real functions. Passes executed after the virtual
57 clone were introduced also perform their Transform stage
58 on new functions, so for a pass there is no significant
59 difference between operating on a real function or a virtual
60 clone introduced before its Execute stage.
62 Optimization passes then work on virtual clones introduced before
63 their Execute stage as if they were real functions. The
64 only difference is that clones are not visible during the
65 Generate Summary stage. */
69 #include "coretypes.h"
75 #include "stringpool.h"
77 #include "lto-streamer.h"
80 #include "tree-inline.h"
82 #include "gimple-pretty-print.h"
83 #include "alloc-pool.h"
84 #include "symbol-summary.h"
87 #include "ipa-fnsummary.h"
89 /* Create clone of edge in the node N represented by CALL_EXPR
93 cgraph_edge::clone (cgraph_node
*n
, gcall
*call_stmt
, unsigned stmt_uid
,
94 profile_count num
, profile_count den
,
97 cgraph_edge
*new_edge
;
98 profile_count::adjust_for_ipa_scaling (&num
, &den
);
99 profile_count prof_count
= count
.apply_scale (num
, den
);
101 if (indirect_unknown_callee
)
105 if (call_stmt
&& (decl
= gimple_call_fndecl (call_stmt
))
106 /* When the call is speculative, we need to resolve it
107 via cgraph_resolve_speculation and not here. */
110 cgraph_node
*callee
= cgraph_node::get (decl
);
111 gcc_checking_assert (callee
);
112 new_edge
= n
->create_edge (callee
, call_stmt
, prof_count
, true);
116 new_edge
= n
->create_indirect_edge (call_stmt
,
117 indirect_info
->ecf_flags
,
119 *new_edge
->indirect_info
= *indirect_info
;
124 new_edge
= n
->create_edge (callee
, call_stmt
, prof_count
, true);
127 new_edge
->indirect_info
128 = ggc_cleared_alloc
<cgraph_indirect_call_info
> ();
129 *new_edge
->indirect_info
= *indirect_info
;
133 new_edge
->inline_failed
= inline_failed
;
134 new_edge
->indirect_inlining_edge
= indirect_inlining_edge
;
136 new_edge
->lto_stmt_uid
= stmt_uid
;
137 new_edge
->speculative_id
= speculative_id
;
138 /* Clone flags that depend on call_stmt availability manually. */
139 new_edge
->can_throw_external
= can_throw_external
;
140 new_edge
->call_stmt_cannot_inline_p
= call_stmt_cannot_inline_p
;
141 new_edge
->speculative
= speculative
;
142 new_edge
->in_polymorphic_cdtor
= in_polymorphic_cdtor
;
144 /* Update IPA profile. Local profiles need no updating in original. */
146 count
= count
.combine_with_ipa_count_within (count
.ipa ()
147 - new_edge
->count
.ipa (),
149 symtab
->call_edge_duplication_hooks (this, new_edge
);
153 /* Set flags of NEW_NODE and its decl. NEW_NODE is a newly created private
154 clone or its thunk. */
157 set_new_clone_decl_and_node_flags (cgraph_node
*new_node
)
159 DECL_EXTERNAL (new_node
->decl
) = 0;
160 TREE_PUBLIC (new_node
->decl
) = 0;
161 DECL_COMDAT (new_node
->decl
) = 0;
162 DECL_WEAK (new_node
->decl
) = 0;
163 DECL_VIRTUAL_P (new_node
->decl
) = 0;
164 DECL_STATIC_CONSTRUCTOR (new_node
->decl
) = 0;
165 DECL_STATIC_DESTRUCTOR (new_node
->decl
) = 0;
166 DECL_SET_IS_OPERATOR_NEW (new_node
->decl
, 0);
167 DECL_SET_IS_OPERATOR_DELETE (new_node
->decl
, 0);
169 new_node
->externally_visible
= 0;
171 new_node
->lowered
= true;
174 /* Duplicate thunk THUNK if necessary but make it to refer to NODE.
175 ARGS_TO_SKIP, if non-NULL, determines which parameters should be omitted.
176 Function can return NODE if no thunk is necessary, which can happen when
177 thunk is this_adjusting but we are removing this parameter. */
180 duplicate_thunk_for_node (cgraph_node
*thunk
, cgraph_node
*node
)
182 cgraph_node
*new_thunk
, *thunk_of
;
183 thunk_of
= thunk
->callees
->callee
->ultimate_alias_target ();
185 if (thunk_of
->thunk
.thunk_p
)
186 node
= duplicate_thunk_for_node (thunk_of
, node
);
188 if (!DECL_ARGUMENTS (thunk
->decl
))
189 thunk
->get_untransformed_body ();
192 for (cs
= node
->callers
; cs
; cs
= cs
->next_caller
)
193 if (cs
->caller
->thunk
.thunk_p
194 && cs
->caller
->thunk
.fixed_offset
== thunk
->thunk
.fixed_offset
195 && cs
->caller
->thunk
.virtual_value
== thunk
->thunk
.virtual_value
196 && cs
->caller
->thunk
.indirect_offset
== thunk
->thunk
.indirect_offset
197 && cs
->caller
->thunk
.this_adjusting
== thunk
->thunk
.this_adjusting
198 && cs
->caller
->thunk
.virtual_offset_p
== thunk
->thunk
.virtual_offset_p
)
202 if (node
->clone
.param_adjustments
)
204 /* We do not need to duplicate this_adjusting thunks if we have removed
206 if (thunk
->thunk
.this_adjusting
207 && !node
->clone
.param_adjustments
->first_param_intact_p ())
210 new_decl
= copy_node (thunk
->decl
);
211 ipa_param_body_adjustments
body_adj (node
->clone
.param_adjustments
,
213 body_adj
.modify_formal_parameters ();
216 new_decl
= copy_node (thunk
->decl
);
218 gcc_checking_assert (!DECL_STRUCT_FUNCTION (new_decl
));
219 gcc_checking_assert (!DECL_INITIAL (new_decl
));
220 gcc_checking_assert (!DECL_RESULT (new_decl
));
221 gcc_checking_assert (!DECL_RTL_SET_P (new_decl
));
223 DECL_NAME (new_decl
) = clone_function_name_numbered (thunk
->decl
,
225 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
227 /* We need to force DECL_IGNORED_P because the new thunk is created after
228 early debug was run. */
229 DECL_IGNORED_P (new_decl
) = 1;
231 new_thunk
= cgraph_node::create (new_decl
);
232 set_new_clone_decl_and_node_flags (new_thunk
);
233 new_thunk
->definition
= true;
234 new_thunk
->can_change_signature
= node
->can_change_signature
;
235 new_thunk
->thunk
= thunk
->thunk
;
236 new_thunk
->unique_name
= in_lto_p
;
237 new_thunk
->former_clone_of
= thunk
->decl
;
238 new_thunk
->clone
.param_adjustments
= node
->clone
.param_adjustments
;
239 new_thunk
->unit_id
= thunk
->unit_id
;
240 new_thunk
->merged_comdat
= thunk
->merged_comdat
;
241 new_thunk
->merged_extern_inline
= thunk
->merged_extern_inline
;
243 cgraph_edge
*e
= new_thunk
->create_edge (node
, NULL
, new_thunk
->count
);
244 symtab
->call_edge_duplication_hooks (thunk
->callees
, e
);
245 symtab
->call_cgraph_duplication_hooks (thunk
, new_thunk
);
249 /* If E does not lead to a thunk, simply redirect it to N. Otherwise create
250 one or more equivalent thunks for N and redirect E to the first in the
251 chain. Note that it is then necessary to call
252 n->expand_all_artificial_thunks once all callers are redirected. */
255 cgraph_edge::redirect_callee_duplicating_thunks (cgraph_node
*n
)
257 cgraph_node
*orig_to
= callee
->ultimate_alias_target ();
258 if (orig_to
->thunk
.thunk_p
)
259 n
= duplicate_thunk_for_node (orig_to
, n
);
264 /* Call expand_thunk on all callers that are thunks and if analyze those nodes
265 that were expanded. */
268 cgraph_node::expand_all_artificial_thunks ()
271 for (e
= callers
; e
;)
272 if (e
->caller
->thunk
.thunk_p
)
274 cgraph_node
*thunk
= e
->caller
;
277 if (thunk
->expand_thunk (false, false))
279 thunk
->thunk
.thunk_p
= false;
281 ipa_analyze_node (thunk
);
282 inline_analyze_function (thunk
);
284 thunk
->expand_all_artificial_thunks ();
291 dump_callgraph_transformation (const cgraph_node
*original
,
292 const cgraph_node
*clone
,
295 if (symtab
->ipa_clones_dump_file
)
297 fprintf (symtab
->ipa_clones_dump_file
,
298 "Callgraph clone;%s;%d;%s;%d;%d;%s;%d;%s;%d;%d;%s\n",
299 original
->asm_name (), original
->order
,
300 DECL_SOURCE_FILE (original
->decl
),
301 DECL_SOURCE_LINE (original
->decl
),
302 DECL_SOURCE_COLUMN (original
->decl
), clone
->asm_name (),
303 clone
->order
, DECL_SOURCE_FILE (clone
->decl
),
304 DECL_SOURCE_LINE (clone
->decl
), DECL_SOURCE_COLUMN (clone
->decl
),
307 symtab
->cloned_nodes
.add (original
);
308 symtab
->cloned_nodes
.add (clone
);
312 /* Turn profile of N to local profile. */
315 localize_profile (cgraph_node
*n
)
317 n
->count
= n
->count
.guessed_local ();
318 for (cgraph_edge
*e
= n
->callees
; e
; e
=e
->next_callee
)
320 e
->count
= e
->count
.guessed_local ();
321 if (!e
->inline_failed
)
322 localize_profile (e
->callee
);
324 for (cgraph_edge
*e
= n
->indirect_calls
; e
; e
=e
->next_callee
)
325 e
->count
= e
->count
.guessed_local ();
328 /* Create node representing clone of N executed COUNT times. Decrease
329 the execution counts from original node too.
330 The new clone will have decl set to DECL that may or may not be the same
333 When UPDATE_ORIGINAL is true, the counts are subtracted from the original
334 function's profile to reflect the fact that part of execution is handled
336 When CALL_DUPLICATION_HOOK is true, the ipa passes are acknowledged about
337 the new clone. Otherwise the caller is responsible for doing so later.
339 If the new node is being inlined into another one, NEW_INLINED_TO should be
340 the outline function the new one is (even indirectly) inlined to. All hooks
341 will see this in node's inlined_to, when invoked. Can be NULL if the
344 If PARAM_ADJUSTMENTS is non-NULL, the parameter manipulation information
345 will be overwritten by the new structure. Otherwise the new node will
346 share parameter manipulation information with the original node. */
349 cgraph_node::create_clone (tree new_decl
, profile_count prof_count
,
350 bool update_original
,
351 vec
<cgraph_edge
*> redirect_callers
,
352 bool call_duplication_hook
,
353 cgraph_node
*new_inlined_to
,
354 ipa_param_adjustments
*param_adjustments
,
357 cgraph_node
*new_node
= symtab
->create_empty ();
360 profile_count old_count
= count
;
361 bool nonzero
= count
.ipa ().nonzero_p ();
364 dump_callgraph_transformation (this, new_inlined_to
, "inlining to");
366 /* When inlining we scale precisely to prof_count, when cloning we can
367 preserve local profile. */
369 prof_count
= count
.combine_with_ipa_count (prof_count
);
370 new_node
->count
= prof_count
;
372 /* Update IPA profile. Local profiles need no updating in original. */
376 count
= count
.combine_with_ipa_count_within (count
.ipa ()
380 count
= count
.combine_with_ipa_count (count
.ipa () - prof_count
.ipa ());
382 new_node
->decl
= new_decl
;
383 new_node
->register_symbol ();
384 new_node
->origin
= origin
;
385 new_node
->lto_file_data
= lto_file_data
;
386 if (new_node
->origin
)
388 new_node
->next_nested
= new_node
->origin
->nested
;
389 new_node
->origin
->nested
= new_node
;
391 new_node
->analyzed
= analyzed
;
392 new_node
->definition
= definition
;
393 new_node
->versionable
= versionable
;
394 new_node
->can_change_signature
= can_change_signature
;
395 new_node
->redefined_extern_inline
= redefined_extern_inline
;
396 new_node
->tm_may_enter_irr
= tm_may_enter_irr
;
397 new_node
->externally_visible
= false;
398 new_node
->no_reorder
= no_reorder
;
399 new_node
->local
= true;
400 new_node
->inlined_to
= new_inlined_to
;
402 new_node
->frequency
= frequency
;
403 new_node
->tp_first_run
= tp_first_run
;
404 new_node
->tm_clone
= tm_clone
;
405 new_node
->icf_merged
= icf_merged
;
406 new_node
->merged_comdat
= merged_comdat
;
407 new_node
->thunk
= thunk
;
408 new_node
->unit_id
= unit_id
;
409 new_node
->merged_comdat
= merged_comdat
;
410 new_node
->merged_extern_inline
= merged_extern_inline
;
412 if (param_adjustments
)
413 new_node
->clone
.param_adjustments
= param_adjustments
;
415 new_node
->clone
.param_adjustments
= clone
.param_adjustments
;
416 new_node
->clone
.tree_map
= NULL
;
417 new_node
->clone
.performed_splits
= vec_safe_copy (clone
.performed_splits
);
418 new_node
->split_part
= split_part
;
420 FOR_EACH_VEC_ELT (redirect_callers
, i
, e
)
422 /* Redirect calls to the old version node to point to its new
423 version. The only exception is when the edge was proved to
424 be unreachable during the cloning procedure. */
426 || !fndecl_built_in_p (e
->callee
->decl
, BUILT_IN_UNREACHABLE
))
427 e
->redirect_callee_duplicating_thunks (new_node
);
429 new_node
->expand_all_artificial_thunks ();
431 for (e
= callees
;e
; e
=e
->next_callee
)
432 e
->clone (new_node
, e
->call_stmt
, e
->lto_stmt_uid
, new_node
->count
, old_count
,
435 for (e
= indirect_calls
; e
; e
= e
->next_callee
)
436 e
->clone (new_node
, e
->call_stmt
, e
->lto_stmt_uid
,
437 new_node
->count
, old_count
, update_original
);
438 new_node
->clone_references (this);
440 new_node
->next_sibling_clone
= clones
;
442 clones
->prev_sibling_clone
= new_node
;
444 new_node
->clone_of
= this;
446 if (call_duplication_hook
)
447 symtab
->call_cgraph_duplication_hooks (this, new_node
);
448 /* With partial train run we do not want to assume that original's
449 count is zero whenever we redurect all executed edges to clone.
450 Simply drop profile to local one in this case. */
452 && opt_for_fn (decl
, flag_profile_partial_training
)
455 && !count
.ipa ().nonzero_p ()
457 localize_profile (this);
460 dump_callgraph_transformation (this, new_node
, suffix
);
465 static GTY(()) hash_map
<const char *, unsigned> *clone_fn_ids
;
467 /* Return a new assembler name for a clone of decl named NAME. Apart
468 from the string SUFFIX, the new name will end with a unique (for
469 each NAME) unspecified number. If clone numbering is not needed
470 then the two argument clone_function_name should be used instead.
471 Should not be called directly except for by
472 lto-partition.c:privatize_symbol_name_1. */
475 clone_function_name_numbered (const char *name
, const char *suffix
)
477 /* Initialize the function->counter mapping the first time it's
480 clone_fn_ids
= hash_map
<const char *, unsigned int>::create_ggc (64);
481 unsigned int &suffix_counter
= clone_fn_ids
->get_or_insert (
482 IDENTIFIER_POINTER (get_identifier (name
)));
483 return clone_function_name (name
, suffix
, suffix_counter
++);
486 /* Return a new assembler name for a clone of DECL. Apart from string
487 SUFFIX, the new name will end with a unique (for each DECL
488 assembler name) unspecified number. If clone numbering is not
489 needed then the two argument clone_function_name should be used
493 clone_function_name_numbered (tree decl
, const char *suffix
)
495 tree name
= DECL_ASSEMBLER_NAME (decl
);
496 return clone_function_name_numbered (IDENTIFIER_POINTER (name
),
500 /* Return a new assembler name for a clone of decl named NAME. Apart
501 from the string SUFFIX, the new name will end with the specified
502 NUMBER. If clone numbering is not needed then the two argument
503 clone_function_name should be used instead. */
506 clone_function_name (const char *name
, const char *suffix
,
507 unsigned long number
)
509 size_t len
= strlen (name
);
510 char *tmp_name
, *prefix
;
512 prefix
= XALLOCAVEC (char, len
+ strlen (suffix
) + 2);
513 memcpy (prefix
, name
, len
);
514 strcpy (prefix
+ len
+ 1, suffix
);
515 prefix
[len
] = symbol_table::symbol_suffix_separator ();
516 ASM_FORMAT_PRIVATE_NAME (tmp_name
, prefix
, number
);
517 return get_identifier (tmp_name
);
520 /* Return a new assembler name for a clone of DECL. Apart from the
521 string SUFFIX, the new name will end with the specified NUMBER. If
522 clone numbering is not needed then the two argument
523 clone_function_name should be used instead. */
526 clone_function_name (tree decl
, const char *suffix
,
527 unsigned long number
)
529 return clone_function_name (
530 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl
)), suffix
, number
);
533 /* Return a new assembler name ending with the string SUFFIX for a
537 clone_function_name (tree decl
, const char *suffix
)
539 tree identifier
= DECL_ASSEMBLER_NAME (decl
);
540 /* For consistency this needs to behave the same way as
541 ASM_FORMAT_PRIVATE_NAME does, but without the final number
543 char *separator
= XALLOCAVEC (char, 2);
544 separator
[0] = symbol_table::symbol_suffix_separator ();
546 #if defined (NO_DOT_IN_LABEL) && defined (NO_DOLLAR_IN_LABEL)
547 const char *prefix
= "__";
549 const char *prefix
= "";
551 char *result
= ACONCAT ((prefix
,
552 IDENTIFIER_POINTER (identifier
),
556 return get_identifier (result
);
560 /* Create callgraph node clone with new declaration. The actual body will be
561 copied later at compilation stage. The name of the new clone will be
562 constructed from the name of the original node, SUFFIX and NUM_SUFFIX.
564 TODO: after merging in ipa-sra use function call notes instead of args_to_skip
568 cgraph_node::create_virtual_clone (vec
<cgraph_edge
*> redirect_callers
,
569 vec
<ipa_replace_map
*, va_gc
> *tree_map
,
570 ipa_param_adjustments
*param_adjustments
,
571 const char * suffix
, unsigned num_suffix
)
573 tree old_decl
= decl
;
574 cgraph_node
*new_node
= NULL
;
577 ipa_replace_map
*map
;
580 gcc_checking_assert (versionable
);
581 /* TODO: It would be nice if we could recognize that param_adjustments do not
582 actually perform any changes, but at the moment let's require it simply
584 gcc_assert (can_change_signature
|| !param_adjustments
);
586 /* Make a new FUNCTION_DECL tree node */
587 if (!param_adjustments
)
588 new_decl
= copy_node (old_decl
);
590 new_decl
= param_adjustments
->adjust_decl (old_decl
);
592 /* These pointers represent function body and will be populated only when clone
594 gcc_assert (new_decl
!= old_decl
);
595 DECL_STRUCT_FUNCTION (new_decl
) = NULL
;
596 DECL_ARGUMENTS (new_decl
) = NULL
;
597 DECL_INITIAL (new_decl
) = NULL
;
598 DECL_RESULT (new_decl
) = NULL
;
599 /* We cannot do DECL_RESULT (new_decl) = NULL; here because of LTO partitioning
600 sometimes storing only clone decl instead of original. */
602 /* Generate a new name for the new version. */
603 len
= IDENTIFIER_LENGTH (DECL_NAME (old_decl
));
604 name
= XALLOCAVEC (char, len
+ strlen (suffix
) + 2);
605 memcpy (name
, IDENTIFIER_POINTER (DECL_NAME (old_decl
)), len
);
606 strcpy (name
+ len
+ 1, suffix
);
608 DECL_NAME (new_decl
) = get_identifier (name
);
609 SET_DECL_ASSEMBLER_NAME (new_decl
,
610 clone_function_name (old_decl
, suffix
, num_suffix
));
611 SET_DECL_RTL (new_decl
, NULL
);
613 new_node
= create_clone (new_decl
, count
, false,
614 redirect_callers
, false, NULL
, param_adjustments
,
617 /* Update the properties.
618 Make clone visible only within this translation unit. Make sure
619 that is not weak also.
620 ??? We cannot use COMDAT linkage because there is no
621 ABI support for this. */
622 set_new_clone_decl_and_node_flags (new_node
);
623 new_node
->ipcp_clone
= ipcp_clone
;
624 new_node
->clone
.tree_map
= tree_map
;
625 if (!implicit_section
)
626 new_node
->set_section (get_section ());
628 /* Clones of global symbols or symbols with unique names are unique. */
629 if ((TREE_PUBLIC (old_decl
)
630 && !DECL_EXTERNAL (old_decl
)
631 && !DECL_WEAK (old_decl
)
632 && !DECL_COMDAT (old_decl
))
634 new_node
->unique_name
= true;
635 FOR_EACH_VEC_SAFE_ELT (tree_map
, i
, map
)
636 new_node
->maybe_create_reference (map
->new_tree
, NULL
);
638 if (ipa_transforms_to_apply
.exists ())
639 new_node
->ipa_transforms_to_apply
640 = ipa_transforms_to_apply
.copy ();
642 symtab
->call_cgraph_duplication_hooks (this, new_node
);
647 /* callgraph node being removed from symbol table; see if its entry can be
648 replaced by other inline clone. */
650 cgraph_node::find_replacement (void)
652 cgraph_node
*next_inline_clone
, *replacement
;
654 for (next_inline_clone
= clones
;
656 && next_inline_clone
->decl
!= decl
;
657 next_inline_clone
= next_inline_clone
->next_sibling_clone
)
660 /* If there is inline clone of the node being removed, we need
661 to put it into the position of removed node and reorganize all
662 other clones to be based on it. */
663 if (next_inline_clone
)
666 cgraph_node
*new_clones
;
668 replacement
= next_inline_clone
;
670 /* Unlink inline clone from the list of clones of removed node. */
671 if (next_inline_clone
->next_sibling_clone
)
672 next_inline_clone
->next_sibling_clone
->prev_sibling_clone
673 = next_inline_clone
->prev_sibling_clone
;
674 if (next_inline_clone
->prev_sibling_clone
)
676 gcc_assert (clones
!= next_inline_clone
);
677 next_inline_clone
->prev_sibling_clone
->next_sibling_clone
678 = next_inline_clone
->next_sibling_clone
;
682 gcc_assert (clones
== next_inline_clone
);
683 clones
= next_inline_clone
->next_sibling_clone
;
689 /* Copy clone info. */
690 next_inline_clone
->clone
= clone
;
692 /* Now place it into clone tree at same level at NODE. */
693 next_inline_clone
->clone_of
= clone_of
;
694 next_inline_clone
->prev_sibling_clone
= NULL
;
695 next_inline_clone
->next_sibling_clone
= NULL
;
698 if (clone_of
->clones
)
699 clone_of
->clones
->prev_sibling_clone
= next_inline_clone
;
700 next_inline_clone
->next_sibling_clone
= clone_of
->clones
;
701 clone_of
->clones
= next_inline_clone
;
704 /* Merge the clone list. */
707 if (!next_inline_clone
->clones
)
708 next_inline_clone
->clones
= new_clones
;
711 n
= next_inline_clone
->clones
;
712 while (n
->next_sibling_clone
)
713 n
= n
->next_sibling_clone
;
714 n
->next_sibling_clone
= new_clones
;
715 new_clones
->prev_sibling_clone
= n
;
719 /* Update clone_of pointers. */
723 n
->clone_of
= next_inline_clone
;
724 n
= n
->next_sibling_clone
;
727 /* Update order in order to be able to find a LTO section
728 with function body. */
729 replacement
->order
= order
;
737 /* Like cgraph_set_call_stmt but walk the clone tree and update all
738 clones sharing the same function body.
739 When WHOLE_SPECULATIVE_EDGES is true, all three components of
740 speculative edge gets updated. Otherwise we update only direct
744 cgraph_node::set_call_stmt_including_clones (gimple
*old_stmt
,
746 bool update_speculative
)
749 cgraph_edge
*master_edge
= get_edge (old_stmt
);
752 cgraph_edge::set_call_stmt (master_edge
, new_stmt
, update_speculative
);
758 cgraph_edge
*edge
= node
->get_edge (old_stmt
);
761 edge
= cgraph_edge::set_call_stmt (edge
, new_stmt
,
763 /* If UPDATE_SPECULATIVE is false, it means that we are turning
764 speculative call into a real code sequence. Update the
766 if (edge
->speculative
&& !update_speculative
)
768 cgraph_edge
*indirect
= edge
->speculative_call_indirect_edge ();
770 for (cgraph_edge
*next
, *direct
771 = edge
->first_speculative_call_target ();
775 next
= direct
->next_speculative_call_target ();
776 direct
->speculative_call_target_ref ()->speculative
= false;
777 direct
->speculative
= false;
779 indirect
->speculative
= false;
784 else if (node
->next_sibling_clone
)
785 node
= node
->next_sibling_clone
;
788 while (node
!= this && !node
->next_sibling_clone
)
789 node
= node
->clone_of
;
791 node
= node
->next_sibling_clone
;
796 /* Like cgraph_create_edge walk the clone tree and update all clones sharing
797 same function body. If clones already have edge for OLD_STMT; only
798 update the edge same way as cgraph_set_call_stmt_including_clones does.
800 TODO: COUNT and LOOP_DEPTH should be properly distributed based on relative
801 frequencies of the clones. */
804 cgraph_node::create_edge_including_clones (cgraph_node
*callee
,
805 gimple
*old_stmt
, gcall
*stmt
,
807 cgraph_inline_failed_t reason
)
811 if (!get_edge (stmt
))
813 cgraph_edge
*edge
= create_edge (callee
, stmt
, count
);
814 edge
->inline_failed
= reason
;
820 /* Thunk clones do not get updated while copying inline function body. */
821 if (!node
->thunk
.thunk_p
)
823 cgraph_edge
*edge
= node
->get_edge (old_stmt
);
825 /* It is possible that clones already contain the edge while
826 master didn't. Either we promoted indirect call into direct
827 call in the clone or we are processing clones of unreachable
828 master where edges has been removed. */
830 edge
= cgraph_edge::set_call_stmt (edge
, stmt
);
831 else if (! node
->get_edge (stmt
))
833 edge
= node
->create_edge (callee
, stmt
, count
);
834 edge
->inline_failed
= reason
;
839 else if (node
->next_sibling_clone
)
840 node
= node
->next_sibling_clone
;
843 while (node
!= this && !node
->next_sibling_clone
)
844 node
= node
->clone_of
;
846 node
= node
->next_sibling_clone
;
851 /* Remove the node from cgraph and all inline clones inlined into it.
852 Skip however removal of FORBIDDEN_NODE and return true if it needs to be
853 removed. This allows to call the function from outer loop walking clone
857 cgraph_node::remove_symbol_and_inline_clones (cgraph_node
*forbidden_node
)
859 cgraph_edge
*e
, *next
;
862 if (this == forbidden_node
)
864 cgraph_edge::remove (callers
);
867 for (e
= callees
; e
; e
= next
)
869 next
= e
->next_callee
;
870 if (!e
->inline_failed
)
871 found
|= e
->callee
->remove_symbol_and_inline_clones (forbidden_node
);
877 /* The edges representing the callers of the NEW_VERSION node were
878 fixed by cgraph_function_versioning (), now the call_expr in their
879 respective tree code should be updated to call the NEW_VERSION. */
882 update_call_expr (cgraph_node
*new_version
)
886 gcc_assert (new_version
);
888 /* Update the call expr on the edges to call the new version. */
889 for (e
= new_version
->callers
; e
; e
= e
->next_caller
)
891 function
*inner_function
= DECL_STRUCT_FUNCTION (e
->caller
->decl
);
892 gimple_call_set_fndecl (e
->call_stmt
, new_version
->decl
);
893 maybe_clean_eh_stmt_fn (inner_function
, e
->call_stmt
);
898 /* Create a new cgraph node which is the new version of
899 callgraph node. REDIRECT_CALLERS holds the callers
900 edges which should be redirected to point to
901 NEW_VERSION. ALL the callees edges of the node
902 are cloned to the new version node. Return the new
905 If non-NULL BLOCK_TO_COPY determine what basic blocks
906 was copied to prevent duplications of calls that are dead
910 cgraph_node::create_version_clone (tree new_decl
,
911 vec
<cgraph_edge
*> redirect_callers
,
915 cgraph_node
*new_version
;
919 new_version
= cgraph_node::create (new_decl
);
921 new_version
->analyzed
= analyzed
;
922 new_version
->definition
= definition
;
923 new_version
->local
= local
;
924 new_version
->externally_visible
= false;
925 new_version
->no_reorder
= no_reorder
;
926 new_version
->local
= new_version
->definition
;
927 new_version
->inlined_to
= inlined_to
;
928 new_version
->rtl
= rtl
;
929 new_version
->count
= count
;
930 new_version
->unit_id
= unit_id
;
931 new_version
->merged_comdat
= merged_comdat
;
932 new_version
->merged_extern_inline
= merged_extern_inline
;
934 for (e
= callees
; e
; e
=e
->next_callee
)
936 || bitmap_bit_p (bbs_to_copy
, gimple_bb (e
->call_stmt
)->index
))
937 e
->clone (new_version
, e
->call_stmt
,
938 e
->lto_stmt_uid
, count
, count
,
940 for (e
= indirect_calls
; e
; e
=e
->next_callee
)
942 || bitmap_bit_p (bbs_to_copy
, gimple_bb (e
->call_stmt
)->index
))
943 e
->clone (new_version
, e
->call_stmt
,
944 e
->lto_stmt_uid
, count
, count
,
946 FOR_EACH_VEC_ELT (redirect_callers
, i
, e
)
948 /* Redirect calls to the old version node to point to its new
950 e
->redirect_callee (new_version
);
953 dump_callgraph_transformation (this, new_version
, suffix
);
958 /* Perform function versioning.
959 Function versioning includes copying of the tree and
960 a callgraph update (creating a new cgraph node and updating
961 its callees and callers).
963 REDIRECT_CALLERS varray includes the edges to be redirected
966 TREE_MAP is a mapping of tree nodes we want to replace with
967 new ones (according to results of prior analysis).
969 If non-NULL ARGS_TO_SKIP determine function parameters to remove
971 If SKIP_RETURN is true, the new version will return void.
972 If non-NULL BLOCK_TO_COPY determine what basic blocks to copy.
973 If non_NULL NEW_ENTRY determine new entry BB of the clone.
975 If TARGET_ATTRIBUTES is non-null, when creating a new declaration,
976 add the attributes to DECL_ATTRIBUTES. And call valid_attribute_p
977 that will promote value of the attribute DECL_FUNCTION_SPECIFIC_TARGET
980 Return the new version's cgraph node. */
983 cgraph_node::create_version_clone_with_body
984 (vec
<cgraph_edge
*> redirect_callers
,
985 vec
<ipa_replace_map
*, va_gc
> *tree_map
,
986 ipa_param_adjustments
*param_adjustments
,
987 bitmap bbs_to_copy
, basic_block new_entry_block
, const char *suffix
,
988 tree target_attributes
)
990 tree old_decl
= decl
;
991 cgraph_node
*new_version_node
= NULL
;
994 if (!tree_versionable_function_p (old_decl
))
997 /* TODO: Restore an assert that we do not change signature if
998 can_change_signature is false. We cannot just check that
999 param_adjustments is NULL because unfortunately ipa-split removes return
1000 values from such functions. */
1002 /* Make a new FUNCTION_DECL tree node for the new version. */
1003 if (param_adjustments
)
1004 new_decl
= param_adjustments
->adjust_decl (old_decl
);
1006 new_decl
= copy_node (old_decl
);
1008 /* Generate a new name for the new version. */
1009 DECL_NAME (new_decl
) = clone_function_name_numbered (old_decl
, suffix
);
1010 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
1011 SET_DECL_RTL (new_decl
, NULL
);
1013 DECL_VIRTUAL_P (new_decl
) = 0;
1015 if (target_attributes
)
1017 DECL_ATTRIBUTES (new_decl
) = target_attributes
;
1019 location_t saved_loc
= input_location
;
1020 tree v
= TREE_VALUE (target_attributes
);
1021 input_location
= DECL_SOURCE_LOCATION (new_decl
);
1022 bool r
= targetm
.target_option
.valid_attribute_p (new_decl
, NULL
, v
, 1);
1023 input_location
= saved_loc
;
1028 /* When the old decl was a con-/destructor make sure the clone isn't. */
1029 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
1030 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
1031 DECL_SET_IS_OPERATOR_NEW (new_decl
, 0);
1032 DECL_SET_IS_OPERATOR_DELETE (new_decl
, 0);
1034 /* Create the new version's call-graph node.
1035 and update the edges of the new node. */
1036 new_version_node
= create_version_clone (new_decl
, redirect_callers
,
1037 bbs_to_copy
, suffix
);
1039 if (ipa_transforms_to_apply
.exists ())
1040 new_version_node
->ipa_transforms_to_apply
1041 = ipa_transforms_to_apply
.copy ();
1042 /* Copy the OLD_VERSION_NODE function tree to the new version. */
1043 tree_function_versioning (old_decl
, new_decl
, tree_map
, param_adjustments
,
1044 false, bbs_to_copy
, new_entry_block
);
1046 /* Update the new version's properties.
1047 Make The new version visible only within this translation unit. Make sure
1048 that is not weak also.
1049 ??? We cannot use COMDAT linkage because there is no
1050 ABI support for this. */
1051 new_version_node
->make_decl_local ();
1052 DECL_VIRTUAL_P (new_version_node
->decl
) = 0;
1053 new_version_node
->externally_visible
= 0;
1054 new_version_node
->local
= 1;
1055 new_version_node
->lowered
= true;
1056 if (!implicit_section
)
1057 new_version_node
->set_section (get_section ());
1058 /* Clones of global symbols or symbols with unique names are unique. */
1059 if ((TREE_PUBLIC (old_decl
)
1060 && !DECL_EXTERNAL (old_decl
)
1061 && !DECL_WEAK (old_decl
)
1062 && !DECL_COMDAT (old_decl
))
1064 new_version_node
->unique_name
= true;
1066 /* Update the call_expr on the edges to call the new version node. */
1067 update_call_expr (new_version_node
);
1069 symtab
->call_cgraph_insertion_hooks (new_version_node
);
1070 return new_version_node
;
1073 /* Remove the node from the tree of virtual and inline clones and make it a
1074 standalone node - not a clone any more. */
1076 void cgraph_node::remove_from_clone_tree ()
1078 if (next_sibling_clone
)
1079 next_sibling_clone
->prev_sibling_clone
= prev_sibling_clone
;
1080 if (prev_sibling_clone
)
1081 prev_sibling_clone
->next_sibling_clone
= next_sibling_clone
;
1083 clone_of
->clones
= next_sibling_clone
;
1084 next_sibling_clone
= NULL
;
1085 prev_sibling_clone
= NULL
;
1089 /* Given virtual clone, turn it into actual clone. */
1092 cgraph_materialize_clone (cgraph_node
*node
)
1094 bitmap_obstack_initialize (NULL
);
1095 node
->former_clone_of
= node
->clone_of
->decl
;
1096 if (node
->clone_of
->former_clone_of
)
1097 node
->former_clone_of
= node
->clone_of
->former_clone_of
;
1098 /* Copy the OLD_VERSION_NODE function tree to the new version. */
1099 tree_function_versioning (node
->clone_of
->decl
, node
->decl
,
1100 node
->clone
.tree_map
, node
->clone
.param_adjustments
,
1102 if (symtab
->dump_file
)
1104 dump_function_to_file (node
->clone_of
->decl
, symtab
->dump_file
,
1106 dump_function_to_file (node
->decl
, symtab
->dump_file
, dump_flags
);
1109 cgraph_node
*clone_of
= node
->clone_of
;
1110 /* Function is no longer clone. */
1111 node
->remove_from_clone_tree ();
1112 if (!clone_of
->analyzed
&& !clone_of
->clones
)
1114 clone_of
->release_body ();
1115 clone_of
->remove_callees ();
1116 clone_of
->remove_all_references ();
1118 bitmap_obstack_release (NULL
);
1121 /* Once all functions from compilation unit are in memory, produce all clones
1122 and update all calls. We might also do this on demand if we don't want to
1123 bring all functions to memory prior compilation, but current WHOPR
1124 implementation does that and it is a bit easier to keep everything right in
1128 symbol_table::materialize_all_clones (void)
1131 bool stabilized
= false;
1134 if (symtab
->dump_file
)
1135 fprintf (symtab
->dump_file
, "Materializing clones\n");
1137 cgraph_node::checking_verify_cgraph_nodes ();
1139 /* We can also do topological order, but number of iterations should be
1140 bounded by number of IPA passes since single IPA pass is probably not
1141 going to create clones of clones it created itself. */
1145 FOR_EACH_FUNCTION (node
)
1147 if (node
->clone_of
&& node
->decl
!= node
->clone_of
->decl
1148 && !gimple_has_body_p (node
->decl
))
1150 if (!node
->clone_of
->clone_of
)
1151 node
->clone_of
->get_untransformed_body ();
1152 if (gimple_has_body_p (node
->clone_of
->decl
))
1154 if (symtab
->dump_file
)
1156 fprintf (symtab
->dump_file
, "cloning %s to %s\n",
1157 node
->clone_of
->dump_name (),
1158 node
->dump_name ());
1159 if (node
->clone
.tree_map
)
1162 fprintf (symtab
->dump_file
, " replace map: ");
1164 i
< vec_safe_length (node
->clone
.tree_map
);
1167 ipa_replace_map
*replace_info
;
1168 replace_info
= (*node
->clone
.tree_map
)[i
];
1169 fprintf (symtab
->dump_file
, "%i -> ",
1170 (*node
->clone
.tree_map
)[i
]->parm_num
);
1171 print_generic_expr (symtab
->dump_file
,
1172 replace_info
->new_tree
);
1174 fprintf (symtab
->dump_file
, "\n");
1176 if (node
->clone
.param_adjustments
)
1177 node
->clone
.param_adjustments
->dump (symtab
->dump_file
);
1179 cgraph_materialize_clone (node
);
1185 FOR_EACH_FUNCTION (node
)
1186 if (!node
->analyzed
&& node
->callees
)
1188 node
->remove_callees ();
1189 node
->remove_all_references ();
1192 node
->clear_stmts_in_references ();
1193 if (symtab
->dump_file
)
1194 fprintf (symtab
->dump_file
, "Materialization Call site updates done.\n");
1196 cgraph_node::checking_verify_cgraph_nodes ();
1198 symtab
->remove_unreachable_nodes (symtab
->dump_file
);
1201 #include "gt-cgraphclones.h"