1 /* Function splitting pass
2 Copyright (C) 2010, 2011
3 Free Software Foundation, Inc.
4 Contributed by Jan Hubicka <jh@suse.cz>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 /* The purpose of this pass is to split function bodies to improve
23 inlining. I.e. for function of the form:
48 When func becomes inlinable and when cheap_test is often true, inlining func,
49 but not fund.part leads to performance improvement similar as inlining
50 original func while the code size growth is smaller.
52 The pass is organized in three stages:
53 1) Collect local info about basic block into BB_INFO structure and
54 compute function body estimated size and time.
55 2) Via DFS walk find all possible basic blocks where we can split
57 3) If split point is found, split at the specified BB by creating a clone
58 and updating function to call it.
60 The decisions what functions to split are in execute_split_functions
63 There are several possible future improvements for this pass including:
65 1) Splitting to break up large functions
66 2) Splitting to reduce stack frame usage
67 3) Allow split part of function to use values computed in the header part.
68 The values needs to be passed to split function, perhaps via same
69 interface as for nested functions or as argument.
70 4) Support for simple rematerialization. I.e. when split part use
71 value computed in header from function parameter in very cheap way, we
72 can just recompute it.
73 5) Support splitting of nested functions.
74 6) Support non-SSA arguments.
75 7) There is nothing preventing us from producing multiple parts of single function
76 when needed or splitting also the parts. */
80 #include "coretypes.h"
85 #include "tree-flow.h"
86 #include "tree-pass.h"
89 #include "diagnostic.h"
90 #include "tree-dump.h"
91 #include "tree-inline.h"
94 #include "gimple-pretty-print.h"
96 /* Per basic block info. */
104 DEF_VEC_ALLOC_O(bb_info
,heap
);
106 static VEC(bb_info
, heap
) *bb_info_vec
;
108 /* Description of split point. */
112 /* Size of the partitions. */
113 unsigned int header_time
, header_size
, split_time
, split_size
;
115 /* SSA names that need to be passed into spit function. */
116 bitmap ssa_names_to_pass
;
118 /* Basic block where we split (that will become entry point of new function. */
119 basic_block entry_bb
;
121 /* Basic blocks we are splitting away. */
124 /* True when return value is computed on split part and thus it needs
126 bool split_part_set_retval
;
129 /* Best split point found. */
131 struct split_point best_split_point
;
133 static tree
find_retval (basic_block return_bb
);
135 /* Callback for walk_stmt_load_store_addr_ops. If T is non-SSA automatic
136 variable, check it if it is present in bitmap passed via DATA. */
139 test_nonssa_use (gimple stmt ATTRIBUTE_UNUSED
, tree t
, void *data
)
141 t
= get_base_address (t
);
143 if (!t
|| is_gimple_reg (t
))
146 if (TREE_CODE (t
) == PARM_DECL
147 || (TREE_CODE (t
) == VAR_DECL
148 && auto_var_in_fn_p (t
, current_function_decl
))
149 || TREE_CODE (t
) == RESULT_DECL
150 || TREE_CODE (t
) == LABEL_DECL
)
151 return bitmap_bit_p ((bitmap
)data
, DECL_UID (t
));
153 /* For DECL_BY_REFERENCE, the return value is actually a pointer. We want
154 to pretend that the value pointed to is actual result decl. */
155 if ((TREE_CODE (t
) == MEM_REF
|| INDIRECT_REF_P (t
))
156 && TREE_CODE (TREE_OPERAND (t
, 0)) == SSA_NAME
157 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (t
, 0))) == RESULT_DECL
158 && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl
)))
160 bitmap_bit_p ((bitmap
)data
,
161 DECL_UID (DECL_RESULT (current_function_decl
)));
166 /* Dump split point CURRENT. */
169 dump_split_point (FILE * file
, struct split_point
*current
)
172 "Split point at BB %i header time:%i header size: %i"
173 " split time: %i split size: %i\n bbs: ",
174 current
->entry_bb
->index
, current
->header_time
,
175 current
->header_size
, current
->split_time
, current
->split_size
);
176 dump_bitmap (file
, current
->split_bbs
);
177 fprintf (file
, " SSA names to pass: ");
178 dump_bitmap (file
, current
->ssa_names_to_pass
);
181 /* Look for all BBs in header that might lead to the split part and verify
182 that they are not defining any non-SSA var used by the split part.
183 Parameters are the same as for consider_split. */
186 verify_non_ssa_vars (struct split_point
*current
, bitmap non_ssa_vars
,
187 basic_block return_bb
)
189 bitmap seen
= BITMAP_ALLOC (NULL
);
190 VEC (basic_block
,heap
) *worklist
= NULL
;
195 FOR_EACH_EDGE (e
, ei
, current
->entry_bb
->preds
)
196 if (e
->src
!= ENTRY_BLOCK_PTR
197 && !bitmap_bit_p (current
->split_bbs
, e
->src
->index
))
199 VEC_safe_push (basic_block
, heap
, worklist
, e
->src
);
200 bitmap_set_bit (seen
, e
->src
->index
);
203 while (!VEC_empty (basic_block
, worklist
))
205 gimple_stmt_iterator bsi
;
206 basic_block bb
= VEC_pop (basic_block
, worklist
);
208 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
209 if (e
->src
!= ENTRY_BLOCK_PTR
210 && bitmap_set_bit (seen
, e
->src
->index
))
212 gcc_checking_assert (!bitmap_bit_p (current
->split_bbs
,
214 VEC_safe_push (basic_block
, heap
, worklist
, e
->src
);
216 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
218 gimple stmt
= gsi_stmt (bsi
);
219 if (is_gimple_debug (stmt
))
221 if (walk_stmt_load_store_addr_ops
222 (stmt
, non_ssa_vars
, test_nonssa_use
, test_nonssa_use
,
228 if (gimple_code (stmt
) == GIMPLE_LABEL
229 && test_nonssa_use (stmt
, gimple_label_label (stmt
),
236 for (bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
238 if (walk_stmt_load_store_addr_ops
239 (gsi_stmt (bsi
), non_ssa_vars
, test_nonssa_use
, test_nonssa_use
,
246 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
248 if (e
->dest
!= return_bb
)
250 for (bsi
= gsi_start_phis (return_bb
); !gsi_end_p (bsi
);
253 gimple stmt
= gsi_stmt (bsi
);
254 tree op
= gimple_phi_arg_def (stmt
, e
->dest_idx
);
256 if (!is_gimple_reg (gimple_phi_result (stmt
)))
258 if (TREE_CODE (op
) != SSA_NAME
259 && test_nonssa_use (stmt
, op
, non_ssa_vars
))
269 VEC_free (basic_block
, heap
, worklist
);
273 /* We found an split_point CURRENT. NON_SSA_VARS is bitmap of all non ssa
274 variables used and RETURN_BB is return basic block.
275 See if we can split function here. */
278 consider_split (struct split_point
*current
, bitmap non_ssa_vars
,
279 basic_block return_bb
)
282 unsigned int num_args
= 0;
283 unsigned int call_overhead
;
286 gimple_stmt_iterator bsi
;
288 int incoming_freq
= 0;
291 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
292 dump_split_point (dump_file
, current
);
294 FOR_EACH_EDGE (e
, ei
, current
->entry_bb
->preds
)
295 if (!bitmap_bit_p (current
->split_bbs
, e
->src
->index
))
296 incoming_freq
+= EDGE_FREQUENCY (e
);
298 /* Do not split when we would end up calling function anyway. */
300 >= (ENTRY_BLOCK_PTR
->frequency
301 * PARAM_VALUE (PARAM_PARTIAL_INLINING_ENTRY_PROBABILITY
) / 100))
303 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
305 " Refused: incoming frequency is too large.\n");
309 if (!current
->header_size
)
311 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
312 fprintf (dump_file
, " Refused: header empty\n");
316 /* Verify that PHI args on entry are either virtual or all their operands
317 incoming from header are the same. */
318 for (bsi
= gsi_start_phis (current
->entry_bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
320 gimple stmt
= gsi_stmt (bsi
);
323 if (!is_gimple_reg (gimple_phi_result (stmt
)))
325 for (i
= 0; i
< gimple_phi_num_args (stmt
); i
++)
327 edge e
= gimple_phi_arg_edge (stmt
, i
);
328 if (!bitmap_bit_p (current
->split_bbs
, e
->src
->index
))
330 tree edge_val
= gimple_phi_arg_def (stmt
, i
);
331 if (val
&& edge_val
!= val
)
333 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
335 " Refused: entry BB has PHI with multiple variants\n");
344 /* See what argument we will pass to the split function and compute
346 call_overhead
= eni_size_weights
.call_cost
;
347 for (parm
= DECL_ARGUMENTS (current_function_decl
); parm
;
348 parm
= DECL_CHAIN (parm
))
350 if (!is_gimple_reg (parm
))
352 if (bitmap_bit_p (non_ssa_vars
, DECL_UID (parm
)))
354 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
356 " Refused: need to pass non-ssa param values\n");
360 else if (gimple_default_def (cfun
, parm
)
361 && bitmap_bit_p (current
->ssa_names_to_pass
,
362 SSA_NAME_VERSION (gimple_default_def
365 if (!VOID_TYPE_P (TREE_TYPE (parm
)))
366 call_overhead
+= estimate_move_cost (TREE_TYPE (parm
));
370 if (!VOID_TYPE_P (TREE_TYPE (current_function_decl
)))
371 call_overhead
+= estimate_move_cost (TREE_TYPE (current_function_decl
));
373 if (current
->split_size
<= call_overhead
)
375 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
377 " Refused: split size is smaller than call overhead\n");
380 if (current
->header_size
+ call_overhead
381 >= (unsigned int)(DECL_DECLARED_INLINE_P (current_function_decl
)
382 ? MAX_INLINE_INSNS_SINGLE
383 : MAX_INLINE_INSNS_AUTO
))
385 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
387 " Refused: header size is too large for inline candidate\n");
391 /* FIXME: we currently can pass only SSA function parameters to the split
392 arguments. Once parm_adjustment infrastructure is supported by cloning,
393 we can pass more than that. */
394 if (num_args
!= bitmap_count_bits (current
->ssa_names_to_pass
))
397 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
399 " Refused: need to pass non-param values\n");
403 /* When there are non-ssa vars used in the split region, see if they
404 are used in the header region. If so, reject the split.
405 FIXME: we can use nested function support to access both. */
406 if (!bitmap_empty_p (non_ssa_vars
)
407 && !verify_non_ssa_vars (current
, non_ssa_vars
, return_bb
))
409 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
411 " Refused: split part has non-ssa uses\n");
414 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
415 fprintf (dump_file
, " Accepted!\n");
417 /* See if retval used by return bb is computed by header or split part.
418 When it is computed by split part, we need to produce return statement
419 in the split part and add code to header to pass it around.
421 This is bit tricky to test:
422 1) When there is no return_bb or no return value, we always pass
424 2) Invariants are always computed by caller.
425 3) For SSA we need to look if defining statement is in header or split part
426 4) For non-SSA we need to look where the var is computed. */
427 retval
= find_retval (return_bb
);
429 current
->split_part_set_retval
= true;
430 else if (is_gimple_min_invariant (retval
))
431 current
->split_part_set_retval
= false;
432 /* Special case is value returned by reference we record as if it was non-ssa
433 set to result_decl. */
434 else if (TREE_CODE (retval
) == SSA_NAME
435 && TREE_CODE (SSA_NAME_VAR (retval
)) == RESULT_DECL
436 && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl
)))
437 current
->split_part_set_retval
438 = bitmap_bit_p (non_ssa_vars
, DECL_UID (SSA_NAME_VAR (retval
)));
439 else if (TREE_CODE (retval
) == SSA_NAME
)
440 current
->split_part_set_retval
441 = (!SSA_NAME_IS_DEFAULT_DEF (retval
)
442 && (bitmap_bit_p (current
->split_bbs
,
443 gimple_bb (SSA_NAME_DEF_STMT (retval
))->index
)
444 || gimple_bb (SSA_NAME_DEF_STMT (retval
)) == return_bb
));
445 else if (TREE_CODE (retval
) == PARM_DECL
)
446 current
->split_part_set_retval
= false;
447 else if (TREE_CODE (retval
) == VAR_DECL
448 || TREE_CODE (retval
) == RESULT_DECL
)
449 current
->split_part_set_retval
450 = bitmap_bit_p (non_ssa_vars
, DECL_UID (retval
));
452 current
->split_part_set_retval
= true;
454 /* At the moment chose split point with lowest frequency and that leaves
455 out smallest size of header.
456 In future we might re-consider this heuristics. */
457 if (!best_split_point
.split_bbs
458 || best_split_point
.entry_bb
->frequency
> current
->entry_bb
->frequency
459 || (best_split_point
.entry_bb
->frequency
== current
->entry_bb
->frequency
460 && best_split_point
.split_size
< current
->split_size
))
463 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
464 fprintf (dump_file
, " New best split point!\n");
465 if (best_split_point
.ssa_names_to_pass
)
467 BITMAP_FREE (best_split_point
.ssa_names_to_pass
);
468 BITMAP_FREE (best_split_point
.split_bbs
);
470 best_split_point
= *current
;
471 best_split_point
.ssa_names_to_pass
= BITMAP_ALLOC (NULL
);
472 bitmap_copy (best_split_point
.ssa_names_to_pass
,
473 current
->ssa_names_to_pass
);
474 best_split_point
.split_bbs
= BITMAP_ALLOC (NULL
);
475 bitmap_copy (best_split_point
.split_bbs
, current
->split_bbs
);
479 /* Return basic block containing RETURN statement. We allow basic blocks
483 but return_bb can not be more complex than this.
484 If nothing is found, return EXIT_BLOCK_PTR.
486 When there are multiple RETURN statement, chose one with return value,
487 since that one is more likely shared by multiple code paths.
489 Return BB is special, because for function splitting it is the only
490 basic block that is duplicated in between header and split part of the
493 TODO: We might support multiple return blocks. */
496 find_return_bb (void)
499 basic_block return_bb
= EXIT_BLOCK_PTR
;
500 gimple_stmt_iterator bsi
;
501 bool found_return
= false;
502 tree retval
= NULL_TREE
;
504 if (!single_pred_p (EXIT_BLOCK_PTR
))
507 e
= single_pred_edge (EXIT_BLOCK_PTR
);
508 for (bsi
= gsi_last_bb (e
->src
); !gsi_end_p (bsi
); gsi_prev (&bsi
))
510 gimple stmt
= gsi_stmt (bsi
);
511 if (gimple_code (stmt
) == GIMPLE_LABEL
|| is_gimple_debug (stmt
))
513 else if (gimple_code (stmt
) == GIMPLE_ASSIGN
515 && gimple_assign_single_p (stmt
)
516 && (auto_var_in_fn_p (gimple_assign_rhs1 (stmt
),
517 current_function_decl
)
518 || is_gimple_min_invariant (gimple_assign_rhs1 (stmt
)))
519 && retval
== gimple_assign_lhs (stmt
))
521 else if (gimple_code (stmt
) == GIMPLE_RETURN
)
524 retval
= gimple_return_retval (stmt
);
529 if (gsi_end_p (bsi
) && found_return
)
535 /* Given return basic block RETURN_BB, see where return value is really
538 find_retval (basic_block return_bb
)
540 gimple_stmt_iterator bsi
;
541 for (bsi
= gsi_start_bb (return_bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
542 if (gimple_code (gsi_stmt (bsi
)) == GIMPLE_RETURN
)
543 return gimple_return_retval (gsi_stmt (bsi
));
544 else if (gimple_code (gsi_stmt (bsi
)) == GIMPLE_ASSIGN
)
545 return gimple_assign_rhs1 (gsi_stmt (bsi
));
549 /* Callback for walk_stmt_load_store_addr_ops. If T is non-SSA automatic
550 variable, mark it as used in bitmap passed via DATA.
551 Return true when access to T prevents splitting the function. */
554 mark_nonssa_use (gimple stmt ATTRIBUTE_UNUSED
, tree t
, void *data
)
556 t
= get_base_address (t
);
558 if (!t
|| is_gimple_reg (t
))
561 /* At present we can't pass non-SSA arguments to split function.
562 FIXME: this can be relaxed by passing references to arguments. */
563 if (TREE_CODE (t
) == PARM_DECL
)
565 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
567 "Cannot split: use of non-ssa function parameter.\n");
571 if ((TREE_CODE (t
) == VAR_DECL
572 && auto_var_in_fn_p (t
, current_function_decl
))
573 || TREE_CODE (t
) == RESULT_DECL
574 || TREE_CODE (t
) == LABEL_DECL
)
575 bitmap_set_bit ((bitmap
)data
, DECL_UID (t
));
577 /* For DECL_BY_REFERENCE, the return value is actually a pointer. We want
578 to pretend that the value pointed to is actual result decl. */
579 if ((TREE_CODE (t
) == MEM_REF
|| INDIRECT_REF_P (t
))
580 && TREE_CODE (TREE_OPERAND (t
, 0)) == SSA_NAME
581 && TREE_CODE (SSA_NAME_VAR (TREE_OPERAND (t
, 0))) == RESULT_DECL
582 && DECL_BY_REFERENCE (DECL_RESULT (current_function_decl
)))
584 bitmap_bit_p ((bitmap
)data
,
585 DECL_UID (DECL_RESULT (current_function_decl
)));
590 /* Compute local properties of basic block BB we collect when looking for
591 split points. We look for ssa defs and store them in SET_SSA_NAMES,
592 for ssa uses and store them in USED_SSA_NAMES and for any non-SSA automatic
593 vars stored in NON_SSA_VARS.
595 When BB has edge to RETURN_BB, collect uses in RETURN_BB too.
597 Return false when BB contains something that prevents it from being put into
601 visit_bb (basic_block bb
, basic_block return_bb
,
602 bitmap set_ssa_names
, bitmap used_ssa_names
,
605 gimple_stmt_iterator bsi
;
608 bool can_split
= true;
610 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
612 gimple stmt
= gsi_stmt (bsi
);
617 if (is_gimple_debug (stmt
))
620 /* FIXME: We can split regions containing EH. We can not however
621 split RESX, EH_DISPATCH and EH_POINTER referring to same region
622 into different partitions. This would require tracking of
623 EH regions and checking in consider_split_point if they
624 are not used elsewhere. */
625 if (gimple_code (stmt
) == GIMPLE_RESX
626 && stmt_can_throw_external (stmt
))
628 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
629 fprintf (dump_file
, "Cannot split: external resx.\n");
632 if (gimple_code (stmt
) == GIMPLE_EH_DISPATCH
)
634 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
635 fprintf (dump_file
, "Cannot split: eh dispatch.\n");
639 /* Check builtins that prevent splitting. */
640 if (gimple_code (stmt
) == GIMPLE_CALL
641 && (decl
= gimple_call_fndecl (stmt
)) != NULL_TREE
642 && DECL_BUILT_IN (decl
)
643 && DECL_BUILT_IN_CLASS (decl
) == BUILT_IN_NORMAL
)
644 switch (DECL_FUNCTION_CODE (decl
))
646 /* FIXME: once we will allow passing non-parm values to split part,
647 we need to be sure to handle correct builtin_stack_save and
648 builtin_stack_restore. At the moment we are safe; there is no
649 way to store builtin_stack_save result in non-SSA variable
650 since all calls to those are compiler generated. */
652 case BUILT_IN_VA_START
:
653 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
655 "Cannot split: builtin_apply and va_start.\n");
658 case BUILT_IN_EH_POINTER
:
659 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
660 fprintf (dump_file
, "Cannot split: builtin_eh_pointer.\n");
667 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, iter
, SSA_OP_DEF
)
668 bitmap_set_bit (set_ssa_names
, SSA_NAME_VERSION (op
));
669 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, iter
, SSA_OP_USE
)
670 bitmap_set_bit (used_ssa_names
, SSA_NAME_VERSION (op
));
671 can_split
&= !walk_stmt_load_store_addr_ops (stmt
, non_ssa_vars
,
676 for (bsi
= gsi_start_phis (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
678 gimple stmt
= gsi_stmt (bsi
);
681 if (is_gimple_debug (stmt
))
683 if (!is_gimple_reg (gimple_phi_result (stmt
)))
685 bitmap_set_bit (set_ssa_names
,
686 SSA_NAME_VERSION (gimple_phi_result (stmt
)));
687 for (i
= 0; i
< gimple_phi_num_args (stmt
); i
++)
689 tree op
= gimple_phi_arg_def (stmt
, i
);
690 if (TREE_CODE (op
) == SSA_NAME
)
691 bitmap_set_bit (used_ssa_names
, SSA_NAME_VERSION (op
));
693 can_split
&= !walk_stmt_load_store_addr_ops (stmt
, non_ssa_vars
,
698 /* Record also uses coming from PHI operand in return BB. */
699 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
700 if (e
->dest
== return_bb
)
702 for (bsi
= gsi_start_phis (return_bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
704 gimple stmt
= gsi_stmt (bsi
);
705 tree op
= gimple_phi_arg_def (stmt
, e
->dest_idx
);
707 if (is_gimple_debug (stmt
))
709 if (!is_gimple_reg (gimple_phi_result (stmt
)))
711 if (TREE_CODE (op
) == SSA_NAME
)
712 bitmap_set_bit (used_ssa_names
, SSA_NAME_VERSION (op
));
714 can_split
&= !mark_nonssa_use (stmt
, op
, non_ssa_vars
);
720 /* Stack entry for recursive DFS walk in find_split_point. */
724 /* Basic block we are examining. */
727 /* SSA names set and used by the BB and all BBs reachable
728 from it via DFS walk. */
729 bitmap set_ssa_names
, used_ssa_names
;
732 /* All BBS visited from this BB via DFS walk. */
735 /* Last examined edge in DFS walk. Since we walk unoriented graph,
736 the value is up to sum of incoming and outgoing edges of BB. */
737 unsigned int edge_num
;
739 /* Stack entry index of earliest BB reachable from current BB
740 or any BB visited later in DFS walk. */
743 /* Overall time and size of all BBs reached from this BB in DFS walk. */
744 int overall_time
, overall_size
;
746 /* When false we can not split on this BB. */
749 DEF_VEC_O(stack_entry
);
750 DEF_VEC_ALLOC_O(stack_entry
,heap
);
753 /* Find all articulations and call consider_split on them.
754 OVERALL_TIME and OVERALL_SIZE is time and size of the function.
756 We perform basic algorithm for finding an articulation in a graph
757 created from CFG by considering it to be an unoriented graph.
759 The articulation is discovered via DFS walk. We collect earliest
760 basic block on stack that is reachable via backward edge. Articulation
761 is any basic block such that there is no backward edge bypassing it.
762 To reduce stack usage we maintain heap allocated stack in STACK vector.
763 AUX pointer of BB is set to index it appears in the stack or -1 once
764 it is visited and popped off the stack.
766 The algorithm finds articulation after visiting the whole component
767 reachable by it. This makes it convenient to collect information about
768 the component used by consider_split. */
771 find_split_points (int overall_time
, int overall_size
)
774 VEC(stack_entry
, heap
) *stack
= NULL
;
776 basic_block return_bb
= find_return_bb ();
777 struct split_point current
;
779 current
.header_time
= overall_time
;
780 current
.header_size
= overall_size
;
781 current
.split_time
= 0;
782 current
.split_size
= 0;
783 current
.ssa_names_to_pass
= BITMAP_ALLOC (NULL
);
785 first
.bb
= ENTRY_BLOCK_PTR
;
787 first
.overall_time
= 0;
788 first
.overall_size
= 0;
789 first
.earliest
= INT_MAX
;
790 first
.set_ssa_names
= 0;
791 first
.used_ssa_names
= 0;
792 first
.bbs_visited
= 0;
793 VEC_safe_push (stack_entry
, heap
, stack
, &first
);
794 ENTRY_BLOCK_PTR
->aux
= (void *)(intptr_t)-1;
796 while (!VEC_empty (stack_entry
, stack
))
798 stack_entry
*entry
= VEC_last (stack_entry
, stack
);
800 /* We are walking an acyclic graph, so edge_num counts
801 succ and pred edges together. However when considering
802 articulation, we want to have processed everything reachable
803 from articulation but nothing that reaches into it. */
804 if (entry
->edge_num
== EDGE_COUNT (entry
->bb
->succs
)
805 && entry
->bb
!= ENTRY_BLOCK_PTR
)
807 int pos
= VEC_length (stack_entry
, stack
);
808 entry
->can_split
&= visit_bb (entry
->bb
, return_bb
,
809 entry
->set_ssa_names
,
810 entry
->used_ssa_names
,
811 entry
->non_ssa_vars
);
812 if (pos
<= entry
->earliest
&& !entry
->can_split
813 && dump_file
&& (dump_flags
& TDF_DETAILS
))
815 "found articulation at bb %i but can not split\n",
817 if (pos
<= entry
->earliest
&& entry
->can_split
)
819 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
820 fprintf (dump_file
, "found articulation at bb %i\n",
822 current
.entry_bb
= entry
->bb
;
823 current
.ssa_names_to_pass
= BITMAP_ALLOC (NULL
);
824 bitmap_and_compl (current
.ssa_names_to_pass
,
825 entry
->used_ssa_names
, entry
->set_ssa_names
);
826 current
.header_time
= overall_time
- entry
->overall_time
;
827 current
.header_size
= overall_size
- entry
->overall_size
;
828 current
.split_time
= entry
->overall_time
;
829 current
.split_size
= entry
->overall_size
;
830 current
.split_bbs
= entry
->bbs_visited
;
831 consider_split (¤t
, entry
->non_ssa_vars
, return_bb
);
832 BITMAP_FREE (current
.ssa_names_to_pass
);
835 /* Do actual DFS walk. */
837 < (EDGE_COUNT (entry
->bb
->succs
)
838 + EDGE_COUNT (entry
->bb
->preds
)))
842 if (entry
->edge_num
< EDGE_COUNT (entry
->bb
->succs
))
844 e
= EDGE_SUCC (entry
->bb
, entry
->edge_num
);
849 e
= EDGE_PRED (entry
->bb
, entry
->edge_num
850 - EDGE_COUNT (entry
->bb
->succs
));
856 /* New BB to visit, push it to the stack. */
857 if (dest
!= return_bb
&& dest
!= EXIT_BLOCK_PTR
860 stack_entry new_entry
;
863 new_entry
.edge_num
= 0;
864 new_entry
.overall_time
865 = VEC_index (bb_info
, bb_info_vec
, dest
->index
)->time
;
866 new_entry
.overall_size
867 = VEC_index (bb_info
, bb_info_vec
, dest
->index
)->size
;
868 new_entry
.earliest
= INT_MAX
;
869 new_entry
.set_ssa_names
= BITMAP_ALLOC (NULL
);
870 new_entry
.used_ssa_names
= BITMAP_ALLOC (NULL
);
871 new_entry
.bbs_visited
= BITMAP_ALLOC (NULL
);
872 new_entry
.non_ssa_vars
= BITMAP_ALLOC (NULL
);
873 new_entry
.can_split
= true;
874 bitmap_set_bit (new_entry
.bbs_visited
, dest
->index
);
875 VEC_safe_push (stack_entry
, heap
, stack
, &new_entry
);
876 dest
->aux
= (void *)(intptr_t)VEC_length (stack_entry
, stack
);
878 /* Back edge found, record the earliest point. */
879 else if ((intptr_t)dest
->aux
> 0
880 && (intptr_t)dest
->aux
< entry
->earliest
)
881 entry
->earliest
= (intptr_t)dest
->aux
;
883 /* We are done with examining the edges. Pop off the value from stack
884 and merge stuff we accumulate during the walk. */
885 else if (entry
->bb
!= ENTRY_BLOCK_PTR
)
887 stack_entry
*prev
= VEC_index (stack_entry
, stack
,
888 VEC_length (stack_entry
, stack
) - 2);
890 entry
->bb
->aux
= (void *)(intptr_t)-1;
891 prev
->can_split
&= entry
->can_split
;
892 if (prev
->set_ssa_names
)
894 bitmap_ior_into (prev
->set_ssa_names
, entry
->set_ssa_names
);
895 bitmap_ior_into (prev
->used_ssa_names
, entry
->used_ssa_names
);
896 bitmap_ior_into (prev
->bbs_visited
, entry
->bbs_visited
);
897 bitmap_ior_into (prev
->non_ssa_vars
, entry
->non_ssa_vars
);
899 if (prev
->earliest
> entry
->earliest
)
900 prev
->earliest
= entry
->earliest
;
901 prev
->overall_time
+= entry
->overall_time
;
902 prev
->overall_size
+= entry
->overall_size
;
903 BITMAP_FREE (entry
->set_ssa_names
);
904 BITMAP_FREE (entry
->used_ssa_names
);
905 BITMAP_FREE (entry
->bbs_visited
);
906 BITMAP_FREE (entry
->non_ssa_vars
);
907 VEC_pop (stack_entry
, stack
);
910 VEC_pop (stack_entry
, stack
);
912 ENTRY_BLOCK_PTR
->aux
= NULL
;
915 VEC_free (stack_entry
, heap
, stack
);
916 BITMAP_FREE (current
.ssa_names_to_pass
);
919 /* Split function at SPLIT_POINT. */
922 split_function (struct split_point
*split_point
)
924 VEC (tree
, heap
) *args_to_pass
= NULL
;
925 bitmap args_to_skip
= BITMAP_ALLOC (NULL
);
928 struct cgraph_node
*node
;
929 basic_block return_bb
= find_return_bb ();
931 gimple_stmt_iterator gsi
;
935 tree retval
= NULL
, real_retval
= NULL
;
936 bool split_part_return_p
= false;
937 gimple last_stmt
= NULL
;
938 bool conv_needed
= false;
944 fprintf (dump_file
, "\n\nSplitting function at:\n");
945 dump_split_point (dump_file
, split_point
);
948 /* Collect the parameters of new function and args_to_skip bitmap. */
949 for (parm
= DECL_ARGUMENTS (current_function_decl
);
950 parm
; parm
= DECL_CHAIN (parm
), num
++)
951 if (!is_gimple_reg (parm
)
952 || !gimple_default_def (cfun
, parm
)
953 || !bitmap_bit_p (split_point
->ssa_names_to_pass
,
954 SSA_NAME_VERSION (gimple_default_def (cfun
, parm
))))
955 bitmap_set_bit (args_to_skip
, num
);
958 arg
= gimple_default_def (cfun
, parm
);
959 if (TYPE_MAIN_VARIANT (DECL_ARG_TYPE (parm
))
960 != TYPE_MAIN_VARIANT (TREE_TYPE (arg
)))
963 arg
= fold_convert (DECL_ARG_TYPE (parm
), arg
);
965 VEC_safe_push (tree
, heap
, args_to_pass
, arg
);
968 /* See if the split function will return. */
969 FOR_EACH_EDGE (e
, ei
, return_bb
->preds
)
970 if (bitmap_bit_p (split_point
->split_bbs
, e
->src
->index
))
973 split_part_return_p
= true;
975 /* Add return block to what will become the split function.
976 We do not return; no return block is needed. */
977 if (!split_part_return_p
)
979 /* We have no return block, so nothing is needed. */
980 else if (return_bb
== EXIT_BLOCK_PTR
)
982 /* When we do not want to return value, we need to construct
983 new return block with empty return statement.
984 FIXME: Once we are able to change return type, we should change function
985 to return void instead of just outputting function with undefined return
986 value. For structures this affects quality of codegen. */
987 else if (!split_point
->split_part_set_retval
988 && find_retval (return_bb
))
990 bool redirected
= true;
991 basic_block new_return_bb
= create_basic_block (NULL
, 0, return_bb
);
992 gimple_stmt_iterator gsi
= gsi_start_bb (new_return_bb
);
993 gsi_insert_after (&gsi
, gimple_build_return (NULL
), GSI_NEW_STMT
);
997 FOR_EACH_EDGE (e
, ei
, return_bb
->preds
)
998 if (bitmap_bit_p (split_point
->split_bbs
, e
->src
->index
))
1000 new_return_bb
->count
+= e
->count
;
1001 new_return_bb
->frequency
+= EDGE_FREQUENCY (e
);
1002 redirect_edge_and_branch (e
, new_return_bb
);
1007 e
= make_edge (new_return_bb
, EXIT_BLOCK_PTR
, 0);
1008 e
->probability
= REG_BR_PROB_BASE
;
1009 e
->count
= new_return_bb
->count
;
1010 bitmap_set_bit (split_point
->split_bbs
, new_return_bb
->index
);
1012 /* When we pass around the value, use existing return block. */
1014 bitmap_set_bit (split_point
->split_bbs
, return_bb
->index
);
1016 /* If RETURN_BB has virtual operand PHIs, they must be removed and the
1017 virtual operand marked for renaming as we change the CFG in a way that
1018 tree-inline is not able to compensate for.
1020 Note this can happen whether or not we have a return value. If we have
1021 a return value, then RETURN_BB may have PHIs for real operands too. */
1022 if (return_bb
!= EXIT_BLOCK_PTR
)
1024 for (gsi
= gsi_start_phis (return_bb
); !gsi_end_p (gsi
);)
1026 gimple stmt
= gsi_stmt (gsi
);
1027 if (is_gimple_reg (gimple_phi_result (stmt
)))
1032 mark_virtual_phi_result_for_renaming (stmt
);
1033 remove_phi_node (&gsi
, true);
1037 /* Now create the actual clone. */
1038 rebuild_cgraph_edges ();
1039 node
= cgraph_function_versioning (cgraph_node (current_function_decl
),
1042 split_point
->split_bbs
,
1043 split_point
->entry_bb
, "part");
1044 /* For usual cloning it is enough to clear builtin only when signature
1045 changes. For partial inlining we however can not expect the part
1046 of builtin implementation to have same semantic as the whole. */
1047 if (DECL_BUILT_IN (node
->decl
))
1049 DECL_BUILT_IN_CLASS (node
->decl
) = NOT_BUILT_IN
;
1050 DECL_FUNCTION_CODE (node
->decl
) = (enum built_in_function
) 0;
1052 cgraph_node_remove_callees (cgraph_node (current_function_decl
));
1053 if (!split_part_return_p
)
1054 TREE_THIS_VOLATILE (node
->decl
) = 1;
1056 dump_function_to_file (node
->decl
, dump_file
, dump_flags
);
1058 /* Create the basic block we place call into. It is the entry basic block
1059 split after last label. */
1060 call_bb
= split_point
->entry_bb
;
1061 for (gsi
= gsi_start_bb (call_bb
); !gsi_end_p (gsi
);)
1062 if (gimple_code (gsi_stmt (gsi
)) == GIMPLE_LABEL
)
1064 last_stmt
= gsi_stmt (gsi
);
1069 e
= split_block (split_point
->entry_bb
, last_stmt
);
1072 /* Produce the call statement. */
1073 gsi
= gsi_last_bb (call_bb
);
1075 FOR_EACH_VEC_ELT (tree
, args_to_pass
, i
, arg
)
1076 if (!is_gimple_val (arg
))
1078 arg
= force_gimple_operand_gsi (&gsi
, arg
, true, NULL_TREE
,
1079 false, GSI_NEW_STMT
);
1080 VEC_replace (tree
, args_to_pass
, i
, arg
);
1082 call
= gimple_build_call_vec (node
->decl
, args_to_pass
);
1083 gimple_set_block (call
, DECL_INITIAL (current_function_decl
));
1085 /* We avoid address being taken on any variable used by split part,
1086 so return slot optimization is always possible. Moreover this is
1087 required to make DECL_BY_REFERENCE work. */
1088 if (aggregate_value_p (DECL_RESULT (current_function_decl
),
1089 TREE_TYPE (current_function_decl
)))
1090 gimple_call_set_return_slot_opt (call
, true);
1092 /* Update return value. This is bit tricky. When we do not return,
1093 do nothing. When we return we might need to update return_bb
1094 or produce a new return statement. */
1095 if (!split_part_return_p
)
1096 gsi_insert_after (&gsi
, call
, GSI_NEW_STMT
);
1099 e
= make_edge (call_bb
, return_bb
,
1100 return_bb
== EXIT_BLOCK_PTR
? 0 : EDGE_FALLTHRU
);
1101 e
->count
= call_bb
->count
;
1102 e
->probability
= REG_BR_PROB_BASE
;
1104 /* If there is return basic block, see what value we need to store
1105 return value into and put call just before it. */
1106 if (return_bb
!= EXIT_BLOCK_PTR
)
1108 real_retval
= retval
= find_retval (return_bb
);
1110 if (real_retval
&& split_point
->split_part_set_retval
)
1112 gimple_stmt_iterator psi
;
1114 /* See if we need new SSA_NAME for the result.
1115 When DECL_BY_REFERENCE is true, retval is actually pointer to
1116 return value and it is constant in whole function. */
1117 if (TREE_CODE (retval
) == SSA_NAME
1118 && !DECL_BY_REFERENCE (DECL_RESULT (current_function_decl
)))
1120 retval
= make_ssa_name (SSA_NAME_VAR (retval
), call
);
1122 /* See if there is PHI defining return value. */
1123 for (psi
= gsi_start_phis (return_bb
);
1124 !gsi_end_p (psi
); gsi_next (&psi
))
1125 if (is_gimple_reg (gimple_phi_result (gsi_stmt (psi
))))
1128 /* When there is PHI, just update its value. */
1129 if (TREE_CODE (retval
) == SSA_NAME
1130 && !gsi_end_p (psi
))
1131 add_phi_arg (gsi_stmt (psi
), retval
, e
, UNKNOWN_LOCATION
);
1132 /* Otherwise update the return BB itself.
1133 find_return_bb allows at most one assignment to return value,
1134 so update first statement. */
1137 gimple_stmt_iterator bsi
;
1138 for (bsi
= gsi_start_bb (return_bb
); !gsi_end_p (bsi
);
1140 if (gimple_code (gsi_stmt (bsi
)) == GIMPLE_RETURN
)
1142 gimple_return_set_retval (gsi_stmt (bsi
), retval
);
1145 else if (gimple_code (gsi_stmt (bsi
)) == GIMPLE_ASSIGN
)
1147 gimple_assign_set_rhs1 (gsi_stmt (bsi
), retval
);
1150 update_stmt (gsi_stmt (bsi
));
1153 if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl
)))
1154 gimple_call_set_lhs (call
, build_simple_mem_ref (retval
));
1156 gimple_call_set_lhs (call
, retval
);
1158 gsi_insert_after (&gsi
, call
, GSI_NEW_STMT
);
1160 /* We don't use return block (there is either no return in function or
1161 multiple of them). So create new basic block with return statement.
1166 if (split_point
->split_part_set_retval
1167 && !VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl
))))
1169 retval
= DECL_RESULT (current_function_decl
);
1171 /* We use temporary register to hold value when aggregate_value_p
1172 is false. Similarly for DECL_BY_REFERENCE we must avoid extra
1174 if (!aggregate_value_p (retval
, TREE_TYPE (current_function_decl
))
1175 && !DECL_BY_REFERENCE (retval
))
1176 retval
= create_tmp_reg (TREE_TYPE (retval
), NULL
);
1177 if (is_gimple_reg (retval
))
1179 /* When returning by reference, there is only one SSA name
1180 assigned to RESULT_DECL (that is pointer to return value).
1181 Look it up or create new one if it is missing. */
1182 if (DECL_BY_REFERENCE (retval
))
1185 if ((retval_name
= gimple_default_def (cfun
, retval
))
1187 retval
= retval_name
;
1190 retval_name
= make_ssa_name (retval
,
1191 gimple_build_nop ());
1192 set_default_def (retval
, retval_name
);
1193 retval
= retval_name
;
1196 /* Otherwise produce new SSA name for return value. */
1198 retval
= make_ssa_name (retval
, call
);
1200 if (DECL_BY_REFERENCE (DECL_RESULT (current_function_decl
)))
1201 gimple_call_set_lhs (call
, build_simple_mem_ref (retval
));
1203 gimple_call_set_lhs (call
, retval
);
1205 gsi_insert_after (&gsi
, call
, GSI_NEW_STMT
);
1206 ret
= gimple_build_return (retval
);
1207 gsi_insert_after (&gsi
, ret
, GSI_NEW_STMT
);
1210 free_dominance_info (CDI_DOMINATORS
);
1211 free_dominance_info (CDI_POST_DOMINATORS
);
1212 compute_inline_parameters (node
);
1215 /* Execute function splitting pass. */
1218 execute_split_functions (void)
1220 gimple_stmt_iterator bsi
;
1222 int overall_time
= 0, overall_size
= 0;
1224 struct cgraph_node
*node
= cgraph_node (current_function_decl
);
1226 if (flags_from_decl_or_type (current_function_decl
) & ECF_NORETURN
)
1229 fprintf (dump_file
, "Not splitting: noreturn function.\n");
1232 if (MAIN_NAME_P (DECL_NAME (current_function_decl
)))
1235 fprintf (dump_file
, "Not splitting: main function.\n");
1238 /* This can be relaxed; function might become inlinable after splitting
1239 away the uninlinable part. */
1240 if (!node
->local
.inlinable
)
1243 fprintf (dump_file
, "Not splitting: not inlinable.\n");
1246 if (node
->local
.disregard_inline_limits
)
1249 fprintf (dump_file
, "Not splitting: disregarding inline limits.\n");
1252 /* This can be relaxed; most of versioning tests actually prevents
1254 if (!tree_versionable_function_p (current_function_decl
))
1257 fprintf (dump_file
, "Not splitting: not versionable.\n");
1260 /* FIXME: we could support this. */
1261 if (DECL_STRUCT_FUNCTION (current_function_decl
)->static_chain_decl
)
1264 fprintf (dump_file
, "Not splitting: nested function.\n");
1268 /* See if it makes sense to try to split.
1269 It makes sense to split if we inline, that is if we have direct calls to
1270 handle or direct calls are possibly going to appear as result of indirect
1271 inlining or LTO. Also handle -fprofile-generate as LTO to allow non-LTO
1272 training for LTO -fprofile-use build.
1274 Note that we are not completely conservative about disqualifying functions
1275 called once. It is possible that the caller is called more then once and
1276 then inlining would still benefit. */
1277 if ((!node
->callers
|| !node
->callers
->next_caller
)
1278 && !node
->address_taken
1279 && (!flag_lto
|| !node
->local
.externally_visible
))
1282 fprintf (dump_file
, "Not splitting: not called directly "
1283 "or called once.\n");
1287 /* FIXME: We can actually split if splitting reduces call overhead. */
1288 if (!flag_inline_small_functions
1289 && !DECL_DECLARED_INLINE_P (current_function_decl
))
1292 fprintf (dump_file
, "Not splitting: not autoinlining and function"
1293 " is not inline.\n");
1297 /* Compute local info about basic blocks and determine function size/time. */
1298 VEC_safe_grow_cleared (bb_info
, heap
, bb_info_vec
, last_basic_block
+ 1);
1299 memset (&best_split_point
, 0, sizeof (best_split_point
));
1304 int freq
= compute_call_stmt_bb_frequency (current_function_decl
, bb
);
1306 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1307 fprintf (dump_file
, "Basic block %i\n", bb
->index
);
1309 for (bsi
= gsi_start_bb (bb
); !gsi_end_p (bsi
); gsi_next (&bsi
))
1311 int this_time
, this_size
;
1312 gimple stmt
= gsi_stmt (bsi
);
1314 this_size
= estimate_num_insns (stmt
, &eni_size_weights
);
1315 this_time
= estimate_num_insns (stmt
, &eni_time_weights
) * freq
;
1319 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1321 fprintf (dump_file
, " freq:%6i size:%3i time:%3i ",
1322 freq
, this_size
, this_time
);
1323 print_gimple_stmt (dump_file
, stmt
, 0, 0);
1326 overall_time
+= time
;
1327 overall_size
+= size
;
1328 VEC_index (bb_info
, bb_info_vec
, bb
->index
)->time
= time
;
1329 VEC_index (bb_info
, bb_info_vec
, bb
->index
)->size
= size
;
1331 find_split_points (overall_time
, overall_size
);
1332 if (best_split_point
.split_bbs
)
1334 split_function (&best_split_point
);
1335 BITMAP_FREE (best_split_point
.ssa_names_to_pass
);
1336 BITMAP_FREE (best_split_point
.split_bbs
);
1337 todo
= TODO_update_ssa
| TODO_cleanup_cfg
;
1339 VEC_free (bb_info
, heap
, bb_info_vec
);
1344 /* Gate function splitting pass. When doing profile feedback, we want
1345 to execute the pass after profiling is read. So disable one in
1346 early optimization. */
1349 gate_split_functions (void)
1351 return (flag_partial_inlining
1352 && !profile_arc_flag
&& !flag_branch_probabilities
);
1355 struct gimple_opt_pass pass_split_functions
=
1359 "fnsplit", /* name */
1360 gate_split_functions
, /* gate */
1361 execute_split_functions
, /* execute */
1364 0, /* static_pass_number */
1365 TV_IPA_FNSPLIT
, /* tv_id */
1366 PROP_cfg
, /* properties_required */
1367 0, /* properties_provided */
1368 0, /* properties_destroyed */
1369 0, /* todo_flags_start */
1370 TODO_dump_func
/* todo_flags_finish */
1374 /* Gate feedback driven function splitting pass.
1375 We don't need to split when profiling at all, we are producing
1376 lousy code anyway. */
1379 gate_feedback_split_functions (void)
1381 return (flag_partial_inlining
1382 && flag_branch_probabilities
);
1385 /* Execute function splitting pass. */
1388 execute_feedback_split_functions (void)
1390 unsigned int retval
= execute_split_functions ();
1392 retval
|= TODO_rebuild_cgraph_edges
;
1396 struct gimple_opt_pass pass_feedback_split_functions
=
1400 "feedback_fnsplit", /* name */
1401 gate_feedback_split_functions
, /* gate */
1402 execute_feedback_split_functions
, /* execute */
1405 0, /* static_pass_number */
1406 TV_IPA_FNSPLIT
, /* tv_id */
1407 PROP_cfg
, /* properties_required */
1408 0, /* properties_provided */
1409 0, /* properties_destroyed */
1410 0, /* todo_flags_start */
1411 TODO_dump_func
/* todo_flags_finish */