1 /* Vectorizer Specific Loop Manipulations
2 Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2012
3 Free Software Foundation, Inc.
4 Contributed by Dorit Naishlos <dorit@il.ibm.com>
5 and Ira Rosen <irar@il.ibm.com>
7 This file is part of GCC.
9 GCC is free software; you can redistribute it and/or modify it under
10 the terms of the GNU General Public License as published by the Free
11 Software Foundation; either version 3, or (at your option) any later
14 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 You should have received a copy of the GNU General Public License
20 along with GCC; see the file COPYING3. If not see
21 <http://www.gnu.org/licenses/>. */
25 #include "coretypes.h"
29 #include "basic-block.h"
30 #include "tree-pretty-print.h"
31 #include "gimple-pretty-print.h"
32 #include "tree-flow.h"
33 #include "tree-dump.h"
35 #include "cfglayout.h"
36 #include "diagnostic-core.h"
37 #include "tree-scalar-evolution.h"
38 #include "tree-vectorizer.h"
39 #include "langhooks.h"
41 /*************************************************************************
42 Simple Loop Peeling Utilities
44 Utilities to support loop peeling for vectorization purposes.
45 *************************************************************************/
48 /* Renames the use *OP_P. */
51 rename_use_op (use_operand_p op_p
)
55 if (TREE_CODE (USE_FROM_PTR (op_p
)) != SSA_NAME
)
58 new_name
= get_current_def (USE_FROM_PTR (op_p
));
60 /* Something defined outside of the loop. */
64 /* An ordinary ssa name defined in the loop. */
66 SET_USE (op_p
, new_name
);
70 /* Renames the variables in basic block BB. */
73 rename_variables_in_bb (basic_block bb
)
75 gimple_stmt_iterator gsi
;
81 struct loop
*loop
= bb
->loop_father
;
83 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
85 stmt
= gsi_stmt (gsi
);
86 FOR_EACH_SSA_USE_OPERAND (use_p
, stmt
, iter
, SSA_OP_ALL_USES
)
87 rename_use_op (use_p
);
90 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
92 if (!flow_bb_inside_loop_p (loop
, e
->dest
))
94 for (gsi
= gsi_start_phis (e
->dest
); !gsi_end_p (gsi
); gsi_next (&gsi
))
95 rename_use_op (PHI_ARG_DEF_PTR_FROM_EDGE (gsi_stmt (gsi
), e
));
100 /* Renames variables in new generated LOOP. */
103 rename_variables_in_loop (struct loop
*loop
)
108 bbs
= get_loop_body (loop
);
110 for (i
= 0; i
< loop
->num_nodes
; i
++)
111 rename_variables_in_bb (bbs
[i
]);
122 DEF_VEC_O(adjust_info
);
123 DEF_VEC_ALLOC_O_STACK(adjust_info
);
124 #define VEC_adjust_info_stack_alloc(alloc) VEC_stack_alloc (adjust_info, alloc)
126 /* A stack of values to be adjusted in debug stmts. We have to
127 process them LIFO, so that the closest substitution applies. If we
128 processed them FIFO, without the stack, we might substitute uses
129 with a PHI DEF that would soon become non-dominant, and when we got
130 to the suitable one, it wouldn't have anything to substitute any
132 static VEC(adjust_info
, stack
) *adjust_vec
;
134 /* Adjust any debug stmts that referenced AI->from values to use the
135 loop-closed AI->to, if the references are dominated by AI->bb and
136 not by the definition of AI->from. */
139 adjust_debug_stmts_now (adjust_info
*ai
)
141 basic_block bbphi
= ai
->bb
;
142 tree orig_def
= ai
->from
;
143 tree new_def
= ai
->to
;
144 imm_use_iterator imm_iter
;
146 basic_block bbdef
= gimple_bb (SSA_NAME_DEF_STMT (orig_def
));
148 gcc_assert (dom_info_available_p (CDI_DOMINATORS
));
150 /* Adjust any debug stmts that held onto non-loop-closed
152 FOR_EACH_IMM_USE_STMT (stmt
, imm_iter
, orig_def
)
157 if (!is_gimple_debug (stmt
))
160 gcc_assert (gimple_debug_bind_p (stmt
));
162 bbuse
= gimple_bb (stmt
);
165 || dominated_by_p (CDI_DOMINATORS
, bbuse
, bbphi
))
167 || dominated_by_p (CDI_DOMINATORS
, bbuse
, bbdef
)))
170 FOR_EACH_IMM_USE_ON_STMT (use_p
, imm_iter
)
171 SET_USE (use_p
, new_def
);
174 gimple_debug_bind_reset_value (stmt
);
181 /* Adjust debug stmts as scheduled before. */
184 adjust_vec_debug_stmts (void)
186 if (!MAY_HAVE_DEBUG_STMTS
)
189 gcc_assert (adjust_vec
);
191 while (!VEC_empty (adjust_info
, adjust_vec
))
193 adjust_debug_stmts_now (VEC_last (adjust_info
, adjust_vec
));
194 VEC_pop (adjust_info
, adjust_vec
);
197 VEC_free (adjust_info
, stack
, adjust_vec
);
200 /* Adjust any debug stmts that referenced FROM values to use the
201 loop-closed TO, if the references are dominated by BB and not by
202 the definition of FROM. If adjust_vec is non-NULL, adjustments
203 will be postponed until adjust_vec_debug_stmts is called. */
206 adjust_debug_stmts (tree from
, tree to
, basic_block bb
)
210 if (MAY_HAVE_DEBUG_STMTS
&& TREE_CODE (from
) == SSA_NAME
211 && SSA_NAME_VAR (from
) != gimple_vop (cfun
))
218 VEC_safe_push (adjust_info
, stack
, adjust_vec
, &ai
);
220 adjust_debug_stmts_now (&ai
);
224 /* Change E's phi arg in UPDATE_PHI to NEW_DEF, and record information
225 to adjust any debug stmts that referenced the old phi arg,
226 presumably non-loop-closed references left over from other
230 adjust_phi_and_debug_stmts (gimple update_phi
, edge e
, tree new_def
)
232 tree orig_def
= PHI_ARG_DEF_FROM_EDGE (update_phi
, e
);
234 SET_PHI_ARG_DEF (update_phi
, e
->dest_idx
, new_def
);
236 if (MAY_HAVE_DEBUG_STMTS
)
237 adjust_debug_stmts (orig_def
, PHI_RESULT (update_phi
),
238 gimple_bb (update_phi
));
242 /* Update the PHI nodes of NEW_LOOP.
244 NEW_LOOP is a duplicate of ORIG_LOOP.
245 AFTER indicates whether NEW_LOOP executes before or after ORIG_LOOP:
246 AFTER is true if NEW_LOOP executes after ORIG_LOOP, and false if it
247 executes before it. */
250 slpeel_update_phis_for_duplicate_loop (struct loop
*orig_loop
,
251 struct loop
*new_loop
, bool after
)
254 gimple phi_new
, phi_orig
;
256 edge orig_loop_latch
= loop_latch_edge (orig_loop
);
257 edge orig_entry_e
= loop_preheader_edge (orig_loop
);
258 edge new_loop_exit_e
= single_exit (new_loop
);
259 edge new_loop_entry_e
= loop_preheader_edge (new_loop
);
260 edge entry_arg_e
= (after
? orig_loop_latch
: orig_entry_e
);
261 gimple_stmt_iterator gsi_new
, gsi_orig
;
264 step 1. For each loop-header-phi:
265 Add the first phi argument for the phi in NEW_LOOP
266 (the one associated with the entry of NEW_LOOP)
268 step 2. For each loop-header-phi:
269 Add the second phi argument for the phi in NEW_LOOP
270 (the one associated with the latch of NEW_LOOP)
272 step 3. Update the phis in the successor block of NEW_LOOP.
274 case 1: NEW_LOOP was placed before ORIG_LOOP:
275 The successor block of NEW_LOOP is the header of ORIG_LOOP.
276 Updating the phis in the successor block can therefore be done
277 along with the scanning of the loop header phis, because the
278 header blocks of ORIG_LOOP and NEW_LOOP have exactly the same
279 phi nodes, organized in the same order.
281 case 2: NEW_LOOP was placed after ORIG_LOOP:
282 The successor block of NEW_LOOP is the original exit block of
283 ORIG_LOOP - the phis to be updated are the loop-closed-ssa phis.
284 We postpone updating these phis to a later stage (when
285 loop guards are added).
289 /* Scan the phis in the headers of the old and new loops
290 (they are organized in exactly the same order). */
292 for (gsi_new
= gsi_start_phis (new_loop
->header
),
293 gsi_orig
= gsi_start_phis (orig_loop
->header
);
294 !gsi_end_p (gsi_new
) && !gsi_end_p (gsi_orig
);
295 gsi_next (&gsi_new
), gsi_next (&gsi_orig
))
297 source_location locus
;
298 phi_new
= gsi_stmt (gsi_new
);
299 phi_orig
= gsi_stmt (gsi_orig
);
302 def
= PHI_ARG_DEF_FROM_EDGE (phi_orig
, entry_arg_e
);
303 locus
= gimple_phi_arg_location_from_edge (phi_orig
, entry_arg_e
);
304 add_phi_arg (phi_new
, def
, new_loop_entry_e
, locus
);
307 def
= PHI_ARG_DEF_FROM_EDGE (phi_orig
, orig_loop_latch
);
308 locus
= gimple_phi_arg_location_from_edge (phi_orig
, orig_loop_latch
);
309 if (TREE_CODE (def
) != SSA_NAME
)
312 new_ssa_name
= get_current_def (def
);
315 /* This only happens if there are no definitions
316 inside the loop. use the phi_result in this case. */
317 new_ssa_name
= PHI_RESULT (phi_new
);
320 /* An ordinary ssa name defined in the loop. */
321 add_phi_arg (phi_new
, new_ssa_name
, loop_latch_edge (new_loop
), locus
);
323 /* Drop any debug references outside the loop, if they would
324 become ill-formed SSA. */
325 adjust_debug_stmts (def
, NULL
, single_exit (orig_loop
)->dest
);
327 /* step 3 (case 1). */
330 gcc_assert (new_loop_exit_e
== orig_entry_e
);
331 adjust_phi_and_debug_stmts (phi_orig
, new_loop_exit_e
, new_ssa_name
);
337 /* Update PHI nodes for a guard of the LOOP.
340 - LOOP, GUARD_EDGE: LOOP is a loop for which we added guard code that
341 controls whether LOOP is to be executed. GUARD_EDGE is the edge that
342 originates from the guard-bb, skips LOOP and reaches the (unique) exit
343 bb of LOOP. This loop-exit-bb is an empty bb with one successor.
344 We denote this bb NEW_MERGE_BB because before the guard code was added
345 it had a single predecessor (the LOOP header), and now it became a merge
346 point of two paths - the path that ends with the LOOP exit-edge, and
347 the path that ends with GUARD_EDGE.
348 - NEW_EXIT_BB: New basic block that is added by this function between LOOP
349 and NEW_MERGE_BB. It is used to place loop-closed-ssa-form exit-phis.
351 ===> The CFG before the guard-code was added:
354 if (exit_loop) goto update_bb
355 else goto LOOP_header_bb
358 ==> The CFG after the guard-code was added:
360 if (LOOP_guard_condition) goto new_merge_bb
361 else goto LOOP_header_bb
364 if (exit_loop_condition) goto new_merge_bb
365 else goto LOOP_header_bb
370 ==> The CFG after this function:
372 if (LOOP_guard_condition) goto new_merge_bb
373 else goto LOOP_header_bb
376 if (exit_loop_condition) goto new_exit_bb
377 else goto LOOP_header_bb
384 1. creates and updates the relevant phi nodes to account for the new
385 incoming edge (GUARD_EDGE) into NEW_MERGE_BB. This involves:
386 1.1. Create phi nodes at NEW_MERGE_BB.
387 1.2. Update the phi nodes at the successor of NEW_MERGE_BB (denoted
388 UPDATE_BB). UPDATE_BB was the exit-bb of LOOP before NEW_MERGE_BB
389 2. preserves loop-closed-ssa-form by creating the required phi nodes
390 at the exit of LOOP (i.e, in NEW_EXIT_BB).
392 There are two flavors to this function:
394 slpeel_update_phi_nodes_for_guard1:
395 Here the guard controls whether we enter or skip LOOP, where LOOP is a
396 prolog_loop (loop1 below), and the new phis created in NEW_MERGE_BB are
397 for variables that have phis in the loop header.
399 slpeel_update_phi_nodes_for_guard2:
400 Here the guard controls whether we enter or skip LOOP, where LOOP is an
401 epilog_loop (loop2 below), and the new phis created in NEW_MERGE_BB are
402 for variables that have phis in the loop exit.
404 I.E., the overall structure is:
407 guard1 (goto loop1/merge1_bb)
410 guard2 (goto merge1_bb/merge2_bb)
417 slpeel_update_phi_nodes_for_guard1 takes care of creating phis in
418 loop1_exit_bb and merge1_bb. These are entry phis (phis for the vars
419 that have phis in loop1->header).
421 slpeel_update_phi_nodes_for_guard2 takes care of creating phis in
422 loop2_exit_bb and merge2_bb. These are exit phis (phis for the vars
423 that have phis in next_bb). It also adds some of these phis to
426 slpeel_update_phi_nodes_for_guard1 is always called before
427 slpeel_update_phi_nodes_for_guard2. They are both needed in order
428 to create correct data-flow and loop-closed-ssa-form.
430 Generally slpeel_update_phi_nodes_for_guard1 creates phis for variables
431 that change between iterations of a loop (and therefore have a phi-node
432 at the loop entry), whereas slpeel_update_phi_nodes_for_guard2 creates
433 phis for variables that are used out of the loop (and therefore have
434 loop-closed exit phis). Some variables may be both updated between
435 iterations and used after the loop. This is why in loop1_exit_bb we
436 may need both entry_phis (created by slpeel_update_phi_nodes_for_guard1)
437 and exit phis (created by slpeel_update_phi_nodes_for_guard2).
439 - IS_NEW_LOOP: if IS_NEW_LOOP is true, then LOOP is a newly created copy of
440 an original loop. i.e., we have:
443 guard_bb (goto LOOP/new_merge)
449 If IS_NEW_LOOP is false, then LOOP is an original loop, in which case we
453 guard_bb (goto LOOP/new_merge)
459 The SSA names defined in the original loop have a current
460 reaching definition that that records the corresponding new
461 ssa-name used in the new duplicated loop copy.
464 /* Function slpeel_update_phi_nodes_for_guard1
467 - GUARD_EDGE, LOOP, IS_NEW_LOOP, NEW_EXIT_BB - as explained above.
468 - DEFS - a bitmap of ssa names to mark new names for which we recorded
471 In the context of the overall structure, we have:
474 guard1 (goto loop1/merge1_bb)
477 guard2 (goto merge1_bb/merge2_bb)
484 For each name updated between loop iterations (i.e - for each name that has
485 an entry (loop-header) phi in LOOP) we create a new phi in:
486 1. merge1_bb (to account for the edge from guard1)
487 2. loop1_exit_bb (an exit-phi to keep LOOP in loop-closed form)
491 slpeel_update_phi_nodes_for_guard1 (edge guard_edge
, struct loop
*loop
,
492 bool is_new_loop
, basic_block
*new_exit_bb
,
495 gimple orig_phi
, new_phi
;
496 gimple update_phi
, update_phi2
;
497 tree guard_arg
, loop_arg
;
498 basic_block new_merge_bb
= guard_edge
->dest
;
499 edge e
= EDGE_SUCC (new_merge_bb
, 0);
500 basic_block update_bb
= e
->dest
;
501 basic_block orig_bb
= loop
->header
;
503 tree current_new_name
;
504 gimple_stmt_iterator gsi_orig
, gsi_update
;
506 /* Create new bb between loop and new_merge_bb. */
507 *new_exit_bb
= split_edge (single_exit (loop
));
509 new_exit_e
= EDGE_SUCC (*new_exit_bb
, 0);
511 for (gsi_orig
= gsi_start_phis (orig_bb
),
512 gsi_update
= gsi_start_phis (update_bb
);
513 !gsi_end_p (gsi_orig
) && !gsi_end_p (gsi_update
);
514 gsi_next (&gsi_orig
), gsi_next (&gsi_update
))
516 source_location loop_locus
, guard_locus
;
517 orig_phi
= gsi_stmt (gsi_orig
);
518 update_phi
= gsi_stmt (gsi_update
);
520 /** 1. Handle new-merge-point phis **/
522 /* 1.1. Generate new phi node in NEW_MERGE_BB: */
523 new_phi
= create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi
)),
526 /* 1.2. NEW_MERGE_BB has two incoming edges: GUARD_EDGE and the exit-edge
527 of LOOP. Set the two phi args in NEW_PHI for these edges: */
528 loop_arg
= PHI_ARG_DEF_FROM_EDGE (orig_phi
, EDGE_SUCC (loop
->latch
, 0));
529 loop_locus
= gimple_phi_arg_location_from_edge (orig_phi
,
530 EDGE_SUCC (loop
->latch
,
532 guard_arg
= PHI_ARG_DEF_FROM_EDGE (orig_phi
, loop_preheader_edge (loop
));
534 = gimple_phi_arg_location_from_edge (orig_phi
,
535 loop_preheader_edge (loop
));
537 add_phi_arg (new_phi
, loop_arg
, new_exit_e
, loop_locus
);
538 add_phi_arg (new_phi
, guard_arg
, guard_edge
, guard_locus
);
540 /* 1.3. Update phi in successor block. */
541 gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi
, e
) == loop_arg
542 || PHI_ARG_DEF_FROM_EDGE (update_phi
, e
) == guard_arg
);
543 adjust_phi_and_debug_stmts (update_phi
, e
, PHI_RESULT (new_phi
));
544 update_phi2
= new_phi
;
547 /** 2. Handle loop-closed-ssa-form phis **/
549 if (!is_gimple_reg (PHI_RESULT (orig_phi
)))
552 /* 2.1. Generate new phi node in NEW_EXIT_BB: */
553 new_phi
= create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi
)),
556 /* 2.2. NEW_EXIT_BB has one incoming edge: the exit-edge of the loop. */
557 add_phi_arg (new_phi
, loop_arg
, single_exit (loop
), loop_locus
);
559 /* 2.3. Update phi in successor of NEW_EXIT_BB: */
560 gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi2
, new_exit_e
) == loop_arg
);
561 adjust_phi_and_debug_stmts (update_phi2
, new_exit_e
,
562 PHI_RESULT (new_phi
));
564 /* 2.4. Record the newly created name with set_current_def.
565 We want to find a name such that
566 name = get_current_def (orig_loop_name)
567 and to set its current definition as follows:
568 set_current_def (name, new_phi_name)
570 If LOOP is a new loop then loop_arg is already the name we're
571 looking for. If LOOP is the original loop, then loop_arg is
572 the orig_loop_name and the relevant name is recorded in its
573 current reaching definition. */
575 current_new_name
= loop_arg
;
578 current_new_name
= get_current_def (loop_arg
);
579 /* current_def is not available only if the variable does not
580 change inside the loop, in which case we also don't care
581 about recording a current_def for it because we won't be
582 trying to create loop-exit-phis for it. */
583 if (!current_new_name
)
586 gcc_assert (get_current_def (current_new_name
) == NULL_TREE
);
588 set_current_def (current_new_name
, PHI_RESULT (new_phi
));
589 bitmap_set_bit (*defs
, SSA_NAME_VERSION (current_new_name
));
594 /* Function slpeel_update_phi_nodes_for_guard2
597 - GUARD_EDGE, LOOP, IS_NEW_LOOP, NEW_EXIT_BB - as explained above.
599 In the context of the overall structure, we have:
602 guard1 (goto loop1/merge1_bb)
605 guard2 (goto merge1_bb/merge2_bb)
612 For each name used out side the loop (i.e - for each name that has an exit
613 phi in next_bb) we create a new phi in:
614 1. merge2_bb (to account for the edge from guard_bb)
615 2. loop2_exit_bb (an exit-phi to keep LOOP in loop-closed form)
616 3. guard2 bb (an exit phi to keep the preceding loop in loop-closed form),
617 if needed (if it wasn't handled by slpeel_update_phis_nodes_for_phi1).
621 slpeel_update_phi_nodes_for_guard2 (edge guard_edge
, struct loop
*loop
,
622 bool is_new_loop
, basic_block
*new_exit_bb
)
624 gimple orig_phi
, new_phi
;
625 gimple update_phi
, update_phi2
;
626 tree guard_arg
, loop_arg
;
627 basic_block new_merge_bb
= guard_edge
->dest
;
628 edge e
= EDGE_SUCC (new_merge_bb
, 0);
629 basic_block update_bb
= e
->dest
;
631 tree orig_def
, orig_def_new_name
;
632 tree new_name
, new_name2
;
634 gimple_stmt_iterator gsi
;
636 /* Create new bb between loop and new_merge_bb. */
637 *new_exit_bb
= split_edge (single_exit (loop
));
639 new_exit_e
= EDGE_SUCC (*new_exit_bb
, 0);
641 for (gsi
= gsi_start_phis (update_bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
643 update_phi
= gsi_stmt (gsi
);
644 orig_phi
= update_phi
;
645 orig_def
= PHI_ARG_DEF_FROM_EDGE (orig_phi
, e
);
646 /* This loop-closed-phi actually doesn't represent a use
647 out of the loop - the phi arg is a constant. */
648 if (TREE_CODE (orig_def
) != SSA_NAME
)
650 orig_def_new_name
= get_current_def (orig_def
);
653 /** 1. Handle new-merge-point phis **/
655 /* 1.1. Generate new phi node in NEW_MERGE_BB: */
656 new_phi
= create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi
)),
659 /* 1.2. NEW_MERGE_BB has two incoming edges: GUARD_EDGE and the exit-edge
660 of LOOP. Set the two PHI args in NEW_PHI for these edges: */
662 new_name2
= NULL_TREE
;
663 if (orig_def_new_name
)
665 new_name
= orig_def_new_name
;
666 /* Some variables have both loop-entry-phis and loop-exit-phis.
667 Such variables were given yet newer names by phis placed in
668 guard_bb by slpeel_update_phi_nodes_for_guard1. I.e:
669 new_name2 = get_current_def (get_current_def (orig_name)). */
670 new_name2
= get_current_def (new_name
);
675 guard_arg
= orig_def
;
680 guard_arg
= new_name
;
684 guard_arg
= new_name2
;
686 add_phi_arg (new_phi
, loop_arg
, new_exit_e
, UNKNOWN_LOCATION
);
687 add_phi_arg (new_phi
, guard_arg
, guard_edge
, UNKNOWN_LOCATION
);
689 /* 1.3. Update phi in successor block. */
690 gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi
, e
) == orig_def
);
691 adjust_phi_and_debug_stmts (update_phi
, e
, PHI_RESULT (new_phi
));
692 update_phi2
= new_phi
;
695 /** 2. Handle loop-closed-ssa-form phis **/
697 /* 2.1. Generate new phi node in NEW_EXIT_BB: */
698 new_phi
= create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi
)),
701 /* 2.2. NEW_EXIT_BB has one incoming edge: the exit-edge of the loop. */
702 add_phi_arg (new_phi
, loop_arg
, single_exit (loop
), UNKNOWN_LOCATION
);
704 /* 2.3. Update phi in successor of NEW_EXIT_BB: */
705 gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi2
, new_exit_e
) == loop_arg
);
706 adjust_phi_and_debug_stmts (update_phi2
, new_exit_e
,
707 PHI_RESULT (new_phi
));
710 /** 3. Handle loop-closed-ssa-form phis for first loop **/
712 /* 3.1. Find the relevant names that need an exit-phi in
713 GUARD_BB, i.e. names for which
714 slpeel_update_phi_nodes_for_guard1 had not already created a
715 phi node. This is the case for names that are used outside
716 the loop (and therefore need an exit phi) but are not updated
717 across loop iterations (and therefore don't have a
720 slpeel_update_phi_nodes_for_guard1 is responsible for
721 creating loop-exit phis in GUARD_BB for names that have a
722 loop-header-phi. When such a phi is created we also record
723 the new name in its current definition. If this new name
724 exists, then guard_arg was set to this new name (see 1.2
725 above). Therefore, if guard_arg is not this new name, this
726 is an indication that an exit-phi in GUARD_BB was not yet
727 created, so we take care of it here. */
728 if (guard_arg
== new_name2
)
732 /* 3.2. Generate new phi node in GUARD_BB: */
733 new_phi
= create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi
)),
736 /* 3.3. GUARD_BB has one incoming edge: */
737 gcc_assert (EDGE_COUNT (guard_edge
->src
->preds
) == 1);
738 add_phi_arg (new_phi
, arg
, EDGE_PRED (guard_edge
->src
, 0),
741 /* 3.4. Update phi in successor of GUARD_BB: */
742 gcc_assert (PHI_ARG_DEF_FROM_EDGE (update_phi2
, guard_edge
)
744 adjust_phi_and_debug_stmts (update_phi2
, guard_edge
,
745 PHI_RESULT (new_phi
));
750 /* Make the LOOP iterate NITERS times. This is done by adding a new IV
751 that starts at zero, increases by one and its limit is NITERS.
753 Assumption: the exit-condition of LOOP is the last stmt in the loop. */
756 slpeel_make_loop_iterate_ntimes (struct loop
*loop
, tree niters
)
758 tree indx_before_incr
, indx_after_incr
;
761 edge exit_edge
= single_exit (loop
);
762 gimple_stmt_iterator loop_cond_gsi
;
763 gimple_stmt_iterator incr_gsi
;
765 tree init
= build_int_cst (TREE_TYPE (niters
), 0);
766 tree step
= build_int_cst (TREE_TYPE (niters
), 1);
770 orig_cond
= get_loop_exit_condition (loop
);
771 gcc_assert (orig_cond
);
772 loop_cond_gsi
= gsi_for_stmt (orig_cond
);
774 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
775 create_iv (init
, step
, NULL_TREE
, loop
,
776 &incr_gsi
, insert_after
, &indx_before_incr
, &indx_after_incr
);
778 indx_after_incr
= force_gimple_operand_gsi (&loop_cond_gsi
, indx_after_incr
,
779 true, NULL_TREE
, true,
781 niters
= force_gimple_operand_gsi (&loop_cond_gsi
, niters
, true, NULL_TREE
,
782 true, GSI_SAME_STMT
);
784 code
= (exit_edge
->flags
& EDGE_TRUE_VALUE
) ? GE_EXPR
: LT_EXPR
;
785 cond_stmt
= gimple_build_cond (code
, indx_after_incr
, niters
, NULL_TREE
,
788 gsi_insert_before (&loop_cond_gsi
, cond_stmt
, GSI_SAME_STMT
);
790 /* Remove old loop exit test: */
791 gsi_remove (&loop_cond_gsi
, true);
793 loop_loc
= find_loop_location (loop
);
794 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
796 if (loop_loc
!= UNKNOWN_LOC
)
797 fprintf (dump_file
, "\nloop at %s:%d: ",
798 LOC_FILE (loop_loc
), LOC_LINE (loop_loc
));
799 print_gimple_stmt (dump_file
, cond_stmt
, 0, TDF_SLIM
);
802 loop
->nb_iterations
= niters
;
806 /* Given LOOP this function generates a new copy of it and puts it
807 on E which is either the entry or exit of LOOP. */
810 slpeel_tree_duplicate_loop_to_edge_cfg (struct loop
*loop
, edge e
)
812 struct loop
*new_loop
;
813 basic_block
*new_bbs
, *bbs
;
816 basic_block exit_dest
;
820 gimple_stmt_iterator gsi
;
822 at_exit
= (e
== single_exit (loop
));
823 if (!at_exit
&& e
!= loop_preheader_edge (loop
))
826 bbs
= get_loop_body (loop
);
828 /* Check whether duplication is possible. */
829 if (!can_copy_bbs_p (bbs
, loop
->num_nodes
))
835 /* Generate new loop structure. */
836 new_loop
= duplicate_loop (loop
, loop_outer (loop
));
843 exit_dest
= single_exit (loop
)->dest
;
844 was_imm_dom
= (get_immediate_dominator (CDI_DOMINATORS
,
845 exit_dest
) == loop
->header
?
848 new_bbs
= XNEWVEC (basic_block
, loop
->num_nodes
);
850 exit
= single_exit (loop
);
851 copy_bbs (bbs
, loop
->num_nodes
, new_bbs
,
852 &exit
, 1, &new_exit
, NULL
,
855 /* Duplicating phi args at exit bbs as coming
856 also from exit of duplicated loop. */
857 for (gsi
= gsi_start_phis (exit_dest
); !gsi_end_p (gsi
); gsi_next (&gsi
))
859 phi
= gsi_stmt (gsi
);
860 phi_arg
= PHI_ARG_DEF_FROM_EDGE (phi
, single_exit (loop
));
863 edge new_loop_exit_edge
;
864 source_location locus
;
866 locus
= gimple_phi_arg_location_from_edge (phi
, single_exit (loop
));
867 if (EDGE_SUCC (new_loop
->header
, 0)->dest
== new_loop
->latch
)
868 new_loop_exit_edge
= EDGE_SUCC (new_loop
->header
, 1);
870 new_loop_exit_edge
= EDGE_SUCC (new_loop
->header
, 0);
872 add_phi_arg (phi
, phi_arg
, new_loop_exit_edge
, locus
);
876 if (at_exit
) /* Add the loop copy at exit. */
878 redirect_edge_and_branch_force (e
, new_loop
->header
);
879 PENDING_STMT (e
) = NULL
;
880 set_immediate_dominator (CDI_DOMINATORS
, new_loop
->header
, e
->src
);
882 set_immediate_dominator (CDI_DOMINATORS
, exit_dest
, new_loop
->header
);
884 else /* Add the copy at entry. */
887 edge entry_e
= loop_preheader_edge (loop
);
888 basic_block preheader
= entry_e
->src
;
890 if (!flow_bb_inside_loop_p (new_loop
,
891 EDGE_SUCC (new_loop
->header
, 0)->dest
))
892 new_exit_e
= EDGE_SUCC (new_loop
->header
, 0);
894 new_exit_e
= EDGE_SUCC (new_loop
->header
, 1);
896 redirect_edge_and_branch_force (new_exit_e
, loop
->header
);
897 PENDING_STMT (new_exit_e
) = NULL
;
898 set_immediate_dominator (CDI_DOMINATORS
, loop
->header
,
901 /* We have to add phi args to the loop->header here as coming
902 from new_exit_e edge. */
903 for (gsi
= gsi_start_phis (loop
->header
);
907 phi
= gsi_stmt (gsi
);
908 phi_arg
= PHI_ARG_DEF_FROM_EDGE (phi
, entry_e
);
910 add_phi_arg (phi
, phi_arg
, new_exit_e
,
911 gimple_phi_arg_location_from_edge (phi
, entry_e
));
914 redirect_edge_and_branch_force (entry_e
, new_loop
->header
);
915 PENDING_STMT (entry_e
) = NULL
;
916 set_immediate_dominator (CDI_DOMINATORS
, new_loop
->header
, preheader
);
926 /* Given the condition statement COND, put it as the last statement
927 of GUARD_BB; EXIT_BB is the basic block to skip the loop;
928 Assumes that this is the single exit of the guarded loop.
929 Returns the skip edge, inserts new stmts on the COND_EXPR_STMT_LIST. */
932 slpeel_add_loop_guard (basic_block guard_bb
, tree cond
,
933 gimple_seq cond_expr_stmt_list
,
934 basic_block exit_bb
, basic_block dom_bb
)
936 gimple_stmt_iterator gsi
;
939 gimple_seq gimplify_stmt_list
= NULL
;
941 enter_e
= EDGE_SUCC (guard_bb
, 0);
942 enter_e
->flags
&= ~EDGE_FALLTHRU
;
943 enter_e
->flags
|= EDGE_FALSE_VALUE
;
944 gsi
= gsi_last_bb (guard_bb
);
946 cond
= force_gimple_operand_1 (cond
, &gimplify_stmt_list
, is_gimple_condexpr
,
948 if (gimplify_stmt_list
)
949 gimple_seq_add_seq (&cond_expr_stmt_list
, gimplify_stmt_list
);
950 cond_stmt
= gimple_build_cond_from_tree (cond
, NULL_TREE
, NULL_TREE
);
951 if (cond_expr_stmt_list
)
952 gsi_insert_seq_after (&gsi
, cond_expr_stmt_list
, GSI_NEW_STMT
);
954 gsi
= gsi_last_bb (guard_bb
);
955 gsi_insert_after (&gsi
, cond_stmt
, GSI_NEW_STMT
);
957 /* Add new edge to connect guard block to the merge/loop-exit block. */
958 new_e
= make_edge (guard_bb
, exit_bb
, EDGE_TRUE_VALUE
);
959 set_immediate_dominator (CDI_DOMINATORS
, exit_bb
, dom_bb
);
964 /* This function verifies that the following restrictions apply to LOOP:
966 (2) it consists of exactly 2 basic blocks - header, and an empty latch.
967 (3) it is single entry, single exit
968 (4) its exit condition is the last stmt in the header
969 (5) E is the entry/exit edge of LOOP.
973 slpeel_can_duplicate_loop_p (const struct loop
*loop
, const_edge e
)
975 edge exit_e
= single_exit (loop
);
976 edge entry_e
= loop_preheader_edge (loop
);
977 gimple orig_cond
= get_loop_exit_condition (loop
);
978 gimple_stmt_iterator loop_exit_gsi
= gsi_last_bb (exit_e
->src
);
980 if (need_ssa_update_p (cfun
))
984 /* All loops have an outer scope; the only case loop->outer is NULL is for
985 the function itself. */
986 || !loop_outer (loop
)
987 || loop
->num_nodes
!= 2
988 || !empty_block_p (loop
->latch
)
989 || !single_exit (loop
)
990 /* Verify that new loop exit condition can be trivially modified. */
991 || (!orig_cond
|| orig_cond
!= gsi_stmt (loop_exit_gsi
))
992 || (e
!= exit_e
&& e
!= entry_e
))
998 #ifdef ENABLE_CHECKING
1000 slpeel_verify_cfg_after_peeling (struct loop
*first_loop
,
1001 struct loop
*second_loop
)
1003 basic_block loop1_exit_bb
= single_exit (first_loop
)->dest
;
1004 basic_block loop2_entry_bb
= loop_preheader_edge (second_loop
)->src
;
1005 basic_block loop1_entry_bb
= loop_preheader_edge (first_loop
)->src
;
1007 /* A guard that controls whether the second_loop is to be executed or skipped
1008 is placed in first_loop->exit. first_loop->exit therefore has two
1009 successors - one is the preheader of second_loop, and the other is a bb
1012 gcc_assert (EDGE_COUNT (loop1_exit_bb
->succs
) == 2);
1014 /* 1. Verify that one of the successors of first_loop->exit is the preheader
1017 /* The preheader of new_loop is expected to have two predecessors:
1018 first_loop->exit and the block that precedes first_loop. */
1020 gcc_assert (EDGE_COUNT (loop2_entry_bb
->preds
) == 2
1021 && ((EDGE_PRED (loop2_entry_bb
, 0)->src
== loop1_exit_bb
1022 && EDGE_PRED (loop2_entry_bb
, 1)->src
== loop1_entry_bb
)
1023 || (EDGE_PRED (loop2_entry_bb
, 1)->src
== loop1_exit_bb
1024 && EDGE_PRED (loop2_entry_bb
, 0)->src
== loop1_entry_bb
)));
1026 /* Verify that the other successor of first_loop->exit is after the
1032 /* If the run time cost model check determines that vectorization is
1033 not profitable and hence scalar loop should be generated then set
1034 FIRST_NITERS to prologue peeled iterations. This will allow all the
1035 iterations to be executed in the prologue peeled scalar loop. */
1038 set_prologue_iterations (basic_block bb_before_first_loop
,
1044 basic_block cond_bb
, then_bb
;
1045 tree var
, prologue_after_cost_adjust_name
;
1046 gimple_stmt_iterator gsi
;
1048 edge e_true
, e_false
, e_fallthru
;
1050 gimple_seq stmts
= NULL
;
1051 tree cost_pre_condition
= NULL_TREE
;
1052 tree scalar_loop_iters
=
1053 unshare_expr (LOOP_VINFO_NITERS_UNCHANGED (loop_vec_info_for_loop (loop
)));
1055 e
= single_pred_edge (bb_before_first_loop
);
1056 cond_bb
= split_edge(e
);
1058 e
= single_pred_edge (bb_before_first_loop
);
1059 then_bb
= split_edge(e
);
1060 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
1062 e_false
= make_single_succ_edge (cond_bb
, bb_before_first_loop
,
1064 set_immediate_dominator (CDI_DOMINATORS
, bb_before_first_loop
, cond_bb
);
1066 e_true
= EDGE_PRED (then_bb
, 0);
1067 e_true
->flags
&= ~EDGE_FALLTHRU
;
1068 e_true
->flags
|= EDGE_TRUE_VALUE
;
1070 e_fallthru
= EDGE_SUCC (then_bb
, 0);
1072 gsi
= gsi_last_bb (cond_bb
);
1073 cost_pre_condition
=
1074 fold_build2 (LE_EXPR
, boolean_type_node
, scalar_loop_iters
,
1075 build_int_cst (TREE_TYPE (scalar_loop_iters
), th
));
1076 cost_pre_condition
=
1077 force_gimple_operand_gsi_1 (&gsi
, cost_pre_condition
, is_gimple_condexpr
,
1078 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
1079 cond_stmt
= gimple_build_cond_from_tree (cost_pre_condition
,
1080 NULL_TREE
, NULL_TREE
);
1081 gsi_insert_after (&gsi
, cond_stmt
, GSI_NEW_STMT
);
1083 var
= create_tmp_var (TREE_TYPE (scalar_loop_iters
),
1084 "prologue_after_cost_adjust");
1085 add_referenced_var (var
);
1086 prologue_after_cost_adjust_name
=
1087 force_gimple_operand (scalar_loop_iters
, &stmts
, false, var
);
1089 gsi
= gsi_last_bb (then_bb
);
1091 gsi_insert_seq_after (&gsi
, stmts
, GSI_NEW_STMT
);
1093 newphi
= create_phi_node (var
, bb_before_first_loop
);
1094 add_phi_arg (newphi
, prologue_after_cost_adjust_name
, e_fallthru
,
1096 add_phi_arg (newphi
, *first_niters
, e_false
, UNKNOWN_LOCATION
);
1098 *first_niters
= PHI_RESULT (newphi
);
1101 /* Function slpeel_tree_peel_loop_to_edge.
1103 Peel the first (last) iterations of LOOP into a new prolog (epilog) loop
1104 that is placed on the entry (exit) edge E of LOOP. After this transformation
1105 we have two loops one after the other - first-loop iterates FIRST_NITERS
1106 times, and second-loop iterates the remainder NITERS - FIRST_NITERS times.
1107 If the cost model indicates that it is profitable to emit a scalar
1108 loop instead of the vector one, then the prolog (epilog) loop will iterate
1109 for the entire unchanged scalar iterations of the loop.
1112 - LOOP: the loop to be peeled.
1113 - E: the exit or entry edge of LOOP.
1114 If it is the entry edge, we peel the first iterations of LOOP. In this
1115 case first-loop is LOOP, and second-loop is the newly created loop.
1116 If it is the exit edge, we peel the last iterations of LOOP. In this
1117 case, first-loop is the newly created loop, and second-loop is LOOP.
1118 - NITERS: the number of iterations that LOOP iterates.
1119 - FIRST_NITERS: the number of iterations that the first-loop should iterate.
1120 - UPDATE_FIRST_LOOP_COUNT: specified whether this function is responsible
1121 for updating the loop bound of the first-loop to FIRST_NITERS. If it
1122 is false, the caller of this function may want to take care of this
1123 (this can be useful if we don't want new stmts added to first-loop).
1124 - TH: cost model profitability threshold of iterations for vectorization.
1125 - CHECK_PROFITABILITY: specify whether cost model check has not occurred
1126 during versioning and hence needs to occur during
1127 prologue generation or whether cost model check
1128 has not occurred during prologue generation and hence
1129 needs to occur during epilogue generation.
1133 The function returns a pointer to the new loop-copy, or NULL if it failed
1134 to perform the transformation.
1136 The function generates two if-then-else guards: one before the first loop,
1137 and the other before the second loop:
1139 if (FIRST_NITERS == 0) then skip the first loop,
1140 and go directly to the second loop.
1141 The second guard is:
1142 if (FIRST_NITERS == NITERS) then skip the second loop.
1144 If the optional COND_EXPR and COND_EXPR_STMT_LIST arguments are given
1145 then the generated condition is combined with COND_EXPR and the
1146 statements in COND_EXPR_STMT_LIST are emitted together with it.
1148 FORNOW only simple loops are supported (see slpeel_can_duplicate_loop_p).
1149 FORNOW the resulting code will not be in loop-closed-ssa form.
1153 slpeel_tree_peel_loop_to_edge (struct loop
*loop
,
1154 edge e
, tree
*first_niters
,
1155 tree niters
, bool update_first_loop_count
,
1156 unsigned int th
, bool check_profitability
,
1157 tree cond_expr
, gimple_seq cond_expr_stmt_list
)
1159 struct loop
*new_loop
= NULL
, *first_loop
, *second_loop
;
1161 tree pre_condition
= NULL_TREE
;
1163 basic_block bb_before_second_loop
, bb_after_second_loop
;
1164 basic_block bb_before_first_loop
;
1165 basic_block bb_between_loops
;
1166 basic_block new_exit_bb
;
1167 gimple_stmt_iterator gsi
;
1168 edge exit_e
= single_exit (loop
);
1170 tree cost_pre_condition
= NULL_TREE
;
1172 if (!slpeel_can_duplicate_loop_p (loop
, e
))
1175 /* We have to initialize cfg_hooks. Then, when calling
1176 cfg_hooks->split_edge, the function tree_split_edge
1177 is actually called and, when calling cfg_hooks->duplicate_block,
1178 the function tree_duplicate_bb is called. */
1179 gimple_register_cfg_hooks ();
1181 /* If the loop has a virtual PHI, but exit bb doesn't, create a virtual PHI
1182 in the exit bb and rename all the uses after the loop. This simplifies
1183 the *guard[12] routines, which assume loop closed SSA form for all PHIs
1184 (but normally loop closed SSA form doesn't require virtual PHIs to be
1185 in the same form). Doing this early simplifies the checking what
1186 uses should be renamed. */
1187 for (gsi
= gsi_start_phis (loop
->header
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1188 if (!is_gimple_reg (gimple_phi_result (gsi_stmt (gsi
))))
1190 gimple phi
= gsi_stmt (gsi
);
1191 for (gsi
= gsi_start_phis (exit_e
->dest
);
1192 !gsi_end_p (gsi
); gsi_next (&gsi
))
1193 if (!is_gimple_reg (gimple_phi_result (gsi_stmt (gsi
))))
1195 if (gsi_end_p (gsi
))
1197 gimple new_phi
= create_phi_node (SSA_NAME_VAR (PHI_RESULT (phi
)),
1199 tree vop
= PHI_ARG_DEF_FROM_EDGE (phi
, EDGE_SUCC (loop
->latch
, 0));
1200 imm_use_iterator imm_iter
;
1202 tree new_vop
= make_ssa_name (SSA_NAME_VAR (PHI_RESULT (phi
)),
1204 use_operand_p use_p
;
1206 add_phi_arg (new_phi
, vop
, exit_e
, UNKNOWN_LOCATION
);
1207 gimple_phi_set_result (new_phi
, new_vop
);
1208 FOR_EACH_IMM_USE_STMT (stmt
, imm_iter
, vop
)
1209 if (stmt
!= new_phi
&& gimple_bb (stmt
) != loop
->header
)
1210 FOR_EACH_IMM_USE_ON_STMT (use_p
, imm_iter
)
1211 SET_USE (use_p
, new_vop
);
1216 /* 1. Generate a copy of LOOP and put it on E (E is the entry/exit of LOOP).
1217 Resulting CFG would be:
1230 if (!(new_loop
= slpeel_tree_duplicate_loop_to_edge_cfg (loop
, e
)))
1232 loop_loc
= find_loop_location (loop
);
1233 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
1235 if (loop_loc
!= UNKNOWN_LOC
)
1236 fprintf (dump_file
, "\n%s:%d: note: ",
1237 LOC_FILE (loop_loc
), LOC_LINE (loop_loc
));
1238 fprintf (dump_file
, "tree_duplicate_loop_to_edge_cfg failed.\n");
1243 if (MAY_HAVE_DEBUG_STMTS
)
1245 gcc_assert (!adjust_vec
);
1246 adjust_vec
= VEC_alloc (adjust_info
, stack
, 32);
1251 /* NEW_LOOP was placed after LOOP. */
1253 second_loop
= new_loop
;
1257 /* NEW_LOOP was placed before LOOP. */
1258 first_loop
= new_loop
;
1262 definitions
= ssa_names_to_replace ();
1263 slpeel_update_phis_for_duplicate_loop (loop
, new_loop
, e
== exit_e
);
1264 rename_variables_in_loop (new_loop
);
1267 /* 2. Add the guard code in one of the following ways:
1269 2.a Add the guard that controls whether the first loop is executed.
1270 This occurs when this function is invoked for prologue or epilogue
1271 generation and when the cost model check can be done at compile time.
1273 Resulting CFG would be:
1275 bb_before_first_loop:
1276 if (FIRST_NITERS == 0) GOTO bb_before_second_loop
1283 bb_before_second_loop:
1291 2.b Add the cost model check that allows the prologue
1292 to iterate for the entire unchanged scalar
1293 iterations of the loop in the event that the cost
1294 model indicates that the scalar loop is more
1295 profitable than the vector one. This occurs when
1296 this function is invoked for prologue generation
1297 and the cost model check needs to be done at run
1300 Resulting CFG after prologue peeling would be:
1302 if (scalar_loop_iterations <= th)
1303 FIRST_NITERS = scalar_loop_iterations
1305 bb_before_first_loop:
1306 if (FIRST_NITERS == 0) GOTO bb_before_second_loop
1313 bb_before_second_loop:
1321 2.c Add the cost model check that allows the epilogue
1322 to iterate for the entire unchanged scalar
1323 iterations of the loop in the event that the cost
1324 model indicates that the scalar loop is more
1325 profitable than the vector one. This occurs when
1326 this function is invoked for epilogue generation
1327 and the cost model check needs to be done at run
1328 time. This check is combined with any pre-existing
1329 check in COND_EXPR to avoid versioning.
1331 Resulting CFG after prologue peeling would be:
1333 bb_before_first_loop:
1334 if ((scalar_loop_iterations <= th)
1336 FIRST_NITERS == 0) GOTO bb_before_second_loop
1343 bb_before_second_loop:
1352 bb_before_first_loop
= split_edge (loop_preheader_edge (first_loop
));
1353 bb_before_second_loop
= split_edge (single_exit (first_loop
));
1355 /* Epilogue peeling. */
1356 if (!update_first_loop_count
)
1359 fold_build2 (LE_EXPR
, boolean_type_node
, *first_niters
,
1360 build_int_cst (TREE_TYPE (*first_niters
), 0));
1361 if (check_profitability
)
1363 tree scalar_loop_iters
1364 = unshare_expr (LOOP_VINFO_NITERS_UNCHANGED
1365 (loop_vec_info_for_loop (loop
)));
1366 cost_pre_condition
=
1367 fold_build2 (LE_EXPR
, boolean_type_node
, scalar_loop_iters
,
1368 build_int_cst (TREE_TYPE (scalar_loop_iters
), th
));
1370 pre_condition
= fold_build2 (TRUTH_OR_EXPR
, boolean_type_node
,
1371 cost_pre_condition
, pre_condition
);
1376 fold_build2 (TRUTH_OR_EXPR
, boolean_type_node
,
1378 fold_build1 (TRUTH_NOT_EXPR
, boolean_type_node
,
1383 /* Prologue peeling. */
1386 if (check_profitability
)
1387 set_prologue_iterations (bb_before_first_loop
, first_niters
,
1391 fold_build2 (LE_EXPR
, boolean_type_node
, *first_niters
,
1392 build_int_cst (TREE_TYPE (*first_niters
), 0));
1395 skip_e
= slpeel_add_loop_guard (bb_before_first_loop
, pre_condition
,
1396 cond_expr_stmt_list
,
1397 bb_before_second_loop
, bb_before_first_loop
);
1398 slpeel_update_phi_nodes_for_guard1 (skip_e
, first_loop
,
1399 first_loop
== new_loop
,
1400 &new_exit_bb
, &definitions
);
1403 /* 3. Add the guard that controls whether the second loop is executed.
1404 Resulting CFG would be:
1406 bb_before_first_loop:
1407 if (FIRST_NITERS == 0) GOTO bb_before_second_loop (skip first loop)
1415 if (FIRST_NITERS == NITERS) GOTO bb_after_second_loop (skip second loop)
1416 GOTO bb_before_second_loop
1418 bb_before_second_loop:
1424 bb_after_second_loop:
1429 bb_between_loops
= new_exit_bb
;
1430 bb_after_second_loop
= split_edge (single_exit (second_loop
));
1433 fold_build2 (EQ_EXPR
, boolean_type_node
, *first_niters
, niters
);
1434 skip_e
= slpeel_add_loop_guard (bb_between_loops
, pre_condition
, NULL
,
1435 bb_after_second_loop
, bb_before_first_loop
);
1436 slpeel_update_phi_nodes_for_guard2 (skip_e
, second_loop
,
1437 second_loop
== new_loop
, &new_exit_bb
);
1439 /* 4. Make first-loop iterate FIRST_NITERS times, if requested.
1441 if (update_first_loop_count
)
1442 slpeel_make_loop_iterate_ntimes (first_loop
, *first_niters
);
1444 BITMAP_FREE (definitions
);
1445 delete_update_ssa ();
1447 adjust_vec_debug_stmts ();
1452 /* Function vect_get_loop_location.
1454 Extract the location of the loop in the source code.
1455 If the loop is not well formed for vectorization, an estimated
1456 location is calculated.
1457 Return the loop location if succeed and NULL if not. */
1460 find_loop_location (struct loop
*loop
)
1464 gimple_stmt_iterator si
;
1469 stmt
= get_loop_exit_condition (loop
);
1471 if (stmt
&& gimple_location (stmt
) != UNKNOWN_LOC
)
1472 return gimple_location (stmt
);
1474 /* If we got here the loop is probably not "well formed",
1475 try to estimate the loop location */
1482 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
1484 stmt
= gsi_stmt (si
);
1485 if (gimple_location (stmt
) != UNKNOWN_LOC
)
1486 return gimple_location (stmt
);
1493 /* This function builds ni_name = number of iterations loop executes
1494 on the loop preheader. If SEQ is given the stmt is instead emitted
1498 vect_build_loop_niters (loop_vec_info loop_vinfo
, gimple_seq seq
)
1501 gimple_seq stmts
= NULL
;
1503 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1504 tree ni
= unshare_expr (LOOP_VINFO_NITERS (loop_vinfo
));
1506 var
= create_tmp_var (TREE_TYPE (ni
), "niters");
1507 add_referenced_var (var
);
1508 ni_name
= force_gimple_operand (ni
, &stmts
, false, var
);
1510 pe
= loop_preheader_edge (loop
);
1514 gimple_seq_add_seq (&seq
, stmts
);
1517 basic_block new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
1518 gcc_assert (!new_bb
);
1526 /* This function generates the following statements:
1528 ni_name = number of iterations loop executes
1529 ratio = ni_name / vf
1530 ratio_mult_vf_name = ratio * vf
1532 and places them at the loop preheader edge or in COND_EXPR_STMT_LIST
1533 if that is non-NULL. */
1536 vect_generate_tmps_on_preheader (loop_vec_info loop_vinfo
,
1538 tree
*ratio_mult_vf_name_ptr
,
1539 tree
*ratio_name_ptr
,
1540 gimple_seq cond_expr_stmt_list
)
1546 tree ni_name
, ni_minus_gap_name
;
1549 tree ratio_mult_vf_name
;
1550 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1551 tree ni
= LOOP_VINFO_NITERS (loop_vinfo
);
1552 int vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
1555 pe
= loop_preheader_edge (loop
);
1557 /* Generate temporary variable that contains
1558 number of iterations loop executes. */
1560 ni_name
= vect_build_loop_niters (loop_vinfo
, cond_expr_stmt_list
);
1561 log_vf
= build_int_cst (TREE_TYPE (ni
), exact_log2 (vf
));
1563 /* If epilogue loop is required because of data accesses with gaps, we
1564 subtract one iteration from the total number of iterations here for
1565 correct calculation of RATIO. */
1566 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
))
1568 ni_minus_gap_name
= fold_build2 (MINUS_EXPR
, TREE_TYPE (ni_name
),
1570 build_one_cst (TREE_TYPE (ni_name
)));
1571 if (!is_gimple_val (ni_minus_gap_name
))
1573 var
= create_tmp_var (TREE_TYPE (ni
), "ni_gap");
1574 add_referenced_var (var
);
1577 ni_minus_gap_name
= force_gimple_operand (ni_minus_gap_name
, &stmts
,
1579 if (cond_expr_stmt_list
)
1580 gimple_seq_add_seq (&cond_expr_stmt_list
, stmts
);
1583 pe
= loop_preheader_edge (loop
);
1584 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
1585 gcc_assert (!new_bb
);
1590 ni_minus_gap_name
= ni_name
;
1592 /* Create: ratio = ni >> log2(vf) */
1594 ratio_name
= fold_build2 (RSHIFT_EXPR
, TREE_TYPE (ni_minus_gap_name
),
1595 ni_minus_gap_name
, log_vf
);
1596 if (!is_gimple_val (ratio_name
))
1598 var
= create_tmp_var (TREE_TYPE (ni
), "bnd");
1599 add_referenced_var (var
);
1602 ratio_name
= force_gimple_operand (ratio_name
, &stmts
, true, var
);
1603 if (cond_expr_stmt_list
)
1604 gimple_seq_add_seq (&cond_expr_stmt_list
, stmts
);
1607 pe
= loop_preheader_edge (loop
);
1608 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
1609 gcc_assert (!new_bb
);
1613 /* Create: ratio_mult_vf = ratio << log2 (vf). */
1615 ratio_mult_vf_name
= fold_build2 (LSHIFT_EXPR
, TREE_TYPE (ratio_name
),
1616 ratio_name
, log_vf
);
1617 if (!is_gimple_val (ratio_mult_vf_name
))
1619 var
= create_tmp_var (TREE_TYPE (ni
), "ratio_mult_vf");
1620 add_referenced_var (var
);
1623 ratio_mult_vf_name
= force_gimple_operand (ratio_mult_vf_name
, &stmts
,
1625 if (cond_expr_stmt_list
)
1626 gimple_seq_add_seq (&cond_expr_stmt_list
, stmts
);
1629 pe
= loop_preheader_edge (loop
);
1630 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
1631 gcc_assert (!new_bb
);
1635 *ni_name_ptr
= ni_name
;
1636 *ratio_mult_vf_name_ptr
= ratio_mult_vf_name
;
1637 *ratio_name_ptr
= ratio_name
;
1642 /* Function vect_can_advance_ivs_p
1644 In case the number of iterations that LOOP iterates is unknown at compile
1645 time, an epilog loop will be generated, and the loop induction variables
1646 (IVs) will be "advanced" to the value they are supposed to take just before
1647 the epilog loop. Here we check that the access function of the loop IVs
1648 and the expression that represents the loop bound are simple enough.
1649 These restrictions will be relaxed in the future. */
1652 vect_can_advance_ivs_p (loop_vec_info loop_vinfo
)
1654 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1655 basic_block bb
= loop
->header
;
1657 gimple_stmt_iterator gsi
;
1659 /* Analyze phi functions of the loop header. */
1661 if (vect_print_dump_info (REPORT_DETAILS
))
1662 fprintf (vect_dump
, "vect_can_advance_ivs_p:");
1664 for (gsi
= gsi_start_phis (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1666 tree access_fn
= NULL
;
1667 tree evolution_part
;
1669 phi
= gsi_stmt (gsi
);
1670 if (vect_print_dump_info (REPORT_DETAILS
))
1672 fprintf (vect_dump
, "Analyze phi: ");
1673 print_gimple_stmt (vect_dump
, phi
, 0, TDF_SLIM
);
1676 /* Skip virtual phi's. The data dependences that are associated with
1677 virtual defs/uses (i.e., memory accesses) are analyzed elsewhere. */
1679 if (!is_gimple_reg (SSA_NAME_VAR (PHI_RESULT (phi
))))
1681 if (vect_print_dump_info (REPORT_DETAILS
))
1682 fprintf (vect_dump
, "virtual phi. skip.");
1686 /* Skip reduction phis. */
1688 if (STMT_VINFO_DEF_TYPE (vinfo_for_stmt (phi
)) == vect_reduction_def
)
1690 if (vect_print_dump_info (REPORT_DETAILS
))
1691 fprintf (vect_dump
, "reduc phi. skip.");
1695 /* Analyze the evolution function. */
1697 access_fn
= instantiate_parameters
1698 (loop
, analyze_scalar_evolution (loop
, PHI_RESULT (phi
)));
1702 if (vect_print_dump_info (REPORT_DETAILS
))
1703 fprintf (vect_dump
, "No Access function.");
1707 if (vect_print_dump_info (REPORT_DETAILS
))
1709 fprintf (vect_dump
, "Access function of PHI: ");
1710 print_generic_expr (vect_dump
, access_fn
, TDF_SLIM
);
1713 evolution_part
= evolution_part_in_loop_num (access_fn
, loop
->num
);
1715 if (evolution_part
== NULL_TREE
)
1717 if (vect_print_dump_info (REPORT_DETAILS
))
1718 fprintf (vect_dump
, "No evolution.");
1722 /* FORNOW: We do not transform initial conditions of IVs
1723 which evolution functions are a polynomial of degree >= 2. */
1725 if (tree_is_chrec (evolution_part
))
1733 /* Function vect_update_ivs_after_vectorizer.
1735 "Advance" the induction variables of LOOP to the value they should take
1736 after the execution of LOOP. This is currently necessary because the
1737 vectorizer does not handle induction variables that are used after the
1738 loop. Such a situation occurs when the last iterations of LOOP are
1740 1. We introduced new uses after LOOP for IVs that were not originally used
1741 after LOOP: the IVs of LOOP are now used by an epilog loop.
1742 2. LOOP is going to be vectorized; this means that it will iterate N/VF
1743 times, whereas the loop IVs should be bumped N times.
1746 - LOOP - a loop that is going to be vectorized. The last few iterations
1747 of LOOP were peeled.
1748 - NITERS - the number of iterations that LOOP executes (before it is
1749 vectorized). i.e, the number of times the ivs should be bumped.
1750 - UPDATE_E - a successor edge of LOOP->exit that is on the (only) path
1751 coming out from LOOP on which there are uses of the LOOP ivs
1752 (this is the path from LOOP->exit to epilog_loop->preheader).
1754 The new definitions of the ivs are placed in LOOP->exit.
1755 The phi args associated with the edge UPDATE_E in the bb
1756 UPDATE_E->dest are updated accordingly.
1758 Assumption 1: Like the rest of the vectorizer, this function assumes
1759 a single loop exit that has a single predecessor.
1761 Assumption 2: The phi nodes in the LOOP header and in update_bb are
1762 organized in the same order.
1764 Assumption 3: The access function of the ivs is simple enough (see
1765 vect_can_advance_ivs_p). This assumption will be relaxed in the future.
1767 Assumption 4: Exactly one of the successors of LOOP exit-bb is on a path
1768 coming out of LOOP on which the ivs of LOOP are used (this is the path
1769 that leads to the epilog loop; other paths skip the epilog loop). This
1770 path starts with the edge UPDATE_E, and its destination (denoted update_bb)
1771 needs to have its phis updated.
1775 vect_update_ivs_after_vectorizer (loop_vec_info loop_vinfo
, tree niters
,
1778 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1779 basic_block exit_bb
= single_exit (loop
)->dest
;
1781 gimple_stmt_iterator gsi
, gsi1
;
1782 basic_block update_bb
= update_e
->dest
;
1784 /* gcc_assert (vect_can_advance_ivs_p (loop_vinfo)); */
1786 /* Make sure there exists a single-predecessor exit bb: */
1787 gcc_assert (single_pred_p (exit_bb
));
1789 for (gsi
= gsi_start_phis (loop
->header
), gsi1
= gsi_start_phis (update_bb
);
1790 !gsi_end_p (gsi
) && !gsi_end_p (gsi1
);
1791 gsi_next (&gsi
), gsi_next (&gsi1
))
1794 tree step_expr
, off
;
1796 tree var
, ni
, ni_name
;
1797 gimple_stmt_iterator last_gsi
;
1798 stmt_vec_info stmt_info
;
1800 phi
= gsi_stmt (gsi
);
1801 phi1
= gsi_stmt (gsi1
);
1802 if (vect_print_dump_info (REPORT_DETAILS
))
1804 fprintf (vect_dump
, "vect_update_ivs_after_vectorizer: phi: ");
1805 print_gimple_stmt (vect_dump
, phi
, 0, TDF_SLIM
);
1808 /* Skip virtual phi's. */
1809 if (!is_gimple_reg (SSA_NAME_VAR (PHI_RESULT (phi
))))
1811 if (vect_print_dump_info (REPORT_DETAILS
))
1812 fprintf (vect_dump
, "virtual phi. skip.");
1816 /* Skip reduction phis. */
1817 stmt_info
= vinfo_for_stmt (phi
);
1818 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
)
1820 if (vect_print_dump_info (REPORT_DETAILS
))
1821 fprintf (vect_dump
, "reduc phi. skip.");
1825 type
= TREE_TYPE (gimple_phi_result (phi
));
1826 step_expr
= STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info
);
1827 step_expr
= unshare_expr (step_expr
);
1829 /* FORNOW: We do not support IVs whose evolution function is a polynomial
1830 of degree >= 2 or exponential. */
1831 gcc_assert (!tree_is_chrec (step_expr
));
1833 init_expr
= PHI_ARG_DEF_FROM_EDGE (phi
, loop_preheader_edge (loop
));
1835 off
= fold_build2 (MULT_EXPR
, TREE_TYPE (step_expr
),
1836 fold_convert (TREE_TYPE (step_expr
), niters
),
1838 if (POINTER_TYPE_P (type
))
1839 ni
= fold_build_pointer_plus (init_expr
, off
);
1841 ni
= fold_build2 (PLUS_EXPR
, type
,
1842 init_expr
, fold_convert (type
, off
));
1844 var
= create_tmp_var (type
, "tmp");
1845 add_referenced_var (var
);
1847 last_gsi
= gsi_last_bb (exit_bb
);
1848 ni_name
= force_gimple_operand_gsi (&last_gsi
, ni
, false, var
,
1849 true, GSI_SAME_STMT
);
1851 /* Fix phi expressions in the successor bb. */
1852 adjust_phi_and_debug_stmts (phi1
, update_e
, ni_name
);
1856 /* Return the more conservative threshold between the
1857 min_profitable_iters returned by the cost model and the user
1858 specified threshold, if provided. */
1861 conservative_cost_threshold (loop_vec_info loop_vinfo
,
1862 int min_profitable_iters
)
1865 int min_scalar_loop_bound
;
1867 min_scalar_loop_bound
= ((PARAM_VALUE (PARAM_MIN_VECT_LOOP_BOUND
)
1868 * LOOP_VINFO_VECT_FACTOR (loop_vinfo
)) - 1);
1870 /* Use the cost model only if it is more conservative than user specified
1872 th
= (unsigned) min_scalar_loop_bound
;
1873 if (min_profitable_iters
1874 && (!min_scalar_loop_bound
1875 || min_profitable_iters
> min_scalar_loop_bound
))
1876 th
= (unsigned) min_profitable_iters
;
1878 if (th
&& vect_print_dump_info (REPORT_COST
))
1879 fprintf (vect_dump
, "Profitability threshold is %u loop iterations.", th
);
1884 /* Function vect_do_peeling_for_loop_bound
1886 Peel the last iterations of the loop represented by LOOP_VINFO.
1887 The peeled iterations form a new epilog loop. Given that the loop now
1888 iterates NITERS times, the new epilog loop iterates
1889 NITERS % VECTORIZATION_FACTOR times.
1891 The original loop will later be made to iterate
1892 NITERS / VECTORIZATION_FACTOR times (this value is placed into RATIO).
1894 COND_EXPR and COND_EXPR_STMT_LIST are combined with a new generated
1898 vect_do_peeling_for_loop_bound (loop_vec_info loop_vinfo
, tree
*ratio
,
1899 tree cond_expr
, gimple_seq cond_expr_stmt_list
)
1901 tree ni_name
, ratio_mult_vf_name
;
1902 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1903 struct loop
*new_loop
;
1905 basic_block preheader
;
1907 bool check_profitability
= false;
1908 unsigned int th
= 0;
1909 int min_profitable_iters
;
1911 if (vect_print_dump_info (REPORT_DETAILS
))
1912 fprintf (vect_dump
, "=== vect_do_peeling_for_loop_bound ===");
1914 initialize_original_copy_tables ();
1916 /* Generate the following variables on the preheader of original loop:
1918 ni_name = number of iteration the original loop executes
1919 ratio = ni_name / vf
1920 ratio_mult_vf_name = ratio * vf */
1921 vect_generate_tmps_on_preheader (loop_vinfo
, &ni_name
,
1922 &ratio_mult_vf_name
, ratio
,
1923 cond_expr_stmt_list
);
1925 loop_num
= loop
->num
;
1927 /* If cost model check not done during versioning and
1928 peeling for alignment. */
1929 if (!LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo
)
1930 && !LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo
)
1931 && !LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo
)
1934 check_profitability
= true;
1936 /* Get profitability threshold for vectorized loop. */
1937 min_profitable_iters
= LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo
);
1939 th
= conservative_cost_threshold (loop_vinfo
,
1940 min_profitable_iters
);
1943 new_loop
= slpeel_tree_peel_loop_to_edge (loop
, single_exit (loop
),
1944 &ratio_mult_vf_name
, ni_name
, false,
1945 th
, check_profitability
,
1946 cond_expr
, cond_expr_stmt_list
);
1947 gcc_assert (new_loop
);
1948 gcc_assert (loop_num
== loop
->num
);
1949 #ifdef ENABLE_CHECKING
1950 slpeel_verify_cfg_after_peeling (loop
, new_loop
);
1953 /* A guard that controls whether the new_loop is to be executed or skipped
1954 is placed in LOOP->exit. LOOP->exit therefore has two successors - one
1955 is the preheader of NEW_LOOP, where the IVs from LOOP are used. The other
1956 is a bb after NEW_LOOP, where these IVs are not used. Find the edge that
1957 is on the path where the LOOP IVs are used and need to be updated. */
1959 preheader
= loop_preheader_edge (new_loop
)->src
;
1960 if (EDGE_PRED (preheader
, 0)->src
== single_exit (loop
)->dest
)
1961 update_e
= EDGE_PRED (preheader
, 0);
1963 update_e
= EDGE_PRED (preheader
, 1);
1965 /* Update IVs of original loop as if they were advanced
1966 by ratio_mult_vf_name steps. */
1967 vect_update_ivs_after_vectorizer (loop_vinfo
, ratio_mult_vf_name
, update_e
);
1969 /* After peeling we have to reset scalar evolution analyzer. */
1972 free_original_copy_tables ();
1976 /* Function vect_gen_niters_for_prolog_loop
1978 Set the number of iterations for the loop represented by LOOP_VINFO
1979 to the minimum between LOOP_NITERS (the original iteration count of the loop)
1980 and the misalignment of DR - the data reference recorded in
1981 LOOP_VINFO_UNALIGNED_DR (LOOP_VINFO). As a result, after the execution of
1982 this loop, the data reference DR will refer to an aligned location.
1984 The following computation is generated:
1986 If the misalignment of DR is known at compile time:
1987 addr_mis = int mis = DR_MISALIGNMENT (dr);
1988 Else, compute address misalignment in bytes:
1989 addr_mis = addr & (vectype_size - 1)
1991 prolog_niters = min (LOOP_NITERS, ((VF - addr_mis/elem_size)&(VF-1))/step)
1993 (elem_size = element type size; an element is the scalar element whose type
1994 is the inner type of the vectype)
1996 When the step of the data-ref in the loop is not 1 (as in interleaved data
1997 and SLP), the number of iterations of the prolog must be divided by the step
1998 (which is equal to the size of interleaved group).
2000 The above formulas assume that VF == number of elements in the vector. This
2001 may not hold when there are multiple-types in the loop.
2002 In this case, for some data-references in the loop the VF does not represent
2003 the number of elements that fit in the vector. Therefore, instead of VF we
2004 use TYPE_VECTOR_SUBPARTS. */
2007 vect_gen_niters_for_prolog_loop (loop_vec_info loop_vinfo
, tree loop_niters
)
2009 struct data_reference
*dr
= LOOP_VINFO_UNALIGNED_DR (loop_vinfo
);
2010 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
2013 tree iters
, iters_name
;
2016 gimple dr_stmt
= DR_STMT (dr
);
2017 stmt_vec_info stmt_info
= vinfo_for_stmt (dr_stmt
);
2018 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
2019 int vectype_align
= TYPE_ALIGN (vectype
) / BITS_PER_UNIT
;
2020 tree niters_type
= TREE_TYPE (loop_niters
);
2021 int nelements
= TYPE_VECTOR_SUBPARTS (vectype
);
2023 pe
= loop_preheader_edge (loop
);
2025 if (LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo
) > 0)
2027 int npeel
= LOOP_PEELING_FOR_ALIGNMENT (loop_vinfo
);
2029 if (vect_print_dump_info (REPORT_DETAILS
))
2030 fprintf (vect_dump
, "known peeling = %d.", npeel
);
2032 iters
= build_int_cst (niters_type
, npeel
);
2036 gimple_seq new_stmts
= NULL
;
2037 bool negative
= tree_int_cst_compare (DR_STEP (dr
), size_zero_node
) < 0;
2038 tree offset
= negative
2039 ? size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1) : NULL_TREE
;
2040 tree start_addr
= vect_create_addr_base_for_vector_ref (dr_stmt
,
2041 &new_stmts
, offset
, loop
);
2042 tree type
= unsigned_type_for (TREE_TYPE (start_addr
));
2043 tree vectype_size_minus_1
= build_int_cst (type
, vectype_align
- 1);
2044 tree elem_size_log
=
2045 build_int_cst (type
, exact_log2 (vectype_align
/nelements
));
2046 tree nelements_minus_1
= build_int_cst (type
, nelements
- 1);
2047 tree nelements_tree
= build_int_cst (type
, nelements
);
2051 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, new_stmts
);
2052 gcc_assert (!new_bb
);
2054 /* Create: byte_misalign = addr & (vectype_size - 1) */
2056 fold_build2 (BIT_AND_EXPR
, type
, fold_convert (type
, start_addr
),
2057 vectype_size_minus_1
);
2059 /* Create: elem_misalign = byte_misalign / element_size */
2061 fold_build2 (RSHIFT_EXPR
, type
, byte_misalign
, elem_size_log
);
2063 /* Create: (niters_type) (nelements - elem_misalign)&(nelements - 1) */
2065 iters
= fold_build2 (MINUS_EXPR
, type
, elem_misalign
, nelements_tree
);
2067 iters
= fold_build2 (MINUS_EXPR
, type
, nelements_tree
, elem_misalign
);
2068 iters
= fold_build2 (BIT_AND_EXPR
, type
, iters
, nelements_minus_1
);
2069 iters
= fold_convert (niters_type
, iters
);
2072 /* Create: prolog_loop_niters = min (iters, loop_niters) */
2073 /* If the loop bound is known at compile time we already verified that it is
2074 greater than vf; since the misalignment ('iters') is at most vf, there's
2075 no need to generate the MIN_EXPR in this case. */
2076 if (TREE_CODE (loop_niters
) != INTEGER_CST
)
2077 iters
= fold_build2 (MIN_EXPR
, niters_type
, iters
, loop_niters
);
2079 if (vect_print_dump_info (REPORT_DETAILS
))
2081 fprintf (vect_dump
, "niters for prolog loop: ");
2082 print_generic_expr (vect_dump
, iters
, TDF_SLIM
);
2085 var
= create_tmp_var (niters_type
, "prolog_loop_niters");
2086 add_referenced_var (var
);
2088 iters_name
= force_gimple_operand (iters
, &stmts
, false, var
);
2090 /* Insert stmt on loop preheader edge. */
2093 basic_block new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
2094 gcc_assert (!new_bb
);
2101 /* Function vect_update_init_of_dr
2103 NITERS iterations were peeled from LOOP. DR represents a data reference
2104 in LOOP. This function updates the information recorded in DR to
2105 account for the fact that the first NITERS iterations had already been
2106 executed. Specifically, it updates the OFFSET field of DR. */
2109 vect_update_init_of_dr (struct data_reference
*dr
, tree niters
)
2111 tree offset
= DR_OFFSET (dr
);
2113 niters
= fold_build2 (MULT_EXPR
, sizetype
,
2114 fold_convert (sizetype
, niters
),
2115 fold_convert (sizetype
, DR_STEP (dr
)));
2116 offset
= fold_build2 (PLUS_EXPR
, sizetype
,
2117 fold_convert (sizetype
, offset
), niters
);
2118 DR_OFFSET (dr
) = offset
;
2122 /* Function vect_update_inits_of_drs
2124 NITERS iterations were peeled from the loop represented by LOOP_VINFO.
2125 This function updates the information recorded for the data references in
2126 the loop to account for the fact that the first NITERS iterations had
2127 already been executed. Specifically, it updates the initial_condition of
2128 the access_function of all the data_references in the loop. */
2131 vect_update_inits_of_drs (loop_vec_info loop_vinfo
, tree niters
)
2134 VEC (data_reference_p
, heap
) *datarefs
= LOOP_VINFO_DATAREFS (loop_vinfo
);
2135 struct data_reference
*dr
;
2137 if (vect_print_dump_info (REPORT_DETAILS
))
2138 fprintf (vect_dump
, "=== vect_update_inits_of_dr ===");
2140 FOR_EACH_VEC_ELT (data_reference_p
, datarefs
, i
, dr
)
2141 vect_update_init_of_dr (dr
, niters
);
2145 /* Function vect_do_peeling_for_alignment
2147 Peel the first 'niters' iterations of the loop represented by LOOP_VINFO.
2148 'niters' is set to the misalignment of one of the data references in the
2149 loop, thereby forcing it to refer to an aligned location at the beginning
2150 of the execution of this loop. The data reference for which we are
2151 peeling is recorded in LOOP_VINFO_UNALIGNED_DR. */
2154 vect_do_peeling_for_alignment (loop_vec_info loop_vinfo
)
2156 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
2157 tree niters_of_prolog_loop
, ni_name
;
2159 tree wide_prolog_niters
;
2160 struct loop
*new_loop
;
2161 unsigned int th
= 0;
2162 int min_profitable_iters
;
2165 if (vect_print_dump_info (REPORT_DETAILS
))
2166 fprintf (vect_dump
, "=== vect_do_peeling_for_alignment ===");
2168 initialize_original_copy_tables ();
2170 ni_name
= vect_build_loop_niters (loop_vinfo
, NULL
);
2171 niters_of_prolog_loop
= vect_gen_niters_for_prolog_loop (loop_vinfo
,
2174 /* Get profitability threshold for vectorized loop. */
2175 min_profitable_iters
= LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo
);
2176 th
= conservative_cost_threshold (loop_vinfo
,
2177 min_profitable_iters
);
2179 /* Peel the prolog loop and iterate it niters_of_prolog_loop. */
2181 slpeel_tree_peel_loop_to_edge (loop
, loop_preheader_edge (loop
),
2182 &niters_of_prolog_loop
, ni_name
, true,
2183 th
, true, NULL_TREE
, NULL
);
2185 gcc_assert (new_loop
);
2186 #ifdef ENABLE_CHECKING
2187 slpeel_verify_cfg_after_peeling (new_loop
, loop
);
2189 max_iter
= MAX (LOOP_VINFO_VECT_FACTOR (loop_vinfo
) - 1, (int) th
);
2190 record_niter_bound (new_loop
, shwi_to_double_int (max_iter
), false, true);
2191 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2192 fprintf (dump_file
, "Setting upper bound of nb iterations for prologue "
2193 "loop to %d\n", max_iter
);
2195 /* Update number of times loop executes. */
2196 n_iters
= LOOP_VINFO_NITERS (loop_vinfo
);
2197 LOOP_VINFO_NITERS (loop_vinfo
) = fold_build2 (MINUS_EXPR
,
2198 TREE_TYPE (n_iters
), n_iters
, niters_of_prolog_loop
);
2200 if (types_compatible_p (sizetype
, TREE_TYPE (niters_of_prolog_loop
)))
2201 wide_prolog_niters
= niters_of_prolog_loop
;
2204 gimple_seq seq
= NULL
;
2205 edge pe
= loop_preheader_edge (loop
);
2206 tree wide_iters
= fold_convert (sizetype
, niters_of_prolog_loop
);
2207 tree var
= create_tmp_var (sizetype
, "prolog_loop_adjusted_niters");
2208 add_referenced_var (var
);
2209 wide_prolog_niters
= force_gimple_operand (wide_iters
, &seq
, false,
2213 /* Insert stmt on loop preheader edge. */
2214 basic_block new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
2215 gcc_assert (!new_bb
);
2219 /* Update the init conditions of the access functions of all data refs. */
2220 vect_update_inits_of_drs (loop_vinfo
, wide_prolog_niters
);
2222 /* After peeling we have to reset scalar evolution analyzer. */
2225 free_original_copy_tables ();
2229 /* Function vect_create_cond_for_align_checks.
2231 Create a conditional expression that represents the alignment checks for
2232 all of data references (array element references) whose alignment must be
2236 COND_EXPR - input conditional expression. New conditions will be chained
2237 with logical AND operation.
2238 LOOP_VINFO - two fields of the loop information are used.
2239 LOOP_VINFO_PTR_MASK is the mask used to check the alignment.
2240 LOOP_VINFO_MAY_MISALIGN_STMTS contains the refs to be checked.
2243 COND_EXPR_STMT_LIST - statements needed to construct the conditional
2245 The returned value is the conditional expression to be used in the if
2246 statement that controls which version of the loop gets executed at runtime.
2248 The algorithm makes two assumptions:
2249 1) The number of bytes "n" in a vector is a power of 2.
2250 2) An address "a" is aligned if a%n is zero and that this
2251 test can be done as a&(n-1) == 0. For example, for 16
2252 byte vectors the test is a&0xf == 0. */
2255 vect_create_cond_for_align_checks (loop_vec_info loop_vinfo
,
2257 gimple_seq
*cond_expr_stmt_list
)
2259 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
2260 VEC(gimple
,heap
) *may_misalign_stmts
2261 = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo
);
2263 int mask
= LOOP_VINFO_PTR_MASK (loop_vinfo
);
2266 tree int_ptrsize_type
;
2268 tree or_tmp_name
= NULL_TREE
;
2269 tree and_tmp
, and_tmp_name
;
2272 tree part_cond_expr
;
2274 /* Check that mask is one less than a power of 2, i.e., mask is
2275 all zeros followed by all ones. */
2276 gcc_assert ((mask
!= 0) && ((mask
& (mask
+1)) == 0));
2278 int_ptrsize_type
= signed_type_for (ptr_type_node
);
2280 /* Create expression (mask & (dr_1 || ... || dr_n)) where dr_i is the address
2281 of the first vector of the i'th data reference. */
2283 FOR_EACH_VEC_ELT (gimple
, may_misalign_stmts
, i
, ref_stmt
)
2285 gimple_seq new_stmt_list
= NULL
;
2287 tree addr_tmp
, addr_tmp_name
;
2288 tree or_tmp
, new_or_tmp_name
;
2289 gimple addr_stmt
, or_stmt
;
2290 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (ref_stmt
);
2291 tree vectype
= STMT_VINFO_VECTYPE (stmt_vinfo
);
2292 bool negative
= tree_int_cst_compare
2293 (DR_STEP (STMT_VINFO_DATA_REF (stmt_vinfo
)), size_zero_node
) < 0;
2294 tree offset
= negative
2295 ? size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1) : NULL_TREE
;
2297 /* create: addr_tmp = (int)(address_of_first_vector) */
2299 vect_create_addr_base_for_vector_ref (ref_stmt
, &new_stmt_list
,
2301 if (new_stmt_list
!= NULL
)
2302 gimple_seq_add_seq (cond_expr_stmt_list
, new_stmt_list
);
2304 sprintf (tmp_name
, "%s%d", "addr2int", i
);
2305 addr_tmp
= create_tmp_var (int_ptrsize_type
, tmp_name
);
2306 add_referenced_var (addr_tmp
);
2307 addr_tmp_name
= make_ssa_name (addr_tmp
, NULL
);
2308 addr_stmt
= gimple_build_assign_with_ops (NOP_EXPR
, addr_tmp_name
,
2309 addr_base
, NULL_TREE
);
2310 SSA_NAME_DEF_STMT (addr_tmp_name
) = addr_stmt
;
2311 gimple_seq_add_stmt (cond_expr_stmt_list
, addr_stmt
);
2313 /* The addresses are OR together. */
2315 if (or_tmp_name
!= NULL_TREE
)
2317 /* create: or_tmp = or_tmp | addr_tmp */
2318 sprintf (tmp_name
, "%s%d", "orptrs", i
);
2319 or_tmp
= create_tmp_var (int_ptrsize_type
, tmp_name
);
2320 add_referenced_var (or_tmp
);
2321 new_or_tmp_name
= make_ssa_name (or_tmp
, NULL
);
2322 or_stmt
= gimple_build_assign_with_ops (BIT_IOR_EXPR
,
2324 or_tmp_name
, addr_tmp_name
);
2325 SSA_NAME_DEF_STMT (new_or_tmp_name
) = or_stmt
;
2326 gimple_seq_add_stmt (cond_expr_stmt_list
, or_stmt
);
2327 or_tmp_name
= new_or_tmp_name
;
2330 or_tmp_name
= addr_tmp_name
;
2334 mask_cst
= build_int_cst (int_ptrsize_type
, mask
);
2336 /* create: and_tmp = or_tmp & mask */
2337 and_tmp
= create_tmp_var (int_ptrsize_type
, "andmask" );
2338 add_referenced_var (and_tmp
);
2339 and_tmp_name
= make_ssa_name (and_tmp
, NULL
);
2341 and_stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, and_tmp_name
,
2342 or_tmp_name
, mask_cst
);
2343 SSA_NAME_DEF_STMT (and_tmp_name
) = and_stmt
;
2344 gimple_seq_add_stmt (cond_expr_stmt_list
, and_stmt
);
2346 /* Make and_tmp the left operand of the conditional test against zero.
2347 if and_tmp has a nonzero bit then some address is unaligned. */
2348 ptrsize_zero
= build_int_cst (int_ptrsize_type
, 0);
2349 part_cond_expr
= fold_build2 (EQ_EXPR
, boolean_type_node
,
2350 and_tmp_name
, ptrsize_zero
);
2352 *cond_expr
= fold_build2 (TRUTH_AND_EXPR
, boolean_type_node
,
2353 *cond_expr
, part_cond_expr
);
2355 *cond_expr
= part_cond_expr
;
2359 /* Function vect_vfa_segment_size.
2361 Create an expression that computes the size of segment
2362 that will be accessed for a data reference. The functions takes into
2363 account that realignment loads may access one more vector.
2366 DR: The data reference.
2367 LENGTH_FACTOR: segment length to consider.
2369 Return an expression whose value is the size of segment which will be
2373 vect_vfa_segment_size (struct data_reference
*dr
, tree length_factor
)
2375 tree segment_length
;
2377 if (!compare_tree_int (DR_STEP (dr
), 0))
2378 segment_length
= TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr
)));
2380 segment_length
= size_binop (MULT_EXPR
,
2381 fold_convert (sizetype
, DR_STEP (dr
)),
2382 fold_convert (sizetype
, length_factor
));
2384 if (vect_supportable_dr_alignment (dr
, false)
2385 == dr_explicit_realign_optimized
)
2387 tree vector_size
= TYPE_SIZE_UNIT
2388 (STMT_VINFO_VECTYPE (vinfo_for_stmt (DR_STMT (dr
))));
2390 segment_length
= size_binop (PLUS_EXPR
, segment_length
, vector_size
);
2392 return segment_length
;
2396 /* Function vect_create_cond_for_alias_checks.
2398 Create a conditional expression that represents the run-time checks for
2399 overlapping of address ranges represented by a list of data references
2400 relations passed as input.
2403 COND_EXPR - input conditional expression. New conditions will be chained
2404 with logical AND operation.
2405 LOOP_VINFO - field LOOP_VINFO_MAY_ALIAS_STMTS contains the list of ddrs
2409 COND_EXPR - conditional expression.
2410 COND_EXPR_STMT_LIST - statements needed to construct the conditional
2414 The returned value is the conditional expression to be used in the if
2415 statement that controls which version of the loop gets executed at runtime.
2419 vect_create_cond_for_alias_checks (loop_vec_info loop_vinfo
,
2421 gimple_seq
* cond_expr_stmt_list
)
2423 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
2424 VEC (ddr_p
, heap
) * may_alias_ddrs
=
2425 LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo
);
2426 int vect_factor
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
2427 tree scalar_loop_iters
= LOOP_VINFO_NITERS (loop_vinfo
);
2431 tree part_cond_expr
, length_factor
;
2433 /* Create expression
2434 ((store_ptr_0 + store_segment_length_0) <= load_ptr_0)
2435 || (load_ptr_0 + load_segment_length_0) <= store_ptr_0))
2439 ((store_ptr_n + store_segment_length_n) <= load_ptr_n)
2440 || (load_ptr_n + load_segment_length_n) <= store_ptr_n)) */
2442 if (VEC_empty (ddr_p
, may_alias_ddrs
))
2445 FOR_EACH_VEC_ELT (ddr_p
, may_alias_ddrs
, i
, ddr
)
2447 struct data_reference
*dr_a
, *dr_b
;
2448 gimple dr_group_first_a
, dr_group_first_b
;
2449 tree addr_base_a
, addr_base_b
;
2450 tree segment_length_a
, segment_length_b
;
2451 gimple stmt_a
, stmt_b
;
2452 tree seg_a_min
, seg_a_max
, seg_b_min
, seg_b_max
;
2455 stmt_a
= DR_STMT (DDR_A (ddr
));
2456 dr_group_first_a
= GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt_a
));
2457 if (dr_group_first_a
)
2459 stmt_a
= dr_group_first_a
;
2460 dr_a
= STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt_a
));
2464 stmt_b
= DR_STMT (DDR_B (ddr
));
2465 dr_group_first_b
= GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt_b
));
2466 if (dr_group_first_b
)
2468 stmt_b
= dr_group_first_b
;
2469 dr_b
= STMT_VINFO_DATA_REF (vinfo_for_stmt (stmt_b
));
2473 vect_create_addr_base_for_vector_ref (stmt_a
, cond_expr_stmt_list
,
2476 vect_create_addr_base_for_vector_ref (stmt_b
, cond_expr_stmt_list
,
2479 if (!operand_equal_p (DR_STEP (dr_a
), DR_STEP (dr_b
), 0))
2480 length_factor
= scalar_loop_iters
;
2482 length_factor
= size_int (vect_factor
);
2483 segment_length_a
= vect_vfa_segment_size (dr_a
, length_factor
);
2484 segment_length_b
= vect_vfa_segment_size (dr_b
, length_factor
);
2486 if (vect_print_dump_info (REPORT_DR_DETAILS
))
2489 "create runtime check for data references ");
2490 print_generic_expr (vect_dump
, DR_REF (dr_a
), TDF_SLIM
);
2491 fprintf (vect_dump
, " and ");
2492 print_generic_expr (vect_dump
, DR_REF (dr_b
), TDF_SLIM
);
2495 seg_a_min
= addr_base_a
;
2496 seg_a_max
= fold_build_pointer_plus (addr_base_a
, segment_length_a
);
2497 if (tree_int_cst_compare (DR_STEP (dr_a
), size_zero_node
) < 0)
2498 seg_a_min
= seg_a_max
, seg_a_max
= addr_base_a
;
2500 seg_b_min
= addr_base_b
;
2501 seg_b_max
= fold_build_pointer_plus (addr_base_b
, segment_length_b
);
2502 if (tree_int_cst_compare (DR_STEP (dr_b
), size_zero_node
) < 0)
2503 seg_b_min
= seg_b_max
, seg_b_max
= addr_base_b
;
2506 fold_build2 (TRUTH_OR_EXPR
, boolean_type_node
,
2507 fold_build2 (LE_EXPR
, boolean_type_node
, seg_a_max
, seg_b_min
),
2508 fold_build2 (LE_EXPR
, boolean_type_node
, seg_b_max
, seg_a_min
));
2511 *cond_expr
= fold_build2 (TRUTH_AND_EXPR
, boolean_type_node
,
2512 *cond_expr
, part_cond_expr
);
2514 *cond_expr
= part_cond_expr
;
2517 if (vect_print_dump_info (REPORT_VECTORIZED_LOCATIONS
))
2518 fprintf (vect_dump
, "created %u versioning for alias checks.\n",
2519 VEC_length (ddr_p
, may_alias_ddrs
));
2523 /* Function vect_loop_versioning.
2525 If the loop has data references that may or may not be aligned or/and
2526 has data reference relations whose independence was not proven then
2527 two versions of the loop need to be generated, one which is vectorized
2528 and one which isn't. A test is then generated to control which of the
2529 loops is executed. The test checks for the alignment of all of the
2530 data references that may or may not be aligned. An additional
2531 sequence of runtime tests is generated for each pairs of DDRs whose
2532 independence was not proven. The vectorized version of loop is
2533 executed only if both alias and alignment tests are passed.
2535 The test generated to check which version of loop is executed
2536 is modified to also check for profitability as indicated by the
2537 cost model initially.
2539 The versioning precondition(s) are placed in *COND_EXPR and
2540 *COND_EXPR_STMT_LIST. If DO_VERSIONING is true versioning is
2541 also performed, otherwise only the conditions are generated. */
2544 vect_loop_versioning (loop_vec_info loop_vinfo
, bool do_versioning
,
2545 tree
*cond_expr
, gimple_seq
*cond_expr_stmt_list
)
2547 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
2548 basic_block condition_bb
;
2549 gimple_stmt_iterator gsi
, cond_exp_gsi
;
2550 basic_block merge_bb
;
2551 basic_block new_exit_bb
;
2553 gimple orig_phi
, new_phi
;
2555 unsigned prob
= 4 * REG_BR_PROB_BASE
/ 5;
2556 gimple_seq gimplify_stmt_list
= NULL
;
2557 tree scalar_loop_iters
= LOOP_VINFO_NITERS (loop_vinfo
);
2558 int min_profitable_iters
= 0;
2561 /* Get profitability threshold for vectorized loop. */
2562 min_profitable_iters
= LOOP_VINFO_COST_MODEL_MIN_ITERS (loop_vinfo
);
2564 th
= conservative_cost_threshold (loop_vinfo
,
2565 min_profitable_iters
);
2567 *cond_expr
= fold_build2 (GT_EXPR
, boolean_type_node
, scalar_loop_iters
,
2568 build_int_cst (TREE_TYPE (scalar_loop_iters
), th
));
2569 *cond_expr
= force_gimple_operand_1 (*cond_expr
, cond_expr_stmt_list
,
2570 is_gimple_condexpr
, NULL_TREE
);
2572 if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo
))
2573 vect_create_cond_for_align_checks (loop_vinfo
, cond_expr
,
2574 cond_expr_stmt_list
);
2576 if (LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo
))
2577 vect_create_cond_for_alias_checks (loop_vinfo
, cond_expr
,
2578 cond_expr_stmt_list
);
2580 *cond_expr
= force_gimple_operand_1 (*cond_expr
, &gimplify_stmt_list
,
2581 is_gimple_condexpr
, NULL_TREE
);
2582 gimple_seq_add_seq (cond_expr_stmt_list
, gimplify_stmt_list
);
2584 /* If we only needed the extra conditions and a new loop copy
2589 initialize_original_copy_tables ();
2590 loop_version (loop
, *cond_expr
, &condition_bb
,
2591 prob
, prob
, REG_BR_PROB_BASE
- prob
, true);
2592 free_original_copy_tables();
2594 /* Loop versioning violates an assumption we try to maintain during
2595 vectorization - that the loop exit block has a single predecessor.
2596 After versioning, the exit block of both loop versions is the same
2597 basic block (i.e. it has two predecessors). Just in order to simplify
2598 following transformations in the vectorizer, we fix this situation
2599 here by adding a new (empty) block on the exit-edge of the loop,
2600 with the proper loop-exit phis to maintain loop-closed-form. */
2602 merge_bb
= single_exit (loop
)->dest
;
2603 gcc_assert (EDGE_COUNT (merge_bb
->preds
) == 2);
2604 new_exit_bb
= split_edge (single_exit (loop
));
2605 new_exit_e
= single_exit (loop
);
2606 e
= EDGE_SUCC (new_exit_bb
, 0);
2608 for (gsi
= gsi_start_phis (merge_bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
2610 orig_phi
= gsi_stmt (gsi
);
2611 new_phi
= create_phi_node (SSA_NAME_VAR (PHI_RESULT (orig_phi
)),
2613 arg
= PHI_ARG_DEF_FROM_EDGE (orig_phi
, e
);
2614 add_phi_arg (new_phi
, arg
, new_exit_e
,
2615 gimple_phi_arg_location_from_edge (orig_phi
, e
));
2616 adjust_phi_and_debug_stmts (orig_phi
, e
, PHI_RESULT (new_phi
));
2619 /* End loop-exit-fixes after versioning. */
2621 update_ssa (TODO_update_ssa
);
2622 if (*cond_expr_stmt_list
)
2624 cond_exp_gsi
= gsi_last_bb (condition_bb
);
2625 gsi_insert_seq_before (&cond_exp_gsi
, *cond_expr_stmt_list
,
2627 *cond_expr_stmt_list
= NULL
;
2629 *cond_expr
= NULL_TREE
;