2 Copyright (C) 2003-2019 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com> and
4 Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
31 #include "tree-pass.h"
33 #include "optabs-tree.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "tree-ssa-loop-ivopts.h"
42 #include "tree-ssa-loop-manip.h"
43 #include "tree-ssa-loop-niter.h"
44 #include "tree-ssa-loop.h"
47 #include "tree-scalar-evolution.h"
48 #include "tree-vectorizer.h"
49 #include "gimple-fold.h"
52 #include "tree-if-conv.h"
53 #include "internal-fn.h"
54 #include "tree-vector-builder.h"
55 #include "vec-perm-indices.h"
58 /* Loop Vectorization Pass.
60 This pass tries to vectorize loops.
62 For example, the vectorizer transforms the following simple loop:
64 short a[N]; short b[N]; short c[N]; int i;
70 as if it was manually vectorized by rewriting the source code into:
72 typedef int __attribute__((mode(V8HI))) v8hi;
73 short a[N]; short b[N]; short c[N]; int i;
74 v8hi *pa = (v8hi*)a, *pb = (v8hi*)b, *pc = (v8hi*)c;
77 for (i=0; i<N/8; i++){
84 The main entry to this pass is vectorize_loops(), in which
85 the vectorizer applies a set of analyses on a given set of loops,
86 followed by the actual vectorization transformation for the loops that
87 had successfully passed the analysis phase.
88 Throughout this pass we make a distinction between two types of
89 data: scalars (which are represented by SSA_NAMES), and memory references
90 ("data-refs"). These two types of data require different handling both
91 during analysis and transformation. The types of data-refs that the
92 vectorizer currently supports are ARRAY_REFS which base is an array DECL
93 (not a pointer), and INDIRECT_REFS through pointers; both array and pointer
94 accesses are required to have a simple (consecutive) access pattern.
98 The driver for the analysis phase is vect_analyze_loop().
99 It applies a set of analyses, some of which rely on the scalar evolution
100 analyzer (scev) developed by Sebastian Pop.
102 During the analysis phase the vectorizer records some information
103 per stmt in a "stmt_vec_info" struct which is attached to each stmt in the
104 loop, as well as general information about the loop as a whole, which is
105 recorded in a "loop_vec_info" struct attached to each loop.
107 Transformation phase:
108 =====================
109 The loop transformation phase scans all the stmts in the loop, and
110 creates a vector stmt (or a sequence of stmts) for each scalar stmt S in
111 the loop that needs to be vectorized. It inserts the vector code sequence
112 just before the scalar stmt S, and records a pointer to the vector code
113 in STMT_VINFO_VEC_STMT (stmt_info) (stmt_info is the stmt_vec_info struct
114 attached to S). This pointer will be used for the vectorization of following
115 stmts which use the def of stmt S. Stmt S is removed if it writes to memory;
116 otherwise, we rely on dead code elimination for removing it.
118 For example, say stmt S1 was vectorized into stmt VS1:
121 S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
124 To vectorize stmt S2, the vectorizer first finds the stmt that defines
125 the operand 'b' (S1), and gets the relevant vector def 'vb' from the
126 vector stmt VS1 pointed to by STMT_VINFO_VEC_STMT (stmt_info (S1)). The
127 resulting sequence would be:
130 S1: b = x[i]; STMT_VINFO_VEC_STMT (stmt_info (S1)) = VS1
132 S2: a = b; STMT_VINFO_VEC_STMT (stmt_info (S2)) = VS2
134 Operands that are not SSA_NAMEs, are data-refs that appear in
135 load/store operations (like 'x[i]' in S1), and are handled differently.
139 Currently the only target specific information that is used is the
140 size of the vector (in bytes) - "TARGET_VECTORIZE_UNITS_PER_SIMD_WORD".
141 Targets that can support different sizes of vectors, for now will need
142 to specify one value for "TARGET_VECTORIZE_UNITS_PER_SIMD_WORD". More
143 flexibility will be added in the future.
145 Since we only vectorize operations which vector form can be
146 expressed using existing tree codes, to verify that an operation is
147 supported, the vectorizer checks the relevant optab at the relevant
148 machine_mode (e.g, optab_handler (add_optab, V8HImode)). If
149 the value found is CODE_FOR_nothing, then there's no target support, and
150 we can't vectorize the stmt.
152 For additional information on this project see:
153 http://gcc.gnu.org/projects/tree-ssa/vectorization.html
156 static void vect_estimate_min_profitable_iters (loop_vec_info
, int *, int *);
157 static stmt_vec_info
vect_is_simple_reduction (loop_vec_info
, stmt_vec_info
,
160 /* Subroutine of vect_determine_vf_for_stmt that handles only one
161 statement. VECTYPE_MAYBE_SET_P is true if STMT_VINFO_VECTYPE
162 may already be set for general statements (not just data refs). */
165 vect_determine_vf_for_stmt_1 (stmt_vec_info stmt_info
,
166 bool vectype_maybe_set_p
,
168 vec
<stmt_vec_info
> *mask_producers
)
170 gimple
*stmt
= stmt_info
->stmt
;
172 if ((!STMT_VINFO_RELEVANT_P (stmt_info
)
173 && !STMT_VINFO_LIVE_P (stmt_info
))
174 || gimple_clobber_p (stmt
))
176 if (dump_enabled_p ())
177 dump_printf_loc (MSG_NOTE
, vect_location
, "skip.\n");
178 return opt_result::success ();
181 tree stmt_vectype
, nunits_vectype
;
182 opt_result res
= vect_get_vector_types_for_stmt (stmt_info
, &stmt_vectype
,
189 if (STMT_VINFO_VECTYPE (stmt_info
))
190 /* The only case when a vectype had been already set is for stmts
191 that contain a data ref, or for "pattern-stmts" (stmts generated
192 by the vectorizer to represent/replace a certain idiom). */
193 gcc_assert ((STMT_VINFO_DATA_REF (stmt_info
)
194 || vectype_maybe_set_p
)
195 && STMT_VINFO_VECTYPE (stmt_info
) == stmt_vectype
);
196 else if (stmt_vectype
== boolean_type_node
)
197 mask_producers
->safe_push (stmt_info
);
199 STMT_VINFO_VECTYPE (stmt_info
) = stmt_vectype
;
203 vect_update_max_nunits (vf
, nunits_vectype
);
205 return opt_result::success ();
208 /* Subroutine of vect_determine_vectorization_factor. Set the vector
209 types of STMT_INFO and all attached pattern statements and update
210 the vectorization factor VF accordingly. If some of the statements
211 produce a mask result whose vector type can only be calculated later,
212 add them to MASK_PRODUCERS. Return true on success or false if
213 something prevented vectorization. */
216 vect_determine_vf_for_stmt (stmt_vec_info stmt_info
, poly_uint64
*vf
,
217 vec
<stmt_vec_info
> *mask_producers
)
219 vec_info
*vinfo
= stmt_info
->vinfo
;
220 if (dump_enabled_p ())
221 dump_printf_loc (MSG_NOTE
, vect_location
, "==> examining statement: %G",
224 = vect_determine_vf_for_stmt_1 (stmt_info
, false, vf
, mask_producers
);
228 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
229 && STMT_VINFO_RELATED_STMT (stmt_info
))
231 gimple
*pattern_def_seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
);
232 stmt_info
= STMT_VINFO_RELATED_STMT (stmt_info
);
234 /* If a pattern statement has def stmts, analyze them too. */
235 for (gimple_stmt_iterator si
= gsi_start (pattern_def_seq
);
236 !gsi_end_p (si
); gsi_next (&si
))
238 stmt_vec_info def_stmt_info
= vinfo
->lookup_stmt (gsi_stmt (si
));
239 if (dump_enabled_p ())
240 dump_printf_loc (MSG_NOTE
, vect_location
,
241 "==> examining pattern def stmt: %G",
242 def_stmt_info
->stmt
);
243 if (!vect_determine_vf_for_stmt_1 (def_stmt_info
, true,
245 res
= vect_determine_vf_for_stmt_1 (def_stmt_info
, true,
251 if (dump_enabled_p ())
252 dump_printf_loc (MSG_NOTE
, vect_location
,
253 "==> examining pattern statement: %G",
255 res
= vect_determine_vf_for_stmt_1 (stmt_info
, true, vf
, mask_producers
);
260 return opt_result::success ();
263 /* Function vect_determine_vectorization_factor
265 Determine the vectorization factor (VF). VF is the number of data elements
266 that are operated upon in parallel in a single iteration of the vectorized
267 loop. For example, when vectorizing a loop that operates on 4byte elements,
268 on a target with vector size (VS) 16byte, the VF is set to 4, since 4
269 elements can fit in a single vector register.
271 We currently support vectorization of loops in which all types operated upon
272 are of the same size. Therefore this function currently sets VF according to
273 the size of the types operated upon, and fails if there are multiple sizes
276 VF is also the factor by which the loop iterations are strip-mined, e.g.:
283 for (i=0; i<N; i+=VF){
284 a[i:VF] = b[i:VF] + c[i:VF];
289 vect_determine_vectorization_factor (loop_vec_info loop_vinfo
)
291 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
292 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
293 unsigned nbbs
= loop
->num_nodes
;
294 poly_uint64 vectorization_factor
= 1;
295 tree scalar_type
= NULL_TREE
;
298 stmt_vec_info stmt_info
;
300 auto_vec
<stmt_vec_info
> mask_producers
;
302 DUMP_VECT_SCOPE ("vect_determine_vectorization_factor");
304 for (i
= 0; i
< nbbs
; i
++)
306 basic_block bb
= bbs
[i
];
308 for (gphi_iterator si
= gsi_start_phis (bb
); !gsi_end_p (si
);
312 stmt_info
= loop_vinfo
->lookup_stmt (phi
);
313 if (dump_enabled_p ())
314 dump_printf_loc (MSG_NOTE
, vect_location
, "==> examining phi: %G",
317 gcc_assert (stmt_info
);
319 if (STMT_VINFO_RELEVANT_P (stmt_info
)
320 || STMT_VINFO_LIVE_P (stmt_info
))
322 gcc_assert (!STMT_VINFO_VECTYPE (stmt_info
));
323 scalar_type
= TREE_TYPE (PHI_RESULT (phi
));
325 if (dump_enabled_p ())
326 dump_printf_loc (MSG_NOTE
, vect_location
,
327 "get vectype for scalar type: %T\n",
330 vectype
= get_vectype_for_scalar_type (loop_vinfo
, scalar_type
);
332 return opt_result::failure_at (phi
,
333 "not vectorized: unsupported "
336 STMT_VINFO_VECTYPE (stmt_info
) = vectype
;
338 if (dump_enabled_p ())
339 dump_printf_loc (MSG_NOTE
, vect_location
, "vectype: %T\n",
342 if (dump_enabled_p ())
344 dump_printf_loc (MSG_NOTE
, vect_location
, "nunits = ");
345 dump_dec (MSG_NOTE
, TYPE_VECTOR_SUBPARTS (vectype
));
346 dump_printf (MSG_NOTE
, "\n");
349 vect_update_max_nunits (&vectorization_factor
, vectype
);
353 for (gimple_stmt_iterator si
= gsi_start_bb (bb
); !gsi_end_p (si
);
356 stmt_info
= loop_vinfo
->lookup_stmt (gsi_stmt (si
));
358 = vect_determine_vf_for_stmt (stmt_info
, &vectorization_factor
,
365 /* TODO: Analyze cost. Decide if worth while to vectorize. */
366 if (dump_enabled_p ())
368 dump_printf_loc (MSG_NOTE
, vect_location
, "vectorization factor = ");
369 dump_dec (MSG_NOTE
, vectorization_factor
);
370 dump_printf (MSG_NOTE
, "\n");
373 if (known_le (vectorization_factor
, 1U))
374 return opt_result::failure_at (vect_location
,
375 "not vectorized: unsupported data-type\n");
376 LOOP_VINFO_VECT_FACTOR (loop_vinfo
) = vectorization_factor
;
378 for (i
= 0; i
< mask_producers
.length (); i
++)
380 stmt_info
= mask_producers
[i
];
381 opt_tree mask_type
= vect_get_mask_type_for_stmt (stmt_info
);
383 return opt_result::propagate_failure (mask_type
);
384 STMT_VINFO_VECTYPE (stmt_info
) = mask_type
;
387 return opt_result::success ();
391 /* Function vect_is_simple_iv_evolution.
393 FORNOW: A simple evolution of an induction variables in the loop is
394 considered a polynomial evolution. */
397 vect_is_simple_iv_evolution (unsigned loop_nb
, tree access_fn
, tree
* init
,
402 tree evolution_part
= evolution_part_in_loop_num (access_fn
, loop_nb
);
405 /* When there is no evolution in this loop, the evolution function
407 if (evolution_part
== NULL_TREE
)
410 /* When the evolution is a polynomial of degree >= 2
411 the evolution function is not "simple". */
412 if (tree_is_chrec (evolution_part
))
415 step_expr
= evolution_part
;
416 init_expr
= unshare_expr (initial_condition_in_loop_num (access_fn
, loop_nb
));
418 if (dump_enabled_p ())
419 dump_printf_loc (MSG_NOTE
, vect_location
, "step: %T, init: %T\n",
420 step_expr
, init_expr
);
425 if (TREE_CODE (step_expr
) != INTEGER_CST
426 && (TREE_CODE (step_expr
) != SSA_NAME
427 || ((bb
= gimple_bb (SSA_NAME_DEF_STMT (step_expr
)))
428 && flow_bb_inside_loop_p (get_loop (cfun
, loop_nb
), bb
))
429 || (!INTEGRAL_TYPE_P (TREE_TYPE (step_expr
))
430 && (!SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr
))
431 || !flag_associative_math
)))
432 && (TREE_CODE (step_expr
) != REAL_CST
433 || !flag_associative_math
))
435 if (dump_enabled_p ())
436 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
444 /* Return true if PHI, described by STMT_INFO, is the inner PHI in
445 what we are assuming is a double reduction. For example, given
446 a structure like this:
449 x_1 = PHI <x_4(outer2), ...>;
453 x_2 = PHI <x_1(outer1), ...>;
459 x_4 = PHI <x_3(inner)>;
462 outer loop analysis would treat x_1 as a double reduction phi and
463 this function would then return true for x_2. */
466 vect_inner_phi_in_double_reduction_p (stmt_vec_info stmt_info
, gphi
*phi
)
468 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
471 FOR_EACH_PHI_ARG (use_p
, phi
, op_iter
, SSA_OP_USE
)
472 if (stmt_vec_info def_info
= loop_vinfo
->lookup_def (USE_FROM_PTR (use_p
)))
473 if (STMT_VINFO_DEF_TYPE (def_info
) == vect_double_reduction_def
)
478 /* Function vect_analyze_scalar_cycles_1.
480 Examine the cross iteration def-use cycles of scalar variables
481 in LOOP. LOOP_VINFO represents the loop that is now being
482 considered for vectorization (can be LOOP, or an outer-loop
486 vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo
, class loop
*loop
)
488 basic_block bb
= loop
->header
;
490 auto_vec
<stmt_vec_info
, 64> worklist
;
492 bool double_reduc
, reduc_chain
;
494 DUMP_VECT_SCOPE ("vect_analyze_scalar_cycles");
496 /* First - identify all inductions. Reduction detection assumes that all the
497 inductions have been identified, therefore, this order must not be
499 for (gsi
= gsi_start_phis (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
501 gphi
*phi
= gsi
.phi ();
502 tree access_fn
= NULL
;
503 tree def
= PHI_RESULT (phi
);
504 stmt_vec_info stmt_vinfo
= loop_vinfo
->lookup_stmt (phi
);
506 if (dump_enabled_p ())
507 dump_printf_loc (MSG_NOTE
, vect_location
, "Analyze phi: %G", phi
);
509 /* Skip virtual phi's. The data dependences that are associated with
510 virtual defs/uses (i.e., memory accesses) are analyzed elsewhere. */
511 if (virtual_operand_p (def
))
514 STMT_VINFO_DEF_TYPE (stmt_vinfo
) = vect_unknown_def_type
;
516 /* Analyze the evolution function. */
517 access_fn
= analyze_scalar_evolution (loop
, def
);
520 STRIP_NOPS (access_fn
);
521 if (dump_enabled_p ())
522 dump_printf_loc (MSG_NOTE
, vect_location
,
523 "Access function of PHI: %T\n", access_fn
);
524 STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_vinfo
)
525 = initial_condition_in_loop_num (access_fn
, loop
->num
);
526 STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo
)
527 = evolution_part_in_loop_num (access_fn
, loop
->num
);
531 || vect_inner_phi_in_double_reduction_p (stmt_vinfo
, phi
)
532 || !vect_is_simple_iv_evolution (loop
->num
, access_fn
, &init
, &step
)
533 || (LOOP_VINFO_LOOP (loop_vinfo
) != loop
534 && TREE_CODE (step
) != INTEGER_CST
))
536 worklist
.safe_push (stmt_vinfo
);
540 gcc_assert (STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_vinfo
)
542 gcc_assert (STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo
) != NULL_TREE
);
544 if (dump_enabled_p ())
545 dump_printf_loc (MSG_NOTE
, vect_location
, "Detected induction.\n");
546 STMT_VINFO_DEF_TYPE (stmt_vinfo
) = vect_induction_def
;
550 /* Second - identify all reductions and nested cycles. */
551 while (worklist
.length () > 0)
553 stmt_vec_info stmt_vinfo
= worklist
.pop ();
554 gphi
*phi
= as_a
<gphi
*> (stmt_vinfo
->stmt
);
555 tree def
= PHI_RESULT (phi
);
557 if (dump_enabled_p ())
558 dump_printf_loc (MSG_NOTE
, vect_location
, "Analyze phi: %G", phi
);
560 gcc_assert (!virtual_operand_p (def
)
561 && STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_unknown_def_type
);
563 stmt_vec_info reduc_stmt_info
564 = vect_is_simple_reduction (loop_vinfo
, stmt_vinfo
, &double_reduc
,
568 STMT_VINFO_REDUC_DEF (stmt_vinfo
) = reduc_stmt_info
;
569 STMT_VINFO_REDUC_DEF (reduc_stmt_info
) = stmt_vinfo
;
572 if (dump_enabled_p ())
573 dump_printf_loc (MSG_NOTE
, vect_location
,
574 "Detected double reduction.\n");
576 STMT_VINFO_DEF_TYPE (stmt_vinfo
) = vect_double_reduction_def
;
577 STMT_VINFO_DEF_TYPE (reduc_stmt_info
) = vect_double_reduction_def
;
581 if (loop
!= LOOP_VINFO_LOOP (loop_vinfo
))
583 if (dump_enabled_p ())
584 dump_printf_loc (MSG_NOTE
, vect_location
,
585 "Detected vectorizable nested cycle.\n");
587 STMT_VINFO_DEF_TYPE (stmt_vinfo
) = vect_nested_cycle
;
591 if (dump_enabled_p ())
592 dump_printf_loc (MSG_NOTE
, vect_location
,
593 "Detected reduction.\n");
595 STMT_VINFO_DEF_TYPE (stmt_vinfo
) = vect_reduction_def
;
596 STMT_VINFO_DEF_TYPE (reduc_stmt_info
) = vect_reduction_def
;
597 /* Store the reduction cycles for possible vectorization in
598 loop-aware SLP if it was not detected as reduction
601 LOOP_VINFO_REDUCTIONS (loop_vinfo
).safe_push
607 if (dump_enabled_p ())
608 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
609 "Unknown def-use cycle pattern.\n");
614 /* Function vect_analyze_scalar_cycles.
616 Examine the cross iteration def-use cycles of scalar variables, by
617 analyzing the loop-header PHIs of scalar variables. Classify each
618 cycle as one of the following: invariant, induction, reduction, unknown.
619 We do that for the loop represented by LOOP_VINFO, and also to its
620 inner-loop, if exists.
621 Examples for scalar cycles:
636 vect_analyze_scalar_cycles (loop_vec_info loop_vinfo
)
638 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
640 vect_analyze_scalar_cycles_1 (loop_vinfo
, loop
);
642 /* When vectorizing an outer-loop, the inner-loop is executed sequentially.
643 Reductions in such inner-loop therefore have different properties than
644 the reductions in the nest that gets vectorized:
645 1. When vectorized, they are executed in the same order as in the original
646 scalar loop, so we can't change the order of computation when
648 2. FIXME: Inner-loop reductions can be used in the inner-loop, so the
649 current checks are too strict. */
652 vect_analyze_scalar_cycles_1 (loop_vinfo
, loop
->inner
);
655 /* Transfer group and reduction information from STMT_INFO to its
659 vect_fixup_reduc_chain (stmt_vec_info stmt_info
)
661 stmt_vec_info firstp
= STMT_VINFO_RELATED_STMT (stmt_info
);
663 gcc_assert (!REDUC_GROUP_FIRST_ELEMENT (firstp
)
664 && REDUC_GROUP_FIRST_ELEMENT (stmt_info
));
665 REDUC_GROUP_SIZE (firstp
) = REDUC_GROUP_SIZE (stmt_info
);
668 stmtp
= STMT_VINFO_RELATED_STMT (stmt_info
);
669 REDUC_GROUP_FIRST_ELEMENT (stmtp
) = firstp
;
670 stmt_info
= REDUC_GROUP_NEXT_ELEMENT (stmt_info
);
672 REDUC_GROUP_NEXT_ELEMENT (stmtp
)
673 = STMT_VINFO_RELATED_STMT (stmt_info
);
676 STMT_VINFO_DEF_TYPE (stmtp
) = vect_reduction_def
;
679 /* Fixup scalar cycles that now have their stmts detected as patterns. */
682 vect_fixup_scalar_cycles_with_patterns (loop_vec_info loop_vinfo
)
687 FOR_EACH_VEC_ELT (LOOP_VINFO_REDUCTION_CHAINS (loop_vinfo
), i
, first
)
688 if (STMT_VINFO_IN_PATTERN_P (first
))
690 stmt_vec_info next
= REDUC_GROUP_NEXT_ELEMENT (first
);
693 if (! STMT_VINFO_IN_PATTERN_P (next
)
694 || STMT_VINFO_REDUC_IDX (STMT_VINFO_RELATED_STMT (next
)) == -1)
696 next
= REDUC_GROUP_NEXT_ELEMENT (next
);
698 /* If not all stmt in the chain are patterns or if we failed
699 to update STMT_VINFO_REDUC_IDX try to handle the chain
702 && STMT_VINFO_REDUC_IDX (STMT_VINFO_RELATED_STMT (first
)) != -1)
704 vect_fixup_reduc_chain (first
);
705 LOOP_VINFO_REDUCTION_CHAINS (loop_vinfo
)[i
]
706 = STMT_VINFO_RELATED_STMT (first
);
711 /* Function vect_get_loop_niters.
713 Determine how many iterations the loop is executed and place it
714 in NUMBER_OF_ITERATIONS. Place the number of latch iterations
715 in NUMBER_OF_ITERATIONSM1. Place the condition under which the
716 niter information holds in ASSUMPTIONS.
718 Return the loop exit condition. */
722 vect_get_loop_niters (class loop
*loop
, tree
*assumptions
,
723 tree
*number_of_iterations
, tree
*number_of_iterationsm1
)
725 edge exit
= single_exit (loop
);
726 class tree_niter_desc niter_desc
;
727 tree niter_assumptions
, niter
, may_be_zero
;
728 gcond
*cond
= get_loop_exit_condition (loop
);
730 *assumptions
= boolean_true_node
;
731 *number_of_iterationsm1
= chrec_dont_know
;
732 *number_of_iterations
= chrec_dont_know
;
733 DUMP_VECT_SCOPE ("get_loop_niters");
738 may_be_zero
= NULL_TREE
;
739 if (!number_of_iterations_exit_assumptions (loop
, exit
, &niter_desc
, NULL
)
740 || chrec_contains_undetermined (niter_desc
.niter
))
743 niter_assumptions
= niter_desc
.assumptions
;
744 may_be_zero
= niter_desc
.may_be_zero
;
745 niter
= niter_desc
.niter
;
747 if (may_be_zero
&& integer_zerop (may_be_zero
))
748 may_be_zero
= NULL_TREE
;
752 if (COMPARISON_CLASS_P (may_be_zero
))
754 /* Try to combine may_be_zero with assumptions, this can simplify
755 computation of niter expression. */
756 if (niter_assumptions
&& !integer_nonzerop (niter_assumptions
))
757 niter_assumptions
= fold_build2 (TRUTH_AND_EXPR
, boolean_type_node
,
759 fold_build1 (TRUTH_NOT_EXPR
,
763 niter
= fold_build3 (COND_EXPR
, TREE_TYPE (niter
), may_be_zero
,
764 build_int_cst (TREE_TYPE (niter
), 0),
765 rewrite_to_non_trapping_overflow (niter
));
767 may_be_zero
= NULL_TREE
;
769 else if (integer_nonzerop (may_be_zero
))
771 *number_of_iterationsm1
= build_int_cst (TREE_TYPE (niter
), 0);
772 *number_of_iterations
= build_int_cst (TREE_TYPE (niter
), 1);
779 *assumptions
= niter_assumptions
;
780 *number_of_iterationsm1
= niter
;
782 /* We want the number of loop header executions which is the number
783 of latch executions plus one.
784 ??? For UINT_MAX latch executions this number overflows to zero
785 for loops like do { n++; } while (n != 0); */
786 if (niter
&& !chrec_contains_undetermined (niter
))
787 niter
= fold_build2 (PLUS_EXPR
, TREE_TYPE (niter
), unshare_expr (niter
),
788 build_int_cst (TREE_TYPE (niter
), 1));
789 *number_of_iterations
= niter
;
794 /* Function bb_in_loop_p
796 Used as predicate for dfs order traversal of the loop bbs. */
799 bb_in_loop_p (const_basic_block bb
, const void *data
)
801 const class loop
*const loop
= (const class loop
*)data
;
802 if (flow_bb_inside_loop_p (loop
, bb
))
808 /* Create and initialize a new loop_vec_info struct for LOOP_IN, as well as
809 stmt_vec_info structs for all the stmts in LOOP_IN. */
811 _loop_vec_info::_loop_vec_info (class loop
*loop_in
, vec_info_shared
*shared
)
812 : vec_info (vec_info::loop
, init_cost (loop_in
), shared
),
814 bbs (XCNEWVEC (basic_block
, loop
->num_nodes
)),
815 num_itersm1 (NULL_TREE
),
816 num_iters (NULL_TREE
),
817 num_iters_unchanged (NULL_TREE
),
818 num_iters_assumptions (NULL_TREE
),
820 versioning_threshold (0),
821 vectorization_factor (0),
822 max_vectorization_factor (0),
823 mask_skip_niters (NULL_TREE
),
824 mask_compare_type (NULL_TREE
),
825 simd_if_cond (NULL_TREE
),
827 peeling_for_alignment (0),
831 slp_unrolling_factor (1),
832 single_scalar_iteration_cost (0),
833 vectorizable (false),
834 can_fully_mask_p (true),
835 fully_masked_p (false),
836 peeling_for_gaps (false),
837 peeling_for_niter (false),
838 no_data_dependencies (false),
839 has_mask_store (false),
840 scalar_loop_scaling (profile_probability::uninitialized ()),
842 orig_loop_info (NULL
)
844 /* CHECKME: We want to visit all BBs before their successors (except for
845 latch blocks, for which this assertion wouldn't hold). In the simple
846 case of the loop forms we allow, a dfs order of the BBs would the same
847 as reversed postorder traversal, so we are safe. */
849 unsigned int nbbs
= dfs_enumerate_from (loop
->header
, 0, bb_in_loop_p
,
850 bbs
, loop
->num_nodes
, loop
);
851 gcc_assert (nbbs
== loop
->num_nodes
);
853 for (unsigned int i
= 0; i
< nbbs
; i
++)
855 basic_block bb
= bbs
[i
];
856 gimple_stmt_iterator si
;
858 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
860 gimple
*phi
= gsi_stmt (si
);
861 gimple_set_uid (phi
, 0);
865 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
867 gimple
*stmt
= gsi_stmt (si
);
868 gimple_set_uid (stmt
, 0);
870 /* If .GOMP_SIMD_LANE call for the current loop has 3 arguments, the
871 third argument is the #pragma omp simd if (x) condition, when 0,
872 loop shouldn't be vectorized, when non-zero constant, it should
873 be vectorized normally, otherwise versioned with vectorized loop
874 done if the condition is non-zero at runtime. */
876 && is_gimple_call (stmt
)
877 && gimple_call_internal_p (stmt
)
878 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
879 && gimple_call_num_args (stmt
) >= 3
880 && TREE_CODE (gimple_call_arg (stmt
, 0)) == SSA_NAME
882 == SSA_NAME_VAR (gimple_call_arg (stmt
, 0))))
884 tree arg
= gimple_call_arg (stmt
, 2);
885 if (integer_zerop (arg
) || TREE_CODE (arg
) == SSA_NAME
)
888 gcc_assert (integer_nonzerop (arg
));
893 epilogue_vinfos
.create (6);
896 /* Free all levels of MASKS. */
899 release_vec_loop_masks (vec_loop_masks
*masks
)
903 FOR_EACH_VEC_ELT (*masks
, i
, rgm
)
904 rgm
->masks
.release ();
908 /* Free all memory used by the _loop_vec_info, as well as all the
909 stmt_vec_info structs of all the stmts in the loop. */
911 _loop_vec_info::~_loop_vec_info ()
915 release_vec_loop_masks (&masks
);
918 epilogue_vinfos
.release ();
923 /* Return an invariant or register for EXPR and emit necessary
924 computations in the LOOP_VINFO loop preheader. */
927 cse_and_gimplify_to_preheader (loop_vec_info loop_vinfo
, tree expr
)
929 if (is_gimple_reg (expr
)
930 || is_gimple_min_invariant (expr
))
933 if (! loop_vinfo
->ivexpr_map
)
934 loop_vinfo
->ivexpr_map
= new hash_map
<tree_operand_hash
, tree
>;
935 tree
&cached
= loop_vinfo
->ivexpr_map
->get_or_insert (expr
);
938 gimple_seq stmts
= NULL
;
939 cached
= force_gimple_operand (unshare_expr (expr
),
940 &stmts
, true, NULL_TREE
);
943 edge e
= loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo
));
944 gsi_insert_seq_on_edge_immediate (e
, stmts
);
950 /* Return true if we can use CMP_TYPE as the comparison type to produce
951 all masks required to mask LOOP_VINFO. */
954 can_produce_all_loop_masks_p (loop_vec_info loop_vinfo
, tree cmp_type
)
958 FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo
), i
, rgm
)
959 if (rgm
->mask_type
!= NULL_TREE
960 && !direct_internal_fn_supported_p (IFN_WHILE_ULT
,
961 cmp_type
, rgm
->mask_type
,
967 /* Calculate the maximum number of scalars per iteration for every
968 rgroup in LOOP_VINFO. */
971 vect_get_max_nscalars_per_iter (loop_vec_info loop_vinfo
)
973 unsigned int res
= 1;
976 FOR_EACH_VEC_ELT (LOOP_VINFO_MASKS (loop_vinfo
), i
, rgm
)
977 res
= MAX (res
, rgm
->max_nscalars_per_iter
);
981 /* Each statement in LOOP_VINFO can be masked where necessary. Check
982 whether we can actually generate the masks required. Return true if so,
983 storing the type of the scalar IV in LOOP_VINFO_MASK_COMPARE_TYPE. */
986 vect_verify_full_masking (loop_vec_info loop_vinfo
)
988 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
989 unsigned int min_ni_width
;
990 unsigned int max_nscalars_per_iter
991 = vect_get_max_nscalars_per_iter (loop_vinfo
);
993 /* Use a normal loop if there are no statements that need masking.
994 This only happens in rare degenerate cases: it means that the loop
995 has no loads, no stores, and no live-out values. */
996 if (LOOP_VINFO_MASKS (loop_vinfo
).is_empty ())
999 /* Get the maximum number of iterations that is representable
1000 in the counter type. */
1001 tree ni_type
= TREE_TYPE (LOOP_VINFO_NITERSM1 (loop_vinfo
));
1002 widest_int max_ni
= wi::to_widest (TYPE_MAX_VALUE (ni_type
)) + 1;
1004 /* Get a more refined estimate for the number of iterations. */
1005 widest_int max_back_edges
;
1006 if (max_loop_iterations (loop
, &max_back_edges
))
1007 max_ni
= wi::smin (max_ni
, max_back_edges
+ 1);
1009 /* Account for rgroup masks, in which each bit is replicated N times. */
1010 max_ni
*= max_nscalars_per_iter
;
1012 /* Work out how many bits we need to represent the limit. */
1013 min_ni_width
= wi::min_precision (max_ni
, UNSIGNED
);
1015 /* Find a scalar mode for which WHILE_ULT is supported. */
1016 opt_scalar_int_mode cmp_mode_iter
;
1017 tree cmp_type
= NULL_TREE
;
1018 tree iv_type
= NULL_TREE
;
1019 widest_int iv_limit
= vect_iv_limit_for_full_masking (loop_vinfo
);
1020 unsigned int iv_precision
= UINT_MAX
;
1023 iv_precision
= wi::min_precision (iv_limit
* max_nscalars_per_iter
,
1026 FOR_EACH_MODE_IN_CLASS (cmp_mode_iter
, MODE_INT
)
1028 unsigned int cmp_bits
= GET_MODE_BITSIZE (cmp_mode_iter
.require ());
1029 if (cmp_bits
>= min_ni_width
1030 && targetm
.scalar_mode_supported_p (cmp_mode_iter
.require ()))
1032 tree this_type
= build_nonstandard_integer_type (cmp_bits
, true);
1034 && can_produce_all_loop_masks_p (loop_vinfo
, this_type
))
1036 /* Although we could stop as soon as we find a valid mode,
1037 there are at least two reasons why that's not always the
1040 - An IV that's Pmode or wider is more likely to be reusable
1041 in address calculations than an IV that's narrower than
1044 - Doing the comparison in IV_PRECISION or wider allows
1045 a natural 0-based IV, whereas using a narrower comparison
1046 type requires mitigations against wrap-around.
1048 Conversely, if the IV limit is variable, doing the comparison
1049 in a wider type than the original type can introduce
1050 unnecessary extensions, so picking the widest valid mode
1051 is not always a good choice either.
1053 Here we prefer the first IV type that's Pmode or wider,
1054 and the first comparison type that's IV_PRECISION or wider.
1055 (The comparison type must be no wider than the IV type,
1056 to avoid extensions in the vector loop.)
1058 ??? We might want to try continuing beyond Pmode for ILP32
1059 targets if CMP_BITS < IV_PRECISION. */
1060 iv_type
= this_type
;
1061 if (!cmp_type
|| iv_precision
> TYPE_PRECISION (cmp_type
))
1062 cmp_type
= this_type
;
1063 if (cmp_bits
>= GET_MODE_BITSIZE (Pmode
))
1072 LOOP_VINFO_MASK_COMPARE_TYPE (loop_vinfo
) = cmp_type
;
1073 LOOP_VINFO_MASK_IV_TYPE (loop_vinfo
) = iv_type
;
1077 /* Calculate the cost of one scalar iteration of the loop. */
1079 vect_compute_single_scalar_iteration_cost (loop_vec_info loop_vinfo
)
1081 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1082 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
1083 int nbbs
= loop
->num_nodes
, factor
;
1084 int innerloop_iters
, i
;
1086 DUMP_VECT_SCOPE ("vect_compute_single_scalar_iteration_cost");
1088 /* Gather costs for statements in the scalar loop. */
1091 innerloop_iters
= 1;
1093 innerloop_iters
= 50; /* FIXME */
1095 for (i
= 0; i
< nbbs
; i
++)
1097 gimple_stmt_iterator si
;
1098 basic_block bb
= bbs
[i
];
1100 if (bb
->loop_father
== loop
->inner
)
1101 factor
= innerloop_iters
;
1105 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
1107 gimple
*stmt
= gsi_stmt (si
);
1108 stmt_vec_info stmt_info
= loop_vinfo
->lookup_stmt (stmt
);
1110 if (!is_gimple_assign (stmt
) && !is_gimple_call (stmt
))
1113 /* Skip stmts that are not vectorized inside the loop. */
1114 stmt_vec_info vstmt_info
= vect_stmt_to_vectorize (stmt_info
);
1115 if (!STMT_VINFO_RELEVANT_P (vstmt_info
)
1116 && (!STMT_VINFO_LIVE_P (vstmt_info
)
1117 || !VECTORIZABLE_CYCLE_DEF
1118 (STMT_VINFO_DEF_TYPE (vstmt_info
))))
1121 vect_cost_for_stmt kind
;
1122 if (STMT_VINFO_DATA_REF (stmt_info
))
1124 if (DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info
)))
1127 kind
= scalar_store
;
1132 record_stmt_cost (&LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo
),
1133 factor
, kind
, stmt_info
, 0, vect_prologue
);
1137 /* Now accumulate cost. */
1138 void *target_cost_data
= init_cost (loop
);
1139 stmt_info_for_cost
*si
;
1141 FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo
),
1143 (void) add_stmt_cost (target_cost_data
, si
->count
,
1144 si
->kind
, si
->stmt_info
, si
->misalign
,
1146 unsigned dummy
, body_cost
= 0;
1147 finish_cost (target_cost_data
, &dummy
, &body_cost
, &dummy
);
1148 destroy_cost_data (target_cost_data
);
1149 LOOP_VINFO_SINGLE_SCALAR_ITERATION_COST (loop_vinfo
) = body_cost
;
1153 /* Function vect_analyze_loop_form_1.
1155 Verify that certain CFG restrictions hold, including:
1156 - the loop has a pre-header
1157 - the loop has a single entry and exit
1158 - the loop exit condition is simple enough
1159 - the number of iterations can be analyzed, i.e, a countable loop. The
1160 niter could be analyzed under some assumptions. */
1163 vect_analyze_loop_form_1 (class loop
*loop
, gcond
**loop_cond
,
1164 tree
*assumptions
, tree
*number_of_iterationsm1
,
1165 tree
*number_of_iterations
, gcond
**inner_loop_cond
)
1167 DUMP_VECT_SCOPE ("vect_analyze_loop_form");
1169 /* Different restrictions apply when we are considering an inner-most loop,
1170 vs. an outer (nested) loop.
1171 (FORNOW. May want to relax some of these restrictions in the future). */
1175 /* Inner-most loop. We currently require that the number of BBs is
1176 exactly 2 (the header and latch). Vectorizable inner-most loops
1187 if (loop
->num_nodes
!= 2)
1188 return opt_result::failure_at (vect_location
,
1190 " control flow in loop.\n");
1192 if (empty_block_p (loop
->header
))
1193 return opt_result::failure_at (vect_location
,
1194 "not vectorized: empty loop.\n");
1198 class loop
*innerloop
= loop
->inner
;
1201 /* Nested loop. We currently require that the loop is doubly-nested,
1202 contains a single inner loop, and the number of BBs is exactly 5.
1203 Vectorizable outer-loops look like this:
1215 The inner-loop has the properties expected of inner-most loops
1216 as described above. */
1218 if ((loop
->inner
)->inner
|| (loop
->inner
)->next
)
1219 return opt_result::failure_at (vect_location
,
1221 " multiple nested loops.\n");
1223 if (loop
->num_nodes
!= 5)
1224 return opt_result::failure_at (vect_location
,
1226 " control flow in loop.\n");
1228 entryedge
= loop_preheader_edge (innerloop
);
1229 if (entryedge
->src
!= loop
->header
1230 || !single_exit (innerloop
)
1231 || single_exit (innerloop
)->dest
!= EDGE_PRED (loop
->latch
, 0)->src
)
1232 return opt_result::failure_at (vect_location
,
1234 " unsupported outerloop form.\n");
1236 /* Analyze the inner-loop. */
1237 tree inner_niterm1
, inner_niter
, inner_assumptions
;
1239 = vect_analyze_loop_form_1 (loop
->inner
, inner_loop_cond
,
1240 &inner_assumptions
, &inner_niterm1
,
1241 &inner_niter
, NULL
);
1244 if (dump_enabled_p ())
1245 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1246 "not vectorized: Bad inner loop.\n");
1250 /* Don't support analyzing niter under assumptions for inner
1252 if (!integer_onep (inner_assumptions
))
1253 return opt_result::failure_at (vect_location
,
1254 "not vectorized: Bad inner loop.\n");
1256 if (!expr_invariant_in_loop_p (loop
, inner_niter
))
1257 return opt_result::failure_at (vect_location
,
1258 "not vectorized: inner-loop count not"
1261 if (dump_enabled_p ())
1262 dump_printf_loc (MSG_NOTE
, vect_location
,
1263 "Considering outer-loop vectorization.\n");
1266 if (!single_exit (loop
))
1267 return opt_result::failure_at (vect_location
,
1268 "not vectorized: multiple exits.\n");
1269 if (EDGE_COUNT (loop
->header
->preds
) != 2)
1270 return opt_result::failure_at (vect_location
,
1272 " too many incoming edges.\n");
1274 /* We assume that the loop exit condition is at the end of the loop. i.e,
1275 that the loop is represented as a do-while (with a proper if-guard
1276 before the loop if needed), where the loop header contains all the
1277 executable statements, and the latch is empty. */
1278 if (!empty_block_p (loop
->latch
)
1279 || !gimple_seq_empty_p (phi_nodes (loop
->latch
)))
1280 return opt_result::failure_at (vect_location
,
1281 "not vectorized: latch block not empty.\n");
1283 /* Make sure the exit is not abnormal. */
1284 edge e
= single_exit (loop
);
1285 if (e
->flags
& EDGE_ABNORMAL
)
1286 return opt_result::failure_at (vect_location
,
1288 " abnormal loop exit edge.\n");
1290 *loop_cond
= vect_get_loop_niters (loop
, assumptions
, number_of_iterations
,
1291 number_of_iterationsm1
);
1293 return opt_result::failure_at
1295 "not vectorized: complicated exit condition.\n");
1297 if (integer_zerop (*assumptions
)
1298 || !*number_of_iterations
1299 || chrec_contains_undetermined (*number_of_iterations
))
1300 return opt_result::failure_at
1302 "not vectorized: number of iterations cannot be computed.\n");
1304 if (integer_zerop (*number_of_iterations
))
1305 return opt_result::failure_at
1307 "not vectorized: number of iterations = 0.\n");
1309 return opt_result::success ();
1312 /* Analyze LOOP form and return a loop_vec_info if it is of suitable form. */
1315 vect_analyze_loop_form (class loop
*loop
, vec_info_shared
*shared
)
1317 tree assumptions
, number_of_iterations
, number_of_iterationsm1
;
1318 gcond
*loop_cond
, *inner_loop_cond
= NULL
;
1321 = vect_analyze_loop_form_1 (loop
, &loop_cond
,
1322 &assumptions
, &number_of_iterationsm1
,
1323 &number_of_iterations
, &inner_loop_cond
);
1325 return opt_loop_vec_info::propagate_failure (res
);
1327 loop_vec_info loop_vinfo
= new _loop_vec_info (loop
, shared
);
1328 LOOP_VINFO_NITERSM1 (loop_vinfo
) = number_of_iterationsm1
;
1329 LOOP_VINFO_NITERS (loop_vinfo
) = number_of_iterations
;
1330 LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo
) = number_of_iterations
;
1331 if (!integer_onep (assumptions
))
1333 /* We consider to vectorize this loop by versioning it under
1334 some assumptions. In order to do this, we need to clear
1335 existing information computed by scev and niter analyzer. */
1337 free_numbers_of_iterations_estimates (loop
);
1338 /* Also set flag for this loop so that following scev and niter
1339 analysis are done under the assumptions. */
1340 loop_constraint_set (loop
, LOOP_C_FINITE
);
1341 /* Also record the assumptions for versioning. */
1342 LOOP_VINFO_NITERS_ASSUMPTIONS (loop_vinfo
) = assumptions
;
1345 if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
))
1347 if (dump_enabled_p ())
1349 dump_printf_loc (MSG_NOTE
, vect_location
,
1350 "Symbolic number of iterations is ");
1351 dump_generic_expr (MSG_NOTE
, TDF_DETAILS
, number_of_iterations
);
1352 dump_printf (MSG_NOTE
, "\n");
1356 stmt_vec_info loop_cond_info
= loop_vinfo
->lookup_stmt (loop_cond
);
1357 STMT_VINFO_TYPE (loop_cond_info
) = loop_exit_ctrl_vec_info_type
;
1358 if (inner_loop_cond
)
1360 stmt_vec_info inner_loop_cond_info
1361 = loop_vinfo
->lookup_stmt (inner_loop_cond
);
1362 STMT_VINFO_TYPE (inner_loop_cond_info
) = loop_exit_ctrl_vec_info_type
;
1365 gcc_assert (!loop
->aux
);
1366 loop
->aux
= loop_vinfo
;
1367 return opt_loop_vec_info::success (loop_vinfo
);
1372 /* Scan the loop stmts and dependent on whether there are any (non-)SLP
1373 statements update the vectorization factor. */
1376 vect_update_vf_for_slp (loop_vec_info loop_vinfo
)
1378 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1379 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
1380 int nbbs
= loop
->num_nodes
;
1381 poly_uint64 vectorization_factor
;
1384 DUMP_VECT_SCOPE ("vect_update_vf_for_slp");
1386 vectorization_factor
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
1387 gcc_assert (known_ne (vectorization_factor
, 0U));
1389 /* If all the stmts in the loop can be SLPed, we perform only SLP, and
1390 vectorization factor of the loop is the unrolling factor required by
1391 the SLP instances. If that unrolling factor is 1, we say, that we
1392 perform pure SLP on loop - cross iteration parallelism is not
1394 bool only_slp_in_loop
= true;
1395 for (i
= 0; i
< nbbs
; i
++)
1397 basic_block bb
= bbs
[i
];
1398 for (gimple_stmt_iterator si
= gsi_start_bb (bb
); !gsi_end_p (si
);
1401 stmt_vec_info stmt_info
= loop_vinfo
->lookup_stmt (gsi_stmt (si
));
1402 stmt_info
= vect_stmt_to_vectorize (stmt_info
);
1403 if ((STMT_VINFO_RELEVANT_P (stmt_info
)
1404 || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info
)))
1405 && !PURE_SLP_STMT (stmt_info
))
1406 /* STMT needs both SLP and loop-based vectorization. */
1407 only_slp_in_loop
= false;
1411 if (only_slp_in_loop
)
1413 if (dump_enabled_p ())
1414 dump_printf_loc (MSG_NOTE
, vect_location
,
1415 "Loop contains only SLP stmts\n");
1416 vectorization_factor
= LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo
);
1420 if (dump_enabled_p ())
1421 dump_printf_loc (MSG_NOTE
, vect_location
,
1422 "Loop contains SLP and non-SLP stmts\n");
1423 /* Both the vectorization factor and unroll factor have the form
1424 loop_vinfo->vector_size * X for some rational X, so they must have
1425 a common multiple. */
1426 vectorization_factor
1427 = force_common_multiple (vectorization_factor
,
1428 LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo
));
1431 LOOP_VINFO_VECT_FACTOR (loop_vinfo
) = vectorization_factor
;
1432 if (dump_enabled_p ())
1434 dump_printf_loc (MSG_NOTE
, vect_location
,
1435 "Updating vectorization factor to ");
1436 dump_dec (MSG_NOTE
, vectorization_factor
);
1437 dump_printf (MSG_NOTE
, ".\n");
1441 /* Return true if STMT_INFO describes a double reduction phi and if
1442 the other phi in the reduction is also relevant for vectorization.
1443 This rejects cases such as:
1446 x_1 = PHI <x_3(outer2), ...>;
1454 x_3 = PHI <x_2(inner)>;
1456 if nothing in x_2 or elsewhere makes x_1 relevant. */
1459 vect_active_double_reduction_p (stmt_vec_info stmt_info
)
1461 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_double_reduction_def
)
1464 return STMT_VINFO_RELEVANT_P (STMT_VINFO_REDUC_DEF (stmt_info
));
1467 /* Function vect_analyze_loop_operations.
1469 Scan the loop stmts and make sure they are all vectorizable. */
1472 vect_analyze_loop_operations (loop_vec_info loop_vinfo
)
1474 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1475 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
1476 int nbbs
= loop
->num_nodes
;
1478 stmt_vec_info stmt_info
;
1479 bool need_to_vectorize
= false;
1482 DUMP_VECT_SCOPE ("vect_analyze_loop_operations");
1484 auto_vec
<stmt_info_for_cost
> cost_vec
;
1486 for (i
= 0; i
< nbbs
; i
++)
1488 basic_block bb
= bbs
[i
];
1490 for (gphi_iterator si
= gsi_start_phis (bb
); !gsi_end_p (si
);
1493 gphi
*phi
= si
.phi ();
1496 stmt_info
= loop_vinfo
->lookup_stmt (phi
);
1497 if (dump_enabled_p ())
1498 dump_printf_loc (MSG_NOTE
, vect_location
, "examining phi: %G", phi
);
1499 if (virtual_operand_p (gimple_phi_result (phi
)))
1502 /* Inner-loop loop-closed exit phi in outer-loop vectorization
1503 (i.e., a phi in the tail of the outer-loop). */
1504 if (! is_loop_header_bb_p (bb
))
1506 /* FORNOW: we currently don't support the case that these phis
1507 are not used in the outerloop (unless it is double reduction,
1508 i.e., this phi is vect_reduction_def), cause this case
1509 requires to actually do something here. */
1510 if (STMT_VINFO_LIVE_P (stmt_info
)
1511 && !vect_active_double_reduction_p (stmt_info
))
1512 return opt_result::failure_at (phi
,
1513 "Unsupported loop-closed phi"
1514 " in outer-loop.\n");
1516 /* If PHI is used in the outer loop, we check that its operand
1517 is defined in the inner loop. */
1518 if (STMT_VINFO_RELEVANT_P (stmt_info
))
1522 if (gimple_phi_num_args (phi
) != 1)
1523 return opt_result::failure_at (phi
, "unsupported phi");
1525 phi_op
= PHI_ARG_DEF (phi
, 0);
1526 stmt_vec_info op_def_info
= loop_vinfo
->lookup_def (phi_op
);
1528 return opt_result::failure_at (phi
, "unsupported phi\n");
1530 if (STMT_VINFO_RELEVANT (op_def_info
) != vect_used_in_outer
1531 && (STMT_VINFO_RELEVANT (op_def_info
)
1532 != vect_used_in_outer_by_reduction
))
1533 return opt_result::failure_at (phi
, "unsupported phi\n");
1535 if ((STMT_VINFO_DEF_TYPE (stmt_info
) == vect_internal_def
1536 || (STMT_VINFO_DEF_TYPE (stmt_info
)
1537 == vect_double_reduction_def
))
1538 && !vectorizable_lc_phi (stmt_info
, NULL
, NULL
))
1539 return opt_result::failure_at (phi
, "unsupported phi\n");
1545 gcc_assert (stmt_info
);
1547 if ((STMT_VINFO_RELEVANT (stmt_info
) == vect_used_in_scope
1548 || STMT_VINFO_LIVE_P (stmt_info
))
1549 && STMT_VINFO_DEF_TYPE (stmt_info
) != vect_induction_def
)
1550 /* A scalar-dependence cycle that we don't support. */
1551 return opt_result::failure_at (phi
,
1553 " scalar dependence cycle.\n");
1555 if (STMT_VINFO_RELEVANT_P (stmt_info
))
1557 need_to_vectorize
= true;
1558 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_induction_def
1559 && ! PURE_SLP_STMT (stmt_info
))
1560 ok
= vectorizable_induction (stmt_info
, NULL
, NULL
, NULL
,
1562 else if ((STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
1563 || (STMT_VINFO_DEF_TYPE (stmt_info
)
1564 == vect_double_reduction_def
)
1565 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
)
1566 && ! PURE_SLP_STMT (stmt_info
))
1567 ok
= vectorizable_reduction (stmt_info
, NULL
, NULL
, &cost_vec
);
1570 /* SLP PHIs are tested by vect_slp_analyze_node_operations. */
1572 && STMT_VINFO_LIVE_P (stmt_info
)
1573 && !PURE_SLP_STMT (stmt_info
))
1574 ok
= vectorizable_live_operation (stmt_info
, NULL
, NULL
, NULL
,
1575 -1, false, &cost_vec
);
1578 return opt_result::failure_at (phi
,
1579 "not vectorized: relevant phi not "
1581 static_cast <gimple
*> (phi
));
1584 for (gimple_stmt_iterator si
= gsi_start_bb (bb
); !gsi_end_p (si
);
1587 gimple
*stmt
= gsi_stmt (si
);
1588 if (!gimple_clobber_p (stmt
))
1591 = vect_analyze_stmt (loop_vinfo
->lookup_stmt (stmt
),
1593 NULL
, NULL
, &cost_vec
);
1600 add_stmt_costs (loop_vinfo
->target_cost_data
, &cost_vec
);
1602 /* All operations in the loop are either irrelevant (deal with loop
1603 control, or dead), or only used outside the loop and can be moved
1604 out of the loop (e.g. invariants, inductions). The loop can be
1605 optimized away by scalar optimizations. We're better off not
1606 touching this loop. */
1607 if (!need_to_vectorize
)
1609 if (dump_enabled_p ())
1610 dump_printf_loc (MSG_NOTE
, vect_location
,
1611 "All the computation can be taken out of the loop.\n");
1612 return opt_result::failure_at
1614 "not vectorized: redundant loop. no profit to vectorize.\n");
1617 return opt_result::success ();
1620 /* Analyze the cost of the loop described by LOOP_VINFO. Decide if it
1621 is worthwhile to vectorize. Return 1 if definitely yes, 0 if
1622 definitely no, or -1 if it's worth retrying. */
1625 vect_analyze_loop_costing (loop_vec_info loop_vinfo
)
1627 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1628 unsigned int assumed_vf
= vect_vf_for_cost (loop_vinfo
);
1630 /* Only fully-masked loops can have iteration counts less than the
1631 vectorization factor. */
1632 if (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
1634 HOST_WIDE_INT max_niter
;
1636 if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
))
1637 max_niter
= LOOP_VINFO_INT_NITERS (loop_vinfo
);
1639 max_niter
= max_stmt_executions_int (loop
);
1642 && (unsigned HOST_WIDE_INT
) max_niter
< assumed_vf
)
1644 if (dump_enabled_p ())
1645 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1646 "not vectorized: iteration count smaller than "
1647 "vectorization factor.\n");
1652 int min_profitable_iters
, min_profitable_estimate
;
1653 vect_estimate_min_profitable_iters (loop_vinfo
, &min_profitable_iters
,
1654 &min_profitable_estimate
);
1656 if (min_profitable_iters
< 0)
1658 if (dump_enabled_p ())
1659 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1660 "not vectorized: vectorization not profitable.\n");
1661 if (dump_enabled_p ())
1662 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1663 "not vectorized: vector version will never be "
1668 int min_scalar_loop_bound
= (PARAM_VALUE (PARAM_MIN_VECT_LOOP_BOUND
)
1671 /* Use the cost model only if it is more conservative than user specified
1673 unsigned int th
= (unsigned) MAX (min_scalar_loop_bound
,
1674 min_profitable_iters
);
1676 LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo
) = th
;
1678 if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
1679 && LOOP_VINFO_INT_NITERS (loop_vinfo
) < th
)
1681 if (dump_enabled_p ())
1682 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1683 "not vectorized: vectorization not profitable.\n");
1684 if (dump_enabled_p ())
1685 dump_printf_loc (MSG_NOTE
, vect_location
,
1686 "not vectorized: iteration count smaller than user "
1687 "specified loop bound parameter or minimum profitable "
1688 "iterations (whichever is more conservative).\n");
1692 HOST_WIDE_INT estimated_niter
;
1694 /* If we are vectorizing an epilogue then we know the maximum number of
1695 scalar iterations it will cover is at least one lower than the
1696 vectorization factor of the main loop. */
1697 if (LOOP_VINFO_EPILOGUE_P (loop_vinfo
))
1699 = vect_vf_for_cost (LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo
)) - 1;
1702 estimated_niter
= estimated_stmt_executions_int (loop
);
1703 if (estimated_niter
== -1)
1704 estimated_niter
= likely_max_stmt_executions_int (loop
);
1706 if (estimated_niter
!= -1
1707 && ((unsigned HOST_WIDE_INT
) estimated_niter
1708 < MAX (th
, (unsigned) min_profitable_estimate
)))
1710 if (dump_enabled_p ())
1711 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1712 "not vectorized: estimated iteration count too "
1714 if (dump_enabled_p ())
1715 dump_printf_loc (MSG_NOTE
, vect_location
,
1716 "not vectorized: estimated iteration count smaller "
1717 "than specified loop bound parameter or minimum "
1718 "profitable iterations (whichever is more "
1719 "conservative).\n");
1727 vect_get_datarefs_in_loop (loop_p loop
, basic_block
*bbs
,
1728 vec
<data_reference_p
> *datarefs
,
1729 unsigned int *n_stmts
)
1732 for (unsigned i
= 0; i
< loop
->num_nodes
; i
++)
1733 for (gimple_stmt_iterator gsi
= gsi_start_bb (bbs
[i
]);
1734 !gsi_end_p (gsi
); gsi_next (&gsi
))
1736 gimple
*stmt
= gsi_stmt (gsi
);
1737 if (is_gimple_debug (stmt
))
1740 opt_result res
= vect_find_stmt_data_reference (loop
, stmt
, datarefs
);
1743 if (is_gimple_call (stmt
) && loop
->safelen
)
1745 tree fndecl
= gimple_call_fndecl (stmt
), op
;
1746 if (fndecl
!= NULL_TREE
)
1748 cgraph_node
*node
= cgraph_node::get (fndecl
);
1749 if (node
!= NULL
&& node
->simd_clones
!= NULL
)
1751 unsigned int j
, n
= gimple_call_num_args (stmt
);
1752 for (j
= 0; j
< n
; j
++)
1754 op
= gimple_call_arg (stmt
, j
);
1756 || (REFERENCE_CLASS_P (op
)
1757 && get_base_address (op
)))
1760 op
= gimple_call_lhs (stmt
);
1761 /* Ignore #pragma omp declare simd functions
1762 if they don't have data references in the
1763 call stmt itself. */
1767 || (REFERENCE_CLASS_P (op
)
1768 && get_base_address (op
)))))
1775 /* If dependence analysis will give up due to the limit on the
1776 number of datarefs stop here and fail fatally. */
1777 if (datarefs
->length ()
1778 > (unsigned)PARAM_VALUE (PARAM_LOOP_MAX_DATAREFS_FOR_DATADEPS
))
1779 return opt_result::failure_at (stmt
, "exceeded param "
1780 "loop-max-datarefs-for-datadeps\n");
1782 return opt_result::success ();
1785 /* Look for SLP-only access groups and turn each individual access into its own
1788 vect_dissolve_slp_only_groups (loop_vec_info loop_vinfo
)
1791 struct data_reference
*dr
;
1793 DUMP_VECT_SCOPE ("vect_dissolve_slp_only_groups");
1795 vec
<data_reference_p
> datarefs
= loop_vinfo
->shared
->datarefs
;
1796 FOR_EACH_VEC_ELT (datarefs
, i
, dr
)
1798 gcc_assert (DR_REF (dr
));
1799 stmt_vec_info stmt_info
= loop_vinfo
->lookup_stmt (DR_STMT (dr
));
1801 /* Check if the load is a part of an interleaving chain. */
1802 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1804 stmt_vec_info first_element
= DR_GROUP_FIRST_ELEMENT (stmt_info
);
1805 unsigned int group_size
= DR_GROUP_SIZE (first_element
);
1807 /* Check if SLP-only groups. */
1808 if (!STMT_SLP_TYPE (stmt_info
)
1809 && STMT_VINFO_SLP_VECT_ONLY (first_element
))
1811 /* Dissolve the group. */
1812 STMT_VINFO_SLP_VECT_ONLY (first_element
) = false;
1814 stmt_vec_info vinfo
= first_element
;
1817 stmt_vec_info next
= DR_GROUP_NEXT_ELEMENT (vinfo
);
1818 DR_GROUP_FIRST_ELEMENT (vinfo
) = vinfo
;
1819 DR_GROUP_NEXT_ELEMENT (vinfo
) = NULL
;
1820 DR_GROUP_SIZE (vinfo
) = 1;
1821 DR_GROUP_GAP (vinfo
) = group_size
- 1;
1830 /* Decides whether we need to create an epilogue loop to handle
1831 remaining scalar iterations and sets PEELING_FOR_NITERS accordingly. */
1834 determine_peel_for_niter (loop_vec_info loop_vinfo
)
1836 LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo
) = false;
1838 unsigned HOST_WIDE_INT const_vf
;
1839 HOST_WIDE_INT max_niter
1840 = likely_max_stmt_executions_int (LOOP_VINFO_LOOP (loop_vinfo
));
1842 unsigned th
= LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo
);
1843 if (!th
&& LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo
))
1844 th
= LOOP_VINFO_COST_MODEL_THRESHOLD (LOOP_VINFO_ORIG_LOOP_INFO
1847 if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
1848 /* The main loop handles all iterations. */
1849 LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo
) = false;
1850 else if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
1851 && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
) >= 0)
1853 /* Work out the (constant) number of iterations that need to be
1854 peeled for reasons other than niters. */
1855 unsigned int peel_niter
= LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
);
1856 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
))
1858 if (!multiple_p (LOOP_VINFO_INT_NITERS (loop_vinfo
) - peel_niter
,
1859 LOOP_VINFO_VECT_FACTOR (loop_vinfo
)))
1860 LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo
) = true;
1862 else if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
)
1863 /* ??? When peeling for gaps but not alignment, we could
1864 try to check whether the (variable) niters is known to be
1865 VF * N + 1. That's something of a niche case though. */
1866 || LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
)
1867 || !LOOP_VINFO_VECT_FACTOR (loop_vinfo
).is_constant (&const_vf
)
1868 || ((tree_ctz (LOOP_VINFO_NITERS (loop_vinfo
))
1869 < (unsigned) exact_log2 (const_vf
))
1870 /* In case of versioning, check if the maximum number of
1871 iterations is greater than th. If they are identical,
1872 the epilogue is unnecessary. */
1873 && (!LOOP_REQUIRES_VERSIONING (loop_vinfo
)
1874 || ((unsigned HOST_WIDE_INT
) max_niter
1875 > (th
/ const_vf
) * const_vf
))))
1876 LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo
) = true;
1880 /* Function vect_analyze_loop_2.
1882 Apply a set of analyses on LOOP, and create a loop_vec_info struct
1883 for it. The different analyses will record information in the
1884 loop_vec_info struct. */
1886 vect_analyze_loop_2 (loop_vec_info loop_vinfo
, bool &fatal
, unsigned *n_stmts
)
1888 opt_result ok
= opt_result::success ();
1890 unsigned int max_vf
= MAX_VECTORIZATION_FACTOR
;
1891 poly_uint64 min_vf
= 2;
1892 loop_vec_info orig_loop_vinfo
= NULL
;
1894 /* If we are dealing with an epilogue then orig_loop_vinfo points to the
1895 loop_vec_info of the first vectorized loop. */
1896 if (LOOP_VINFO_EPILOGUE_P (loop_vinfo
))
1897 orig_loop_vinfo
= LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo
);
1899 orig_loop_vinfo
= loop_vinfo
;
1900 gcc_assert (orig_loop_vinfo
);
1902 /* The first group of checks is independent of the vector size. */
1905 if (LOOP_VINFO_SIMD_IF_COND (loop_vinfo
)
1906 && integer_zerop (LOOP_VINFO_SIMD_IF_COND (loop_vinfo
)))
1907 return opt_result::failure_at (vect_location
,
1908 "not vectorized: simd if(0)\n");
1910 /* Find all data references in the loop (which correspond to vdefs/vuses)
1911 and analyze their evolution in the loop. */
1913 loop_p loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1915 /* Gather the data references and count stmts in the loop. */
1916 if (!LOOP_VINFO_DATAREFS (loop_vinfo
).exists ())
1919 = vect_get_datarefs_in_loop (loop
, LOOP_VINFO_BBS (loop_vinfo
),
1920 &LOOP_VINFO_DATAREFS (loop_vinfo
),
1924 if (dump_enabled_p ())
1925 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1926 "not vectorized: loop contains function "
1927 "calls or data references that cannot "
1931 loop_vinfo
->shared
->save_datarefs ();
1934 loop_vinfo
->shared
->check_datarefs ();
1936 /* Analyze the data references and also adjust the minimal
1937 vectorization factor according to the loads and stores. */
1939 ok
= vect_analyze_data_refs (loop_vinfo
, &min_vf
, &fatal
);
1942 if (dump_enabled_p ())
1943 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1944 "bad data references.\n");
1948 /* Classify all cross-iteration scalar data-flow cycles.
1949 Cross-iteration cycles caused by virtual phis are analyzed separately. */
1950 vect_analyze_scalar_cycles (loop_vinfo
);
1952 vect_pattern_recog (loop_vinfo
);
1954 vect_fixup_scalar_cycles_with_patterns (loop_vinfo
);
1956 /* Analyze the access patterns of the data-refs in the loop (consecutive,
1957 complex, etc.). FORNOW: Only handle consecutive access pattern. */
1959 ok
= vect_analyze_data_ref_accesses (loop_vinfo
);
1962 if (dump_enabled_p ())
1963 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1964 "bad data access.\n");
1968 /* Data-flow analysis to detect stmts that do not need to be vectorized. */
1970 ok
= vect_mark_stmts_to_be_vectorized (loop_vinfo
, &fatal
);
1973 if (dump_enabled_p ())
1974 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1975 "unexpected pattern.\n");
1979 /* While the rest of the analysis below depends on it in some way. */
1982 /* Analyze data dependences between the data-refs in the loop
1983 and adjust the maximum vectorization factor according to
1985 FORNOW: fail at the first data dependence that we encounter. */
1987 ok
= vect_analyze_data_ref_dependences (loop_vinfo
, &max_vf
);
1990 if (dump_enabled_p ())
1991 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1992 "bad data dependence.\n");
1995 if (max_vf
!= MAX_VECTORIZATION_FACTOR
1996 && maybe_lt (max_vf
, min_vf
))
1997 return opt_result::failure_at (vect_location
, "bad data dependence.\n");
1998 LOOP_VINFO_MAX_VECT_FACTOR (loop_vinfo
) = max_vf
;
2000 ok
= vect_determine_vectorization_factor (loop_vinfo
);
2003 if (dump_enabled_p ())
2004 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2005 "can't determine vectorization factor.\n");
2008 if (max_vf
!= MAX_VECTORIZATION_FACTOR
2009 && maybe_lt (max_vf
, LOOP_VINFO_VECT_FACTOR (loop_vinfo
)))
2010 return opt_result::failure_at (vect_location
, "bad data dependence.\n");
2012 /* Compute the scalar iteration cost. */
2013 vect_compute_single_scalar_iteration_cost (loop_vinfo
);
2015 poly_uint64 saved_vectorization_factor
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
2017 /* Check the SLP opportunities in the loop, analyze and build SLP trees. */
2018 ok
= vect_analyze_slp (loop_vinfo
, *n_stmts
);
2022 /* If there are any SLP instances mark them as pure_slp. */
2023 bool slp
= vect_make_slp_decision (loop_vinfo
);
2026 /* Find stmts that need to be both vectorized and SLPed. */
2027 vect_detect_hybrid_slp (loop_vinfo
);
2029 /* Update the vectorization factor based on the SLP decision. */
2030 vect_update_vf_for_slp (loop_vinfo
);
2033 bool saved_can_fully_mask_p
= LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo
);
2035 /* We don't expect to have to roll back to anything other than an empty
2037 gcc_assert (LOOP_VINFO_MASKS (loop_vinfo
).is_empty ());
2039 /* This is the point where we can re-start analysis with SLP forced off. */
2042 /* Now the vectorization factor is final. */
2043 poly_uint64 vectorization_factor
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
2044 gcc_assert (known_ne (vectorization_factor
, 0U));
2046 if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
) && dump_enabled_p ())
2048 dump_printf_loc (MSG_NOTE
, vect_location
,
2049 "vectorization_factor = ");
2050 dump_dec (MSG_NOTE
, vectorization_factor
);
2051 dump_printf (MSG_NOTE
, ", niters = %wd\n",
2052 LOOP_VINFO_INT_NITERS (loop_vinfo
));
2055 /* Analyze the alignment of the data-refs in the loop.
2056 Fail if a data reference is found that cannot be vectorized. */
2058 ok
= vect_analyze_data_refs_alignment (loop_vinfo
);
2061 if (dump_enabled_p ())
2062 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2063 "bad data alignment.\n");
2067 /* Prune the list of ddrs to be tested at run-time by versioning for alias.
2068 It is important to call pruning after vect_analyze_data_ref_accesses,
2069 since we use grouping information gathered by interleaving analysis. */
2070 ok
= vect_prune_runtime_alias_test_list (loop_vinfo
);
2074 /* Do not invoke vect_enhance_data_refs_alignment for epilogue
2075 vectorization, since we do not want to add extra peeling or
2076 add versioning for alignment. */
2077 if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo
))
2078 /* This pass will decide on using loop versioning and/or loop peeling in
2079 order to enhance the alignment of data references in the loop. */
2080 ok
= vect_enhance_data_refs_alignment (loop_vinfo
);
2082 ok
= vect_verify_datarefs_alignment (loop_vinfo
);
2088 /* Analyze operations in the SLP instances. Note this may
2089 remove unsupported SLP instances which makes the above
2090 SLP kind detection invalid. */
2091 unsigned old_size
= LOOP_VINFO_SLP_INSTANCES (loop_vinfo
).length ();
2092 vect_slp_analyze_operations (loop_vinfo
);
2093 if (LOOP_VINFO_SLP_INSTANCES (loop_vinfo
).length () != old_size
)
2095 ok
= opt_result::failure_at (vect_location
,
2096 "unsupported SLP instances\n");
2101 /* Dissolve SLP-only groups. */
2102 vect_dissolve_slp_only_groups (loop_vinfo
);
2104 /* Scan all the remaining operations in the loop that are not subject
2105 to SLP and make sure they are vectorizable. */
2106 ok
= vect_analyze_loop_operations (loop_vinfo
);
2109 if (dump_enabled_p ())
2110 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2111 "bad operation or unsupported loop bound.\n");
2115 /* Decide whether to use a fully-masked loop for this vectorization
2117 LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
)
2118 = (LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo
)
2119 && vect_verify_full_masking (loop_vinfo
));
2120 if (dump_enabled_p ())
2122 if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
2123 dump_printf_loc (MSG_NOTE
, vect_location
,
2124 "using a fully-masked loop.\n");
2126 dump_printf_loc (MSG_NOTE
, vect_location
,
2127 "not using a fully-masked loop.\n");
2130 /* If epilog loop is required because of data accesses with gaps,
2131 one additional iteration needs to be peeled. Check if there is
2132 enough iterations for vectorization. */
2133 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
)
2134 && LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
2135 && !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
2137 poly_uint64 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
2138 tree scalar_niters
= LOOP_VINFO_NITERSM1 (loop_vinfo
);
2140 if (known_lt (wi::to_widest (scalar_niters
), vf
))
2141 return opt_result::failure_at (vect_location
,
2142 "loop has no enough iterations to"
2143 " support peeling for gaps.\n");
2146 /* If we're vectorizing an epilogue loop, we either need a fully-masked
2147 loop or a loop that has a lower VF than the main loop. */
2148 if (LOOP_VINFO_EPILOGUE_P (loop_vinfo
)
2149 && !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
)
2150 && maybe_ge (LOOP_VINFO_VECT_FACTOR (loop_vinfo
),
2151 LOOP_VINFO_VECT_FACTOR (orig_loop_vinfo
)))
2152 return opt_result::failure_at (vect_location
,
2153 "Vectorization factor too high for"
2154 " epilogue loop.\n");
2156 /* Check the costings of the loop make vectorizing worthwhile. */
2157 res
= vect_analyze_loop_costing (loop_vinfo
);
2160 ok
= opt_result::failure_at (vect_location
,
2161 "Loop costings may not be worthwhile.\n");
2165 return opt_result::failure_at (vect_location
,
2166 "Loop costings not worthwhile.\n");
2168 determine_peel_for_niter (loop_vinfo
);
2169 /* If an epilogue loop is required make sure we can create one. */
2170 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
)
2171 || LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo
))
2173 if (dump_enabled_p ())
2174 dump_printf_loc (MSG_NOTE
, vect_location
, "epilog loop required\n");
2175 if (!vect_can_advance_ivs_p (loop_vinfo
)
2176 || !slpeel_can_duplicate_loop_p (LOOP_VINFO_LOOP (loop_vinfo
),
2177 single_exit (LOOP_VINFO_LOOP
2180 ok
= opt_result::failure_at (vect_location
,
2181 "not vectorized: can't create required "
2187 /* During peeling, we need to check if number of loop iterations is
2188 enough for both peeled prolog loop and vector loop. This check
2189 can be merged along with threshold check of loop versioning, so
2190 increase threshold for this case if necessary.
2192 If we are analyzing an epilogue we still want to check what its
2193 versioning threshold would be. If we decide to vectorize the epilogues we
2194 will want to use the lowest versioning threshold of all epilogues and main
2195 loop. This will enable us to enter a vectorized epilogue even when
2196 versioning the loop. We can't simply check whether the epilogue requires
2197 versioning though since we may have skipped some versioning checks when
2198 analyzing the epilogue. For instance, checks for alias versioning will be
2199 skipped when dealing with epilogues as we assume we already checked them
2200 for the main loop. So instead we always check the 'orig_loop_vinfo'. */
2201 if (LOOP_REQUIRES_VERSIONING (orig_loop_vinfo
))
2203 poly_uint64 niters_th
= 0;
2204 unsigned int th
= LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo
);
2206 if (!vect_use_loop_mask_for_alignment_p (loop_vinfo
))
2208 /* Niters for peeled prolog loop. */
2209 if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
) < 0)
2211 dr_vec_info
*dr_info
= LOOP_VINFO_UNALIGNED_DR (loop_vinfo
);
2212 tree vectype
= STMT_VINFO_VECTYPE (dr_info
->stmt
);
2213 niters_th
+= TYPE_VECTOR_SUBPARTS (vectype
) - 1;
2216 niters_th
+= LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
);
2219 /* Niters for at least one iteration of vectorized loop. */
2220 if (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
2221 niters_th
+= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
2222 /* One additional iteration because of peeling for gap. */
2223 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
))
2226 /* Use the same condition as vect_transform_loop to decide when to use
2227 the cost to determine a versioning threshold. */
2228 if (th
>= vect_vf_for_cost (loop_vinfo
)
2229 && !LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
2230 && ordered_p (th
, niters_th
))
2231 niters_th
= ordered_max (poly_uint64 (th
), niters_th
);
2233 LOOP_VINFO_VERSIONING_THRESHOLD (loop_vinfo
) = niters_th
;
2236 gcc_assert (known_eq (vectorization_factor
,
2237 LOOP_VINFO_VECT_FACTOR (loop_vinfo
)));
2239 /* Ok to vectorize! */
2240 return opt_result::success ();
2243 /* Ensure that "ok" is false (with an opt_problem if dumping is enabled). */
2246 /* Try again with SLP forced off but if we didn't do any SLP there is
2247 no point in re-trying. */
2251 /* If there are reduction chains re-trying will fail anyway. */
2252 if (! LOOP_VINFO_REDUCTION_CHAINS (loop_vinfo
).is_empty ())
2255 /* Likewise if the grouped loads or stores in the SLP cannot be handled
2256 via interleaving or lane instructions. */
2257 slp_instance instance
;
2260 FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo
), i
, instance
)
2262 stmt_vec_info vinfo
;
2263 vinfo
= SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance
))[0];
2264 if (! STMT_VINFO_GROUPED_ACCESS (vinfo
))
2266 vinfo
= DR_GROUP_FIRST_ELEMENT (vinfo
);
2267 unsigned int size
= DR_GROUP_SIZE (vinfo
);
2268 tree vectype
= STMT_VINFO_VECTYPE (vinfo
);
2269 if (! vect_store_lanes_supported (vectype
, size
, false)
2270 && ! known_eq (TYPE_VECTOR_SUBPARTS (vectype
), 1U)
2271 && ! vect_grouped_store_supported (vectype
, size
))
2272 return opt_result::failure_at (vinfo
->stmt
,
2273 "unsupported grouped store\n");
2274 FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance
), j
, node
)
2276 vinfo
= SLP_TREE_SCALAR_STMTS (node
)[0];
2277 vinfo
= DR_GROUP_FIRST_ELEMENT (vinfo
);
2278 bool single_element_p
= !DR_GROUP_NEXT_ELEMENT (vinfo
);
2279 size
= DR_GROUP_SIZE (vinfo
);
2280 vectype
= STMT_VINFO_VECTYPE (vinfo
);
2281 if (! vect_load_lanes_supported (vectype
, size
, false)
2282 && ! vect_grouped_load_supported (vectype
, single_element_p
,
2284 return opt_result::failure_at (vinfo
->stmt
,
2285 "unsupported grouped load\n");
2289 if (dump_enabled_p ())
2290 dump_printf_loc (MSG_NOTE
, vect_location
,
2291 "re-trying with SLP disabled\n");
2293 /* Roll back state appropriately. No SLP this time. */
2295 /* Restore vectorization factor as it were without SLP. */
2296 LOOP_VINFO_VECT_FACTOR (loop_vinfo
) = saved_vectorization_factor
;
2297 /* Free the SLP instances. */
2298 FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo
), j
, instance
)
2299 vect_free_slp_instance (instance
, false);
2300 LOOP_VINFO_SLP_INSTANCES (loop_vinfo
).release ();
2301 /* Reset SLP type to loop_vect on all stmts. */
2302 for (i
= 0; i
< LOOP_VINFO_LOOP (loop_vinfo
)->num_nodes
; ++i
)
2304 basic_block bb
= LOOP_VINFO_BBS (loop_vinfo
)[i
];
2305 for (gimple_stmt_iterator si
= gsi_start_phis (bb
);
2306 !gsi_end_p (si
); gsi_next (&si
))
2308 stmt_vec_info stmt_info
= loop_vinfo
->lookup_stmt (gsi_stmt (si
));
2309 STMT_SLP_TYPE (stmt_info
) = loop_vect
;
2310 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
2311 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_double_reduction_def
)
2313 /* vectorizable_reduction adjusts reduction stmt def-types,
2314 restore them to that of the PHI. */
2315 STMT_VINFO_DEF_TYPE (STMT_VINFO_REDUC_DEF (stmt_info
))
2316 = STMT_VINFO_DEF_TYPE (stmt_info
);
2317 STMT_VINFO_DEF_TYPE (vect_stmt_to_vectorize
2318 (STMT_VINFO_REDUC_DEF (stmt_info
)))
2319 = STMT_VINFO_DEF_TYPE (stmt_info
);
2322 for (gimple_stmt_iterator si
= gsi_start_bb (bb
);
2323 !gsi_end_p (si
); gsi_next (&si
))
2325 stmt_vec_info stmt_info
= loop_vinfo
->lookup_stmt (gsi_stmt (si
));
2326 STMT_SLP_TYPE (stmt_info
) = loop_vect
;
2327 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
2329 gimple
*pattern_def_seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
);
2330 stmt_info
= STMT_VINFO_RELATED_STMT (stmt_info
);
2331 STMT_SLP_TYPE (stmt_info
) = loop_vect
;
2332 for (gimple_stmt_iterator pi
= gsi_start (pattern_def_seq
);
2333 !gsi_end_p (pi
); gsi_next (&pi
))
2334 STMT_SLP_TYPE (loop_vinfo
->lookup_stmt (gsi_stmt (pi
)))
2339 /* Free optimized alias test DDRS. */
2340 LOOP_VINFO_LOWER_BOUNDS (loop_vinfo
).truncate (0);
2341 LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo
).release ();
2342 LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo
).release ();
2343 /* Reset target cost data. */
2344 destroy_cost_data (LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
));
2345 LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
)
2346 = init_cost (LOOP_VINFO_LOOP (loop_vinfo
));
2347 /* Reset accumulated rgroup information. */
2348 release_vec_loop_masks (&LOOP_VINFO_MASKS (loop_vinfo
));
2349 /* Reset assorted flags. */
2350 LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo
) = false;
2351 LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
) = false;
2352 LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo
) = 0;
2353 LOOP_VINFO_VERSIONING_THRESHOLD (loop_vinfo
) = 0;
2354 LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo
) = saved_can_fully_mask_p
;
2359 /* Function vect_analyze_loop.
2361 Apply a set of analyses on LOOP, and create a loop_vec_info struct
2362 for it. The different analyses will record information in the
2363 loop_vec_info struct. */
2365 vect_analyze_loop (class loop
*loop
, vec_info_shared
*shared
)
2367 auto_vector_sizes vector_sizes
;
2369 /* Autodetect first vector size we try. */
2370 targetm
.vectorize
.autovectorize_vector_sizes (&vector_sizes
,
2371 loop
->simdlen
!= 0);
2372 unsigned int next_size
= 0;
2374 DUMP_VECT_SCOPE ("analyze_loop_nest");
2376 if (loop_outer (loop
)
2377 && loop_vec_info_for_loop (loop_outer (loop
))
2378 && LOOP_VINFO_VECTORIZABLE_P (loop_vec_info_for_loop (loop_outer (loop
))))
2379 return opt_loop_vec_info::failure_at (vect_location
,
2380 "outer-loop already vectorized.\n");
2382 if (!find_loop_nest (loop
, &shared
->loop_nest
))
2383 return opt_loop_vec_info::failure_at
2385 "not vectorized: loop nest containing two or more consecutive inner"
2386 " loops cannot be vectorized\n");
2388 unsigned n_stmts
= 0;
2389 poly_uint64 autodetected_vector_size
= 0;
2390 opt_loop_vec_info first_loop_vinfo
= opt_loop_vec_info::success (NULL
);
2391 poly_uint64 next_vector_size
= 0;
2392 poly_uint64 lowest_th
= 0;
2393 unsigned vectorized_loops
= 0;
2395 bool vect_epilogues
= false;
2396 opt_result res
= opt_result::success ();
2397 unsigned HOST_WIDE_INT simdlen
= loop
->simdlen
;
2400 /* Check the CFG characteristics of the loop (nesting, entry/exit). */
2401 opt_loop_vec_info loop_vinfo
= vect_analyze_loop_form (loop
, shared
);
2404 if (dump_enabled_p ())
2405 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2406 "bad loop form.\n");
2407 gcc_checking_assert (first_loop_vinfo
== NULL
);
2410 loop_vinfo
->vector_size
= next_vector_size
;
2415 LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo
) = first_loop_vinfo
;
2417 res
= vect_analyze_loop_2 (loop_vinfo
, fatal
, &n_stmts
);
2419 autodetected_vector_size
= loop_vinfo
->vector_size
;
2424 LOOP_VINFO_VECTORIZABLE_P (loop_vinfo
) = 1;
2427 /* Once we hit the desired simdlen for the first time,
2428 discard any previous attempts. */
2430 && known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo
), simdlen
))
2432 delete first_loop_vinfo
;
2433 first_loop_vinfo
= opt_loop_vec_info::success (NULL
);
2434 LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo
) = NULL
;
2438 if (first_loop_vinfo
== NULL
)
2440 first_loop_vinfo
= loop_vinfo
;
2441 lowest_th
= LOOP_VINFO_VERSIONING_THRESHOLD (first_loop_vinfo
);
2443 else if (vect_epilogues
)
2445 first_loop_vinfo
->epilogue_vinfos
.safe_push (loop_vinfo
);
2446 poly_uint64 th
= LOOP_VINFO_VERSIONING_THRESHOLD (loop_vinfo
);
2447 gcc_assert (!LOOP_REQUIRES_VERSIONING (loop_vinfo
)
2448 || maybe_ne (lowest_th
, 0U));
2449 /* Keep track of the known smallest versioning
2451 if (ordered_p (lowest_th
, th
))
2452 lowest_th
= ordered_min (lowest_th
, th
);
2457 /* Only vectorize epilogues if PARAM_VECT_EPILOGUES_NOMASK is
2458 enabled, this is not a simd loop and it is the innermost loop. */
2459 vect_epilogues
= (!loop
->simdlen
2460 && loop
->inner
== NULL
2461 && PARAM_VALUE (PARAM_VECT_EPILOGUES_NOMASK
)
2462 /* For now only allow one epilogue loop. */
2463 && first_loop_vinfo
->epilogue_vinfos
.is_empty ());
2465 /* Commit to first_loop_vinfo if we have no reason to try
2467 if (!simdlen
&& !vect_epilogues
)
2475 gcc_checking_assert (first_loop_vinfo
== NULL
);
2480 if (next_size
< vector_sizes
.length ()
2481 && known_eq (vector_sizes
[next_size
], autodetected_vector_size
))
2484 if (next_size
== vector_sizes
.length ()
2485 || known_eq (autodetected_vector_size
, 0U))
2488 /* Try the next biggest vector size. */
2489 next_vector_size
= vector_sizes
[next_size
++];
2490 if (dump_enabled_p ())
2492 dump_printf_loc (MSG_NOTE
, vect_location
,
2493 "***** Re-trying analysis with "
2495 dump_dec (MSG_NOTE
, next_vector_size
);
2496 dump_printf (MSG_NOTE
, "\n");
2500 if (first_loop_vinfo
)
2502 loop
->aux
= (loop_vec_info
) first_loop_vinfo
;
2503 if (dump_enabled_p ())
2505 dump_printf_loc (MSG_NOTE
, vect_location
,
2506 "***** Choosing vector size ");
2507 dump_dec (MSG_NOTE
, first_loop_vinfo
->vector_size
);
2508 dump_printf (MSG_NOTE
, "\n");
2510 LOOP_VINFO_VERSIONING_THRESHOLD (first_loop_vinfo
) = lowest_th
;
2511 return first_loop_vinfo
;
2514 return opt_loop_vec_info::propagate_failure (res
);
2517 /* Return true if there is an in-order reduction function for CODE, storing
2518 it in *REDUC_FN if so. */
2521 fold_left_reduction_fn (tree_code code
, internal_fn
*reduc_fn
)
2526 *reduc_fn
= IFN_FOLD_LEFT_PLUS
;
2534 /* Function reduction_fn_for_scalar_code
2537 CODE - tree_code of a reduction operations.
2540 REDUC_FN - the corresponding internal function to be used to reduce the
2541 vector of partial results into a single scalar result, or IFN_LAST
2542 if the operation is a supported reduction operation, but does not have
2543 such an internal function.
2545 Return FALSE if CODE currently cannot be vectorized as reduction. */
2548 reduction_fn_for_scalar_code (enum tree_code code
, internal_fn
*reduc_fn
)
2553 *reduc_fn
= IFN_REDUC_MAX
;
2557 *reduc_fn
= IFN_REDUC_MIN
;
2561 *reduc_fn
= IFN_REDUC_PLUS
;
2565 *reduc_fn
= IFN_REDUC_AND
;
2569 *reduc_fn
= IFN_REDUC_IOR
;
2573 *reduc_fn
= IFN_REDUC_XOR
;
2578 *reduc_fn
= IFN_LAST
;
2586 /* If there is a neutral value X such that SLP reduction NODE would not
2587 be affected by the introduction of additional X elements, return that X,
2588 otherwise return null. CODE is the code of the reduction. REDUC_CHAIN
2589 is true if the SLP statements perform a single reduction, false if each
2590 statement performs an independent reduction. */
2593 neutral_op_for_slp_reduction (slp_tree slp_node
, tree_code code
,
2596 vec
<stmt_vec_info
> stmts
= SLP_TREE_SCALAR_STMTS (slp_node
);
2597 stmt_vec_info stmt_vinfo
= stmts
[0];
2598 tree vector_type
= STMT_VINFO_VECTYPE (stmt_vinfo
);
2599 tree scalar_type
= TREE_TYPE (vector_type
);
2600 class loop
*loop
= gimple_bb (stmt_vinfo
->stmt
)->loop_father
;
2605 case WIDEN_SUM_EXPR
:
2612 return build_zero_cst (scalar_type
);
2615 return build_one_cst (scalar_type
);
2618 return build_all_ones_cst (scalar_type
);
2622 /* For MIN/MAX the initial values are neutral. A reduction chain
2623 has only a single initial value, so that value is neutral for
2626 return PHI_ARG_DEF_FROM_EDGE (stmt_vinfo
->stmt
,
2627 loop_preheader_edge (loop
));
2635 /* Error reporting helper for vect_is_simple_reduction below. GIMPLE statement
2636 STMT is printed with a message MSG. */
2639 report_vect_op (dump_flags_t msg_type
, gimple
*stmt
, const char *msg
)
2641 dump_printf_loc (msg_type
, vect_location
, "%s%G", msg
, stmt
);
2644 /* Return true if we need an in-order reduction for operation CODE
2645 on type TYPE. NEED_WRAPPING_INTEGRAL_OVERFLOW is true if integer
2646 overflow must wrap. */
2649 needs_fold_left_reduction_p (tree type
, tree_code code
)
2651 /* CHECKME: check for !flag_finite_math_only too? */
2652 if (SCALAR_FLOAT_TYPE_P (type
))
2660 return !flag_associative_math
;
2663 if (INTEGRAL_TYPE_P (type
))
2665 if (!operation_no_trapping_overflow (type
, code
))
2670 if (SAT_FIXED_POINT_TYPE_P (type
))
2676 /* Return true if the reduction PHI in LOOP with latch arg LOOP_ARG and
2677 has a handled computation expression. Store the main reduction
2678 operation in *CODE. */
2681 check_reduction_path (dump_user_location_t loc
, loop_p loop
, gphi
*phi
,
2682 tree loop_arg
, enum tree_code
*code
,
2683 vec
<std::pair
<ssa_op_iter
, use_operand_p
> > &path
)
2685 auto_bitmap visited
;
2686 tree lookfor
= PHI_RESULT (phi
);
2688 use_operand_p curr
= op_iter_init_phiuse (&curri
, phi
, SSA_OP_USE
);
2689 while (USE_FROM_PTR (curr
) != loop_arg
)
2690 curr
= op_iter_next_use (&curri
);
2691 curri
.i
= curri
.numops
;
2694 path
.safe_push (std::make_pair (curri
, curr
));
2695 tree use
= USE_FROM_PTR (curr
);
2698 gimple
*def
= SSA_NAME_DEF_STMT (use
);
2699 if (gimple_nop_p (def
)
2700 || ! flow_bb_inside_loop_p (loop
, gimple_bb (def
)))
2705 std::pair
<ssa_op_iter
, use_operand_p
> x
= path
.pop ();
2709 curr
= op_iter_next_use (&curri
);
2710 /* Skip already visited or non-SSA operands (from iterating
2712 while (curr
!= NULL_USE_OPERAND_P
2713 && (TREE_CODE (USE_FROM_PTR (curr
)) != SSA_NAME
2714 || ! bitmap_set_bit (visited
,
2716 (USE_FROM_PTR (curr
)))));
2718 while (curr
== NULL_USE_OPERAND_P
&& ! path
.is_empty ());
2719 if (curr
== NULL_USE_OPERAND_P
)
2724 if (gimple_code (def
) == GIMPLE_PHI
)
2725 curr
= op_iter_init_phiuse (&curri
, as_a
<gphi
*>(def
), SSA_OP_USE
);
2727 curr
= op_iter_init_use (&curri
, def
, SSA_OP_USE
);
2728 while (curr
!= NULL_USE_OPERAND_P
2729 && (TREE_CODE (USE_FROM_PTR (curr
)) != SSA_NAME
2730 || ! bitmap_set_bit (visited
,
2732 (USE_FROM_PTR (curr
)))))
2733 curr
= op_iter_next_use (&curri
);
2734 if (curr
== NULL_USE_OPERAND_P
)
2739 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
2741 dump_printf_loc (MSG_NOTE
, loc
, "reduction path: ");
2743 std::pair
<ssa_op_iter
, use_operand_p
> *x
;
2744 FOR_EACH_VEC_ELT (path
, i
, x
)
2745 dump_printf (MSG_NOTE
, "%T ", USE_FROM_PTR (x
->second
));
2746 dump_printf (MSG_NOTE
, "\n");
2749 /* Check whether the reduction path detected is valid. */
2750 bool fail
= path
.length () == 0;
2754 for (unsigned i
= 1; i
< path
.length (); ++i
)
2756 gimple
*use_stmt
= USE_STMT (path
[i
].second
);
2757 tree op
= USE_FROM_PTR (path
[i
].second
);
2758 if (! is_gimple_assign (use_stmt
)
2759 /* The following make sure we can compute the operand index
2760 easily plus it mostly disallows chaining via COND_EXPR condition
2762 || (gimple_assign_rhs1 (use_stmt
) != op
2763 && gimple_assign_rhs2 (use_stmt
) != op
2764 && gimple_assign_rhs3 (use_stmt
) != op
))
2769 /* Check there's only a single stmt the op is used on inside
2771 imm_use_iterator imm_iter
;
2772 gimple
*op_use_stmt
;
2774 FOR_EACH_IMM_USE_STMT (op_use_stmt
, imm_iter
, op
)
2775 if (!is_gimple_debug (op_use_stmt
)
2776 && flow_bb_inside_loop_p (loop
, gimple_bb (op_use_stmt
)))
2783 tree_code use_code
= gimple_assign_rhs_code (use_stmt
);
2784 if (use_code
== MINUS_EXPR
)
2786 use_code
= PLUS_EXPR
;
2787 /* Track whether we negate the reduction value each iteration. */
2788 if (gimple_assign_rhs2 (use_stmt
) == op
)
2791 if (CONVERT_EXPR_CODE_P (use_code
)
2792 && tree_nop_conversion_p (TREE_TYPE (gimple_assign_lhs (use_stmt
)),
2793 TREE_TYPE (gimple_assign_rhs1 (use_stmt
))))
2795 else if (*code
== ERROR_MARK
)
2798 sign
= TYPE_SIGN (TREE_TYPE (gimple_assign_lhs (use_stmt
)));
2800 else if (use_code
!= *code
)
2805 else if ((use_code
== MIN_EXPR
2806 || use_code
== MAX_EXPR
)
2807 && sign
!= TYPE_SIGN (TREE_TYPE (gimple_assign_lhs (use_stmt
))))
2813 return ! fail
&& ! neg
&& *code
!= ERROR_MARK
;
2817 check_reduction_path (dump_user_location_t loc
, loop_p loop
, gphi
*phi
,
2818 tree loop_arg
, enum tree_code code
)
2820 auto_vec
<std::pair
<ssa_op_iter
, use_operand_p
> > path
;
2821 enum tree_code code_
;
2822 return (check_reduction_path (loc
, loop
, phi
, loop_arg
, &code_
, path
)
2828 /* Function vect_is_simple_reduction
2830 (1) Detect a cross-iteration def-use cycle that represents a simple
2831 reduction computation. We look for the following pattern:
2836 a2 = operation (a3, a1)
2843 a2 = operation (a3, a1)
2846 1. operation is commutative and associative and it is safe to
2847 change the order of the computation
2848 2. no uses for a2 in the loop (a2 is used out of the loop)
2849 3. no uses of a1 in the loop besides the reduction operation
2850 4. no uses of a1 outside the loop.
2852 Conditions 1,4 are tested here.
2853 Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized.
2855 (2) Detect a cross-iteration def-use cycle in nested loops, i.e.,
2858 (3) Detect cycles of phi nodes in outer-loop vectorization, i.e., double
2862 inner loop (def of a3)
2865 (4) Detect condition expressions, ie:
2866 for (int i = 0; i < N; i++)
2872 static stmt_vec_info
2873 vect_is_simple_reduction (loop_vec_info loop_info
, stmt_vec_info phi_info
,
2874 bool *double_reduc
, bool *reduc_chain_p
)
2876 gphi
*phi
= as_a
<gphi
*> (phi_info
->stmt
);
2877 gimple
*phi_use_stmt
= NULL
;
2878 imm_use_iterator imm_iter
;
2879 use_operand_p use_p
;
2881 *double_reduc
= false;
2882 *reduc_chain_p
= false;
2883 STMT_VINFO_REDUC_TYPE (phi_info
) = TREE_CODE_REDUCTION
;
2885 tree phi_name
= PHI_RESULT (phi
);
2886 /* ??? If there are no uses of the PHI result the inner loop reduction
2887 won't be detected as possibly double-reduction by vectorizable_reduction
2888 because that tries to walk the PHI arg from the preheader edge which
2889 can be constant. See PR60382. */
2890 if (has_zero_uses (phi_name
))
2892 class loop
*loop
= (gimple_bb (phi
))->loop_father
;
2893 unsigned nphi_def_loop_uses
= 0;
2894 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, phi_name
)
2896 gimple
*use_stmt
= USE_STMT (use_p
);
2897 if (is_gimple_debug (use_stmt
))
2900 if (!flow_bb_inside_loop_p (loop
, gimple_bb (use_stmt
)))
2902 if (dump_enabled_p ())
2903 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2904 "intermediate value used outside loop.\n");
2909 nphi_def_loop_uses
++;
2910 phi_use_stmt
= use_stmt
;
2913 tree latch_def
= PHI_ARG_DEF_FROM_EDGE (phi
, loop_latch_edge (loop
));
2914 if (TREE_CODE (latch_def
) != SSA_NAME
)
2916 if (dump_enabled_p ())
2917 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2918 "reduction: not ssa_name: %T\n", latch_def
);
2922 stmt_vec_info def_stmt_info
= loop_info
->lookup_def (latch_def
);
2924 || !flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt_info
->stmt
)))
2927 bool nested_in_vect_loop
2928 = flow_loop_nested_p (LOOP_VINFO_LOOP (loop_info
), loop
);
2929 unsigned nlatch_def_loop_uses
= 0;
2930 auto_vec
<gphi
*, 3> lcphis
;
2931 bool inner_loop_of_double_reduc
= false;
2932 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, latch_def
)
2934 gimple
*use_stmt
= USE_STMT (use_p
);
2935 if (is_gimple_debug (use_stmt
))
2937 if (flow_bb_inside_loop_p (loop
, gimple_bb (use_stmt
)))
2938 nlatch_def_loop_uses
++;
2941 /* We can have more than one loop-closed PHI. */
2942 lcphis
.safe_push (as_a
<gphi
*> (use_stmt
));
2943 if (nested_in_vect_loop
2944 && (STMT_VINFO_DEF_TYPE (loop_info
->lookup_stmt (use_stmt
))
2945 == vect_double_reduction_def
))
2946 inner_loop_of_double_reduc
= true;
2950 /* If we are vectorizing an inner reduction we are executing that
2951 in the original order only in case we are not dealing with a
2952 double reduction. */
2953 if (nested_in_vect_loop
&& !inner_loop_of_double_reduc
)
2955 if (dump_enabled_p ())
2956 report_vect_op (MSG_NOTE
, def_stmt_info
->stmt
,
2957 "detected nested cycle: ");
2958 return def_stmt_info
;
2961 /* If this isn't a nested cycle or if the nested cycle reduction value
2962 is used ouside of the inner loop we cannot handle uses of the reduction
2964 if (nlatch_def_loop_uses
> 1 || nphi_def_loop_uses
> 1)
2966 if (dump_enabled_p ())
2967 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2968 "reduction used in loop.\n");
2972 /* If DEF_STMT is a phi node itself, we expect it to have a single argument
2973 defined in the inner loop. */
2974 if (gphi
*def_stmt
= dyn_cast
<gphi
*> (def_stmt_info
->stmt
))
2976 tree op1
= PHI_ARG_DEF (def_stmt
, 0);
2977 if (gimple_phi_num_args (def_stmt
) != 1
2978 || TREE_CODE (op1
) != SSA_NAME
)
2980 if (dump_enabled_p ())
2981 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2982 "unsupported phi node definition.\n");
2987 gimple
*def1
= SSA_NAME_DEF_STMT (op1
);
2988 if (gimple_bb (def1
)
2989 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
))
2991 && flow_bb_inside_loop_p (loop
->inner
, gimple_bb (def1
))
2992 && is_gimple_assign (def1
)
2993 && is_a
<gphi
*> (phi_use_stmt
)
2994 && flow_bb_inside_loop_p (loop
->inner
, gimple_bb (phi_use_stmt
)))
2996 if (dump_enabled_p ())
2997 report_vect_op (MSG_NOTE
, def_stmt
,
2998 "detected double reduction: ");
3000 *double_reduc
= true;
3001 return def_stmt_info
;
3007 /* Look for the expression computing latch_def from then loop PHI result. */
3008 auto_vec
<std::pair
<ssa_op_iter
, use_operand_p
> > path
;
3009 enum tree_code code
;
3010 if (check_reduction_path (vect_location
, loop
, phi
, latch_def
, &code
,
3013 STMT_VINFO_REDUC_CODE (phi_info
) = code
;
3014 if (code
== COND_EXPR
&& !nested_in_vect_loop
)
3015 STMT_VINFO_REDUC_TYPE (phi_info
) = COND_REDUCTION
;
3017 /* Fill in STMT_VINFO_REDUC_IDX and gather stmts for an SLP
3018 reduction chain for which the additional restriction is that
3019 all operations in the chain are the same. */
3020 auto_vec
<stmt_vec_info
, 8> reduc_chain
;
3022 bool is_slp_reduc
= !nested_in_vect_loop
&& code
!= COND_EXPR
;
3023 for (i
= path
.length () - 1; i
>= 1; --i
)
3025 gimple
*stmt
= USE_STMT (path
[i
].second
);
3026 stmt_vec_info stmt_info
= loop_info
->lookup_stmt (stmt
);
3027 STMT_VINFO_REDUC_IDX (stmt_info
)
3028 = path
[i
].second
->use
- gimple_assign_rhs1_ptr (stmt
);
3029 enum tree_code stmt_code
= gimple_assign_rhs_code (stmt
);
3030 bool leading_conversion
= (CONVERT_EXPR_CODE_P (stmt_code
)
3031 && (i
== 1 || i
== path
.length () - 1));
3032 if ((stmt_code
!= code
&& !leading_conversion
)
3033 /* We can only handle the final value in epilogue
3034 generation for reduction chains. */
3035 || (i
!= 1 && !has_single_use (gimple_assign_lhs (stmt
))))
3036 is_slp_reduc
= false;
3037 /* For reduction chains we support a trailing/leading
3038 conversions. We do not store those in the actual chain. */
3039 if (leading_conversion
)
3041 reduc_chain
.safe_push (stmt_info
);
3043 if (is_slp_reduc
&& reduc_chain
.length () > 1)
3045 for (unsigned i
= 0; i
< reduc_chain
.length () - 1; ++i
)
3047 REDUC_GROUP_FIRST_ELEMENT (reduc_chain
[i
]) = reduc_chain
[0];
3048 REDUC_GROUP_NEXT_ELEMENT (reduc_chain
[i
]) = reduc_chain
[i
+1];
3050 REDUC_GROUP_FIRST_ELEMENT (reduc_chain
.last ()) = reduc_chain
[0];
3051 REDUC_GROUP_NEXT_ELEMENT (reduc_chain
.last ()) = NULL
;
3053 /* Save the chain for further analysis in SLP detection. */
3054 LOOP_VINFO_REDUCTION_CHAINS (loop_info
).safe_push (reduc_chain
[0]);
3055 REDUC_GROUP_SIZE (reduc_chain
[0]) = reduc_chain
.length ();
3057 *reduc_chain_p
= true;
3058 if (dump_enabled_p ())
3059 dump_printf_loc (MSG_NOTE
, vect_location
,
3060 "reduction: detected reduction chain\n");
3062 else if (dump_enabled_p ())
3063 dump_printf_loc (MSG_NOTE
, vect_location
,
3064 "reduction: detected reduction\n");
3066 return def_stmt_info
;
3069 if (dump_enabled_p ())
3070 dump_printf_loc (MSG_NOTE
, vect_location
,
3071 "reduction: unknown pattern\n");
3076 /* Calculate cost of peeling the loop PEEL_ITERS_PROLOGUE times. */
3078 vect_get_known_peeling_cost (loop_vec_info loop_vinfo
, int peel_iters_prologue
,
3079 int *peel_iters_epilogue
,
3080 stmt_vector_for_cost
*scalar_cost_vec
,
3081 stmt_vector_for_cost
*prologue_cost_vec
,
3082 stmt_vector_for_cost
*epilogue_cost_vec
)
3085 int assumed_vf
= vect_vf_for_cost (loop_vinfo
);
3087 if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
))
3089 *peel_iters_epilogue
= assumed_vf
/ 2;
3090 if (dump_enabled_p ())
3091 dump_printf_loc (MSG_NOTE
, vect_location
,
3092 "cost model: epilogue peel iters set to vf/2 "
3093 "because loop iterations are unknown .\n");
3095 /* If peeled iterations are known but number of scalar loop
3096 iterations are unknown, count a taken branch per peeled loop. */
3097 retval
= record_stmt_cost (prologue_cost_vec
, 1, cond_branch_taken
,
3098 NULL
, 0, vect_prologue
);
3099 retval
+= record_stmt_cost (epilogue_cost_vec
, 1, cond_branch_taken
,
3100 NULL
, 0, vect_epilogue
);
3104 int niters
= LOOP_VINFO_INT_NITERS (loop_vinfo
);
3105 peel_iters_prologue
= niters
< peel_iters_prologue
?
3106 niters
: peel_iters_prologue
;
3107 *peel_iters_epilogue
= (niters
- peel_iters_prologue
) % assumed_vf
;
3108 /* If we need to peel for gaps, but no peeling is required, we have to
3109 peel VF iterations. */
3110 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
) && !*peel_iters_epilogue
)
3111 *peel_iters_epilogue
= assumed_vf
;
3114 stmt_info_for_cost
*si
;
3116 if (peel_iters_prologue
)
3117 FOR_EACH_VEC_ELT (*scalar_cost_vec
, j
, si
)
3118 retval
+= record_stmt_cost (prologue_cost_vec
,
3119 si
->count
* peel_iters_prologue
,
3120 si
->kind
, si
->stmt_info
, si
->misalign
,
3122 if (*peel_iters_epilogue
)
3123 FOR_EACH_VEC_ELT (*scalar_cost_vec
, j
, si
)
3124 retval
+= record_stmt_cost (epilogue_cost_vec
,
3125 si
->count
* *peel_iters_epilogue
,
3126 si
->kind
, si
->stmt_info
, si
->misalign
,
3132 /* Function vect_estimate_min_profitable_iters
3134 Return the number of iterations required for the vector version of the
3135 loop to be profitable relative to the cost of the scalar version of the
3138 *RET_MIN_PROFITABLE_NITERS is a cost model profitability threshold
3139 of iterations for vectorization. -1 value means loop vectorization
3140 is not profitable. This returned value may be used for dynamic
3141 profitability check.
3143 *RET_MIN_PROFITABLE_ESTIMATE is a profitability threshold to be used
3144 for static check against estimated number of iterations. */
3147 vect_estimate_min_profitable_iters (loop_vec_info loop_vinfo
,
3148 int *ret_min_profitable_niters
,
3149 int *ret_min_profitable_estimate
)
3151 int min_profitable_iters
;
3152 int min_profitable_estimate
;
3153 int peel_iters_prologue
;
3154 int peel_iters_epilogue
;
3155 unsigned vec_inside_cost
= 0;
3156 int vec_outside_cost
= 0;
3157 unsigned vec_prologue_cost
= 0;
3158 unsigned vec_epilogue_cost
= 0;
3159 int scalar_single_iter_cost
= 0;
3160 int scalar_outside_cost
= 0;
3161 int assumed_vf
= vect_vf_for_cost (loop_vinfo
);
3162 int npeel
= LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
);
3163 void *target_cost_data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
3165 /* Cost model disabled. */
3166 if (unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo
)))
3168 if (dump_enabled_p ())
3169 dump_printf_loc (MSG_NOTE
, vect_location
, "cost model disabled.\n");
3170 *ret_min_profitable_niters
= 0;
3171 *ret_min_profitable_estimate
= 0;
3175 /* Requires loop versioning tests to handle misalignment. */
3176 if (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo
))
3178 /* FIXME: Make cost depend on complexity of individual check. */
3179 unsigned len
= LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo
).length ();
3180 (void) add_stmt_cost (target_cost_data
, len
, vector_stmt
, NULL
, 0,
3182 if (dump_enabled_p ())
3183 dump_printf (MSG_NOTE
,
3184 "cost model: Adding cost of checks for loop "
3185 "versioning to treat misalignment.\n");
3188 /* Requires loop versioning with alias checks. */
3189 if (LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo
))
3191 /* FIXME: Make cost depend on complexity of individual check. */
3192 unsigned len
= LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo
).length ();
3193 (void) add_stmt_cost (target_cost_data
, len
, vector_stmt
, NULL
, 0,
3195 len
= LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo
).length ();
3197 /* Count LEN - 1 ANDs and LEN comparisons. */
3198 (void) add_stmt_cost (target_cost_data
, len
* 2 - 1, scalar_stmt
,
3199 NULL
, 0, vect_prologue
);
3200 len
= LOOP_VINFO_LOWER_BOUNDS (loop_vinfo
).length ();
3203 /* Count LEN - 1 ANDs and LEN comparisons. */
3204 unsigned int nstmts
= len
* 2 - 1;
3205 /* +1 for each bias that needs adding. */
3206 for (unsigned int i
= 0; i
< len
; ++i
)
3207 if (!LOOP_VINFO_LOWER_BOUNDS (loop_vinfo
)[i
].unsigned_p
)
3209 (void) add_stmt_cost (target_cost_data
, nstmts
, scalar_stmt
,
3210 NULL
, 0, vect_prologue
);
3212 if (dump_enabled_p ())
3213 dump_printf (MSG_NOTE
,
3214 "cost model: Adding cost of checks for loop "
3215 "versioning aliasing.\n");
3218 /* Requires loop versioning with niter checks. */
3219 if (LOOP_REQUIRES_VERSIONING_FOR_NITERS (loop_vinfo
))
3221 /* FIXME: Make cost depend on complexity of individual check. */
3222 (void) add_stmt_cost (target_cost_data
, 1, vector_stmt
, NULL
, 0,
3224 if (dump_enabled_p ())
3225 dump_printf (MSG_NOTE
,
3226 "cost model: Adding cost of checks for loop "
3227 "versioning niters.\n");
3230 if (LOOP_REQUIRES_VERSIONING (loop_vinfo
))
3231 (void) add_stmt_cost (target_cost_data
, 1, cond_branch_taken
, NULL
, 0,
3234 /* Count statements in scalar loop. Using this as scalar cost for a single
3237 TODO: Add outer loop support.
3239 TODO: Consider assigning different costs to different scalar
3242 scalar_single_iter_cost
3243 = LOOP_VINFO_SINGLE_SCALAR_ITERATION_COST (loop_vinfo
);
3245 /* Add additional cost for the peeled instructions in prologue and epilogue
3246 loop. (For fully-masked loops there will be no peeling.)
3248 FORNOW: If we don't know the value of peel_iters for prologue or epilogue
3249 at compile-time - we assume it's vf/2 (the worst would be vf-1).
3251 TODO: Build an expression that represents peel_iters for prologue and
3252 epilogue to be used in a run-time test. */
3254 if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
3256 peel_iters_prologue
= 0;
3257 peel_iters_epilogue
= 0;
3259 if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
))
3261 /* We need to peel exactly one iteration. */
3262 peel_iters_epilogue
+= 1;
3263 stmt_info_for_cost
*si
;
3265 FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo
),
3267 (void) add_stmt_cost (target_cost_data
, si
->count
,
3268 si
->kind
, si
->stmt_info
, si
->misalign
,
3274 peel_iters_prologue
= assumed_vf
/ 2;
3275 if (dump_enabled_p ())
3276 dump_printf (MSG_NOTE
, "cost model: "
3277 "prologue peel iters set to vf/2.\n");
3279 /* If peeling for alignment is unknown, loop bound of main loop becomes
3281 peel_iters_epilogue
= assumed_vf
/ 2;
3282 if (dump_enabled_p ())
3283 dump_printf (MSG_NOTE
, "cost model: "
3284 "epilogue peel iters set to vf/2 because "
3285 "peeling for alignment is unknown.\n");
3287 /* If peeled iterations are unknown, count a taken branch and a not taken
3288 branch per peeled loop. Even if scalar loop iterations are known,
3289 vector iterations are not known since peeled prologue iterations are
3290 not known. Hence guards remain the same. */
3291 (void) add_stmt_cost (target_cost_data
, 1, cond_branch_taken
,
3292 NULL
, 0, vect_prologue
);
3293 (void) add_stmt_cost (target_cost_data
, 1, cond_branch_not_taken
,
3294 NULL
, 0, vect_prologue
);
3295 (void) add_stmt_cost (target_cost_data
, 1, cond_branch_taken
,
3296 NULL
, 0, vect_epilogue
);
3297 (void) add_stmt_cost (target_cost_data
, 1, cond_branch_not_taken
,
3298 NULL
, 0, vect_epilogue
);
3299 stmt_info_for_cost
*si
;
3301 FOR_EACH_VEC_ELT (LOOP_VINFO_SCALAR_ITERATION_COST (loop_vinfo
), j
, si
)
3303 (void) add_stmt_cost (target_cost_data
,
3304 si
->count
* peel_iters_prologue
,
3305 si
->kind
, si
->stmt_info
, si
->misalign
,
3307 (void) add_stmt_cost (target_cost_data
,
3308 si
->count
* peel_iters_epilogue
,
3309 si
->kind
, si
->stmt_info
, si
->misalign
,
3315 stmt_vector_for_cost prologue_cost_vec
, epilogue_cost_vec
;
3316 stmt_info_for_cost
*si
;
3318 void *data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
3320 prologue_cost_vec
.create (2);
3321 epilogue_cost_vec
.create (2);
3322 peel_iters_prologue
= npeel
;
3324 (void) vect_get_known_peeling_cost (loop_vinfo
, peel_iters_prologue
,
3325 &peel_iters_epilogue
,
3326 &LOOP_VINFO_SCALAR_ITERATION_COST
3329 &epilogue_cost_vec
);
3331 FOR_EACH_VEC_ELT (prologue_cost_vec
, j
, si
)
3332 (void) add_stmt_cost (data
, si
->count
, si
->kind
, si
->stmt_info
,
3333 si
->misalign
, vect_prologue
);
3335 FOR_EACH_VEC_ELT (epilogue_cost_vec
, j
, si
)
3336 (void) add_stmt_cost (data
, si
->count
, si
->kind
, si
->stmt_info
,
3337 si
->misalign
, vect_epilogue
);
3339 prologue_cost_vec
.release ();
3340 epilogue_cost_vec
.release ();
3343 /* FORNOW: The scalar outside cost is incremented in one of the
3346 1. The vectorizer checks for alignment and aliasing and generates
3347 a condition that allows dynamic vectorization. A cost model
3348 check is ANDED with the versioning condition. Hence scalar code
3349 path now has the added cost of the versioning check.
3351 if (cost > th & versioning_check)
3354 Hence run-time scalar is incremented by not-taken branch cost.
3356 2. The vectorizer then checks if a prologue is required. If the
3357 cost model check was not done before during versioning, it has to
3358 be done before the prologue check.
3361 prologue = scalar_iters
3366 if (prologue == num_iters)
3369 Hence the run-time scalar cost is incremented by a taken branch,
3370 plus a not-taken branch, plus a taken branch cost.
3372 3. The vectorizer then checks if an epilogue is required. If the
3373 cost model check was not done before during prologue check, it
3374 has to be done with the epilogue check.
3380 if (prologue == num_iters)
3383 if ((cost <= th) | (scalar_iters-prologue-epilogue == 0))
3386 Hence the run-time scalar cost should be incremented by 2 taken
3389 TODO: The back end may reorder the BBS's differently and reverse
3390 conditions/branch directions. Change the estimates below to
3391 something more reasonable. */
3393 /* If the number of iterations is known and we do not do versioning, we can
3394 decide whether to vectorize at compile time. Hence the scalar version
3395 do not carry cost model guard costs. */
3396 if (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
3397 || LOOP_REQUIRES_VERSIONING (loop_vinfo
))
3399 /* Cost model check occurs at versioning. */
3400 if (LOOP_REQUIRES_VERSIONING (loop_vinfo
))
3401 scalar_outside_cost
+= vect_get_stmt_cost (cond_branch_not_taken
);
3404 /* Cost model check occurs at prologue generation. */
3405 if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
) < 0)
3406 scalar_outside_cost
+= 2 * vect_get_stmt_cost (cond_branch_taken
)
3407 + vect_get_stmt_cost (cond_branch_not_taken
);
3408 /* Cost model check occurs at epilogue generation. */
3410 scalar_outside_cost
+= 2 * vect_get_stmt_cost (cond_branch_taken
);
3414 /* Complete the target-specific cost calculations. */
3415 finish_cost (LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
), &vec_prologue_cost
,
3416 &vec_inside_cost
, &vec_epilogue_cost
);
3418 vec_outside_cost
= (int)(vec_prologue_cost
+ vec_epilogue_cost
);
3420 if (dump_enabled_p ())
3422 dump_printf_loc (MSG_NOTE
, vect_location
, "Cost model analysis: \n");
3423 dump_printf (MSG_NOTE
, " Vector inside of loop cost: %d\n",
3425 dump_printf (MSG_NOTE
, " Vector prologue cost: %d\n",
3427 dump_printf (MSG_NOTE
, " Vector epilogue cost: %d\n",
3429 dump_printf (MSG_NOTE
, " Scalar iteration cost: %d\n",
3430 scalar_single_iter_cost
);
3431 dump_printf (MSG_NOTE
, " Scalar outside cost: %d\n",
3432 scalar_outside_cost
);
3433 dump_printf (MSG_NOTE
, " Vector outside cost: %d\n",
3435 dump_printf (MSG_NOTE
, " prologue iterations: %d\n",
3436 peel_iters_prologue
);
3437 dump_printf (MSG_NOTE
, " epilogue iterations: %d\n",
3438 peel_iters_epilogue
);
3441 /* Calculate number of iterations required to make the vector version
3442 profitable, relative to the loop bodies only. The following condition
3444 SIC * niters + SOC > VIC * ((niters - NPEEL) / VF) + VOC
3446 SIC = scalar iteration cost, VIC = vector iteration cost,
3447 VOC = vector outside cost, VF = vectorization factor,
3448 NPEEL = prologue iterations + epilogue iterations,
3449 SOC = scalar outside cost for run time cost model check. */
3451 int saving_per_viter
= (scalar_single_iter_cost
* assumed_vf
3453 if (saving_per_viter
<= 0)
3455 if (LOOP_VINFO_LOOP (loop_vinfo
)->force_vectorize
)
3456 warning_at (vect_location
.get_location_t (), OPT_Wopenmp_simd
,
3457 "vectorization did not happen for a simd loop");
3459 if (dump_enabled_p ())
3460 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3461 "cost model: the vector iteration cost = %d "
3462 "divided by the scalar iteration cost = %d "
3463 "is greater or equal to the vectorization factor = %d"
3465 vec_inside_cost
, scalar_single_iter_cost
, assumed_vf
);
3466 *ret_min_profitable_niters
= -1;
3467 *ret_min_profitable_estimate
= -1;
3471 /* ??? The "if" arm is written to handle all cases; see below for what
3472 we would do for !LOOP_VINFO_FULLY_MASKED_P. */
3473 if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
3475 /* Rewriting the condition above in terms of the number of
3476 vector iterations (vniters) rather than the number of
3477 scalar iterations (niters) gives:
3479 SIC * (vniters * VF + NPEEL) + SOC > VIC * vniters + VOC
3481 <==> vniters * (SIC * VF - VIC) > VOC - SIC * NPEEL - SOC
3483 For integer N, X and Y when X > 0:
3485 N * X > Y <==> N >= (Y /[floor] X) + 1. */
3486 int outside_overhead
= (vec_outside_cost
3487 - scalar_single_iter_cost
* peel_iters_prologue
3488 - scalar_single_iter_cost
* peel_iters_epilogue
3489 - scalar_outside_cost
);
3490 /* We're only interested in cases that require at least one
3491 vector iteration. */
3492 int min_vec_niters
= 1;
3493 if (outside_overhead
> 0)
3494 min_vec_niters
= outside_overhead
/ saving_per_viter
+ 1;
3496 if (dump_enabled_p ())
3497 dump_printf (MSG_NOTE
, " Minimum number of vector iterations: %d\n",
3500 if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
3502 /* Now that we know the minimum number of vector iterations,
3503 find the minimum niters for which the scalar cost is larger:
3505 SIC * niters > VIC * vniters + VOC - SOC
3507 We know that the minimum niters is no more than
3508 vniters * VF + NPEEL, but it might be (and often is) less
3509 than that if a partial vector iteration is cheaper than the
3510 equivalent scalar code. */
3511 int threshold
= (vec_inside_cost
* min_vec_niters
3513 - scalar_outside_cost
);
3515 min_profitable_iters
= 1;
3517 min_profitable_iters
= threshold
/ scalar_single_iter_cost
+ 1;
3520 /* Convert the number of vector iterations into a number of
3521 scalar iterations. */
3522 min_profitable_iters
= (min_vec_niters
* assumed_vf
3523 + peel_iters_prologue
3524 + peel_iters_epilogue
);
3528 min_profitable_iters
= ((vec_outside_cost
- scalar_outside_cost
)
3530 - vec_inside_cost
* peel_iters_prologue
3531 - vec_inside_cost
* peel_iters_epilogue
);
3532 if (min_profitable_iters
<= 0)
3533 min_profitable_iters
= 0;
3536 min_profitable_iters
/= saving_per_viter
;
3538 if ((scalar_single_iter_cost
* assumed_vf
* min_profitable_iters
)
3539 <= (((int) vec_inside_cost
* min_profitable_iters
)
3540 + (((int) vec_outside_cost
- scalar_outside_cost
)
3542 min_profitable_iters
++;
3546 if (dump_enabled_p ())
3547 dump_printf (MSG_NOTE
,
3548 " Calculated minimum iters for profitability: %d\n",
3549 min_profitable_iters
);
3551 if (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
)
3552 && min_profitable_iters
< (assumed_vf
+ peel_iters_prologue
))
3553 /* We want the vectorized loop to execute at least once. */
3554 min_profitable_iters
= assumed_vf
+ peel_iters_prologue
;
3556 if (dump_enabled_p ())
3557 dump_printf_loc (MSG_NOTE
, vect_location
,
3558 " Runtime profitability threshold = %d\n",
3559 min_profitable_iters
);
3561 *ret_min_profitable_niters
= min_profitable_iters
;
3563 /* Calculate number of iterations required to make the vector version
3564 profitable, relative to the loop bodies only.
3566 Non-vectorized variant is SIC * niters and it must win over vector
3567 variant on the expected loop trip count. The following condition must hold true:
3568 SIC * niters > VIC * ((niters - NPEEL) / VF) + VOC + SOC */
3570 if (vec_outside_cost
<= 0)
3571 min_profitable_estimate
= 0;
3572 else if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
3574 /* This is a repeat of the code above, but with + SOC rather
3576 int outside_overhead
= (vec_outside_cost
3577 - scalar_single_iter_cost
* peel_iters_prologue
3578 - scalar_single_iter_cost
* peel_iters_epilogue
3579 + scalar_outside_cost
);
3580 int min_vec_niters
= 1;
3581 if (outside_overhead
> 0)
3582 min_vec_niters
= outside_overhead
/ saving_per_viter
+ 1;
3584 if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
3586 int threshold
= (vec_inside_cost
* min_vec_niters
3588 + scalar_outside_cost
);
3589 min_profitable_estimate
= threshold
/ scalar_single_iter_cost
+ 1;
3592 min_profitable_estimate
= (min_vec_niters
* assumed_vf
3593 + peel_iters_prologue
3594 + peel_iters_epilogue
);
3598 min_profitable_estimate
= ((vec_outside_cost
+ scalar_outside_cost
)
3600 - vec_inside_cost
* peel_iters_prologue
3601 - vec_inside_cost
* peel_iters_epilogue
)
3602 / ((scalar_single_iter_cost
* assumed_vf
)
3605 min_profitable_estimate
= MAX (min_profitable_estimate
, min_profitable_iters
);
3606 if (dump_enabled_p ())
3607 dump_printf_loc (MSG_NOTE
, vect_location
,
3608 " Static estimate profitability threshold = %d\n",
3609 min_profitable_estimate
);
3611 *ret_min_profitable_estimate
= min_profitable_estimate
;
3614 /* Writes into SEL a mask for a vec_perm, equivalent to a vec_shr by OFFSET
3615 vector elements (not bits) for a vector with NELT elements. */
3617 calc_vec_perm_mask_for_shift (unsigned int offset
, unsigned int nelt
,
3618 vec_perm_builder
*sel
)
3620 /* The encoding is a single stepped pattern. Any wrap-around is handled
3621 by vec_perm_indices. */
3622 sel
->new_vector (nelt
, 1, 3);
3623 for (unsigned int i
= 0; i
< 3; i
++)
3624 sel
->quick_push (i
+ offset
);
3627 /* Checks whether the target supports whole-vector shifts for vectors of mode
3628 MODE. This is the case if _either_ the platform handles vec_shr_optab, _or_
3629 it supports vec_perm_const with masks for all necessary shift amounts. */
3631 have_whole_vector_shift (machine_mode mode
)
3633 if (optab_handler (vec_shr_optab
, mode
) != CODE_FOR_nothing
)
3636 /* Variable-length vectors should be handled via the optab. */
3638 if (!GET_MODE_NUNITS (mode
).is_constant (&nelt
))
3641 vec_perm_builder sel
;
3642 vec_perm_indices indices
;
3643 for (unsigned int i
= nelt
/ 2; i
>= 1; i
/= 2)
3645 calc_vec_perm_mask_for_shift (i
, nelt
, &sel
);
3646 indices
.new_vector (sel
, 2, nelt
);
3647 if (!can_vec_perm_const_p (mode
, indices
, false))
3653 /* TODO: Close dependency between vect_model_*_cost and vectorizable_*
3654 functions. Design better to avoid maintenance issues. */
3656 /* Function vect_model_reduction_cost.
3658 Models cost for a reduction operation, including the vector ops
3659 generated within the strip-mine loop, the initial definition before
3660 the loop, and the epilogue code that must be generated. */
3663 vect_model_reduction_cost (stmt_vec_info stmt_info
, internal_fn reduc_fn
,
3664 vect_reduction_type reduction_type
,
3665 int ncopies
, stmt_vector_for_cost
*cost_vec
)
3667 int prologue_cost
= 0, epilogue_cost
= 0, inside_cost
;
3668 enum tree_code code
;
3672 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
3673 class loop
*loop
= NULL
;
3676 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
3678 /* Condition reductions generate two reductions in the loop. */
3679 if (reduction_type
== COND_REDUCTION
)
3682 vectype
= STMT_VINFO_VECTYPE (stmt_info
);
3683 mode
= TYPE_MODE (vectype
);
3684 stmt_vec_info orig_stmt_info
= vect_orig_stmt (stmt_info
);
3686 code
= gimple_assign_rhs_code (orig_stmt_info
->stmt
);
3688 if (reduction_type
== EXTRACT_LAST_REDUCTION
3689 || reduction_type
== FOLD_LEFT_REDUCTION
)
3691 /* No extra instructions needed in the prologue. */
3694 if (reduction_type
== EXTRACT_LAST_REDUCTION
|| reduc_fn
!= IFN_LAST
)
3695 /* Count one reduction-like operation per vector. */
3696 inside_cost
= record_stmt_cost (cost_vec
, ncopies
, vec_to_scalar
,
3697 stmt_info
, 0, vect_body
);
3700 /* Use NELEMENTS extracts and NELEMENTS scalar ops. */
3701 unsigned int nelements
= ncopies
* vect_nunits_for_cost (vectype
);
3702 inside_cost
= record_stmt_cost (cost_vec
, nelements
,
3703 vec_to_scalar
, stmt_info
, 0,
3705 inside_cost
+= record_stmt_cost (cost_vec
, nelements
,
3706 scalar_stmt
, stmt_info
, 0,
3712 /* Add in cost for initial definition.
3713 For cond reduction we have four vectors: initial index, step,
3714 initial result of the data reduction, initial value of the index
3716 int prologue_stmts
= reduction_type
== COND_REDUCTION
? 4 : 1;
3717 prologue_cost
+= record_stmt_cost (cost_vec
, prologue_stmts
,
3718 scalar_to_vec
, stmt_info
, 0,
3721 /* Cost of reduction op inside loop. */
3722 inside_cost
= record_stmt_cost (cost_vec
, ncopies
, vector_stmt
,
3723 stmt_info
, 0, vect_body
);
3726 /* Determine cost of epilogue code.
3728 We have a reduction operator that will reduce the vector in one statement.
3729 Also requires scalar extract. */
3731 if (!loop
|| !nested_in_vect_loop_p (loop
, orig_stmt_info
))
3733 if (reduc_fn
!= IFN_LAST
)
3735 if (reduction_type
== COND_REDUCTION
)
3737 /* An EQ stmt and an COND_EXPR stmt. */
3738 epilogue_cost
+= record_stmt_cost (cost_vec
, 2,
3739 vector_stmt
, stmt_info
, 0,
3741 /* Reduction of the max index and a reduction of the found
3743 epilogue_cost
+= record_stmt_cost (cost_vec
, 2,
3744 vec_to_scalar
, stmt_info
, 0,
3746 /* A broadcast of the max value. */
3747 epilogue_cost
+= record_stmt_cost (cost_vec
, 1,
3748 scalar_to_vec
, stmt_info
, 0,
3753 epilogue_cost
+= record_stmt_cost (cost_vec
, 1, vector_stmt
,
3754 stmt_info
, 0, vect_epilogue
);
3755 epilogue_cost
+= record_stmt_cost (cost_vec
, 1,
3756 vec_to_scalar
, stmt_info
, 0,
3760 else if (reduction_type
== COND_REDUCTION
)
3762 unsigned estimated_nunits
= vect_nunits_for_cost (vectype
);
3763 /* Extraction of scalar elements. */
3764 epilogue_cost
+= record_stmt_cost (cost_vec
,
3765 2 * estimated_nunits
,
3766 vec_to_scalar
, stmt_info
, 0,
3768 /* Scalar max reductions via COND_EXPR / MAX_EXPR. */
3769 epilogue_cost
+= record_stmt_cost (cost_vec
,
3770 2 * estimated_nunits
- 3,
3771 scalar_stmt
, stmt_info
, 0,
3774 else if (reduction_type
== EXTRACT_LAST_REDUCTION
3775 || reduction_type
== FOLD_LEFT_REDUCTION
)
3776 /* No extra instructions need in the epilogue. */
3780 int vec_size_in_bits
= tree_to_uhwi (TYPE_SIZE (vectype
));
3782 TYPE_SIZE (TREE_TYPE (gimple_assign_lhs (orig_stmt_info
->stmt
)));
3783 int element_bitsize
= tree_to_uhwi (bitsize
);
3784 int nelements
= vec_size_in_bits
/ element_bitsize
;
3786 if (code
== COND_EXPR
)
3789 optab
= optab_for_tree_code (code
, vectype
, optab_default
);
3791 /* We have a whole vector shift available. */
3792 if (optab
!= unknown_optab
3793 && VECTOR_MODE_P (mode
)
3794 && optab_handler (optab
, mode
) != CODE_FOR_nothing
3795 && have_whole_vector_shift (mode
))
3797 /* Final reduction via vector shifts and the reduction operator.
3798 Also requires scalar extract. */
3799 epilogue_cost
+= record_stmt_cost (cost_vec
,
3800 exact_log2 (nelements
) * 2,
3801 vector_stmt
, stmt_info
, 0,
3803 epilogue_cost
+= record_stmt_cost (cost_vec
, 1,
3804 vec_to_scalar
, stmt_info
, 0,
3808 /* Use extracts and reduction op for final reduction. For N
3809 elements, we have N extracts and N-1 reduction ops. */
3810 epilogue_cost
+= record_stmt_cost (cost_vec
,
3811 nelements
+ nelements
- 1,
3812 vector_stmt
, stmt_info
, 0,
3817 if (dump_enabled_p ())
3818 dump_printf (MSG_NOTE
,
3819 "vect_model_reduction_cost: inside_cost = %d, "
3820 "prologue_cost = %d, epilogue_cost = %d .\n", inside_cost
,
3821 prologue_cost
, epilogue_cost
);
3825 /* Function vect_model_induction_cost.
3827 Models cost for induction operations. */
3830 vect_model_induction_cost (stmt_vec_info stmt_info
, int ncopies
,
3831 stmt_vector_for_cost
*cost_vec
)
3833 unsigned inside_cost
, prologue_cost
;
3835 if (PURE_SLP_STMT (stmt_info
))
3838 /* loop cost for vec_loop. */
3839 inside_cost
= record_stmt_cost (cost_vec
, ncopies
, vector_stmt
,
3840 stmt_info
, 0, vect_body
);
3842 /* prologue cost for vec_init and vec_step. */
3843 prologue_cost
= record_stmt_cost (cost_vec
, 2, scalar_to_vec
,
3844 stmt_info
, 0, vect_prologue
);
3846 if (dump_enabled_p ())
3847 dump_printf_loc (MSG_NOTE
, vect_location
,
3848 "vect_model_induction_cost: inside_cost = %d, "
3849 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
3854 /* Function get_initial_def_for_reduction
3857 STMT_VINFO - a stmt that performs a reduction operation in the loop.
3858 INIT_VAL - the initial value of the reduction variable
3861 ADJUSTMENT_DEF - a tree that holds a value to be added to the final result
3862 of the reduction (used for adjusting the epilog - see below).
3863 Return a vector variable, initialized according to the operation that
3864 STMT_VINFO performs. This vector will be used as the initial value
3865 of the vector of partial results.
3867 Option1 (adjust in epilog): Initialize the vector as follows:
3868 add/bit or/xor: [0,0,...,0,0]
3869 mult/bit and: [1,1,...,1,1]
3870 min/max/cond_expr: [init_val,init_val,..,init_val,init_val]
3871 and when necessary (e.g. add/mult case) let the caller know
3872 that it needs to adjust the result by init_val.
3874 Option2: Initialize the vector as follows:
3875 add/bit or/xor: [init_val,0,0,...,0]
3876 mult/bit and: [init_val,1,1,...,1]
3877 min/max/cond_expr: [init_val,init_val,...,init_val]
3878 and no adjustments are needed.
3880 For example, for the following code:
3886 STMT_VINFO is 's = s + a[i]', and the reduction variable is 's'.
3887 For a vector of 4 units, we want to return either [0,0,0,init_val],
3888 or [0,0,0,0] and let the caller know that it needs to adjust
3889 the result at the end by 'init_val'.
3891 FORNOW, we are using the 'adjust in epilog' scheme, because this way the
3892 initialization vector is simpler (same element in all entries), if
3893 ADJUSTMENT_DEF is not NULL, and Option2 otherwise.
3895 A cost model should help decide between these two schemes. */
3898 get_initial_def_for_reduction (stmt_vec_info stmt_vinfo
,
3899 enum tree_code code
, tree init_val
,
3900 tree
*adjustment_def
)
3902 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
3903 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
3904 tree scalar_type
= TREE_TYPE (init_val
);
3905 tree vectype
= get_vectype_for_scalar_type (loop_vinfo
, scalar_type
);
3908 REAL_VALUE_TYPE real_init_val
= dconst0
;
3909 int int_init_val
= 0;
3910 gimple_seq stmts
= NULL
;
3912 gcc_assert (vectype
);
3914 gcc_assert (POINTER_TYPE_P (scalar_type
) || INTEGRAL_TYPE_P (scalar_type
)
3915 || SCALAR_FLOAT_TYPE_P (scalar_type
));
3917 gcc_assert (nested_in_vect_loop_p (loop
, stmt_vinfo
)
3918 || loop
== (gimple_bb (stmt_vinfo
->stmt
))->loop_father
);
3920 /* ADJUSTMENT_DEF is NULL when called from
3921 vect_create_epilog_for_reduction to vectorize double reduction. */
3923 *adjustment_def
= NULL
;
3927 case WIDEN_SUM_EXPR
:
3937 if (code
== MULT_EXPR
)
3939 real_init_val
= dconst1
;
3943 if (code
== BIT_AND_EXPR
)
3946 if (SCALAR_FLOAT_TYPE_P (scalar_type
))
3947 def_for_init
= build_real (scalar_type
, real_init_val
);
3949 def_for_init
= build_int_cst (scalar_type
, int_init_val
);
3951 if (adjustment_def
|| operand_equal_p (def_for_init
, init_val
, 0))
3953 /* Option1: the first element is '0' or '1' as well. */
3954 if (!operand_equal_p (def_for_init
, init_val
, 0))
3955 *adjustment_def
= init_val
;
3956 init_def
= gimple_build_vector_from_val (&stmts
, vectype
,
3959 else if (!TYPE_VECTOR_SUBPARTS (vectype
).is_constant ())
3961 /* Option2 (variable length): the first element is INIT_VAL. */
3962 init_def
= gimple_build_vector_from_val (&stmts
, vectype
,
3964 init_def
= gimple_build (&stmts
, CFN_VEC_SHL_INSERT
,
3965 vectype
, init_def
, init_val
);
3969 /* Option2: the first element is INIT_VAL. */
3970 tree_vector_builder
elts (vectype
, 1, 2);
3971 elts
.quick_push (init_val
);
3972 elts
.quick_push (def_for_init
);
3973 init_def
= gimple_build_vector (&stmts
, &elts
);
3982 init_val
= gimple_convert (&stmts
, TREE_TYPE (vectype
), init_val
);
3983 init_def
= gimple_build_vector_from_val (&stmts
, vectype
, init_val
);
3992 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
3996 /* Get at the initial defs for the reduction PHIs in SLP_NODE.
3997 NUMBER_OF_VECTORS is the number of vector defs to create.
3998 If NEUTRAL_OP is nonnull, introducing extra elements of that
3999 value will not change the result. */
4002 get_initial_defs_for_reduction (slp_tree slp_node
,
4003 vec
<tree
> *vec_oprnds
,
4004 unsigned int number_of_vectors
,
4005 bool reduc_chain
, tree neutral_op
)
4007 vec
<stmt_vec_info
> stmts
= SLP_TREE_SCALAR_STMTS (slp_node
);
4008 stmt_vec_info stmt_vinfo
= stmts
[0];
4009 vec_info
*vinfo
= stmt_vinfo
->vinfo
;
4010 unsigned HOST_WIDE_INT nunits
;
4011 unsigned j
, number_of_places_left_in_vector
;
4013 unsigned int group_size
= stmts
.length ();
4017 vector_type
= STMT_VINFO_VECTYPE (stmt_vinfo
);
4019 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
);
4021 loop
= (gimple_bb (stmt_vinfo
->stmt
))->loop_father
;
4023 edge pe
= loop_preheader_edge (loop
);
4025 gcc_assert (!reduc_chain
|| neutral_op
);
4027 /* NUMBER_OF_COPIES is the number of times we need to use the same values in
4028 created vectors. It is greater than 1 if unrolling is performed.
4030 For example, we have two scalar operands, s1 and s2 (e.g., group of
4031 strided accesses of size two), while NUNITS is four (i.e., four scalars
4032 of this type can be packed in a vector). The output vector will contain
4033 two copies of each scalar operand: {s1, s2, s1, s2}. (NUMBER_OF_COPIES
4036 If REDUC_GROUP_SIZE > NUNITS, the scalars will be split into several
4037 vectors containing the operands.
4039 For example, NUNITS is four as before, and the group size is 8
4040 (s1, s2, ..., s8). We will create two vectors {s1, s2, s3, s4} and
4041 {s5, s6, s7, s8}. */
4043 if (!TYPE_VECTOR_SUBPARTS (vector_type
).is_constant (&nunits
))
4044 nunits
= group_size
;
4046 number_of_places_left_in_vector
= nunits
;
4047 bool constant_p
= true;
4048 tree_vector_builder
elts (vector_type
, nunits
, 1);
4049 elts
.quick_grow (nunits
);
4050 gimple_seq ctor_seq
= NULL
;
4051 for (j
= 0; j
< nunits
* number_of_vectors
; ++j
)
4055 stmt_vinfo
= stmts
[i
];
4057 /* Get the def before the loop. In reduction chain we have only
4058 one initial value. Else we have as many as PHIs in the group. */
4060 op
= j
!= 0 ? neutral_op
: PHI_ARG_DEF_FROM_EDGE (stmt_vinfo
->stmt
, pe
);
4061 else if (((vec_oprnds
->length () + 1) * nunits
4062 - number_of_places_left_in_vector
>= group_size
)
4066 op
= PHI_ARG_DEF_FROM_EDGE (stmt_vinfo
->stmt
, pe
);
4068 /* Create 'vect_ = {op0,op1,...,opn}'. */
4069 number_of_places_left_in_vector
--;
4070 elts
[nunits
- number_of_places_left_in_vector
- 1] = op
;
4071 if (!CONSTANT_CLASS_P (op
))
4074 if (number_of_places_left_in_vector
== 0)
4077 if (constant_p
&& !neutral_op
4078 ? multiple_p (TYPE_VECTOR_SUBPARTS (vector_type
), nunits
)
4079 : known_eq (TYPE_VECTOR_SUBPARTS (vector_type
), nunits
))
4080 /* Build the vector directly from ELTS. */
4081 init
= gimple_build_vector (&ctor_seq
, &elts
);
4082 else if (neutral_op
)
4084 /* Build a vector of the neutral value and shift the
4085 other elements into place. */
4086 init
= gimple_build_vector_from_val (&ctor_seq
, vector_type
,
4089 while (k
> 0 && elts
[k
- 1] == neutral_op
)
4094 init
= gimple_build (&ctor_seq
, CFN_VEC_SHL_INSERT
,
4095 vector_type
, init
, elts
[k
]);
4100 /* First time round, duplicate ELTS to fill the
4101 required number of vectors. */
4102 duplicate_and_interleave (vinfo
, &ctor_seq
, vector_type
, elts
,
4103 number_of_vectors
, *vec_oprnds
);
4106 vec_oprnds
->quick_push (init
);
4108 number_of_places_left_in_vector
= nunits
;
4109 elts
.new_vector (vector_type
, nunits
, 1);
4110 elts
.quick_grow (nunits
);
4114 if (ctor_seq
!= NULL
)
4115 gsi_insert_seq_on_edge_immediate (pe
, ctor_seq
);
4118 /* For a statement STMT_INFO taking part in a reduction operation return
4119 the stmt_vec_info the meta information is stored on. */
4122 info_for_reduction (stmt_vec_info stmt_info
)
4124 stmt_info
= vect_orig_stmt (stmt_info
);
4125 gcc_assert (STMT_VINFO_REDUC_DEF (stmt_info
));
4126 if (!is_a
<gphi
*> (stmt_info
->stmt
))
4127 stmt_info
= STMT_VINFO_REDUC_DEF (stmt_info
);
4128 gphi
*phi
= as_a
<gphi
*> (stmt_info
->stmt
);
4129 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_double_reduction_def
)
4131 if (gimple_phi_num_args (phi
) == 1)
4132 stmt_info
= STMT_VINFO_REDUC_DEF (stmt_info
);
4134 else if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
)
4136 edge pe
= loop_preheader_edge (gimple_bb (phi
)->loop_father
);
4138 = stmt_info
->vinfo
->lookup_def (PHI_ARG_DEF_FROM_EDGE (phi
, pe
));
4139 if (info
&& STMT_VINFO_DEF_TYPE (info
) == vect_double_reduction_def
)
4145 /* Function vect_create_epilog_for_reduction
4147 Create code at the loop-epilog to finalize the result of a reduction
4150 STMT_INFO is the scalar reduction stmt that is being vectorized.
4151 SLP_NODE is an SLP node containing a group of reduction statements. The
4152 first one in this group is STMT_INFO.
4153 SLP_NODE_INSTANCE is the SLP node instance containing SLP_NODE
4154 REDUC_INDEX says which rhs operand of the STMT_INFO is the reduction phi
4158 1. Completes the reduction def-use cycles.
4159 2. "Reduces" each vector of partial results VECT_DEFS into a single result,
4160 by calling the function specified by REDUC_FN if available, or by
4161 other means (whole-vector shifts or a scalar loop).
4162 The function also creates a new phi node at the loop exit to preserve
4163 loop-closed form, as illustrated below.
4165 The flow at the entry to this function:
4168 vec_def = phi <vec_init, null> # REDUCTION_PHI
4169 VECT_DEF = vector_stmt # vectorized form of STMT_INFO
4170 s_loop = scalar_stmt # (scalar) STMT_INFO
4172 s_out0 = phi <s_loop> # (scalar) EXIT_PHI
4176 The above is transformed by this function into:
4179 vec_def = phi <vec_init, VECT_DEF> # REDUCTION_PHI
4180 VECT_DEF = vector_stmt # vectorized form of STMT_INFO
4181 s_loop = scalar_stmt # (scalar) STMT_INFO
4183 s_out0 = phi <s_loop> # (scalar) EXIT_PHI
4184 v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
4185 v_out2 = reduce <v_out1>
4186 s_out3 = extract_field <v_out2, 0>
4187 s_out4 = adjust_result <s_out3>
4193 vect_create_epilog_for_reduction (stmt_vec_info stmt_info
,
4195 slp_instance slp_node_instance
)
4197 stmt_vec_info reduc_info
= info_for_reduction (stmt_info
);
4198 gcc_assert (reduc_info
->is_reduc_info
);
4199 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4200 /* For double reductions we need to get at the inner loop reduction
4201 stmt which has the meta info attached. Our stmt_info is that of the
4202 loop-closed PHI of the inner loop which we remember as
4203 def for the reduction PHI generation. */
4204 bool double_reduc
= false;
4205 stmt_vec_info rdef_info
= stmt_info
;
4206 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_double_reduction_def
)
4208 gcc_assert (!slp_node
);
4209 double_reduc
= true;
4210 stmt_info
= loop_vinfo
->lookup_def (gimple_phi_arg_def
4211 (stmt_info
->stmt
, 0));
4212 stmt_info
= vect_stmt_to_vectorize (stmt_info
);
4214 gphi
*reduc_def_stmt
4215 = as_a
<gphi
*> (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info
))->stmt
);
4216 enum tree_code code
= STMT_VINFO_REDUC_CODE (reduc_info
);
4217 internal_fn reduc_fn
= STMT_VINFO_REDUC_FN (reduc_info
);
4218 tree neutral_op
= NULL_TREE
;
4221 = neutral_op_for_slp_reduction (slp_node_instance
->reduc_phis
, code
,
4222 REDUC_GROUP_FIRST_ELEMENT (stmt_info
));
4223 stmt_vec_info prev_phi_info
;
4226 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
), *outer_loop
= NULL
;
4227 basic_block exit_bb
;
4230 gimple
*new_phi
= NULL
, *phi
;
4231 stmt_vec_info phi_info
;
4232 gimple_stmt_iterator exit_gsi
;
4234 tree new_temp
= NULL_TREE
, new_name
, new_scalar_dest
;
4235 gimple
*epilog_stmt
= NULL
;
4239 tree orig_name
, scalar_result
;
4240 imm_use_iterator imm_iter
, phi_imm_iter
;
4241 use_operand_p use_p
, phi_use_p
;
4243 bool nested_in_vect_loop
= false;
4244 auto_vec
<gimple
*> new_phis
;
4246 auto_vec
<tree
> scalar_results
;
4247 unsigned int group_size
= 1, k
;
4248 auto_vec
<gimple
*> phis
;
4249 bool slp_reduc
= false;
4250 bool direct_slp_reduc
;
4251 tree new_phi_result
;
4252 tree induction_index
= NULL_TREE
;
4255 group_size
= SLP_TREE_SCALAR_STMTS (slp_node
).length ();
4257 if (nested_in_vect_loop_p (loop
, stmt_info
))
4261 nested_in_vect_loop
= true;
4262 gcc_assert (!slp_node
);
4264 gcc_assert (!nested_in_vect_loop
|| double_reduc
);
4266 vectype
= STMT_VINFO_VECTYPE (stmt_info
);
4267 gcc_assert (vectype
);
4268 mode
= TYPE_MODE (vectype
);
4270 tree initial_def
= NULL
;
4271 tree induc_val
= NULL_TREE
;
4272 tree adjustment_def
= NULL
;
4277 /* Get at the scalar def before the loop, that defines the initial value
4278 of the reduction variable. */
4279 initial_def
= PHI_ARG_DEF_FROM_EDGE (reduc_def_stmt
,
4280 loop_preheader_edge (loop
));
4281 /* Optimize: for induction condition reduction, if we can't use zero
4282 for induc_val, use initial_def. */
4283 if (STMT_VINFO_REDUC_TYPE (reduc_info
) == INTEGER_INDUC_COND_REDUCTION
)
4284 induc_val
= STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_info
);
4285 else if (double_reduc
)
4287 else if (nested_in_vect_loop
)
4290 adjustment_def
= STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (reduc_info
);
4297 vec_num
= SLP_TREE_VEC_STMTS (slp_node_instance
->reduc_phis
).length ();
4304 phi_info
= STMT_VINFO_VEC_STMT (loop_vinfo
->lookup_stmt (reduc_def_stmt
));
4308 phi_info
= STMT_VINFO_RELATED_STMT (phi_info
);
4313 /* For cond reductions we want to create a new vector (INDEX_COND_EXPR)
4314 which is updated with the current index of the loop for every match of
4315 the original loop's cond_expr (VEC_STMT). This results in a vector
4316 containing the last time the condition passed for that vector lane.
4317 The first match will be a 1 to allow 0 to be used for non-matching
4318 indexes. If there are no matches at all then the vector will be all
4320 if (STMT_VINFO_REDUC_TYPE (reduc_info
) == COND_REDUCTION
)
4322 tree indx_before_incr
, indx_after_incr
;
4323 poly_uint64 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype
);
4325 gimple
*vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
)->stmt
;
4326 gcc_assert (gimple_assign_rhs_code (vec_stmt
) == VEC_COND_EXPR
);
4328 int scalar_precision
4329 = GET_MODE_PRECISION (SCALAR_TYPE_MODE (TREE_TYPE (vectype
)));
4330 tree cr_index_scalar_type
= make_unsigned_type (scalar_precision
);
4331 tree cr_index_vector_type
= build_vector_type
4332 (cr_index_scalar_type
, TYPE_VECTOR_SUBPARTS (vectype
));
4334 /* First we create a simple vector induction variable which starts
4335 with the values {1,2,3,...} (SERIES_VECT) and increments by the
4336 vector size (STEP). */
4338 /* Create a {1,2,3,...} vector. */
4339 tree series_vect
= build_index_vector (cr_index_vector_type
, 1, 1);
4341 /* Create a vector of the step value. */
4342 tree step
= build_int_cst (cr_index_scalar_type
, nunits_out
);
4343 tree vec_step
= build_vector_from_val (cr_index_vector_type
, step
);
4345 /* Create an induction variable. */
4346 gimple_stmt_iterator incr_gsi
;
4348 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
4349 create_iv (series_vect
, vec_step
, NULL_TREE
, loop
, &incr_gsi
,
4350 insert_after
, &indx_before_incr
, &indx_after_incr
);
4352 /* Next create a new phi node vector (NEW_PHI_TREE) which starts
4353 filled with zeros (VEC_ZERO). */
4355 /* Create a vector of 0s. */
4356 tree zero
= build_zero_cst (cr_index_scalar_type
);
4357 tree vec_zero
= build_vector_from_val (cr_index_vector_type
, zero
);
4359 /* Create a vector phi node. */
4360 tree new_phi_tree
= make_ssa_name (cr_index_vector_type
);
4361 new_phi
= create_phi_node (new_phi_tree
, loop
->header
);
4362 loop_vinfo
->add_stmt (new_phi
);
4363 add_phi_arg (as_a
<gphi
*> (new_phi
), vec_zero
,
4364 loop_preheader_edge (loop
), UNKNOWN_LOCATION
);
4366 /* Now take the condition from the loops original cond_expr
4367 (VEC_STMT) and produce a new cond_expr (INDEX_COND_EXPR) which for
4368 every match uses values from the induction variable
4369 (INDEX_BEFORE_INCR) otherwise uses values from the phi node
4371 Finally, we update the phi (NEW_PHI_TREE) to take the value of
4372 the new cond_expr (INDEX_COND_EXPR). */
4374 /* Duplicate the condition from vec_stmt. */
4375 tree ccompare
= unshare_expr (gimple_assign_rhs1 (vec_stmt
));
4377 /* Create a conditional, where the condition is taken from vec_stmt
4378 (CCOMPARE). The then and else values mirror the main VEC_COND_EXPR:
4379 the reduction phi corresponds to NEW_PHI_TREE and the new values
4380 correspond to INDEX_BEFORE_INCR. */
4381 gcc_assert (STMT_VINFO_REDUC_IDX (stmt_info
) >= 1);
4382 tree index_cond_expr
;
4383 if (STMT_VINFO_REDUC_IDX (stmt_info
) == 2)
4384 index_cond_expr
= build3 (VEC_COND_EXPR
, cr_index_vector_type
,
4385 ccompare
, indx_before_incr
, new_phi_tree
);
4387 index_cond_expr
= build3 (VEC_COND_EXPR
, cr_index_vector_type
,
4388 ccompare
, new_phi_tree
, indx_before_incr
);
4389 induction_index
= make_ssa_name (cr_index_vector_type
);
4390 gimple
*index_condition
= gimple_build_assign (induction_index
,
4392 gsi_insert_before (&incr_gsi
, index_condition
, GSI_SAME_STMT
);
4393 stmt_vec_info index_vec_info
= loop_vinfo
->add_stmt (index_condition
);
4394 STMT_VINFO_VECTYPE (index_vec_info
) = cr_index_vector_type
;
4396 /* Update the phi with the vec cond. */
4397 add_phi_arg (as_a
<gphi
*> (new_phi
), induction_index
,
4398 loop_latch_edge (loop
), UNKNOWN_LOCATION
);
4401 /* 2. Create epilog code.
4402 The reduction epilog code operates across the elements of the vector
4403 of partial results computed by the vectorized loop.
4404 The reduction epilog code consists of:
4406 step 1: compute the scalar result in a vector (v_out2)
4407 step 2: extract the scalar result (s_out3) from the vector (v_out2)
4408 step 3: adjust the scalar result (s_out3) if needed.
4410 Step 1 can be accomplished using one the following three schemes:
4411 (scheme 1) using reduc_fn, if available.
4412 (scheme 2) using whole-vector shifts, if available.
4413 (scheme 3) using a scalar loop. In this case steps 1+2 above are
4416 The overall epilog code looks like this:
4418 s_out0 = phi <s_loop> # original EXIT_PHI
4419 v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
4420 v_out2 = reduce <v_out1> # step 1
4421 s_out3 = extract_field <v_out2, 0> # step 2
4422 s_out4 = adjust_result <s_out3> # step 3
4424 (step 3 is optional, and steps 1 and 2 may be combined).
4425 Lastly, the uses of s_out0 are replaced by s_out4. */
4428 /* 2.1 Create new loop-exit-phis to preserve loop-closed form:
4429 v_out1 = phi <VECT_DEF>
4430 Store them in NEW_PHIS. */
4433 exit_bb
= single_exit (loop
)->dest
;
4434 prev_phi_info
= NULL
;
4435 new_phis
.create (slp_node
? vec_num
: ncopies
);
4436 for (unsigned i
= 0; i
< vec_num
; i
++)
4439 def
= gimple_get_lhs (SLP_TREE_VEC_STMTS (slp_node
)[i
]->stmt
);
4441 def
= gimple_get_lhs (STMT_VINFO_VEC_STMT (rdef_info
)->stmt
);
4442 for (j
= 0; j
< ncopies
; j
++)
4444 tree new_def
= copy_ssa_name (def
);
4445 phi
= create_phi_node (new_def
, exit_bb
);
4446 stmt_vec_info phi_info
= loop_vinfo
->add_stmt (phi
);
4448 new_phis
.quick_push (phi
);
4451 def
= vect_get_vec_def_for_stmt_copy (loop_vinfo
, def
);
4452 STMT_VINFO_RELATED_STMT (prev_phi_info
) = phi_info
;
4455 SET_PHI_ARG_DEF (phi
, single_exit (loop
)->dest_idx
, def
);
4456 prev_phi_info
= phi_info
;
4460 exit_gsi
= gsi_after_labels (exit_bb
);
4462 /* 2.2 Get the relevant tree-code to use in the epilog for schemes 2,3
4463 (i.e. when reduc_fn is not available) and in the final adjustment
4464 code (if needed). Also get the original scalar reduction variable as
4465 defined in the loop. In case STMT is a "pattern-stmt" (i.e. - it
4466 represents a reduction pattern), the tree-code and scalar-def are
4467 taken from the original stmt that the pattern-stmt (STMT) replaces.
4468 Otherwise (it is a regular reduction) - the tree-code and scalar-def
4469 are taken from STMT. */
4471 stmt_vec_info orig_stmt_info
= vect_orig_stmt (stmt_info
);
4472 if (orig_stmt_info
!= stmt_info
)
4474 /* Reduction pattern */
4475 gcc_assert (STMT_VINFO_IN_PATTERN_P (orig_stmt_info
));
4476 gcc_assert (STMT_VINFO_RELATED_STMT (orig_stmt_info
) == stmt_info
);
4479 scalar_dest
= gimple_assign_lhs (orig_stmt_info
->stmt
);
4480 scalar_type
= TREE_TYPE (scalar_dest
);
4481 scalar_results
.create (group_size
);
4482 new_scalar_dest
= vect_create_destination_var (scalar_dest
, NULL
);
4483 bitsize
= TYPE_SIZE (scalar_type
);
4485 /* SLP reduction without reduction chain, e.g.,
4489 b2 = operation (b1) */
4490 slp_reduc
= (slp_node
&& !REDUC_GROUP_FIRST_ELEMENT (stmt_info
));
4492 /* True if we should implement SLP_REDUC using native reduction operations
4493 instead of scalar operations. */
4494 direct_slp_reduc
= (reduc_fn
!= IFN_LAST
4496 && !TYPE_VECTOR_SUBPARTS (vectype
).is_constant ());
4498 /* In case of reduction chain, e.g.,
4501 a3 = operation (a2),
4503 we may end up with more than one vector result. Here we reduce them to
4505 if (REDUC_GROUP_FIRST_ELEMENT (stmt_info
) || direct_slp_reduc
)
4507 tree first_vect
= PHI_RESULT (new_phis
[0]);
4508 gassign
*new_vec_stmt
= NULL
;
4509 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4510 for (k
= 1; k
< new_phis
.length (); k
++)
4512 gimple
*next_phi
= new_phis
[k
];
4513 tree second_vect
= PHI_RESULT (next_phi
);
4514 tree tem
= make_ssa_name (vec_dest
, new_vec_stmt
);
4515 new_vec_stmt
= gimple_build_assign (tem
, code
,
4516 first_vect
, second_vect
);
4517 gsi_insert_before (&exit_gsi
, new_vec_stmt
, GSI_SAME_STMT
);
4521 new_phi_result
= first_vect
;
4524 new_phis
.truncate (0);
4525 new_phis
.safe_push (new_vec_stmt
);
4528 /* Likewise if we couldn't use a single defuse cycle. */
4529 else if (ncopies
> 1)
4531 gcc_assert (new_phis
.length () == 1);
4532 tree first_vect
= PHI_RESULT (new_phis
[0]);
4533 gassign
*new_vec_stmt
= NULL
;
4534 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4535 stmt_vec_info next_phi_info
= loop_vinfo
->lookup_stmt (new_phis
[0]);
4536 for (int k
= 1; k
< ncopies
; ++k
)
4538 next_phi_info
= STMT_VINFO_RELATED_STMT (next_phi_info
);
4539 tree second_vect
= PHI_RESULT (next_phi_info
->stmt
);
4540 tree tem
= make_ssa_name (vec_dest
, new_vec_stmt
);
4541 new_vec_stmt
= gimple_build_assign (tem
, code
,
4542 first_vect
, second_vect
);
4543 gsi_insert_before (&exit_gsi
, new_vec_stmt
, GSI_SAME_STMT
);
4546 new_phi_result
= first_vect
;
4547 new_phis
.truncate (0);
4548 new_phis
.safe_push (new_vec_stmt
);
4551 new_phi_result
= PHI_RESULT (new_phis
[0]);
4553 if (STMT_VINFO_REDUC_TYPE (reduc_info
) == COND_REDUCTION
4554 && reduc_fn
!= IFN_LAST
)
4556 /* For condition reductions, we have a vector (NEW_PHI_RESULT) containing
4557 various data values where the condition matched and another vector
4558 (INDUCTION_INDEX) containing all the indexes of those matches. We
4559 need to extract the last matching index (which will be the index with
4560 highest value) and use this to index into the data vector.
4561 For the case where there were no matches, the data vector will contain
4562 all default values and the index vector will be all zeros. */
4564 /* Get various versions of the type of the vector of indexes. */
4565 tree index_vec_type
= TREE_TYPE (induction_index
);
4566 gcc_checking_assert (TYPE_UNSIGNED (index_vec_type
));
4567 tree index_scalar_type
= TREE_TYPE (index_vec_type
);
4568 tree index_vec_cmp_type
= build_same_sized_truth_vector_type
4571 /* Get an unsigned integer version of the type of the data vector. */
4572 int scalar_precision
4573 = GET_MODE_PRECISION (SCALAR_TYPE_MODE (scalar_type
));
4574 tree scalar_type_unsigned
= make_unsigned_type (scalar_precision
);
4575 tree vectype_unsigned
= build_vector_type
4576 (scalar_type_unsigned
, TYPE_VECTOR_SUBPARTS (vectype
));
4578 /* First we need to create a vector (ZERO_VEC) of zeros and another
4579 vector (MAX_INDEX_VEC) filled with the last matching index, which we
4580 can create using a MAX reduction and then expanding.
4581 In the case where the loop never made any matches, the max index will
4584 /* Vector of {0, 0, 0,...}. */
4585 tree zero_vec
= make_ssa_name (vectype
);
4586 tree zero_vec_rhs
= build_zero_cst (vectype
);
4587 gimple
*zero_vec_stmt
= gimple_build_assign (zero_vec
, zero_vec_rhs
);
4588 gsi_insert_before (&exit_gsi
, zero_vec_stmt
, GSI_SAME_STMT
);
4590 /* Find maximum value from the vector of found indexes. */
4591 tree max_index
= make_ssa_name (index_scalar_type
);
4592 gcall
*max_index_stmt
= gimple_build_call_internal (IFN_REDUC_MAX
,
4593 1, induction_index
);
4594 gimple_call_set_lhs (max_index_stmt
, max_index
);
4595 gsi_insert_before (&exit_gsi
, max_index_stmt
, GSI_SAME_STMT
);
4597 /* Vector of {max_index, max_index, max_index,...}. */
4598 tree max_index_vec
= make_ssa_name (index_vec_type
);
4599 tree max_index_vec_rhs
= build_vector_from_val (index_vec_type
,
4601 gimple
*max_index_vec_stmt
= gimple_build_assign (max_index_vec
,
4603 gsi_insert_before (&exit_gsi
, max_index_vec_stmt
, GSI_SAME_STMT
);
4605 /* Next we compare the new vector (MAX_INDEX_VEC) full of max indexes
4606 with the vector (INDUCTION_INDEX) of found indexes, choosing values
4607 from the data vector (NEW_PHI_RESULT) for matches, 0 (ZERO_VEC)
4608 otherwise. Only one value should match, resulting in a vector
4609 (VEC_COND) with one data value and the rest zeros.
4610 In the case where the loop never made any matches, every index will
4611 match, resulting in a vector with all data values (which will all be
4612 the default value). */
4614 /* Compare the max index vector to the vector of found indexes to find
4615 the position of the max value. */
4616 tree vec_compare
= make_ssa_name (index_vec_cmp_type
);
4617 gimple
*vec_compare_stmt
= gimple_build_assign (vec_compare
, EQ_EXPR
,
4620 gsi_insert_before (&exit_gsi
, vec_compare_stmt
, GSI_SAME_STMT
);
4622 /* Use the compare to choose either values from the data vector or
4624 tree vec_cond
= make_ssa_name (vectype
);
4625 gimple
*vec_cond_stmt
= gimple_build_assign (vec_cond
, VEC_COND_EXPR
,
4626 vec_compare
, new_phi_result
,
4628 gsi_insert_before (&exit_gsi
, vec_cond_stmt
, GSI_SAME_STMT
);
4630 /* Finally we need to extract the data value from the vector (VEC_COND)
4631 into a scalar (MATCHED_DATA_REDUC). Logically we want to do a OR
4632 reduction, but because this doesn't exist, we can use a MAX reduction
4633 instead. The data value might be signed or a float so we need to cast
4635 In the case where the loop never made any matches, the data values are
4636 all identical, and so will reduce down correctly. */
4638 /* Make the matched data values unsigned. */
4639 tree vec_cond_cast
= make_ssa_name (vectype_unsigned
);
4640 tree vec_cond_cast_rhs
= build1 (VIEW_CONVERT_EXPR
, vectype_unsigned
,
4642 gimple
*vec_cond_cast_stmt
= gimple_build_assign (vec_cond_cast
,
4645 gsi_insert_before (&exit_gsi
, vec_cond_cast_stmt
, GSI_SAME_STMT
);
4647 /* Reduce down to a scalar value. */
4648 tree data_reduc
= make_ssa_name (scalar_type_unsigned
);
4649 gcall
*data_reduc_stmt
= gimple_build_call_internal (IFN_REDUC_MAX
,
4651 gimple_call_set_lhs (data_reduc_stmt
, data_reduc
);
4652 gsi_insert_before (&exit_gsi
, data_reduc_stmt
, GSI_SAME_STMT
);
4654 /* Convert the reduced value back to the result type and set as the
4656 gimple_seq stmts
= NULL
;
4657 new_temp
= gimple_build (&stmts
, VIEW_CONVERT_EXPR
, scalar_type
,
4659 gsi_insert_seq_before (&exit_gsi
, stmts
, GSI_SAME_STMT
);
4660 scalar_results
.safe_push (new_temp
);
4662 else if (STMT_VINFO_REDUC_TYPE (reduc_info
) == COND_REDUCTION
4663 && reduc_fn
== IFN_LAST
)
4665 /* Condition reduction without supported IFN_REDUC_MAX. Generate
4667 idx_val = induction_index[0];
4668 val = data_reduc[0];
4669 for (idx = 0, val = init, i = 0; i < nelts; ++i)
4670 if (induction_index[i] > idx_val)
4671 val = data_reduc[i], idx_val = induction_index[i];
4674 tree data_eltype
= TREE_TYPE (TREE_TYPE (new_phi_result
));
4675 tree idx_eltype
= TREE_TYPE (TREE_TYPE (induction_index
));
4676 unsigned HOST_WIDE_INT el_size
= tree_to_uhwi (TYPE_SIZE (idx_eltype
));
4677 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (induction_index
));
4678 /* Enforced by vectorizable_reduction, which ensures we have target
4679 support before allowing a conditional reduction on variable-length
4681 unsigned HOST_WIDE_INT v_size
= el_size
* nunits
.to_constant ();
4682 tree idx_val
= NULL_TREE
, val
= NULL_TREE
;
4683 for (unsigned HOST_WIDE_INT off
= 0; off
< v_size
; off
+= el_size
)
4685 tree old_idx_val
= idx_val
;
4687 idx_val
= make_ssa_name (idx_eltype
);
4688 epilog_stmt
= gimple_build_assign (idx_val
, BIT_FIELD_REF
,
4689 build3 (BIT_FIELD_REF
, idx_eltype
,
4691 bitsize_int (el_size
),
4692 bitsize_int (off
)));
4693 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4694 val
= make_ssa_name (data_eltype
);
4695 epilog_stmt
= gimple_build_assign (val
, BIT_FIELD_REF
,
4696 build3 (BIT_FIELD_REF
,
4699 bitsize_int (el_size
),
4700 bitsize_int (off
)));
4701 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4704 tree new_idx_val
= idx_val
;
4705 if (off
!= v_size
- el_size
)
4707 new_idx_val
= make_ssa_name (idx_eltype
);
4708 epilog_stmt
= gimple_build_assign (new_idx_val
,
4711 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4713 tree new_val
= make_ssa_name (data_eltype
);
4714 epilog_stmt
= gimple_build_assign (new_val
,
4721 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4722 idx_val
= new_idx_val
;
4726 /* Convert the reduced value back to the result type and set as the
4728 gimple_seq stmts
= NULL
;
4729 val
= gimple_convert (&stmts
, scalar_type
, val
);
4730 gsi_insert_seq_before (&exit_gsi
, stmts
, GSI_SAME_STMT
);
4731 scalar_results
.safe_push (val
);
4734 /* 2.3 Create the reduction code, using one of the three schemes described
4735 above. In SLP we simply need to extract all the elements from the
4736 vector (without reducing them), so we use scalar shifts. */
4737 else if (reduc_fn
!= IFN_LAST
&& !slp_reduc
)
4743 v_out2 = reduc_expr <v_out1> */
4745 if (dump_enabled_p ())
4746 dump_printf_loc (MSG_NOTE
, vect_location
,
4747 "Reduce using direct vector reduction.\n");
4749 vec_elem_type
= TREE_TYPE (TREE_TYPE (new_phi_result
));
4750 if (!useless_type_conversion_p (scalar_type
, vec_elem_type
))
4753 = vect_create_destination_var (scalar_dest
, vec_elem_type
);
4754 epilog_stmt
= gimple_build_call_internal (reduc_fn
, 1,
4756 gimple_set_lhs (epilog_stmt
, tmp_dest
);
4757 new_temp
= make_ssa_name (tmp_dest
, epilog_stmt
);
4758 gimple_set_lhs (epilog_stmt
, new_temp
);
4759 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4761 epilog_stmt
= gimple_build_assign (new_scalar_dest
, NOP_EXPR
,
4766 epilog_stmt
= gimple_build_call_internal (reduc_fn
, 1,
4768 gimple_set_lhs (epilog_stmt
, new_scalar_dest
);
4771 new_temp
= make_ssa_name (new_scalar_dest
, epilog_stmt
);
4772 gimple_set_lhs (epilog_stmt
, new_temp
);
4773 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4775 if ((STMT_VINFO_REDUC_TYPE (reduc_info
) == INTEGER_INDUC_COND_REDUCTION
)
4778 /* Earlier we set the initial value to be a vector if induc_val
4779 values. Check the result and if it is induc_val then replace
4780 with the original initial value, unless induc_val is
4781 the same as initial_def already. */
4782 tree zcompare
= build2 (EQ_EXPR
, boolean_type_node
, new_temp
,
4785 tmp
= make_ssa_name (new_scalar_dest
);
4786 epilog_stmt
= gimple_build_assign (tmp
, COND_EXPR
, zcompare
,
4787 initial_def
, new_temp
);
4788 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4792 scalar_results
.safe_push (new_temp
);
4794 else if (direct_slp_reduc
)
4796 /* Here we create one vector for each of the REDUC_GROUP_SIZE results,
4797 with the elements for other SLP statements replaced with the
4798 neutral value. We can then do a normal reduction on each vector. */
4800 /* Enforced by vectorizable_reduction. */
4801 gcc_assert (new_phis
.length () == 1);
4802 gcc_assert (pow2p_hwi (group_size
));
4804 slp_tree orig_phis_slp_node
= slp_node_instance
->reduc_phis
;
4805 vec
<stmt_vec_info
> orig_phis
4806 = SLP_TREE_SCALAR_STMTS (orig_phis_slp_node
);
4807 gimple_seq seq
= NULL
;
4809 /* Build a vector {0, 1, 2, ...}, with the same number of elements
4810 and the same element size as VECTYPE. */
4811 tree index
= build_index_vector (vectype
, 0, 1);
4812 tree index_type
= TREE_TYPE (index
);
4813 tree index_elt_type
= TREE_TYPE (index_type
);
4814 tree mask_type
= build_same_sized_truth_vector_type (index_type
);
4816 /* Create a vector that, for each element, identifies which of
4817 the REDUC_GROUP_SIZE results should use it. */
4818 tree index_mask
= build_int_cst (index_elt_type
, group_size
- 1);
4819 index
= gimple_build (&seq
, BIT_AND_EXPR
, index_type
, index
,
4820 build_vector_from_val (index_type
, index_mask
));
4822 /* Get a neutral vector value. This is simply a splat of the neutral
4823 scalar value if we have one, otherwise the initial scalar value
4824 is itself a neutral value. */
4825 tree vector_identity
= NULL_TREE
;
4827 vector_identity
= gimple_build_vector_from_val (&seq
, vectype
,
4829 for (unsigned int i
= 0; i
< group_size
; ++i
)
4831 /* If there's no univeral neutral value, we can use the
4832 initial scalar value from the original PHI. This is used
4833 for MIN and MAX reduction, for example. */
4837 = PHI_ARG_DEF_FROM_EDGE (orig_phis
[i
]->stmt
,
4838 loop_preheader_edge (loop
));
4839 vector_identity
= gimple_build_vector_from_val (&seq
, vectype
,
4843 /* Calculate the equivalent of:
4845 sel[j] = (index[j] == i);
4847 which selects the elements of NEW_PHI_RESULT that should
4848 be included in the result. */
4849 tree compare_val
= build_int_cst (index_elt_type
, i
);
4850 compare_val
= build_vector_from_val (index_type
, compare_val
);
4851 tree sel
= gimple_build (&seq
, EQ_EXPR
, mask_type
,
4852 index
, compare_val
);
4854 /* Calculate the equivalent of:
4856 vec = seq ? new_phi_result : vector_identity;
4858 VEC is now suitable for a full vector reduction. */
4859 tree vec
= gimple_build (&seq
, VEC_COND_EXPR
, vectype
,
4860 sel
, new_phi_result
, vector_identity
);
4862 /* Do the reduction and convert it to the appropriate type. */
4863 tree scalar
= gimple_build (&seq
, as_combined_fn (reduc_fn
),
4864 TREE_TYPE (vectype
), vec
);
4865 scalar
= gimple_convert (&seq
, scalar_type
, scalar
);
4866 scalar_results
.safe_push (scalar
);
4868 gsi_insert_seq_before (&exit_gsi
, seq
, GSI_SAME_STMT
);
4872 bool reduce_with_shift
;
4875 /* See if the target wants to do the final (shift) reduction
4876 in a vector mode of smaller size and first reduce upper/lower
4877 halves against each other. */
4878 enum machine_mode mode1
= mode
;
4879 unsigned sz
= tree_to_uhwi (TYPE_SIZE_UNIT (vectype
));
4882 && (mode1
= targetm
.vectorize
.split_reduction (mode
)) != mode
)
4883 sz1
= GET_MODE_SIZE (mode1
).to_constant ();
4885 tree vectype1
= get_vectype_for_scalar_type_and_size (scalar_type
, sz1
);
4886 reduce_with_shift
= have_whole_vector_shift (mode1
);
4887 if (!VECTOR_MODE_P (mode1
))
4888 reduce_with_shift
= false;
4891 optab optab
= optab_for_tree_code (code
, vectype1
, optab_default
);
4892 if (optab_handler (optab
, mode1
) == CODE_FOR_nothing
)
4893 reduce_with_shift
= false;
4896 /* First reduce the vector to the desired vector size we should
4897 do shift reduction on by combining upper and lower halves. */
4898 new_temp
= new_phi_result
;
4901 gcc_assert (!slp_reduc
);
4903 vectype1
= get_vectype_for_scalar_type_and_size (scalar_type
, sz
);
4905 /* The target has to make sure we support lowpart/highpart
4906 extraction, either via direct vector extract or through
4907 an integer mode punning. */
4909 if (convert_optab_handler (vec_extract_optab
,
4910 TYPE_MODE (TREE_TYPE (new_temp
)),
4911 TYPE_MODE (vectype1
))
4912 != CODE_FOR_nothing
)
4914 /* Extract sub-vectors directly once vec_extract becomes
4915 a conversion optab. */
4916 dst1
= make_ssa_name (vectype1
);
4918 = gimple_build_assign (dst1
, BIT_FIELD_REF
,
4919 build3 (BIT_FIELD_REF
, vectype1
,
4920 new_temp
, TYPE_SIZE (vectype1
),
4922 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4923 dst2
= make_ssa_name (vectype1
);
4925 = gimple_build_assign (dst2
, BIT_FIELD_REF
,
4926 build3 (BIT_FIELD_REF
, vectype1
,
4927 new_temp
, TYPE_SIZE (vectype1
),
4928 bitsize_int (sz
* BITS_PER_UNIT
)));
4929 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4933 /* Extract via punning to appropriately sized integer mode
4935 tree eltype
= build_nonstandard_integer_type (sz
* BITS_PER_UNIT
,
4937 tree etype
= build_vector_type (eltype
, 2);
4938 gcc_assert (convert_optab_handler (vec_extract_optab
,
4941 != CODE_FOR_nothing
);
4942 tree tem
= make_ssa_name (etype
);
4943 epilog_stmt
= gimple_build_assign (tem
, VIEW_CONVERT_EXPR
,
4944 build1 (VIEW_CONVERT_EXPR
,
4946 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4948 tem
= make_ssa_name (eltype
);
4950 = gimple_build_assign (tem
, BIT_FIELD_REF
,
4951 build3 (BIT_FIELD_REF
, eltype
,
4952 new_temp
, TYPE_SIZE (eltype
),
4954 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4955 dst1
= make_ssa_name (vectype1
);
4956 epilog_stmt
= gimple_build_assign (dst1
, VIEW_CONVERT_EXPR
,
4957 build1 (VIEW_CONVERT_EXPR
,
4959 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4960 tem
= make_ssa_name (eltype
);
4962 = gimple_build_assign (tem
, BIT_FIELD_REF
,
4963 build3 (BIT_FIELD_REF
, eltype
,
4964 new_temp
, TYPE_SIZE (eltype
),
4965 bitsize_int (sz
* BITS_PER_UNIT
)));
4966 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4967 dst2
= make_ssa_name (vectype1
);
4968 epilog_stmt
= gimple_build_assign (dst2
, VIEW_CONVERT_EXPR
,
4969 build1 (VIEW_CONVERT_EXPR
,
4971 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4974 new_temp
= make_ssa_name (vectype1
);
4975 epilog_stmt
= gimple_build_assign (new_temp
, code
, dst1
, dst2
);
4976 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
4979 if (reduce_with_shift
&& !slp_reduc
)
4981 int element_bitsize
= tree_to_uhwi (bitsize
);
4982 /* Enforced by vectorizable_reduction, which disallows SLP reductions
4983 for variable-length vectors and also requires direct target support
4984 for loop reductions. */
4985 int vec_size_in_bits
= tree_to_uhwi (TYPE_SIZE (vectype1
));
4986 int nelements
= vec_size_in_bits
/ element_bitsize
;
4987 vec_perm_builder sel
;
4988 vec_perm_indices indices
;
4992 tree zero_vec
= build_zero_cst (vectype1
);
4994 for (offset = nelements/2; offset >= 1; offset/=2)
4996 Create: va' = vec_shift <va, offset>
4997 Create: va = vop <va, va'>
5002 if (dump_enabled_p ())
5003 dump_printf_loc (MSG_NOTE
, vect_location
,
5004 "Reduce using vector shifts\n");
5006 vec_dest
= vect_create_destination_var (scalar_dest
, vectype1
);
5007 for (elt_offset
= nelements
/ 2;
5011 calc_vec_perm_mask_for_shift (elt_offset
, nelements
, &sel
);
5012 indices
.new_vector (sel
, 2, nelements
);
5013 tree mask
= vect_gen_perm_mask_any (vectype1
, indices
);
5014 epilog_stmt
= gimple_build_assign (vec_dest
, VEC_PERM_EXPR
,
5015 new_temp
, zero_vec
, mask
);
5016 new_name
= make_ssa_name (vec_dest
, epilog_stmt
);
5017 gimple_assign_set_lhs (epilog_stmt
, new_name
);
5018 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
5020 epilog_stmt
= gimple_build_assign (vec_dest
, code
, new_name
,
5022 new_temp
= make_ssa_name (vec_dest
, epilog_stmt
);
5023 gimple_assign_set_lhs (epilog_stmt
, new_temp
);
5024 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
5027 /* 2.4 Extract the final scalar result. Create:
5028 s_out3 = extract_field <v_out2, bitpos> */
5030 if (dump_enabled_p ())
5031 dump_printf_loc (MSG_NOTE
, vect_location
,
5032 "extract scalar result\n");
5034 rhs
= build3 (BIT_FIELD_REF
, scalar_type
, new_temp
,
5035 bitsize
, bitsize_zero_node
);
5036 epilog_stmt
= gimple_build_assign (new_scalar_dest
, rhs
);
5037 new_temp
= make_ssa_name (new_scalar_dest
, epilog_stmt
);
5038 gimple_assign_set_lhs (epilog_stmt
, new_temp
);
5039 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
5040 scalar_results
.safe_push (new_temp
);
5045 s = extract_field <v_out2, 0>
5046 for (offset = element_size;
5047 offset < vector_size;
5048 offset += element_size;)
5050 Create: s' = extract_field <v_out2, offset>
5051 Create: s = op <s, s'> // For non SLP cases
5054 if (dump_enabled_p ())
5055 dump_printf_loc (MSG_NOTE
, vect_location
,
5056 "Reduce using scalar code.\n");
5058 int vec_size_in_bits
= tree_to_uhwi (TYPE_SIZE (vectype1
));
5059 int element_bitsize
= tree_to_uhwi (bitsize
);
5060 FOR_EACH_VEC_ELT (new_phis
, i
, new_phi
)
5063 if (gimple_code (new_phi
) == GIMPLE_PHI
)
5064 vec_temp
= PHI_RESULT (new_phi
);
5066 vec_temp
= gimple_assign_lhs (new_phi
);
5067 tree rhs
= build3 (BIT_FIELD_REF
, scalar_type
, vec_temp
, bitsize
,
5069 epilog_stmt
= gimple_build_assign (new_scalar_dest
, rhs
);
5070 new_temp
= make_ssa_name (new_scalar_dest
, epilog_stmt
);
5071 gimple_assign_set_lhs (epilog_stmt
, new_temp
);
5072 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
5074 /* In SLP we don't need to apply reduction operation, so we just
5075 collect s' values in SCALAR_RESULTS. */
5077 scalar_results
.safe_push (new_temp
);
5079 for (bit_offset
= element_bitsize
;
5080 bit_offset
< vec_size_in_bits
;
5081 bit_offset
+= element_bitsize
)
5083 tree bitpos
= bitsize_int (bit_offset
);
5084 tree rhs
= build3 (BIT_FIELD_REF
, scalar_type
, vec_temp
,
5087 epilog_stmt
= gimple_build_assign (new_scalar_dest
, rhs
);
5088 new_name
= make_ssa_name (new_scalar_dest
, epilog_stmt
);
5089 gimple_assign_set_lhs (epilog_stmt
, new_name
);
5090 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
5094 /* In SLP we don't need to apply reduction operation, so
5095 we just collect s' values in SCALAR_RESULTS. */
5096 new_temp
= new_name
;
5097 scalar_results
.safe_push (new_name
);
5101 epilog_stmt
= gimple_build_assign (new_scalar_dest
, code
,
5102 new_name
, new_temp
);
5103 new_temp
= make_ssa_name (new_scalar_dest
, epilog_stmt
);
5104 gimple_assign_set_lhs (epilog_stmt
, new_temp
);
5105 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
5110 /* The only case where we need to reduce scalar results in SLP, is
5111 unrolling. If the size of SCALAR_RESULTS is greater than
5112 REDUC_GROUP_SIZE, we reduce them combining elements modulo
5113 REDUC_GROUP_SIZE. */
5116 tree res
, first_res
, new_res
;
5119 /* Reduce multiple scalar results in case of SLP unrolling. */
5120 for (j
= group_size
; scalar_results
.iterate (j
, &res
);
5123 first_res
= scalar_results
[j
% group_size
];
5124 new_stmt
= gimple_build_assign (new_scalar_dest
, code
,
5126 new_res
= make_ssa_name (new_scalar_dest
, new_stmt
);
5127 gimple_assign_set_lhs (new_stmt
, new_res
);
5128 gsi_insert_before (&exit_gsi
, new_stmt
, GSI_SAME_STMT
);
5129 scalar_results
[j
% group_size
] = new_res
;
5133 /* Not SLP - we have one scalar to keep in SCALAR_RESULTS. */
5134 scalar_results
.safe_push (new_temp
);
5137 if ((STMT_VINFO_REDUC_TYPE (reduc_info
) == INTEGER_INDUC_COND_REDUCTION
)
5140 /* Earlier we set the initial value to be a vector if induc_val
5141 values. Check the result and if it is induc_val then replace
5142 with the original initial value, unless induc_val is
5143 the same as initial_def already. */
5144 tree zcompare
= build2 (EQ_EXPR
, boolean_type_node
, new_temp
,
5147 tree tmp
= make_ssa_name (new_scalar_dest
);
5148 epilog_stmt
= gimple_build_assign (tmp
, COND_EXPR
, zcompare
,
5149 initial_def
, new_temp
);
5150 gsi_insert_before (&exit_gsi
, epilog_stmt
, GSI_SAME_STMT
);
5151 scalar_results
[0] = tmp
;
5155 /* 2.5 Adjust the final result by the initial value of the reduction
5156 variable. (When such adjustment is not needed, then
5157 'adjustment_def' is zero). For example, if code is PLUS we create:
5158 new_temp = loop_exit_def + adjustment_def */
5162 gcc_assert (!slp_reduc
);
5163 gimple_seq stmts
= NULL
;
5164 if (nested_in_vect_loop
)
5166 new_phi
= new_phis
[0];
5167 gcc_assert (VECTOR_TYPE_P (TREE_TYPE (adjustment_def
)));
5168 adjustment_def
= gimple_convert (&stmts
, vectype
, adjustment_def
);
5169 new_temp
= gimple_build (&stmts
, code
, vectype
,
5170 PHI_RESULT (new_phi
), adjustment_def
);
5174 new_temp
= scalar_results
[0];
5175 gcc_assert (TREE_CODE (TREE_TYPE (adjustment_def
)) != VECTOR_TYPE
);
5176 adjustment_def
= gimple_convert (&stmts
, scalar_type
, adjustment_def
);
5177 new_temp
= gimple_build (&stmts
, code
, scalar_type
,
5178 new_temp
, adjustment_def
);
5181 epilog_stmt
= gimple_seq_last_stmt (stmts
);
5182 gsi_insert_seq_before (&exit_gsi
, stmts
, GSI_SAME_STMT
);
5183 if (nested_in_vect_loop
)
5185 stmt_vec_info epilog_stmt_info
= loop_vinfo
->add_stmt (epilog_stmt
);
5186 STMT_VINFO_RELATED_STMT (epilog_stmt_info
)
5187 = STMT_VINFO_RELATED_STMT (loop_vinfo
->lookup_stmt (new_phi
));
5190 scalar_results
.quick_push (new_temp
);
5192 scalar_results
[0] = new_temp
;
5195 scalar_results
[0] = new_temp
;
5197 new_phis
[0] = epilog_stmt
;
5203 /* 2.6 Handle the loop-exit phis. Replace the uses of scalar loop-exit
5204 phis with new adjusted scalar results, i.e., replace use <s_out0>
5209 s_out0 = phi <s_loop> # (scalar) EXIT_PHI
5210 v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
5211 v_out2 = reduce <v_out1>
5212 s_out3 = extract_field <v_out2, 0>
5213 s_out4 = adjust_result <s_out3>
5220 s_out0 = phi <s_loop> # (scalar) EXIT_PHI
5221 v_out1 = phi <VECT_DEF> # NEW_EXIT_PHI
5222 v_out2 = reduce <v_out1>
5223 s_out3 = extract_field <v_out2, 0>
5224 s_out4 = adjust_result <s_out3>
5229 /* In SLP reduction chain we reduce vector results into one vector if
5230 necessary, hence we set here REDUC_GROUP_SIZE to 1. SCALAR_DEST is the
5231 LHS of the last stmt in the reduction chain, since we are looking for
5232 the loop exit phi node. */
5233 if (REDUC_GROUP_FIRST_ELEMENT (stmt_info
))
5235 stmt_vec_info dest_stmt_info
5236 = vect_orig_stmt (SLP_TREE_SCALAR_STMTS (slp_node
)[group_size
- 1]);
5237 scalar_dest
= gimple_assign_lhs (dest_stmt_info
->stmt
);
5241 /* In SLP we may have several statements in NEW_PHIS and REDUCTION_PHIS (in
5242 case that REDUC_GROUP_SIZE is greater than vectorization factor).
5243 Therefore, we need to match SCALAR_RESULTS with corresponding statements.
5244 The first (REDUC_GROUP_SIZE / number of new vector stmts) scalar results
5245 correspond to the first vector stmt, etc.
5246 (RATIO is equal to (REDUC_GROUP_SIZE / number of new vector stmts)). */
5247 if (group_size
> new_phis
.length ())
5248 gcc_assert (!(group_size
% new_phis
.length ()));
5250 for (k
= 0; k
< group_size
; k
++)
5254 stmt_vec_info scalar_stmt_info
= SLP_TREE_SCALAR_STMTS (slp_node
)[k
];
5256 orig_stmt_info
= STMT_VINFO_RELATED_STMT (scalar_stmt_info
);
5257 /* SLP statements can't participate in patterns. */
5258 gcc_assert (!orig_stmt_info
);
5259 scalar_dest
= gimple_assign_lhs (scalar_stmt_info
->stmt
);
5262 if (nested_in_vect_loop
)
5271 /* Find the loop-closed-use at the loop exit of the original scalar
5272 result. (The reduction result is expected to have two immediate uses,
5273 one at the latch block, and one at the loop exit). For double
5274 reductions we are looking for exit phis of the outer loop. */
5275 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, scalar_dest
)
5277 if (!flow_bb_inside_loop_p (loop
, gimple_bb (USE_STMT (use_p
))))
5279 if (!is_gimple_debug (USE_STMT (use_p
)))
5280 phis
.safe_push (USE_STMT (use_p
));
5284 if (double_reduc
&& gimple_code (USE_STMT (use_p
)) == GIMPLE_PHI
)
5286 tree phi_res
= PHI_RESULT (USE_STMT (use_p
));
5288 FOR_EACH_IMM_USE_FAST (phi_use_p
, phi_imm_iter
, phi_res
)
5290 if (!flow_bb_inside_loop_p (loop
,
5291 gimple_bb (USE_STMT (phi_use_p
)))
5292 && !is_gimple_debug (USE_STMT (phi_use_p
)))
5293 phis
.safe_push (USE_STMT (phi_use_p
));
5299 FOR_EACH_VEC_ELT (phis
, i
, exit_phi
)
5301 /* Replace the uses: */
5302 orig_name
= PHI_RESULT (exit_phi
);
5303 scalar_result
= scalar_results
[k
];
5304 FOR_EACH_IMM_USE_STMT (use_stmt
, imm_iter
, orig_name
)
5305 FOR_EACH_IMM_USE_ON_STMT (use_p
, imm_iter
)
5306 SET_USE (use_p
, scalar_result
);
5313 /* Return a vector of type VECTYPE that is equal to the vector select
5314 operation "MASK ? VEC : IDENTITY". Insert the select statements
5318 merge_with_identity (gimple_stmt_iterator
*gsi
, tree mask
, tree vectype
,
5319 tree vec
, tree identity
)
5321 tree cond
= make_temp_ssa_name (vectype
, NULL
, "cond");
5322 gimple
*new_stmt
= gimple_build_assign (cond
, VEC_COND_EXPR
,
5323 mask
, vec
, identity
);
5324 gsi_insert_before (gsi
, new_stmt
, GSI_SAME_STMT
);
5328 /* Successively apply CODE to each element of VECTOR_RHS, in left-to-right
5329 order, starting with LHS. Insert the extraction statements before GSI and
5330 associate the new scalar SSA names with variable SCALAR_DEST.
5331 Return the SSA name for the result. */
5334 vect_expand_fold_left (gimple_stmt_iterator
*gsi
, tree scalar_dest
,
5335 tree_code code
, tree lhs
, tree vector_rhs
)
5337 tree vectype
= TREE_TYPE (vector_rhs
);
5338 tree scalar_type
= TREE_TYPE (vectype
);
5339 tree bitsize
= TYPE_SIZE (scalar_type
);
5340 unsigned HOST_WIDE_INT vec_size_in_bits
= tree_to_uhwi (TYPE_SIZE (vectype
));
5341 unsigned HOST_WIDE_INT element_bitsize
= tree_to_uhwi (bitsize
);
5343 for (unsigned HOST_WIDE_INT bit_offset
= 0;
5344 bit_offset
< vec_size_in_bits
;
5345 bit_offset
+= element_bitsize
)
5347 tree bitpos
= bitsize_int (bit_offset
);
5348 tree rhs
= build3 (BIT_FIELD_REF
, scalar_type
, vector_rhs
,
5351 gassign
*stmt
= gimple_build_assign (scalar_dest
, rhs
);
5352 rhs
= make_ssa_name (scalar_dest
, stmt
);
5353 gimple_assign_set_lhs (stmt
, rhs
);
5354 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5356 stmt
= gimple_build_assign (scalar_dest
, code
, lhs
, rhs
);
5357 tree new_name
= make_ssa_name (scalar_dest
, stmt
);
5358 gimple_assign_set_lhs (stmt
, new_name
);
5359 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5365 /* Get a masked internal function equivalent to REDUC_FN. VECTYPE_IN is the
5366 type of the vector input. */
5369 get_masked_reduction_fn (internal_fn reduc_fn
, tree vectype_in
)
5371 internal_fn mask_reduc_fn
;
5375 case IFN_FOLD_LEFT_PLUS
:
5376 mask_reduc_fn
= IFN_MASK_FOLD_LEFT_PLUS
;
5383 if (direct_internal_fn_supported_p (mask_reduc_fn
, vectype_in
,
5384 OPTIMIZE_FOR_SPEED
))
5385 return mask_reduc_fn
;
5389 /* Perform an in-order reduction (FOLD_LEFT_REDUCTION). STMT_INFO is the
5390 statement that sets the live-out value. REDUC_DEF_STMT is the phi
5391 statement. CODE is the operation performed by STMT_INFO and OPS are
5392 its scalar operands. REDUC_INDEX is the index of the operand in
5393 OPS that is set by REDUC_DEF_STMT. REDUC_FN is the function that
5394 implements in-order reduction, or IFN_LAST if we should open-code it.
5395 VECTYPE_IN is the type of the vector input. MASKS specifies the masks
5396 that should be used to control the operation in a fully-masked loop. */
5399 vectorize_fold_left_reduction (stmt_vec_info stmt_info
,
5400 gimple_stmt_iterator
*gsi
,
5401 stmt_vec_info
*vec_stmt
, slp_tree slp_node
,
5402 gimple
*reduc_def_stmt
,
5403 tree_code code
, internal_fn reduc_fn
,
5404 tree ops
[3], tree vectype_in
,
5405 int reduc_index
, vec_loop_masks
*masks
)
5407 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5408 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5409 tree vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
5410 stmt_vec_info new_stmt_info
= NULL
;
5411 internal_fn mask_reduc_fn
= get_masked_reduction_fn (reduc_fn
, vectype_in
);
5417 ncopies
= vect_get_num_copies (loop_vinfo
, vectype_in
);
5419 gcc_assert (!nested_in_vect_loop_p (loop
, stmt_info
));
5420 gcc_assert (ncopies
== 1);
5421 gcc_assert (TREE_CODE_LENGTH (code
) == binary_op
);
5424 gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype_out
),
5425 TYPE_VECTOR_SUBPARTS (vectype_in
)));
5427 tree op0
= ops
[1 - reduc_index
];
5430 stmt_vec_info scalar_dest_def_info
;
5431 auto_vec
<tree
> vec_oprnds0
;
5434 auto_vec
<vec
<tree
> > vec_defs (2);
5435 vect_get_slp_defs (slp_node
, &vec_defs
);
5436 vec_oprnds0
.safe_splice (vec_defs
[1 - reduc_index
]);
5437 vec_defs
[0].release ();
5438 vec_defs
[1].release ();
5439 group_size
= SLP_TREE_SCALAR_STMTS (slp_node
).length ();
5440 scalar_dest_def_info
= SLP_TREE_SCALAR_STMTS (slp_node
)[group_size
- 1];
5444 tree loop_vec_def0
= vect_get_vec_def_for_operand (op0
, stmt_info
);
5445 vec_oprnds0
.create (1);
5446 vec_oprnds0
.quick_push (loop_vec_def0
);
5447 scalar_dest_def_info
= stmt_info
;
5450 tree scalar_dest
= gimple_assign_lhs (scalar_dest_def_info
->stmt
);
5451 tree scalar_type
= TREE_TYPE (scalar_dest
);
5452 tree reduc_var
= gimple_phi_result (reduc_def_stmt
);
5454 int vec_num
= vec_oprnds0
.length ();
5455 gcc_assert (vec_num
== 1 || slp_node
);
5456 tree vec_elem_type
= TREE_TYPE (vectype_out
);
5457 gcc_checking_assert (useless_type_conversion_p (scalar_type
, vec_elem_type
));
5459 tree vector_identity
= NULL_TREE
;
5460 if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
5461 vector_identity
= build_zero_cst (vectype_out
);
5463 tree scalar_dest_var
= vect_create_destination_var (scalar_dest
, NULL
);
5466 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, def0
)
5469 tree mask
= NULL_TREE
;
5470 if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
5471 mask
= vect_get_loop_mask (gsi
, masks
, vec_num
, vectype_in
, i
);
5473 /* Handle MINUS by adding the negative. */
5474 if (reduc_fn
!= IFN_LAST
&& code
== MINUS_EXPR
)
5476 tree negated
= make_ssa_name (vectype_out
);
5477 new_stmt
= gimple_build_assign (negated
, NEGATE_EXPR
, def0
);
5478 gsi_insert_before (gsi
, new_stmt
, GSI_SAME_STMT
);
5482 if (mask
&& mask_reduc_fn
== IFN_LAST
)
5483 def0
= merge_with_identity (gsi
, mask
, vectype_out
, def0
,
5486 /* On the first iteration the input is simply the scalar phi
5487 result, and for subsequent iterations it is the output of
5488 the preceding operation. */
5489 if (reduc_fn
!= IFN_LAST
|| (mask
&& mask_reduc_fn
!= IFN_LAST
))
5491 if (mask
&& mask_reduc_fn
!= IFN_LAST
)
5492 new_stmt
= gimple_build_call_internal (mask_reduc_fn
, 3, reduc_var
,
5495 new_stmt
= gimple_build_call_internal (reduc_fn
, 2, reduc_var
,
5497 /* For chained SLP reductions the output of the previous reduction
5498 operation serves as the input of the next. For the final statement
5499 the output cannot be a temporary - we reuse the original
5500 scalar destination of the last statement. */
5501 if (i
!= vec_num
- 1)
5503 gimple_set_lhs (new_stmt
, scalar_dest_var
);
5504 reduc_var
= make_ssa_name (scalar_dest_var
, new_stmt
);
5505 gimple_set_lhs (new_stmt
, reduc_var
);
5510 reduc_var
= vect_expand_fold_left (gsi
, scalar_dest_var
, code
,
5512 new_stmt
= SSA_NAME_DEF_STMT (reduc_var
);
5513 /* Remove the statement, so that we can use the same code paths
5514 as for statements that we've just created. */
5515 gimple_stmt_iterator tmp_gsi
= gsi_for_stmt (new_stmt
);
5516 gsi_remove (&tmp_gsi
, true);
5519 if (i
== vec_num
- 1)
5521 gimple_set_lhs (new_stmt
, scalar_dest
);
5522 new_stmt_info
= vect_finish_replace_stmt (scalar_dest_def_info
,
5526 new_stmt_info
= vect_finish_stmt_generation (scalar_dest_def_info
,
5530 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt_info
);
5534 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
5539 /* Function is_nonwrapping_integer_induction.
5541 Check if STMT_VINO (which is part of loop LOOP) both increments and
5542 does not cause overflow. */
5545 is_nonwrapping_integer_induction (stmt_vec_info stmt_vinfo
, class loop
*loop
)
5547 gphi
*phi
= as_a
<gphi
*> (stmt_vinfo
->stmt
);
5548 tree base
= STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (stmt_vinfo
);
5549 tree step
= STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_vinfo
);
5550 tree lhs_type
= TREE_TYPE (gimple_phi_result (phi
));
5551 widest_int ni
, max_loop_value
, lhs_max
;
5552 wi::overflow_type overflow
= wi::OVF_NONE
;
5554 /* Make sure the loop is integer based. */
5555 if (TREE_CODE (base
) != INTEGER_CST
5556 || TREE_CODE (step
) != INTEGER_CST
)
5559 /* Check that the max size of the loop will not wrap. */
5561 if (TYPE_OVERFLOW_UNDEFINED (lhs_type
))
5564 if (! max_stmt_executions (loop
, &ni
))
5567 max_loop_value
= wi::mul (wi::to_widest (step
), ni
, TYPE_SIGN (lhs_type
),
5572 max_loop_value
= wi::add (wi::to_widest (base
), max_loop_value
,
5573 TYPE_SIGN (lhs_type
), &overflow
);
5577 return (wi::min_precision (max_loop_value
, TYPE_SIGN (lhs_type
))
5578 <= TYPE_PRECISION (lhs_type
));
5581 /* Check if masking can be supported by inserting a conditional expression.
5582 CODE is the code for the operation. COND_FN is the conditional internal
5583 function, if it exists. VECTYPE_IN is the type of the vector input. */
5585 use_mask_by_cond_expr_p (enum tree_code code
, internal_fn cond_fn
,
5588 if (cond_fn
!= IFN_LAST
5589 && direct_internal_fn_supported_p (cond_fn
, vectype_in
,
5590 OPTIMIZE_FOR_SPEED
))
5604 /* Insert a conditional expression to enable masked vectorization. CODE is the
5605 code for the operation. VOP is the array of operands. MASK is the loop
5606 mask. GSI is a statement iterator used to place the new conditional
5609 build_vect_cond_expr (enum tree_code code
, tree vop
[3], tree mask
,
5610 gimple_stmt_iterator
*gsi
)
5616 tree vectype
= TREE_TYPE (vop
[1]);
5617 tree zero
= build_zero_cst (vectype
);
5618 tree masked_op1
= make_temp_ssa_name (vectype
, NULL
, "masked_op1");
5619 gassign
*select
= gimple_build_assign (masked_op1
, VEC_COND_EXPR
,
5620 mask
, vop
[1], zero
);
5621 gsi_insert_before (gsi
, select
, GSI_SAME_STMT
);
5622 vop
[1] = masked_op1
;
5628 tree vectype
= TREE_TYPE (vop
[1]);
5629 tree masked_op1
= make_temp_ssa_name (vectype
, NULL
, "masked_op1");
5630 gassign
*select
= gimple_build_assign (masked_op1
, VEC_COND_EXPR
,
5631 mask
, vop
[1], vop
[0]);
5632 gsi_insert_before (gsi
, select
, GSI_SAME_STMT
);
5633 vop
[1] = masked_op1
;
5642 /* Function vectorizable_reduction.
5644 Check if STMT_INFO performs a reduction operation that can be vectorized.
5645 If VEC_STMT is also passed, vectorize STMT_INFO: create a vectorized
5646 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
5647 Return true if STMT_INFO is vectorizable in this way.
5649 This function also handles reduction idioms (patterns) that have been
5650 recognized in advance during vect_pattern_recog. In this case, STMT_INFO
5651 may be of this form:
5652 X = pattern_expr (arg0, arg1, ..., X)
5653 and its STMT_VINFO_RELATED_STMT points to the last stmt in the original
5654 sequence that had been detected and replaced by the pattern-stmt
5657 This function also handles reduction of condition expressions, for example:
5658 for (int i = 0; i < N; i++)
5661 This is handled by vectorising the loop and creating an additional vector
5662 containing the loop indexes for which "a[i] < value" was true. In the
5663 function epilogue this is reduced to a single max value and then used to
5664 index into the vector of results.
5666 In some cases of reduction patterns, the type of the reduction variable X is
5667 different than the type of the other arguments of STMT_INFO.
5668 In such cases, the vectype that is used when transforming STMT_INFO into
5669 a vector stmt is different than the vectype that is used to determine the
5670 vectorization factor, because it consists of a different number of elements
5671 than the actual number of elements that are being operated upon in parallel.
5673 For example, consider an accumulation of shorts into an int accumulator.
5674 On some targets it's possible to vectorize this pattern operating on 8
5675 shorts at a time (hence, the vectype for purposes of determining the
5676 vectorization factor should be V8HI); on the other hand, the vectype that
5677 is used to create the vector form is actually V4SI (the type of the result).
5679 Upon entry to this function, STMT_VINFO_VECTYPE records the vectype that
5680 indicates what is the actual level of parallelism (V8HI in the example), so
5681 that the right vectorization factor would be derived. This vectype
5682 corresponds to the type of arguments to the reduction stmt, and should *NOT*
5683 be used to create the vectorized stmt. The right vectype for the vectorized
5684 stmt is obtained from the type of the result X:
5685 get_vectype_for_scalar_type (vinfo, TREE_TYPE (X))
5687 This means that, contrary to "regular" reductions (or "regular" stmts in
5688 general), the following equation:
5689 STMT_VINFO_VECTYPE == get_vectype_for_scalar_type (vinfo, TREE_TYPE (X))
5690 does *NOT* necessarily hold for reduction patterns. */
5693 vectorizable_reduction (stmt_vec_info stmt_info
, slp_tree slp_node
,
5694 slp_instance slp_node_instance
,
5695 stmt_vector_for_cost
*cost_vec
)
5698 tree vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
5699 tree vectype_in
= NULL_TREE
;
5700 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5701 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5702 enum vect_def_type cond_reduc_dt
= vect_unknown_def_type
;
5703 stmt_vec_info cond_stmt_vinfo
= NULL
;
5707 bool single_defuse_cycle
= false;
5708 bool nested_cycle
= false;
5709 bool double_reduc
= false;
5712 tree cr_index_scalar_type
= NULL_TREE
, cr_index_vector_type
= NULL_TREE
;
5713 tree cond_reduc_val
= NULL_TREE
;
5715 /* Make sure it was already recognized as a reduction computation. */
5716 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_reduction_def
5717 && STMT_VINFO_DEF_TYPE (stmt_info
) != vect_double_reduction_def
5718 && STMT_VINFO_DEF_TYPE (stmt_info
) != vect_nested_cycle
)
5721 /* The stmt we store reduction analysis meta on. */
5722 stmt_vec_info reduc_info
= info_for_reduction (stmt_info
);
5723 reduc_info
->is_reduc_info
= true;
5725 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
)
5727 if (is_a
<gphi
*> (stmt_info
->stmt
))
5728 /* Analysis for double-reduction is done on the outer
5729 loop PHI, nested cycles have no further restrictions. */
5730 STMT_VINFO_TYPE (stmt_info
) = cycle_phi_info_type
;
5732 STMT_VINFO_TYPE (stmt_info
) = reduc_vec_info_type
;
5736 stmt_vec_info orig_stmt_of_analysis
= stmt_info
;
5737 stmt_vec_info phi_info
= stmt_info
;
5738 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
5739 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_double_reduction_def
)
5741 if (!is_a
<gphi
*> (stmt_info
->stmt
))
5743 STMT_VINFO_TYPE (stmt_info
) = reduc_vec_info_type
;
5748 slp_node_instance
->reduc_phis
= slp_node
;
5749 /* ??? We're leaving slp_node to point to the PHIs, we only
5750 need it to get at the number of vector stmts which wasn't
5751 yet initialized for the instance root. */
5753 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
)
5754 stmt_info
= vect_stmt_to_vectorize (STMT_VINFO_REDUC_DEF (stmt_info
));
5755 else /* STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def */
5757 use_operand_p use_p
;
5759 bool res
= single_imm_use (gimple_phi_result (stmt_info
->stmt
),
5762 phi_info
= loop_vinfo
->lookup_stmt (use_stmt
);
5763 stmt_info
= vect_stmt_to_vectorize (STMT_VINFO_REDUC_DEF (phi_info
));
5765 /* STMT_VINFO_REDUC_DEF doesn't point to the first but the last
5767 if (slp_node
&& REDUC_GROUP_FIRST_ELEMENT (stmt_info
))
5769 gcc_assert (!REDUC_GROUP_NEXT_ELEMENT (stmt_info
));
5770 stmt_info
= REDUC_GROUP_FIRST_ELEMENT (stmt_info
);
5773 /* PHIs should not participate in patterns. */
5774 gcc_assert (!STMT_VINFO_RELATED_STMT (phi_info
));
5776 if (nested_in_vect_loop_p (loop
, stmt_info
))
5779 nested_cycle
= true;
5782 if (REDUC_GROUP_FIRST_ELEMENT (stmt_info
))
5783 gcc_assert (slp_node
5784 && REDUC_GROUP_FIRST_ELEMENT (stmt_info
) == stmt_info
);
5786 /* 1. Is vectorizable reduction? */
5787 /* Not supportable if the reduction variable is used in the loop, unless
5788 it's a reduction chain. */
5789 if (STMT_VINFO_RELEVANT (stmt_info
) > vect_used_in_outer
5790 && !REDUC_GROUP_FIRST_ELEMENT (stmt_info
))
5793 /* Reductions that are not used even in an enclosing outer-loop,
5794 are expected to be "live" (used out of the loop). */
5795 if (STMT_VINFO_RELEVANT (stmt_info
) == vect_unused_in_scope
5796 && !STMT_VINFO_LIVE_P (stmt_info
))
5799 /* 2. Has this been recognized as a reduction pattern?
5801 Check if STMT represents a pattern that has been recognized
5802 in earlier analysis stages. For stmts that represent a pattern,
5803 the STMT_VINFO_RELATED_STMT field records the last stmt in
5804 the original sequence that constitutes the pattern. */
5806 stmt_vec_info orig_stmt_info
= STMT_VINFO_RELATED_STMT (stmt_info
);
5809 gcc_assert (STMT_VINFO_IN_PATTERN_P (orig_stmt_info
));
5810 gcc_assert (!STMT_VINFO_IN_PATTERN_P (stmt_info
));
5813 /* 3. Check the operands of the operation. The first operands are defined
5814 inside the loop body. The last operand is the reduction variable,
5815 which is defined by the loop-header-phi. */
5817 gassign
*stmt
= as_a
<gassign
*> (stmt_info
->stmt
);
5818 enum tree_code code
= gimple_assign_rhs_code (stmt
);
5819 bool lane_reduc_code_p
5820 = (code
== DOT_PROD_EXPR
|| code
== WIDEN_SUM_EXPR
|| code
== SAD_EXPR
);
5821 int op_type
= TREE_CODE_LENGTH (code
);
5823 scalar_dest
= gimple_assign_lhs (stmt
);
5824 scalar_type
= TREE_TYPE (scalar_dest
);
5825 if (!POINTER_TYPE_P (scalar_type
) && !INTEGRAL_TYPE_P (scalar_type
)
5826 && !SCALAR_FLOAT_TYPE_P (scalar_type
))
5829 /* Do not try to vectorize bit-precision reductions. */
5830 if (!type_has_mode_precision_p (scalar_type
))
5833 /* All uses but the last are expected to be defined in the loop.
5834 The last use is the reduction variable. In case of nested cycle this
5835 assumption is not true: we use reduc_index to record the index of the
5836 reduction variable. */
5837 gphi
*reduc_def_phi
= as_a
<gphi
*> (phi_info
->stmt
);
5839 /* Verify following REDUC_IDX from the latch def leads us back to the PHI
5840 and compute the reduction chain length. */
5841 tree reduc_def
= PHI_ARG_DEF_FROM_EDGE (reduc_def_phi
,
5842 loop_latch_edge (loop
));
5843 unsigned reduc_chain_length
= 0;
5844 bool only_slp_reduc_chain
= true;
5845 while (reduc_def
!= PHI_RESULT (reduc_def_phi
))
5847 stmt_vec_info def
= loop_vinfo
->lookup_def (reduc_def
);
5848 stmt_vec_info vdef
= vect_stmt_to_vectorize (def
);
5849 if (STMT_VINFO_REDUC_IDX (vdef
) == -1)
5851 if (dump_enabled_p ())
5852 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5853 "reduction chain broken by patterns.\n");
5856 if (!REDUC_GROUP_FIRST_ELEMENT (vdef
))
5857 only_slp_reduc_chain
= false;
5858 /* ??? For epilogue generation live members of the chain need
5859 to point back to the PHI via their original stmt for
5860 info_for_reduction to work. */
5861 if (STMT_VINFO_LIVE_P (vdef
))
5862 STMT_VINFO_REDUC_DEF (def
) = phi_info
;
5863 reduc_def
= gimple_op (vdef
->stmt
, 1 + STMT_VINFO_REDUC_IDX (vdef
));
5864 reduc_chain_length
++;
5867 reduc_def
= PHI_RESULT (reduc_def_phi
);
5868 for (i
= 0; i
< op_type
; i
++)
5870 tree op
= gimple_op (stmt
, i
+ 1);
5871 /* The condition of COND_EXPR is checked in vectorizable_condition(). */
5872 if (i
== 0 && code
== COND_EXPR
)
5875 stmt_vec_info def_stmt_info
;
5876 enum vect_def_type dt
;
5877 if (!vect_is_simple_use (op
, loop_vinfo
, &dt
, &tem
,
5880 if (dump_enabled_p ())
5881 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5882 "use not simple.\n");
5885 if (i
== STMT_VINFO_REDUC_IDX (stmt_info
))
5888 /* There should be only one cycle def in the stmt, the one
5889 leading to reduc_def. */
5890 if (VECTORIZABLE_CYCLE_DEF (dt
))
5893 /* To properly compute ncopies we are interested in the widest
5894 non-reduction input type in case we're looking at a widening
5895 accumulation that we later handle in vect_transform_reduction. */
5896 if (lane_reduc_code_p
5899 || (GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (vectype_in
)))
5900 < GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (tem
))))))
5903 if (code
== COND_EXPR
)
5905 /* Record how the non-reduction-def value of COND_EXPR is defined. */
5906 if (dt
== vect_constant_def
)
5909 cond_reduc_val
= op
;
5911 if (dt
== vect_induction_def
5913 && is_nonwrapping_integer_induction (def_stmt_info
, loop
))
5916 cond_stmt_vinfo
= def_stmt_info
;
5921 vectype_in
= vectype_out
;
5922 STMT_VINFO_REDUC_VECTYPE_IN (reduc_info
) = vectype_in
;
5924 enum vect_reduction_type v_reduc_type
= STMT_VINFO_REDUC_TYPE (phi_info
);
5925 STMT_VINFO_REDUC_TYPE (reduc_info
) = v_reduc_type
;
5926 /* If we have a condition reduction, see if we can simplify it further. */
5927 if (v_reduc_type
== COND_REDUCTION
)
5932 /* When the condition uses the reduction value in the condition, fail. */
5933 if (STMT_VINFO_REDUC_IDX (stmt_info
) == 0)
5935 if (dump_enabled_p ())
5936 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5937 "condition depends on previous iteration\n");
5941 if (direct_internal_fn_supported_p (IFN_FOLD_EXTRACT_LAST
,
5942 vectype_in
, OPTIMIZE_FOR_SPEED
))
5944 if (dump_enabled_p ())
5945 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5946 "optimizing condition reduction with"
5947 " FOLD_EXTRACT_LAST.\n");
5948 STMT_VINFO_REDUC_TYPE (reduc_info
) = EXTRACT_LAST_REDUCTION
;
5950 else if (cond_reduc_dt
== vect_induction_def
)
5953 = STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED (cond_stmt_vinfo
);
5954 tree step
= STMT_VINFO_LOOP_PHI_EVOLUTION_PART (cond_stmt_vinfo
);
5956 gcc_assert (TREE_CODE (base
) == INTEGER_CST
5957 && TREE_CODE (step
) == INTEGER_CST
);
5958 cond_reduc_val
= NULL_TREE
;
5959 enum tree_code cond_reduc_op_code
= ERROR_MARK
;
5960 tree res
= PHI_RESULT (STMT_VINFO_STMT (cond_stmt_vinfo
));
5961 if (!types_compatible_p (TREE_TYPE (res
), TREE_TYPE (base
)))
5963 /* Find a suitable value, for MAX_EXPR below base, for MIN_EXPR
5964 above base; punt if base is the minimum value of the type for
5965 MAX_EXPR or maximum value of the type for MIN_EXPR for now. */
5966 else if (tree_int_cst_sgn (step
) == -1)
5968 cond_reduc_op_code
= MIN_EXPR
;
5969 if (tree_int_cst_sgn (base
) == -1)
5970 cond_reduc_val
= build_int_cst (TREE_TYPE (base
), 0);
5971 else if (tree_int_cst_lt (base
,
5972 TYPE_MAX_VALUE (TREE_TYPE (base
))))
5974 = int_const_binop (PLUS_EXPR
, base
, integer_one_node
);
5978 cond_reduc_op_code
= MAX_EXPR
;
5979 if (tree_int_cst_sgn (base
) == 1)
5980 cond_reduc_val
= build_int_cst (TREE_TYPE (base
), 0);
5981 else if (tree_int_cst_lt (TYPE_MIN_VALUE (TREE_TYPE (base
)),
5984 = int_const_binop (MINUS_EXPR
, base
, integer_one_node
);
5988 if (dump_enabled_p ())
5989 dump_printf_loc (MSG_NOTE
, vect_location
,
5990 "condition expression based on "
5991 "integer induction.\n");
5992 STMT_VINFO_REDUC_CODE (reduc_info
) = cond_reduc_op_code
;
5993 STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_info
)
5995 STMT_VINFO_REDUC_TYPE (reduc_info
) = INTEGER_INDUC_COND_REDUCTION
;
5998 else if (cond_reduc_dt
== vect_constant_def
)
6000 enum vect_def_type cond_initial_dt
;
6001 tree cond_initial_val
6002 = PHI_ARG_DEF_FROM_EDGE (reduc_def_phi
, loop_preheader_edge (loop
));
6004 gcc_assert (cond_reduc_val
!= NULL_TREE
);
6005 vect_is_simple_use (cond_initial_val
, loop_vinfo
, &cond_initial_dt
);
6006 if (cond_initial_dt
== vect_constant_def
6007 && types_compatible_p (TREE_TYPE (cond_initial_val
),
6008 TREE_TYPE (cond_reduc_val
)))
6010 tree e
= fold_binary (LE_EXPR
, boolean_type_node
,
6011 cond_initial_val
, cond_reduc_val
);
6012 if (e
&& (integer_onep (e
) || integer_zerop (e
)))
6014 if (dump_enabled_p ())
6015 dump_printf_loc (MSG_NOTE
, vect_location
,
6016 "condition expression based on "
6017 "compile time constant.\n");
6018 /* Record reduction code at analysis stage. */
6019 STMT_VINFO_REDUC_CODE (reduc_info
)
6020 = integer_onep (e
) ? MAX_EXPR
: MIN_EXPR
;
6021 STMT_VINFO_REDUC_TYPE (reduc_info
) = CONST_COND_REDUCTION
;
6027 if (REDUC_GROUP_FIRST_ELEMENT (stmt_info
))
6028 /* We changed STMT to be the first stmt in reduction chain, hence we
6029 check that in this case the first element in the chain is STMT. */
6030 gcc_assert (REDUC_GROUP_FIRST_ELEMENT (STMT_VINFO_REDUC_DEF (phi_info
))
6031 == vect_orig_stmt (stmt_info
));
6033 if (STMT_VINFO_LIVE_P (phi_info
))
6039 ncopies
= vect_get_num_copies (loop_vinfo
, vectype_in
);
6041 gcc_assert (ncopies
>= 1);
6043 poly_uint64 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
6047 gcc_assert (STMT_VINFO_DEF_TYPE (reduc_info
)
6048 == vect_double_reduction_def
);
6049 double_reduc
= true;
6052 /* 4.2. Check support for the epilog operation.
6054 If STMT represents a reduction pattern, then the type of the
6055 reduction variable may be different than the type of the rest
6056 of the arguments. For example, consider the case of accumulation
6057 of shorts into an int accumulator; The original code:
6058 S1: int_a = (int) short_a;
6059 orig_stmt-> S2: int_acc = plus <int_a ,int_acc>;
6062 STMT: int_acc = widen_sum <short_a, int_acc>
6065 1. The tree-code that is used to create the vector operation in the
6066 epilog code (that reduces the partial results) is not the
6067 tree-code of STMT, but is rather the tree-code of the original
6068 stmt from the pattern that STMT is replacing. I.e, in the example
6069 above we want to use 'widen_sum' in the loop, but 'plus' in the
6071 2. The type (mode) we use to check available target support
6072 for the vector operation to be created in the *epilog*, is
6073 determined by the type of the reduction variable (in the example
6074 above we'd check this: optab_handler (plus_optab, vect_int_mode])).
6075 However the type (mode) we use to check available target support
6076 for the vector operation to be created *inside the loop*, is
6077 determined by the type of the other arguments to STMT (in the
6078 example we'd check this: optab_handler (widen_sum_optab,
6081 This is contrary to "regular" reductions, in which the types of all
6082 the arguments are the same as the type of the reduction variable.
6083 For "regular" reductions we can therefore use the same vector type
6084 (and also the same tree-code) when generating the epilog code and
6085 when generating the code inside the loop. */
6087 enum tree_code orig_code
= STMT_VINFO_REDUC_CODE (phi_info
);
6088 STMT_VINFO_REDUC_CODE (reduc_info
) = orig_code
;
6090 vect_reduction_type reduction_type
= STMT_VINFO_REDUC_TYPE (reduc_info
);
6091 if (reduction_type
== TREE_CODE_REDUCTION
)
6093 /* Check whether it's ok to change the order of the computation.
6094 Generally, when vectorizing a reduction we change the order of the
6095 computation. This may change the behavior of the program in some
6096 cases, so we need to check that this is ok. One exception is when
6097 vectorizing an outer-loop: the inner-loop is executed sequentially,
6098 and therefore vectorizing reductions in the inner-loop during
6099 outer-loop vectorization is safe. */
6100 if (needs_fold_left_reduction_p (scalar_type
, orig_code
))
6102 /* When vectorizing a reduction chain w/o SLP the reduction PHI
6103 is not directy used in stmt. */
6104 if (!only_slp_reduc_chain
6105 && reduc_chain_length
!= 1)
6107 if (dump_enabled_p ())
6108 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6109 "in-order reduction chain without SLP.\n");
6112 STMT_VINFO_REDUC_TYPE (reduc_info
)
6113 = reduction_type
= FOLD_LEFT_REDUCTION
;
6115 else if (!commutative_tree_code (orig_code
)
6116 || !associative_tree_code (orig_code
))
6118 if (dump_enabled_p ())
6119 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6120 "reduction: not commutative/associative");
6125 if ((double_reduc
|| reduction_type
!= TREE_CODE_REDUCTION
)
6128 if (dump_enabled_p ())
6129 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6130 "multiple types in double reduction or condition "
6131 "reduction or fold-left reduction.\n");
6135 internal_fn reduc_fn
= IFN_LAST
;
6136 if (reduction_type
== TREE_CODE_REDUCTION
6137 || reduction_type
== FOLD_LEFT_REDUCTION
6138 || reduction_type
== INTEGER_INDUC_COND_REDUCTION
6139 || reduction_type
== CONST_COND_REDUCTION
)
6141 if (reduction_type
== FOLD_LEFT_REDUCTION
6142 ? fold_left_reduction_fn (orig_code
, &reduc_fn
)
6143 : reduction_fn_for_scalar_code (orig_code
, &reduc_fn
))
6145 if (reduc_fn
!= IFN_LAST
6146 && !direct_internal_fn_supported_p (reduc_fn
, vectype_out
,
6147 OPTIMIZE_FOR_SPEED
))
6149 if (dump_enabled_p ())
6150 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6151 "reduc op not supported by target.\n");
6153 reduc_fn
= IFN_LAST
;
6158 if (!nested_cycle
|| double_reduc
)
6160 if (dump_enabled_p ())
6161 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6162 "no reduc code for scalar code.\n");
6168 else if (reduction_type
== COND_REDUCTION
)
6170 int scalar_precision
6171 = GET_MODE_PRECISION (SCALAR_TYPE_MODE (scalar_type
));
6172 cr_index_scalar_type
= make_unsigned_type (scalar_precision
);
6173 cr_index_vector_type
= build_vector_type (cr_index_scalar_type
,
6176 if (direct_internal_fn_supported_p (IFN_REDUC_MAX
, cr_index_vector_type
,
6177 OPTIMIZE_FOR_SPEED
))
6178 reduc_fn
= IFN_REDUC_MAX
;
6180 STMT_VINFO_REDUC_FN (reduc_info
) = reduc_fn
;
6182 if (reduction_type
!= EXTRACT_LAST_REDUCTION
6183 && (!nested_cycle
|| double_reduc
)
6184 && reduc_fn
== IFN_LAST
6185 && !nunits_out
.is_constant ())
6187 if (dump_enabled_p ())
6188 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6189 "missing target support for reduction on"
6190 " variable-length vectors.\n");
6194 /* For SLP reductions, see if there is a neutral value we can use. */
6195 tree neutral_op
= NULL_TREE
;
6197 neutral_op
= neutral_op_for_slp_reduction
6198 (slp_node_instance
->reduc_phis
, orig_code
,
6199 REDUC_GROUP_FIRST_ELEMENT (stmt_info
) != NULL
);
6201 if (double_reduc
&& reduction_type
== FOLD_LEFT_REDUCTION
)
6203 /* We can't support in-order reductions of code such as this:
6205 for (int i = 0; i < n1; ++i)
6206 for (int j = 0; j < n2; ++j)
6209 since GCC effectively transforms the loop when vectorizing:
6211 for (int i = 0; i < n1 / VF; ++i)
6212 for (int j = 0; j < n2; ++j)
6213 for (int k = 0; k < VF; ++k)
6216 which is a reassociation of the original operation. */
6217 if (dump_enabled_p ())
6218 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6219 "in-order double reduction not supported.\n");
6224 if (reduction_type
== FOLD_LEFT_REDUCTION
6226 && !REDUC_GROUP_FIRST_ELEMENT (stmt_info
))
6228 /* We cannot use in-order reductions in this case because there is
6229 an implicit reassociation of the operations involved. */
6230 if (dump_enabled_p ())
6231 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6232 "in-order unchained SLP reductions not supported.\n");
6236 /* For double reductions, and for SLP reductions with a neutral value,
6237 we construct a variable-length initial vector by loading a vector
6238 full of the neutral value and then shift-and-inserting the start
6239 values into the low-numbered elements. */
6240 if ((double_reduc
|| neutral_op
)
6241 && !nunits_out
.is_constant ()
6242 && !direct_internal_fn_supported_p (IFN_VEC_SHL_INSERT
,
6243 vectype_out
, OPTIMIZE_FOR_SPEED
))
6245 if (dump_enabled_p ())
6246 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6247 "reduction on variable-length vectors requires"
6248 " target support for a vector-shift-and-insert"
6253 /* Check extra constraints for variable-length unchained SLP reductions. */
6254 if (STMT_SLP_TYPE (stmt_info
)
6255 && !REDUC_GROUP_FIRST_ELEMENT (stmt_info
)
6256 && !nunits_out
.is_constant ())
6258 /* We checked above that we could build the initial vector when
6259 there's a neutral element value. Check here for the case in
6260 which each SLP statement has its own initial value and in which
6261 that value needs to be repeated for every instance of the
6262 statement within the initial vector. */
6263 unsigned int group_size
= SLP_INSTANCE_GROUP_SIZE (slp_node_instance
);
6264 scalar_mode elt_mode
= SCALAR_TYPE_MODE (TREE_TYPE (vectype_out
));
6266 && !can_duplicate_and_interleave_p (loop_vinfo
, group_size
,
6269 if (dump_enabled_p ())
6270 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6271 "unsupported form of SLP reduction for"
6272 " variable-length vectors: cannot build"
6273 " initial vector.\n");
6276 /* The epilogue code relies on the number of elements being a multiple
6277 of the group size. The duplicate-and-interleave approach to setting
6278 up the the initial vector does too. */
6279 if (!multiple_p (nunits_out
, group_size
))
6281 if (dump_enabled_p ())
6282 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6283 "unsupported form of SLP reduction for"
6284 " variable-length vectors: the vector size"
6285 " is not a multiple of the number of results.\n");
6290 if (reduction_type
== COND_REDUCTION
)
6294 if (! max_loop_iterations (loop
, &ni
))
6296 if (dump_enabled_p ())
6297 dump_printf_loc (MSG_NOTE
, vect_location
,
6298 "loop count not known, cannot create cond "
6302 /* Convert backedges to iterations. */
6305 /* The additional index will be the same type as the condition. Check
6306 that the loop can fit into this less one (because we'll use up the
6307 zero slot for when there are no matches). */
6308 tree max_index
= TYPE_MAX_VALUE (cr_index_scalar_type
);
6309 if (wi::geu_p (ni
, wi::to_widest (max_index
)))
6311 if (dump_enabled_p ())
6312 dump_printf_loc (MSG_NOTE
, vect_location
,
6313 "loop size is greater than data size.\n");
6318 /* In case the vectorization factor (VF) is bigger than the number
6319 of elements that we can fit in a vectype (nunits), we have to generate
6320 more than one vector stmt - i.e - we need to "unroll" the
6321 vector stmt by a factor VF/nunits. For more details see documentation
6322 in vectorizable_operation. */
6324 /* If the reduction is used in an outer loop we need to generate
6325 VF intermediate results, like so (e.g. for ncopies=2):
6330 (i.e. we generate VF results in 2 registers).
6331 In this case we have a separate def-use cycle for each copy, and therefore
6332 for each copy we get the vector def for the reduction variable from the
6333 respective phi node created for this copy.
6335 Otherwise (the reduction is unused in the loop nest), we can combine
6336 together intermediate results, like so (e.g. for ncopies=2):
6340 (i.e. we generate VF/2 results in a single register).
6341 In this case for each copy we get the vector def for the reduction variable
6342 from the vectorized reduction operation generated in the previous iteration.
6344 This only works when we see both the reduction PHI and its only consumer
6345 in vectorizable_reduction and there are no intermediate stmts
6348 && (STMT_VINFO_RELEVANT (stmt_info
) <= vect_used_only_live
)
6349 && reduc_chain_length
== 1)
6350 single_defuse_cycle
= true;
6352 if (single_defuse_cycle
|| lane_reduc_code_p
)
6354 gcc_assert (code
!= COND_EXPR
);
6356 /* 4. Supportable by target? */
6359 /* 4.1. check support for the operation in the loop */
6360 optab optab
= optab_for_tree_code (code
, vectype_in
, optab_vector
);
6363 if (dump_enabled_p ())
6364 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6369 machine_mode vec_mode
= TYPE_MODE (vectype_in
);
6370 if (ok
&& optab_handler (optab
, vec_mode
) == CODE_FOR_nothing
)
6372 if (dump_enabled_p ())
6373 dump_printf (MSG_NOTE
, "op not supported by target.\n");
6374 if (maybe_ne (GET_MODE_SIZE (vec_mode
), UNITS_PER_WORD
)
6375 || !vect_worthwhile_without_simd_p (loop_vinfo
, code
))
6378 if (dump_enabled_p ())
6379 dump_printf (MSG_NOTE
, "proceeding using word mode.\n");
6382 /* Worthwhile without SIMD support? */
6384 && !VECTOR_MODE_P (TYPE_MODE (vectype_in
))
6385 && !vect_worthwhile_without_simd_p (loop_vinfo
, code
))
6387 if (dump_enabled_p ())
6388 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6389 "not worthwhile without SIMD support.\n");
6393 /* lane-reducing operations have to go through vect_transform_reduction.
6394 For the other cases try without the single cycle optimization. */
6397 if (lane_reduc_code_p
)
6400 single_defuse_cycle
= false;
6403 STMT_VINFO_FORCE_SINGLE_CYCLE (reduc_info
) = single_defuse_cycle
;
6405 /* If the reduction stmt is one of the patterns that have lane
6406 reduction embedded we cannot handle the case of ! single_defuse_cycle. */
6407 if ((ncopies
> 1 && ! single_defuse_cycle
)
6408 && lane_reduc_code_p
)
6410 if (dump_enabled_p ())
6411 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6412 "multi def-use cycle not possible for lane-reducing "
6413 "reduction operation\n");
6418 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6422 vect_model_reduction_cost (stmt_info
, reduc_fn
, reduction_type
, ncopies
,
6424 if (dump_enabled_p ()
6425 && reduction_type
== FOLD_LEFT_REDUCTION
)
6426 dump_printf_loc (MSG_NOTE
, vect_location
,
6427 "using an in-order (fold-left) reduction.\n");
6428 STMT_VINFO_TYPE (orig_stmt_of_analysis
) = cycle_phi_info_type
;
6429 /* All but single defuse-cycle optimized, lane-reducing and fold-left
6430 reductions go through their own vectorizable_* routines. */
6431 if (!single_defuse_cycle
6432 && code
!= DOT_PROD_EXPR
6433 && code
!= WIDEN_SUM_EXPR
6435 && reduction_type
!= FOLD_LEFT_REDUCTION
)
6437 STMT_VINFO_DEF_TYPE (stmt_info
) = vect_internal_def
;
6438 STMT_VINFO_DEF_TYPE (vect_orig_stmt (stmt_info
)) = vect_internal_def
;
6440 else if (loop_vinfo
&& LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo
))
6442 vec_loop_masks
*masks
= &LOOP_VINFO_MASKS (loop_vinfo
);
6443 internal_fn cond_fn
= get_conditional_internal_fn (code
);
6445 if (reduction_type
!= FOLD_LEFT_REDUCTION
6446 && !use_mask_by_cond_expr_p (code
, cond_fn
, vectype_in
)
6447 && (cond_fn
== IFN_LAST
6448 || !direct_internal_fn_supported_p (cond_fn
, vectype_in
,
6449 OPTIMIZE_FOR_SPEED
)))
6451 if (dump_enabled_p ())
6452 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6453 "can't use a fully-masked loop because no"
6454 " conditional operation is available.\n");
6455 LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo
) = false;
6458 vect_record_loop_mask (loop_vinfo
, masks
, ncopies
* vec_num
,
6464 /* Transform the definition stmt STMT_INFO of a reduction PHI backedge
6468 vect_transform_reduction (stmt_vec_info stmt_info
, gimple_stmt_iterator
*gsi
,
6469 stmt_vec_info
*vec_stmt
, slp_tree slp_node
)
6471 tree vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
6472 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
6473 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
6479 stmt_vec_info reduc_info
= info_for_reduction (stmt_info
);
6480 gcc_assert (reduc_info
->is_reduc_info
);
6482 if (nested_in_vect_loop_p (loop
, stmt_info
))
6485 gcc_assert (STMT_VINFO_DEF_TYPE (reduc_info
) == vect_double_reduction_def
);
6488 gassign
*stmt
= as_a
<gassign
*> (stmt_info
->stmt
);
6489 enum tree_code code
= gimple_assign_rhs_code (stmt
);
6490 int op_type
= TREE_CODE_LENGTH (code
);
6494 switch (get_gimple_rhs_class (code
))
6496 case GIMPLE_TERNARY_RHS
:
6497 ops
[2] = gimple_assign_rhs3 (stmt
);
6499 case GIMPLE_BINARY_RHS
:
6500 ops
[0] = gimple_assign_rhs1 (stmt
);
6501 ops
[1] = gimple_assign_rhs2 (stmt
);
6507 /* All uses but the last are expected to be defined in the loop.
6508 The last use is the reduction variable. In case of nested cycle this
6509 assumption is not true: we use reduc_index to record the index of the
6510 reduction variable. */
6511 stmt_vec_info phi_info
= STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info
));
6512 gphi
*reduc_def_phi
= as_a
<gphi
*> (phi_info
->stmt
);
6513 int reduc_index
= STMT_VINFO_REDUC_IDX (stmt_info
);
6514 tree vectype_in
= STMT_VINFO_REDUC_VECTYPE_IN (reduc_info
);
6519 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6523 ncopies
= vect_get_num_copies (loop_vinfo
, vectype_in
);
6527 internal_fn cond_fn
= get_conditional_internal_fn (code
);
6528 vec_loop_masks
*masks
= &LOOP_VINFO_MASKS (loop_vinfo
);
6529 bool mask_by_cond_expr
= use_mask_by_cond_expr_p (code
, cond_fn
, vectype_in
);
6532 stmt_vec_info new_stmt_info
= NULL
;
6533 stmt_vec_info prev_stmt_info
;
6534 tree new_temp
= NULL_TREE
;
6535 auto_vec
<tree
> vec_oprnds0
;
6536 auto_vec
<tree
> vec_oprnds1
;
6537 auto_vec
<tree
> vec_oprnds2
;
6540 if (dump_enabled_p ())
6541 dump_printf_loc (MSG_NOTE
, vect_location
, "transform reduction.\n");
6543 /* FORNOW: Multiple types are not supported for condition. */
6544 if (code
== COND_EXPR
)
6545 gcc_assert (ncopies
== 1);
6547 bool masked_loop_p
= LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
);
6549 vect_reduction_type reduction_type
= STMT_VINFO_REDUC_TYPE (reduc_info
);
6550 if (reduction_type
== FOLD_LEFT_REDUCTION
)
6552 internal_fn reduc_fn
= STMT_VINFO_REDUC_FN (reduc_info
);
6553 return vectorize_fold_left_reduction
6554 (stmt_info
, gsi
, vec_stmt
, slp_node
, reduc_def_phi
, code
,
6555 reduc_fn
, ops
, vectype_in
, reduc_index
, masks
);
6558 bool single_defuse_cycle
= STMT_VINFO_FORCE_SINGLE_CYCLE (reduc_info
);
6559 gcc_assert (single_defuse_cycle
6560 || code
== DOT_PROD_EXPR
6561 || code
== WIDEN_SUM_EXPR
6562 || code
== SAD_EXPR
);
6564 /* Create the destination vector */
6565 tree scalar_dest
= gimple_assign_lhs (stmt
);
6566 tree vec_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
6568 prev_stmt_info
= NULL
;
6571 vec_oprnds0
.create (1);
6572 vec_oprnds1
.create (1);
6573 if (op_type
== ternary_op
)
6574 vec_oprnds2
.create (1);
6577 for (j
= 0; j
< ncopies
; j
++)
6584 /* Get vec defs for all the operands except the reduction index,
6585 ensuring the ordering of the ops in the vector is kept. */
6586 auto_vec
<vec
<tree
>, 3> vec_defs
;
6587 vect_get_slp_defs (slp_node
, &vec_defs
);
6588 vec_oprnds0
.safe_splice (vec_defs
[0]);
6589 vec_defs
[0].release ();
6590 vec_oprnds1
.safe_splice (vec_defs
[1]);
6591 vec_defs
[1].release ();
6592 if (op_type
== ternary_op
)
6594 vec_oprnds2
.safe_splice (vec_defs
[2]);
6595 vec_defs
[2].release ();
6600 vec_oprnds0
.quick_push
6601 (vect_get_vec_def_for_operand (ops
[0], stmt_info
));
6602 vec_oprnds1
.quick_push
6603 (vect_get_vec_def_for_operand (ops
[1], stmt_info
));
6604 if (op_type
== ternary_op
)
6605 vec_oprnds2
.quick_push
6606 (vect_get_vec_def_for_operand (ops
[2], stmt_info
));
6613 gcc_assert (reduc_index
!= -1 || ! single_defuse_cycle
);
6615 if (single_defuse_cycle
&& reduc_index
== 0)
6616 vec_oprnds0
[0] = gimple_get_lhs (new_stmt_info
->stmt
);
6619 = vect_get_vec_def_for_stmt_copy (loop_vinfo
,
6621 if (single_defuse_cycle
&& reduc_index
== 1)
6622 vec_oprnds1
[0] = gimple_get_lhs (new_stmt_info
->stmt
);
6625 = vect_get_vec_def_for_stmt_copy (loop_vinfo
,
6627 if (op_type
== ternary_op
)
6629 if (single_defuse_cycle
&& reduc_index
== 2)
6630 vec_oprnds2
[0] = gimple_get_lhs (new_stmt_info
->stmt
);
6633 = vect_get_vec_def_for_stmt_copy (loop_vinfo
,
6639 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, def0
)
6641 tree vop
[3] = { def0
, vec_oprnds1
[i
], NULL_TREE
};
6642 if (masked_loop_p
&& !mask_by_cond_expr
)
6644 /* Make sure that the reduction accumulator is vop[0]. */
6645 if (reduc_index
== 1)
6647 gcc_assert (commutative_tree_code (code
));
6648 std::swap (vop
[0], vop
[1]);
6650 tree mask
= vect_get_loop_mask (gsi
, masks
, vec_num
* ncopies
,
6651 vectype_in
, i
* ncopies
+ j
);
6652 gcall
*call
= gimple_build_call_internal (cond_fn
, 4, mask
,
6655 new_temp
= make_ssa_name (vec_dest
, call
);
6656 gimple_call_set_lhs (call
, new_temp
);
6657 gimple_call_set_nothrow (call
, true);
6659 = vect_finish_stmt_generation (stmt_info
, call
, gsi
);
6663 if (op_type
== ternary_op
)
6664 vop
[2] = vec_oprnds2
[i
];
6666 if (masked_loop_p
&& mask_by_cond_expr
)
6668 tree mask
= vect_get_loop_mask (gsi
, masks
,
6670 vectype_in
, i
* ncopies
+ j
);
6671 build_vect_cond_expr (code
, vop
, mask
, gsi
);
6674 gassign
*new_stmt
= gimple_build_assign (vec_dest
, code
,
6675 vop
[0], vop
[1], vop
[2]);
6676 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
6677 gimple_assign_set_lhs (new_stmt
, new_temp
);
6679 = vect_finish_stmt_generation (stmt_info
, new_stmt
, gsi
);
6683 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt_info
);
6686 if (slp_node
|| single_defuse_cycle
)
6690 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
6692 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt_info
;
6694 prev_stmt_info
= new_stmt_info
;
6697 if (single_defuse_cycle
&& !slp_node
)
6698 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt_info
;
6703 /* Transform phase of a cycle PHI. */
6706 vect_transform_cycle_phi (stmt_vec_info stmt_info
, stmt_vec_info
*vec_stmt
,
6707 slp_tree slp_node
, slp_instance slp_node_instance
)
6709 tree vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
6710 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
6711 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
6714 stmt_vec_info prev_phi_info
;
6716 bool nested_cycle
= false;
6719 if (nested_in_vect_loop_p (loop
, stmt_info
))
6722 nested_cycle
= true;
6725 stmt_vec_info reduc_stmt_info
= STMT_VINFO_REDUC_DEF (stmt_info
);
6726 reduc_stmt_info
= vect_stmt_to_vectorize (reduc_stmt_info
);
6727 stmt_vec_info reduc_info
= info_for_reduction (stmt_info
);
6728 gcc_assert (reduc_info
->is_reduc_info
);
6730 if (STMT_VINFO_REDUC_TYPE (reduc_info
) == EXTRACT_LAST_REDUCTION
6731 || STMT_VINFO_REDUC_TYPE (reduc_info
) == FOLD_LEFT_REDUCTION
)
6732 /* Leave the scalar phi in place. */
6735 tree vectype_in
= STMT_VINFO_REDUC_VECTYPE_IN (reduc_info
);
6736 /* For a nested cycle we do not fill the above. */
6738 vectype_in
= STMT_VINFO_VECTYPE (stmt_info
);
6739 gcc_assert (vectype_in
);
6743 /* The size vect_schedule_slp_instance computes is off for us. */
6744 vec_num
= vect_get_num_vectors
6745 (LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
6746 * SLP_TREE_SCALAR_STMTS (slp_node
).length (), vectype_in
);
6752 ncopies
= vect_get_num_copies (loop_vinfo
, vectype_in
);
6755 /* Check whether we should use a single PHI node and accumulate
6756 vectors to one before the backedge. */
6757 if (STMT_VINFO_FORCE_SINGLE_CYCLE (reduc_info
))
6760 /* Create the destination vector */
6761 gphi
*phi
= as_a
<gphi
*> (stmt_info
->stmt
);
6762 tree vec_dest
= vect_create_destination_var (gimple_phi_result (phi
),
6765 /* Get the loop-entry arguments. */
6766 tree vec_initial_def
;
6767 auto_vec
<tree
> vec_initial_defs
;
6770 vec_initial_defs
.reserve (vec_num
);
6771 gcc_assert (slp_node
== slp_node_instance
->reduc_phis
);
6772 stmt_vec_info first
= REDUC_GROUP_FIRST_ELEMENT (reduc_stmt_info
);
6774 = neutral_op_for_slp_reduction (slp_node
,
6775 STMT_VINFO_REDUC_CODE (reduc_info
),
6777 get_initial_defs_for_reduction (slp_node_instance
->reduc_phis
,
6778 &vec_initial_defs
, vec_num
,
6779 first
!= NULL
, neutral_op
);
6783 /* Get at the scalar def before the loop, that defines the initial
6784 value of the reduction variable. */
6785 tree initial_def
= PHI_ARG_DEF_FROM_EDGE (phi
,
6786 loop_preheader_edge (loop
));
6787 /* Optimize: if initial_def is for REDUC_MAX smaller than the base
6788 and we can't use zero for induc_val, use initial_def. Similarly
6789 for REDUC_MIN and initial_def larger than the base. */
6790 if (STMT_VINFO_REDUC_TYPE (reduc_info
) == INTEGER_INDUC_COND_REDUCTION
)
6792 tree induc_val
= STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_info
);
6793 if (TREE_CODE (initial_def
) == INTEGER_CST
6794 && !integer_zerop (induc_val
)
6795 && ((STMT_VINFO_REDUC_CODE (reduc_info
) == MAX_EXPR
6796 && tree_int_cst_lt (initial_def
, induc_val
))
6797 || (STMT_VINFO_REDUC_CODE (reduc_info
) == MIN_EXPR
6798 && tree_int_cst_lt (induc_val
, initial_def
))))
6800 induc_val
= initial_def
;
6801 /* Communicate we used the initial_def to epilouge
6803 STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_info
) = NULL_TREE
;
6805 vec_initial_def
= build_vector_from_val (vectype_out
, induc_val
);
6807 else if (nested_cycle
)
6809 /* Do not use an adjustment def as that case is not supported
6810 correctly if ncopies is not one. */
6811 vec_initial_def
= vect_get_vec_def_for_operand (initial_def
,
6816 tree adjustment_def
= NULL_TREE
;
6817 tree
*adjustment_defp
= &adjustment_def
;
6818 enum tree_code code
= STMT_VINFO_REDUC_CODE (reduc_info
);
6819 if (STMT_VINFO_DEF_TYPE (stmt_info
) == vect_double_reduction_def
)
6820 adjustment_defp
= NULL
;
6822 = get_initial_def_for_reduction (reduc_stmt_info
, code
,
6823 initial_def
, adjustment_defp
);
6824 STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (reduc_info
) = adjustment_def
;
6826 vec_initial_defs
.create (1);
6827 vec_initial_defs
.quick_push (vec_initial_def
);
6830 /* Generate the reduction PHIs upfront. */
6831 prev_phi_info
= NULL
;
6832 for (i
= 0; i
< vec_num
; i
++)
6834 tree vec_init_def
= vec_initial_defs
[i
];
6835 for (j
= 0; j
< ncopies
; j
++)
6837 /* Create the reduction-phi that defines the reduction
6839 gphi
*new_phi
= create_phi_node (vec_dest
, loop
->header
);
6840 stmt_vec_info new_phi_info
= loop_vinfo
->add_stmt (new_phi
);
6842 /* Set the loop-entry arg of the reduction-phi. */
6843 if (j
!= 0 && nested_cycle
)
6844 vec_init_def
= vect_get_vec_def_for_stmt_copy (loop_vinfo
,
6846 add_phi_arg (new_phi
, vec_init_def
, loop_preheader_edge (loop
),
6849 /* The loop-latch arg is set in epilogue processing. */
6852 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_phi_info
);
6856 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_phi_info
;
6858 STMT_VINFO_RELATED_STMT (prev_phi_info
) = new_phi_info
;
6859 prev_phi_info
= new_phi_info
;
6867 /* Vectorizes LC PHIs. */
6870 vectorizable_lc_phi (stmt_vec_info stmt_info
, stmt_vec_info
*vec_stmt
,
6873 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
6875 || !is_a
<gphi
*> (stmt_info
->stmt
)
6876 || gimple_phi_num_args (stmt_info
->stmt
) != 1)
6879 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
6880 && STMT_VINFO_DEF_TYPE (stmt_info
) != vect_double_reduction_def
)
6883 if (!vec_stmt
) /* transformation not required. */
6885 STMT_VINFO_TYPE (stmt_info
) = lc_phi_info_type
;
6889 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
6890 tree scalar_dest
= gimple_phi_result (stmt_info
->stmt
);
6891 basic_block bb
= gimple_bb (stmt_info
->stmt
);
6892 edge e
= single_pred_edge (bb
);
6893 tree vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6894 vec
<tree
> vec_oprnds
= vNULL
;
6895 vect_get_vec_defs (gimple_phi_arg_def (stmt_info
->stmt
, 0), NULL_TREE
,
6896 stmt_info
, &vec_oprnds
, NULL
, slp_node
);
6899 unsigned vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6900 gcc_assert (vec_oprnds
.length () == vec_num
);
6901 for (unsigned i
= 0; i
< vec_num
; i
++)
6903 /* Create the vectorized LC PHI node. */
6904 gphi
*new_phi
= create_phi_node (vec_dest
, bb
);
6905 add_phi_arg (new_phi
, vec_oprnds
[i
], e
, UNKNOWN_LOCATION
);
6906 stmt_vec_info new_phi_info
= loop_vinfo
->add_stmt (new_phi
);
6907 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_phi_info
);
6912 unsigned ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
6913 stmt_vec_info prev_phi_info
= NULL
;
6914 for (unsigned i
= 0; i
< ncopies
; i
++)
6917 vect_get_vec_defs_for_stmt_copy (loop_vinfo
, &vec_oprnds
, NULL
);
6918 /* Create the vectorized LC PHI node. */
6919 gphi
*new_phi
= create_phi_node (vec_dest
, bb
);
6920 add_phi_arg (new_phi
, vec_oprnds
[0], e
, UNKNOWN_LOCATION
);
6921 stmt_vec_info new_phi_info
= loop_vinfo
->add_stmt (new_phi
);
6923 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_phi_info
;
6925 STMT_VINFO_RELATED_STMT (prev_phi_info
) = new_phi_info
;
6926 prev_phi_info
= new_phi_info
;
6929 vec_oprnds
.release ();
6935 /* Function vect_min_worthwhile_factor.
6937 For a loop where we could vectorize the operation indicated by CODE,
6938 return the minimum vectorization factor that makes it worthwhile
6939 to use generic vectors. */
6941 vect_min_worthwhile_factor (enum tree_code code
)
6961 /* Return true if VINFO indicates we are doing loop vectorization and if
6962 it is worth decomposing CODE operations into scalar operations for
6963 that loop's vectorization factor. */
6966 vect_worthwhile_without_simd_p (vec_info
*vinfo
, tree_code code
)
6968 loop_vec_info loop_vinfo
= dyn_cast
<loop_vec_info
> (vinfo
);
6969 unsigned HOST_WIDE_INT value
;
6971 && LOOP_VINFO_VECT_FACTOR (loop_vinfo
).is_constant (&value
)
6972 && value
>= vect_min_worthwhile_factor (code
));
6975 /* Function vectorizable_induction
6977 Check if STMT_INFO performs an induction computation that can be vectorized.
6978 If VEC_STMT is also passed, vectorize the induction PHI: create a vectorized
6979 phi to replace it, put it in VEC_STMT, and add it to the same basic block.
6980 Return true if STMT_INFO is vectorizable in this way. */
6983 vectorizable_induction (stmt_vec_info stmt_info
,
6984 gimple_stmt_iterator
*gsi ATTRIBUTE_UNUSED
,
6985 stmt_vec_info
*vec_stmt
, slp_tree slp_node
,
6986 stmt_vector_for_cost
*cost_vec
)
6988 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
6989 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
6991 bool nested_in_vect_loop
= false;
6992 class loop
*iv_loop
;
6994 edge pe
= loop_preheader_edge (loop
);
6996 tree new_vec
, vec_init
, vec_step
, t
;
6999 gphi
*induction_phi
;
7000 tree induc_def
, vec_dest
;
7001 tree init_expr
, step_expr
;
7002 poly_uint64 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
7006 imm_use_iterator imm_iter
;
7007 use_operand_p use_p
;
7011 gimple_stmt_iterator si
;
7013 gphi
*phi
= dyn_cast
<gphi
*> (stmt_info
->stmt
);
7017 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
7020 /* Make sure it was recognized as induction computation. */
7021 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_induction_def
)
7024 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
7025 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
7030 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
7031 gcc_assert (ncopies
>= 1);
7033 /* FORNOW. These restrictions should be relaxed. */
7034 if (nested_in_vect_loop_p (loop
, stmt_info
))
7036 imm_use_iterator imm_iter
;
7037 use_operand_p use_p
;
7044 if (dump_enabled_p ())
7045 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7046 "multiple types in nested loop.\n");
7050 /* FORNOW: outer loop induction with SLP not supported. */
7051 if (STMT_SLP_TYPE (stmt_info
))
7055 latch_e
= loop_latch_edge (loop
->inner
);
7056 loop_arg
= PHI_ARG_DEF_FROM_EDGE (phi
, latch_e
);
7057 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, loop_arg
)
7059 gimple
*use_stmt
= USE_STMT (use_p
);
7060 if (is_gimple_debug (use_stmt
))
7063 if (!flow_bb_inside_loop_p (loop
->inner
, gimple_bb (use_stmt
)))
7065 exit_phi
= use_stmt
;
7071 stmt_vec_info exit_phi_vinfo
= loop_vinfo
->lookup_stmt (exit_phi
);
7072 if (!(STMT_VINFO_RELEVANT_P (exit_phi_vinfo
)
7073 && !STMT_VINFO_LIVE_P (exit_phi_vinfo
)))
7075 if (dump_enabled_p ())
7076 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7077 "inner-loop induction only used outside "
7078 "of the outer vectorized loop.\n");
7083 nested_in_vect_loop
= true;
7084 iv_loop
= loop
->inner
;
7088 gcc_assert (iv_loop
== (gimple_bb (phi
))->loop_father
);
7090 if (slp_node
&& !nunits
.is_constant ())
7092 /* The current SLP code creates the initial value element-by-element. */
7093 if (dump_enabled_p ())
7094 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7095 "SLP induction not supported for variable-length"
7100 if (!vec_stmt
) /* transformation not required. */
7102 STMT_VINFO_TYPE (stmt_info
) = induc_vec_info_type
;
7103 DUMP_VECT_SCOPE ("vectorizable_induction");
7104 vect_model_induction_cost (stmt_info
, ncopies
, cost_vec
);
7110 /* Compute a vector variable, initialized with the first VF values of
7111 the induction variable. E.g., for an iv with IV_PHI='X' and
7112 evolution S, for a vector of 4 units, we want to compute:
7113 [X, X + S, X + 2*S, X + 3*S]. */
7115 if (dump_enabled_p ())
7116 dump_printf_loc (MSG_NOTE
, vect_location
, "transform induction phi.\n");
7118 latch_e
= loop_latch_edge (iv_loop
);
7119 loop_arg
= PHI_ARG_DEF_FROM_EDGE (phi
, latch_e
);
7121 step_expr
= STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info
);
7122 gcc_assert (step_expr
!= NULL_TREE
);
7123 tree step_vectype
= get_same_sized_vectype (TREE_TYPE (step_expr
), vectype
);
7125 pe
= loop_preheader_edge (iv_loop
);
7126 init_expr
= PHI_ARG_DEF_FROM_EDGE (phi
,
7127 loop_preheader_edge (iv_loop
));
7130 if (!nested_in_vect_loop
)
7132 /* Convert the initial value to the IV update type. */
7133 tree new_type
= TREE_TYPE (step_expr
);
7134 init_expr
= gimple_convert (&stmts
, new_type
, init_expr
);
7136 /* If we are using the loop mask to "peel" for alignment then we need
7137 to adjust the start value here. */
7138 tree skip_niters
= LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo
);
7139 if (skip_niters
!= NULL_TREE
)
7141 if (FLOAT_TYPE_P (vectype
))
7142 skip_niters
= gimple_build (&stmts
, FLOAT_EXPR
, new_type
,
7145 skip_niters
= gimple_convert (&stmts
, new_type
, skip_niters
);
7146 tree skip_step
= gimple_build (&stmts
, MULT_EXPR
, new_type
,
7147 skip_niters
, step_expr
);
7148 init_expr
= gimple_build (&stmts
, MINUS_EXPR
, new_type
,
7149 init_expr
, skip_step
);
7155 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
7156 gcc_assert (!new_bb
);
7159 /* Find the first insertion point in the BB. */
7160 basic_block bb
= gimple_bb (phi
);
7161 si
= gsi_after_labels (bb
);
7163 /* For SLP induction we have to generate several IVs as for example
7164 with group size 3 we need [i, i, i, i + S] [i + S, i + S, i + 2*S, i + 2*S]
7165 [i + 2*S, i + 3*S, i + 3*S, i + 3*S]. The step is the same uniform
7166 [VF*S, VF*S, VF*S, VF*S] for all. */
7169 /* Enforced above. */
7170 unsigned int const_nunits
= nunits
.to_constant ();
7172 /* Generate [VF*S, VF*S, ... ]. */
7173 if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr
)))
7175 expr
= build_int_cst (integer_type_node
, vf
);
7176 expr
= fold_convert (TREE_TYPE (step_expr
), expr
);
7179 expr
= build_int_cst (TREE_TYPE (step_expr
), vf
);
7180 new_name
= fold_build2 (MULT_EXPR
, TREE_TYPE (step_expr
),
7182 if (! CONSTANT_CLASS_P (new_name
))
7183 new_name
= vect_init_vector (stmt_info
, new_name
,
7184 TREE_TYPE (step_expr
), NULL
);
7185 new_vec
= build_vector_from_val (step_vectype
, new_name
);
7186 vec_step
= vect_init_vector (stmt_info
, new_vec
, step_vectype
, NULL
);
7188 /* Now generate the IVs. */
7189 unsigned group_size
= SLP_TREE_SCALAR_STMTS (slp_node
).length ();
7190 unsigned nvects
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
7191 unsigned elts
= const_nunits
* nvects
;
7192 unsigned nivs
= least_common_multiple (group_size
,
7193 const_nunits
) / const_nunits
;
7194 gcc_assert (elts
% group_size
== 0);
7195 tree elt
= init_expr
;
7197 for (ivn
= 0; ivn
< nivs
; ++ivn
)
7199 tree_vector_builder
elts (step_vectype
, const_nunits
, 1);
7201 for (unsigned eltn
= 0; eltn
< const_nunits
; ++eltn
)
7203 if (ivn
*const_nunits
+ eltn
>= group_size
7204 && (ivn
* const_nunits
+ eltn
) % group_size
== 0)
7205 elt
= gimple_build (&stmts
, PLUS_EXPR
, TREE_TYPE (elt
),
7207 elts
.quick_push (elt
);
7209 vec_init
= gimple_build_vector (&stmts
, &elts
);
7210 vec_init
= gimple_convert (&stmts
, vectype
, vec_init
);
7213 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
7214 gcc_assert (!new_bb
);
7217 /* Create the induction-phi that defines the induction-operand. */
7218 vec_dest
= vect_get_new_vect_var (vectype
, vect_simple_var
, "vec_iv_");
7219 induction_phi
= create_phi_node (vec_dest
, iv_loop
->header
);
7220 stmt_vec_info induction_phi_info
7221 = loop_vinfo
->add_stmt (induction_phi
);
7222 induc_def
= PHI_RESULT (induction_phi
);
7224 /* Create the iv update inside the loop */
7225 gimple_seq stmts
= NULL
;
7226 vec_def
= gimple_convert (&stmts
, step_vectype
, induc_def
);
7227 vec_def
= gimple_build (&stmts
,
7228 PLUS_EXPR
, step_vectype
, vec_def
, vec_step
);
7229 vec_def
= gimple_convert (&stmts
, vectype
, vec_def
);
7230 loop_vinfo
->add_stmt (SSA_NAME_DEF_STMT (vec_def
));
7231 gsi_insert_seq_before (&si
, stmts
, GSI_SAME_STMT
);
7233 /* Set the arguments of the phi node: */
7234 add_phi_arg (induction_phi
, vec_init
, pe
, UNKNOWN_LOCATION
);
7235 add_phi_arg (induction_phi
, vec_def
, loop_latch_edge (iv_loop
),
7238 SLP_TREE_VEC_STMTS (slp_node
).quick_push (induction_phi_info
);
7241 /* Re-use IVs when we can. */
7245 = least_common_multiple (group_size
, const_nunits
) / group_size
;
7246 /* Generate [VF'*S, VF'*S, ... ]. */
7247 if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr
)))
7249 expr
= build_int_cst (integer_type_node
, vfp
);
7250 expr
= fold_convert (TREE_TYPE (step_expr
), expr
);
7253 expr
= build_int_cst (TREE_TYPE (step_expr
), vfp
);
7254 new_name
= fold_build2 (MULT_EXPR
, TREE_TYPE (step_expr
),
7256 if (! CONSTANT_CLASS_P (new_name
))
7257 new_name
= vect_init_vector (stmt_info
, new_name
,
7258 TREE_TYPE (step_expr
), NULL
);
7259 new_vec
= build_vector_from_val (step_vectype
, new_name
);
7260 vec_step
= vect_init_vector (stmt_info
, new_vec
, step_vectype
, NULL
);
7261 for (; ivn
< nvects
; ++ivn
)
7263 gimple
*iv
= SLP_TREE_VEC_STMTS (slp_node
)[ivn
- nivs
]->stmt
;
7265 if (gimple_code (iv
) == GIMPLE_PHI
)
7266 def
= gimple_phi_result (iv
);
7268 def
= gimple_assign_lhs (iv
);
7269 gimple_seq stmts
= NULL
;
7270 def
= gimple_convert (&stmts
, step_vectype
, def
);
7271 def
= gimple_build (&stmts
,
7272 PLUS_EXPR
, step_vectype
, def
, vec_step
);
7273 def
= gimple_convert (&stmts
, vectype
, def
);
7274 if (gimple_code (iv
) == GIMPLE_PHI
)
7275 gsi_insert_seq_before (&si
, stmts
, GSI_SAME_STMT
);
7278 gimple_stmt_iterator tgsi
= gsi_for_stmt (iv
);
7279 gsi_insert_seq_after (&tgsi
, stmts
, GSI_CONTINUE_LINKING
);
7281 SLP_TREE_VEC_STMTS (slp_node
).quick_push
7282 (loop_vinfo
->add_stmt (SSA_NAME_DEF_STMT (def
)));
7289 /* Create the vector that holds the initial_value of the induction. */
7290 if (nested_in_vect_loop
)
7292 /* iv_loop is nested in the loop to be vectorized. init_expr had already
7293 been created during vectorization of previous stmts. We obtain it
7294 from the STMT_VINFO_VEC_STMT of the defining stmt. */
7295 vec_init
= vect_get_vec_def_for_operand (init_expr
, stmt_info
);
7296 /* If the initial value is not of proper type, convert it. */
7297 if (!useless_type_conversion_p (vectype
, TREE_TYPE (vec_init
)))
7300 = gimple_build_assign (vect_get_new_ssa_name (vectype
,
7304 build1 (VIEW_CONVERT_EXPR
, vectype
,
7306 vec_init
= gimple_assign_lhs (new_stmt
);
7307 new_bb
= gsi_insert_on_edge_immediate (loop_preheader_edge (iv_loop
),
7309 gcc_assert (!new_bb
);
7310 loop_vinfo
->add_stmt (new_stmt
);
7315 /* iv_loop is the loop to be vectorized. Create:
7316 vec_init = [X, X+S, X+2*S, X+3*S] (S = step_expr, X = init_expr) */
7318 new_name
= gimple_convert (&stmts
, TREE_TYPE (step_expr
), init_expr
);
7320 unsigned HOST_WIDE_INT const_nunits
;
7321 if (nunits
.is_constant (&const_nunits
))
7323 tree_vector_builder
elts (step_vectype
, const_nunits
, 1);
7324 elts
.quick_push (new_name
);
7325 for (i
= 1; i
< const_nunits
; i
++)
7327 /* Create: new_name_i = new_name + step_expr */
7328 new_name
= gimple_build (&stmts
, PLUS_EXPR
, TREE_TYPE (new_name
),
7329 new_name
, step_expr
);
7330 elts
.quick_push (new_name
);
7332 /* Create a vector from [new_name_0, new_name_1, ...,
7333 new_name_nunits-1] */
7334 vec_init
= gimple_build_vector (&stmts
, &elts
);
7336 else if (INTEGRAL_TYPE_P (TREE_TYPE (step_expr
)))
7337 /* Build the initial value directly from a VEC_SERIES_EXPR. */
7338 vec_init
= gimple_build (&stmts
, VEC_SERIES_EXPR
, step_vectype
,
7339 new_name
, step_expr
);
7343 [base, base, base, ...]
7344 + (vectype) [0, 1, 2, ...] * [step, step, step, ...]. */
7345 gcc_assert (SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr
)));
7346 gcc_assert (flag_associative_math
);
7347 tree index
= build_index_vector (step_vectype
, 0, 1);
7348 tree base_vec
= gimple_build_vector_from_val (&stmts
, step_vectype
,
7350 tree step_vec
= gimple_build_vector_from_val (&stmts
, step_vectype
,
7352 vec_init
= gimple_build (&stmts
, FLOAT_EXPR
, step_vectype
, index
);
7353 vec_init
= gimple_build (&stmts
, MULT_EXPR
, step_vectype
,
7354 vec_init
, step_vec
);
7355 vec_init
= gimple_build (&stmts
, PLUS_EXPR
, step_vectype
,
7356 vec_init
, base_vec
);
7358 vec_init
= gimple_convert (&stmts
, vectype
, vec_init
);
7362 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
7363 gcc_assert (!new_bb
);
7368 /* Create the vector that holds the step of the induction. */
7369 if (nested_in_vect_loop
)
7370 /* iv_loop is nested in the loop to be vectorized. Generate:
7371 vec_step = [S, S, S, S] */
7372 new_name
= step_expr
;
7375 /* iv_loop is the loop to be vectorized. Generate:
7376 vec_step = [VF*S, VF*S, VF*S, VF*S] */
7377 gimple_seq seq
= NULL
;
7378 if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr
)))
7380 expr
= build_int_cst (integer_type_node
, vf
);
7381 expr
= gimple_build (&seq
, FLOAT_EXPR
, TREE_TYPE (step_expr
), expr
);
7384 expr
= build_int_cst (TREE_TYPE (step_expr
), vf
);
7385 new_name
= gimple_build (&seq
, MULT_EXPR
, TREE_TYPE (step_expr
),
7389 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
7390 gcc_assert (!new_bb
);
7394 t
= unshare_expr (new_name
);
7395 gcc_assert (CONSTANT_CLASS_P (new_name
)
7396 || TREE_CODE (new_name
) == SSA_NAME
);
7397 new_vec
= build_vector_from_val (step_vectype
, t
);
7398 vec_step
= vect_init_vector (stmt_info
, new_vec
, step_vectype
, NULL
);
7401 /* Create the following def-use cycle:
7406 vec_iv = PHI <vec_init, vec_loop>
7410 vec_loop = vec_iv + vec_step; */
7412 /* Create the induction-phi that defines the induction-operand. */
7413 vec_dest
= vect_get_new_vect_var (vectype
, vect_simple_var
, "vec_iv_");
7414 induction_phi
= create_phi_node (vec_dest
, iv_loop
->header
);
7415 stmt_vec_info induction_phi_info
= loop_vinfo
->add_stmt (induction_phi
);
7416 induc_def
= PHI_RESULT (induction_phi
);
7418 /* Create the iv update inside the loop */
7420 vec_def
= gimple_convert (&stmts
, step_vectype
, induc_def
);
7421 vec_def
= gimple_build (&stmts
, PLUS_EXPR
, step_vectype
, vec_def
, vec_step
);
7422 vec_def
= gimple_convert (&stmts
, vectype
, vec_def
);
7423 gsi_insert_seq_before (&si
, stmts
, GSI_SAME_STMT
);
7424 new_stmt
= SSA_NAME_DEF_STMT (vec_def
);
7425 stmt_vec_info new_stmt_info
= loop_vinfo
->add_stmt (new_stmt
);
7427 /* Set the arguments of the phi node: */
7428 add_phi_arg (induction_phi
, vec_init
, pe
, UNKNOWN_LOCATION
);
7429 add_phi_arg (induction_phi
, vec_def
, loop_latch_edge (iv_loop
),
7432 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= induction_phi_info
;
7434 /* In case that vectorization factor (VF) is bigger than the number
7435 of elements that we can fit in a vectype (nunits), we have to generate
7436 more than one vector stmt - i.e - we need to "unroll" the
7437 vector stmt by a factor VF/nunits. For more details see documentation
7438 in vectorizable_operation. */
7442 gimple_seq seq
= NULL
;
7443 stmt_vec_info prev_stmt_vinfo
;
7444 /* FORNOW. This restriction should be relaxed. */
7445 gcc_assert (!nested_in_vect_loop
);
7447 /* Create the vector that holds the step of the induction. */
7448 if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr
)))
7450 expr
= build_int_cst (integer_type_node
, nunits
);
7451 expr
= gimple_build (&seq
, FLOAT_EXPR
, TREE_TYPE (step_expr
), expr
);
7454 expr
= build_int_cst (TREE_TYPE (step_expr
), nunits
);
7455 new_name
= gimple_build (&seq
, MULT_EXPR
, TREE_TYPE (step_expr
),
7459 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
7460 gcc_assert (!new_bb
);
7463 t
= unshare_expr (new_name
);
7464 gcc_assert (CONSTANT_CLASS_P (new_name
)
7465 || TREE_CODE (new_name
) == SSA_NAME
);
7466 new_vec
= build_vector_from_val (step_vectype
, t
);
7467 vec_step
= vect_init_vector (stmt_info
, new_vec
, step_vectype
, NULL
);
7469 vec_def
= induc_def
;
7470 prev_stmt_vinfo
= induction_phi_info
;
7471 for (i
= 1; i
< ncopies
; i
++)
7473 /* vec_i = vec_prev + vec_step */
7474 gimple_seq stmts
= NULL
;
7475 vec_def
= gimple_convert (&stmts
, step_vectype
, vec_def
);
7476 vec_def
= gimple_build (&stmts
,
7477 PLUS_EXPR
, step_vectype
, vec_def
, vec_step
);
7478 vec_def
= gimple_convert (&stmts
, vectype
, vec_def
);
7480 gsi_insert_seq_before (&si
, stmts
, GSI_SAME_STMT
);
7481 new_stmt
= SSA_NAME_DEF_STMT (vec_def
);
7482 new_stmt_info
= loop_vinfo
->add_stmt (new_stmt
);
7483 STMT_VINFO_RELATED_STMT (prev_stmt_vinfo
) = new_stmt_info
;
7484 prev_stmt_vinfo
= new_stmt_info
;
7488 if (nested_in_vect_loop
)
7490 /* Find the loop-closed exit-phi of the induction, and record
7491 the final vector of induction results: */
7493 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, loop_arg
)
7495 gimple
*use_stmt
= USE_STMT (use_p
);
7496 if (is_gimple_debug (use_stmt
))
7499 if (!flow_bb_inside_loop_p (iv_loop
, gimple_bb (use_stmt
)))
7501 exit_phi
= use_stmt
;
7507 stmt_vec_info stmt_vinfo
= loop_vinfo
->lookup_stmt (exit_phi
);
7508 /* FORNOW. Currently not supporting the case that an inner-loop induction
7509 is not used in the outer-loop (i.e. only outside the outer-loop). */
7510 gcc_assert (STMT_VINFO_RELEVANT_P (stmt_vinfo
)
7511 && !STMT_VINFO_LIVE_P (stmt_vinfo
));
7513 STMT_VINFO_VEC_STMT (stmt_vinfo
) = new_stmt_info
;
7514 if (dump_enabled_p ())
7515 dump_printf_loc (MSG_NOTE
, vect_location
,
7516 "vector of inductions after inner-loop:%G",
7522 if (dump_enabled_p ())
7523 dump_printf_loc (MSG_NOTE
, vect_location
,
7524 "transform induction: created def-use cycle: %G%G",
7525 induction_phi
, SSA_NAME_DEF_STMT (vec_def
));
7530 /* Function vectorizable_live_operation.
7532 STMT_INFO computes a value that is used outside the loop. Check if
7533 it can be supported. */
7536 vectorizable_live_operation (stmt_vec_info stmt_info
,
7537 gimple_stmt_iterator
*gsi
,
7538 slp_tree slp_node
, slp_instance slp_node_instance
,
7539 int slp_index
, bool vec_stmt_p
,
7540 stmt_vector_for_cost
*)
7542 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
7543 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
7544 imm_use_iterator imm_iter
;
7545 tree lhs
, lhs_type
, bitsize
, vec_bitsize
;
7546 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
7547 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
7550 auto_vec
<tree
> vec_oprnds
;
7552 poly_uint64 vec_index
= 0;
7554 gcc_assert (STMT_VINFO_LIVE_P (stmt_info
));
7556 /* If a stmt of a reduction is live, vectorize it via
7557 vect_create_epilog_for_reduction. vectorizable_reduction assessed
7558 validity so just trigger the transform here. */
7559 if (STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info
)))
7565 /* For reduction chains the meta-info is attached to
7566 the group leader. */
7567 if (REDUC_GROUP_FIRST_ELEMENT (stmt_info
))
7568 stmt_info
= REDUC_GROUP_FIRST_ELEMENT (stmt_info
);
7569 /* For SLP reductions we vectorize the epilogue for
7570 all involved stmts together. */
7571 else if (slp_index
!= 0)
7574 stmt_vec_info reduc_info
= info_for_reduction (stmt_info
);
7575 gcc_assert (reduc_info
->is_reduc_info
);
7576 if (STMT_VINFO_REDUC_TYPE (reduc_info
) == FOLD_LEFT_REDUCTION
7577 || STMT_VINFO_REDUC_TYPE (reduc_info
) == EXTRACT_LAST_REDUCTION
)
7579 vect_create_epilog_for_reduction (stmt_info
, slp_node
,
7584 /* FORNOW. CHECKME. */
7585 if (nested_in_vect_loop_p (loop
, stmt_info
))
7588 /* If STMT is not relevant and it is a simple assignment and its inputs are
7589 invariant then it can remain in place, unvectorized. The original last
7590 scalar value that it computes will be used. */
7591 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
7593 gcc_assert (is_simple_and_all_uses_invariant (stmt_info
, loop_vinfo
));
7594 if (dump_enabled_p ())
7595 dump_printf_loc (MSG_NOTE
, vect_location
,
7596 "statement is simple and uses invariant. Leaving in "
7604 ncopies
= vect_get_num_copies (loop_vinfo
, vectype
);
7608 gcc_assert (slp_index
>= 0);
7610 int num_scalar
= SLP_TREE_SCALAR_STMTS (slp_node
).length ();
7611 int num_vec
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
7613 /* Get the last occurrence of the scalar index from the concatenation of
7614 all the slp vectors. Calculate which slp vector it is and the index
7616 poly_uint64 pos
= (num_vec
* nunits
) - num_scalar
+ slp_index
;
7618 /* Calculate which vector contains the result, and which lane of
7619 that vector we need. */
7620 if (!can_div_trunc_p (pos
, nunits
, &vec_entry
, &vec_index
))
7622 if (dump_enabled_p ())
7623 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7624 "Cannot determine which vector holds the"
7625 " final result.\n");
7632 /* No transformation required. */
7633 if (LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo
))
7635 if (!direct_internal_fn_supported_p (IFN_EXTRACT_LAST
, vectype
,
7636 OPTIMIZE_FOR_SPEED
))
7638 if (dump_enabled_p ())
7639 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7640 "can't use a fully-masked loop because "
7641 "the target doesn't support extract last "
7643 LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo
) = false;
7647 if (dump_enabled_p ())
7648 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7649 "can't use a fully-masked loop because an "
7650 "SLP statement is live after the loop.\n");
7651 LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo
) = false;
7653 else if (ncopies
> 1)
7655 if (dump_enabled_p ())
7656 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7657 "can't use a fully-masked loop because"
7658 " ncopies is greater than 1.\n");
7659 LOOP_VINFO_CAN_FULLY_MASK_P (loop_vinfo
) = false;
7663 gcc_assert (ncopies
== 1 && !slp_node
);
7664 vect_record_loop_mask (loop_vinfo
,
7665 &LOOP_VINFO_MASKS (loop_vinfo
),
7672 /* Use the lhs of the original scalar statement. */
7673 gimple
*stmt
= vect_orig_stmt (stmt_info
)->stmt
;
7675 lhs
= (is_a
<gphi
*> (stmt
)) ? gimple_phi_result (stmt
)
7676 : gimple_get_lhs (stmt
);
7677 lhs_type
= TREE_TYPE (lhs
);
7679 bitsize
= (VECTOR_BOOLEAN_TYPE_P (vectype
)
7680 ? bitsize_int (TYPE_PRECISION (TREE_TYPE (vectype
)))
7681 : TYPE_SIZE (TREE_TYPE (vectype
)));
7682 vec_bitsize
= TYPE_SIZE (vectype
);
7684 /* Get the vectorized lhs of STMT and the lane to use (counted in bits). */
7685 tree vec_lhs
, bitstart
;
7688 gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
));
7690 /* Get the correct slp vectorized stmt. */
7691 gimple
*vec_stmt
= SLP_TREE_VEC_STMTS (slp_node
)[vec_entry
]->stmt
;
7692 if (gphi
*phi
= dyn_cast
<gphi
*> (vec_stmt
))
7693 vec_lhs
= gimple_phi_result (phi
);
7695 vec_lhs
= gimple_get_lhs (vec_stmt
);
7697 /* Get entry to use. */
7698 bitstart
= bitsize_int (vec_index
);
7699 bitstart
= int_const_binop (MULT_EXPR
, bitsize
, bitstart
);
7703 enum vect_def_type dt
= STMT_VINFO_DEF_TYPE (stmt_info
);
7704 vec_lhs
= vect_get_vec_def_for_operand_1 (stmt_info
, dt
);
7705 gcc_checking_assert (ncopies
== 1
7706 || !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
));
7708 /* For multiple copies, get the last copy. */
7709 for (int i
= 1; i
< ncopies
; ++i
)
7710 vec_lhs
= vect_get_vec_def_for_stmt_copy (loop_vinfo
, vec_lhs
);
7712 /* Get the last lane in the vector. */
7713 bitstart
= int_const_binop (MINUS_EXPR
, vec_bitsize
, bitsize
);
7716 gimple_seq stmts
= NULL
;
7718 if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
7722 SCALAR_RES = EXTRACT_LAST <VEC_LHS, MASK>
7724 where VEC_LHS is the vectorized live-out result and MASK is
7725 the loop mask for the final iteration. */
7726 gcc_assert (ncopies
== 1 && !slp_node
);
7727 tree scalar_type
= TREE_TYPE (STMT_VINFO_VECTYPE (stmt_info
));
7728 tree mask
= vect_get_loop_mask (gsi
, &LOOP_VINFO_MASKS (loop_vinfo
),
7730 tree scalar_res
= gimple_build (&stmts
, CFN_EXTRACT_LAST
,
7731 scalar_type
, mask
, vec_lhs
);
7733 /* Convert the extracted vector element to the required scalar type. */
7734 new_tree
= gimple_convert (&stmts
, lhs_type
, scalar_res
);
7738 tree bftype
= TREE_TYPE (vectype
);
7739 if (VECTOR_BOOLEAN_TYPE_P (vectype
))
7740 bftype
= build_nonstandard_integer_type (tree_to_uhwi (bitsize
), 1);
7741 new_tree
= build3 (BIT_FIELD_REF
, bftype
, vec_lhs
, bitsize
, bitstart
);
7742 new_tree
= force_gimple_operand (fold_convert (lhs_type
, new_tree
),
7743 &stmts
, true, NULL_TREE
);
7747 gsi_insert_seq_on_edge_immediate (single_exit (loop
), stmts
);
7749 /* Replace use of lhs with newly computed result. If the use stmt is a
7750 single arg PHI, just replace all uses of PHI result. It's necessary
7751 because lcssa PHI defining lhs may be before newly inserted stmt. */
7752 use_operand_p use_p
;
7753 FOR_EACH_IMM_USE_STMT (use_stmt
, imm_iter
, lhs
)
7754 if (!flow_bb_inside_loop_p (loop
, gimple_bb (use_stmt
))
7755 && !is_gimple_debug (use_stmt
))
7757 if (gimple_code (use_stmt
) == GIMPLE_PHI
7758 && gimple_phi_num_args (use_stmt
) == 1)
7760 replace_uses_by (gimple_phi_result (use_stmt
), new_tree
);
7764 FOR_EACH_IMM_USE_ON_STMT (use_p
, imm_iter
)
7765 SET_USE (use_p
, new_tree
);
7767 update_stmt (use_stmt
);
7773 /* Kill any debug uses outside LOOP of SSA names defined in STMT_INFO. */
7776 vect_loop_kill_debug_uses (class loop
*loop
, stmt_vec_info stmt_info
)
7778 ssa_op_iter op_iter
;
7779 imm_use_iterator imm_iter
;
7780 def_operand_p def_p
;
7783 FOR_EACH_PHI_OR_STMT_DEF (def_p
, stmt_info
->stmt
, op_iter
, SSA_OP_DEF
)
7785 FOR_EACH_IMM_USE_STMT (ustmt
, imm_iter
, DEF_FROM_PTR (def_p
))
7789 if (!is_gimple_debug (ustmt
))
7792 bb
= gimple_bb (ustmt
);
7794 if (!flow_bb_inside_loop_p (loop
, bb
))
7796 if (gimple_debug_bind_p (ustmt
))
7798 if (dump_enabled_p ())
7799 dump_printf_loc (MSG_NOTE
, vect_location
,
7800 "killing debug use\n");
7802 gimple_debug_bind_reset_value (ustmt
);
7803 update_stmt (ustmt
);
7812 /* Given loop represented by LOOP_VINFO, return true if computation of
7813 LOOP_VINFO_NITERS (= LOOP_VINFO_NITERSM1 + 1) doesn't overflow, false
7817 loop_niters_no_overflow (loop_vec_info loop_vinfo
)
7819 /* Constant case. */
7820 if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
))
7822 tree cst_niters
= LOOP_VINFO_NITERS (loop_vinfo
);
7823 tree cst_nitersm1
= LOOP_VINFO_NITERSM1 (loop_vinfo
);
7825 gcc_assert (TREE_CODE (cst_niters
) == INTEGER_CST
);
7826 gcc_assert (TREE_CODE (cst_nitersm1
) == INTEGER_CST
);
7827 if (wi::to_widest (cst_nitersm1
) < wi::to_widest (cst_niters
))
7832 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
7833 /* Check the upper bound of loop niters. */
7834 if (get_max_loop_iterations (loop
, &max
))
7836 tree type
= TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo
));
7837 signop sgn
= TYPE_SIGN (type
);
7838 widest_int type_max
= widest_int::from (wi::max_value (type
), sgn
);
7845 /* Return a mask type with half the number of elements as TYPE. */
7848 vect_halve_mask_nunits (vec_info
*vinfo
, tree type
)
7850 poly_uint64 nunits
= exact_div (TYPE_VECTOR_SUBPARTS (type
), 2);
7851 return build_truth_vector_type (nunits
, vinfo
->vector_size
);
7854 /* Return a mask type with twice as many elements as TYPE. */
7857 vect_double_mask_nunits (vec_info
*vinfo
, tree type
)
7859 poly_uint64 nunits
= TYPE_VECTOR_SUBPARTS (type
) * 2;
7860 return build_truth_vector_type (nunits
, vinfo
->vector_size
);
7863 /* Record that a fully-masked version of LOOP_VINFO would need MASKS to
7864 contain a sequence of NVECTORS masks that each control a vector of type
7865 VECTYPE. If SCALAR_MASK is nonnull, the fully-masked loop would AND
7866 these vector masks with the vector version of SCALAR_MASK. */
7869 vect_record_loop_mask (loop_vec_info loop_vinfo
, vec_loop_masks
*masks
,
7870 unsigned int nvectors
, tree vectype
, tree scalar_mask
)
7872 gcc_assert (nvectors
!= 0);
7873 if (masks
->length () < nvectors
)
7874 masks
->safe_grow_cleared (nvectors
);
7875 rgroup_masks
*rgm
= &(*masks
)[nvectors
- 1];
7876 /* The number of scalars per iteration and the number of vectors are
7877 both compile-time constants. */
7878 unsigned int nscalars_per_iter
7879 = exact_div (nvectors
* TYPE_VECTOR_SUBPARTS (vectype
),
7880 LOOP_VINFO_VECT_FACTOR (loop_vinfo
)).to_constant ();
7884 scalar_cond_masked_key
cond (scalar_mask
, nvectors
);
7885 loop_vinfo
->scalar_cond_masked_set
.add (cond
);
7888 if (rgm
->max_nscalars_per_iter
< nscalars_per_iter
)
7890 rgm
->max_nscalars_per_iter
= nscalars_per_iter
;
7891 rgm
->mask_type
= build_same_sized_truth_vector_type (vectype
);
7895 /* Given a complete set of masks MASKS, extract mask number INDEX
7896 for an rgroup that operates on NVECTORS vectors of type VECTYPE,
7897 where 0 <= INDEX < NVECTORS. Insert any set-up statements before GSI.
7899 See the comment above vec_loop_masks for more details about the mask
7903 vect_get_loop_mask (gimple_stmt_iterator
*gsi
, vec_loop_masks
*masks
,
7904 unsigned int nvectors
, tree vectype
, unsigned int index
)
7906 rgroup_masks
*rgm
= &(*masks
)[nvectors
- 1];
7907 tree mask_type
= rgm
->mask_type
;
7909 /* Populate the rgroup's mask array, if this is the first time we've
7911 if (rgm
->masks
.is_empty ())
7913 rgm
->masks
.safe_grow_cleared (nvectors
);
7914 for (unsigned int i
= 0; i
< nvectors
; ++i
)
7916 tree mask
= make_temp_ssa_name (mask_type
, NULL
, "loop_mask");
7917 /* Provide a dummy definition until the real one is available. */
7918 SSA_NAME_DEF_STMT (mask
) = gimple_build_nop ();
7919 rgm
->masks
[i
] = mask
;
7923 tree mask
= rgm
->masks
[index
];
7924 if (maybe_ne (TYPE_VECTOR_SUBPARTS (mask_type
),
7925 TYPE_VECTOR_SUBPARTS (vectype
)))
7927 /* A loop mask for data type X can be reused for data type Y
7928 if X has N times more elements than Y and if Y's elements
7929 are N times bigger than X's. In this case each sequence
7930 of N elements in the loop mask will be all-zero or all-one.
7931 We can then view-convert the mask so that each sequence of
7932 N elements is replaced by a single element. */
7933 gcc_assert (multiple_p (TYPE_VECTOR_SUBPARTS (mask_type
),
7934 TYPE_VECTOR_SUBPARTS (vectype
)));
7935 gimple_seq seq
= NULL
;
7936 mask_type
= build_same_sized_truth_vector_type (vectype
);
7937 mask
= gimple_build (&seq
, VIEW_CONVERT_EXPR
, mask_type
, mask
);
7939 gsi_insert_seq_before (gsi
, seq
, GSI_SAME_STMT
);
7944 /* Scale profiling counters by estimation for LOOP which is vectorized
7948 scale_profile_for_vect_loop (class loop
*loop
, unsigned vf
)
7950 edge preheader
= loop_preheader_edge (loop
);
7951 /* Reduce loop iterations by the vectorization factor. */
7952 gcov_type new_est_niter
= niter_for_unrolled_loop (loop
, vf
);
7953 profile_count freq_h
= loop
->header
->count
, freq_e
= preheader
->count ();
7955 if (freq_h
.nonzero_p ())
7957 profile_probability p
;
7959 /* Avoid dropping loop body profile counter to 0 because of zero count
7960 in loop's preheader. */
7961 if (!(freq_e
== profile_count::zero ()))
7962 freq_e
= freq_e
.force_nonzero ();
7963 p
= freq_e
.apply_scale (new_est_niter
+ 1, 1).probability_in (freq_h
);
7964 scale_loop_frequencies (loop
, p
);
7967 edge exit_e
= single_exit (loop
);
7968 exit_e
->probability
= profile_probability::always ()
7969 .apply_scale (1, new_est_niter
+ 1);
7971 edge exit_l
= single_pred_edge (loop
->latch
);
7972 profile_probability prob
= exit_l
->probability
;
7973 exit_l
->probability
= exit_e
->probability
.invert ();
7974 if (prob
.initialized_p () && exit_l
->probability
.initialized_p ())
7975 scale_bbs_frequencies (&loop
->latch
, 1, exit_l
->probability
/ prob
);
7978 /* Vectorize STMT_INFO if relevant, inserting any new instructions before GSI.
7979 When vectorizing STMT_INFO as a store, set *SEEN_STORE to its
7983 vect_transform_loop_stmt (loop_vec_info loop_vinfo
, stmt_vec_info stmt_info
,
7984 gimple_stmt_iterator
*gsi
, stmt_vec_info
*seen_store
)
7986 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
7987 poly_uint64 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
7989 if (dump_enabled_p ())
7990 dump_printf_loc (MSG_NOTE
, vect_location
,
7991 "------>vectorizing statement: %G", stmt_info
->stmt
);
7993 if (MAY_HAVE_DEBUG_BIND_STMTS
&& !STMT_VINFO_LIVE_P (stmt_info
))
7994 vect_loop_kill_debug_uses (loop
, stmt_info
);
7996 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
7997 && !STMT_VINFO_LIVE_P (stmt_info
))
8000 if (STMT_VINFO_VECTYPE (stmt_info
))
8003 = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info
));
8004 if (!STMT_SLP_TYPE (stmt_info
)
8005 && maybe_ne (nunits
, vf
)
8006 && dump_enabled_p ())
8007 /* For SLP VF is set according to unrolling factor, and not
8008 to vector size, hence for SLP this print is not valid. */
8009 dump_printf_loc (MSG_NOTE
, vect_location
, "multiple-types.\n");
8012 /* Pure SLP statements have already been vectorized. We still need
8013 to apply loop vectorization to hybrid SLP statements. */
8014 if (PURE_SLP_STMT (stmt_info
))
8017 if (dump_enabled_p ())
8018 dump_printf_loc (MSG_NOTE
, vect_location
, "transform statement.\n");
8020 if (vect_transform_stmt (stmt_info
, gsi
, NULL
, NULL
))
8021 *seen_store
= stmt_info
;
8024 /* Helper function to pass to simplify_replace_tree to enable replacing tree's
8025 in the hash_map with its corresponding values. */
8028 find_in_mapping (tree t
, void *context
)
8030 hash_map
<tree
,tree
>* mapping
= (hash_map
<tree
, tree
>*) context
;
8032 tree
*value
= mapping
->get (t
);
8033 return value
? *value
: t
;
8036 /* Update EPILOGUE's loop_vec_info. EPILOGUE was constructed as a copy of the
8037 original loop that has now been vectorized.
8039 The inits of the data_references need to be advanced with the number of
8040 iterations of the main loop. This has been computed in vect_do_peeling and
8041 is stored in parameter ADVANCE. We first restore the data_references
8042 initial offset with the values recored in ORIG_DRS_INIT.
8044 Since the loop_vec_info of this EPILOGUE was constructed for the original
8045 loop, its stmt_vec_infos all point to the original statements. These need
8046 to be updated to point to their corresponding copies as well as the SSA_NAMES
8047 in their PATTERN_DEF_SEQs and RELATED_STMTs.
8049 The data_reference's connections also need to be updated. Their
8050 corresponding dr_vec_info need to be reconnected to the EPILOGUE's
8051 stmt_vec_infos, their statements need to point to their corresponding copy,
8052 if they are gather loads or scatter stores then their reference needs to be
8053 updated to point to its corresponding copy and finally we set
8054 'base_misaligned' to false as we have already peeled for alignment in the
8055 prologue of the main loop. */
8058 update_epilogue_loop_vinfo (class loop
*epilogue
, tree advance
,
8059 drs_init_vec
&orig_drs_init
)
8061 loop_vec_info epilogue_vinfo
= loop_vec_info_for_loop (epilogue
);
8062 auto_vec
<gimple
*> stmt_worklist
;
8063 hash_map
<tree
,tree
> mapping
;
8064 gimple
*orig_stmt
, *new_stmt
;
8065 gimple_stmt_iterator epilogue_gsi
;
8066 gphi_iterator epilogue_phi_gsi
;
8067 stmt_vec_info stmt_vinfo
= NULL
, related_vinfo
;
8068 basic_block
*epilogue_bbs
= get_loop_body (epilogue
);
8070 LOOP_VINFO_BBS (epilogue_vinfo
) = epilogue_bbs
;
8072 /* Restore original data_reference's offset, before the previous loop and its
8074 std::pair
<data_reference
*, tree
> *dr_init
;
8076 for (i
= 0; orig_drs_init
.iterate (i
, &dr_init
); i
++)
8077 DR_OFFSET (dr_init
->first
) = dr_init
->second
;
8079 /* Advance data_reference's with the number of iterations of the previous
8080 loop and its prologue. */
8081 vect_update_inits_of_drs (epilogue_vinfo
, advance
, PLUS_EXPR
);
8084 /* The EPILOGUE loop is a copy of the original loop so they share the same
8085 gimple UIDs. In this loop we update the loop_vec_info of the EPILOGUE to
8086 point to the copied statements. We also create a mapping of all LHS' in
8087 the original loop and all the LHS' in the EPILOGUE and create worklists to
8088 update teh STMT_VINFO_PATTERN_DEF_SEQs and STMT_VINFO_RELATED_STMTs. */
8089 for (unsigned i
= 0; i
< epilogue
->num_nodes
; ++i
)
8091 for (epilogue_phi_gsi
= gsi_start_phis (epilogue_bbs
[i
]);
8092 !gsi_end_p (epilogue_phi_gsi
); gsi_next (&epilogue_phi_gsi
))
8094 new_stmt
= epilogue_phi_gsi
.phi ();
8096 gcc_assert (gimple_uid (new_stmt
) > 0);
8098 = epilogue_vinfo
->stmt_vec_infos
[gimple_uid (new_stmt
) - 1];
8100 orig_stmt
= STMT_VINFO_STMT (stmt_vinfo
);
8101 STMT_VINFO_STMT (stmt_vinfo
) = new_stmt
;
8103 mapping
.put (gimple_phi_result (orig_stmt
),
8104 gimple_phi_result (new_stmt
));
8105 /* PHI nodes can not have patterns or related statements. */
8106 gcc_assert (STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo
) == NULL
8107 && STMT_VINFO_RELATED_STMT (stmt_vinfo
) == NULL
);
8110 for (epilogue_gsi
= gsi_start_bb (epilogue_bbs
[i
]);
8111 !gsi_end_p (epilogue_gsi
); gsi_next (&epilogue_gsi
))
8113 new_stmt
= gsi_stmt (epilogue_gsi
);
8115 gcc_assert (gimple_uid (new_stmt
) > 0);
8117 = epilogue_vinfo
->stmt_vec_infos
[gimple_uid (new_stmt
) - 1];
8119 orig_stmt
= STMT_VINFO_STMT (stmt_vinfo
);
8120 STMT_VINFO_STMT (stmt_vinfo
) = new_stmt
;
8122 if (tree old_lhs
= gimple_get_lhs (orig_stmt
))
8123 mapping
.put (old_lhs
, gimple_get_lhs (new_stmt
));
8125 if (STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo
))
8127 gimple_seq seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_vinfo
);
8128 for (gimple_stmt_iterator gsi
= gsi_start (seq
);
8129 !gsi_end_p (gsi
); gsi_next (&gsi
))
8130 stmt_worklist
.safe_push (gsi_stmt (gsi
));
8133 related_vinfo
= STMT_VINFO_RELATED_STMT (stmt_vinfo
);
8134 if (related_vinfo
!= NULL
&& related_vinfo
!= stmt_vinfo
)
8136 gimple
*stmt
= STMT_VINFO_STMT (related_vinfo
);
8137 stmt_worklist
.safe_push (stmt
);
8138 /* Set BB such that the assert in
8139 'get_initial_def_for_reduction' is able to determine that
8140 the BB of the related stmt is inside this loop. */
8141 gimple_set_bb (stmt
,
8142 gimple_bb (new_stmt
));
8143 related_vinfo
= STMT_VINFO_RELATED_STMT (related_vinfo
);
8144 gcc_assert (related_vinfo
== NULL
8145 || related_vinfo
== stmt_vinfo
);
8150 /* The PATTERN_DEF_SEQs and RELATED_STMTs in the epilogue were constructed
8151 using the original main loop and thus need to be updated to refer to the
8152 cloned variables used in the epilogue. */
8153 for (unsigned i
= 0; i
< stmt_worklist
.length (); ++i
)
8155 gimple
*stmt
= stmt_worklist
[i
];
8158 for (unsigned j
= 1; j
< gimple_num_ops (stmt
); ++j
)
8160 tree op
= gimple_op (stmt
, j
);
8161 if ((new_op
= mapping
.get(op
)))
8162 gimple_set_op (stmt
, j
, *new_op
);
8165 op
= simplify_replace_tree (op
, NULL_TREE
, NULL_TREE
,
8166 &find_in_mapping
, &mapping
);
8167 gimple_set_op (stmt
, j
, op
);
8172 struct data_reference
*dr
;
8173 vec
<data_reference_p
> datarefs
= epilogue_vinfo
->shared
->datarefs
;
8174 FOR_EACH_VEC_ELT (datarefs
, i
, dr
)
8176 orig_stmt
= DR_STMT (dr
);
8177 gcc_assert (gimple_uid (orig_stmt
) > 0);
8178 stmt_vinfo
= epilogue_vinfo
->stmt_vec_infos
[gimple_uid (orig_stmt
) - 1];
8179 /* Data references for gather loads and scatter stores do not use the
8180 updated offset we set using ADVANCE. Instead we have to make sure the
8181 reference in the data references point to the corresponding copy of
8182 the original in the epilogue. */
8183 if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo
))
8186 = simplify_replace_tree (DR_REF (dr
), NULL_TREE
, NULL_TREE
,
8187 &find_in_mapping
, &mapping
);
8188 DR_BASE_ADDRESS (dr
)
8189 = simplify_replace_tree (DR_BASE_ADDRESS (dr
), NULL_TREE
, NULL_TREE
,
8190 &find_in_mapping
, &mapping
);
8192 DR_STMT (dr
) = STMT_VINFO_STMT (stmt_vinfo
);
8193 stmt_vinfo
->dr_aux
.stmt
= stmt_vinfo
;
8194 /* The vector size of the epilogue is smaller than that of the main loop
8195 so the alignment is either the same or lower. This means the dr will
8196 thus by definition be aligned. */
8197 STMT_VINFO_DR_INFO (stmt_vinfo
)->base_misaligned
= false;
8200 epilogue_vinfo
->shared
->datarefs_copy
.release ();
8201 epilogue_vinfo
->shared
->save_datarefs ();
8204 /* Function vect_transform_loop.
8206 The analysis phase has determined that the loop is vectorizable.
8207 Vectorize the loop - created vectorized stmts to replace the scalar
8208 stmts in the loop, and update the loop exit condition.
8209 Returns scalar epilogue loop if any. */
8212 vect_transform_loop (loop_vec_info loop_vinfo
)
8214 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
8215 class loop
*epilogue
= NULL
;
8216 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
8217 int nbbs
= loop
->num_nodes
;
8219 tree niters_vector
= NULL_TREE
;
8220 tree step_vector
= NULL_TREE
;
8221 tree niters_vector_mult_vf
= NULL_TREE
;
8222 poly_uint64 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
8223 unsigned int lowest_vf
= constant_lower_bound (vf
);
8225 bool check_profitability
= false;
8228 DUMP_VECT_SCOPE ("vec_transform_loop");
8230 loop_vinfo
->shared
->check_datarefs ();
8232 /* Use the more conservative vectorization threshold. If the number
8233 of iterations is constant assume the cost check has been performed
8234 by our caller. If the threshold makes all loops profitable that
8235 run at least the (estimated) vectorization factor number of times
8236 checking is pointless, too. */
8237 th
= LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo
);
8238 if (th
>= vect_vf_for_cost (loop_vinfo
)
8239 && !LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
))
8241 if (dump_enabled_p ())
8242 dump_printf_loc (MSG_NOTE
, vect_location
,
8243 "Profitability threshold is %d loop iterations.\n",
8245 check_profitability
= true;
8248 /* Make sure there exists a single-predecessor exit bb. Do this before
8250 edge e
= single_exit (loop
);
8251 if (! single_pred_p (e
->dest
))
8253 split_loop_exit_edge (e
, true);
8254 if (dump_enabled_p ())
8255 dump_printf (MSG_NOTE
, "split exit edge\n");
8258 /* Version the loop first, if required, so the profitability check
8261 if (LOOP_REQUIRES_VERSIONING (loop_vinfo
))
8264 = vect_loop_versioning (loop_vinfo
);
8265 sloop
->force_vectorize
= false;
8266 check_profitability
= false;
8269 /* Make sure there exists a single-predecessor exit bb also on the
8270 scalar loop copy. Do this after versioning but before peeling
8271 so CFG structure is fine for both scalar and if-converted loop
8272 to make slpeel_duplicate_current_defs_from_edges face matched
8273 loop closed PHI nodes on the exit. */
8274 if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo
))
8276 e
= single_exit (LOOP_VINFO_SCALAR_LOOP (loop_vinfo
));
8277 if (! single_pred_p (e
->dest
))
8279 split_loop_exit_edge (e
, true);
8280 if (dump_enabled_p ())
8281 dump_printf (MSG_NOTE
, "split exit edge of scalar loop\n");
8285 tree niters
= vect_build_loop_niters (loop_vinfo
);
8286 LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo
) = niters
;
8287 tree nitersm1
= unshare_expr (LOOP_VINFO_NITERSM1 (loop_vinfo
));
8288 bool niters_no_overflow
= loop_niters_no_overflow (loop_vinfo
);
8290 drs_init_vec orig_drs_init
;
8292 epilogue
= vect_do_peeling (loop_vinfo
, niters
, nitersm1
, &niters_vector
,
8293 &step_vector
, &niters_vector_mult_vf
, th
,
8294 check_profitability
, niters_no_overflow
,
8295 &advance
, orig_drs_init
);
8297 if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo
)
8298 && LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo
).initialized_p ())
8299 scale_loop_frequencies (LOOP_VINFO_SCALAR_LOOP (loop_vinfo
),
8300 LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo
));
8302 if (niters_vector
== NULL_TREE
)
8304 if (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo
)
8305 && !LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
)
8306 && known_eq (lowest_vf
, vf
))
8309 = build_int_cst (TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo
)),
8310 LOOP_VINFO_INT_NITERS (loop_vinfo
) / lowest_vf
);
8311 step_vector
= build_one_cst (TREE_TYPE (niters
));
8314 vect_gen_vector_loop_niters (loop_vinfo
, niters
, &niters_vector
,
8315 &step_vector
, niters_no_overflow
);
8318 /* 1) Make sure the loop header has exactly two entries
8319 2) Make sure we have a preheader basic block. */
8321 gcc_assert (EDGE_COUNT (loop
->header
->preds
) == 2);
8323 split_edge (loop_preheader_edge (loop
));
8325 if (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
)
8326 && vect_use_loop_mask_for_alignment_p (loop_vinfo
))
8327 /* This will deal with any possible peeling. */
8328 vect_prepare_for_masked_peels (loop_vinfo
);
8330 /* Schedule the SLP instances first, then handle loop vectorization
8332 if (!loop_vinfo
->slp_instances
.is_empty ())
8334 DUMP_VECT_SCOPE ("scheduling SLP instances");
8335 vect_schedule_slp (loop_vinfo
);
8338 /* FORNOW: the vectorizer supports only loops which body consist
8339 of one basic block (header + empty latch). When the vectorizer will
8340 support more involved loop forms, the order by which the BBs are
8341 traversed need to be reconsidered. */
8343 for (i
= 0; i
< nbbs
; i
++)
8345 basic_block bb
= bbs
[i
];
8346 stmt_vec_info stmt_info
;
8348 for (gphi_iterator si
= gsi_start_phis (bb
); !gsi_end_p (si
);
8351 gphi
*phi
= si
.phi ();
8352 if (dump_enabled_p ())
8353 dump_printf_loc (MSG_NOTE
, vect_location
,
8354 "------>vectorizing phi: %G", phi
);
8355 stmt_info
= loop_vinfo
->lookup_stmt (phi
);
8359 if (MAY_HAVE_DEBUG_BIND_STMTS
&& !STMT_VINFO_LIVE_P (stmt_info
))
8360 vect_loop_kill_debug_uses (loop
, stmt_info
);
8362 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
8363 && !STMT_VINFO_LIVE_P (stmt_info
))
8366 if (STMT_VINFO_VECTYPE (stmt_info
)
8368 (TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info
)), vf
))
8369 && dump_enabled_p ())
8370 dump_printf_loc (MSG_NOTE
, vect_location
, "multiple-types.\n");
8372 if ((STMT_VINFO_DEF_TYPE (stmt_info
) == vect_induction_def
8373 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
8374 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_double_reduction_def
8375 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
8376 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_internal_def
)
8377 && ! PURE_SLP_STMT (stmt_info
))
8379 if (dump_enabled_p ())
8380 dump_printf_loc (MSG_NOTE
, vect_location
, "transform phi.\n");
8381 vect_transform_stmt (stmt_info
, NULL
, NULL
, NULL
);
8385 for (gimple_stmt_iterator si
= gsi_start_bb (bb
);
8388 stmt
= gsi_stmt (si
);
8389 /* During vectorization remove existing clobber stmts. */
8390 if (gimple_clobber_p (stmt
))
8392 unlink_stmt_vdef (stmt
);
8393 gsi_remove (&si
, true);
8394 release_defs (stmt
);
8398 stmt_info
= loop_vinfo
->lookup_stmt (stmt
);
8400 /* vector stmts created in the outer-loop during vectorization of
8401 stmts in an inner-loop may not have a stmt_info, and do not
8402 need to be vectorized. */
8403 stmt_vec_info seen_store
= NULL
;
8406 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
8408 gimple
*def_seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
);
8409 for (gimple_stmt_iterator subsi
= gsi_start (def_seq
);
8410 !gsi_end_p (subsi
); gsi_next (&subsi
))
8412 stmt_vec_info pat_stmt_info
8413 = loop_vinfo
->lookup_stmt (gsi_stmt (subsi
));
8414 vect_transform_loop_stmt (loop_vinfo
, pat_stmt_info
,
8417 stmt_vec_info pat_stmt_info
8418 = STMT_VINFO_RELATED_STMT (stmt_info
);
8419 vect_transform_loop_stmt (loop_vinfo
, pat_stmt_info
, &si
,
8422 vect_transform_loop_stmt (loop_vinfo
, stmt_info
, &si
,
8428 if (STMT_VINFO_GROUPED_ACCESS (seen_store
))
8429 /* Interleaving. If IS_STORE is TRUE, the
8430 vectorization of the interleaving chain was
8431 completed - free all the stores in the chain. */
8432 vect_remove_stores (DR_GROUP_FIRST_ELEMENT (seen_store
));
8434 /* Free the attached stmt_vec_info and remove the stmt. */
8435 loop_vinfo
->remove_stmt (stmt_info
);
8440 /* Stub out scalar statements that must not survive vectorization.
8441 Doing this here helps with grouped statements, or statements that
8442 are involved in patterns. */
8443 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
);
8444 !gsi_end_p (gsi
); gsi_next (&gsi
))
8446 gcall
*call
= dyn_cast
<gcall
*> (gsi_stmt (gsi
));
8447 if (call
&& gimple_call_internal_p (call
, IFN_MASK_LOAD
))
8449 tree lhs
= gimple_get_lhs (call
);
8450 if (!VECTOR_TYPE_P (TREE_TYPE (lhs
)))
8452 tree zero
= build_zero_cst (TREE_TYPE (lhs
));
8453 gimple
*new_stmt
= gimple_build_assign (lhs
, zero
);
8454 gsi_replace (&gsi
, new_stmt
, true);
8460 /* The vectorization factor is always > 1, so if we use an IV increment of 1.
8461 a zero NITERS becomes a nonzero NITERS_VECTOR. */
8462 if (integer_onep (step_vector
))
8463 niters_no_overflow
= true;
8464 vect_set_loop_condition (loop
, loop_vinfo
, niters_vector
, step_vector
,
8465 niters_vector_mult_vf
, !niters_no_overflow
);
8467 unsigned int assumed_vf
= vect_vf_for_cost (loop_vinfo
);
8468 scale_profile_for_vect_loop (loop
, assumed_vf
);
8470 /* True if the final iteration might not handle a full vector's
8471 worth of scalar iterations. */
8472 bool final_iter_may_be_partial
= LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
);
8473 /* The minimum number of iterations performed by the epilogue. This
8474 is 1 when peeling for gaps because we always need a final scalar
8476 int min_epilogue_iters
= LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo
) ? 1 : 0;
8477 /* +1 to convert latch counts to loop iteration counts,
8478 -min_epilogue_iters to remove iterations that cannot be performed
8479 by the vector code. */
8480 int bias_for_lowest
= 1 - min_epilogue_iters
;
8481 int bias_for_assumed
= bias_for_lowest
;
8482 int alignment_npeels
= LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
);
8483 if (alignment_npeels
&& LOOP_VINFO_FULLY_MASKED_P (loop_vinfo
))
8485 /* When the amount of peeling is known at compile time, the first
8486 iteration will have exactly alignment_npeels active elements.
8487 In the worst case it will have at least one. */
8488 int min_first_active
= (alignment_npeels
> 0 ? alignment_npeels
: 1);
8489 bias_for_lowest
+= lowest_vf
- min_first_active
;
8490 bias_for_assumed
+= assumed_vf
- min_first_active
;
8492 /* In these calculations the "- 1" converts loop iteration counts
8493 back to latch counts. */
8494 if (loop
->any_upper_bound
)
8495 loop
->nb_iterations_upper_bound
8496 = (final_iter_may_be_partial
8497 ? wi::udiv_ceil (loop
->nb_iterations_upper_bound
+ bias_for_lowest
,
8499 : wi::udiv_floor (loop
->nb_iterations_upper_bound
+ bias_for_lowest
,
8501 if (loop
->any_likely_upper_bound
)
8502 loop
->nb_iterations_likely_upper_bound
8503 = (final_iter_may_be_partial
8504 ? wi::udiv_ceil (loop
->nb_iterations_likely_upper_bound
8505 + bias_for_lowest
, lowest_vf
) - 1
8506 : wi::udiv_floor (loop
->nb_iterations_likely_upper_bound
8507 + bias_for_lowest
, lowest_vf
) - 1);
8508 if (loop
->any_estimate
)
8509 loop
->nb_iterations_estimate
8510 = (final_iter_may_be_partial
8511 ? wi::udiv_ceil (loop
->nb_iterations_estimate
+ bias_for_assumed
,
8513 : wi::udiv_floor (loop
->nb_iterations_estimate
+ bias_for_assumed
,
8516 if (dump_enabled_p ())
8518 if (!LOOP_VINFO_EPILOGUE_P (loop_vinfo
))
8520 dump_printf_loc (MSG_NOTE
, vect_location
,
8521 "LOOP VECTORIZED\n");
8523 dump_printf_loc (MSG_NOTE
, vect_location
,
8524 "OUTER LOOP VECTORIZED\n");
8525 dump_printf (MSG_NOTE
, "\n");
8529 dump_printf_loc (MSG_NOTE
, vect_location
,
8530 "LOOP EPILOGUE VECTORIZED (VS=");
8531 dump_dec (MSG_NOTE
, loop_vinfo
->vector_size
);
8532 dump_printf (MSG_NOTE
, ")\n");
8536 /* Loops vectorized with a variable factor won't benefit from
8537 unrolling/peeling. */
8538 if (!vf
.is_constant ())
8541 if (dump_enabled_p ())
8542 dump_printf_loc (MSG_NOTE
, vect_location
, "Disabling unrolling due to"
8543 " variable-length vectorization factor\n");
8545 /* Free SLP instances here because otherwise stmt reference counting
8547 slp_instance instance
;
8548 FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo
), i
, instance
)
8549 vect_free_slp_instance (instance
, true);
8550 LOOP_VINFO_SLP_INSTANCES (loop_vinfo
).release ();
8551 /* Clear-up safelen field since its value is invalid after vectorization
8552 since vectorized loop can have loop-carried dependencies. */
8557 update_epilogue_loop_vinfo (epilogue
, advance
, orig_drs_init
);
8559 epilogue
->simduid
= loop
->simduid
;
8560 epilogue
->force_vectorize
= loop
->force_vectorize
;
8561 epilogue
->safelen
= loop
->safelen
;
8562 epilogue
->dont_vectorize
= false;
8568 /* The code below is trying to perform simple optimization - revert
8569 if-conversion for masked stores, i.e. if the mask of a store is zero
8570 do not perform it and all stored value producers also if possible.
8578 this transformation will produce the following semi-hammock:
8580 if (!mask__ifc__42.18_165 == { 0, 0, 0, 0, 0, 0, 0, 0 })
8582 vect__11.19_170 = MASK_LOAD (vectp_p1.20_168, 0B, mask__ifc__42.18_165);
8583 vect__12.22_172 = vect__11.19_170 + vect_cst__171;
8584 MASK_STORE (vectp_p1.23_175, 0B, mask__ifc__42.18_165, vect__12.22_172);
8585 vect__18.25_182 = MASK_LOAD (vectp_p3.26_180, 0B, mask__ifc__42.18_165);
8586 vect__19.28_184 = vect__18.25_182 + vect_cst__183;
8587 MASK_STORE (vectp_p2.29_187, 0B, mask__ifc__42.18_165, vect__19.28_184);
8592 optimize_mask_stores (class loop
*loop
)
8594 basic_block
*bbs
= get_loop_body (loop
);
8595 unsigned nbbs
= loop
->num_nodes
;
8598 class loop
*bb_loop
;
8599 gimple_stmt_iterator gsi
;
8601 auto_vec
<gimple
*> worklist
;
8602 auto_purge_vect_location sentinel
;
8604 vect_location
= find_loop_location (loop
);
8605 /* Pick up all masked stores in loop if any. */
8606 for (i
= 0; i
< nbbs
; i
++)
8609 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);
8612 stmt
= gsi_stmt (gsi
);
8613 if (gimple_call_internal_p (stmt
, IFN_MASK_STORE
))
8614 worklist
.safe_push (stmt
);
8619 if (worklist
.is_empty ())
8622 /* Loop has masked stores. */
8623 while (!worklist
.is_empty ())
8625 gimple
*last
, *last_store
;
8628 basic_block store_bb
, join_bb
;
8629 gimple_stmt_iterator gsi_to
;
8630 tree vdef
, new_vdef
;
8635 last
= worklist
.pop ();
8636 mask
= gimple_call_arg (last
, 2);
8637 bb
= gimple_bb (last
);
8638 /* Create then_bb and if-then structure in CFG, then_bb belongs to
8639 the same loop as if_bb. It could be different to LOOP when two
8640 level loop-nest is vectorized and mask_store belongs to the inner
8642 e
= split_block (bb
, last
);
8643 bb_loop
= bb
->loop_father
;
8644 gcc_assert (loop
== bb_loop
|| flow_loop_nested_p (loop
, bb_loop
));
8646 store_bb
= create_empty_bb (bb
);
8647 add_bb_to_loop (store_bb
, bb_loop
);
8648 e
->flags
= EDGE_TRUE_VALUE
;
8649 efalse
= make_edge (bb
, store_bb
, EDGE_FALSE_VALUE
);
8650 /* Put STORE_BB to likely part. */
8651 efalse
->probability
= profile_probability::unlikely ();
8652 store_bb
->count
= efalse
->count ();
8653 make_single_succ_edge (store_bb
, join_bb
, EDGE_FALLTHRU
);
8654 if (dom_info_available_p (CDI_DOMINATORS
))
8655 set_immediate_dominator (CDI_DOMINATORS
, store_bb
, bb
);
8656 if (dump_enabled_p ())
8657 dump_printf_loc (MSG_NOTE
, vect_location
,
8658 "Create new block %d to sink mask stores.",
8660 /* Create vector comparison with boolean result. */
8661 vectype
= TREE_TYPE (mask
);
8662 zero
= build_zero_cst (vectype
);
8663 stmt
= gimple_build_cond (EQ_EXPR
, mask
, zero
, NULL_TREE
, NULL_TREE
);
8664 gsi
= gsi_last_bb (bb
);
8665 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8666 /* Create new PHI node for vdef of the last masked store:
8667 .MEM_2 = VDEF <.MEM_1>
8668 will be converted to
8669 .MEM.3 = VDEF <.MEM_1>
8670 and new PHI node will be created in join bb
8671 .MEM_2 = PHI <.MEM_1, .MEM_3>
8673 vdef
= gimple_vdef (last
);
8674 new_vdef
= make_ssa_name (gimple_vop (cfun
), last
);
8675 gimple_set_vdef (last
, new_vdef
);
8676 phi
= create_phi_node (vdef
, join_bb
);
8677 add_phi_arg (phi
, new_vdef
, EDGE_SUCC (store_bb
, 0), UNKNOWN_LOCATION
);
8679 /* Put all masked stores with the same mask to STORE_BB if possible. */
8682 gimple_stmt_iterator gsi_from
;
8683 gimple
*stmt1
= NULL
;
8685 /* Move masked store to STORE_BB. */
8687 gsi
= gsi_for_stmt (last
);
8689 /* Shift GSI to the previous stmt for further traversal. */
8691 gsi_to
= gsi_start_bb (store_bb
);
8692 gsi_move_before (&gsi_from
, &gsi_to
);
8693 /* Setup GSI_TO to the non-empty block start. */
8694 gsi_to
= gsi_start_bb (store_bb
);
8695 if (dump_enabled_p ())
8696 dump_printf_loc (MSG_NOTE
, vect_location
,
8697 "Move stmt to created bb\n%G", last
);
8698 /* Move all stored value producers if possible. */
8699 while (!gsi_end_p (gsi
))
8702 imm_use_iterator imm_iter
;
8703 use_operand_p use_p
;
8706 /* Skip debug statements. */
8707 if (is_gimple_debug (gsi_stmt (gsi
)))
8712 stmt1
= gsi_stmt (gsi
);
8713 /* Do not consider statements writing to memory or having
8714 volatile operand. */
8715 if (gimple_vdef (stmt1
)
8716 || gimple_has_volatile_ops (stmt1
))
8720 lhs
= gimple_get_lhs (stmt1
);
8724 /* LHS of vectorized stmt must be SSA_NAME. */
8725 if (TREE_CODE (lhs
) != SSA_NAME
)
8728 if (!VECTOR_TYPE_P (TREE_TYPE (lhs
)))
8730 /* Remove dead scalar statement. */
8731 if (has_zero_uses (lhs
))
8733 gsi_remove (&gsi_from
, true);
8738 /* Check that LHS does not have uses outside of STORE_BB. */
8740 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, lhs
)
8743 use_stmt
= USE_STMT (use_p
);
8744 if (is_gimple_debug (use_stmt
))
8746 if (gimple_bb (use_stmt
) != store_bb
)
8755 if (gimple_vuse (stmt1
)
8756 && gimple_vuse (stmt1
) != gimple_vuse (last_store
))
8759 /* Can move STMT1 to STORE_BB. */
8760 if (dump_enabled_p ())
8761 dump_printf_loc (MSG_NOTE
, vect_location
,
8762 "Move stmt to created bb\n%G", stmt1
);
8763 gsi_move_before (&gsi_from
, &gsi_to
);
8764 /* Shift GSI_TO for further insertion. */
8767 /* Put other masked stores with the same mask to STORE_BB. */
8768 if (worklist
.is_empty ()
8769 || gimple_call_arg (worklist
.last (), 2) != mask
8770 || worklist
.last () != stmt1
)
8772 last
= worklist
.pop ();
8774 add_phi_arg (phi
, gimple_vuse (last_store
), e
, UNKNOWN_LOCATION
);
8778 /* Decide whether it is possible to use a zero-based induction variable
8779 when vectorizing LOOP_VINFO with a fully-masked loop. If it is,
8780 return the value that the induction variable must be able to hold
8781 in order to ensure that the loop ends with an all-false mask.
8782 Return -1 otherwise. */
8784 vect_iv_limit_for_full_masking (loop_vec_info loop_vinfo
)
8786 tree niters_skip
= LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo
);
8787 class loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
8788 unsigned HOST_WIDE_INT max_vf
= vect_max_vf (loop_vinfo
);
8790 /* Calculate the value that the induction variable must be able
8791 to hit in order to ensure that we end the loop with an all-false mask.
8792 This involves adding the maximum number of inactive trailing scalar
8794 widest_int iv_limit
= -1;
8795 if (max_loop_iterations (loop
, &iv_limit
))
8799 /* Add the maximum number of skipped iterations to the
8800 maximum iteration count. */
8801 if (TREE_CODE (niters_skip
) == INTEGER_CST
)
8802 iv_limit
+= wi::to_widest (niters_skip
);
8804 iv_limit
+= max_vf
- 1;
8806 else if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo
))
8807 /* Make a conservatively-correct assumption. */
8808 iv_limit
+= max_vf
- 1;
8810 /* IV_LIMIT is the maximum number of latch iterations, which is also
8811 the maximum in-range IV value. Round this value down to the previous
8812 vector alignment boundary and then add an extra full iteration. */
8813 poly_uint64 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
8814 iv_limit
= (iv_limit
& -(int) known_alignment (vf
)) + max_vf
;