1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2015 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
32 #include "fold-const.h"
33 #include "stor-layout.h"
35 #include "gimple-pretty-print.h"
36 #include "internal-fn.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
42 #include "tree-ssa-loop-manip.h"
44 #include "tree-ssa-loop.h"
45 #include "tree-scalar-evolution.h"
47 #include "insn-config.h"
56 #include "recog.h" /* FIXME: for insn_data */
57 #include "insn-codes.h"
59 #include "diagnostic-core.h"
60 #include "tree-vectorizer.h"
64 /* For lang_hooks.types.type_for_mode. */
65 #include "langhooks.h"
67 /* Return the vectorized type for the given statement. */
70 stmt_vectype (struct _stmt_vec_info
*stmt_info
)
72 return STMT_VINFO_VECTYPE (stmt_info
);
75 /* Return TRUE iff the given statement is in an inner loop relative to
76 the loop being vectorized. */
78 stmt_in_inner_loop_p (struct _stmt_vec_info
*stmt_info
)
80 gimple stmt
= STMT_VINFO_STMT (stmt_info
);
81 basic_block bb
= gimple_bb (stmt
);
82 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
88 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
90 return (bb
->loop_father
== loop
->inner
);
93 /* Record the cost of a statement, either by directly informing the
94 target model or by saving it in a vector for later processing.
95 Return a preliminary estimate of the statement's cost. */
98 record_stmt_cost (stmt_vector_for_cost
*body_cost_vec
, int count
,
99 enum vect_cost_for_stmt kind
, stmt_vec_info stmt_info
,
100 int misalign
, enum vect_cost_model_location where
)
104 tree vectype
= stmt_info
? stmt_vectype (stmt_info
) : NULL_TREE
;
105 add_stmt_info_to_vec (body_cost_vec
, count
, kind
,
106 stmt_info
? STMT_VINFO_STMT (stmt_info
) : NULL
,
109 (builtin_vectorization_cost (kind
, vectype
, misalign
) * count
);
114 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
115 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
116 void *target_cost_data
;
119 target_cost_data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
121 target_cost_data
= BB_VINFO_TARGET_COST_DATA (bb_vinfo
);
123 return add_stmt_cost (target_cost_data
, count
, kind
, stmt_info
,
128 /* Return a variable of type ELEM_TYPE[NELEMS]. */
131 create_vector_array (tree elem_type
, unsigned HOST_WIDE_INT nelems
)
133 return create_tmp_var (build_array_type_nelts (elem_type
, nelems
),
137 /* ARRAY is an array of vectors created by create_vector_array.
138 Return an SSA_NAME for the vector in index N. The reference
139 is part of the vectorization of STMT and the vector is associated
140 with scalar destination SCALAR_DEST. */
143 read_vector_array (gimple stmt
, gimple_stmt_iterator
*gsi
, tree scalar_dest
,
144 tree array
, unsigned HOST_WIDE_INT n
)
146 tree vect_type
, vect
, vect_name
, array_ref
;
149 gcc_assert (TREE_CODE (TREE_TYPE (array
)) == ARRAY_TYPE
);
150 vect_type
= TREE_TYPE (TREE_TYPE (array
));
151 vect
= vect_create_destination_var (scalar_dest
, vect_type
);
152 array_ref
= build4 (ARRAY_REF
, vect_type
, array
,
153 build_int_cst (size_type_node
, n
),
154 NULL_TREE
, NULL_TREE
);
156 new_stmt
= gimple_build_assign (vect
, array_ref
);
157 vect_name
= make_ssa_name (vect
, new_stmt
);
158 gimple_assign_set_lhs (new_stmt
, vect_name
);
159 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
164 /* ARRAY is an array of vectors created by create_vector_array.
165 Emit code to store SSA_NAME VECT in index N of the array.
166 The store is part of the vectorization of STMT. */
169 write_vector_array (gimple stmt
, gimple_stmt_iterator
*gsi
, tree vect
,
170 tree array
, unsigned HOST_WIDE_INT n
)
175 array_ref
= build4 (ARRAY_REF
, TREE_TYPE (vect
), array
,
176 build_int_cst (size_type_node
, n
),
177 NULL_TREE
, NULL_TREE
);
179 new_stmt
= gimple_build_assign (array_ref
, vect
);
180 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
183 /* PTR is a pointer to an array of type TYPE. Return a representation
184 of *PTR. The memory reference replaces those in FIRST_DR
188 create_array_ref (tree type
, tree ptr
, struct data_reference
*first_dr
)
190 tree mem_ref
, alias_ptr_type
;
192 alias_ptr_type
= reference_alias_ptr_type (DR_REF (first_dr
));
193 mem_ref
= build2 (MEM_REF
, type
, ptr
, build_int_cst (alias_ptr_type
, 0));
194 /* Arrays have the same alignment as their type. */
195 set_ptr_info_alignment (get_ptr_info (ptr
), TYPE_ALIGN_UNIT (type
), 0);
199 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
201 /* Function vect_mark_relevant.
203 Mark STMT as "relevant for vectorization" and add it to WORKLIST. */
206 vect_mark_relevant (vec
<gimple
> *worklist
, gimple stmt
,
207 enum vect_relevant relevant
, bool live_p
,
208 bool used_in_pattern
)
210 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
211 enum vect_relevant save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
212 bool save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
215 if (dump_enabled_p ())
216 dump_printf_loc (MSG_NOTE
, vect_location
,
217 "mark relevant %d, live %d.\n", relevant
, live_p
);
219 /* If this stmt is an original stmt in a pattern, we might need to mark its
220 related pattern stmt instead of the original stmt. However, such stmts
221 may have their own uses that are not in any pattern, in such cases the
222 stmt itself should be marked. */
223 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
226 if (!used_in_pattern
)
228 imm_use_iterator imm_iter
;
232 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
233 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
235 if (is_gimple_assign (stmt
))
236 lhs
= gimple_assign_lhs (stmt
);
238 lhs
= gimple_call_lhs (stmt
);
240 /* This use is out of pattern use, if LHS has other uses that are
241 pattern uses, we should mark the stmt itself, and not the pattern
243 if (lhs
&& TREE_CODE (lhs
) == SSA_NAME
)
244 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, lhs
)
246 if (is_gimple_debug (USE_STMT (use_p
)))
248 use_stmt
= USE_STMT (use_p
);
250 if (!flow_bb_inside_loop_p (loop
, gimple_bb (use_stmt
)))
253 if (vinfo_for_stmt (use_stmt
)
254 && STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (use_stmt
)))
264 /* This is the last stmt in a sequence that was detected as a
265 pattern that can potentially be vectorized. Don't mark the stmt
266 as relevant/live because it's not going to be vectorized.
267 Instead mark the pattern-stmt that replaces it. */
269 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
271 if (dump_enabled_p ())
272 dump_printf_loc (MSG_NOTE
, vect_location
,
273 "last stmt in pattern. don't mark"
274 " relevant/live.\n");
275 stmt_info
= vinfo_for_stmt (pattern_stmt
);
276 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info
) == stmt
);
277 save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
278 save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
283 STMT_VINFO_LIVE_P (stmt_info
) |= live_p
;
284 if (relevant
> STMT_VINFO_RELEVANT (stmt_info
))
285 STMT_VINFO_RELEVANT (stmt_info
) = relevant
;
287 if (STMT_VINFO_RELEVANT (stmt_info
) == save_relevant
288 && STMT_VINFO_LIVE_P (stmt_info
) == save_live_p
)
290 if (dump_enabled_p ())
291 dump_printf_loc (MSG_NOTE
, vect_location
,
292 "already marked relevant/live.\n");
296 worklist
->safe_push (stmt
);
300 /* Function vect_stmt_relevant_p.
302 Return true if STMT in loop that is represented by LOOP_VINFO is
303 "relevant for vectorization".
305 A stmt is considered "relevant for vectorization" if:
306 - it has uses outside the loop.
307 - it has vdefs (it alters memory).
308 - control stmts in the loop (except for the exit condition).
310 CHECKME: what other side effects would the vectorizer allow? */
313 vect_stmt_relevant_p (gimple stmt
, loop_vec_info loop_vinfo
,
314 enum vect_relevant
*relevant
, bool *live_p
)
316 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
318 imm_use_iterator imm_iter
;
322 *relevant
= vect_unused_in_scope
;
325 /* cond stmt other than loop exit cond. */
326 if (is_ctrl_stmt (stmt
)
327 && STMT_VINFO_TYPE (vinfo_for_stmt (stmt
))
328 != loop_exit_ctrl_vec_info_type
)
329 *relevant
= vect_used_in_scope
;
331 /* changing memory. */
332 if (gimple_code (stmt
) != GIMPLE_PHI
)
333 if (gimple_vdef (stmt
)
334 && !gimple_clobber_p (stmt
))
336 if (dump_enabled_p ())
337 dump_printf_loc (MSG_NOTE
, vect_location
,
338 "vec_stmt_relevant_p: stmt has vdefs.\n");
339 *relevant
= vect_used_in_scope
;
342 /* uses outside the loop. */
343 FOR_EACH_PHI_OR_STMT_DEF (def_p
, stmt
, op_iter
, SSA_OP_DEF
)
345 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, DEF_FROM_PTR (def_p
))
347 basic_block bb
= gimple_bb (USE_STMT (use_p
));
348 if (!flow_bb_inside_loop_p (loop
, bb
))
350 if (dump_enabled_p ())
351 dump_printf_loc (MSG_NOTE
, vect_location
,
352 "vec_stmt_relevant_p: used out of loop.\n");
354 if (is_gimple_debug (USE_STMT (use_p
)))
357 /* We expect all such uses to be in the loop exit phis
358 (because of loop closed form) */
359 gcc_assert (gimple_code (USE_STMT (use_p
)) == GIMPLE_PHI
);
360 gcc_assert (bb
== single_exit (loop
)->dest
);
367 return (*live_p
|| *relevant
);
371 /* Function exist_non_indexing_operands_for_use_p
373 USE is one of the uses attached to STMT. Check if USE is
374 used in STMT for anything other than indexing an array. */
377 exist_non_indexing_operands_for_use_p (tree use
, gimple stmt
)
380 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
382 /* USE corresponds to some operand in STMT. If there is no data
383 reference in STMT, then any operand that corresponds to USE
384 is not indexing an array. */
385 if (!STMT_VINFO_DATA_REF (stmt_info
))
388 /* STMT has a data_ref. FORNOW this means that its of one of
392 (This should have been verified in analyze_data_refs).
394 'var' in the second case corresponds to a def, not a use,
395 so USE cannot correspond to any operands that are not used
398 Therefore, all we need to check is if STMT falls into the
399 first case, and whether var corresponds to USE. */
401 if (!gimple_assign_copy_p (stmt
))
403 if (is_gimple_call (stmt
)
404 && gimple_call_internal_p (stmt
))
405 switch (gimple_call_internal_fn (stmt
))
408 operand
= gimple_call_arg (stmt
, 3);
413 operand
= gimple_call_arg (stmt
, 2);
423 if (TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
425 operand
= gimple_assign_rhs1 (stmt
);
426 if (TREE_CODE (operand
) != SSA_NAME
)
437 Function process_use.
440 - a USE in STMT in a loop represented by LOOP_VINFO
441 - LIVE_P, RELEVANT - enum values to be set in the STMT_VINFO of the stmt
442 that defined USE. This is done by calling mark_relevant and passing it
443 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
444 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
448 Generally, LIVE_P and RELEVANT are used to define the liveness and
449 relevance info of the DEF_STMT of this USE:
450 STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
451 STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
453 - case 1: If USE is used only for address computations (e.g. array indexing),
454 which does not need to be directly vectorized, then the liveness/relevance
455 of the respective DEF_STMT is left unchanged.
456 - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we
457 skip DEF_STMT cause it had already been processed.
458 - case 3: If DEF_STMT and STMT are in different nests, then "relevant" will
459 be modified accordingly.
461 Return true if everything is as expected. Return false otherwise. */
464 process_use (gimple stmt
, tree use
, loop_vec_info loop_vinfo
, bool live_p
,
465 enum vect_relevant relevant
, vec
<gimple
> *worklist
,
468 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
469 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
470 stmt_vec_info dstmt_vinfo
;
471 basic_block bb
, def_bb
;
474 enum vect_def_type dt
;
476 /* case 1: we are only interested in uses that need to be vectorized. Uses
477 that are used for address computation are not considered relevant. */
478 if (!force
&& !exist_non_indexing_operands_for_use_p (use
, stmt
))
481 if (!vect_is_simple_use (use
, stmt
, loop_vinfo
, NULL
, &def_stmt
, &def
, &dt
))
483 if (dump_enabled_p ())
484 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
485 "not vectorized: unsupported use in stmt.\n");
489 if (!def_stmt
|| gimple_nop_p (def_stmt
))
492 def_bb
= gimple_bb (def_stmt
);
493 if (!flow_bb_inside_loop_p (loop
, def_bb
))
495 if (dump_enabled_p ())
496 dump_printf_loc (MSG_NOTE
, vect_location
, "def_stmt is out of loop.\n");
500 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DEF_STMT).
501 DEF_STMT must have already been processed, because this should be the
502 only way that STMT, which is a reduction-phi, was put in the worklist,
503 as there should be no other uses for DEF_STMT in the loop. So we just
504 check that everything is as expected, and we are done. */
505 dstmt_vinfo
= vinfo_for_stmt (def_stmt
);
506 bb
= gimple_bb (stmt
);
507 if (gimple_code (stmt
) == GIMPLE_PHI
508 && STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
509 && gimple_code (def_stmt
) != GIMPLE_PHI
510 && STMT_VINFO_DEF_TYPE (dstmt_vinfo
) == vect_reduction_def
511 && bb
->loop_father
== def_bb
->loop_father
)
513 if (dump_enabled_p ())
514 dump_printf_loc (MSG_NOTE
, vect_location
,
515 "reduc-stmt defining reduc-phi in the same nest.\n");
516 if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo
))
517 dstmt_vinfo
= vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo
));
518 gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo
) < vect_used_by_reduction
);
519 gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo
)
520 || STMT_VINFO_RELEVANT (dstmt_vinfo
) > vect_unused_in_scope
);
524 /* case 3a: outer-loop stmt defining an inner-loop stmt:
525 outer-loop-header-bb:
531 if (flow_loop_nested_p (def_bb
->loop_father
, bb
->loop_father
))
533 if (dump_enabled_p ())
534 dump_printf_loc (MSG_NOTE
, vect_location
,
535 "outer-loop def-stmt defining inner-loop stmt.\n");
539 case vect_unused_in_scope
:
540 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_nested_cycle
) ?
541 vect_used_in_scope
: vect_unused_in_scope
;
544 case vect_used_in_outer_by_reduction
:
545 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
546 relevant
= vect_used_by_reduction
;
549 case vect_used_in_outer
:
550 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
551 relevant
= vect_used_in_scope
;
554 case vect_used_in_scope
:
562 /* case 3b: inner-loop stmt defining an outer-loop stmt:
563 outer-loop-header-bb:
567 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
569 else if (flow_loop_nested_p (bb
->loop_father
, def_bb
->loop_father
))
571 if (dump_enabled_p ())
572 dump_printf_loc (MSG_NOTE
, vect_location
,
573 "inner-loop def-stmt defining outer-loop stmt.\n");
577 case vect_unused_in_scope
:
578 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
579 || STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_double_reduction_def
) ?
580 vect_used_in_outer_by_reduction
: vect_unused_in_scope
;
583 case vect_used_by_reduction
:
584 relevant
= vect_used_in_outer_by_reduction
;
587 case vect_used_in_scope
:
588 relevant
= vect_used_in_outer
;
596 vect_mark_relevant (worklist
, def_stmt
, relevant
, live_p
,
597 is_pattern_stmt_p (stmt_vinfo
));
602 /* Function vect_mark_stmts_to_be_vectorized.
604 Not all stmts in the loop need to be vectorized. For example:
613 Stmt 1 and 3 do not need to be vectorized, because loop control and
614 addressing of vectorized data-refs are handled differently.
616 This pass detects such stmts. */
619 vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo
)
621 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
622 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
623 unsigned int nbbs
= loop
->num_nodes
;
624 gimple_stmt_iterator si
;
627 stmt_vec_info stmt_vinfo
;
631 enum vect_relevant relevant
, tmp_relevant
;
632 enum vect_def_type def_type
;
634 if (dump_enabled_p ())
635 dump_printf_loc (MSG_NOTE
, vect_location
,
636 "=== vect_mark_stmts_to_be_vectorized ===\n");
638 auto_vec
<gimple
, 64> worklist
;
640 /* 1. Init worklist. */
641 for (i
= 0; i
< nbbs
; i
++)
644 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
647 if (dump_enabled_p ())
649 dump_printf_loc (MSG_NOTE
, vect_location
, "init: phi relevant? ");
650 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, phi
, 0);
653 if (vect_stmt_relevant_p (phi
, loop_vinfo
, &relevant
, &live_p
))
654 vect_mark_relevant (&worklist
, phi
, relevant
, live_p
, false);
656 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
658 stmt
= gsi_stmt (si
);
659 if (dump_enabled_p ())
661 dump_printf_loc (MSG_NOTE
, vect_location
, "init: stmt relevant? ");
662 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
665 if (vect_stmt_relevant_p (stmt
, loop_vinfo
, &relevant
, &live_p
))
666 vect_mark_relevant (&worklist
, stmt
, relevant
, live_p
, false);
670 /* 2. Process_worklist */
671 while (worklist
.length () > 0)
676 stmt
= worklist
.pop ();
677 if (dump_enabled_p ())
679 dump_printf_loc (MSG_NOTE
, vect_location
, "worklist: examine stmt: ");
680 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
683 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
684 (DEF_STMT) as relevant/irrelevant and live/dead according to the
685 liveness and relevance properties of STMT. */
686 stmt_vinfo
= vinfo_for_stmt (stmt
);
687 relevant
= STMT_VINFO_RELEVANT (stmt_vinfo
);
688 live_p
= STMT_VINFO_LIVE_P (stmt_vinfo
);
690 /* Generally, the liveness and relevance properties of STMT are
691 propagated as is to the DEF_STMTs of its USEs:
692 live_p <-- STMT_VINFO_LIVE_P (STMT_VINFO)
693 relevant <-- STMT_VINFO_RELEVANT (STMT_VINFO)
695 One exception is when STMT has been identified as defining a reduction
696 variable; in this case we set the liveness/relevance as follows:
698 relevant = vect_used_by_reduction
699 This is because we distinguish between two kinds of relevant stmts -
700 those that are used by a reduction computation, and those that are
701 (also) used by a regular computation. This allows us later on to
702 identify stmts that are used solely by a reduction, and therefore the
703 order of the results that they produce does not have to be kept. */
705 def_type
= STMT_VINFO_DEF_TYPE (stmt_vinfo
);
706 tmp_relevant
= relevant
;
709 case vect_reduction_def
:
710 switch (tmp_relevant
)
712 case vect_unused_in_scope
:
713 relevant
= vect_used_by_reduction
;
716 case vect_used_by_reduction
:
717 if (gimple_code (stmt
) == GIMPLE_PHI
)
722 if (dump_enabled_p ())
723 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
724 "unsupported use of reduction.\n");
731 case vect_nested_cycle
:
732 if (tmp_relevant
!= vect_unused_in_scope
733 && tmp_relevant
!= vect_used_in_outer_by_reduction
734 && tmp_relevant
!= vect_used_in_outer
)
736 if (dump_enabled_p ())
737 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
738 "unsupported use of nested cycle.\n");
746 case vect_double_reduction_def
:
747 if (tmp_relevant
!= vect_unused_in_scope
748 && tmp_relevant
!= vect_used_by_reduction
)
750 if (dump_enabled_p ())
751 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
752 "unsupported use of double reduction.\n");
764 if (is_pattern_stmt_p (stmt_vinfo
))
766 /* Pattern statements are not inserted into the code, so
767 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
768 have to scan the RHS or function arguments instead. */
769 if (is_gimple_assign (stmt
))
771 enum tree_code rhs_code
= gimple_assign_rhs_code (stmt
);
772 tree op
= gimple_assign_rhs1 (stmt
);
775 if (rhs_code
== COND_EXPR
&& COMPARISON_CLASS_P (op
))
777 if (!process_use (stmt
, TREE_OPERAND (op
, 0), loop_vinfo
,
778 live_p
, relevant
, &worklist
, false)
779 || !process_use (stmt
, TREE_OPERAND (op
, 1), loop_vinfo
,
780 live_p
, relevant
, &worklist
, false))
784 for (; i
< gimple_num_ops (stmt
); i
++)
786 op
= gimple_op (stmt
, i
);
787 if (TREE_CODE (op
) == SSA_NAME
788 && !process_use (stmt
, op
, loop_vinfo
, live_p
, relevant
,
793 else if (is_gimple_call (stmt
))
795 for (i
= 0; i
< gimple_call_num_args (stmt
); i
++)
797 tree arg
= gimple_call_arg (stmt
, i
);
798 if (!process_use (stmt
, arg
, loop_vinfo
, live_p
, relevant
,
805 FOR_EACH_PHI_OR_STMT_USE (use_p
, stmt
, iter
, SSA_OP_USE
)
807 tree op
= USE_FROM_PTR (use_p
);
808 if (!process_use (stmt
, op
, loop_vinfo
, live_p
, relevant
,
813 if (STMT_VINFO_GATHER_P (stmt_vinfo
))
816 tree decl
= vect_check_gather (stmt
, loop_vinfo
, NULL
, &off
, NULL
);
818 if (!process_use (stmt
, off
, loop_vinfo
, live_p
, relevant
,
822 } /* while worklist */
828 /* Function vect_model_simple_cost.
830 Models cost for simple operations, i.e. those that only emit ncopies of a
831 single op. Right now, this does not account for multiple insns that could
832 be generated for the single vector op. We will handle that shortly. */
835 vect_model_simple_cost (stmt_vec_info stmt_info
, int ncopies
,
836 enum vect_def_type
*dt
,
837 stmt_vector_for_cost
*prologue_cost_vec
,
838 stmt_vector_for_cost
*body_cost_vec
)
841 int inside_cost
= 0, prologue_cost
= 0;
843 /* The SLP costs were already calculated during SLP tree build. */
844 if (PURE_SLP_STMT (stmt_info
))
847 /* FORNOW: Assuming maximum 2 args per stmts. */
848 for (i
= 0; i
< 2; i
++)
849 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
850 prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1, vector_stmt
,
851 stmt_info
, 0, vect_prologue
);
853 /* Pass the inside-of-loop statements to the target-specific cost model. */
854 inside_cost
= record_stmt_cost (body_cost_vec
, ncopies
, vector_stmt
,
855 stmt_info
, 0, vect_body
);
857 if (dump_enabled_p ())
858 dump_printf_loc (MSG_NOTE
, vect_location
,
859 "vect_model_simple_cost: inside_cost = %d, "
860 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
864 /* Model cost for type demotion and promotion operations. PWR is normally
865 zero for single-step promotions and demotions. It will be one if
866 two-step promotion/demotion is required, and so on. Each additional
867 step doubles the number of instructions required. */
870 vect_model_promotion_demotion_cost (stmt_vec_info stmt_info
,
871 enum vect_def_type
*dt
, int pwr
)
874 int inside_cost
= 0, prologue_cost
= 0;
875 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
876 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
877 void *target_cost_data
;
879 /* The SLP costs were already calculated during SLP tree build. */
880 if (PURE_SLP_STMT (stmt_info
))
884 target_cost_data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
886 target_cost_data
= BB_VINFO_TARGET_COST_DATA (bb_vinfo
);
888 for (i
= 0; i
< pwr
+ 1; i
++)
890 tmp
= (STMT_VINFO_TYPE (stmt_info
) == type_promotion_vec_info_type
) ?
892 inside_cost
+= add_stmt_cost (target_cost_data
, vect_pow2 (tmp
),
893 vec_promote_demote
, stmt_info
, 0,
897 /* FORNOW: Assuming maximum 2 args per stmts. */
898 for (i
= 0; i
< 2; i
++)
899 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
900 prologue_cost
+= add_stmt_cost (target_cost_data
, 1, vector_stmt
,
901 stmt_info
, 0, vect_prologue
);
903 if (dump_enabled_p ())
904 dump_printf_loc (MSG_NOTE
, vect_location
,
905 "vect_model_promotion_demotion_cost: inside_cost = %d, "
906 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
909 /* Function vect_cost_group_size
911 For grouped load or store, return the group_size only if it is the first
912 load or store of a group, else return 1. This ensures that group size is
913 only returned once per group. */
916 vect_cost_group_size (stmt_vec_info stmt_info
)
918 gimple first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
920 if (first_stmt
== STMT_VINFO_STMT (stmt_info
))
921 return GROUP_SIZE (stmt_info
);
927 /* Function vect_model_store_cost
929 Models cost for stores. In the case of grouped accesses, one access
930 has the overhead of the grouped access attributed to it. */
933 vect_model_store_cost (stmt_vec_info stmt_info
, int ncopies
,
934 bool store_lanes_p
, enum vect_def_type dt
,
936 stmt_vector_for_cost
*prologue_cost_vec
,
937 stmt_vector_for_cost
*body_cost_vec
)
940 unsigned int inside_cost
= 0, prologue_cost
= 0;
941 struct data_reference
*first_dr
;
944 if (dt
== vect_constant_def
|| dt
== vect_external_def
)
945 prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1, scalar_to_vec
,
946 stmt_info
, 0, vect_prologue
);
948 /* Grouped access? */
949 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
953 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
958 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
959 group_size
= vect_cost_group_size (stmt_info
);
962 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
964 /* Not a grouped access. */
968 first_dr
= STMT_VINFO_DATA_REF (stmt_info
);
971 /* We assume that the cost of a single store-lanes instruction is
972 equivalent to the cost of GROUP_SIZE separate stores. If a grouped
973 access is instead being provided by a permute-and-store operation,
974 include the cost of the permutes. */
975 if (!store_lanes_p
&& group_size
> 1
976 && !STMT_VINFO_STRIDED_P (stmt_info
))
978 /* Uses a high and low interleave or shuffle operations for each
980 int nstmts
= ncopies
* ceil_log2 (group_size
) * group_size
;
981 inside_cost
= record_stmt_cost (body_cost_vec
, nstmts
, vec_perm
,
982 stmt_info
, 0, vect_body
);
984 if (dump_enabled_p ())
985 dump_printf_loc (MSG_NOTE
, vect_location
,
986 "vect_model_store_cost: strided group_size = %d .\n",
990 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
991 /* Costs of the stores. */
992 if (STMT_VINFO_STRIDED_P (stmt_info
)
993 && !STMT_VINFO_GROUPED_ACCESS (stmt_info
))
995 /* N scalar stores plus extracting the elements. */
996 inside_cost
+= record_stmt_cost (body_cost_vec
,
997 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
998 scalar_store
, stmt_info
, 0, vect_body
);
1001 vect_get_store_cost (first_dr
, ncopies
, &inside_cost
, body_cost_vec
);
1003 if (STMT_VINFO_STRIDED_P (stmt_info
))
1004 inside_cost
+= record_stmt_cost (body_cost_vec
,
1005 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
1006 vec_to_scalar
, stmt_info
, 0, vect_body
);
1008 if (dump_enabled_p ())
1009 dump_printf_loc (MSG_NOTE
, vect_location
,
1010 "vect_model_store_cost: inside_cost = %d, "
1011 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
1015 /* Calculate cost of DR's memory access. */
1017 vect_get_store_cost (struct data_reference
*dr
, int ncopies
,
1018 unsigned int *inside_cost
,
1019 stmt_vector_for_cost
*body_cost_vec
)
1021 int alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
1022 gimple stmt
= DR_STMT (dr
);
1023 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1025 switch (alignment_support_scheme
)
1029 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1030 vector_store
, stmt_info
, 0,
1033 if (dump_enabled_p ())
1034 dump_printf_loc (MSG_NOTE
, vect_location
,
1035 "vect_model_store_cost: aligned.\n");
1039 case dr_unaligned_supported
:
1041 /* Here, we assign an additional cost for the unaligned store. */
1042 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1043 unaligned_store
, stmt_info
,
1044 DR_MISALIGNMENT (dr
), vect_body
);
1045 if (dump_enabled_p ())
1046 dump_printf_loc (MSG_NOTE
, vect_location
,
1047 "vect_model_store_cost: unaligned supported by "
1052 case dr_unaligned_unsupported
:
1054 *inside_cost
= VECT_MAX_COST
;
1056 if (dump_enabled_p ())
1057 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1058 "vect_model_store_cost: unsupported access.\n");
1068 /* Function vect_model_load_cost
1070 Models cost for loads. In the case of grouped accesses, the last access
1071 has the overhead of the grouped access attributed to it. Since unaligned
1072 accesses are supported for loads, we also account for the costs of the
1073 access scheme chosen. */
1076 vect_model_load_cost (stmt_vec_info stmt_info
, int ncopies
,
1077 bool load_lanes_p
, slp_tree slp_node
,
1078 stmt_vector_for_cost
*prologue_cost_vec
,
1079 stmt_vector_for_cost
*body_cost_vec
)
1083 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
;
1084 unsigned int inside_cost
= 0, prologue_cost
= 0;
1086 /* Grouped accesses? */
1087 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
1088 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
) && first_stmt
&& !slp_node
)
1090 group_size
= vect_cost_group_size (stmt_info
);
1091 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
1093 /* Not a grouped access. */
1100 /* We assume that the cost of a single load-lanes instruction is
1101 equivalent to the cost of GROUP_SIZE separate loads. If a grouped
1102 access is instead being provided by a load-and-permute operation,
1103 include the cost of the permutes. */
1104 if (!load_lanes_p
&& group_size
> 1
1105 && !STMT_VINFO_STRIDED_P (stmt_info
))
1107 /* Uses an even and odd extract operations or shuffle operations
1108 for each needed permute. */
1109 int nstmts
= ncopies
* ceil_log2 (group_size
) * group_size
;
1110 inside_cost
= record_stmt_cost (body_cost_vec
, nstmts
, vec_perm
,
1111 stmt_info
, 0, vect_body
);
1113 if (dump_enabled_p ())
1114 dump_printf_loc (MSG_NOTE
, vect_location
,
1115 "vect_model_load_cost: strided group_size = %d .\n",
1119 /* The loads themselves. */
1120 if (STMT_VINFO_STRIDED_P (stmt_info
)
1121 && !STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1123 /* N scalar loads plus gathering them into a vector. */
1124 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1125 inside_cost
+= record_stmt_cost (body_cost_vec
,
1126 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
1127 scalar_load
, stmt_info
, 0, vect_body
);
1130 vect_get_load_cost (first_dr
, ncopies
,
1131 ((!STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1132 || group_size
> 1 || slp_node
),
1133 &inside_cost
, &prologue_cost
,
1134 prologue_cost_vec
, body_cost_vec
, true);
1135 if (STMT_VINFO_STRIDED_P (stmt_info
))
1136 inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vec_construct
,
1137 stmt_info
, 0, vect_body
);
1139 if (dump_enabled_p ())
1140 dump_printf_loc (MSG_NOTE
, vect_location
,
1141 "vect_model_load_cost: inside_cost = %d, "
1142 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
1146 /* Calculate cost of DR's memory access. */
1148 vect_get_load_cost (struct data_reference
*dr
, int ncopies
,
1149 bool add_realign_cost
, unsigned int *inside_cost
,
1150 unsigned int *prologue_cost
,
1151 stmt_vector_for_cost
*prologue_cost_vec
,
1152 stmt_vector_for_cost
*body_cost_vec
,
1153 bool record_prologue_costs
)
1155 int alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
1156 gimple stmt
= DR_STMT (dr
);
1157 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1159 switch (alignment_support_scheme
)
1163 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vector_load
,
1164 stmt_info
, 0, vect_body
);
1166 if (dump_enabled_p ())
1167 dump_printf_loc (MSG_NOTE
, vect_location
,
1168 "vect_model_load_cost: aligned.\n");
1172 case dr_unaligned_supported
:
1174 /* Here, we assign an additional cost for the unaligned load. */
1175 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1176 unaligned_load
, stmt_info
,
1177 DR_MISALIGNMENT (dr
), vect_body
);
1179 if (dump_enabled_p ())
1180 dump_printf_loc (MSG_NOTE
, vect_location
,
1181 "vect_model_load_cost: unaligned supported by "
1186 case dr_explicit_realign
:
1188 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
* 2,
1189 vector_load
, stmt_info
, 0, vect_body
);
1190 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1191 vec_perm
, stmt_info
, 0, vect_body
);
1193 /* FIXME: If the misalignment remains fixed across the iterations of
1194 the containing loop, the following cost should be added to the
1196 if (targetm
.vectorize
.builtin_mask_for_load
)
1197 *inside_cost
+= record_stmt_cost (body_cost_vec
, 1, vector_stmt
,
1198 stmt_info
, 0, vect_body
);
1200 if (dump_enabled_p ())
1201 dump_printf_loc (MSG_NOTE
, vect_location
,
1202 "vect_model_load_cost: explicit realign\n");
1206 case dr_explicit_realign_optimized
:
1208 if (dump_enabled_p ())
1209 dump_printf_loc (MSG_NOTE
, vect_location
,
1210 "vect_model_load_cost: unaligned software "
1213 /* Unaligned software pipeline has a load of an address, an initial
1214 load, and possibly a mask operation to "prime" the loop. However,
1215 if this is an access in a group of loads, which provide grouped
1216 access, then the above cost should only be considered for one
1217 access in the group. Inside the loop, there is a load op
1218 and a realignment op. */
1220 if (add_realign_cost
&& record_prologue_costs
)
1222 *prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 2,
1223 vector_stmt
, stmt_info
,
1225 if (targetm
.vectorize
.builtin_mask_for_load
)
1226 *prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1,
1227 vector_stmt
, stmt_info
,
1231 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vector_load
,
1232 stmt_info
, 0, vect_body
);
1233 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vec_perm
,
1234 stmt_info
, 0, vect_body
);
1236 if (dump_enabled_p ())
1237 dump_printf_loc (MSG_NOTE
, vect_location
,
1238 "vect_model_load_cost: explicit realign optimized"
1244 case dr_unaligned_unsupported
:
1246 *inside_cost
= VECT_MAX_COST
;
1248 if (dump_enabled_p ())
1249 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1250 "vect_model_load_cost: unsupported access.\n");
1259 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1260 the loop preheader for the vectorized stmt STMT. */
1263 vect_init_vector_1 (gimple stmt
, gimple new_stmt
, gimple_stmt_iterator
*gsi
)
1266 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1269 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
1270 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1274 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1278 if (nested_in_vect_loop_p (loop
, stmt
))
1281 pe
= loop_preheader_edge (loop
);
1282 new_bb
= gsi_insert_on_edge_immediate (pe
, new_stmt
);
1283 gcc_assert (!new_bb
);
1287 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_vinfo
);
1289 gimple_stmt_iterator gsi_bb_start
;
1291 gcc_assert (bb_vinfo
);
1292 bb
= BB_VINFO_BB (bb_vinfo
);
1293 gsi_bb_start
= gsi_after_labels (bb
);
1294 gsi_insert_before (&gsi_bb_start
, new_stmt
, GSI_SAME_STMT
);
1298 if (dump_enabled_p ())
1300 dump_printf_loc (MSG_NOTE
, vect_location
,
1301 "created new init_stmt: ");
1302 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, new_stmt
, 0);
1306 /* Function vect_init_vector.
1308 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1309 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1310 vector type a vector with all elements equal to VAL is created first.
1311 Place the initialization at BSI if it is not NULL. Otherwise, place the
1312 initialization at the loop preheader.
1313 Return the DEF of INIT_STMT.
1314 It will be used in the vectorization of STMT. */
1317 vect_init_vector (gimple stmt
, tree val
, tree type
, gimple_stmt_iterator
*gsi
)
1324 if (TREE_CODE (type
) == VECTOR_TYPE
1325 && TREE_CODE (TREE_TYPE (val
)) != VECTOR_TYPE
)
1327 if (!types_compatible_p (TREE_TYPE (type
), TREE_TYPE (val
)))
1329 if (CONSTANT_CLASS_P (val
))
1330 val
= fold_unary (VIEW_CONVERT_EXPR
, TREE_TYPE (type
), val
);
1333 new_temp
= make_ssa_name (TREE_TYPE (type
));
1334 init_stmt
= gimple_build_assign (new_temp
, NOP_EXPR
, val
);
1335 vect_init_vector_1 (stmt
, init_stmt
, gsi
);
1339 val
= build_vector_from_val (type
, val
);
1342 new_var
= vect_get_new_vect_var (type
, vect_simple_var
, "cst_");
1343 init_stmt
= gimple_build_assign (new_var
, val
);
1344 new_temp
= make_ssa_name (new_var
, init_stmt
);
1345 gimple_assign_set_lhs (init_stmt
, new_temp
);
1346 vect_init_vector_1 (stmt
, init_stmt
, gsi
);
1347 vec_oprnd
= gimple_assign_lhs (init_stmt
);
1352 /* Function vect_get_vec_def_for_operand.
1354 OP is an operand in STMT. This function returns a (vector) def that will be
1355 used in the vectorized stmt for STMT.
1357 In the case that OP is an SSA_NAME which is defined in the loop, then
1358 STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
1360 In case OP is an invariant or constant, a new stmt that creates a vector def
1361 needs to be introduced. */
1364 vect_get_vec_def_for_operand (tree op
, gimple stmt
, tree
*scalar_def
)
1369 stmt_vec_info def_stmt_info
= NULL
;
1370 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
1371 unsigned int nunits
;
1372 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1374 enum vect_def_type dt
;
1378 if (dump_enabled_p ())
1380 dump_printf_loc (MSG_NOTE
, vect_location
,
1381 "vect_get_vec_def_for_operand: ");
1382 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, op
);
1383 dump_printf (MSG_NOTE
, "\n");
1386 is_simple_use
= vect_is_simple_use (op
, stmt
, loop_vinfo
, NULL
,
1387 &def_stmt
, &def
, &dt
);
1388 gcc_assert (is_simple_use
);
1389 if (dump_enabled_p ())
1391 int loc_printed
= 0;
1394 dump_printf_loc (MSG_NOTE
, vect_location
, "def = ");
1396 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, def
);
1397 dump_printf (MSG_NOTE
, "\n");
1402 dump_printf (MSG_NOTE
, " def_stmt = ");
1404 dump_printf_loc (MSG_NOTE
, vect_location
, " def_stmt = ");
1405 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, def_stmt
, 0);
1411 /* Case 1: operand is a constant. */
1412 case vect_constant_def
:
1414 vector_type
= get_vectype_for_scalar_type (TREE_TYPE (op
));
1415 gcc_assert (vector_type
);
1416 nunits
= TYPE_VECTOR_SUBPARTS (vector_type
);
1421 /* Create 'vect_cst_ = {cst,cst,...,cst}' */
1422 if (dump_enabled_p ())
1423 dump_printf_loc (MSG_NOTE
, vect_location
,
1424 "Create vector_cst. nunits = %d\n", nunits
);
1426 return vect_init_vector (stmt
, op
, vector_type
, NULL
);
1429 /* Case 2: operand is defined outside the loop - loop invariant. */
1430 case vect_external_def
:
1432 vector_type
= get_vectype_for_scalar_type (TREE_TYPE (def
));
1433 gcc_assert (vector_type
);
1438 /* Create 'vec_inv = {inv,inv,..,inv}' */
1439 if (dump_enabled_p ())
1440 dump_printf_loc (MSG_NOTE
, vect_location
, "Create vector_inv.\n");
1442 return vect_init_vector (stmt
, def
, vector_type
, NULL
);
1445 /* Case 3: operand is defined inside the loop. */
1446 case vect_internal_def
:
1449 *scalar_def
= NULL
/* FIXME tuples: def_stmt*/;
1451 /* Get the def from the vectorized stmt. */
1452 def_stmt_info
= vinfo_for_stmt (def_stmt
);
1454 vec_stmt
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1455 /* Get vectorized pattern statement. */
1457 && STMT_VINFO_IN_PATTERN_P (def_stmt_info
)
1458 && !STMT_VINFO_RELEVANT (def_stmt_info
))
1459 vec_stmt
= STMT_VINFO_VEC_STMT (vinfo_for_stmt (
1460 STMT_VINFO_RELATED_STMT (def_stmt_info
)));
1461 gcc_assert (vec_stmt
);
1462 if (gimple_code (vec_stmt
) == GIMPLE_PHI
)
1463 vec_oprnd
= PHI_RESULT (vec_stmt
);
1464 else if (is_gimple_call (vec_stmt
))
1465 vec_oprnd
= gimple_call_lhs (vec_stmt
);
1467 vec_oprnd
= gimple_assign_lhs (vec_stmt
);
1471 /* Case 4: operand is defined by a loop header phi - reduction */
1472 case vect_reduction_def
:
1473 case vect_double_reduction_def
:
1474 case vect_nested_cycle
:
1478 gcc_assert (gimple_code (def_stmt
) == GIMPLE_PHI
);
1479 loop
= (gimple_bb (def_stmt
))->loop_father
;
1481 /* Get the def before the loop */
1482 op
= PHI_ARG_DEF_FROM_EDGE (def_stmt
, loop_preheader_edge (loop
));
1483 return get_initial_def_for_reduction (stmt
, op
, scalar_def
);
1486 /* Case 5: operand is defined by loop-header phi - induction. */
1487 case vect_induction_def
:
1489 gcc_assert (gimple_code (def_stmt
) == GIMPLE_PHI
);
1491 /* Get the def from the vectorized stmt. */
1492 def_stmt_info
= vinfo_for_stmt (def_stmt
);
1493 vec_stmt
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1494 if (gimple_code (vec_stmt
) == GIMPLE_PHI
)
1495 vec_oprnd
= PHI_RESULT (vec_stmt
);
1497 vec_oprnd
= gimple_get_lhs (vec_stmt
);
1507 /* Function vect_get_vec_def_for_stmt_copy
1509 Return a vector-def for an operand. This function is used when the
1510 vectorized stmt to be created (by the caller to this function) is a "copy"
1511 created in case the vectorized result cannot fit in one vector, and several
1512 copies of the vector-stmt are required. In this case the vector-def is
1513 retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field
1514 of the stmt that defines VEC_OPRND.
1515 DT is the type of the vector def VEC_OPRND.
1518 In case the vectorization factor (VF) is bigger than the number
1519 of elements that can fit in a vectype (nunits), we have to generate
1520 more than one vector stmt to vectorize the scalar stmt. This situation
1521 arises when there are multiple data-types operated upon in the loop; the
1522 smallest data-type determines the VF, and as a result, when vectorizing
1523 stmts operating on wider types we need to create 'VF/nunits' "copies" of the
1524 vector stmt (each computing a vector of 'nunits' results, and together
1525 computing 'VF' results in each iteration). This function is called when
1526 vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
1527 which VF=16 and nunits=4, so the number of copies required is 4):
1529 scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
1531 S1: x = load VS1.0: vx.0 = memref0 VS1.1
1532 VS1.1: vx.1 = memref1 VS1.2
1533 VS1.2: vx.2 = memref2 VS1.3
1534 VS1.3: vx.3 = memref3
1536 S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1
1537 VSnew.1: vz1 = vx.1 + ... VSnew.2
1538 VSnew.2: vz2 = vx.2 + ... VSnew.3
1539 VSnew.3: vz3 = vx.3 + ...
1541 The vectorization of S1 is explained in vectorizable_load.
1542 The vectorization of S2:
1543 To create the first vector-stmt out of the 4 copies - VSnew.0 -
1544 the function 'vect_get_vec_def_for_operand' is called to
1545 get the relevant vector-def for each operand of S2. For operand x it
1546 returns the vector-def 'vx.0'.
1548 To create the remaining copies of the vector-stmt (VSnew.j), this
1549 function is called to get the relevant vector-def for each operand. It is
1550 obtained from the respective VS1.j stmt, which is recorded in the
1551 STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND.
1553 For example, to obtain the vector-def 'vx.1' in order to create the
1554 vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'.
1555 Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the
1556 STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1',
1557 and return its def ('vx.1').
1558 Overall, to create the above sequence this function will be called 3 times:
1559 vx.1 = vect_get_vec_def_for_stmt_copy (dt, vx.0);
1560 vx.2 = vect_get_vec_def_for_stmt_copy (dt, vx.1);
1561 vx.3 = vect_get_vec_def_for_stmt_copy (dt, vx.2); */
1564 vect_get_vec_def_for_stmt_copy (enum vect_def_type dt
, tree vec_oprnd
)
1566 gimple vec_stmt_for_operand
;
1567 stmt_vec_info def_stmt_info
;
1569 /* Do nothing; can reuse same def. */
1570 if (dt
== vect_external_def
|| dt
== vect_constant_def
)
1573 vec_stmt_for_operand
= SSA_NAME_DEF_STMT (vec_oprnd
);
1574 def_stmt_info
= vinfo_for_stmt (vec_stmt_for_operand
);
1575 gcc_assert (def_stmt_info
);
1576 vec_stmt_for_operand
= STMT_VINFO_RELATED_STMT (def_stmt_info
);
1577 gcc_assert (vec_stmt_for_operand
);
1578 vec_oprnd
= gimple_get_lhs (vec_stmt_for_operand
);
1579 if (gimple_code (vec_stmt_for_operand
) == GIMPLE_PHI
)
1580 vec_oprnd
= PHI_RESULT (vec_stmt_for_operand
);
1582 vec_oprnd
= gimple_get_lhs (vec_stmt_for_operand
);
1587 /* Get vectorized definitions for the operands to create a copy of an original
1588 stmt. See vect_get_vec_def_for_stmt_copy () for details. */
1591 vect_get_vec_defs_for_stmt_copy (enum vect_def_type
*dt
,
1592 vec
<tree
> *vec_oprnds0
,
1593 vec
<tree
> *vec_oprnds1
)
1595 tree vec_oprnd
= vec_oprnds0
->pop ();
1597 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd
);
1598 vec_oprnds0
->quick_push (vec_oprnd
);
1600 if (vec_oprnds1
&& vec_oprnds1
->length ())
1602 vec_oprnd
= vec_oprnds1
->pop ();
1603 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
[1], vec_oprnd
);
1604 vec_oprnds1
->quick_push (vec_oprnd
);
1609 /* Get vectorized definitions for OP0 and OP1.
1610 REDUC_INDEX is the index of reduction operand in case of reduction,
1611 and -1 otherwise. */
1614 vect_get_vec_defs (tree op0
, tree op1
, gimple stmt
,
1615 vec
<tree
> *vec_oprnds0
,
1616 vec
<tree
> *vec_oprnds1
,
1617 slp_tree slp_node
, int reduc_index
)
1621 int nops
= (op1
== NULL_TREE
) ? 1 : 2;
1622 auto_vec
<tree
> ops (nops
);
1623 auto_vec
<vec
<tree
> > vec_defs (nops
);
1625 ops
.quick_push (op0
);
1627 ops
.quick_push (op1
);
1629 vect_get_slp_defs (ops
, slp_node
, &vec_defs
, reduc_index
);
1631 *vec_oprnds0
= vec_defs
[0];
1633 *vec_oprnds1
= vec_defs
[1];
1639 vec_oprnds0
->create (1);
1640 vec_oprnd
= vect_get_vec_def_for_operand (op0
, stmt
, NULL
);
1641 vec_oprnds0
->quick_push (vec_oprnd
);
1645 vec_oprnds1
->create (1);
1646 vec_oprnd
= vect_get_vec_def_for_operand (op1
, stmt
, NULL
);
1647 vec_oprnds1
->quick_push (vec_oprnd
);
1653 /* Function vect_finish_stmt_generation.
1655 Insert a new stmt. */
1658 vect_finish_stmt_generation (gimple stmt
, gimple vec_stmt
,
1659 gimple_stmt_iterator
*gsi
)
1661 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1662 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1663 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
1665 gcc_assert (gimple_code (stmt
) != GIMPLE_LABEL
);
1667 if (!gsi_end_p (*gsi
)
1668 && gimple_has_mem_ops (vec_stmt
))
1670 gimple at_stmt
= gsi_stmt (*gsi
);
1671 tree vuse
= gimple_vuse (at_stmt
);
1672 if (vuse
&& TREE_CODE (vuse
) == SSA_NAME
)
1674 tree vdef
= gimple_vdef (at_stmt
);
1675 gimple_set_vuse (vec_stmt
, gimple_vuse (at_stmt
));
1676 /* If we have an SSA vuse and insert a store, update virtual
1677 SSA form to avoid triggering the renamer. Do so only
1678 if we can easily see all uses - which is what almost always
1679 happens with the way vectorized stmts are inserted. */
1680 if ((vdef
&& TREE_CODE (vdef
) == SSA_NAME
)
1681 && ((is_gimple_assign (vec_stmt
)
1682 && !is_gimple_reg (gimple_assign_lhs (vec_stmt
)))
1683 || (is_gimple_call (vec_stmt
)
1684 && !(gimple_call_flags (vec_stmt
)
1685 & (ECF_CONST
|ECF_PURE
|ECF_NOVOPS
)))))
1687 tree new_vdef
= copy_ssa_name (vuse
, vec_stmt
);
1688 gimple_set_vdef (vec_stmt
, new_vdef
);
1689 SET_USE (gimple_vuse_op (at_stmt
), new_vdef
);
1693 gsi_insert_before (gsi
, vec_stmt
, GSI_SAME_STMT
);
1695 set_vinfo_for_stmt (vec_stmt
, new_stmt_vec_info (vec_stmt
, loop_vinfo
,
1698 if (dump_enabled_p ())
1700 dump_printf_loc (MSG_NOTE
, vect_location
, "add new stmt: ");
1701 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, vec_stmt
, 0);
1704 gimple_set_location (vec_stmt
, gimple_location (stmt
));
1706 /* While EH edges will generally prevent vectorization, stmt might
1707 e.g. be in a must-not-throw region. Ensure newly created stmts
1708 that could throw are part of the same region. */
1709 int lp_nr
= lookup_stmt_eh_lp (stmt
);
1710 if (lp_nr
!= 0 && stmt_could_throw_p (vec_stmt
))
1711 add_stmt_to_eh_lp (vec_stmt
, lp_nr
);
1714 /* Checks if CALL can be vectorized in type VECTYPE. Returns
1715 a function declaration if the target has a vectorized version
1716 of the function, or NULL_TREE if the function cannot be vectorized. */
1719 vectorizable_function (gcall
*call
, tree vectype_out
, tree vectype_in
)
1721 tree fndecl
= gimple_call_fndecl (call
);
1723 /* We only handle functions that do not read or clobber memory -- i.e.
1724 const or novops ones. */
1725 if (!(gimple_call_flags (call
) & (ECF_CONST
| ECF_NOVOPS
)))
1729 || TREE_CODE (fndecl
) != FUNCTION_DECL
1730 || !DECL_BUILT_IN (fndecl
))
1733 return targetm
.vectorize
.builtin_vectorized_function (fndecl
, vectype_out
,
1738 static tree
permute_vec_elements (tree
, tree
, tree
, gimple
,
1739 gimple_stmt_iterator
*);
1742 /* Function vectorizable_mask_load_store.
1744 Check if STMT performs a conditional load or store that can be vectorized.
1745 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1746 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
1747 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1750 vectorizable_mask_load_store (gimple stmt
, gimple_stmt_iterator
*gsi
,
1751 gimple
*vec_stmt
, slp_tree slp_node
)
1753 tree vec_dest
= NULL
;
1754 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1755 stmt_vec_info prev_stmt_info
;
1756 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1757 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1758 bool nested_in_vect_loop
= nested_in_vect_loop_p (loop
, stmt
);
1759 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
);
1760 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1764 tree dataref_ptr
= NULL_TREE
;
1766 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
1770 tree gather_base
= NULL_TREE
, gather_off
= NULL_TREE
;
1771 tree gather_off_vectype
= NULL_TREE
, gather_decl
= NULL_TREE
;
1772 int gather_scale
= 1;
1773 enum vect_def_type gather_dt
= vect_unknown_def_type
;
1778 enum vect_def_type dt
;
1780 if (slp_node
!= NULL
)
1783 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
1784 gcc_assert (ncopies
>= 1);
1786 is_store
= gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
;
1787 mask
= gimple_call_arg (stmt
, 2);
1788 if (TYPE_PRECISION (TREE_TYPE (mask
))
1789 != GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (vectype
))))
1792 /* FORNOW. This restriction should be relaxed. */
1793 if (nested_in_vect_loop
&& ncopies
> 1)
1795 if (dump_enabled_p ())
1796 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1797 "multiple types in nested loop.");
1801 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
1804 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
1807 if (!STMT_VINFO_DATA_REF (stmt_info
))
1810 elem_type
= TREE_TYPE (vectype
);
1812 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1815 if (STMT_VINFO_STRIDED_P (stmt_info
))
1818 if (STMT_VINFO_GATHER_P (stmt_info
))
1822 gather_decl
= vect_check_gather (stmt
, loop_vinfo
, &gather_base
,
1823 &gather_off
, &gather_scale
);
1824 gcc_assert (gather_decl
);
1825 if (!vect_is_simple_use_1 (gather_off
, NULL
, loop_vinfo
, NULL
,
1826 &def_stmt
, &def
, &gather_dt
,
1827 &gather_off_vectype
))
1829 if (dump_enabled_p ())
1830 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1831 "gather index use not simple.");
1835 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gather_decl
));
1837 = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist
))));
1838 if (TREE_CODE (masktype
) == INTEGER_TYPE
)
1840 if (dump_enabled_p ())
1841 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1842 "masked gather with integer mask not supported.");
1846 else if (tree_int_cst_compare (nested_in_vect_loop
1847 ? STMT_VINFO_DR_STEP (stmt_info
)
1848 : DR_STEP (dr
), size_zero_node
) <= 0)
1850 else if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
1851 || !can_vec_mask_load_store_p (TYPE_MODE (vectype
), !is_store
))
1854 if (TREE_CODE (mask
) != SSA_NAME
)
1857 if (!vect_is_simple_use (mask
, stmt
, loop_vinfo
, NULL
,
1858 &def_stmt
, &def
, &dt
))
1863 tree rhs
= gimple_call_arg (stmt
, 3);
1864 if (!vect_is_simple_use (rhs
, stmt
, loop_vinfo
, NULL
,
1865 &def_stmt
, &def
, &dt
))
1869 if (!vec_stmt
) /* transformation not required. */
1871 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
1873 vect_model_store_cost (stmt_info
, ncopies
, false, dt
,
1876 vect_model_load_cost (stmt_info
, ncopies
, false, NULL
, NULL
, NULL
);
1882 if (STMT_VINFO_GATHER_P (stmt_info
))
1884 tree vec_oprnd0
= NULL_TREE
, op
;
1885 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gather_decl
));
1886 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
1887 tree ptr
, vec_mask
= NULL_TREE
, mask_op
= NULL_TREE
, var
, scale
;
1888 tree perm_mask
= NULL_TREE
, prev_res
= NULL_TREE
;
1889 tree mask_perm_mask
= NULL_TREE
;
1890 edge pe
= loop_preheader_edge (loop
);
1893 enum { NARROW
, NONE
, WIDEN
} modifier
;
1894 int gather_off_nunits
= TYPE_VECTOR_SUBPARTS (gather_off_vectype
);
1896 rettype
= TREE_TYPE (TREE_TYPE (gather_decl
));
1897 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1898 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1899 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1900 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1901 scaletype
= TREE_VALUE (arglist
);
1902 gcc_checking_assert (types_compatible_p (srctype
, rettype
)
1903 && types_compatible_p (srctype
, masktype
));
1905 if (nunits
== gather_off_nunits
)
1907 else if (nunits
== gather_off_nunits
/ 2)
1909 unsigned char *sel
= XALLOCAVEC (unsigned char, gather_off_nunits
);
1912 for (i
= 0; i
< gather_off_nunits
; ++i
)
1913 sel
[i
] = i
| nunits
;
1915 perm_mask
= vect_gen_perm_mask_checked (gather_off_vectype
, sel
);
1917 else if (nunits
== gather_off_nunits
* 2)
1919 unsigned char *sel
= XALLOCAVEC (unsigned char, nunits
);
1922 for (i
= 0; i
< nunits
; ++i
)
1923 sel
[i
] = i
< gather_off_nunits
1924 ? i
: i
+ nunits
- gather_off_nunits
;
1926 perm_mask
= vect_gen_perm_mask_checked (vectype
, sel
);
1928 for (i
= 0; i
< nunits
; ++i
)
1929 sel
[i
] = i
| gather_off_nunits
;
1930 mask_perm_mask
= vect_gen_perm_mask_checked (masktype
, sel
);
1935 vec_dest
= vect_create_destination_var (gimple_call_lhs (stmt
), vectype
);
1937 ptr
= fold_convert (ptrtype
, gather_base
);
1938 if (!is_gimple_min_invariant (ptr
))
1940 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
1941 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
1942 gcc_assert (!new_bb
);
1945 scale
= build_int_cst (scaletype
, gather_scale
);
1947 prev_stmt_info
= NULL
;
1948 for (j
= 0; j
< ncopies
; ++j
)
1950 if (modifier
== WIDEN
&& (j
& 1))
1951 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
,
1952 perm_mask
, stmt
, gsi
);
1955 = vect_get_vec_def_for_operand (gather_off
, stmt
, NULL
);
1958 = vect_get_vec_def_for_stmt_copy (gather_dt
, vec_oprnd0
);
1960 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
1962 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
1963 == TYPE_VECTOR_SUBPARTS (idxtype
));
1964 var
= vect_get_new_vect_var (idxtype
, vect_simple_var
, NULL
);
1965 var
= make_ssa_name (var
);
1966 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
1968 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
1969 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1973 if (mask_perm_mask
&& (j
& 1))
1974 mask_op
= permute_vec_elements (mask_op
, mask_op
,
1975 mask_perm_mask
, stmt
, gsi
);
1979 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
, NULL
);
1982 vect_is_simple_use (vec_mask
, NULL
, loop_vinfo
, NULL
,
1983 &def_stmt
, &def
, &dt
);
1984 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
1988 if (!useless_type_conversion_p (masktype
, TREE_TYPE (vec_mask
)))
1990 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask_op
))
1991 == TYPE_VECTOR_SUBPARTS (masktype
));
1992 var
= vect_get_new_vect_var (masktype
, vect_simple_var
,
1994 var
= make_ssa_name (var
);
1995 mask_op
= build1 (VIEW_CONVERT_EXPR
, masktype
, mask_op
);
1997 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, mask_op
);
1998 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2004 = gimple_build_call (gather_decl
, 5, mask_op
, ptr
, op
, mask_op
,
2007 if (!useless_type_conversion_p (vectype
, rettype
))
2009 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype
)
2010 == TYPE_VECTOR_SUBPARTS (rettype
));
2011 var
= vect_get_new_vect_var (rettype
, vect_simple_var
, NULL
);
2012 op
= make_ssa_name (var
, new_stmt
);
2013 gimple_call_set_lhs (new_stmt
, op
);
2014 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2015 var
= make_ssa_name (vec_dest
);
2016 op
= build1 (VIEW_CONVERT_EXPR
, vectype
, op
);
2017 new_stmt
= gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
2021 var
= make_ssa_name (vec_dest
, new_stmt
);
2022 gimple_call_set_lhs (new_stmt
, var
);
2025 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2027 if (modifier
== NARROW
)
2034 var
= permute_vec_elements (prev_res
, var
,
2035 perm_mask
, stmt
, gsi
);
2036 new_stmt
= SSA_NAME_DEF_STMT (var
);
2039 if (prev_stmt_info
== NULL
)
2040 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2042 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2043 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2046 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
2048 tree lhs
= gimple_call_lhs (stmt
);
2049 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
2050 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2051 set_vinfo_for_stmt (stmt
, NULL
);
2052 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2053 gsi_replace (gsi
, new_stmt
, true);
2058 tree vec_rhs
= NULL_TREE
, vec_mask
= NULL_TREE
;
2059 prev_stmt_info
= NULL
;
2060 for (i
= 0; i
< ncopies
; i
++)
2062 unsigned align
, misalign
;
2066 tree rhs
= gimple_call_arg (stmt
, 3);
2067 vec_rhs
= vect_get_vec_def_for_operand (rhs
, stmt
, NULL
);
2068 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
, NULL
);
2069 /* We should have catched mismatched types earlier. */
2070 gcc_assert (useless_type_conversion_p (vectype
,
2071 TREE_TYPE (vec_rhs
)));
2072 dataref_ptr
= vect_create_data_ref_ptr (stmt
, vectype
, NULL
,
2073 NULL_TREE
, &dummy
, gsi
,
2074 &ptr_incr
, false, &inv_p
);
2075 gcc_assert (!inv_p
);
2079 vect_is_simple_use (vec_rhs
, NULL
, loop_vinfo
, NULL
, &def_stmt
,
2081 vec_rhs
= vect_get_vec_def_for_stmt_copy (dt
, vec_rhs
);
2082 vect_is_simple_use (vec_mask
, NULL
, loop_vinfo
, NULL
, &def_stmt
,
2084 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2085 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
2086 TYPE_SIZE_UNIT (vectype
));
2089 align
= TYPE_ALIGN_UNIT (vectype
);
2090 if (aligned_access_p (dr
))
2092 else if (DR_MISALIGNMENT (dr
) == -1)
2094 align
= TYPE_ALIGN_UNIT (elem_type
);
2098 misalign
= DR_MISALIGNMENT (dr
);
2099 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
2102 = gimple_build_call_internal (IFN_MASK_STORE
, 4, dataref_ptr
,
2103 gimple_call_arg (stmt
, 1),
2105 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2107 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2109 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2110 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2115 tree vec_mask
= NULL_TREE
;
2116 prev_stmt_info
= NULL
;
2117 vec_dest
= vect_create_destination_var (gimple_call_lhs (stmt
), vectype
);
2118 for (i
= 0; i
< ncopies
; i
++)
2120 unsigned align
, misalign
;
2124 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
, NULL
);
2125 dataref_ptr
= vect_create_data_ref_ptr (stmt
, vectype
, NULL
,
2126 NULL_TREE
, &dummy
, gsi
,
2127 &ptr_incr
, false, &inv_p
);
2128 gcc_assert (!inv_p
);
2132 vect_is_simple_use (vec_mask
, NULL
, loop_vinfo
, NULL
, &def_stmt
,
2134 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2135 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
2136 TYPE_SIZE_UNIT (vectype
));
2139 align
= TYPE_ALIGN_UNIT (vectype
);
2140 if (aligned_access_p (dr
))
2142 else if (DR_MISALIGNMENT (dr
) == -1)
2144 align
= TYPE_ALIGN_UNIT (elem_type
);
2148 misalign
= DR_MISALIGNMENT (dr
);
2149 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
2152 = gimple_build_call_internal (IFN_MASK_LOAD
, 3, dataref_ptr
,
2153 gimple_call_arg (stmt
, 1),
2155 gimple_call_set_lhs (new_stmt
, make_ssa_name (vec_dest
));
2156 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2158 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2160 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2161 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2167 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
2169 tree lhs
= gimple_call_lhs (stmt
);
2170 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
2171 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2172 set_vinfo_for_stmt (stmt
, NULL
);
2173 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2174 gsi_replace (gsi
, new_stmt
, true);
2181 /* Function vectorizable_call.
2183 Check if GS performs a function call that can be vectorized.
2184 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2185 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2186 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2189 vectorizable_call (gimple gs
, gimple_stmt_iterator
*gsi
, gimple
*vec_stmt
,
2196 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
2197 stmt_vec_info stmt_info
= vinfo_for_stmt (gs
), prev_stmt_info
;
2198 tree vectype_out
, vectype_in
;
2201 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2202 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
2203 tree fndecl
, new_temp
, def
, rhs_type
;
2205 enum vect_def_type dt
[3]
2206 = {vect_unknown_def_type
, vect_unknown_def_type
, vect_unknown_def_type
};
2207 gimple new_stmt
= NULL
;
2209 vec
<tree
> vargs
= vNULL
;
2210 enum { NARROW
, NONE
, WIDEN
} modifier
;
2214 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
2217 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
2220 /* Is GS a vectorizable call? */
2221 stmt
= dyn_cast
<gcall
*> (gs
);
2225 if (gimple_call_internal_p (stmt
)
2226 && (gimple_call_internal_fn (stmt
) == IFN_MASK_LOAD
2227 || gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
))
2228 return vectorizable_mask_load_store (stmt
, gsi
, vec_stmt
,
2231 if (gimple_call_lhs (stmt
) == NULL_TREE
2232 || TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
2235 gcc_checking_assert (!stmt_can_throw_internal (stmt
));
2237 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
2239 /* Process function arguments. */
2240 rhs_type
= NULL_TREE
;
2241 vectype_in
= NULL_TREE
;
2242 nargs
= gimple_call_num_args (stmt
);
2244 /* Bail out if the function has more than three arguments, we do not have
2245 interesting builtin functions to vectorize with more than two arguments
2246 except for fma. No arguments is also not good. */
2247 if (nargs
== 0 || nargs
> 3)
2250 /* Ignore the argument of IFN_GOMP_SIMD_LANE, it is magic. */
2251 if (gimple_call_internal_p (stmt
)
2252 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
)
2255 rhs_type
= unsigned_type_node
;
2258 for (i
= 0; i
< nargs
; i
++)
2262 op
= gimple_call_arg (stmt
, i
);
2264 /* We can only handle calls with arguments of the same type. */
2266 && !types_compatible_p (rhs_type
, TREE_TYPE (op
)))
2268 if (dump_enabled_p ())
2269 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2270 "argument types differ.\n");
2274 rhs_type
= TREE_TYPE (op
);
2276 if (!vect_is_simple_use_1 (op
, stmt
, loop_vinfo
, bb_vinfo
,
2277 &def_stmt
, &def
, &dt
[i
], &opvectype
))
2279 if (dump_enabled_p ())
2280 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2281 "use not simple.\n");
2286 vectype_in
= opvectype
;
2288 && opvectype
!= vectype_in
)
2290 if (dump_enabled_p ())
2291 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2292 "argument vector types differ.\n");
2296 /* If all arguments are external or constant defs use a vector type with
2297 the same size as the output vector type. */
2299 vectype_in
= get_same_sized_vectype (rhs_type
, vectype_out
);
2301 gcc_assert (vectype_in
);
2304 if (dump_enabled_p ())
2306 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2307 "no vectype for scalar type ");
2308 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
2309 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
2316 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
2317 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
2318 if (nunits_in
== nunits_out
/ 2)
2320 else if (nunits_out
== nunits_in
)
2322 else if (nunits_out
== nunits_in
/ 2)
2327 /* For now, we only vectorize functions if a target specific builtin
2328 is available. TODO -- in some cases, it might be profitable to
2329 insert the calls for pieces of the vector, in order to be able
2330 to vectorize other operations in the loop. */
2331 fndecl
= vectorizable_function (stmt
, vectype_out
, vectype_in
);
2332 if (fndecl
== NULL_TREE
)
2334 if (gimple_call_internal_p (stmt
)
2335 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
2338 && LOOP_VINFO_LOOP (loop_vinfo
)->simduid
2339 && TREE_CODE (gimple_call_arg (stmt
, 0)) == SSA_NAME
2340 && LOOP_VINFO_LOOP (loop_vinfo
)->simduid
2341 == SSA_NAME_VAR (gimple_call_arg (stmt
, 0)))
2343 /* We can handle IFN_GOMP_SIMD_LANE by returning a
2344 { 0, 1, 2, ... vf - 1 } vector. */
2345 gcc_assert (nargs
== 0);
2349 if (dump_enabled_p ())
2350 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2351 "function is not vectorizable.\n");
2356 gcc_assert (!gimple_vuse (stmt
));
2358 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
2360 else if (modifier
== NARROW
)
2361 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_out
;
2363 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
2365 /* Sanity check: make sure that at least one copy of the vectorized stmt
2366 needs to be generated. */
2367 gcc_assert (ncopies
>= 1);
2369 if (!vec_stmt
) /* transformation not required. */
2371 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
2372 if (dump_enabled_p ())
2373 dump_printf_loc (MSG_NOTE
, vect_location
, "=== vectorizable_call ==="
2375 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
2381 if (dump_enabled_p ())
2382 dump_printf_loc (MSG_NOTE
, vect_location
, "transform call.\n");
2385 scalar_dest
= gimple_call_lhs (stmt
);
2386 vec_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
2388 prev_stmt_info
= NULL
;
2392 for (j
= 0; j
< ncopies
; ++j
)
2394 /* Build argument list for the vectorized call. */
2396 vargs
.create (nargs
);
2402 auto_vec
<vec
<tree
> > vec_defs (nargs
);
2403 vec
<tree
> vec_oprnds0
;
2405 for (i
= 0; i
< nargs
; i
++)
2406 vargs
.quick_push (gimple_call_arg (stmt
, i
));
2407 vect_get_slp_defs (vargs
, slp_node
, &vec_defs
, -1);
2408 vec_oprnds0
= vec_defs
[0];
2410 /* Arguments are ready. Create the new vector stmt. */
2411 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_oprnd0
)
2414 for (k
= 0; k
< nargs
; k
++)
2416 vec
<tree
> vec_oprndsk
= vec_defs
[k
];
2417 vargs
[k
] = vec_oprndsk
[i
];
2419 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2420 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2421 gimple_call_set_lhs (new_stmt
, new_temp
);
2422 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2423 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
2426 for (i
= 0; i
< nargs
; i
++)
2428 vec
<tree
> vec_oprndsi
= vec_defs
[i
];
2429 vec_oprndsi
.release ();
2434 for (i
= 0; i
< nargs
; i
++)
2436 op
= gimple_call_arg (stmt
, i
);
2439 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
2442 vec_oprnd0
= gimple_call_arg (new_stmt
, i
);
2444 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2447 vargs
.quick_push (vec_oprnd0
);
2450 if (gimple_call_internal_p (stmt
)
2451 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
)
2453 tree
*v
= XALLOCAVEC (tree
, nunits_out
);
2455 for (k
= 0; k
< nunits_out
; ++k
)
2456 v
[k
] = build_int_cst (unsigned_type_node
, j
* nunits_out
+ k
);
2457 tree cst
= build_vector (vectype_out
, v
);
2459 = vect_get_new_vect_var (vectype_out
, vect_simple_var
, "cst_");
2460 gimple init_stmt
= gimple_build_assign (new_var
, cst
);
2461 new_temp
= make_ssa_name (new_var
, init_stmt
);
2462 gimple_assign_set_lhs (init_stmt
, new_temp
);
2463 vect_init_vector_1 (stmt
, init_stmt
, NULL
);
2464 new_temp
= make_ssa_name (vec_dest
);
2465 new_stmt
= gimple_build_assign (new_temp
,
2466 gimple_assign_lhs (init_stmt
));
2470 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2471 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2472 gimple_call_set_lhs (new_stmt
, new_temp
);
2474 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2477 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2479 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2481 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2487 for (j
= 0; j
< ncopies
; ++j
)
2489 /* Build argument list for the vectorized call. */
2491 vargs
.create (nargs
* 2);
2497 auto_vec
<vec
<tree
> > vec_defs (nargs
);
2498 vec
<tree
> vec_oprnds0
;
2500 for (i
= 0; i
< nargs
; i
++)
2501 vargs
.quick_push (gimple_call_arg (stmt
, i
));
2502 vect_get_slp_defs (vargs
, slp_node
, &vec_defs
, -1);
2503 vec_oprnds0
= vec_defs
[0];
2505 /* Arguments are ready. Create the new vector stmt. */
2506 for (i
= 0; vec_oprnds0
.iterate (i
, &vec_oprnd0
); i
+= 2)
2510 for (k
= 0; k
< nargs
; k
++)
2512 vec
<tree
> vec_oprndsk
= vec_defs
[k
];
2513 vargs
.quick_push (vec_oprndsk
[i
]);
2514 vargs
.quick_push (vec_oprndsk
[i
+ 1]);
2516 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2517 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2518 gimple_call_set_lhs (new_stmt
, new_temp
);
2519 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2520 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
2523 for (i
= 0; i
< nargs
; i
++)
2525 vec
<tree
> vec_oprndsi
= vec_defs
[i
];
2526 vec_oprndsi
.release ();
2531 for (i
= 0; i
< nargs
; i
++)
2533 op
= gimple_call_arg (stmt
, i
);
2537 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
2539 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2543 vec_oprnd1
= gimple_call_arg (new_stmt
, 2*i
+ 1);
2545 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd1
);
2547 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2550 vargs
.quick_push (vec_oprnd0
);
2551 vargs
.quick_push (vec_oprnd1
);
2554 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2555 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2556 gimple_call_set_lhs (new_stmt
, new_temp
);
2557 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2560 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
2562 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2564 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2567 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
2572 /* No current target implements this case. */
2578 /* The call in STMT might prevent it from being removed in dce.
2579 We however cannot remove it here, due to the way the ssa name
2580 it defines is mapped to the new definition. So just replace
2581 rhs of the statement with something harmless. */
2586 type
= TREE_TYPE (scalar_dest
);
2587 if (is_pattern_stmt_p (stmt_info
))
2588 lhs
= gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info
));
2590 lhs
= gimple_call_lhs (stmt
);
2592 if (gimple_call_internal_p (stmt
)
2593 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
)
2595 /* Replace uses of the lhs of GOMP_SIMD_LANE call outside the loop
2596 with vf - 1 rather than 0, that is the last iteration of the
2598 imm_use_iterator iter
;
2599 use_operand_p use_p
;
2601 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, lhs
)
2603 basic_block use_bb
= gimple_bb (use_stmt
);
2605 && !flow_bb_inside_loop_p (LOOP_VINFO_LOOP (loop_vinfo
), use_bb
))
2607 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
2608 SET_USE (use_p
, build_int_cst (TREE_TYPE (lhs
),
2609 ncopies
* nunits_out
- 1));
2610 update_stmt (use_stmt
);
2615 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (type
));
2616 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2617 set_vinfo_for_stmt (stmt
, NULL
);
2618 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2619 gsi_replace (gsi
, new_stmt
, false);
2625 struct simd_call_arg_info
2629 enum vect_def_type dt
;
2630 HOST_WIDE_INT linear_step
;
2634 /* Function vectorizable_simd_clone_call.
2636 Check if STMT performs a function call that can be vectorized
2637 by calling a simd clone of the function.
2638 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2639 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2640 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2643 vectorizable_simd_clone_call (gimple stmt
, gimple_stmt_iterator
*gsi
,
2644 gimple
*vec_stmt
, slp_tree slp_node
)
2649 tree vec_oprnd0
= NULL_TREE
;
2650 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
), prev_stmt_info
;
2652 unsigned int nunits
;
2653 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2654 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
2655 struct loop
*loop
= loop_vinfo
? LOOP_VINFO_LOOP (loop_vinfo
) : NULL
;
2656 tree fndecl
, new_temp
, def
;
2658 gimple new_stmt
= NULL
;
2660 vec
<simd_call_arg_info
> arginfo
= vNULL
;
2661 vec
<tree
> vargs
= vNULL
;
2663 tree lhs
, rtype
, ratype
;
2664 vec
<constructor_elt
, va_gc
> *ret_ctor_elts
;
2666 /* Is STMT a vectorizable call? */
2667 if (!is_gimple_call (stmt
))
2670 fndecl
= gimple_call_fndecl (stmt
);
2671 if (fndecl
== NULL_TREE
)
2674 struct cgraph_node
*node
= cgraph_node::get (fndecl
);
2675 if (node
== NULL
|| node
->simd_clones
== NULL
)
2678 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
2681 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
2684 if (gimple_call_lhs (stmt
)
2685 && TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
2688 gcc_checking_assert (!stmt_can_throw_internal (stmt
));
2690 vectype
= STMT_VINFO_VECTYPE (stmt_info
);
2692 if (loop_vinfo
&& nested_in_vect_loop_p (loop
, stmt
))
2696 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
2699 /* Process function arguments. */
2700 nargs
= gimple_call_num_args (stmt
);
2702 /* Bail out if the function has zero arguments. */
2706 arginfo
.create (nargs
);
2708 for (i
= 0; i
< nargs
; i
++)
2710 simd_call_arg_info thisarginfo
;
2713 thisarginfo
.linear_step
= 0;
2714 thisarginfo
.align
= 0;
2715 thisarginfo
.op
= NULL_TREE
;
2717 op
= gimple_call_arg (stmt
, i
);
2718 if (!vect_is_simple_use_1 (op
, stmt
, loop_vinfo
, bb_vinfo
,
2719 &def_stmt
, &def
, &thisarginfo
.dt
,
2720 &thisarginfo
.vectype
)
2721 || thisarginfo
.dt
== vect_uninitialized_def
)
2723 if (dump_enabled_p ())
2724 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2725 "use not simple.\n");
2730 if (thisarginfo
.dt
== vect_constant_def
2731 || thisarginfo
.dt
== vect_external_def
)
2732 gcc_assert (thisarginfo
.vectype
== NULL_TREE
);
2734 gcc_assert (thisarginfo
.vectype
!= NULL_TREE
);
2736 /* For linear arguments, the analyze phase should have saved
2737 the base and step in STMT_VINFO_SIMD_CLONE_INFO. */
2738 if (i
* 2 + 3 <= STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).length ()
2739 && STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 2 + 2])
2741 gcc_assert (vec_stmt
);
2742 thisarginfo
.linear_step
2743 = tree_to_shwi (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 2 + 2]);
2745 = STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 2 + 1];
2746 /* If loop has been peeled for alignment, we need to adjust it. */
2747 tree n1
= LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo
);
2748 tree n2
= LOOP_VINFO_NITERS (loop_vinfo
);
2751 tree bias
= fold_build2 (MINUS_EXPR
, TREE_TYPE (n1
), n1
, n2
);
2752 tree step
= STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 2 + 2];
2753 tree opt
= TREE_TYPE (thisarginfo
.op
);
2754 bias
= fold_convert (TREE_TYPE (step
), bias
);
2755 bias
= fold_build2 (MULT_EXPR
, TREE_TYPE (step
), bias
, step
);
2757 = fold_build2 (POINTER_TYPE_P (opt
)
2758 ? POINTER_PLUS_EXPR
: PLUS_EXPR
, opt
,
2759 thisarginfo
.op
, bias
);
2763 && thisarginfo
.dt
!= vect_constant_def
2764 && thisarginfo
.dt
!= vect_external_def
2766 && TREE_CODE (op
) == SSA_NAME
2767 && simple_iv (loop
, loop_containing_stmt (stmt
), op
,
2769 && tree_fits_shwi_p (iv
.step
))
2771 thisarginfo
.linear_step
= tree_to_shwi (iv
.step
);
2772 thisarginfo
.op
= iv
.base
;
2774 else if ((thisarginfo
.dt
== vect_constant_def
2775 || thisarginfo
.dt
== vect_external_def
)
2776 && POINTER_TYPE_P (TREE_TYPE (op
)))
2777 thisarginfo
.align
= get_pointer_alignment (op
) / BITS_PER_UNIT
;
2779 arginfo
.quick_push (thisarginfo
);
2782 unsigned int badness
= 0;
2783 struct cgraph_node
*bestn
= NULL
;
2784 if (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).exists ())
2785 bestn
= cgraph_node::get (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[0]);
2787 for (struct cgraph_node
*n
= node
->simd_clones
; n
!= NULL
;
2788 n
= n
->simdclone
->next_clone
)
2790 unsigned int this_badness
= 0;
2791 if (n
->simdclone
->simdlen
2792 > (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
2793 || n
->simdclone
->nargs
!= nargs
)
2795 if (n
->simdclone
->simdlen
2796 < (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo
))
2797 this_badness
+= (exact_log2 (LOOP_VINFO_VECT_FACTOR (loop_vinfo
))
2798 - exact_log2 (n
->simdclone
->simdlen
)) * 1024;
2799 if (n
->simdclone
->inbranch
)
2800 this_badness
+= 2048;
2801 int target_badness
= targetm
.simd_clone
.usable (n
);
2802 if (target_badness
< 0)
2804 this_badness
+= target_badness
* 512;
2805 /* FORNOW: Have to add code to add the mask argument. */
2806 if (n
->simdclone
->inbranch
)
2808 for (i
= 0; i
< nargs
; i
++)
2810 switch (n
->simdclone
->args
[i
].arg_type
)
2812 case SIMD_CLONE_ARG_TYPE_VECTOR
:
2813 if (!useless_type_conversion_p
2814 (n
->simdclone
->args
[i
].orig_type
,
2815 TREE_TYPE (gimple_call_arg (stmt
, i
))))
2817 else if (arginfo
[i
].dt
== vect_constant_def
2818 || arginfo
[i
].dt
== vect_external_def
2819 || arginfo
[i
].linear_step
)
2822 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
2823 if (arginfo
[i
].dt
!= vect_constant_def
2824 && arginfo
[i
].dt
!= vect_external_def
)
2827 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
2828 if (arginfo
[i
].dt
== vect_constant_def
2829 || arginfo
[i
].dt
== vect_external_def
2830 || (arginfo
[i
].linear_step
2831 != n
->simdclone
->args
[i
].linear_step
))
2834 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
2838 case SIMD_CLONE_ARG_TYPE_MASK
:
2841 if (i
== (size_t) -1)
2843 if (n
->simdclone
->args
[i
].alignment
> arginfo
[i
].align
)
2848 if (arginfo
[i
].align
)
2849 this_badness
+= (exact_log2 (arginfo
[i
].align
)
2850 - exact_log2 (n
->simdclone
->args
[i
].alignment
));
2852 if (i
== (size_t) -1)
2854 if (bestn
== NULL
|| this_badness
< badness
)
2857 badness
= this_badness
;
2867 for (i
= 0; i
< nargs
; i
++)
2868 if ((arginfo
[i
].dt
== vect_constant_def
2869 || arginfo
[i
].dt
== vect_external_def
)
2870 && bestn
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
2873 = get_vectype_for_scalar_type (TREE_TYPE (gimple_call_arg (stmt
,
2875 if (arginfo
[i
].vectype
== NULL
2876 || (TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
)
2877 > bestn
->simdclone
->simdlen
))
2884 fndecl
= bestn
->decl
;
2885 nunits
= bestn
->simdclone
->simdlen
;
2886 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
2888 /* If the function isn't const, only allow it in simd loops where user
2889 has asserted that at least nunits consecutive iterations can be
2890 performed using SIMD instructions. */
2891 if ((loop
== NULL
|| (unsigned) loop
->safelen
< nunits
)
2892 && gimple_vuse (stmt
))
2898 /* Sanity check: make sure that at least one copy of the vectorized stmt
2899 needs to be generated. */
2900 gcc_assert (ncopies
>= 1);
2902 if (!vec_stmt
) /* transformation not required. */
2904 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (bestn
->decl
);
2905 for (i
= 0; i
< nargs
; i
++)
2906 if (bestn
->simdclone
->args
[i
].arg_type
2907 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
2909 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_grow_cleared (i
* 2
2911 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (arginfo
[i
].op
);
2912 tree lst
= POINTER_TYPE_P (TREE_TYPE (arginfo
[i
].op
))
2913 ? size_type_node
: TREE_TYPE (arginfo
[i
].op
);
2914 tree ls
= build_int_cst (lst
, arginfo
[i
].linear_step
);
2915 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (ls
);
2917 STMT_VINFO_TYPE (stmt_info
) = call_simd_clone_vec_info_type
;
2918 if (dump_enabled_p ())
2919 dump_printf_loc (MSG_NOTE
, vect_location
,
2920 "=== vectorizable_simd_clone_call ===\n");
2921 /* vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL); */
2928 if (dump_enabled_p ())
2929 dump_printf_loc (MSG_NOTE
, vect_location
, "transform call.\n");
2932 scalar_dest
= gimple_call_lhs (stmt
);
2933 vec_dest
= NULL_TREE
;
2938 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
2939 rtype
= TREE_TYPE (TREE_TYPE (fndecl
));
2940 if (TREE_CODE (rtype
) == ARRAY_TYPE
)
2943 rtype
= TREE_TYPE (ratype
);
2947 prev_stmt_info
= NULL
;
2948 for (j
= 0; j
< ncopies
; ++j
)
2950 /* Build argument list for the vectorized call. */
2952 vargs
.create (nargs
);
2956 for (i
= 0; i
< nargs
; i
++)
2958 unsigned int k
, l
, m
, o
;
2960 op
= gimple_call_arg (stmt
, i
);
2961 switch (bestn
->simdclone
->args
[i
].arg_type
)
2963 case SIMD_CLONE_ARG_TYPE_VECTOR
:
2964 atype
= bestn
->simdclone
->args
[i
].vector_type
;
2965 o
= nunits
/ TYPE_VECTOR_SUBPARTS (atype
);
2966 for (m
= j
* o
; m
< (j
+ 1) * o
; m
++)
2968 if (TYPE_VECTOR_SUBPARTS (atype
)
2969 < TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
))
2971 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (atype
));
2972 k
= (TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
)
2973 / TYPE_VECTOR_SUBPARTS (atype
));
2974 gcc_assert ((k
& (k
- 1)) == 0);
2977 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
2980 vec_oprnd0
= arginfo
[i
].op
;
2981 if ((m
& (k
- 1)) == 0)
2983 = vect_get_vec_def_for_stmt_copy (arginfo
[i
].dt
,
2986 arginfo
[i
].op
= vec_oprnd0
;
2988 = build3 (BIT_FIELD_REF
, atype
, vec_oprnd0
,
2990 bitsize_int ((m
& (k
- 1)) * prec
));
2992 = gimple_build_assign (make_ssa_name (atype
),
2994 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2995 vargs
.safe_push (gimple_assign_lhs (new_stmt
));
2999 k
= (TYPE_VECTOR_SUBPARTS (atype
)
3000 / TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
));
3001 gcc_assert ((k
& (k
- 1)) == 0);
3002 vec
<constructor_elt
, va_gc
> *ctor_elts
;
3004 vec_alloc (ctor_elts
, k
);
3007 for (l
= 0; l
< k
; l
++)
3009 if (m
== 0 && l
== 0)
3011 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
3014 = vect_get_vec_def_for_stmt_copy (arginfo
[i
].dt
,
3016 arginfo
[i
].op
= vec_oprnd0
;
3019 CONSTRUCTOR_APPEND_ELT (ctor_elts
, NULL_TREE
,
3023 vargs
.safe_push (vec_oprnd0
);
3026 vec_oprnd0
= build_constructor (atype
, ctor_elts
);
3028 = gimple_build_assign (make_ssa_name (atype
),
3030 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3031 vargs
.safe_push (gimple_assign_lhs (new_stmt
));
3036 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
3037 vargs
.safe_push (op
);
3039 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
3044 = force_gimple_operand (arginfo
[i
].op
, &stmts
, true,
3049 edge pe
= loop_preheader_edge (loop
);
3050 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
3051 gcc_assert (!new_bb
);
3053 tree phi_res
= copy_ssa_name (op
);
3054 gphi
*new_phi
= create_phi_node (phi_res
, loop
->header
);
3055 set_vinfo_for_stmt (new_phi
,
3056 new_stmt_vec_info (new_phi
, loop_vinfo
,
3058 add_phi_arg (new_phi
, arginfo
[i
].op
,
3059 loop_preheader_edge (loop
), UNKNOWN_LOCATION
);
3061 = POINTER_TYPE_P (TREE_TYPE (op
))
3062 ? POINTER_PLUS_EXPR
: PLUS_EXPR
;
3063 tree type
= POINTER_TYPE_P (TREE_TYPE (op
))
3064 ? sizetype
: TREE_TYPE (op
);
3066 = wi::mul (bestn
->simdclone
->args
[i
].linear_step
,
3068 tree tcst
= wide_int_to_tree (type
, cst
);
3069 tree phi_arg
= copy_ssa_name (op
);
3071 = gimple_build_assign (phi_arg
, code
, phi_res
, tcst
);
3072 gimple_stmt_iterator si
= gsi_after_labels (loop
->header
);
3073 gsi_insert_after (&si
, new_stmt
, GSI_NEW_STMT
);
3074 set_vinfo_for_stmt (new_stmt
,
3075 new_stmt_vec_info (new_stmt
, loop_vinfo
,
3077 add_phi_arg (new_phi
, phi_arg
, loop_latch_edge (loop
),
3079 arginfo
[i
].op
= phi_res
;
3080 vargs
.safe_push (phi_res
);
3085 = POINTER_TYPE_P (TREE_TYPE (op
))
3086 ? POINTER_PLUS_EXPR
: PLUS_EXPR
;
3087 tree type
= POINTER_TYPE_P (TREE_TYPE (op
))
3088 ? sizetype
: TREE_TYPE (op
);
3090 = wi::mul (bestn
->simdclone
->args
[i
].linear_step
,
3092 tree tcst
= wide_int_to_tree (type
, cst
);
3093 new_temp
= make_ssa_name (TREE_TYPE (op
));
3094 new_stmt
= gimple_build_assign (new_temp
, code
,
3095 arginfo
[i
].op
, tcst
);
3096 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3097 vargs
.safe_push (new_temp
);
3100 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
3106 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
3109 gcc_assert (ratype
|| TYPE_VECTOR_SUBPARTS (rtype
) == nunits
);
3111 new_temp
= create_tmp_var (ratype
);
3112 else if (TYPE_VECTOR_SUBPARTS (vectype
)
3113 == TYPE_VECTOR_SUBPARTS (rtype
))
3114 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3116 new_temp
= make_ssa_name (rtype
, new_stmt
);
3117 gimple_call_set_lhs (new_stmt
, new_temp
);
3119 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3123 if (TYPE_VECTOR_SUBPARTS (vectype
) < nunits
)
3126 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (vectype
));
3127 k
= nunits
/ TYPE_VECTOR_SUBPARTS (vectype
);
3128 gcc_assert ((k
& (k
- 1)) == 0);
3129 for (l
= 0; l
< k
; l
++)
3134 t
= build_fold_addr_expr (new_temp
);
3135 t
= build2 (MEM_REF
, vectype
, t
,
3136 build_int_cst (TREE_TYPE (t
),
3137 l
* prec
/ BITS_PER_UNIT
));
3140 t
= build3 (BIT_FIELD_REF
, vectype
, new_temp
,
3141 size_int (prec
), bitsize_int (l
* prec
));
3143 = gimple_build_assign (make_ssa_name (vectype
), t
);
3144 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3145 if (j
== 0 && l
== 0)
3146 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3148 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3150 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3155 tree clobber
= build_constructor (ratype
, NULL
);
3156 TREE_THIS_VOLATILE (clobber
) = 1;
3157 new_stmt
= gimple_build_assign (new_temp
, clobber
);
3158 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3162 else if (TYPE_VECTOR_SUBPARTS (vectype
) > nunits
)
3164 unsigned int k
= (TYPE_VECTOR_SUBPARTS (vectype
)
3165 / TYPE_VECTOR_SUBPARTS (rtype
));
3166 gcc_assert ((k
& (k
- 1)) == 0);
3167 if ((j
& (k
- 1)) == 0)
3168 vec_alloc (ret_ctor_elts
, k
);
3171 unsigned int m
, o
= nunits
/ TYPE_VECTOR_SUBPARTS (rtype
);
3172 for (m
= 0; m
< o
; m
++)
3174 tree tem
= build4 (ARRAY_REF
, rtype
, new_temp
,
3175 size_int (m
), NULL_TREE
, NULL_TREE
);
3177 = gimple_build_assign (make_ssa_name (rtype
), tem
);
3178 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3179 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts
, NULL_TREE
,
3180 gimple_assign_lhs (new_stmt
));
3182 tree clobber
= build_constructor (ratype
, NULL
);
3183 TREE_THIS_VOLATILE (clobber
) = 1;
3184 new_stmt
= gimple_build_assign (new_temp
, clobber
);
3185 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3188 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts
, NULL_TREE
, new_temp
);
3189 if ((j
& (k
- 1)) != k
- 1)
3191 vec_oprnd0
= build_constructor (vectype
, ret_ctor_elts
);
3193 = gimple_build_assign (make_ssa_name (vec_dest
), vec_oprnd0
);
3194 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3196 if ((unsigned) j
== k
- 1)
3197 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3199 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3201 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3206 tree t
= build_fold_addr_expr (new_temp
);
3207 t
= build2 (MEM_REF
, vectype
, t
,
3208 build_int_cst (TREE_TYPE (t
), 0));
3210 = gimple_build_assign (make_ssa_name (vec_dest
), t
);
3211 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3212 tree clobber
= build_constructor (ratype
, NULL
);
3213 TREE_THIS_VOLATILE (clobber
) = 1;
3214 vect_finish_stmt_generation (stmt
,
3215 gimple_build_assign (new_temp
,
3221 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3223 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3225 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3230 /* The call in STMT might prevent it from being removed in dce.
3231 We however cannot remove it here, due to the way the ssa name
3232 it defines is mapped to the new definition. So just replace
3233 rhs of the statement with something harmless. */
3240 type
= TREE_TYPE (scalar_dest
);
3241 if (is_pattern_stmt_p (stmt_info
))
3242 lhs
= gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info
));
3244 lhs
= gimple_call_lhs (stmt
);
3245 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (type
));
3248 new_stmt
= gimple_build_nop ();
3249 set_vinfo_for_stmt (new_stmt
, stmt_info
);
3250 set_vinfo_for_stmt (stmt
, NULL
);
3251 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
3252 gsi_replace (gsi
, new_stmt
, true);
3253 unlink_stmt_vdef (stmt
);
3259 /* Function vect_gen_widened_results_half
3261 Create a vector stmt whose code, type, number of arguments, and result
3262 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
3263 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at BSI.
3264 In the case that CODE is a CALL_EXPR, this means that a call to DECL
3265 needs to be created (DECL is a function-decl of a target-builtin).
3266 STMT is the original scalar stmt that we are vectorizing. */
3269 vect_gen_widened_results_half (enum tree_code code
,
3271 tree vec_oprnd0
, tree vec_oprnd1
, int op_type
,
3272 tree vec_dest
, gimple_stmt_iterator
*gsi
,
3278 /* Generate half of the widened result: */
3279 if (code
== CALL_EXPR
)
3281 /* Target specific support */
3282 if (op_type
== binary_op
)
3283 new_stmt
= gimple_build_call (decl
, 2, vec_oprnd0
, vec_oprnd1
);
3285 new_stmt
= gimple_build_call (decl
, 1, vec_oprnd0
);
3286 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3287 gimple_call_set_lhs (new_stmt
, new_temp
);
3291 /* Generic support */
3292 gcc_assert (op_type
== TREE_CODE_LENGTH (code
));
3293 if (op_type
!= binary_op
)
3295 new_stmt
= gimple_build_assign (vec_dest
, code
, vec_oprnd0
, vec_oprnd1
);
3296 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3297 gimple_assign_set_lhs (new_stmt
, new_temp
);
3299 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3305 /* Get vectorized definitions for loop-based vectorization. For the first
3306 operand we call vect_get_vec_def_for_operand() (with OPRND containing
3307 scalar operand), and for the rest we get a copy with
3308 vect_get_vec_def_for_stmt_copy() using the previous vector definition
3309 (stored in OPRND). See vect_get_vec_def_for_stmt_copy() for details.
3310 The vectors are collected into VEC_OPRNDS. */
3313 vect_get_loop_based_defs (tree
*oprnd
, gimple stmt
, enum vect_def_type dt
,
3314 vec
<tree
> *vec_oprnds
, int multi_step_cvt
)
3318 /* Get first vector operand. */
3319 /* All the vector operands except the very first one (that is scalar oprnd)
3321 if (TREE_CODE (TREE_TYPE (*oprnd
)) != VECTOR_TYPE
)
3322 vec_oprnd
= vect_get_vec_def_for_operand (*oprnd
, stmt
, NULL
);
3324 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, *oprnd
);
3326 vec_oprnds
->quick_push (vec_oprnd
);
3328 /* Get second vector operand. */
3329 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, vec_oprnd
);
3330 vec_oprnds
->quick_push (vec_oprnd
);
3334 /* For conversion in multiple steps, continue to get operands
3337 vect_get_loop_based_defs (oprnd
, stmt
, dt
, vec_oprnds
, multi_step_cvt
- 1);
3341 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
3342 For multi-step conversions store the resulting vectors and call the function
3346 vect_create_vectorized_demotion_stmts (vec
<tree
> *vec_oprnds
,
3347 int multi_step_cvt
, gimple stmt
,
3349 gimple_stmt_iterator
*gsi
,
3350 slp_tree slp_node
, enum tree_code code
,
3351 stmt_vec_info
*prev_stmt_info
)
3354 tree vop0
, vop1
, new_tmp
, vec_dest
;
3356 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3358 vec_dest
= vec_dsts
.pop ();
3360 for (i
= 0; i
< vec_oprnds
->length (); i
+= 2)
3362 /* Create demotion operation. */
3363 vop0
= (*vec_oprnds
)[i
];
3364 vop1
= (*vec_oprnds
)[i
+ 1];
3365 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
);
3366 new_tmp
= make_ssa_name (vec_dest
, new_stmt
);
3367 gimple_assign_set_lhs (new_stmt
, new_tmp
);
3368 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3371 /* Store the resulting vector for next recursive call. */
3372 (*vec_oprnds
)[i
/2] = new_tmp
;
3375 /* This is the last step of the conversion sequence. Store the
3376 vectors in SLP_NODE or in vector info of the scalar statement
3377 (or in STMT_VINFO_RELATED_STMT chain). */
3379 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3382 if (!*prev_stmt_info
)
3383 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
3385 STMT_VINFO_RELATED_STMT (*prev_stmt_info
) = new_stmt
;
3387 *prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3392 /* For multi-step demotion operations we first generate demotion operations
3393 from the source type to the intermediate types, and then combine the
3394 results (stored in VEC_OPRNDS) in demotion operation to the destination
3398 /* At each level of recursion we have half of the operands we had at the
3400 vec_oprnds
->truncate ((i
+1)/2);
3401 vect_create_vectorized_demotion_stmts (vec_oprnds
, multi_step_cvt
- 1,
3402 stmt
, vec_dsts
, gsi
, slp_node
,
3403 VEC_PACK_TRUNC_EXPR
,
3407 vec_dsts
.quick_push (vec_dest
);
3411 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
3412 and VEC_OPRNDS1 (for binary operations). For multi-step conversions store
3413 the resulting vectors and call the function recursively. */
3416 vect_create_vectorized_promotion_stmts (vec
<tree
> *vec_oprnds0
,
3417 vec
<tree
> *vec_oprnds1
,
3418 gimple stmt
, tree vec_dest
,
3419 gimple_stmt_iterator
*gsi
,
3420 enum tree_code code1
,
3421 enum tree_code code2
, tree decl1
,
3422 tree decl2
, int op_type
)
3425 tree vop0
, vop1
, new_tmp1
, new_tmp2
;
3426 gimple new_stmt1
, new_stmt2
;
3427 vec
<tree
> vec_tmp
= vNULL
;
3429 vec_tmp
.create (vec_oprnds0
->length () * 2);
3430 FOR_EACH_VEC_ELT (*vec_oprnds0
, i
, vop0
)
3432 if (op_type
== binary_op
)
3433 vop1
= (*vec_oprnds1
)[i
];
3437 /* Generate the two halves of promotion operation. */
3438 new_stmt1
= vect_gen_widened_results_half (code1
, decl1
, vop0
, vop1
,
3439 op_type
, vec_dest
, gsi
, stmt
);
3440 new_stmt2
= vect_gen_widened_results_half (code2
, decl2
, vop0
, vop1
,
3441 op_type
, vec_dest
, gsi
, stmt
);
3442 if (is_gimple_call (new_stmt1
))
3444 new_tmp1
= gimple_call_lhs (new_stmt1
);
3445 new_tmp2
= gimple_call_lhs (new_stmt2
);
3449 new_tmp1
= gimple_assign_lhs (new_stmt1
);
3450 new_tmp2
= gimple_assign_lhs (new_stmt2
);
3453 /* Store the results for the next step. */
3454 vec_tmp
.quick_push (new_tmp1
);
3455 vec_tmp
.quick_push (new_tmp2
);
3458 vec_oprnds0
->release ();
3459 *vec_oprnds0
= vec_tmp
;
3463 /* Check if STMT performs a conversion operation, that can be vectorized.
3464 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3465 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3466 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3469 vectorizable_conversion (gimple stmt
, gimple_stmt_iterator
*gsi
,
3470 gimple
*vec_stmt
, slp_tree slp_node
)
3474 tree op0
, op1
= NULL_TREE
;
3475 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
3476 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3477 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
3478 enum tree_code code
, code1
= ERROR_MARK
, code2
= ERROR_MARK
;
3479 enum tree_code codecvt1
= ERROR_MARK
, codecvt2
= ERROR_MARK
;
3480 tree decl1
= NULL_TREE
, decl2
= NULL_TREE
;
3484 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
3485 gimple new_stmt
= NULL
;
3486 stmt_vec_info prev_stmt_info
;
3489 tree vectype_out
, vectype_in
;
3491 tree lhs_type
, rhs_type
;
3492 enum { NARROW
, NONE
, WIDEN
} modifier
;
3493 vec
<tree
> vec_oprnds0
= vNULL
;
3494 vec
<tree
> vec_oprnds1
= vNULL
;
3496 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
3497 int multi_step_cvt
= 0;
3498 vec
<tree
> vec_dsts
= vNULL
;
3499 vec
<tree
> interm_types
= vNULL
;
3500 tree last_oprnd
, intermediate_type
, cvt_type
= NULL_TREE
;
3502 machine_mode rhs_mode
;
3503 unsigned short fltsz
;
3505 /* Is STMT a vectorizable conversion? */
3507 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
3510 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
3513 if (!is_gimple_assign (stmt
))
3516 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
3519 code
= gimple_assign_rhs_code (stmt
);
3520 if (!CONVERT_EXPR_CODE_P (code
)
3521 && code
!= FIX_TRUNC_EXPR
3522 && code
!= FLOAT_EXPR
3523 && code
!= WIDEN_MULT_EXPR
3524 && code
!= WIDEN_LSHIFT_EXPR
)
3527 op_type
= TREE_CODE_LENGTH (code
);
3529 /* Check types of lhs and rhs. */
3530 scalar_dest
= gimple_assign_lhs (stmt
);
3531 lhs_type
= TREE_TYPE (scalar_dest
);
3532 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
3534 op0
= gimple_assign_rhs1 (stmt
);
3535 rhs_type
= TREE_TYPE (op0
);
3537 if ((code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
3538 && !((INTEGRAL_TYPE_P (lhs_type
)
3539 && INTEGRAL_TYPE_P (rhs_type
))
3540 || (SCALAR_FLOAT_TYPE_P (lhs_type
)
3541 && SCALAR_FLOAT_TYPE_P (rhs_type
))))
3544 if ((INTEGRAL_TYPE_P (lhs_type
)
3545 && (TYPE_PRECISION (lhs_type
)
3546 != GET_MODE_PRECISION (TYPE_MODE (lhs_type
))))
3547 || (INTEGRAL_TYPE_P (rhs_type
)
3548 && (TYPE_PRECISION (rhs_type
)
3549 != GET_MODE_PRECISION (TYPE_MODE (rhs_type
)))))
3551 if (dump_enabled_p ())
3552 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3553 "type conversion to/from bit-precision unsupported."
3558 /* Check the operands of the operation. */
3559 if (!vect_is_simple_use_1 (op0
, stmt
, loop_vinfo
, bb_vinfo
,
3560 &def_stmt
, &def
, &dt
[0], &vectype_in
))
3562 if (dump_enabled_p ())
3563 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3564 "use not simple.\n");
3567 if (op_type
== binary_op
)
3571 op1
= gimple_assign_rhs2 (stmt
);
3572 gcc_assert (code
== WIDEN_MULT_EXPR
|| code
== WIDEN_LSHIFT_EXPR
);
3573 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
3575 if (CONSTANT_CLASS_P (op0
))
3576 ok
= vect_is_simple_use_1 (op1
, stmt
, loop_vinfo
, bb_vinfo
,
3577 &def_stmt
, &def
, &dt
[1], &vectype_in
);
3579 ok
= vect_is_simple_use (op1
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
3584 if (dump_enabled_p ())
3585 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3586 "use not simple.\n");
3591 /* If op0 is an external or constant defs use a vector type of
3592 the same size as the output vector type. */
3594 vectype_in
= get_same_sized_vectype (rhs_type
, vectype_out
);
3596 gcc_assert (vectype_in
);
3599 if (dump_enabled_p ())
3601 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3602 "no vectype for scalar type ");
3603 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
3604 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
3610 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
3611 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
3612 if (nunits_in
< nunits_out
)
3614 else if (nunits_out
== nunits_in
)
3619 /* Multiple types in SLP are handled by creating the appropriate number of
3620 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3622 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
3624 else if (modifier
== NARROW
)
3625 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_out
;
3627 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
3629 /* Sanity check: make sure that at least one copy of the vectorized stmt
3630 needs to be generated. */
3631 gcc_assert (ncopies
>= 1);
3633 /* Supportable by target? */
3637 if (code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
3639 if (supportable_convert_operation (code
, vectype_out
, vectype_in
,
3644 if (dump_enabled_p ())
3645 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3646 "conversion not supported by target.\n");
3650 if (supportable_widening_operation (code
, stmt
, vectype_out
, vectype_in
,
3651 &code1
, &code2
, &multi_step_cvt
,
3654 /* Binary widening operation can only be supported directly by the
3656 gcc_assert (!(multi_step_cvt
&& op_type
== binary_op
));
3660 if (code
!= FLOAT_EXPR
3661 || (GET_MODE_SIZE (TYPE_MODE (lhs_type
))
3662 <= GET_MODE_SIZE (TYPE_MODE (rhs_type
))))
3665 rhs_mode
= TYPE_MODE (rhs_type
);
3666 fltsz
= GET_MODE_SIZE (TYPE_MODE (lhs_type
));
3667 for (rhs_mode
= GET_MODE_2XWIDER_MODE (TYPE_MODE (rhs_type
));
3668 rhs_mode
!= VOIDmode
&& GET_MODE_SIZE (rhs_mode
) <= fltsz
;
3669 rhs_mode
= GET_MODE_2XWIDER_MODE (rhs_mode
))
3672 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode
), 0);
3673 cvt_type
= get_same_sized_vectype (cvt_type
, vectype_in
);
3674 if (cvt_type
== NULL_TREE
)
3677 if (GET_MODE_SIZE (rhs_mode
) == fltsz
)
3679 if (!supportable_convert_operation (code
, vectype_out
,
3680 cvt_type
, &decl1
, &codecvt1
))
3683 else if (!supportable_widening_operation (code
, stmt
, vectype_out
,
3684 cvt_type
, &codecvt1
,
3685 &codecvt2
, &multi_step_cvt
,
3689 gcc_assert (multi_step_cvt
== 0);
3691 if (supportable_widening_operation (NOP_EXPR
, stmt
, cvt_type
,
3692 vectype_in
, &code1
, &code2
,
3693 &multi_step_cvt
, &interm_types
))
3697 if (rhs_mode
== VOIDmode
|| GET_MODE_SIZE (rhs_mode
) > fltsz
)
3700 if (GET_MODE_SIZE (rhs_mode
) == fltsz
)
3701 codecvt2
= ERROR_MARK
;
3705 interm_types
.safe_push (cvt_type
);
3706 cvt_type
= NULL_TREE
;
3711 gcc_assert (op_type
== unary_op
);
3712 if (supportable_narrowing_operation (code
, vectype_out
, vectype_in
,
3713 &code1
, &multi_step_cvt
,
3717 if (code
!= FIX_TRUNC_EXPR
3718 || (GET_MODE_SIZE (TYPE_MODE (lhs_type
))
3719 >= GET_MODE_SIZE (TYPE_MODE (rhs_type
))))
3722 rhs_mode
= TYPE_MODE (rhs_type
);
3724 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode
), 0);
3725 cvt_type
= get_same_sized_vectype (cvt_type
, vectype_in
);
3726 if (cvt_type
== NULL_TREE
)
3728 if (!supportable_convert_operation (code
, cvt_type
, vectype_in
,
3731 if (supportable_narrowing_operation (NOP_EXPR
, vectype_out
, cvt_type
,
3732 &code1
, &multi_step_cvt
,
3741 if (!vec_stmt
) /* transformation not required. */
3743 if (dump_enabled_p ())
3744 dump_printf_loc (MSG_NOTE
, vect_location
,
3745 "=== vectorizable_conversion ===\n");
3746 if (code
== FIX_TRUNC_EXPR
|| code
== FLOAT_EXPR
)
3748 STMT_VINFO_TYPE (stmt_info
) = type_conversion_vec_info_type
;
3749 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
3751 else if (modifier
== NARROW
)
3753 STMT_VINFO_TYPE (stmt_info
) = type_demotion_vec_info_type
;
3754 vect_model_promotion_demotion_cost (stmt_info
, dt
, multi_step_cvt
);
3758 STMT_VINFO_TYPE (stmt_info
) = type_promotion_vec_info_type
;
3759 vect_model_promotion_demotion_cost (stmt_info
, dt
, multi_step_cvt
);
3761 interm_types
.release ();
3766 if (dump_enabled_p ())
3767 dump_printf_loc (MSG_NOTE
, vect_location
,
3768 "transform conversion. ncopies = %d.\n", ncopies
);
3770 if (op_type
== binary_op
)
3772 if (CONSTANT_CLASS_P (op0
))
3773 op0
= fold_convert (TREE_TYPE (op1
), op0
);
3774 else if (CONSTANT_CLASS_P (op1
))
3775 op1
= fold_convert (TREE_TYPE (op0
), op1
);
3778 /* In case of multi-step conversion, we first generate conversion operations
3779 to the intermediate types, and then from that types to the final one.
3780 We create vector destinations for the intermediate type (TYPES) received
3781 from supportable_*_operation, and store them in the correct order
3782 for future use in vect_create_vectorized_*_stmts (). */
3783 vec_dsts
.create (multi_step_cvt
+ 1);
3784 vec_dest
= vect_create_destination_var (scalar_dest
,
3785 (cvt_type
&& modifier
== WIDEN
)
3786 ? cvt_type
: vectype_out
);
3787 vec_dsts
.quick_push (vec_dest
);
3791 for (i
= interm_types
.length () - 1;
3792 interm_types
.iterate (i
, &intermediate_type
); i
--)
3794 vec_dest
= vect_create_destination_var (scalar_dest
,
3796 vec_dsts
.quick_push (vec_dest
);
3801 vec_dest
= vect_create_destination_var (scalar_dest
,
3803 ? vectype_out
: cvt_type
);
3807 if (modifier
== WIDEN
)
3809 vec_oprnds0
.create (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1);
3810 if (op_type
== binary_op
)
3811 vec_oprnds1
.create (1);
3813 else if (modifier
== NARROW
)
3814 vec_oprnds0
.create (
3815 2 * (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1));
3817 else if (code
== WIDEN_LSHIFT_EXPR
)
3818 vec_oprnds1
.create (slp_node
->vec_stmts_size
);
3821 prev_stmt_info
= NULL
;
3825 for (j
= 0; j
< ncopies
; j
++)
3828 vect_get_vec_defs (op0
, NULL
, stmt
, &vec_oprnds0
, NULL
, slp_node
,
3831 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, NULL
);
3833 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
3835 /* Arguments are ready, create the new vector stmt. */
3836 if (code1
== CALL_EXPR
)
3838 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
3839 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3840 gimple_call_set_lhs (new_stmt
, new_temp
);
3844 gcc_assert (TREE_CODE_LENGTH (code1
) == unary_op
);
3845 new_stmt
= gimple_build_assign (vec_dest
, code1
, vop0
);
3846 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3847 gimple_assign_set_lhs (new_stmt
, new_temp
);
3850 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3852 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3855 if (!prev_stmt_info
)
3856 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3858 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3859 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3866 /* In case the vectorization factor (VF) is bigger than the number
3867 of elements that we can fit in a vectype (nunits), we have to
3868 generate more than one vector stmt - i.e - we need to "unroll"
3869 the vector stmt by a factor VF/nunits. */
3870 for (j
= 0; j
< ncopies
; j
++)
3877 if (code
== WIDEN_LSHIFT_EXPR
)
3882 /* Store vec_oprnd1 for every vector stmt to be created
3883 for SLP_NODE. We check during the analysis that all
3884 the shift arguments are the same. */
3885 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
3886 vec_oprnds1
.quick_push (vec_oprnd1
);
3888 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
3892 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
,
3893 &vec_oprnds1
, slp_node
, -1);
3897 vec_oprnd0
= vect_get_vec_def_for_operand (op0
, stmt
, NULL
);
3898 vec_oprnds0
.quick_push (vec_oprnd0
);
3899 if (op_type
== binary_op
)
3901 if (code
== WIDEN_LSHIFT_EXPR
)
3904 vec_oprnd1
= vect_get_vec_def_for_operand (op1
, stmt
,
3906 vec_oprnds1
.quick_push (vec_oprnd1
);
3912 vec_oprnd0
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd0
);
3913 vec_oprnds0
.truncate (0);
3914 vec_oprnds0
.quick_push (vec_oprnd0
);
3915 if (op_type
== binary_op
)
3917 if (code
== WIDEN_LSHIFT_EXPR
)
3920 vec_oprnd1
= vect_get_vec_def_for_stmt_copy (dt
[1],
3922 vec_oprnds1
.truncate (0);
3923 vec_oprnds1
.quick_push (vec_oprnd1
);
3927 /* Arguments are ready. Create the new vector stmts. */
3928 for (i
= multi_step_cvt
; i
>= 0; i
--)
3930 tree this_dest
= vec_dsts
[i
];
3931 enum tree_code c1
= code1
, c2
= code2
;
3932 if (i
== 0 && codecvt2
!= ERROR_MARK
)
3937 vect_create_vectorized_promotion_stmts (&vec_oprnds0
,
3939 stmt
, this_dest
, gsi
,
3940 c1
, c2
, decl1
, decl2
,
3944 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
3948 if (codecvt1
== CALL_EXPR
)
3950 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
3951 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3952 gimple_call_set_lhs (new_stmt
, new_temp
);
3956 gcc_assert (TREE_CODE_LENGTH (codecvt1
) == unary_op
);
3957 new_temp
= make_ssa_name (vec_dest
);
3958 new_stmt
= gimple_build_assign (new_temp
, codecvt1
,
3962 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3965 new_stmt
= SSA_NAME_DEF_STMT (vop0
);
3968 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3971 if (!prev_stmt_info
)
3972 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
3974 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3975 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3980 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
3984 /* In case the vectorization factor (VF) is bigger than the number
3985 of elements that we can fit in a vectype (nunits), we have to
3986 generate more than one vector stmt - i.e - we need to "unroll"
3987 the vector stmt by a factor VF/nunits. */
3988 for (j
= 0; j
< ncopies
; j
++)
3992 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
3996 vec_oprnds0
.truncate (0);
3997 vect_get_loop_based_defs (&last_oprnd
, stmt
, dt
[0], &vec_oprnds0
,
3998 vect_pow2 (multi_step_cvt
) - 1);
4001 /* Arguments are ready. Create the new vector stmts. */
4003 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4005 if (codecvt1
== CALL_EXPR
)
4007 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
4008 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4009 gimple_call_set_lhs (new_stmt
, new_temp
);
4013 gcc_assert (TREE_CODE_LENGTH (codecvt1
) == unary_op
);
4014 new_temp
= make_ssa_name (vec_dest
);
4015 new_stmt
= gimple_build_assign (new_temp
, codecvt1
,
4019 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4020 vec_oprnds0
[i
] = new_temp
;
4023 vect_create_vectorized_demotion_stmts (&vec_oprnds0
, multi_step_cvt
,
4024 stmt
, vec_dsts
, gsi
,
4029 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
4033 vec_oprnds0
.release ();
4034 vec_oprnds1
.release ();
4035 vec_dsts
.release ();
4036 interm_types
.release ();
4042 /* Function vectorizable_assignment.
4044 Check if STMT performs an assignment (copy) that can be vectorized.
4045 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4046 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4047 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4050 vectorizable_assignment (gimple stmt
, gimple_stmt_iterator
*gsi
,
4051 gimple
*vec_stmt
, slp_tree slp_node
)
4056 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4057 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4061 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
4064 vec
<tree
> vec_oprnds
= vNULL
;
4066 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4067 gimple new_stmt
= NULL
;
4068 stmt_vec_info prev_stmt_info
= NULL
;
4069 enum tree_code code
;
4072 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4075 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
4078 /* Is vectorizable assignment? */
4079 if (!is_gimple_assign (stmt
))
4082 scalar_dest
= gimple_assign_lhs (stmt
);
4083 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
4086 code
= gimple_assign_rhs_code (stmt
);
4087 if (gimple_assign_single_p (stmt
)
4088 || code
== PAREN_EXPR
4089 || CONVERT_EXPR_CODE_P (code
))
4090 op
= gimple_assign_rhs1 (stmt
);
4094 if (code
== VIEW_CONVERT_EXPR
)
4095 op
= TREE_OPERAND (op
, 0);
4097 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
4098 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
4100 /* Multiple types in SLP are handled by creating the appropriate number of
4101 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4103 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
4106 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
4108 gcc_assert (ncopies
>= 1);
4110 if (!vect_is_simple_use_1 (op
, stmt
, loop_vinfo
, bb_vinfo
,
4111 &def_stmt
, &def
, &dt
[0], &vectype_in
))
4113 if (dump_enabled_p ())
4114 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4115 "use not simple.\n");
4119 /* We can handle NOP_EXPR conversions that do not change the number
4120 of elements or the vector size. */
4121 if ((CONVERT_EXPR_CODE_P (code
)
4122 || code
== VIEW_CONVERT_EXPR
)
4124 || TYPE_VECTOR_SUBPARTS (vectype_in
) != nunits
4125 || (GET_MODE_SIZE (TYPE_MODE (vectype
))
4126 != GET_MODE_SIZE (TYPE_MODE (vectype_in
)))))
4129 /* We do not handle bit-precision changes. */
4130 if ((CONVERT_EXPR_CODE_P (code
)
4131 || code
== VIEW_CONVERT_EXPR
)
4132 && INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest
))
4133 && ((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4134 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4135 || ((TYPE_PRECISION (TREE_TYPE (op
))
4136 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (op
))))))
4137 /* But a conversion that does not change the bit-pattern is ok. */
4138 && !((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4139 > TYPE_PRECISION (TREE_TYPE (op
)))
4140 && TYPE_UNSIGNED (TREE_TYPE (op
))))
4142 if (dump_enabled_p ())
4143 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4144 "type conversion to/from bit-precision "
4149 if (!vec_stmt
) /* transformation not required. */
4151 STMT_VINFO_TYPE (stmt_info
) = assignment_vec_info_type
;
4152 if (dump_enabled_p ())
4153 dump_printf_loc (MSG_NOTE
, vect_location
,
4154 "=== vectorizable_assignment ===\n");
4155 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4160 if (dump_enabled_p ())
4161 dump_printf_loc (MSG_NOTE
, vect_location
, "transform assignment.\n");
4164 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4167 for (j
= 0; j
< ncopies
; j
++)
4171 vect_get_vec_defs (op
, NULL
, stmt
, &vec_oprnds
, NULL
, slp_node
, -1);
4173 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds
, NULL
);
4175 /* Arguments are ready. create the new vector stmt. */
4176 FOR_EACH_VEC_ELT (vec_oprnds
, i
, vop
)
4178 if (CONVERT_EXPR_CODE_P (code
)
4179 || code
== VIEW_CONVERT_EXPR
)
4180 vop
= build1 (VIEW_CONVERT_EXPR
, vectype
, vop
);
4181 new_stmt
= gimple_build_assign (vec_dest
, vop
);
4182 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4183 gimple_assign_set_lhs (new_stmt
, new_temp
);
4184 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4186 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4193 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4195 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4197 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4200 vec_oprnds
.release ();
4205 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
4206 either as shift by a scalar or by a vector. */
4209 vect_supportable_shift (enum tree_code code
, tree scalar_type
)
4212 machine_mode vec_mode
;
4217 vectype
= get_vectype_for_scalar_type (scalar_type
);
4221 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
4223 || optab_handler (optab
, TYPE_MODE (vectype
)) == CODE_FOR_nothing
)
4225 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4227 || (optab_handler (optab
, TYPE_MODE (vectype
))
4228 == CODE_FOR_nothing
))
4232 vec_mode
= TYPE_MODE (vectype
);
4233 icode
= (int) optab_handler (optab
, vec_mode
);
4234 if (icode
== CODE_FOR_nothing
)
4241 /* Function vectorizable_shift.
4243 Check if STMT performs a shift operation that can be vectorized.
4244 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4245 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4246 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4249 vectorizable_shift (gimple stmt
, gimple_stmt_iterator
*gsi
,
4250 gimple
*vec_stmt
, slp_tree slp_node
)
4254 tree op0
, op1
= NULL
;
4255 tree vec_oprnd1
= NULL_TREE
;
4256 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4258 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4259 enum tree_code code
;
4260 machine_mode vec_mode
;
4264 machine_mode optab_op2_mode
;
4267 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
4268 gimple new_stmt
= NULL
;
4269 stmt_vec_info prev_stmt_info
;
4276 vec
<tree
> vec_oprnds0
= vNULL
;
4277 vec
<tree
> vec_oprnds1
= vNULL
;
4280 bool scalar_shift_arg
= true;
4281 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4284 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4287 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
4290 /* Is STMT a vectorizable binary/unary operation? */
4291 if (!is_gimple_assign (stmt
))
4294 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
4297 code
= gimple_assign_rhs_code (stmt
);
4299 if (!(code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
4300 || code
== RROTATE_EXPR
))
4303 scalar_dest
= gimple_assign_lhs (stmt
);
4304 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
4305 if (TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4306 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4308 if (dump_enabled_p ())
4309 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4310 "bit-precision shifts not supported.\n");
4314 op0
= gimple_assign_rhs1 (stmt
);
4315 if (!vect_is_simple_use_1 (op0
, stmt
, loop_vinfo
, bb_vinfo
,
4316 &def_stmt
, &def
, &dt
[0], &vectype
))
4318 if (dump_enabled_p ())
4319 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4320 "use not simple.\n");
4323 /* If op0 is an external or constant def use a vector type with
4324 the same size as the output vector type. */
4326 vectype
= get_same_sized_vectype (TREE_TYPE (op0
), vectype_out
);
4328 gcc_assert (vectype
);
4331 if (dump_enabled_p ())
4332 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4333 "no vectype for scalar type\n");
4337 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
4338 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
4339 if (nunits_out
!= nunits_in
)
4342 op1
= gimple_assign_rhs2 (stmt
);
4343 if (!vect_is_simple_use_1 (op1
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
4344 &def
, &dt
[1], &op1_vectype
))
4346 if (dump_enabled_p ())
4347 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4348 "use not simple.\n");
4353 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
4357 /* Multiple types in SLP are handled by creating the appropriate number of
4358 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4360 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
4363 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
4365 gcc_assert (ncopies
>= 1);
4367 /* Determine whether the shift amount is a vector, or scalar. If the
4368 shift/rotate amount is a vector, use the vector/vector shift optabs. */
4370 if (dt
[1] == vect_internal_def
&& !slp_node
)
4371 scalar_shift_arg
= false;
4372 else if (dt
[1] == vect_constant_def
4373 || dt
[1] == vect_external_def
4374 || dt
[1] == vect_internal_def
)
4376 /* In SLP, need to check whether the shift count is the same,
4377 in loops if it is a constant or invariant, it is always
4381 vec
<gimple
> stmts
= SLP_TREE_SCALAR_STMTS (slp_node
);
4384 FOR_EACH_VEC_ELT (stmts
, k
, slpstmt
)
4385 if (!operand_equal_p (gimple_assign_rhs2 (slpstmt
), op1
, 0))
4386 scalar_shift_arg
= false;
4391 if (dump_enabled_p ())
4392 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4393 "operand mode requires invariant argument.\n");
4397 /* Vector shifted by vector. */
4398 if (!scalar_shift_arg
)
4400 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4401 if (dump_enabled_p ())
4402 dump_printf_loc (MSG_NOTE
, vect_location
,
4403 "vector/vector shift/rotate found.\n");
4406 op1_vectype
= get_same_sized_vectype (TREE_TYPE (op1
), vectype_out
);
4407 if (op1_vectype
== NULL_TREE
4408 || TYPE_MODE (op1_vectype
) != TYPE_MODE (vectype
))
4410 if (dump_enabled_p ())
4411 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4412 "unusable type for last operand in"
4413 " vector/vector shift/rotate.\n");
4417 /* See if the machine has a vector shifted by scalar insn and if not
4418 then see if it has a vector shifted by vector insn. */
4421 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
4423 && optab_handler (optab
, TYPE_MODE (vectype
)) != CODE_FOR_nothing
)
4425 if (dump_enabled_p ())
4426 dump_printf_loc (MSG_NOTE
, vect_location
,
4427 "vector/scalar shift/rotate found.\n");
4431 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4433 && (optab_handler (optab
, TYPE_MODE (vectype
))
4434 != CODE_FOR_nothing
))
4436 scalar_shift_arg
= false;
4438 if (dump_enabled_p ())
4439 dump_printf_loc (MSG_NOTE
, vect_location
,
4440 "vector/vector shift/rotate found.\n");
4442 /* Unlike the other binary operators, shifts/rotates have
4443 the rhs being int, instead of the same type as the lhs,
4444 so make sure the scalar is the right type if we are
4445 dealing with vectors of long long/long/short/char. */
4446 if (dt
[1] == vect_constant_def
)
4447 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
4448 else if (!useless_type_conversion_p (TREE_TYPE (vectype
),
4452 && TYPE_MODE (TREE_TYPE (vectype
))
4453 != TYPE_MODE (TREE_TYPE (op1
)))
4455 if (dump_enabled_p ())
4456 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4457 "unusable type for last operand in"
4458 " vector/vector shift/rotate.\n");
4461 if (vec_stmt
&& !slp_node
)
4463 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
4464 op1
= vect_init_vector (stmt
, op1
,
4465 TREE_TYPE (vectype
), NULL
);
4472 /* Supportable by target? */
4475 if (dump_enabled_p ())
4476 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4480 vec_mode
= TYPE_MODE (vectype
);
4481 icode
= (int) optab_handler (optab
, vec_mode
);
4482 if (icode
== CODE_FOR_nothing
)
4484 if (dump_enabled_p ())
4485 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4486 "op not supported by target.\n");
4487 /* Check only during analysis. */
4488 if (GET_MODE_SIZE (vec_mode
) != UNITS_PER_WORD
4489 || (vf
< vect_min_worthwhile_factor (code
)
4492 if (dump_enabled_p ())
4493 dump_printf_loc (MSG_NOTE
, vect_location
,
4494 "proceeding using word mode.\n");
4497 /* Worthwhile without SIMD support? Check only during analysis. */
4498 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
4499 && vf
< vect_min_worthwhile_factor (code
)
4502 if (dump_enabled_p ())
4503 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4504 "not worthwhile without SIMD support.\n");
4508 if (!vec_stmt
) /* transformation not required. */
4510 STMT_VINFO_TYPE (stmt_info
) = shift_vec_info_type
;
4511 if (dump_enabled_p ())
4512 dump_printf_loc (MSG_NOTE
, vect_location
,
4513 "=== vectorizable_shift ===\n");
4514 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4520 if (dump_enabled_p ())
4521 dump_printf_loc (MSG_NOTE
, vect_location
,
4522 "transform binary/unary operation.\n");
4525 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4527 prev_stmt_info
= NULL
;
4528 for (j
= 0; j
< ncopies
; j
++)
4533 if (scalar_shift_arg
)
4535 /* Vector shl and shr insn patterns can be defined with scalar
4536 operand 2 (shift operand). In this case, use constant or loop
4537 invariant op1 directly, without extending it to vector mode
4539 optab_op2_mode
= insn_data
[icode
].operand
[2].mode
;
4540 if (!VECTOR_MODE_P (optab_op2_mode
))
4542 if (dump_enabled_p ())
4543 dump_printf_loc (MSG_NOTE
, vect_location
,
4544 "operand 1 using scalar mode.\n");
4546 vec_oprnds1
.create (slp_node
? slp_node
->vec_stmts_size
: 1);
4547 vec_oprnds1
.quick_push (vec_oprnd1
);
4550 /* Store vec_oprnd1 for every vector stmt to be created
4551 for SLP_NODE. We check during the analysis that all
4552 the shift arguments are the same.
4553 TODO: Allow different constants for different vector
4554 stmts generated for an SLP instance. */
4555 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
4556 vec_oprnds1
.quick_push (vec_oprnd1
);
4561 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
4562 (a special case for certain kind of vector shifts); otherwise,
4563 operand 1 should be of a vector type (the usual case). */
4565 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
4568 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
, &vec_oprnds1
,
4572 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, &vec_oprnds1
);
4574 /* Arguments are ready. Create the new vector stmt. */
4575 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4577 vop1
= vec_oprnds1
[i
];
4578 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
);
4579 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4580 gimple_assign_set_lhs (new_stmt
, new_temp
);
4581 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4583 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4590 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4592 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4593 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4596 vec_oprnds0
.release ();
4597 vec_oprnds1
.release ();
4603 /* Function vectorizable_operation.
4605 Check if STMT performs a binary, unary or ternary operation that can
4607 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4608 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4609 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4612 vectorizable_operation (gimple stmt
, gimple_stmt_iterator
*gsi
,
4613 gimple
*vec_stmt
, slp_tree slp_node
)
4617 tree op0
, op1
= NULL_TREE
, op2
= NULL_TREE
;
4618 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4620 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4621 enum tree_code code
;
4622 machine_mode vec_mode
;
4629 enum vect_def_type dt
[3]
4630 = {vect_unknown_def_type
, vect_unknown_def_type
, vect_unknown_def_type
};
4631 gimple new_stmt
= NULL
;
4632 stmt_vec_info prev_stmt_info
;
4638 vec
<tree
> vec_oprnds0
= vNULL
;
4639 vec
<tree
> vec_oprnds1
= vNULL
;
4640 vec
<tree
> vec_oprnds2
= vNULL
;
4641 tree vop0
, vop1
, vop2
;
4642 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4645 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4648 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
4651 /* Is STMT a vectorizable binary/unary operation? */
4652 if (!is_gimple_assign (stmt
))
4655 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
4658 code
= gimple_assign_rhs_code (stmt
);
4660 /* For pointer addition, we should use the normal plus for
4661 the vector addition. */
4662 if (code
== POINTER_PLUS_EXPR
)
4665 /* Support only unary or binary operations. */
4666 op_type
= TREE_CODE_LENGTH (code
);
4667 if (op_type
!= unary_op
&& op_type
!= binary_op
&& op_type
!= ternary_op
)
4669 if (dump_enabled_p ())
4670 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4671 "num. args = %d (not unary/binary/ternary op).\n",
4676 scalar_dest
= gimple_assign_lhs (stmt
);
4677 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
4679 /* Most operations cannot handle bit-precision types without extra
4681 if ((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4682 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4683 /* Exception are bitwise binary operations. */
4684 && code
!= BIT_IOR_EXPR
4685 && code
!= BIT_XOR_EXPR
4686 && code
!= BIT_AND_EXPR
)
4688 if (dump_enabled_p ())
4689 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4690 "bit-precision arithmetic not supported.\n");
4694 op0
= gimple_assign_rhs1 (stmt
);
4695 if (!vect_is_simple_use_1 (op0
, stmt
, loop_vinfo
, bb_vinfo
,
4696 &def_stmt
, &def
, &dt
[0], &vectype
))
4698 if (dump_enabled_p ())
4699 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4700 "use not simple.\n");
4703 /* If op0 is an external or constant def use a vector type with
4704 the same size as the output vector type. */
4706 vectype
= get_same_sized_vectype (TREE_TYPE (op0
), vectype_out
);
4708 gcc_assert (vectype
);
4711 if (dump_enabled_p ())
4713 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4714 "no vectype for scalar type ");
4715 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
4717 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
4723 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
4724 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
4725 if (nunits_out
!= nunits_in
)
4728 if (op_type
== binary_op
|| op_type
== ternary_op
)
4730 op1
= gimple_assign_rhs2 (stmt
);
4731 if (!vect_is_simple_use (op1
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
4734 if (dump_enabled_p ())
4735 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4736 "use not simple.\n");
4740 if (op_type
== ternary_op
)
4742 op2
= gimple_assign_rhs3 (stmt
);
4743 if (!vect_is_simple_use (op2
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
4746 if (dump_enabled_p ())
4747 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4748 "use not simple.\n");
4754 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
4758 /* Multiple types in SLP are handled by creating the appropriate number of
4759 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4761 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
4764 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
4766 gcc_assert (ncopies
>= 1);
4768 /* Shifts are handled in vectorizable_shift (). */
4769 if (code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
4770 || code
== RROTATE_EXPR
)
4773 /* Supportable by target? */
4775 vec_mode
= TYPE_MODE (vectype
);
4776 if (code
== MULT_HIGHPART_EXPR
)
4778 if (can_mult_highpart_p (vec_mode
, TYPE_UNSIGNED (vectype
)))
4779 icode
= LAST_INSN_CODE
;
4781 icode
= CODE_FOR_nothing
;
4785 optab
= optab_for_tree_code (code
, vectype
, optab_default
);
4788 if (dump_enabled_p ())
4789 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4793 icode
= (int) optab_handler (optab
, vec_mode
);
4796 if (icode
== CODE_FOR_nothing
)
4798 if (dump_enabled_p ())
4799 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4800 "op not supported by target.\n");
4801 /* Check only during analysis. */
4802 if (GET_MODE_SIZE (vec_mode
) != UNITS_PER_WORD
4803 || (!vec_stmt
&& vf
< vect_min_worthwhile_factor (code
)))
4805 if (dump_enabled_p ())
4806 dump_printf_loc (MSG_NOTE
, vect_location
,
4807 "proceeding using word mode.\n");
4810 /* Worthwhile without SIMD support? Check only during analysis. */
4811 if (!VECTOR_MODE_P (vec_mode
)
4813 && vf
< vect_min_worthwhile_factor (code
))
4815 if (dump_enabled_p ())
4816 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4817 "not worthwhile without SIMD support.\n");
4821 if (!vec_stmt
) /* transformation not required. */
4823 STMT_VINFO_TYPE (stmt_info
) = op_vec_info_type
;
4824 if (dump_enabled_p ())
4825 dump_printf_loc (MSG_NOTE
, vect_location
,
4826 "=== vectorizable_operation ===\n");
4827 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4833 if (dump_enabled_p ())
4834 dump_printf_loc (MSG_NOTE
, vect_location
,
4835 "transform binary/unary operation.\n");
4838 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4840 /* In case the vectorization factor (VF) is bigger than the number
4841 of elements that we can fit in a vectype (nunits), we have to generate
4842 more than one vector stmt - i.e - we need to "unroll" the
4843 vector stmt by a factor VF/nunits. In doing so, we record a pointer
4844 from one copy of the vector stmt to the next, in the field
4845 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
4846 stages to find the correct vector defs to be used when vectorizing
4847 stmts that use the defs of the current stmt. The example below
4848 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
4849 we need to create 4 vectorized stmts):
4851 before vectorization:
4852 RELATED_STMT VEC_STMT
4856 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
4858 RELATED_STMT VEC_STMT
4859 VS1_0: vx0 = memref0 VS1_1 -
4860 VS1_1: vx1 = memref1 VS1_2 -
4861 VS1_2: vx2 = memref2 VS1_3 -
4862 VS1_3: vx3 = memref3 - -
4863 S1: x = load - VS1_0
4866 step2: vectorize stmt S2 (done here):
4867 To vectorize stmt S2 we first need to find the relevant vector
4868 def for the first operand 'x'. This is, as usual, obtained from
4869 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
4870 that defines 'x' (S1). This way we find the stmt VS1_0, and the
4871 relevant vector def 'vx0'. Having found 'vx0' we can generate
4872 the vector stmt VS2_0, and as usual, record it in the
4873 STMT_VINFO_VEC_STMT of stmt S2.
4874 When creating the second copy (VS2_1), we obtain the relevant vector
4875 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
4876 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
4877 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
4878 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
4879 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
4880 chain of stmts and pointers:
4881 RELATED_STMT VEC_STMT
4882 VS1_0: vx0 = memref0 VS1_1 -
4883 VS1_1: vx1 = memref1 VS1_2 -
4884 VS1_2: vx2 = memref2 VS1_3 -
4885 VS1_3: vx3 = memref3 - -
4886 S1: x = load - VS1_0
4887 VS2_0: vz0 = vx0 + v1 VS2_1 -
4888 VS2_1: vz1 = vx1 + v1 VS2_2 -
4889 VS2_2: vz2 = vx2 + v1 VS2_3 -
4890 VS2_3: vz3 = vx3 + v1 - -
4891 S2: z = x + 1 - VS2_0 */
4893 prev_stmt_info
= NULL
;
4894 for (j
= 0; j
< ncopies
; j
++)
4899 if (op_type
== binary_op
|| op_type
== ternary_op
)
4900 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
, &vec_oprnds1
,
4903 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
4905 if (op_type
== ternary_op
)
4907 vec_oprnds2
.create (1);
4908 vec_oprnds2
.quick_push (vect_get_vec_def_for_operand (op2
,
4915 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, &vec_oprnds1
);
4916 if (op_type
== ternary_op
)
4918 tree vec_oprnd
= vec_oprnds2
.pop ();
4919 vec_oprnds2
.quick_push (vect_get_vec_def_for_stmt_copy (dt
[2],
4924 /* Arguments are ready. Create the new vector stmt. */
4925 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4927 vop1
= ((op_type
== binary_op
|| op_type
== ternary_op
)
4928 ? vec_oprnds1
[i
] : NULL_TREE
);
4929 vop2
= ((op_type
== ternary_op
)
4930 ? vec_oprnds2
[i
] : NULL_TREE
);
4931 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
, vop2
);
4932 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4933 gimple_assign_set_lhs (new_stmt
, new_temp
);
4934 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4936 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4943 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4945 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4946 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4949 vec_oprnds0
.release ();
4950 vec_oprnds1
.release ();
4951 vec_oprnds2
.release ();
4956 /* A helper function to ensure data reference DR's base alignment
4960 ensure_base_align (stmt_vec_info stmt_info
, struct data_reference
*dr
)
4965 if (((dataref_aux
*)dr
->aux
)->base_misaligned
)
4967 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
4968 tree base_decl
= ((dataref_aux
*)dr
->aux
)->base_decl
;
4970 if (decl_in_symtab_p (base_decl
))
4971 symtab_node::get (base_decl
)->increase_alignment (TYPE_ALIGN (vectype
));
4974 DECL_ALIGN (base_decl
) = TYPE_ALIGN (vectype
);
4975 DECL_USER_ALIGN (base_decl
) = 1;
4977 ((dataref_aux
*)dr
->aux
)->base_misaligned
= false;
4982 /* Given a vector type VECTYPE returns the VECTOR_CST mask that implements
4983 reversal of the vector elements. If that is impossible to do,
4987 perm_mask_for_reverse (tree vectype
)
4992 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
4993 sel
= XALLOCAVEC (unsigned char, nunits
);
4995 for (i
= 0; i
< nunits
; ++i
)
4996 sel
[i
] = nunits
- 1 - i
;
4998 if (!can_vec_perm_p (TYPE_MODE (vectype
), false, sel
))
5000 return vect_gen_perm_mask_checked (vectype
, sel
);
5003 /* Function vectorizable_store.
5005 Check if STMT defines a non scalar data-ref (array/pointer/structure) that
5007 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
5008 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
5009 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5012 vectorizable_store (gimple stmt
, gimple_stmt_iterator
*gsi
, gimple
*vec_stmt
,
5018 tree vec_oprnd
= NULL_TREE
;
5019 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
5020 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
= NULL
;
5022 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5023 struct loop
*loop
= NULL
;
5024 machine_mode vec_mode
;
5026 enum dr_alignment_support alignment_support_scheme
;
5029 enum vect_def_type dt
;
5030 stmt_vec_info prev_stmt_info
= NULL
;
5031 tree dataref_ptr
= NULL_TREE
;
5032 tree dataref_offset
= NULL_TREE
;
5033 gimple ptr_incr
= NULL
;
5036 gimple next_stmt
, first_stmt
= NULL
;
5037 bool grouped_store
= false;
5038 bool store_lanes_p
= false;
5039 unsigned int group_size
, i
;
5040 vec
<tree
> dr_chain
= vNULL
;
5041 vec
<tree
> oprnds
= vNULL
;
5042 vec
<tree
> result_chain
= vNULL
;
5044 bool negative
= false;
5045 tree offset
= NULL_TREE
;
5046 vec
<tree
> vec_oprnds
= vNULL
;
5047 bool slp
= (slp_node
!= NULL
);
5048 unsigned int vec_num
;
5049 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
5052 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
5055 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
5058 /* Is vectorizable store? */
5060 if (!is_gimple_assign (stmt
))
5063 scalar_dest
= gimple_assign_lhs (stmt
);
5064 if (TREE_CODE (scalar_dest
) == VIEW_CONVERT_EXPR
5065 && is_pattern_stmt_p (stmt_info
))
5066 scalar_dest
= TREE_OPERAND (scalar_dest
, 0);
5067 if (TREE_CODE (scalar_dest
) != ARRAY_REF
5068 && TREE_CODE (scalar_dest
) != BIT_FIELD_REF
5069 && TREE_CODE (scalar_dest
) != INDIRECT_REF
5070 && TREE_CODE (scalar_dest
) != COMPONENT_REF
5071 && TREE_CODE (scalar_dest
) != IMAGPART_EXPR
5072 && TREE_CODE (scalar_dest
) != REALPART_EXPR
5073 && TREE_CODE (scalar_dest
) != MEM_REF
)
5076 gcc_assert (gimple_assign_single_p (stmt
));
5078 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
5079 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5082 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5084 /* Multiple types in SLP are handled by creating the appropriate number of
5085 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5087 if (slp
|| PURE_SLP_STMT (stmt_info
))
5090 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
5092 gcc_assert (ncopies
>= 1);
5094 /* FORNOW. This restriction should be relaxed. */
5095 if (loop
&& nested_in_vect_loop_p (loop
, stmt
) && ncopies
> 1)
5097 if (dump_enabled_p ())
5098 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5099 "multiple types in nested loop.\n");
5103 op
= gimple_assign_rhs1 (stmt
);
5104 if (!vect_is_simple_use (op
, stmt
, loop_vinfo
, bb_vinfo
, &def_stmt
,
5107 if (dump_enabled_p ())
5108 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5109 "use not simple.\n");
5113 elem_type
= TREE_TYPE (vectype
);
5114 vec_mode
= TYPE_MODE (vectype
);
5116 /* FORNOW. In some cases can vectorize even if data-type not supported
5117 (e.g. - array initialization with 0). */
5118 if (optab_handler (mov_optab
, vec_mode
) == CODE_FOR_nothing
)
5121 if (!STMT_VINFO_DATA_REF (stmt_info
))
5124 if (!STMT_VINFO_STRIDED_P (stmt_info
))
5127 tree_int_cst_compare (loop
&& nested_in_vect_loop_p (loop
, stmt
)
5128 ? STMT_VINFO_DR_STEP (stmt_info
) : DR_STEP (dr
),
5129 size_zero_node
) < 0;
5130 if (negative
&& ncopies
> 1)
5132 if (dump_enabled_p ())
5133 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5134 "multiple types with negative step.\n");
5139 gcc_assert (!grouped_store
);
5140 alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
5141 if (alignment_support_scheme
!= dr_aligned
5142 && alignment_support_scheme
!= dr_unaligned_supported
)
5144 if (dump_enabled_p ())
5145 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5146 "negative step but alignment required.\n");
5149 if (dt
!= vect_constant_def
5150 && dt
!= vect_external_def
5151 && !perm_mask_for_reverse (vectype
))
5153 if (dump_enabled_p ())
5154 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5155 "negative step and reversing not supported.\n");
5161 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
5163 grouped_store
= true;
5164 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
5165 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
5167 && !PURE_SLP_STMT (stmt_info
)
5168 && !STMT_VINFO_STRIDED_P (stmt_info
))
5170 if (vect_store_lanes_supported (vectype
, group_size
))
5171 store_lanes_p
= true;
5172 else if (!vect_grouped_store_supported (vectype
, group_size
))
5176 if (STMT_VINFO_STRIDED_P (stmt_info
)
5177 && (slp
|| PURE_SLP_STMT (stmt_info
))
5178 && (group_size
> nunits
5179 || nunits
% group_size
!= 0))
5181 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5182 "unhandled strided group store\n");
5186 if (first_stmt
== stmt
)
5188 /* STMT is the leader of the group. Check the operands of all the
5189 stmts of the group. */
5190 next_stmt
= GROUP_NEXT_ELEMENT (stmt_info
);
5193 gcc_assert (gimple_assign_single_p (next_stmt
));
5194 op
= gimple_assign_rhs1 (next_stmt
);
5195 if (!vect_is_simple_use (op
, next_stmt
, loop_vinfo
, bb_vinfo
,
5196 &def_stmt
, &def
, &dt
))
5198 if (dump_enabled_p ())
5199 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5200 "use not simple.\n");
5203 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5208 if (!vec_stmt
) /* transformation not required. */
5210 STMT_VINFO_TYPE (stmt_info
) = store_vec_info_type
;
5211 /* The SLP costs are calculated during SLP analysis. */
5212 if (!PURE_SLP_STMT (stmt_info
))
5213 vect_model_store_cost (stmt_info
, ncopies
, store_lanes_p
, dt
,
5220 ensure_base_align (stmt_info
, dr
);
5224 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
5225 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
5227 GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt
))++;
5230 gcc_assert (!loop
|| !nested_in_vect_loop_p (loop
, stmt
));
5232 /* We vectorize all the stmts of the interleaving group when we
5233 reach the last stmt in the group. */
5234 if (GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt
))
5235 < GROUP_SIZE (vinfo_for_stmt (first_stmt
))
5244 grouped_store
= false;
5245 /* VEC_NUM is the number of vect stmts to be created for this
5247 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
5248 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
5249 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
5250 op
= gimple_assign_rhs1 (first_stmt
);
5253 /* VEC_NUM is the number of vect stmts to be created for this
5255 vec_num
= group_size
;
5261 group_size
= vec_num
= 1;
5264 if (dump_enabled_p ())
5265 dump_printf_loc (MSG_NOTE
, vect_location
,
5266 "transform store. ncopies = %d\n", ncopies
);
5268 if (STMT_VINFO_STRIDED_P (stmt_info
))
5270 gimple_stmt_iterator incr_gsi
;
5276 gimple_seq stmts
= NULL
;
5277 tree stride_base
, stride_step
, alias_off
;
5281 gcc_assert (!nested_in_vect_loop_p (loop
, stmt
));
5284 = fold_build_pointer_plus
5285 (unshare_expr (DR_BASE_ADDRESS (first_dr
)),
5286 size_binop (PLUS_EXPR
,
5287 convert_to_ptrofftype (unshare_expr (DR_OFFSET (first_dr
))),
5288 convert_to_ptrofftype (DR_INIT(first_dr
))));
5289 stride_step
= fold_convert (sizetype
, unshare_expr (DR_STEP (first_dr
)));
5291 /* For a store with loop-invariant (but other than power-of-2)
5292 stride (i.e. not a grouped access) like so:
5294 for (i = 0; i < n; i += stride)
5297 we generate a new induction variable and new stores from
5298 the components of the (vectorized) rhs:
5300 for (j = 0; ; j += VF*stride)
5305 array[j + stride] = tmp2;
5309 unsigned nstores
= nunits
;
5310 tree ltype
= elem_type
;
5313 nstores
= nunits
/ group_size
;
5314 if (group_size
< nunits
)
5315 ltype
= build_vector_type (elem_type
, group_size
);
5318 ltype
= build_aligned_type (ltype
, TYPE_ALIGN (elem_type
));
5319 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
5323 ivstep
= stride_step
;
5324 ivstep
= fold_build2 (MULT_EXPR
, TREE_TYPE (ivstep
), ivstep
,
5325 build_int_cst (TREE_TYPE (ivstep
),
5326 ncopies
* nstores
));
5328 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
5330 create_iv (stride_base
, ivstep
, NULL
,
5331 loop
, &incr_gsi
, insert_after
,
5333 incr
= gsi_stmt (incr_gsi
);
5334 set_vinfo_for_stmt (incr
, new_stmt_vec_info (incr
, loop_vinfo
, NULL
));
5336 stride_step
= force_gimple_operand (stride_step
, &stmts
, true, NULL_TREE
);
5338 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
5340 prev_stmt_info
= NULL
;
5341 alias_off
= build_int_cst (reference_alias_ptr_type (DR_REF (first_dr
)), 0);
5342 next_stmt
= first_stmt
;
5343 for (g
= 0; g
< group_size
; g
++)
5345 running_off
= offvar
;
5348 tree size
= TYPE_SIZE_UNIT (ltype
);
5349 tree pos
= fold_build2 (MULT_EXPR
, sizetype
, size_int (g
),
5351 tree newoff
= copy_ssa_name (running_off
, NULL
);
5352 incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
5354 vect_finish_stmt_generation (stmt
, incr
, gsi
);
5355 running_off
= newoff
;
5357 for (j
= 0; j
< ncopies
; j
++)
5359 /* We've set op and dt above, from gimple_assign_rhs1(stmt),
5360 and first_stmt == stmt. */
5365 vect_get_vec_defs (op
, NULL_TREE
, stmt
, &vec_oprnds
, NULL
,
5367 vec_oprnd
= vec_oprnds
[0];
5371 gcc_assert (gimple_assign_single_p (next_stmt
));
5372 op
= gimple_assign_rhs1 (next_stmt
);
5373 vec_oprnd
= vect_get_vec_def_for_operand (op
, next_stmt
,
5380 vec_oprnd
= vec_oprnds
[j
];
5383 vect_is_simple_use (vec_oprnd
, NULL
, loop_vinfo
,
5384 bb_vinfo
, &def_stmt
, &def
, &dt
);
5385 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, vec_oprnd
);
5389 for (i
= 0; i
< nstores
; i
++)
5391 tree newref
, newoff
;
5392 gimple incr
, assign
;
5393 tree size
= TYPE_SIZE (ltype
);
5394 /* Extract the i'th component. */
5395 tree pos
= fold_build2 (MULT_EXPR
, bitsizetype
,
5396 bitsize_int (i
), size
);
5397 tree elem
= fold_build3 (BIT_FIELD_REF
, ltype
, vec_oprnd
,
5400 elem
= force_gimple_operand_gsi (gsi
, elem
, true,
5404 newref
= build2 (MEM_REF
, ltype
,
5405 running_off
, alias_off
);
5407 /* And store it to *running_off. */
5408 assign
= gimple_build_assign (newref
, elem
);
5409 vect_finish_stmt_generation (stmt
, assign
, gsi
);
5411 newoff
= copy_ssa_name (running_off
, NULL
);
5412 incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
5413 running_off
, stride_step
);
5414 vect_finish_stmt_generation (stmt
, incr
, gsi
);
5416 running_off
= newoff
;
5417 if (g
== group_size
- 1
5420 if (j
== 0 && i
== 0)
5421 STMT_VINFO_VEC_STMT (stmt_info
)
5422 = *vec_stmt
= assign
;
5424 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = assign
;
5425 prev_stmt_info
= vinfo_for_stmt (assign
);
5429 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5434 dr_chain
.create (group_size
);
5435 oprnds
.create (group_size
);
5437 alignment_support_scheme
= vect_supportable_dr_alignment (first_dr
, false);
5438 gcc_assert (alignment_support_scheme
);
5439 /* Targets with store-lane instructions must not require explicit
5441 gcc_assert (!store_lanes_p
5442 || alignment_support_scheme
== dr_aligned
5443 || alignment_support_scheme
== dr_unaligned_supported
);
5446 offset
= size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1);
5449 aggr_type
= build_array_type_nelts (elem_type
, vec_num
* nunits
);
5451 aggr_type
= vectype
;
5453 /* In case the vectorization factor (VF) is bigger than the number
5454 of elements that we can fit in a vectype (nunits), we have to generate
5455 more than one vector stmt - i.e - we need to "unroll" the
5456 vector stmt by a factor VF/nunits. For more details see documentation in
5457 vect_get_vec_def_for_copy_stmt. */
5459 /* In case of interleaving (non-unit grouped access):
5466 We create vectorized stores starting from base address (the access of the
5467 first stmt in the chain (S2 in the above example), when the last store stmt
5468 of the chain (S4) is reached:
5471 VS2: &base + vec_size*1 = vx0
5472 VS3: &base + vec_size*2 = vx1
5473 VS4: &base + vec_size*3 = vx3
5475 Then permutation statements are generated:
5477 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
5478 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
5481 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
5482 (the order of the data-refs in the output of vect_permute_store_chain
5483 corresponds to the order of scalar stmts in the interleaving chain - see
5484 the documentation of vect_permute_store_chain()).
5486 In case of both multiple types and interleaving, above vector stores and
5487 permutation stmts are created for every copy. The result vector stmts are
5488 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
5489 STMT_VINFO_RELATED_STMT for the next copies.
5492 prev_stmt_info
= NULL
;
5493 for (j
= 0; j
< ncopies
; j
++)
5501 /* Get vectorized arguments for SLP_NODE. */
5502 vect_get_vec_defs (op
, NULL_TREE
, stmt
, &vec_oprnds
,
5503 NULL
, slp_node
, -1);
5505 vec_oprnd
= vec_oprnds
[0];
5509 /* For interleaved stores we collect vectorized defs for all the
5510 stores in the group in DR_CHAIN and OPRNDS. DR_CHAIN is then
5511 used as an input to vect_permute_store_chain(), and OPRNDS as
5512 an input to vect_get_vec_def_for_stmt_copy() for the next copy.
5514 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
5515 OPRNDS are of size 1. */
5516 next_stmt
= first_stmt
;
5517 for (i
= 0; i
< group_size
; i
++)
5519 /* Since gaps are not supported for interleaved stores,
5520 GROUP_SIZE is the exact number of stmts in the chain.
5521 Therefore, NEXT_STMT can't be NULL_TREE. In case that
5522 there is no interleaving, GROUP_SIZE is 1, and only one
5523 iteration of the loop will be executed. */
5524 gcc_assert (next_stmt
5525 && gimple_assign_single_p (next_stmt
));
5526 op
= gimple_assign_rhs1 (next_stmt
);
5528 vec_oprnd
= vect_get_vec_def_for_operand (op
, next_stmt
,
5530 dr_chain
.quick_push (vec_oprnd
);
5531 oprnds
.quick_push (vec_oprnd
);
5532 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5536 /* We should have catched mismatched types earlier. */
5537 gcc_assert (useless_type_conversion_p (vectype
,
5538 TREE_TYPE (vec_oprnd
)));
5539 bool simd_lane_access_p
5540 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info
);
5541 if (simd_lane_access_p
5542 && TREE_CODE (DR_BASE_ADDRESS (first_dr
)) == ADDR_EXPR
5543 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr
), 0))
5544 && integer_zerop (DR_OFFSET (first_dr
))
5545 && integer_zerop (DR_INIT (first_dr
))
5546 && alias_sets_conflict_p (get_alias_set (aggr_type
),
5547 get_alias_set (DR_REF (first_dr
))))
5549 dataref_ptr
= unshare_expr (DR_BASE_ADDRESS (first_dr
));
5550 dataref_offset
= build_int_cst (reference_alias_ptr_type
5551 (DR_REF (first_dr
)), 0);
5556 = vect_create_data_ref_ptr (first_stmt
, aggr_type
,
5557 simd_lane_access_p
? loop
: NULL
,
5558 offset
, &dummy
, gsi
, &ptr_incr
,
5559 simd_lane_access_p
, &inv_p
);
5560 gcc_assert (bb_vinfo
|| !inv_p
);
5564 /* For interleaved stores we created vectorized defs for all the
5565 defs stored in OPRNDS in the previous iteration (previous copy).
5566 DR_CHAIN is then used as an input to vect_permute_store_chain(),
5567 and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
5569 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
5570 OPRNDS are of size 1. */
5571 for (i
= 0; i
< group_size
; i
++)
5574 vect_is_simple_use (op
, NULL
, loop_vinfo
, bb_vinfo
, &def_stmt
,
5576 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, op
);
5577 dr_chain
[i
] = vec_oprnd
;
5578 oprnds
[i
] = vec_oprnd
;
5582 = int_const_binop (PLUS_EXPR
, dataref_offset
,
5583 TYPE_SIZE_UNIT (aggr_type
));
5585 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
5586 TYPE_SIZE_UNIT (aggr_type
));
5593 /* Combine all the vectors into an array. */
5594 vec_array
= create_vector_array (vectype
, vec_num
);
5595 for (i
= 0; i
< vec_num
; i
++)
5597 vec_oprnd
= dr_chain
[i
];
5598 write_vector_array (stmt
, gsi
, vec_oprnd
, vec_array
, i
);
5602 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
5603 data_ref
= create_array_ref (aggr_type
, dataref_ptr
, first_dr
);
5604 new_stmt
= gimple_build_call_internal (IFN_STORE_LANES
, 1, vec_array
);
5605 gimple_call_set_lhs (new_stmt
, data_ref
);
5606 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5614 result_chain
.create (group_size
);
5616 vect_permute_store_chain (dr_chain
, group_size
, stmt
, gsi
,
5620 next_stmt
= first_stmt
;
5621 for (i
= 0; i
< vec_num
; i
++)
5623 unsigned align
, misalign
;
5626 /* Bump the vector pointer. */
5627 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
5631 vec_oprnd
= vec_oprnds
[i
];
5632 else if (grouped_store
)
5633 /* For grouped stores vectorized defs are interleaved in
5634 vect_permute_store_chain(). */
5635 vec_oprnd
= result_chain
[i
];
5637 data_ref
= fold_build2 (MEM_REF
, TREE_TYPE (vec_oprnd
),
5641 : build_int_cst (reference_alias_ptr_type
5642 (DR_REF (first_dr
)), 0));
5643 align
= TYPE_ALIGN_UNIT (vectype
);
5644 if (aligned_access_p (first_dr
))
5646 else if (DR_MISALIGNMENT (first_dr
) == -1)
5648 TREE_TYPE (data_ref
)
5649 = build_aligned_type (TREE_TYPE (data_ref
),
5650 TYPE_ALIGN (elem_type
));
5651 align
= TYPE_ALIGN_UNIT (elem_type
);
5656 TREE_TYPE (data_ref
)
5657 = build_aligned_type (TREE_TYPE (data_ref
),
5658 TYPE_ALIGN (elem_type
));
5659 misalign
= DR_MISALIGNMENT (first_dr
);
5661 if (dataref_offset
== NULL_TREE
5662 && TREE_CODE (dataref_ptr
) == SSA_NAME
)
5663 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
5667 && dt
!= vect_constant_def
5668 && dt
!= vect_external_def
)
5670 tree perm_mask
= perm_mask_for_reverse (vectype
);
5672 = vect_create_destination_var (gimple_assign_rhs1 (stmt
),
5674 tree new_temp
= make_ssa_name (perm_dest
);
5676 /* Generate the permute statement. */
5678 = gimple_build_assign (new_temp
, VEC_PERM_EXPR
, vec_oprnd
,
5679 vec_oprnd
, perm_mask
);
5680 vect_finish_stmt_generation (stmt
, perm_stmt
, gsi
);
5682 perm_stmt
= SSA_NAME_DEF_STMT (new_temp
);
5683 vec_oprnd
= new_temp
;
5686 /* Arguments are ready. Create the new vector stmt. */
5687 new_stmt
= gimple_build_assign (data_ref
, vec_oprnd
);
5688 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5693 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5701 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
5703 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
5704 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
5708 dr_chain
.release ();
5710 result_chain
.release ();
5711 vec_oprnds
.release ();
5716 /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
5717 VECTOR_CST mask. No checks are made that the target platform supports the
5718 mask, so callers may wish to test can_vec_perm_p separately, or use
5719 vect_gen_perm_mask_checked. */
5722 vect_gen_perm_mask_any (tree vectype
, const unsigned char *sel
)
5724 tree mask_elt_type
, mask_type
, mask_vec
, *mask_elts
;
5727 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5729 mask_elt_type
= lang_hooks
.types
.type_for_mode
5730 (int_mode_for_mode (TYPE_MODE (TREE_TYPE (vectype
))), 1);
5731 mask_type
= get_vectype_for_scalar_type (mask_elt_type
);
5733 mask_elts
= XALLOCAVEC (tree
, nunits
);
5734 for (i
= nunits
- 1; i
>= 0; i
--)
5735 mask_elts
[i
] = build_int_cst (mask_elt_type
, sel
[i
]);
5736 mask_vec
= build_vector (mask_type
, mask_elts
);
5741 /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_p,
5742 i.e. that the target supports the pattern _for arbitrary input vectors_. */
5745 vect_gen_perm_mask_checked (tree vectype
, const unsigned char *sel
)
5747 gcc_assert (can_vec_perm_p (TYPE_MODE (vectype
), false, sel
));
5748 return vect_gen_perm_mask_any (vectype
, sel
);
5751 /* Given a vector variable X and Y, that was generated for the scalar
5752 STMT, generate instructions to permute the vector elements of X and Y
5753 using permutation mask MASK_VEC, insert them at *GSI and return the
5754 permuted vector variable. */
5757 permute_vec_elements (tree x
, tree y
, tree mask_vec
, gimple stmt
,
5758 gimple_stmt_iterator
*gsi
)
5760 tree vectype
= TREE_TYPE (x
);
5761 tree perm_dest
, data_ref
;
5764 perm_dest
= vect_create_destination_var (gimple_get_lhs (stmt
), vectype
);
5765 data_ref
= make_ssa_name (perm_dest
);
5767 /* Generate the permute statement. */
5768 perm_stmt
= gimple_build_assign (data_ref
, VEC_PERM_EXPR
, x
, y
, mask_vec
);
5769 vect_finish_stmt_generation (stmt
, perm_stmt
, gsi
);
5774 /* Hoist the definitions of all SSA uses on STMT out of the loop LOOP,
5775 inserting them on the loops preheader edge. Returns true if we
5776 were successful in doing so (and thus STMT can be moved then),
5777 otherwise returns false. */
5780 hoist_defs_of_uses (gimple stmt
, struct loop
*loop
)
5786 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_USE
)
5788 gimple def_stmt
= SSA_NAME_DEF_STMT (op
);
5789 if (!gimple_nop_p (def_stmt
)
5790 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
)))
5792 /* Make sure we don't need to recurse. While we could do
5793 so in simple cases when there are more complex use webs
5794 we don't have an easy way to preserve stmt order to fulfil
5795 dependencies within them. */
5798 if (gimple_code (def_stmt
) == GIMPLE_PHI
)
5800 FOR_EACH_SSA_TREE_OPERAND (op2
, def_stmt
, i2
, SSA_OP_USE
)
5802 gimple def_stmt2
= SSA_NAME_DEF_STMT (op2
);
5803 if (!gimple_nop_p (def_stmt2
)
5804 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt2
)))
5814 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_USE
)
5816 gimple def_stmt
= SSA_NAME_DEF_STMT (op
);
5817 if (!gimple_nop_p (def_stmt
)
5818 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
)))
5820 gimple_stmt_iterator gsi
= gsi_for_stmt (def_stmt
);
5821 gsi_remove (&gsi
, false);
5822 gsi_insert_on_edge_immediate (loop_preheader_edge (loop
), def_stmt
);
5829 /* vectorizable_load.
5831 Check if STMT reads a non scalar data-ref (array/pointer/structure) that
5833 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
5834 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
5835 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5838 vectorizable_load (gimple stmt
, gimple_stmt_iterator
*gsi
, gimple
*vec_stmt
,
5839 slp_tree slp_node
, slp_instance slp_node_instance
)
5842 tree vec_dest
= NULL
;
5843 tree data_ref
= NULL
;
5844 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
5845 stmt_vec_info prev_stmt_info
;
5846 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5847 struct loop
*loop
= NULL
;
5848 struct loop
*containing_loop
= (gimple_bb (stmt
))->loop_father
;
5849 bool nested_in_vect_loop
= false;
5850 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
= NULL
;
5854 gimple new_stmt
= NULL
;
5856 enum dr_alignment_support alignment_support_scheme
;
5857 tree dataref_ptr
= NULL_TREE
;
5858 tree dataref_offset
= NULL_TREE
;
5859 gimple ptr_incr
= NULL
;
5861 int i
, j
, group_size
= -1, group_gap_adj
;
5862 tree msq
= NULL_TREE
, lsq
;
5863 tree offset
= NULL_TREE
;
5864 tree byte_offset
= NULL_TREE
;
5865 tree realignment_token
= NULL_TREE
;
5867 vec
<tree
> dr_chain
= vNULL
;
5868 bool grouped_load
= false;
5869 bool load_lanes_p
= false;
5872 bool negative
= false;
5873 bool compute_in_loop
= false;
5874 struct loop
*at_loop
;
5876 bool slp
= (slp_node
!= NULL
);
5877 bool slp_perm
= false;
5878 enum tree_code code
;
5879 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
5882 tree gather_base
= NULL_TREE
, gather_off
= NULL_TREE
;
5883 tree gather_off_vectype
= NULL_TREE
, gather_decl
= NULL_TREE
;
5884 int gather_scale
= 1;
5885 enum vect_def_type gather_dt
= vect_unknown_def_type
;
5887 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
5890 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
5893 /* Is vectorizable load? */
5894 if (!is_gimple_assign (stmt
))
5897 scalar_dest
= gimple_assign_lhs (stmt
);
5898 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
5901 code
= gimple_assign_rhs_code (stmt
);
5902 if (code
!= ARRAY_REF
5903 && code
!= BIT_FIELD_REF
5904 && code
!= INDIRECT_REF
5905 && code
!= COMPONENT_REF
5906 && code
!= IMAGPART_EXPR
5907 && code
!= REALPART_EXPR
5909 && TREE_CODE_CLASS (code
) != tcc_declaration
)
5912 if (!STMT_VINFO_DATA_REF (stmt_info
))
5915 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
5916 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5920 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5921 nested_in_vect_loop
= nested_in_vect_loop_p (loop
, stmt
);
5922 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
5927 /* Multiple types in SLP are handled by creating the appropriate number of
5928 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5930 if (slp
|| PURE_SLP_STMT (stmt_info
))
5933 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
5935 gcc_assert (ncopies
>= 1);
5937 /* FORNOW. This restriction should be relaxed. */
5938 if (nested_in_vect_loop
&& ncopies
> 1)
5940 if (dump_enabled_p ())
5941 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5942 "multiple types in nested loop.\n");
5946 /* Invalidate assumptions made by dependence analysis when vectorization
5947 on the unrolled body effectively re-orders stmts. */
5949 && STMT_VINFO_MIN_NEG_DIST (stmt_info
) != 0
5950 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
5951 > STMT_VINFO_MIN_NEG_DIST (stmt_info
)))
5953 if (dump_enabled_p ())
5954 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5955 "cannot perform implicit CSE when unrolling "
5956 "with negative dependence distance\n");
5960 elem_type
= TREE_TYPE (vectype
);
5961 mode
= TYPE_MODE (vectype
);
5963 /* FORNOW. In some cases can vectorize even if data-type not supported
5964 (e.g. - data copies). */
5965 if (optab_handler (mov_optab
, mode
) == CODE_FOR_nothing
)
5967 if (dump_enabled_p ())
5968 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5969 "Aligned load, but unsupported type.\n");
5973 /* Check if the load is a part of an interleaving chain. */
5974 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
5976 grouped_load
= true;
5978 gcc_assert (! nested_in_vect_loop
&& !STMT_VINFO_GATHER_P (stmt_info
));
5980 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
5982 /* If this is single-element interleaving with an element distance
5983 that leaves unused vector loads around punt - we at least create
5984 very sub-optimal code in that case (and blow up memory,
5986 if (first_stmt
== stmt
5987 && !GROUP_NEXT_ELEMENT (stmt_info
)
5988 && GROUP_SIZE (stmt_info
) > TYPE_VECTOR_SUBPARTS (vectype
))
5990 if (dump_enabled_p ())
5991 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5992 "single-element interleaving not supported "
5993 "for not adjacent vector loads\n");
5997 if (slp
&& SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ())
6000 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
6002 && !PURE_SLP_STMT (stmt_info
)
6003 && !STMT_VINFO_STRIDED_P (stmt_info
))
6005 if (vect_load_lanes_supported (vectype
, group_size
))
6006 load_lanes_p
= true;
6007 else if (!vect_grouped_load_supported (vectype
, group_size
))
6011 /* Invalidate assumptions made by dependence analysis when vectorization
6012 on the unrolled body effectively re-orders stmts. */
6013 if (!PURE_SLP_STMT (stmt_info
)
6014 && STMT_VINFO_MIN_NEG_DIST (stmt_info
) != 0
6015 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
6016 > STMT_VINFO_MIN_NEG_DIST (stmt_info
)))
6018 if (dump_enabled_p ())
6019 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6020 "cannot perform implicit CSE when performing "
6021 "group loads with negative dependence distance\n");
6025 /* Similarly when the stmt is a load that is both part of a SLP
6026 instance and a loop vectorized stmt via the same-dr mechanism
6027 we have to give up. */
6028 if (STMT_VINFO_GROUP_SAME_DR_STMT (stmt_info
)
6029 && (STMT_SLP_TYPE (stmt_info
)
6030 != STMT_SLP_TYPE (vinfo_for_stmt
6031 (STMT_VINFO_GROUP_SAME_DR_STMT (stmt_info
)))))
6033 if (dump_enabled_p ())
6034 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6035 "conflicting SLP types for CSEd load\n");
6041 if (STMT_VINFO_GATHER_P (stmt_info
))
6045 gather_decl
= vect_check_gather (stmt
, loop_vinfo
, &gather_base
,
6046 &gather_off
, &gather_scale
);
6047 gcc_assert (gather_decl
);
6048 if (!vect_is_simple_use_1 (gather_off
, NULL
, loop_vinfo
, bb_vinfo
,
6049 &def_stmt
, &def
, &gather_dt
,
6050 &gather_off_vectype
))
6052 if (dump_enabled_p ())
6053 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6054 "gather index use not simple.\n");
6058 else if (STMT_VINFO_STRIDED_P (stmt_info
))
6061 && (slp
|| PURE_SLP_STMT (stmt_info
)))
6062 && (group_size
> nunits
6063 || nunits
% group_size
!= 0))
6065 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6066 "unhandled strided group load\n");
6072 negative
= tree_int_cst_compare (nested_in_vect_loop
6073 ? STMT_VINFO_DR_STEP (stmt_info
)
6075 size_zero_node
) < 0;
6076 if (negative
&& ncopies
> 1)
6078 if (dump_enabled_p ())
6079 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6080 "multiple types with negative step.\n");
6088 if (dump_enabled_p ())
6089 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6090 "negative step for group load not supported"
6094 alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
6095 if (alignment_support_scheme
!= dr_aligned
6096 && alignment_support_scheme
!= dr_unaligned_supported
)
6098 if (dump_enabled_p ())
6099 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6100 "negative step but alignment required.\n");
6103 if (!perm_mask_for_reverse (vectype
))
6105 if (dump_enabled_p ())
6106 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6107 "negative step and reversing not supported."
6114 if (!vec_stmt
) /* transformation not required. */
6116 STMT_VINFO_TYPE (stmt_info
) = load_vec_info_type
;
6117 /* The SLP costs are calculated during SLP analysis. */
6118 if (!PURE_SLP_STMT (stmt_info
))
6119 vect_model_load_cost (stmt_info
, ncopies
, load_lanes_p
,
6124 if (dump_enabled_p ())
6125 dump_printf_loc (MSG_NOTE
, vect_location
,
6126 "transform load. ncopies = %d\n", ncopies
);
6130 ensure_base_align (stmt_info
, dr
);
6132 if (STMT_VINFO_GATHER_P (stmt_info
))
6134 tree vec_oprnd0
= NULL_TREE
, op
;
6135 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gather_decl
));
6136 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
6137 tree ptr
, mask
, var
, scale
, merge
, perm_mask
= NULL_TREE
, prev_res
= NULL_TREE
;
6138 edge pe
= loop_preheader_edge (loop
);
6141 enum { NARROW
, NONE
, WIDEN
} modifier
;
6142 int gather_off_nunits
= TYPE_VECTOR_SUBPARTS (gather_off_vectype
);
6144 if (nunits
== gather_off_nunits
)
6146 else if (nunits
== gather_off_nunits
/ 2)
6148 unsigned char *sel
= XALLOCAVEC (unsigned char, gather_off_nunits
);
6151 for (i
= 0; i
< gather_off_nunits
; ++i
)
6152 sel
[i
] = i
| nunits
;
6154 perm_mask
= vect_gen_perm_mask_checked (gather_off_vectype
, sel
);
6156 else if (nunits
== gather_off_nunits
* 2)
6158 unsigned char *sel
= XALLOCAVEC (unsigned char, nunits
);
6161 for (i
= 0; i
< nunits
; ++i
)
6162 sel
[i
] = i
< gather_off_nunits
6163 ? i
: i
+ nunits
- gather_off_nunits
;
6165 perm_mask
= vect_gen_perm_mask_checked (vectype
, sel
);
6171 rettype
= TREE_TYPE (TREE_TYPE (gather_decl
));
6172 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6173 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6174 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6175 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6176 scaletype
= TREE_VALUE (arglist
);
6177 gcc_checking_assert (types_compatible_p (srctype
, rettype
));
6179 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6181 ptr
= fold_convert (ptrtype
, gather_base
);
6182 if (!is_gimple_min_invariant (ptr
))
6184 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
6185 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
6186 gcc_assert (!new_bb
);
6189 /* Currently we support only unconditional gather loads,
6190 so mask should be all ones. */
6191 if (TREE_CODE (masktype
) == INTEGER_TYPE
)
6192 mask
= build_int_cst (masktype
, -1);
6193 else if (TREE_CODE (TREE_TYPE (masktype
)) == INTEGER_TYPE
)
6195 mask
= build_int_cst (TREE_TYPE (masktype
), -1);
6196 mask
= build_vector_from_val (masktype
, mask
);
6197 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
6199 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype
)))
6203 for (j
= 0; j
< 6; ++j
)
6205 real_from_target (&r
, tmp
, TYPE_MODE (TREE_TYPE (masktype
)));
6206 mask
= build_real (TREE_TYPE (masktype
), r
);
6207 mask
= build_vector_from_val (masktype
, mask
);
6208 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
6213 scale
= build_int_cst (scaletype
, gather_scale
);
6215 if (TREE_CODE (TREE_TYPE (rettype
)) == INTEGER_TYPE
)
6216 merge
= build_int_cst (TREE_TYPE (rettype
), 0);
6217 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (rettype
)))
6221 for (j
= 0; j
< 6; ++j
)
6223 real_from_target (&r
, tmp
, TYPE_MODE (TREE_TYPE (rettype
)));
6224 merge
= build_real (TREE_TYPE (rettype
), r
);
6228 merge
= build_vector_from_val (rettype
, merge
);
6229 merge
= vect_init_vector (stmt
, merge
, rettype
, NULL
);
6231 prev_stmt_info
= NULL
;
6232 for (j
= 0; j
< ncopies
; ++j
)
6234 if (modifier
== WIDEN
&& (j
& 1))
6235 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
,
6236 perm_mask
, stmt
, gsi
);
6239 = vect_get_vec_def_for_operand (gather_off
, stmt
, NULL
);
6242 = vect_get_vec_def_for_stmt_copy (gather_dt
, vec_oprnd0
);
6244 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
6246 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
6247 == TYPE_VECTOR_SUBPARTS (idxtype
));
6248 var
= vect_get_new_vect_var (idxtype
, vect_simple_var
, NULL
);
6249 var
= make_ssa_name (var
);
6250 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
6252 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
6253 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6258 = gimple_build_call (gather_decl
, 5, merge
, ptr
, op
, mask
, scale
);
6260 if (!useless_type_conversion_p (vectype
, rettype
))
6262 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype
)
6263 == TYPE_VECTOR_SUBPARTS (rettype
));
6264 var
= vect_get_new_vect_var (rettype
, vect_simple_var
, NULL
);
6265 op
= make_ssa_name (var
, new_stmt
);
6266 gimple_call_set_lhs (new_stmt
, op
);
6267 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6268 var
= make_ssa_name (vec_dest
);
6269 op
= build1 (VIEW_CONVERT_EXPR
, vectype
, op
);
6271 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
6275 var
= make_ssa_name (vec_dest
, new_stmt
);
6276 gimple_call_set_lhs (new_stmt
, var
);
6279 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6281 if (modifier
== NARROW
)
6288 var
= permute_vec_elements (prev_res
, var
,
6289 perm_mask
, stmt
, gsi
);
6290 new_stmt
= SSA_NAME_DEF_STMT (var
);
6293 if (prev_stmt_info
== NULL
)
6294 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6296 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6297 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6301 else if (STMT_VINFO_STRIDED_P (stmt_info
))
6303 gimple_stmt_iterator incr_gsi
;
6309 vec
<constructor_elt
, va_gc
> *v
= NULL
;
6310 gimple_seq stmts
= NULL
;
6311 tree stride_base
, stride_step
, alias_off
;
6313 gcc_assert (!nested_in_vect_loop
);
6315 if (slp
&& grouped_load
)
6316 first_dr
= STMT_VINFO_DATA_REF
6317 (vinfo_for_stmt (GROUP_FIRST_ELEMENT (stmt_info
)));
6322 = fold_build_pointer_plus
6323 (DR_BASE_ADDRESS (first_dr
),
6324 size_binop (PLUS_EXPR
,
6325 convert_to_ptrofftype (DR_OFFSET (first_dr
)),
6326 convert_to_ptrofftype (DR_INIT (first_dr
))));
6327 stride_step
= fold_convert (sizetype
, DR_STEP (first_dr
));
6329 /* For a load with loop-invariant (but other than power-of-2)
6330 stride (i.e. not a grouped access) like so:
6332 for (i = 0; i < n; i += stride)
6335 we generate a new induction variable and new accesses to
6336 form a new vector (or vectors, depending on ncopies):
6338 for (j = 0; ; j += VF*stride)
6340 tmp2 = array[j + stride];
6342 vectemp = {tmp1, tmp2, ...}
6345 ivstep
= fold_build2 (MULT_EXPR
, TREE_TYPE (stride_step
), stride_step
,
6346 build_int_cst (TREE_TYPE (stride_step
), vf
));
6348 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
6350 create_iv (unshare_expr (stride_base
), unshare_expr (ivstep
), NULL
,
6351 loop
, &incr_gsi
, insert_after
,
6353 incr
= gsi_stmt (incr_gsi
);
6354 set_vinfo_for_stmt (incr
, new_stmt_vec_info (incr
, loop_vinfo
, NULL
));
6356 stride_step
= force_gimple_operand (unshare_expr (stride_step
),
6357 &stmts
, true, NULL_TREE
);
6359 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
6361 prev_stmt_info
= NULL
;
6362 running_off
= offvar
;
6363 alias_off
= build_int_cst (reference_alias_ptr_type (DR_REF (first_dr
)), 0);
6364 int nloads
= nunits
;
6365 tree ltype
= TREE_TYPE (vectype
);
6366 auto_vec
<tree
> dr_chain
;
6369 nloads
= nunits
/ group_size
;
6370 if (group_size
< nunits
)
6371 ltype
= build_vector_type (TREE_TYPE (vectype
), group_size
);
6374 ltype
= build_aligned_type (ltype
, TYPE_ALIGN (TREE_TYPE (vectype
)));
6375 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6377 dr_chain
.create (ncopies
);
6379 for (j
= 0; j
< ncopies
; j
++)
6385 vec_alloc (v
, nloads
);
6386 for (i
= 0; i
< nloads
; i
++)
6388 tree newref
, newoff
;
6390 newref
= build2 (MEM_REF
, ltype
, running_off
, alias_off
);
6392 newref
= force_gimple_operand_gsi (gsi
, newref
, true,
6395 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
, newref
);
6396 newoff
= copy_ssa_name (running_off
);
6397 incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
6398 running_off
, stride_step
);
6399 vect_finish_stmt_generation (stmt
, incr
, gsi
);
6401 running_off
= newoff
;
6404 vec_inv
= build_constructor (vectype
, v
);
6405 new_temp
= vect_init_vector (stmt
, vec_inv
, vectype
, gsi
);
6406 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
6410 new_stmt
= gimple_build_assign (make_ssa_name (ltype
),
6411 build2 (MEM_REF
, ltype
,
6412 running_off
, alias_off
));
6413 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6415 tree newoff
= copy_ssa_name (running_off
);
6416 gimple incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
6417 running_off
, stride_step
);
6418 vect_finish_stmt_generation (stmt
, incr
, gsi
);
6420 running_off
= newoff
;
6425 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
6427 dr_chain
.quick_push (gimple_assign_lhs (new_stmt
));
6432 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6434 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6435 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6439 vect_transform_slp_perm_load (slp_node
, dr_chain
, gsi
, vf
,
6440 slp_node_instance
, false);
6446 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
6448 && !SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ()
6449 && first_stmt
!= SLP_TREE_SCALAR_STMTS (slp_node
)[0])
6450 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
6452 /* Check if the chain of loads is already vectorized. */
6453 if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt
))
6454 /* For SLP we would need to copy over SLP_TREE_VEC_STMTS.
6455 ??? But we can only do so if there is exactly one
6456 as we have no way to get at the rest. Leave the CSE
6458 ??? With the group load eventually participating
6459 in multiple different permutations (having multiple
6460 slp nodes which refer to the same group) the CSE
6461 is even wrong code. See PR56270. */
6464 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
6467 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
6468 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
6471 /* VEC_NUM is the number of vect stmts to be created for this group. */
6474 grouped_load
= false;
6475 /* For SLP permutation support we need to load the whole group,
6476 not only the number of vector stmts the permutation result
6479 vec_num
= (group_size
* vf
+ nunits
- 1) / nunits
;
6481 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6482 group_gap_adj
= vf
* group_size
- nunits
* vec_num
;
6485 vec_num
= group_size
;
6491 group_size
= vec_num
= 1;
6495 alignment_support_scheme
= vect_supportable_dr_alignment (first_dr
, false);
6496 gcc_assert (alignment_support_scheme
);
6497 /* Targets with load-lane instructions must not require explicit
6499 gcc_assert (!load_lanes_p
6500 || alignment_support_scheme
== dr_aligned
6501 || alignment_support_scheme
== dr_unaligned_supported
);
6503 /* In case the vectorization factor (VF) is bigger than the number
6504 of elements that we can fit in a vectype (nunits), we have to generate
6505 more than one vector stmt - i.e - we need to "unroll" the
6506 vector stmt by a factor VF/nunits. In doing so, we record a pointer
6507 from one copy of the vector stmt to the next, in the field
6508 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
6509 stages to find the correct vector defs to be used when vectorizing
6510 stmts that use the defs of the current stmt. The example below
6511 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
6512 need to create 4 vectorized stmts):
6514 before vectorization:
6515 RELATED_STMT VEC_STMT
6519 step 1: vectorize stmt S1:
6520 We first create the vector stmt VS1_0, and, as usual, record a
6521 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
6522 Next, we create the vector stmt VS1_1, and record a pointer to
6523 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
6524 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
6526 RELATED_STMT VEC_STMT
6527 VS1_0: vx0 = memref0 VS1_1 -
6528 VS1_1: vx1 = memref1 VS1_2 -
6529 VS1_2: vx2 = memref2 VS1_3 -
6530 VS1_3: vx3 = memref3 - -
6531 S1: x = load - VS1_0
6534 See in documentation in vect_get_vec_def_for_stmt_copy for how the
6535 information we recorded in RELATED_STMT field is used to vectorize
6538 /* In case of interleaving (non-unit grouped access):
6545 Vectorized loads are created in the order of memory accesses
6546 starting from the access of the first stmt of the chain:
6549 VS2: vx1 = &base + vec_size*1
6550 VS3: vx3 = &base + vec_size*2
6551 VS4: vx4 = &base + vec_size*3
6553 Then permutation statements are generated:
6555 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
6556 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
6559 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
6560 (the order of the data-refs in the output of vect_permute_load_chain
6561 corresponds to the order of scalar stmts in the interleaving chain - see
6562 the documentation of vect_permute_load_chain()).
6563 The generation of permutation stmts and recording them in
6564 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
6566 In case of both multiple types and interleaving, the vector loads and
6567 permutation stmts above are created for every copy. The result vector
6568 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
6569 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
6571 /* If the data reference is aligned (dr_aligned) or potentially unaligned
6572 on a target that supports unaligned accesses (dr_unaligned_supported)
6573 we generate the following code:
6577 p = p + indx * vectype_size;
6582 Otherwise, the data reference is potentially unaligned on a target that
6583 does not support unaligned accesses (dr_explicit_realign_optimized) -
6584 then generate the following code, in which the data in each iteration is
6585 obtained by two vector loads, one from the previous iteration, and one
6586 from the current iteration:
6588 msq_init = *(floor(p1))
6589 p2 = initial_addr + VS - 1;
6590 realignment_token = call target_builtin;
6593 p2 = p2 + indx * vectype_size
6595 vec_dest = realign_load (msq, lsq, realignment_token)
6600 /* If the misalignment remains the same throughout the execution of the
6601 loop, we can create the init_addr and permutation mask at the loop
6602 preheader. Otherwise, it needs to be created inside the loop.
6603 This can only occur when vectorizing memory accesses in the inner-loop
6604 nested within an outer-loop that is being vectorized. */
6606 if (nested_in_vect_loop
6607 && (TREE_INT_CST_LOW (DR_STEP (dr
))
6608 % GET_MODE_SIZE (TYPE_MODE (vectype
)) != 0))
6610 gcc_assert (alignment_support_scheme
!= dr_explicit_realign_optimized
);
6611 compute_in_loop
= true;
6614 if ((alignment_support_scheme
== dr_explicit_realign_optimized
6615 || alignment_support_scheme
== dr_explicit_realign
)
6616 && !compute_in_loop
)
6618 msq
= vect_setup_realignment (first_stmt
, gsi
, &realignment_token
,
6619 alignment_support_scheme
, NULL_TREE
,
6621 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
6623 phi
= as_a
<gphi
*> (SSA_NAME_DEF_STMT (msq
));
6624 byte_offset
= size_binop (MINUS_EXPR
, TYPE_SIZE_UNIT (vectype
),
6632 offset
= size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1);
6635 aggr_type
= build_array_type_nelts (elem_type
, vec_num
* nunits
);
6637 aggr_type
= vectype
;
6639 prev_stmt_info
= NULL
;
6640 for (j
= 0; j
< ncopies
; j
++)
6642 /* 1. Create the vector or array pointer update chain. */
6645 bool simd_lane_access_p
6646 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info
);
6647 if (simd_lane_access_p
6648 && TREE_CODE (DR_BASE_ADDRESS (first_dr
)) == ADDR_EXPR
6649 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr
), 0))
6650 && integer_zerop (DR_OFFSET (first_dr
))
6651 && integer_zerop (DR_INIT (first_dr
))
6652 && alias_sets_conflict_p (get_alias_set (aggr_type
),
6653 get_alias_set (DR_REF (first_dr
)))
6654 && (alignment_support_scheme
== dr_aligned
6655 || alignment_support_scheme
== dr_unaligned_supported
))
6657 dataref_ptr
= unshare_expr (DR_BASE_ADDRESS (first_dr
));
6658 dataref_offset
= build_int_cst (reference_alias_ptr_type
6659 (DR_REF (first_dr
)), 0);
6664 = vect_create_data_ref_ptr (first_stmt
, aggr_type
, at_loop
,
6665 offset
, &dummy
, gsi
, &ptr_incr
,
6666 simd_lane_access_p
, &inv_p
,
6669 else if (dataref_offset
)
6670 dataref_offset
= int_const_binop (PLUS_EXPR
, dataref_offset
,
6671 TYPE_SIZE_UNIT (aggr_type
));
6673 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
6674 TYPE_SIZE_UNIT (aggr_type
));
6676 if (grouped_load
|| slp_perm
)
6677 dr_chain
.create (vec_num
);
6683 vec_array
= create_vector_array (vectype
, vec_num
);
6686 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
6687 data_ref
= create_array_ref (aggr_type
, dataref_ptr
, first_dr
);
6688 new_stmt
= gimple_build_call_internal (IFN_LOAD_LANES
, 1, data_ref
);
6689 gimple_call_set_lhs (new_stmt
, vec_array
);
6690 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6692 /* Extract each vector into an SSA_NAME. */
6693 for (i
= 0; i
< vec_num
; i
++)
6695 new_temp
= read_vector_array (stmt
, gsi
, scalar_dest
,
6697 dr_chain
.quick_push (new_temp
);
6700 /* Record the mapping between SSA_NAMEs and statements. */
6701 vect_record_grouped_load_vectors (stmt
, dr_chain
);
6705 for (i
= 0; i
< vec_num
; i
++)
6708 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
6711 /* 2. Create the vector-load in the loop. */
6712 switch (alignment_support_scheme
)
6715 case dr_unaligned_supported
:
6717 unsigned int align
, misalign
;
6720 = fold_build2 (MEM_REF
, vectype
, dataref_ptr
,
6723 : build_int_cst (reference_alias_ptr_type
6724 (DR_REF (first_dr
)), 0));
6725 align
= TYPE_ALIGN_UNIT (vectype
);
6726 if (alignment_support_scheme
== dr_aligned
)
6728 gcc_assert (aligned_access_p (first_dr
));
6731 else if (DR_MISALIGNMENT (first_dr
) == -1)
6733 TREE_TYPE (data_ref
)
6734 = build_aligned_type (TREE_TYPE (data_ref
),
6735 TYPE_ALIGN (elem_type
));
6736 align
= TYPE_ALIGN_UNIT (elem_type
);
6741 TREE_TYPE (data_ref
)
6742 = build_aligned_type (TREE_TYPE (data_ref
),
6743 TYPE_ALIGN (elem_type
));
6744 misalign
= DR_MISALIGNMENT (first_dr
);
6746 if (dataref_offset
== NULL_TREE
6747 && TREE_CODE (dataref_ptr
) == SSA_NAME
)
6748 set_ptr_info_alignment (get_ptr_info (dataref_ptr
),
6752 case dr_explicit_realign
:
6756 tree vs
= size_int (TYPE_VECTOR_SUBPARTS (vectype
));
6758 if (compute_in_loop
)
6759 msq
= vect_setup_realignment (first_stmt
, gsi
,
6761 dr_explicit_realign
,
6764 if (TREE_CODE (dataref_ptr
) == SSA_NAME
)
6765 ptr
= copy_ssa_name (dataref_ptr
);
6767 ptr
= make_ssa_name (TREE_TYPE (dataref_ptr
));
6768 new_stmt
= gimple_build_assign
6769 (ptr
, BIT_AND_EXPR
, dataref_ptr
,
6771 (TREE_TYPE (dataref_ptr
),
6772 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
6773 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6775 = build2 (MEM_REF
, vectype
, ptr
,
6776 build_int_cst (reference_alias_ptr_type
6777 (DR_REF (first_dr
)), 0));
6778 vec_dest
= vect_create_destination_var (scalar_dest
,
6780 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
6781 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
6782 gimple_assign_set_lhs (new_stmt
, new_temp
);
6783 gimple_set_vdef (new_stmt
, gimple_vdef (stmt
));
6784 gimple_set_vuse (new_stmt
, gimple_vuse (stmt
));
6785 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6788 bump
= size_binop (MULT_EXPR
, vs
,
6789 TYPE_SIZE_UNIT (elem_type
));
6790 bump
= size_binop (MINUS_EXPR
, bump
, size_one_node
);
6791 ptr
= bump_vector_ptr (dataref_ptr
, NULL
, gsi
, stmt
, bump
);
6792 new_stmt
= gimple_build_assign
6793 (NULL_TREE
, BIT_AND_EXPR
, ptr
,
6796 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
6797 ptr
= copy_ssa_name (ptr
, new_stmt
);
6798 gimple_assign_set_lhs (new_stmt
, ptr
);
6799 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6801 = build2 (MEM_REF
, vectype
, ptr
,
6802 build_int_cst (reference_alias_ptr_type
6803 (DR_REF (first_dr
)), 0));
6806 case dr_explicit_realign_optimized
:
6807 if (TREE_CODE (dataref_ptr
) == SSA_NAME
)
6808 new_temp
= copy_ssa_name (dataref_ptr
);
6810 new_temp
= make_ssa_name (TREE_TYPE (dataref_ptr
));
6811 new_stmt
= gimple_build_assign
6812 (new_temp
, BIT_AND_EXPR
, dataref_ptr
,
6814 (TREE_TYPE (dataref_ptr
),
6815 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
6816 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6818 = build2 (MEM_REF
, vectype
, new_temp
,
6819 build_int_cst (reference_alias_ptr_type
6820 (DR_REF (first_dr
)), 0));
6825 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6826 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
6827 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
6828 gimple_assign_set_lhs (new_stmt
, new_temp
);
6829 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6831 /* 3. Handle explicit realignment if necessary/supported.
6833 vec_dest = realign_load (msq, lsq, realignment_token) */
6834 if (alignment_support_scheme
== dr_explicit_realign_optimized
6835 || alignment_support_scheme
== dr_explicit_realign
)
6837 lsq
= gimple_assign_lhs (new_stmt
);
6838 if (!realignment_token
)
6839 realignment_token
= dataref_ptr
;
6840 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6841 new_stmt
= gimple_build_assign (vec_dest
, REALIGN_LOAD_EXPR
,
6842 msq
, lsq
, realignment_token
);
6843 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
6844 gimple_assign_set_lhs (new_stmt
, new_temp
);
6845 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6847 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
6850 if (i
== vec_num
- 1 && j
== ncopies
- 1)
6851 add_phi_arg (phi
, lsq
,
6852 loop_latch_edge (containing_loop
),
6858 /* 4. Handle invariant-load. */
6859 if (inv_p
&& !bb_vinfo
)
6861 gcc_assert (!grouped_load
);
6862 /* If we have versioned for aliasing or the loop doesn't
6863 have any data dependencies that would preclude this,
6864 then we are sure this is a loop invariant load and
6865 thus we can insert it on the preheader edge. */
6866 if (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo
)
6867 && !nested_in_vect_loop
6868 && hoist_defs_of_uses (stmt
, loop
))
6870 if (dump_enabled_p ())
6872 dump_printf_loc (MSG_NOTE
, vect_location
,
6873 "hoisting out of the vectorized "
6875 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
6877 tree tem
= copy_ssa_name (scalar_dest
);
6878 gsi_insert_on_edge_immediate
6879 (loop_preheader_edge (loop
),
6880 gimple_build_assign (tem
,
6882 (gimple_assign_rhs1 (stmt
))));
6883 new_temp
= vect_init_vector (stmt
, tem
, vectype
, NULL
);
6887 gimple_stmt_iterator gsi2
= *gsi
;
6889 new_temp
= vect_init_vector (stmt
, scalar_dest
,
6892 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
6893 set_vinfo_for_stmt (new_stmt
,
6894 new_stmt_vec_info (new_stmt
, loop_vinfo
,
6900 tree perm_mask
= perm_mask_for_reverse (vectype
);
6901 new_temp
= permute_vec_elements (new_temp
, new_temp
,
6902 perm_mask
, stmt
, gsi
);
6903 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
6906 /* Collect vector loads and later create their permutation in
6907 vect_transform_grouped_load (). */
6908 if (grouped_load
|| slp_perm
)
6909 dr_chain
.quick_push (new_temp
);
6911 /* Store vector loads in the corresponding SLP_NODE. */
6912 if (slp
&& !slp_perm
)
6913 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
6915 /* Bump the vector pointer to account for a gap or for excess
6916 elements loaded for a permuted SLP load. */
6917 if (group_gap_adj
!= 0)
6921 = wide_int_to_tree (sizetype
,
6922 wi::smul (TYPE_SIZE_UNIT (elem_type
),
6923 group_gap_adj
, &ovf
));
6924 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
6929 if (slp
&& !slp_perm
)
6934 if (!vect_transform_slp_perm_load (slp_node
, dr_chain
, gsi
, vf
,
6935 slp_node_instance
, false))
6937 dr_chain
.release ();
6946 vect_transform_grouped_load (stmt
, dr_chain
, group_size
, gsi
);
6947 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
6952 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6954 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6955 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6958 dr_chain
.release ();
6964 /* Function vect_is_simple_cond.
6967 LOOP - the loop that is being vectorized.
6968 COND - Condition that is checked for simple use.
6971 *COMP_VECTYPE - the vector type for the comparison.
6973 Returns whether a COND can be vectorized. Checks whether
6974 condition operands are supportable using vec_is_simple_use. */
6977 vect_is_simple_cond (tree cond
, gimple stmt
, loop_vec_info loop_vinfo
,
6978 bb_vec_info bb_vinfo
, tree
*comp_vectype
)
6982 enum vect_def_type dt
;
6983 tree vectype1
= NULL_TREE
, vectype2
= NULL_TREE
;
6985 if (!COMPARISON_CLASS_P (cond
))
6988 lhs
= TREE_OPERAND (cond
, 0);
6989 rhs
= TREE_OPERAND (cond
, 1);
6991 if (TREE_CODE (lhs
) == SSA_NAME
)
6993 gimple lhs_def_stmt
= SSA_NAME_DEF_STMT (lhs
);
6994 if (!vect_is_simple_use_1 (lhs
, stmt
, loop_vinfo
, bb_vinfo
,
6995 &lhs_def_stmt
, &def
, &dt
, &vectype1
))
6998 else if (TREE_CODE (lhs
) != INTEGER_CST
&& TREE_CODE (lhs
) != REAL_CST
6999 && TREE_CODE (lhs
) != FIXED_CST
)
7002 if (TREE_CODE (rhs
) == SSA_NAME
)
7004 gimple rhs_def_stmt
= SSA_NAME_DEF_STMT (rhs
);
7005 if (!vect_is_simple_use_1 (rhs
, stmt
, loop_vinfo
, bb_vinfo
,
7006 &rhs_def_stmt
, &def
, &dt
, &vectype2
))
7009 else if (TREE_CODE (rhs
) != INTEGER_CST
&& TREE_CODE (rhs
) != REAL_CST
7010 && TREE_CODE (rhs
) != FIXED_CST
)
7013 *comp_vectype
= vectype1
? vectype1
: vectype2
;
7017 /* vectorizable_condition.
7019 Check if STMT is conditional modify expression that can be vectorized.
7020 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
7021 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
7024 When STMT is vectorized as nested cycle, REDUC_DEF is the vector variable
7025 to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in
7026 else caluse if it is 2).
7028 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
7031 vectorizable_condition (gimple stmt
, gimple_stmt_iterator
*gsi
,
7032 gimple
*vec_stmt
, tree reduc_def
, int reduc_index
,
7035 tree scalar_dest
= NULL_TREE
;
7036 tree vec_dest
= NULL_TREE
;
7037 tree cond_expr
, then_clause
, else_clause
;
7038 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7039 tree comp_vectype
= NULL_TREE
;
7040 tree vec_cond_lhs
= NULL_TREE
, vec_cond_rhs
= NULL_TREE
;
7041 tree vec_then_clause
= NULL_TREE
, vec_else_clause
= NULL_TREE
;
7042 tree vec_compare
, vec_cond_expr
;
7044 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
7046 enum vect_def_type dt
, dts
[4];
7048 enum tree_code code
;
7049 stmt_vec_info prev_stmt_info
= NULL
;
7051 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
7052 vec
<tree
> vec_oprnds0
= vNULL
;
7053 vec
<tree
> vec_oprnds1
= vNULL
;
7054 vec
<tree
> vec_oprnds2
= vNULL
;
7055 vec
<tree
> vec_oprnds3
= vNULL
;
7058 if (reduc_index
&& STMT_SLP_TYPE (stmt_info
))
7061 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
7064 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
7065 && !(STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
7069 /* FORNOW: not yet supported. */
7070 if (STMT_VINFO_LIVE_P (stmt_info
))
7072 if (dump_enabled_p ())
7073 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7074 "value used after loop.\n");
7078 /* Is vectorizable conditional operation? */
7079 if (!is_gimple_assign (stmt
))
7082 code
= gimple_assign_rhs_code (stmt
);
7084 if (code
!= COND_EXPR
)
7087 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
7088 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
7090 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
7093 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
7095 gcc_assert (ncopies
>= 1);
7096 if (reduc_index
&& ncopies
> 1)
7097 return false; /* FORNOW */
7099 cond_expr
= gimple_assign_rhs1 (stmt
);
7100 then_clause
= gimple_assign_rhs2 (stmt
);
7101 else_clause
= gimple_assign_rhs3 (stmt
);
7103 if (!vect_is_simple_cond (cond_expr
, stmt
, loop_vinfo
, bb_vinfo
,
7108 if (TREE_CODE (then_clause
) == SSA_NAME
)
7110 gimple then_def_stmt
= SSA_NAME_DEF_STMT (then_clause
);
7111 if (!vect_is_simple_use (then_clause
, stmt
, loop_vinfo
, bb_vinfo
,
7112 &then_def_stmt
, &def
, &dt
))
7115 else if (TREE_CODE (then_clause
) != INTEGER_CST
7116 && TREE_CODE (then_clause
) != REAL_CST
7117 && TREE_CODE (then_clause
) != FIXED_CST
)
7120 if (TREE_CODE (else_clause
) == SSA_NAME
)
7122 gimple else_def_stmt
= SSA_NAME_DEF_STMT (else_clause
);
7123 if (!vect_is_simple_use (else_clause
, stmt
, loop_vinfo
, bb_vinfo
,
7124 &else_def_stmt
, &def
, &dt
))
7127 else if (TREE_CODE (else_clause
) != INTEGER_CST
7128 && TREE_CODE (else_clause
) != REAL_CST
7129 && TREE_CODE (else_clause
) != FIXED_CST
)
7132 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (vectype
)));
7133 /* The result of a vector comparison should be signed type. */
7134 tree cmp_type
= build_nonstandard_integer_type (prec
, 0);
7135 vec_cmp_type
= get_same_sized_vectype (cmp_type
, vectype
);
7136 if (vec_cmp_type
== NULL_TREE
)
7141 STMT_VINFO_TYPE (stmt_info
) = condition_vec_info_type
;
7142 return expand_vec_cond_expr_p (vectype
, comp_vectype
);
7149 vec_oprnds0
.create (1);
7150 vec_oprnds1
.create (1);
7151 vec_oprnds2
.create (1);
7152 vec_oprnds3
.create (1);
7156 scalar_dest
= gimple_assign_lhs (stmt
);
7157 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
7159 /* Handle cond expr. */
7160 for (j
= 0; j
< ncopies
; j
++)
7162 gassign
*new_stmt
= NULL
;
7167 auto_vec
<tree
, 4> ops
;
7168 auto_vec
<vec
<tree
>, 4> vec_defs
;
7170 ops
.safe_push (TREE_OPERAND (cond_expr
, 0));
7171 ops
.safe_push (TREE_OPERAND (cond_expr
, 1));
7172 ops
.safe_push (then_clause
);
7173 ops
.safe_push (else_clause
);
7174 vect_get_slp_defs (ops
, slp_node
, &vec_defs
, -1);
7175 vec_oprnds3
= vec_defs
.pop ();
7176 vec_oprnds2
= vec_defs
.pop ();
7177 vec_oprnds1
= vec_defs
.pop ();
7178 vec_oprnds0
= vec_defs
.pop ();
7181 vec_defs
.release ();
7187 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr
, 0),
7189 vect_is_simple_use (TREE_OPERAND (cond_expr
, 0), stmt
,
7190 loop_vinfo
, NULL
, >emp
, &def
, &dts
[0]);
7193 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr
, 1),
7195 vect_is_simple_use (TREE_OPERAND (cond_expr
, 1), stmt
,
7196 loop_vinfo
, NULL
, >emp
, &def
, &dts
[1]);
7197 if (reduc_index
== 1)
7198 vec_then_clause
= reduc_def
;
7201 vec_then_clause
= vect_get_vec_def_for_operand (then_clause
,
7203 vect_is_simple_use (then_clause
, stmt
, loop_vinfo
,
7204 NULL
, >emp
, &def
, &dts
[2]);
7206 if (reduc_index
== 2)
7207 vec_else_clause
= reduc_def
;
7210 vec_else_clause
= vect_get_vec_def_for_operand (else_clause
,
7212 vect_is_simple_use (else_clause
, stmt
, loop_vinfo
,
7213 NULL
, >emp
, &def
, &dts
[3]);
7219 vec_cond_lhs
= vect_get_vec_def_for_stmt_copy (dts
[0],
7220 vec_oprnds0
.pop ());
7221 vec_cond_rhs
= vect_get_vec_def_for_stmt_copy (dts
[1],
7222 vec_oprnds1
.pop ());
7223 vec_then_clause
= vect_get_vec_def_for_stmt_copy (dts
[2],
7224 vec_oprnds2
.pop ());
7225 vec_else_clause
= vect_get_vec_def_for_stmt_copy (dts
[3],
7226 vec_oprnds3
.pop ());
7231 vec_oprnds0
.quick_push (vec_cond_lhs
);
7232 vec_oprnds1
.quick_push (vec_cond_rhs
);
7233 vec_oprnds2
.quick_push (vec_then_clause
);
7234 vec_oprnds3
.quick_push (vec_else_clause
);
7237 /* Arguments are ready. Create the new vector stmt. */
7238 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_cond_lhs
)
7240 vec_cond_rhs
= vec_oprnds1
[i
];
7241 vec_then_clause
= vec_oprnds2
[i
];
7242 vec_else_clause
= vec_oprnds3
[i
];
7244 vec_compare
= build2 (TREE_CODE (cond_expr
), vec_cmp_type
,
7245 vec_cond_lhs
, vec_cond_rhs
);
7246 vec_cond_expr
= build3 (VEC_COND_EXPR
, vectype
,
7247 vec_compare
, vec_then_clause
, vec_else_clause
);
7249 new_stmt
= gimple_build_assign (vec_dest
, vec_cond_expr
);
7250 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
7251 gimple_assign_set_lhs (new_stmt
, new_temp
);
7252 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7254 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
7261 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
7263 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
7265 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
7268 vec_oprnds0
.release ();
7269 vec_oprnds1
.release ();
7270 vec_oprnds2
.release ();
7271 vec_oprnds3
.release ();
7277 /* Make sure the statement is vectorizable. */
7280 vect_analyze_stmt (gimple stmt
, bool *need_to_vectorize
, slp_tree node
)
7282 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7283 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
7284 enum vect_relevant relevance
= STMT_VINFO_RELEVANT (stmt_info
);
7286 tree scalar_type
, vectype
;
7287 gimple pattern_stmt
;
7288 gimple_seq pattern_def_seq
;
7290 if (dump_enabled_p ())
7292 dump_printf_loc (MSG_NOTE
, vect_location
, "==> examining statement: ");
7293 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7296 if (gimple_has_volatile_ops (stmt
))
7298 if (dump_enabled_p ())
7299 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7300 "not vectorized: stmt has volatile operands\n");
7305 /* Skip stmts that do not need to be vectorized. In loops this is expected
7307 - the COND_EXPR which is the loop exit condition
7308 - any LABEL_EXPRs in the loop
7309 - computations that are used only for array indexing or loop control.
7310 In basic blocks we only analyze statements that are a part of some SLP
7311 instance, therefore, all the statements are relevant.
7313 Pattern statement needs to be analyzed instead of the original statement
7314 if the original statement is not relevant. Otherwise, we analyze both
7315 statements. In basic blocks we are called from some SLP instance
7316 traversal, don't analyze pattern stmts instead, the pattern stmts
7317 already will be part of SLP instance. */
7319 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
7320 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
7321 && !STMT_VINFO_LIVE_P (stmt_info
))
7323 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
7325 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
7326 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
7328 /* Analyze PATTERN_STMT instead of the original stmt. */
7329 stmt
= pattern_stmt
;
7330 stmt_info
= vinfo_for_stmt (pattern_stmt
);
7331 if (dump_enabled_p ())
7333 dump_printf_loc (MSG_NOTE
, vect_location
,
7334 "==> examining pattern statement: ");
7335 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7340 if (dump_enabled_p ())
7341 dump_printf_loc (MSG_NOTE
, vect_location
, "irrelevant.\n");
7346 else if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
7349 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
7350 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
7352 /* Analyze PATTERN_STMT too. */
7353 if (dump_enabled_p ())
7355 dump_printf_loc (MSG_NOTE
, vect_location
,
7356 "==> examining pattern statement: ");
7357 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7360 if (!vect_analyze_stmt (pattern_stmt
, need_to_vectorize
, node
))
7364 if (is_pattern_stmt_p (stmt_info
)
7366 && (pattern_def_seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
)))
7368 gimple_stmt_iterator si
;
7370 for (si
= gsi_start (pattern_def_seq
); !gsi_end_p (si
); gsi_next (&si
))
7372 gimple pattern_def_stmt
= gsi_stmt (si
);
7373 if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt
))
7374 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt
)))
7376 /* Analyze def stmt of STMT if it's a pattern stmt. */
7377 if (dump_enabled_p ())
7379 dump_printf_loc (MSG_NOTE
, vect_location
,
7380 "==> examining pattern def statement: ");
7381 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, pattern_def_stmt
, 0);
7384 if (!vect_analyze_stmt (pattern_def_stmt
,
7385 need_to_vectorize
, node
))
7391 switch (STMT_VINFO_DEF_TYPE (stmt_info
))
7393 case vect_internal_def
:
7396 case vect_reduction_def
:
7397 case vect_nested_cycle
:
7398 gcc_assert (!bb_vinfo
7399 && (relevance
== vect_used_in_outer
7400 || relevance
== vect_used_in_outer_by_reduction
7401 || relevance
== vect_used_by_reduction
7402 || relevance
== vect_unused_in_scope
));
7405 case vect_induction_def
:
7406 case vect_constant_def
:
7407 case vect_external_def
:
7408 case vect_unknown_def_type
:
7415 gcc_assert (PURE_SLP_STMT (stmt_info
));
7417 scalar_type
= TREE_TYPE (gimple_get_lhs (stmt
));
7418 if (dump_enabled_p ())
7420 dump_printf_loc (MSG_NOTE
, vect_location
,
7421 "get vectype for scalar type: ");
7422 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, scalar_type
);
7423 dump_printf (MSG_NOTE
, "\n");
7426 vectype
= get_vectype_for_scalar_type (scalar_type
);
7429 if (dump_enabled_p ())
7431 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7432 "not SLPed: unsupported data-type ");
7433 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
7435 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
7440 if (dump_enabled_p ())
7442 dump_printf_loc (MSG_NOTE
, vect_location
, "vectype: ");
7443 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, vectype
);
7444 dump_printf (MSG_NOTE
, "\n");
7447 STMT_VINFO_VECTYPE (stmt_info
) = vectype
;
7450 if (STMT_VINFO_RELEVANT_P (stmt_info
))
7452 gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt
))));
7453 gcc_assert (STMT_VINFO_VECTYPE (stmt_info
)
7454 || (is_gimple_call (stmt
)
7455 && gimple_call_lhs (stmt
) == NULL_TREE
));
7456 *need_to_vectorize
= true;
7459 if (PURE_SLP_STMT (stmt_info
) && !node
)
7461 dump_printf_loc (MSG_NOTE
, vect_location
,
7462 "handled only by SLP analysis\n");
7468 && (STMT_VINFO_RELEVANT_P (stmt_info
)
7469 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
))
7470 ok
= (vectorizable_simd_clone_call (stmt
, NULL
, NULL
, node
)
7471 || vectorizable_conversion (stmt
, NULL
, NULL
, node
)
7472 || vectorizable_shift (stmt
, NULL
, NULL
, node
)
7473 || vectorizable_operation (stmt
, NULL
, NULL
, node
)
7474 || vectorizable_assignment (stmt
, NULL
, NULL
, node
)
7475 || vectorizable_load (stmt
, NULL
, NULL
, node
, NULL
)
7476 || vectorizable_call (stmt
, NULL
, NULL
, node
)
7477 || vectorizable_store (stmt
, NULL
, NULL
, node
)
7478 || vectorizable_reduction (stmt
, NULL
, NULL
, node
)
7479 || vectorizable_condition (stmt
, NULL
, NULL
, NULL
, 0, node
));
7483 ok
= (vectorizable_simd_clone_call (stmt
, NULL
, NULL
, node
)
7484 || vectorizable_conversion (stmt
, NULL
, NULL
, node
)
7485 || vectorizable_shift (stmt
, NULL
, NULL
, node
)
7486 || vectorizable_operation (stmt
, NULL
, NULL
, node
)
7487 || vectorizable_assignment (stmt
, NULL
, NULL
, node
)
7488 || vectorizable_load (stmt
, NULL
, NULL
, node
, NULL
)
7489 || vectorizable_call (stmt
, NULL
, NULL
, node
)
7490 || vectorizable_store (stmt
, NULL
, NULL
, node
)
7491 || vectorizable_condition (stmt
, NULL
, NULL
, NULL
, 0, node
));
7496 if (dump_enabled_p ())
7498 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7499 "not vectorized: relevant stmt not ");
7500 dump_printf (MSG_MISSED_OPTIMIZATION
, "supported: ");
7501 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
7510 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
7511 need extra handling, except for vectorizable reductions. */
7512 if (STMT_VINFO_LIVE_P (stmt_info
)
7513 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
)
7514 ok
= vectorizable_live_operation (stmt
, NULL
, NULL
);
7518 if (dump_enabled_p ())
7520 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7521 "not vectorized: live stmt not ");
7522 dump_printf (MSG_MISSED_OPTIMIZATION
, "supported: ");
7523 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
7533 /* Function vect_transform_stmt.
7535 Create a vectorized stmt to replace STMT, and insert it at BSI. */
7538 vect_transform_stmt (gimple stmt
, gimple_stmt_iterator
*gsi
,
7539 bool *grouped_store
, slp_tree slp_node
,
7540 slp_instance slp_node_instance
)
7542 bool is_store
= false;
7543 gimple vec_stmt
= NULL
;
7544 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7547 gimple old_vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
7549 switch (STMT_VINFO_TYPE (stmt_info
))
7551 case type_demotion_vec_info_type
:
7552 case type_promotion_vec_info_type
:
7553 case type_conversion_vec_info_type
:
7554 done
= vectorizable_conversion (stmt
, gsi
, &vec_stmt
, slp_node
);
7558 case induc_vec_info_type
:
7559 gcc_assert (!slp_node
);
7560 done
= vectorizable_induction (stmt
, gsi
, &vec_stmt
);
7564 case shift_vec_info_type
:
7565 done
= vectorizable_shift (stmt
, gsi
, &vec_stmt
, slp_node
);
7569 case op_vec_info_type
:
7570 done
= vectorizable_operation (stmt
, gsi
, &vec_stmt
, slp_node
);
7574 case assignment_vec_info_type
:
7575 done
= vectorizable_assignment (stmt
, gsi
, &vec_stmt
, slp_node
);
7579 case load_vec_info_type
:
7580 done
= vectorizable_load (stmt
, gsi
, &vec_stmt
, slp_node
,
7585 case store_vec_info_type
:
7586 done
= vectorizable_store (stmt
, gsi
, &vec_stmt
, slp_node
);
7588 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
) && !slp_node
)
7590 /* In case of interleaving, the whole chain is vectorized when the
7591 last store in the chain is reached. Store stmts before the last
7592 one are skipped, and there vec_stmt_info shouldn't be freed
7594 *grouped_store
= true;
7595 if (STMT_VINFO_VEC_STMT (stmt_info
))
7602 case condition_vec_info_type
:
7603 done
= vectorizable_condition (stmt
, gsi
, &vec_stmt
, NULL
, 0, slp_node
);
7607 case call_vec_info_type
:
7608 done
= vectorizable_call (stmt
, gsi
, &vec_stmt
, slp_node
);
7609 stmt
= gsi_stmt (*gsi
);
7610 if (is_gimple_call (stmt
)
7611 && gimple_call_internal_p (stmt
)
7612 && gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
)
7616 case call_simd_clone_vec_info_type
:
7617 done
= vectorizable_simd_clone_call (stmt
, gsi
, &vec_stmt
, slp_node
);
7618 stmt
= gsi_stmt (*gsi
);
7621 case reduc_vec_info_type
:
7622 done
= vectorizable_reduction (stmt
, gsi
, &vec_stmt
, slp_node
);
7627 if (!STMT_VINFO_LIVE_P (stmt_info
))
7629 if (dump_enabled_p ())
7630 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7631 "stmt not supported.\n");
7636 /* Verify SLP vectorization doesn't mess with STMT_VINFO_VEC_STMT.
7637 This would break hybrid SLP vectorization. */
7639 gcc_assert (!vec_stmt
7640 && STMT_VINFO_VEC_STMT (stmt_info
) == old_vec_stmt
);
7642 /* Handle inner-loop stmts whose DEF is used in the loop-nest that
7643 is being vectorized, but outside the immediately enclosing loop. */
7645 && STMT_VINFO_LOOP_VINFO (stmt_info
)
7646 && nested_in_vect_loop_p (LOOP_VINFO_LOOP (
7647 STMT_VINFO_LOOP_VINFO (stmt_info
)), stmt
)
7648 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
7649 && (STMT_VINFO_RELEVANT (stmt_info
) == vect_used_in_outer
7650 || STMT_VINFO_RELEVANT (stmt_info
) ==
7651 vect_used_in_outer_by_reduction
))
7653 struct loop
*innerloop
= LOOP_VINFO_LOOP (
7654 STMT_VINFO_LOOP_VINFO (stmt_info
))->inner
;
7655 imm_use_iterator imm_iter
;
7656 use_operand_p use_p
;
7660 if (dump_enabled_p ())
7661 dump_printf_loc (MSG_NOTE
, vect_location
,
7662 "Record the vdef for outer-loop vectorization.\n");
7664 /* Find the relevant loop-exit phi-node, and reord the vec_stmt there
7665 (to be used when vectorizing outer-loop stmts that use the DEF of
7667 if (gimple_code (stmt
) == GIMPLE_PHI
)
7668 scalar_dest
= PHI_RESULT (stmt
);
7670 scalar_dest
= gimple_assign_lhs (stmt
);
7672 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, scalar_dest
)
7674 if (!flow_bb_inside_loop_p (innerloop
, gimple_bb (USE_STMT (use_p
))))
7676 exit_phi
= USE_STMT (use_p
);
7677 STMT_VINFO_VEC_STMT (vinfo_for_stmt (exit_phi
)) = vec_stmt
;
7682 /* Handle stmts whose DEF is used outside the loop-nest that is
7683 being vectorized. */
7684 if (STMT_VINFO_LIVE_P (stmt_info
)
7685 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
)
7687 done
= vectorizable_live_operation (stmt
, gsi
, &vec_stmt
);
7692 STMT_VINFO_VEC_STMT (stmt_info
) = vec_stmt
;
7698 /* Remove a group of stores (for SLP or interleaving), free their
7702 vect_remove_stores (gimple first_stmt
)
7704 gimple next
= first_stmt
;
7706 gimple_stmt_iterator next_si
;
7710 stmt_vec_info stmt_info
= vinfo_for_stmt (next
);
7712 tmp
= GROUP_NEXT_ELEMENT (stmt_info
);
7713 if (is_pattern_stmt_p (stmt_info
))
7714 next
= STMT_VINFO_RELATED_STMT (stmt_info
);
7715 /* Free the attached stmt_vec_info and remove the stmt. */
7716 next_si
= gsi_for_stmt (next
);
7717 unlink_stmt_vdef (next
);
7718 gsi_remove (&next_si
, true);
7719 release_defs (next
);
7720 free_stmt_vec_info (next
);
7726 /* Function new_stmt_vec_info.
7728 Create and initialize a new stmt_vec_info struct for STMT. */
7731 new_stmt_vec_info (gimple stmt
, loop_vec_info loop_vinfo
,
7732 bb_vec_info bb_vinfo
)
7735 res
= (stmt_vec_info
) xcalloc (1, sizeof (struct _stmt_vec_info
));
7737 STMT_VINFO_TYPE (res
) = undef_vec_info_type
;
7738 STMT_VINFO_STMT (res
) = stmt
;
7739 STMT_VINFO_LOOP_VINFO (res
) = loop_vinfo
;
7740 STMT_VINFO_BB_VINFO (res
) = bb_vinfo
;
7741 STMT_VINFO_RELEVANT (res
) = vect_unused_in_scope
;
7742 STMT_VINFO_LIVE_P (res
) = false;
7743 STMT_VINFO_VECTYPE (res
) = NULL
;
7744 STMT_VINFO_VEC_STMT (res
) = NULL
;
7745 STMT_VINFO_VECTORIZABLE (res
) = true;
7746 STMT_VINFO_IN_PATTERN_P (res
) = false;
7747 STMT_VINFO_RELATED_STMT (res
) = NULL
;
7748 STMT_VINFO_PATTERN_DEF_SEQ (res
) = NULL
;
7749 STMT_VINFO_DATA_REF (res
) = NULL
;
7751 STMT_VINFO_DR_BASE_ADDRESS (res
) = NULL
;
7752 STMT_VINFO_DR_OFFSET (res
) = NULL
;
7753 STMT_VINFO_DR_INIT (res
) = NULL
;
7754 STMT_VINFO_DR_STEP (res
) = NULL
;
7755 STMT_VINFO_DR_ALIGNED_TO (res
) = NULL
;
7757 if (gimple_code (stmt
) == GIMPLE_PHI
7758 && is_loop_header_bb_p (gimple_bb (stmt
)))
7759 STMT_VINFO_DEF_TYPE (res
) = vect_unknown_def_type
;
7761 STMT_VINFO_DEF_TYPE (res
) = vect_internal_def
;
7763 STMT_VINFO_SAME_ALIGN_REFS (res
).create (0);
7764 STMT_SLP_TYPE (res
) = loop_vect
;
7765 GROUP_FIRST_ELEMENT (res
) = NULL
;
7766 GROUP_NEXT_ELEMENT (res
) = NULL
;
7767 GROUP_SIZE (res
) = 0;
7768 GROUP_STORE_COUNT (res
) = 0;
7769 GROUP_GAP (res
) = 0;
7770 GROUP_SAME_DR_STMT (res
) = NULL
;
7776 /* Create a hash table for stmt_vec_info. */
7779 init_stmt_vec_info_vec (void)
7781 gcc_assert (!stmt_vec_info_vec
.exists ());
7782 stmt_vec_info_vec
.create (50);
7786 /* Free hash table for stmt_vec_info. */
7789 free_stmt_vec_info_vec (void)
7793 FOR_EACH_VEC_ELT (stmt_vec_info_vec
, i
, info
)
7795 free_stmt_vec_info (STMT_VINFO_STMT ((stmt_vec_info
) info
));
7796 gcc_assert (stmt_vec_info_vec
.exists ());
7797 stmt_vec_info_vec
.release ();
7801 /* Free stmt vectorization related info. */
7804 free_stmt_vec_info (gimple stmt
)
7806 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7811 /* Check if this statement has a related "pattern stmt"
7812 (introduced by the vectorizer during the pattern recognition
7813 pass). Free pattern's stmt_vec_info and def stmt's stmt_vec_info
7815 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
7817 stmt_vec_info patt_info
7818 = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info
));
7821 gimple_seq seq
= STMT_VINFO_PATTERN_DEF_SEQ (patt_info
);
7822 gimple patt_stmt
= STMT_VINFO_STMT (patt_info
);
7823 gimple_set_bb (patt_stmt
, NULL
);
7824 tree lhs
= gimple_get_lhs (patt_stmt
);
7825 if (TREE_CODE (lhs
) == SSA_NAME
)
7826 release_ssa_name (lhs
);
7829 gimple_stmt_iterator si
;
7830 for (si
= gsi_start (seq
); !gsi_end_p (si
); gsi_next (&si
))
7832 gimple seq_stmt
= gsi_stmt (si
);
7833 gimple_set_bb (seq_stmt
, NULL
);
7834 lhs
= gimple_get_lhs (patt_stmt
);
7835 if (TREE_CODE (lhs
) == SSA_NAME
)
7836 release_ssa_name (lhs
);
7837 free_stmt_vec_info (seq_stmt
);
7840 free_stmt_vec_info (patt_stmt
);
7844 STMT_VINFO_SAME_ALIGN_REFS (stmt_info
).release ();
7845 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).release ();
7846 set_vinfo_for_stmt (stmt
, NULL
);
7851 /* Function get_vectype_for_scalar_type_and_size.
7853 Returns the vector type corresponding to SCALAR_TYPE and SIZE as supported
7857 get_vectype_for_scalar_type_and_size (tree scalar_type
, unsigned size
)
7859 machine_mode inner_mode
= TYPE_MODE (scalar_type
);
7860 machine_mode simd_mode
;
7861 unsigned int nbytes
= GET_MODE_SIZE (inner_mode
);
7868 if (GET_MODE_CLASS (inner_mode
) != MODE_INT
7869 && GET_MODE_CLASS (inner_mode
) != MODE_FLOAT
)
7872 /* For vector types of elements whose mode precision doesn't
7873 match their types precision we use a element type of mode
7874 precision. The vectorization routines will have to make sure
7875 they support the proper result truncation/extension.
7876 We also make sure to build vector types with INTEGER_TYPE
7877 component type only. */
7878 if (INTEGRAL_TYPE_P (scalar_type
)
7879 && (GET_MODE_BITSIZE (inner_mode
) != TYPE_PRECISION (scalar_type
)
7880 || TREE_CODE (scalar_type
) != INTEGER_TYPE
))
7881 scalar_type
= build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode
),
7882 TYPE_UNSIGNED (scalar_type
));
7884 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
7885 When the component mode passes the above test simply use a type
7886 corresponding to that mode. The theory is that any use that
7887 would cause problems with this will disable vectorization anyway. */
7888 else if (!SCALAR_FLOAT_TYPE_P (scalar_type
)
7889 && !INTEGRAL_TYPE_P (scalar_type
))
7890 scalar_type
= lang_hooks
.types
.type_for_mode (inner_mode
, 1);
7892 /* We can't build a vector type of elements with alignment bigger than
7894 else if (nbytes
< TYPE_ALIGN_UNIT (scalar_type
))
7895 scalar_type
= lang_hooks
.types
.type_for_mode (inner_mode
,
7896 TYPE_UNSIGNED (scalar_type
));
7898 /* If we felt back to using the mode fail if there was
7899 no scalar type for it. */
7900 if (scalar_type
== NULL_TREE
)
7903 /* If no size was supplied use the mode the target prefers. Otherwise
7904 lookup a vector mode of the specified size. */
7906 simd_mode
= targetm
.vectorize
.preferred_simd_mode (inner_mode
);
7908 simd_mode
= mode_for_vector (inner_mode
, size
/ nbytes
);
7909 nunits
= GET_MODE_SIZE (simd_mode
) / nbytes
;
7913 vectype
= build_vector_type (scalar_type
, nunits
);
7915 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
7916 && !INTEGRAL_MODE_P (TYPE_MODE (vectype
)))
7922 unsigned int current_vector_size
;
7924 /* Function get_vectype_for_scalar_type.
7926 Returns the vector type corresponding to SCALAR_TYPE as supported
7930 get_vectype_for_scalar_type (tree scalar_type
)
7933 vectype
= get_vectype_for_scalar_type_and_size (scalar_type
,
7934 current_vector_size
);
7936 && current_vector_size
== 0)
7937 current_vector_size
= GET_MODE_SIZE (TYPE_MODE (vectype
));
7941 /* Function get_same_sized_vectype
7943 Returns a vector type corresponding to SCALAR_TYPE of size
7944 VECTOR_TYPE if supported by the target. */
7947 get_same_sized_vectype (tree scalar_type
, tree vector_type
)
7949 return get_vectype_for_scalar_type_and_size
7950 (scalar_type
, GET_MODE_SIZE (TYPE_MODE (vector_type
)));
7953 /* Function vect_is_simple_use.
7956 LOOP_VINFO - the vect info of the loop that is being vectorized.
7957 BB_VINFO - the vect info of the basic block that is being vectorized.
7958 OPERAND - operand of STMT in the loop or bb.
7959 DEF - the defining stmt in case OPERAND is an SSA_NAME.
7961 Returns whether a stmt with OPERAND can be vectorized.
7962 For loops, supportable operands are constants, loop invariants, and operands
7963 that are defined by the current iteration of the loop. Unsupportable
7964 operands are those that are defined by a previous iteration of the loop (as
7965 is the case in reduction/induction computations).
7966 For basic blocks, supportable operands are constants and bb invariants.
7967 For now, operands defined outside the basic block are not supported. */
7970 vect_is_simple_use (tree operand
, gimple stmt
, loop_vec_info loop_vinfo
,
7971 bb_vec_info bb_vinfo
, gimple
*def_stmt
,
7972 tree
*def
, enum vect_def_type
*dt
)
7976 *dt
= vect_unknown_def_type
;
7978 if (dump_enabled_p ())
7980 dump_printf_loc (MSG_NOTE
, vect_location
,
7981 "vect_is_simple_use: operand ");
7982 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, operand
);
7983 dump_printf (MSG_NOTE
, "\n");
7986 if (CONSTANT_CLASS_P (operand
))
7988 *dt
= vect_constant_def
;
7992 if (is_gimple_min_invariant (operand
))
7995 *dt
= vect_external_def
;
7999 if (TREE_CODE (operand
) != SSA_NAME
)
8001 if (dump_enabled_p ())
8002 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8007 if (SSA_NAME_IS_DEFAULT_DEF (operand
))
8010 *dt
= vect_external_def
;
8014 *def_stmt
= SSA_NAME_DEF_STMT (operand
);
8015 if (dump_enabled_p ())
8017 dump_printf_loc (MSG_NOTE
, vect_location
, "def_stmt: ");
8018 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, *def_stmt
, 0);
8021 basic_block bb
= gimple_bb (*def_stmt
);
8022 if ((loop_vinfo
&& !flow_bb_inside_loop_p (LOOP_VINFO_LOOP (loop_vinfo
), bb
))
8024 && (bb
!= BB_VINFO_BB (bb_vinfo
)
8025 || gimple_code (*def_stmt
) == GIMPLE_PHI
)))
8026 *dt
= vect_external_def
;
8029 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (*def_stmt
);
8030 if (bb_vinfo
&& !STMT_VINFO_VECTORIZABLE (stmt_vinfo
))
8031 *dt
= vect_external_def
;
8033 *dt
= STMT_VINFO_DEF_TYPE (stmt_vinfo
);
8036 if (dump_enabled_p ())
8038 dump_printf_loc (MSG_NOTE
, vect_location
, "type of def: ");
8041 case vect_uninitialized_def
:
8042 dump_printf (MSG_NOTE
, "uninitialized\n");
8044 case vect_constant_def
:
8045 dump_printf (MSG_NOTE
, "constant\n");
8047 case vect_external_def
:
8048 dump_printf (MSG_NOTE
, "external\n");
8050 case vect_internal_def
:
8051 dump_printf (MSG_NOTE
, "internal\n");
8053 case vect_induction_def
:
8054 dump_printf (MSG_NOTE
, "induction\n");
8056 case vect_reduction_def
:
8057 dump_printf (MSG_NOTE
, "reduction\n");
8059 case vect_double_reduction_def
:
8060 dump_printf (MSG_NOTE
, "double reduction\n");
8062 case vect_nested_cycle
:
8063 dump_printf (MSG_NOTE
, "nested cycle\n");
8065 case vect_unknown_def_type
:
8066 dump_printf (MSG_NOTE
, "unknown\n");
8071 if (*dt
== vect_unknown_def_type
8073 && *dt
== vect_double_reduction_def
8074 && gimple_code (stmt
) != GIMPLE_PHI
))
8076 if (dump_enabled_p ())
8077 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8078 "Unsupported pattern.\n");
8082 switch (gimple_code (*def_stmt
))
8085 *def
= gimple_phi_result (*def_stmt
);
8089 *def
= gimple_assign_lhs (*def_stmt
);
8093 *def
= gimple_call_lhs (*def_stmt
);
8098 if (dump_enabled_p ())
8099 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8100 "unsupported defining stmt:\n");
8107 /* Function vect_is_simple_use_1.
8109 Same as vect_is_simple_use_1 but also determines the vector operand
8110 type of OPERAND and stores it to *VECTYPE. If the definition of
8111 OPERAND is vect_uninitialized_def, vect_constant_def or
8112 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
8113 is responsible to compute the best suited vector type for the
8117 vect_is_simple_use_1 (tree operand
, gimple stmt
, loop_vec_info loop_vinfo
,
8118 bb_vec_info bb_vinfo
, gimple
*def_stmt
,
8119 tree
*def
, enum vect_def_type
*dt
, tree
*vectype
)
8121 if (!vect_is_simple_use (operand
, stmt
, loop_vinfo
, bb_vinfo
, def_stmt
,
8125 /* Now get a vector type if the def is internal, otherwise supply
8126 NULL_TREE and leave it up to the caller to figure out a proper
8127 type for the use stmt. */
8128 if (*dt
== vect_internal_def
8129 || *dt
== vect_induction_def
8130 || *dt
== vect_reduction_def
8131 || *dt
== vect_double_reduction_def
8132 || *dt
== vect_nested_cycle
)
8134 stmt_vec_info stmt_info
= vinfo_for_stmt (*def_stmt
);
8136 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
8137 && !STMT_VINFO_RELEVANT (stmt_info
)
8138 && !STMT_VINFO_LIVE_P (stmt_info
))
8139 stmt_info
= vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info
));
8141 *vectype
= STMT_VINFO_VECTYPE (stmt_info
);
8142 gcc_assert (*vectype
!= NULL_TREE
);
8144 else if (*dt
== vect_uninitialized_def
8145 || *dt
== vect_constant_def
8146 || *dt
== vect_external_def
)
8147 *vectype
= NULL_TREE
;
8155 /* Function supportable_widening_operation
8157 Check whether an operation represented by the code CODE is a
8158 widening operation that is supported by the target platform in
8159 vector form (i.e., when operating on arguments of type VECTYPE_IN
8160 producing a result of type VECTYPE_OUT).
8162 Widening operations we currently support are NOP (CONVERT), FLOAT
8163 and WIDEN_MULT. This function checks if these operations are supported
8164 by the target platform either directly (via vector tree-codes), or via
8168 - CODE1 and CODE2 are codes of vector operations to be used when
8169 vectorizing the operation, if available.
8170 - MULTI_STEP_CVT determines the number of required intermediate steps in
8171 case of multi-step conversion (like char->short->int - in that case
8172 MULTI_STEP_CVT will be 1).
8173 - INTERM_TYPES contains the intermediate type required to perform the
8174 widening operation (short in the above example). */
8177 supportable_widening_operation (enum tree_code code
, gimple stmt
,
8178 tree vectype_out
, tree vectype_in
,
8179 enum tree_code
*code1
, enum tree_code
*code2
,
8180 int *multi_step_cvt
,
8181 vec
<tree
> *interm_types
)
8183 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
8184 loop_vec_info loop_info
= STMT_VINFO_LOOP_VINFO (stmt_info
);
8185 struct loop
*vect_loop
= NULL
;
8186 machine_mode vec_mode
;
8187 enum insn_code icode1
, icode2
;
8188 optab optab1
, optab2
;
8189 tree vectype
= vectype_in
;
8190 tree wide_vectype
= vectype_out
;
8191 enum tree_code c1
, c2
;
8193 tree prev_type
, intermediate_type
;
8194 machine_mode intermediate_mode
, prev_mode
;
8195 optab optab3
, optab4
;
8197 *multi_step_cvt
= 0;
8199 vect_loop
= LOOP_VINFO_LOOP (loop_info
);
8203 case WIDEN_MULT_EXPR
:
8204 /* The result of a vectorized widening operation usually requires
8205 two vectors (because the widened results do not fit into one vector).
8206 The generated vector results would normally be expected to be
8207 generated in the same order as in the original scalar computation,
8208 i.e. if 8 results are generated in each vector iteration, they are
8209 to be organized as follows:
8210 vect1: [res1,res2,res3,res4],
8211 vect2: [res5,res6,res7,res8].
8213 However, in the special case that the result of the widening
8214 operation is used in a reduction computation only, the order doesn't
8215 matter (because when vectorizing a reduction we change the order of
8216 the computation). Some targets can take advantage of this and
8217 generate more efficient code. For example, targets like Altivec,
8218 that support widen_mult using a sequence of {mult_even,mult_odd}
8219 generate the following vectors:
8220 vect1: [res1,res3,res5,res7],
8221 vect2: [res2,res4,res6,res8].
8223 When vectorizing outer-loops, we execute the inner-loop sequentially
8224 (each vectorized inner-loop iteration contributes to VF outer-loop
8225 iterations in parallel). We therefore don't allow to change the
8226 order of the computation in the inner-loop during outer-loop
8228 /* TODO: Another case in which order doesn't *really* matter is when we
8229 widen and then contract again, e.g. (short)((int)x * y >> 8).
8230 Normally, pack_trunc performs an even/odd permute, whereas the
8231 repack from an even/odd expansion would be an interleave, which
8232 would be significantly simpler for e.g. AVX2. */
8233 /* In any case, in order to avoid duplicating the code below, recurse
8234 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
8235 are properly set up for the caller. If we fail, we'll continue with
8236 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
8238 && STMT_VINFO_RELEVANT (stmt_info
) == vect_used_by_reduction
8239 && !nested_in_vect_loop_p (vect_loop
, stmt
)
8240 && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR
,
8241 stmt
, vectype_out
, vectype_in
,
8242 code1
, code2
, multi_step_cvt
,
8245 /* Elements in a vector with vect_used_by_reduction property cannot
8246 be reordered if the use chain with this property does not have the
8247 same operation. One such an example is s += a * b, where elements
8248 in a and b cannot be reordered. Here we check if the vector defined
8249 by STMT is only directly used in the reduction statement. */
8250 tree lhs
= gimple_assign_lhs (stmt
);
8251 use_operand_p dummy
;
8253 stmt_vec_info use_stmt_info
= NULL
;
8254 if (single_imm_use (lhs
, &dummy
, &use_stmt
)
8255 && (use_stmt_info
= vinfo_for_stmt (use_stmt
))
8256 && STMT_VINFO_DEF_TYPE (use_stmt_info
) == vect_reduction_def
)
8259 c1
= VEC_WIDEN_MULT_LO_EXPR
;
8260 c2
= VEC_WIDEN_MULT_HI_EXPR
;
8263 case VEC_WIDEN_MULT_EVEN_EXPR
:
8264 /* Support the recursion induced just above. */
8265 c1
= VEC_WIDEN_MULT_EVEN_EXPR
;
8266 c2
= VEC_WIDEN_MULT_ODD_EXPR
;
8269 case WIDEN_LSHIFT_EXPR
:
8270 c1
= VEC_WIDEN_LSHIFT_LO_EXPR
;
8271 c2
= VEC_WIDEN_LSHIFT_HI_EXPR
;
8275 c1
= VEC_UNPACK_LO_EXPR
;
8276 c2
= VEC_UNPACK_HI_EXPR
;
8280 c1
= VEC_UNPACK_FLOAT_LO_EXPR
;
8281 c2
= VEC_UNPACK_FLOAT_HI_EXPR
;
8284 case FIX_TRUNC_EXPR
:
8285 /* ??? Not yet implemented due to missing VEC_UNPACK_FIX_TRUNC_HI_EXPR/
8286 VEC_UNPACK_FIX_TRUNC_LO_EXPR tree codes and optabs used for
8287 computing the operation. */
8294 if (BYTES_BIG_ENDIAN
&& c1
!= VEC_WIDEN_MULT_EVEN_EXPR
)
8297 if (code
== FIX_TRUNC_EXPR
)
8299 /* The signedness is determined from output operand. */
8300 optab1
= optab_for_tree_code (c1
, vectype_out
, optab_default
);
8301 optab2
= optab_for_tree_code (c2
, vectype_out
, optab_default
);
8305 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
8306 optab2
= optab_for_tree_code (c2
, vectype
, optab_default
);
8309 if (!optab1
|| !optab2
)
8312 vec_mode
= TYPE_MODE (vectype
);
8313 if ((icode1
= optab_handler (optab1
, vec_mode
)) == CODE_FOR_nothing
8314 || (icode2
= optab_handler (optab2
, vec_mode
)) == CODE_FOR_nothing
)
8320 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
8321 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
8324 /* Check if it's a multi-step conversion that can be done using intermediate
8327 prev_type
= vectype
;
8328 prev_mode
= vec_mode
;
8330 if (!CONVERT_EXPR_CODE_P (code
))
8333 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
8334 intermediate steps in promotion sequence. We try
8335 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
8337 interm_types
->create (MAX_INTERM_CVT_STEPS
);
8338 for (i
= 0; i
< MAX_INTERM_CVT_STEPS
; i
++)
8340 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
8342 = lang_hooks
.types
.type_for_mode (intermediate_mode
,
8343 TYPE_UNSIGNED (prev_type
));
8344 optab3
= optab_for_tree_code (c1
, intermediate_type
, optab_default
);
8345 optab4
= optab_for_tree_code (c2
, intermediate_type
, optab_default
);
8347 if (!optab3
|| !optab4
8348 || (icode1
= optab_handler (optab1
, prev_mode
)) == CODE_FOR_nothing
8349 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
8350 || (icode2
= optab_handler (optab2
, prev_mode
)) == CODE_FOR_nothing
8351 || insn_data
[icode2
].operand
[0].mode
!= intermediate_mode
8352 || ((icode1
= optab_handler (optab3
, intermediate_mode
))
8353 == CODE_FOR_nothing
)
8354 || ((icode2
= optab_handler (optab4
, intermediate_mode
))
8355 == CODE_FOR_nothing
))
8358 interm_types
->quick_push (intermediate_type
);
8359 (*multi_step_cvt
)++;
8361 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
8362 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
8365 prev_type
= intermediate_type
;
8366 prev_mode
= intermediate_mode
;
8369 interm_types
->release ();
8374 /* Function supportable_narrowing_operation
8376 Check whether an operation represented by the code CODE is a
8377 narrowing operation that is supported by the target platform in
8378 vector form (i.e., when operating on arguments of type VECTYPE_IN
8379 and producing a result of type VECTYPE_OUT).
8381 Narrowing operations we currently support are NOP (CONVERT) and
8382 FIX_TRUNC. This function checks if these operations are supported by
8383 the target platform directly via vector tree-codes.
8386 - CODE1 is the code of a vector operation to be used when
8387 vectorizing the operation, if available.
8388 - MULTI_STEP_CVT determines the number of required intermediate steps in
8389 case of multi-step conversion (like int->short->char - in that case
8390 MULTI_STEP_CVT will be 1).
8391 - INTERM_TYPES contains the intermediate type required to perform the
8392 narrowing operation (short in the above example). */
8395 supportable_narrowing_operation (enum tree_code code
,
8396 tree vectype_out
, tree vectype_in
,
8397 enum tree_code
*code1
, int *multi_step_cvt
,
8398 vec
<tree
> *interm_types
)
8400 machine_mode vec_mode
;
8401 enum insn_code icode1
;
8402 optab optab1
, interm_optab
;
8403 tree vectype
= vectype_in
;
8404 tree narrow_vectype
= vectype_out
;
8406 tree intermediate_type
;
8407 machine_mode intermediate_mode
, prev_mode
;
8411 *multi_step_cvt
= 0;
8415 c1
= VEC_PACK_TRUNC_EXPR
;
8418 case FIX_TRUNC_EXPR
:
8419 c1
= VEC_PACK_FIX_TRUNC_EXPR
;
8423 /* ??? Not yet implemented due to missing VEC_PACK_FLOAT_EXPR
8424 tree code and optabs used for computing the operation. */
8431 if (code
== FIX_TRUNC_EXPR
)
8432 /* The signedness is determined from output operand. */
8433 optab1
= optab_for_tree_code (c1
, vectype_out
, optab_default
);
8435 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
8440 vec_mode
= TYPE_MODE (vectype
);
8441 if ((icode1
= optab_handler (optab1
, vec_mode
)) == CODE_FOR_nothing
)
8446 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
8449 /* Check if it's a multi-step conversion that can be done using intermediate
8451 prev_mode
= vec_mode
;
8452 if (code
== FIX_TRUNC_EXPR
)
8453 uns
= TYPE_UNSIGNED (vectype_out
);
8455 uns
= TYPE_UNSIGNED (vectype
);
8457 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
8458 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
8459 costly than signed. */
8460 if (code
== FIX_TRUNC_EXPR
&& uns
)
8462 enum insn_code icode2
;
8465 = lang_hooks
.types
.type_for_mode (TYPE_MODE (vectype_out
), 0);
8467 = optab_for_tree_code (c1
, intermediate_type
, optab_default
);
8468 if (interm_optab
!= unknown_optab
8469 && (icode2
= optab_handler (optab1
, vec_mode
)) != CODE_FOR_nothing
8470 && insn_data
[icode1
].operand
[0].mode
8471 == insn_data
[icode2
].operand
[0].mode
)
8474 optab1
= interm_optab
;
8479 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
8480 intermediate steps in promotion sequence. We try
8481 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
8482 interm_types
->create (MAX_INTERM_CVT_STEPS
);
8483 for (i
= 0; i
< MAX_INTERM_CVT_STEPS
; i
++)
8485 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
8487 = lang_hooks
.types
.type_for_mode (intermediate_mode
, uns
);
8489 = optab_for_tree_code (VEC_PACK_TRUNC_EXPR
, intermediate_type
,
8492 || ((icode1
= optab_handler (optab1
, prev_mode
)) == CODE_FOR_nothing
)
8493 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
8494 || ((icode1
= optab_handler (interm_optab
, intermediate_mode
))
8495 == CODE_FOR_nothing
))
8498 interm_types
->quick_push (intermediate_type
);
8499 (*multi_step_cvt
)++;
8501 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
8504 prev_mode
= intermediate_mode
;
8505 optab1
= interm_optab
;
8508 interm_types
->release ();