1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003-2015 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4 and Ira Rosen <irar@il.ibm.com>
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
31 #include "optabs-tree.h"
32 #include "insn-config.h"
33 #include "recog.h" /* FIXME: for insn_data */
37 #include "fold-const.h"
38 #include "stor-layout.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
44 #include "tree-ssa-loop-manip.h"
46 #include "tree-ssa-loop.h"
47 #include "tree-scalar-evolution.h"
48 #include "tree-vectorizer.h"
51 /* For lang_hooks.types.type_for_mode. */
52 #include "langhooks.h"
54 /* Return the vectorized type for the given statement. */
57 stmt_vectype (struct _stmt_vec_info
*stmt_info
)
59 return STMT_VINFO_VECTYPE (stmt_info
);
62 /* Return TRUE iff the given statement is in an inner loop relative to
63 the loop being vectorized. */
65 stmt_in_inner_loop_p (struct _stmt_vec_info
*stmt_info
)
67 gimple
*stmt
= STMT_VINFO_STMT (stmt_info
);
68 basic_block bb
= gimple_bb (stmt
);
69 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
75 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
77 return (bb
->loop_father
== loop
->inner
);
80 /* Record the cost of a statement, either by directly informing the
81 target model or by saving it in a vector for later processing.
82 Return a preliminary estimate of the statement's cost. */
85 record_stmt_cost (stmt_vector_for_cost
*body_cost_vec
, int count
,
86 enum vect_cost_for_stmt kind
, stmt_vec_info stmt_info
,
87 int misalign
, enum vect_cost_model_location where
)
91 tree vectype
= stmt_info
? stmt_vectype (stmt_info
) : NULL_TREE
;
92 stmt_info_for_cost si
= { count
, kind
,
93 stmt_info
? STMT_VINFO_STMT (stmt_info
) : NULL
,
95 body_cost_vec
->safe_push (si
);
97 (builtin_vectorization_cost (kind
, vectype
, misalign
) * count
);
100 return add_stmt_cost (stmt_info
->vinfo
->target_cost_data
,
101 count
, kind
, stmt_info
, misalign
, where
);
104 /* Return a variable of type ELEM_TYPE[NELEMS]. */
107 create_vector_array (tree elem_type
, unsigned HOST_WIDE_INT nelems
)
109 return create_tmp_var (build_array_type_nelts (elem_type
, nelems
),
113 /* ARRAY is an array of vectors created by create_vector_array.
114 Return an SSA_NAME for the vector in index N. The reference
115 is part of the vectorization of STMT and the vector is associated
116 with scalar destination SCALAR_DEST. */
119 read_vector_array (gimple
*stmt
, gimple_stmt_iterator
*gsi
, tree scalar_dest
,
120 tree array
, unsigned HOST_WIDE_INT n
)
122 tree vect_type
, vect
, vect_name
, array_ref
;
125 gcc_assert (TREE_CODE (TREE_TYPE (array
)) == ARRAY_TYPE
);
126 vect_type
= TREE_TYPE (TREE_TYPE (array
));
127 vect
= vect_create_destination_var (scalar_dest
, vect_type
);
128 array_ref
= build4 (ARRAY_REF
, vect_type
, array
,
129 build_int_cst (size_type_node
, n
),
130 NULL_TREE
, NULL_TREE
);
132 new_stmt
= gimple_build_assign (vect
, array_ref
);
133 vect_name
= make_ssa_name (vect
, new_stmt
);
134 gimple_assign_set_lhs (new_stmt
, vect_name
);
135 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
140 /* ARRAY is an array of vectors created by create_vector_array.
141 Emit code to store SSA_NAME VECT in index N of the array.
142 The store is part of the vectorization of STMT. */
145 write_vector_array (gimple
*stmt
, gimple_stmt_iterator
*gsi
, tree vect
,
146 tree array
, unsigned HOST_WIDE_INT n
)
151 array_ref
= build4 (ARRAY_REF
, TREE_TYPE (vect
), array
,
152 build_int_cst (size_type_node
, n
),
153 NULL_TREE
, NULL_TREE
);
155 new_stmt
= gimple_build_assign (array_ref
, vect
);
156 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
159 /* PTR is a pointer to an array of type TYPE. Return a representation
160 of *PTR. The memory reference replaces those in FIRST_DR
164 create_array_ref (tree type
, tree ptr
, struct data_reference
*first_dr
)
166 tree mem_ref
, alias_ptr_type
;
168 alias_ptr_type
= reference_alias_ptr_type (DR_REF (first_dr
));
169 mem_ref
= build2 (MEM_REF
, type
, ptr
, build_int_cst (alias_ptr_type
, 0));
170 /* Arrays have the same alignment as their type. */
171 set_ptr_info_alignment (get_ptr_info (ptr
), TYPE_ALIGN_UNIT (type
), 0);
175 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
177 /* Function vect_mark_relevant.
179 Mark STMT as "relevant for vectorization" and add it to WORKLIST. */
182 vect_mark_relevant (vec
<gimple
*> *worklist
, gimple
*stmt
,
183 enum vect_relevant relevant
, bool live_p
,
184 bool used_in_pattern
)
186 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
187 enum vect_relevant save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
188 bool save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
189 gimple
*pattern_stmt
;
191 if (dump_enabled_p ())
192 dump_printf_loc (MSG_NOTE
, vect_location
,
193 "mark relevant %d, live %d.\n", relevant
, live_p
);
195 /* If this stmt is an original stmt in a pattern, we might need to mark its
196 related pattern stmt instead of the original stmt. However, such stmts
197 may have their own uses that are not in any pattern, in such cases the
198 stmt itself should be marked. */
199 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
202 if (!used_in_pattern
)
204 imm_use_iterator imm_iter
;
208 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
209 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
211 if (is_gimple_assign (stmt
))
212 lhs
= gimple_assign_lhs (stmt
);
214 lhs
= gimple_call_lhs (stmt
);
216 /* This use is out of pattern use, if LHS has other uses that are
217 pattern uses, we should mark the stmt itself, and not the pattern
219 if (lhs
&& TREE_CODE (lhs
) == SSA_NAME
)
220 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, lhs
)
222 if (is_gimple_debug (USE_STMT (use_p
)))
224 use_stmt
= USE_STMT (use_p
);
226 if (!flow_bb_inside_loop_p (loop
, gimple_bb (use_stmt
)))
229 if (vinfo_for_stmt (use_stmt
)
230 && STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (use_stmt
)))
240 /* This is the last stmt in a sequence that was detected as a
241 pattern that can potentially be vectorized. Don't mark the stmt
242 as relevant/live because it's not going to be vectorized.
243 Instead mark the pattern-stmt that replaces it. */
245 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
247 if (dump_enabled_p ())
248 dump_printf_loc (MSG_NOTE
, vect_location
,
249 "last stmt in pattern. don't mark"
250 " relevant/live.\n");
251 stmt_info
= vinfo_for_stmt (pattern_stmt
);
252 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info
) == stmt
);
253 save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
254 save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
259 STMT_VINFO_LIVE_P (stmt_info
) |= live_p
;
260 if (relevant
> STMT_VINFO_RELEVANT (stmt_info
))
261 STMT_VINFO_RELEVANT (stmt_info
) = relevant
;
263 if (STMT_VINFO_RELEVANT (stmt_info
) == save_relevant
264 && STMT_VINFO_LIVE_P (stmt_info
) == save_live_p
)
266 if (dump_enabled_p ())
267 dump_printf_loc (MSG_NOTE
, vect_location
,
268 "already marked relevant/live.\n");
272 worklist
->safe_push (stmt
);
276 /* Function vect_stmt_relevant_p.
278 Return true if STMT in loop that is represented by LOOP_VINFO is
279 "relevant for vectorization".
281 A stmt is considered "relevant for vectorization" if:
282 - it has uses outside the loop.
283 - it has vdefs (it alters memory).
284 - control stmts in the loop (except for the exit condition).
286 CHECKME: what other side effects would the vectorizer allow? */
289 vect_stmt_relevant_p (gimple
*stmt
, loop_vec_info loop_vinfo
,
290 enum vect_relevant
*relevant
, bool *live_p
)
292 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
294 imm_use_iterator imm_iter
;
298 *relevant
= vect_unused_in_scope
;
301 /* cond stmt other than loop exit cond. */
302 if (is_ctrl_stmt (stmt
)
303 && STMT_VINFO_TYPE (vinfo_for_stmt (stmt
))
304 != loop_exit_ctrl_vec_info_type
)
305 *relevant
= vect_used_in_scope
;
307 /* changing memory. */
308 if (gimple_code (stmt
) != GIMPLE_PHI
)
309 if (gimple_vdef (stmt
)
310 && !gimple_clobber_p (stmt
))
312 if (dump_enabled_p ())
313 dump_printf_loc (MSG_NOTE
, vect_location
,
314 "vec_stmt_relevant_p: stmt has vdefs.\n");
315 *relevant
= vect_used_in_scope
;
318 /* uses outside the loop. */
319 FOR_EACH_PHI_OR_STMT_DEF (def_p
, stmt
, op_iter
, SSA_OP_DEF
)
321 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, DEF_FROM_PTR (def_p
))
323 basic_block bb
= gimple_bb (USE_STMT (use_p
));
324 if (!flow_bb_inside_loop_p (loop
, bb
))
326 if (dump_enabled_p ())
327 dump_printf_loc (MSG_NOTE
, vect_location
,
328 "vec_stmt_relevant_p: used out of loop.\n");
330 if (is_gimple_debug (USE_STMT (use_p
)))
333 /* We expect all such uses to be in the loop exit phis
334 (because of loop closed form) */
335 gcc_assert (gimple_code (USE_STMT (use_p
)) == GIMPLE_PHI
);
336 gcc_assert (bb
== single_exit (loop
)->dest
);
343 return (*live_p
|| *relevant
);
347 /* Function exist_non_indexing_operands_for_use_p
349 USE is one of the uses attached to STMT. Check if USE is
350 used in STMT for anything other than indexing an array. */
353 exist_non_indexing_operands_for_use_p (tree use
, gimple
*stmt
)
356 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
358 /* USE corresponds to some operand in STMT. If there is no data
359 reference in STMT, then any operand that corresponds to USE
360 is not indexing an array. */
361 if (!STMT_VINFO_DATA_REF (stmt_info
))
364 /* STMT has a data_ref. FORNOW this means that its of one of
368 (This should have been verified in analyze_data_refs).
370 'var' in the second case corresponds to a def, not a use,
371 so USE cannot correspond to any operands that are not used
374 Therefore, all we need to check is if STMT falls into the
375 first case, and whether var corresponds to USE. */
377 if (!gimple_assign_copy_p (stmt
))
379 if (is_gimple_call (stmt
)
380 && gimple_call_internal_p (stmt
))
381 switch (gimple_call_internal_fn (stmt
))
384 operand
= gimple_call_arg (stmt
, 3);
389 operand
= gimple_call_arg (stmt
, 2);
399 if (TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
401 operand
= gimple_assign_rhs1 (stmt
);
402 if (TREE_CODE (operand
) != SSA_NAME
)
413 Function process_use.
416 - a USE in STMT in a loop represented by LOOP_VINFO
417 - LIVE_P, RELEVANT - enum values to be set in the STMT_VINFO of the stmt
418 that defined USE. This is done by calling mark_relevant and passing it
419 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
420 - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't
424 Generally, LIVE_P and RELEVANT are used to define the liveness and
425 relevance info of the DEF_STMT of this USE:
426 STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
427 STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
429 - case 1: If USE is used only for address computations (e.g. array indexing),
430 which does not need to be directly vectorized, then the liveness/relevance
431 of the respective DEF_STMT is left unchanged.
432 - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we
433 skip DEF_STMT cause it had already been processed.
434 - case 3: If DEF_STMT and STMT are in different nests, then "relevant" will
435 be modified accordingly.
437 Return true if everything is as expected. Return false otherwise. */
440 process_use (gimple
*stmt
, tree use
, loop_vec_info loop_vinfo
, bool live_p
,
441 enum vect_relevant relevant
, vec
<gimple
*> *worklist
,
444 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
445 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
446 stmt_vec_info dstmt_vinfo
;
447 basic_block bb
, def_bb
;
449 enum vect_def_type dt
;
451 /* case 1: we are only interested in uses that need to be vectorized. Uses
452 that are used for address computation are not considered relevant. */
453 if (!force
&& !exist_non_indexing_operands_for_use_p (use
, stmt
))
456 if (!vect_is_simple_use (use
, loop_vinfo
, &def_stmt
, &dt
))
458 if (dump_enabled_p ())
459 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
460 "not vectorized: unsupported use in stmt.\n");
464 if (!def_stmt
|| gimple_nop_p (def_stmt
))
467 def_bb
= gimple_bb (def_stmt
);
468 if (!flow_bb_inside_loop_p (loop
, def_bb
))
470 if (dump_enabled_p ())
471 dump_printf_loc (MSG_NOTE
, vect_location
, "def_stmt is out of loop.\n");
475 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DEF_STMT).
476 DEF_STMT must have already been processed, because this should be the
477 only way that STMT, which is a reduction-phi, was put in the worklist,
478 as there should be no other uses for DEF_STMT in the loop. So we just
479 check that everything is as expected, and we are done. */
480 dstmt_vinfo
= vinfo_for_stmt (def_stmt
);
481 bb
= gimple_bb (stmt
);
482 if (gimple_code (stmt
) == GIMPLE_PHI
483 && STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
484 && gimple_code (def_stmt
) != GIMPLE_PHI
485 && STMT_VINFO_DEF_TYPE (dstmt_vinfo
) == vect_reduction_def
486 && bb
->loop_father
== def_bb
->loop_father
)
488 if (dump_enabled_p ())
489 dump_printf_loc (MSG_NOTE
, vect_location
,
490 "reduc-stmt defining reduc-phi in the same nest.\n");
491 if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo
))
492 dstmt_vinfo
= vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo
));
493 gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo
) < vect_used_by_reduction
);
494 gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo
)
495 || STMT_VINFO_RELEVANT (dstmt_vinfo
) > vect_unused_in_scope
);
499 /* case 3a: outer-loop stmt defining an inner-loop stmt:
500 outer-loop-header-bb:
506 if (flow_loop_nested_p (def_bb
->loop_father
, bb
->loop_father
))
508 if (dump_enabled_p ())
509 dump_printf_loc (MSG_NOTE
, vect_location
,
510 "outer-loop def-stmt defining inner-loop stmt.\n");
514 case vect_unused_in_scope
:
515 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_nested_cycle
) ?
516 vect_used_in_scope
: vect_unused_in_scope
;
519 case vect_used_in_outer_by_reduction
:
520 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
521 relevant
= vect_used_by_reduction
;
524 case vect_used_in_outer
:
525 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
526 relevant
= vect_used_in_scope
;
529 case vect_used_in_scope
:
537 /* case 3b: inner-loop stmt defining an outer-loop stmt:
538 outer-loop-header-bb:
542 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
544 else if (flow_loop_nested_p (bb
->loop_father
, def_bb
->loop_father
))
546 if (dump_enabled_p ())
547 dump_printf_loc (MSG_NOTE
, vect_location
,
548 "inner-loop def-stmt defining outer-loop stmt.\n");
552 case vect_unused_in_scope
:
553 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
554 || STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_double_reduction_def
) ?
555 vect_used_in_outer_by_reduction
: vect_unused_in_scope
;
558 case vect_used_by_reduction
:
559 relevant
= vect_used_in_outer_by_reduction
;
562 case vect_used_in_scope
:
563 relevant
= vect_used_in_outer
;
571 vect_mark_relevant (worklist
, def_stmt
, relevant
, live_p
,
572 is_pattern_stmt_p (stmt_vinfo
));
577 /* Function vect_mark_stmts_to_be_vectorized.
579 Not all stmts in the loop need to be vectorized. For example:
588 Stmt 1 and 3 do not need to be vectorized, because loop control and
589 addressing of vectorized data-refs are handled differently.
591 This pass detects such stmts. */
594 vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo
)
596 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
597 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
598 unsigned int nbbs
= loop
->num_nodes
;
599 gimple_stmt_iterator si
;
602 stmt_vec_info stmt_vinfo
;
606 enum vect_relevant relevant
, tmp_relevant
;
607 enum vect_def_type def_type
;
609 if (dump_enabled_p ())
610 dump_printf_loc (MSG_NOTE
, vect_location
,
611 "=== vect_mark_stmts_to_be_vectorized ===\n");
613 auto_vec
<gimple
*, 64> worklist
;
615 /* 1. Init worklist. */
616 for (i
= 0; i
< nbbs
; i
++)
619 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
622 if (dump_enabled_p ())
624 dump_printf_loc (MSG_NOTE
, vect_location
, "init: phi relevant? ");
625 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, phi
, 0);
628 if (vect_stmt_relevant_p (phi
, loop_vinfo
, &relevant
, &live_p
))
629 vect_mark_relevant (&worklist
, phi
, relevant
, live_p
, false);
631 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
633 stmt
= gsi_stmt (si
);
634 if (dump_enabled_p ())
636 dump_printf_loc (MSG_NOTE
, vect_location
, "init: stmt relevant? ");
637 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
640 if (vect_stmt_relevant_p (stmt
, loop_vinfo
, &relevant
, &live_p
))
641 vect_mark_relevant (&worklist
, stmt
, relevant
, live_p
, false);
645 /* 2. Process_worklist */
646 while (worklist
.length () > 0)
651 stmt
= worklist
.pop ();
652 if (dump_enabled_p ())
654 dump_printf_loc (MSG_NOTE
, vect_location
, "worklist: examine stmt: ");
655 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
658 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
659 (DEF_STMT) as relevant/irrelevant and live/dead according to the
660 liveness and relevance properties of STMT. */
661 stmt_vinfo
= vinfo_for_stmt (stmt
);
662 relevant
= STMT_VINFO_RELEVANT (stmt_vinfo
);
663 live_p
= STMT_VINFO_LIVE_P (stmt_vinfo
);
665 /* Generally, the liveness and relevance properties of STMT are
666 propagated as is to the DEF_STMTs of its USEs:
667 live_p <-- STMT_VINFO_LIVE_P (STMT_VINFO)
668 relevant <-- STMT_VINFO_RELEVANT (STMT_VINFO)
670 One exception is when STMT has been identified as defining a reduction
671 variable; in this case we set the liveness/relevance as follows:
673 relevant = vect_used_by_reduction
674 This is because we distinguish between two kinds of relevant stmts -
675 those that are used by a reduction computation, and those that are
676 (also) used by a regular computation. This allows us later on to
677 identify stmts that are used solely by a reduction, and therefore the
678 order of the results that they produce does not have to be kept. */
680 def_type
= STMT_VINFO_DEF_TYPE (stmt_vinfo
);
681 tmp_relevant
= relevant
;
684 case vect_reduction_def
:
685 switch (tmp_relevant
)
687 case vect_unused_in_scope
:
688 relevant
= vect_used_by_reduction
;
691 case vect_used_by_reduction
:
692 if (gimple_code (stmt
) == GIMPLE_PHI
)
697 if (dump_enabled_p ())
698 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
699 "unsupported use of reduction.\n");
706 case vect_nested_cycle
:
707 if (tmp_relevant
!= vect_unused_in_scope
708 && tmp_relevant
!= vect_used_in_outer_by_reduction
709 && tmp_relevant
!= vect_used_in_outer
)
711 if (dump_enabled_p ())
712 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
713 "unsupported use of nested cycle.\n");
721 case vect_double_reduction_def
:
722 if (tmp_relevant
!= vect_unused_in_scope
723 && tmp_relevant
!= vect_used_by_reduction
)
725 if (dump_enabled_p ())
726 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
727 "unsupported use of double reduction.\n");
739 if (is_pattern_stmt_p (stmt_vinfo
))
741 /* Pattern statements are not inserted into the code, so
742 FOR_EACH_PHI_OR_STMT_USE optimizes their operands out, and we
743 have to scan the RHS or function arguments instead. */
744 if (is_gimple_assign (stmt
))
746 enum tree_code rhs_code
= gimple_assign_rhs_code (stmt
);
747 tree op
= gimple_assign_rhs1 (stmt
);
750 if (rhs_code
== COND_EXPR
&& COMPARISON_CLASS_P (op
))
752 if (!process_use (stmt
, TREE_OPERAND (op
, 0), loop_vinfo
,
753 live_p
, relevant
, &worklist
, false)
754 || !process_use (stmt
, TREE_OPERAND (op
, 1), loop_vinfo
,
755 live_p
, relevant
, &worklist
, false))
759 for (; i
< gimple_num_ops (stmt
); i
++)
761 op
= gimple_op (stmt
, i
);
762 if (TREE_CODE (op
) == SSA_NAME
763 && !process_use (stmt
, op
, loop_vinfo
, live_p
, relevant
,
768 else if (is_gimple_call (stmt
))
770 for (i
= 0; i
< gimple_call_num_args (stmt
); i
++)
772 tree arg
= gimple_call_arg (stmt
, i
);
773 if (!process_use (stmt
, arg
, loop_vinfo
, live_p
, relevant
,
780 FOR_EACH_PHI_OR_STMT_USE (use_p
, stmt
, iter
, SSA_OP_USE
)
782 tree op
= USE_FROM_PTR (use_p
);
783 if (!process_use (stmt
, op
, loop_vinfo
, live_p
, relevant
,
788 if (STMT_VINFO_GATHER_SCATTER_P (stmt_vinfo
))
791 tree decl
= vect_check_gather_scatter (stmt
, loop_vinfo
, NULL
, &off
, NULL
);
793 if (!process_use (stmt
, off
, loop_vinfo
, live_p
, relevant
,
797 } /* while worklist */
803 /* Function vect_model_simple_cost.
805 Models cost for simple operations, i.e. those that only emit ncopies of a
806 single op. Right now, this does not account for multiple insns that could
807 be generated for the single vector op. We will handle that shortly. */
810 vect_model_simple_cost (stmt_vec_info stmt_info
, int ncopies
,
811 enum vect_def_type
*dt
,
812 stmt_vector_for_cost
*prologue_cost_vec
,
813 stmt_vector_for_cost
*body_cost_vec
)
816 int inside_cost
= 0, prologue_cost
= 0;
818 /* The SLP costs were already calculated during SLP tree build. */
819 if (PURE_SLP_STMT (stmt_info
))
822 /* FORNOW: Assuming maximum 2 args per stmts. */
823 for (i
= 0; i
< 2; i
++)
824 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
825 prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1, vector_stmt
,
826 stmt_info
, 0, vect_prologue
);
828 /* Pass the inside-of-loop statements to the target-specific cost model. */
829 inside_cost
= record_stmt_cost (body_cost_vec
, ncopies
, vector_stmt
,
830 stmt_info
, 0, vect_body
);
832 if (dump_enabled_p ())
833 dump_printf_loc (MSG_NOTE
, vect_location
,
834 "vect_model_simple_cost: inside_cost = %d, "
835 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
839 /* Model cost for type demotion and promotion operations. PWR is normally
840 zero for single-step promotions and demotions. It will be one if
841 two-step promotion/demotion is required, and so on. Each additional
842 step doubles the number of instructions required. */
845 vect_model_promotion_demotion_cost (stmt_vec_info stmt_info
,
846 enum vect_def_type
*dt
, int pwr
)
849 int inside_cost
= 0, prologue_cost
= 0;
850 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
851 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
852 void *target_cost_data
;
854 /* The SLP costs were already calculated during SLP tree build. */
855 if (PURE_SLP_STMT (stmt_info
))
859 target_cost_data
= LOOP_VINFO_TARGET_COST_DATA (loop_vinfo
);
861 target_cost_data
= BB_VINFO_TARGET_COST_DATA (bb_vinfo
);
863 for (i
= 0; i
< pwr
+ 1; i
++)
865 tmp
= (STMT_VINFO_TYPE (stmt_info
) == type_promotion_vec_info_type
) ?
867 inside_cost
+= add_stmt_cost (target_cost_data
, vect_pow2 (tmp
),
868 vec_promote_demote
, stmt_info
, 0,
872 /* FORNOW: Assuming maximum 2 args per stmts. */
873 for (i
= 0; i
< 2; i
++)
874 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
875 prologue_cost
+= add_stmt_cost (target_cost_data
, 1, vector_stmt
,
876 stmt_info
, 0, vect_prologue
);
878 if (dump_enabled_p ())
879 dump_printf_loc (MSG_NOTE
, vect_location
,
880 "vect_model_promotion_demotion_cost: inside_cost = %d, "
881 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
884 /* Function vect_cost_group_size
886 For grouped load or store, return the group_size only if it is the first
887 load or store of a group, else return 1. This ensures that group size is
888 only returned once per group. */
891 vect_cost_group_size (stmt_vec_info stmt_info
)
893 gimple
*first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
895 if (first_stmt
== STMT_VINFO_STMT (stmt_info
))
896 return GROUP_SIZE (stmt_info
);
902 /* Function vect_model_store_cost
904 Models cost for stores. In the case of grouped accesses, one access
905 has the overhead of the grouped access attributed to it. */
908 vect_model_store_cost (stmt_vec_info stmt_info
, int ncopies
,
909 bool store_lanes_p
, enum vect_def_type dt
,
911 stmt_vector_for_cost
*prologue_cost_vec
,
912 stmt_vector_for_cost
*body_cost_vec
)
915 unsigned int inside_cost
= 0, prologue_cost
= 0;
916 struct data_reference
*first_dr
;
919 if (dt
== vect_constant_def
|| dt
== vect_external_def
)
920 prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1, scalar_to_vec
,
921 stmt_info
, 0, vect_prologue
);
923 /* Grouped access? */
924 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
928 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
933 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
934 group_size
= vect_cost_group_size (stmt_info
);
937 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
939 /* Not a grouped access. */
943 first_dr
= STMT_VINFO_DATA_REF (stmt_info
);
946 /* We assume that the cost of a single store-lanes instruction is
947 equivalent to the cost of GROUP_SIZE separate stores. If a grouped
948 access is instead being provided by a permute-and-store operation,
949 include the cost of the permutes. */
950 if (!store_lanes_p
&& group_size
> 1
951 && !STMT_VINFO_STRIDED_P (stmt_info
))
953 /* Uses a high and low interleave or shuffle operations for each
955 int nstmts
= ncopies
* ceil_log2 (group_size
) * group_size
;
956 inside_cost
= record_stmt_cost (body_cost_vec
, nstmts
, vec_perm
,
957 stmt_info
, 0, vect_body
);
959 if (dump_enabled_p ())
960 dump_printf_loc (MSG_NOTE
, vect_location
,
961 "vect_model_store_cost: strided group_size = %d .\n",
965 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
966 /* Costs of the stores. */
967 if (STMT_VINFO_STRIDED_P (stmt_info
)
968 && !STMT_VINFO_GROUPED_ACCESS (stmt_info
))
970 /* N scalar stores plus extracting the elements. */
971 inside_cost
+= record_stmt_cost (body_cost_vec
,
972 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
973 scalar_store
, stmt_info
, 0, vect_body
);
976 vect_get_store_cost (first_dr
, ncopies
, &inside_cost
, body_cost_vec
);
978 if (STMT_VINFO_STRIDED_P (stmt_info
))
979 inside_cost
+= record_stmt_cost (body_cost_vec
,
980 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
981 vec_to_scalar
, stmt_info
, 0, vect_body
);
983 if (dump_enabled_p ())
984 dump_printf_loc (MSG_NOTE
, vect_location
,
985 "vect_model_store_cost: inside_cost = %d, "
986 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
990 /* Calculate cost of DR's memory access. */
992 vect_get_store_cost (struct data_reference
*dr
, int ncopies
,
993 unsigned int *inside_cost
,
994 stmt_vector_for_cost
*body_cost_vec
)
996 int alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
997 gimple
*stmt
= DR_STMT (dr
);
998 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1000 switch (alignment_support_scheme
)
1004 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1005 vector_store
, stmt_info
, 0,
1008 if (dump_enabled_p ())
1009 dump_printf_loc (MSG_NOTE
, vect_location
,
1010 "vect_model_store_cost: aligned.\n");
1014 case dr_unaligned_supported
:
1016 /* Here, we assign an additional cost for the unaligned store. */
1017 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1018 unaligned_store
, stmt_info
,
1019 DR_MISALIGNMENT (dr
), vect_body
);
1020 if (dump_enabled_p ())
1021 dump_printf_loc (MSG_NOTE
, vect_location
,
1022 "vect_model_store_cost: unaligned supported by "
1027 case dr_unaligned_unsupported
:
1029 *inside_cost
= VECT_MAX_COST
;
1031 if (dump_enabled_p ())
1032 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1033 "vect_model_store_cost: unsupported access.\n");
1043 /* Function vect_model_load_cost
1045 Models cost for loads. In the case of grouped accesses, the last access
1046 has the overhead of the grouped access attributed to it. Since unaligned
1047 accesses are supported for loads, we also account for the costs of the
1048 access scheme chosen. */
1051 vect_model_load_cost (stmt_vec_info stmt_info
, int ncopies
,
1052 bool load_lanes_p
, slp_tree slp_node
,
1053 stmt_vector_for_cost
*prologue_cost_vec
,
1054 stmt_vector_for_cost
*body_cost_vec
)
1058 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
;
1059 unsigned int inside_cost
= 0, prologue_cost
= 0;
1061 /* Grouped accesses? */
1062 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
1063 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
) && first_stmt
&& !slp_node
)
1065 group_size
= vect_cost_group_size (stmt_info
);
1066 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
1068 /* Not a grouped access. */
1075 /* We assume that the cost of a single load-lanes instruction is
1076 equivalent to the cost of GROUP_SIZE separate loads. If a grouped
1077 access is instead being provided by a load-and-permute operation,
1078 include the cost of the permutes. */
1079 if (!load_lanes_p
&& group_size
> 1
1080 && !STMT_VINFO_STRIDED_P (stmt_info
))
1082 /* Uses an even and odd extract operations or shuffle operations
1083 for each needed permute. */
1084 int nstmts
= ncopies
* ceil_log2 (group_size
) * group_size
;
1085 inside_cost
= record_stmt_cost (body_cost_vec
, nstmts
, vec_perm
,
1086 stmt_info
, 0, vect_body
);
1088 if (dump_enabled_p ())
1089 dump_printf_loc (MSG_NOTE
, vect_location
,
1090 "vect_model_load_cost: strided group_size = %d .\n",
1094 /* The loads themselves. */
1095 if (STMT_VINFO_STRIDED_P (stmt_info
)
1096 && !STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1098 /* N scalar loads plus gathering them into a vector. */
1099 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1100 inside_cost
+= record_stmt_cost (body_cost_vec
,
1101 ncopies
* TYPE_VECTOR_SUBPARTS (vectype
),
1102 scalar_load
, stmt_info
, 0, vect_body
);
1105 vect_get_load_cost (first_dr
, ncopies
,
1106 ((!STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1107 || group_size
> 1 || slp_node
),
1108 &inside_cost
, &prologue_cost
,
1109 prologue_cost_vec
, body_cost_vec
, true);
1110 if (STMT_VINFO_STRIDED_P (stmt_info
))
1111 inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vec_construct
,
1112 stmt_info
, 0, vect_body
);
1114 if (dump_enabled_p ())
1115 dump_printf_loc (MSG_NOTE
, vect_location
,
1116 "vect_model_load_cost: inside_cost = %d, "
1117 "prologue_cost = %d .\n", inside_cost
, prologue_cost
);
1121 /* Calculate cost of DR's memory access. */
1123 vect_get_load_cost (struct data_reference
*dr
, int ncopies
,
1124 bool add_realign_cost
, unsigned int *inside_cost
,
1125 unsigned int *prologue_cost
,
1126 stmt_vector_for_cost
*prologue_cost_vec
,
1127 stmt_vector_for_cost
*body_cost_vec
,
1128 bool record_prologue_costs
)
1130 int alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
1131 gimple
*stmt
= DR_STMT (dr
);
1132 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1134 switch (alignment_support_scheme
)
1138 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vector_load
,
1139 stmt_info
, 0, vect_body
);
1141 if (dump_enabled_p ())
1142 dump_printf_loc (MSG_NOTE
, vect_location
,
1143 "vect_model_load_cost: aligned.\n");
1147 case dr_unaligned_supported
:
1149 /* Here, we assign an additional cost for the unaligned load. */
1150 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1151 unaligned_load
, stmt_info
,
1152 DR_MISALIGNMENT (dr
), vect_body
);
1154 if (dump_enabled_p ())
1155 dump_printf_loc (MSG_NOTE
, vect_location
,
1156 "vect_model_load_cost: unaligned supported by "
1161 case dr_explicit_realign
:
1163 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
* 2,
1164 vector_load
, stmt_info
, 0, vect_body
);
1165 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
,
1166 vec_perm
, stmt_info
, 0, vect_body
);
1168 /* FIXME: If the misalignment remains fixed across the iterations of
1169 the containing loop, the following cost should be added to the
1171 if (targetm
.vectorize
.builtin_mask_for_load
)
1172 *inside_cost
+= record_stmt_cost (body_cost_vec
, 1, vector_stmt
,
1173 stmt_info
, 0, vect_body
);
1175 if (dump_enabled_p ())
1176 dump_printf_loc (MSG_NOTE
, vect_location
,
1177 "vect_model_load_cost: explicit realign\n");
1181 case dr_explicit_realign_optimized
:
1183 if (dump_enabled_p ())
1184 dump_printf_loc (MSG_NOTE
, vect_location
,
1185 "vect_model_load_cost: unaligned software "
1188 /* Unaligned software pipeline has a load of an address, an initial
1189 load, and possibly a mask operation to "prime" the loop. However,
1190 if this is an access in a group of loads, which provide grouped
1191 access, then the above cost should only be considered for one
1192 access in the group. Inside the loop, there is a load op
1193 and a realignment op. */
1195 if (add_realign_cost
&& record_prologue_costs
)
1197 *prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 2,
1198 vector_stmt
, stmt_info
,
1200 if (targetm
.vectorize
.builtin_mask_for_load
)
1201 *prologue_cost
+= record_stmt_cost (prologue_cost_vec
, 1,
1202 vector_stmt
, stmt_info
,
1206 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vector_load
,
1207 stmt_info
, 0, vect_body
);
1208 *inside_cost
+= record_stmt_cost (body_cost_vec
, ncopies
, vec_perm
,
1209 stmt_info
, 0, vect_body
);
1211 if (dump_enabled_p ())
1212 dump_printf_loc (MSG_NOTE
, vect_location
,
1213 "vect_model_load_cost: explicit realign optimized"
1219 case dr_unaligned_unsupported
:
1221 *inside_cost
= VECT_MAX_COST
;
1223 if (dump_enabled_p ())
1224 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1225 "vect_model_load_cost: unsupported access.\n");
1234 /* Insert the new stmt NEW_STMT at *GSI or at the appropriate place in
1235 the loop preheader for the vectorized stmt STMT. */
1238 vect_init_vector_1 (gimple
*stmt
, gimple
*new_stmt
, gimple_stmt_iterator
*gsi
)
1241 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1244 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
1245 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1249 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1253 if (nested_in_vect_loop_p (loop
, stmt
))
1256 pe
= loop_preheader_edge (loop
);
1257 new_bb
= gsi_insert_on_edge_immediate (pe
, new_stmt
);
1258 gcc_assert (!new_bb
);
1262 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_vinfo
);
1264 gimple_stmt_iterator gsi_bb_start
;
1266 gcc_assert (bb_vinfo
);
1267 bb
= BB_VINFO_BB (bb_vinfo
);
1268 gsi_bb_start
= gsi_after_labels (bb
);
1269 gsi_insert_before (&gsi_bb_start
, new_stmt
, GSI_SAME_STMT
);
1273 if (dump_enabled_p ())
1275 dump_printf_loc (MSG_NOTE
, vect_location
,
1276 "created new init_stmt: ");
1277 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, new_stmt
, 0);
1281 /* Function vect_init_vector.
1283 Insert a new stmt (INIT_STMT) that initializes a new variable of type
1284 TYPE with the value VAL. If TYPE is a vector type and VAL does not have
1285 vector type a vector with all elements equal to VAL is created first.
1286 Place the initialization at BSI if it is not NULL. Otherwise, place the
1287 initialization at the loop preheader.
1288 Return the DEF of INIT_STMT.
1289 It will be used in the vectorization of STMT. */
1292 vect_init_vector (gimple
*stmt
, tree val
, tree type
, gimple_stmt_iterator
*gsi
)
1297 if (TREE_CODE (type
) == VECTOR_TYPE
1298 && TREE_CODE (TREE_TYPE (val
)) != VECTOR_TYPE
)
1300 if (!types_compatible_p (TREE_TYPE (type
), TREE_TYPE (val
)))
1302 if (CONSTANT_CLASS_P (val
))
1303 val
= fold_convert (TREE_TYPE (type
), val
);
1306 new_temp
= make_ssa_name (TREE_TYPE (type
));
1307 init_stmt
= gimple_build_assign (new_temp
, NOP_EXPR
, val
);
1308 vect_init_vector_1 (stmt
, init_stmt
, gsi
);
1312 val
= build_vector_from_val (type
, val
);
1315 new_temp
= vect_get_new_ssa_name (type
, vect_simple_var
, "cst_");
1316 init_stmt
= gimple_build_assign (new_temp
, val
);
1317 vect_init_vector_1 (stmt
, init_stmt
, gsi
);
1322 /* Function vect_get_vec_def_for_operand.
1324 OP is an operand in STMT. This function returns a (vector) def that will be
1325 used in the vectorized stmt for STMT.
1327 In the case that OP is an SSA_NAME which is defined in the loop, then
1328 STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
1330 In case OP is an invariant or constant, a new stmt that creates a vector def
1331 needs to be introduced. VECTYPE may be used to specify a required type for
1332 vector invariant. */
1335 vect_get_vec_def_for_operand (tree op
, gimple
*stmt
, tree vectype
)
1340 stmt_vec_info def_stmt_info
= NULL
;
1341 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
1342 tree stmt_vectype
= STMT_VINFO_VECTYPE (stmt_vinfo
);
1343 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
1344 enum vect_def_type dt
;
1348 if (dump_enabled_p ())
1350 dump_printf_loc (MSG_NOTE
, vect_location
,
1351 "vect_get_vec_def_for_operand: ");
1352 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, op
);
1353 dump_printf (MSG_NOTE
, "\n");
1356 is_simple_use
= vect_is_simple_use (op
, loop_vinfo
, &def_stmt
, &dt
);
1357 gcc_assert (is_simple_use
);
1358 if (dump_enabled_p ())
1360 int loc_printed
= 0;
1364 dump_printf (MSG_NOTE
, " def_stmt = ");
1366 dump_printf_loc (MSG_NOTE
, vect_location
, " def_stmt = ");
1367 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, def_stmt
, 0);
1373 /* operand is a constant or a loop invariant. */
1374 case vect_constant_def
:
1375 case vect_external_def
:
1378 vector_type
= vectype
;
1379 else if (TREE_CODE (TREE_TYPE (op
)) == BOOLEAN_TYPE
1380 && VECTOR_BOOLEAN_TYPE_P (stmt_vectype
))
1381 vector_type
= build_same_sized_truth_vector_type (stmt_vectype
);
1383 vector_type
= get_vectype_for_scalar_type (TREE_TYPE (op
));
1385 gcc_assert (vector_type
);
1386 return vect_init_vector (stmt
, op
, vector_type
, NULL
);
1389 /* operand is defined inside the loop. */
1390 case vect_internal_def
:
1392 /* Get the def from the vectorized stmt. */
1393 def_stmt_info
= vinfo_for_stmt (def_stmt
);
1395 vec_stmt
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1396 /* Get vectorized pattern statement. */
1398 && STMT_VINFO_IN_PATTERN_P (def_stmt_info
)
1399 && !STMT_VINFO_RELEVANT (def_stmt_info
))
1400 vec_stmt
= STMT_VINFO_VEC_STMT (vinfo_for_stmt (
1401 STMT_VINFO_RELATED_STMT (def_stmt_info
)));
1402 gcc_assert (vec_stmt
);
1403 if (gimple_code (vec_stmt
) == GIMPLE_PHI
)
1404 vec_oprnd
= PHI_RESULT (vec_stmt
);
1405 else if (is_gimple_call (vec_stmt
))
1406 vec_oprnd
= gimple_call_lhs (vec_stmt
);
1408 vec_oprnd
= gimple_assign_lhs (vec_stmt
);
1412 /* operand is defined by a loop header phi - reduction */
1413 case vect_reduction_def
:
1414 case vect_double_reduction_def
:
1415 case vect_nested_cycle
:
1416 /* Code should use get_initial_def_for_reduction. */
1419 /* operand is defined by loop-header phi - induction. */
1420 case vect_induction_def
:
1422 gcc_assert (gimple_code (def_stmt
) == GIMPLE_PHI
);
1424 /* Get the def from the vectorized stmt. */
1425 def_stmt_info
= vinfo_for_stmt (def_stmt
);
1426 vec_stmt
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1427 if (gimple_code (vec_stmt
) == GIMPLE_PHI
)
1428 vec_oprnd
= PHI_RESULT (vec_stmt
);
1430 vec_oprnd
= gimple_get_lhs (vec_stmt
);
1440 /* Function vect_get_vec_def_for_stmt_copy
1442 Return a vector-def for an operand. This function is used when the
1443 vectorized stmt to be created (by the caller to this function) is a "copy"
1444 created in case the vectorized result cannot fit in one vector, and several
1445 copies of the vector-stmt are required. In this case the vector-def is
1446 retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field
1447 of the stmt that defines VEC_OPRND.
1448 DT is the type of the vector def VEC_OPRND.
1451 In case the vectorization factor (VF) is bigger than the number
1452 of elements that can fit in a vectype (nunits), we have to generate
1453 more than one vector stmt to vectorize the scalar stmt. This situation
1454 arises when there are multiple data-types operated upon in the loop; the
1455 smallest data-type determines the VF, and as a result, when vectorizing
1456 stmts operating on wider types we need to create 'VF/nunits' "copies" of the
1457 vector stmt (each computing a vector of 'nunits' results, and together
1458 computing 'VF' results in each iteration). This function is called when
1459 vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
1460 which VF=16 and nunits=4, so the number of copies required is 4):
1462 scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
1464 S1: x = load VS1.0: vx.0 = memref0 VS1.1
1465 VS1.1: vx.1 = memref1 VS1.2
1466 VS1.2: vx.2 = memref2 VS1.3
1467 VS1.3: vx.3 = memref3
1469 S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1
1470 VSnew.1: vz1 = vx.1 + ... VSnew.2
1471 VSnew.2: vz2 = vx.2 + ... VSnew.3
1472 VSnew.3: vz3 = vx.3 + ...
1474 The vectorization of S1 is explained in vectorizable_load.
1475 The vectorization of S2:
1476 To create the first vector-stmt out of the 4 copies - VSnew.0 -
1477 the function 'vect_get_vec_def_for_operand' is called to
1478 get the relevant vector-def for each operand of S2. For operand x it
1479 returns the vector-def 'vx.0'.
1481 To create the remaining copies of the vector-stmt (VSnew.j), this
1482 function is called to get the relevant vector-def for each operand. It is
1483 obtained from the respective VS1.j stmt, which is recorded in the
1484 STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND.
1486 For example, to obtain the vector-def 'vx.1' in order to create the
1487 vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'.
1488 Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the
1489 STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1',
1490 and return its def ('vx.1').
1491 Overall, to create the above sequence this function will be called 3 times:
1492 vx.1 = vect_get_vec_def_for_stmt_copy (dt, vx.0);
1493 vx.2 = vect_get_vec_def_for_stmt_copy (dt, vx.1);
1494 vx.3 = vect_get_vec_def_for_stmt_copy (dt, vx.2); */
1497 vect_get_vec_def_for_stmt_copy (enum vect_def_type dt
, tree vec_oprnd
)
1499 gimple
*vec_stmt_for_operand
;
1500 stmt_vec_info def_stmt_info
;
1502 /* Do nothing; can reuse same def. */
1503 if (dt
== vect_external_def
|| dt
== vect_constant_def
)
1506 vec_stmt_for_operand
= SSA_NAME_DEF_STMT (vec_oprnd
);
1507 def_stmt_info
= vinfo_for_stmt (vec_stmt_for_operand
);
1508 gcc_assert (def_stmt_info
);
1509 vec_stmt_for_operand
= STMT_VINFO_RELATED_STMT (def_stmt_info
);
1510 gcc_assert (vec_stmt_for_operand
);
1511 if (gimple_code (vec_stmt_for_operand
) == GIMPLE_PHI
)
1512 vec_oprnd
= PHI_RESULT (vec_stmt_for_operand
);
1514 vec_oprnd
= gimple_get_lhs (vec_stmt_for_operand
);
1519 /* Get vectorized definitions for the operands to create a copy of an original
1520 stmt. See vect_get_vec_def_for_stmt_copy () for details. */
1523 vect_get_vec_defs_for_stmt_copy (enum vect_def_type
*dt
,
1524 vec
<tree
> *vec_oprnds0
,
1525 vec
<tree
> *vec_oprnds1
)
1527 tree vec_oprnd
= vec_oprnds0
->pop ();
1529 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd
);
1530 vec_oprnds0
->quick_push (vec_oprnd
);
1532 if (vec_oprnds1
&& vec_oprnds1
->length ())
1534 vec_oprnd
= vec_oprnds1
->pop ();
1535 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
[1], vec_oprnd
);
1536 vec_oprnds1
->quick_push (vec_oprnd
);
1541 /* Get vectorized definitions for OP0 and OP1.
1542 REDUC_INDEX is the index of reduction operand in case of reduction,
1543 and -1 otherwise. */
1546 vect_get_vec_defs (tree op0
, tree op1
, gimple
*stmt
,
1547 vec
<tree
> *vec_oprnds0
,
1548 vec
<tree
> *vec_oprnds1
,
1549 slp_tree slp_node
, int reduc_index
)
1553 int nops
= (op1
== NULL_TREE
) ? 1 : 2;
1554 auto_vec
<tree
> ops (nops
);
1555 auto_vec
<vec
<tree
> > vec_defs (nops
);
1557 ops
.quick_push (op0
);
1559 ops
.quick_push (op1
);
1561 vect_get_slp_defs (ops
, slp_node
, &vec_defs
, reduc_index
);
1563 *vec_oprnds0
= vec_defs
[0];
1565 *vec_oprnds1
= vec_defs
[1];
1571 vec_oprnds0
->create (1);
1572 vec_oprnd
= vect_get_vec_def_for_operand (op0
, stmt
);
1573 vec_oprnds0
->quick_push (vec_oprnd
);
1577 vec_oprnds1
->create (1);
1578 vec_oprnd
= vect_get_vec_def_for_operand (op1
, stmt
);
1579 vec_oprnds1
->quick_push (vec_oprnd
);
1585 /* Function vect_finish_stmt_generation.
1587 Insert a new stmt. */
1590 vect_finish_stmt_generation (gimple
*stmt
, gimple
*vec_stmt
,
1591 gimple_stmt_iterator
*gsi
)
1593 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1594 vec_info
*vinfo
= stmt_info
->vinfo
;
1596 gcc_assert (gimple_code (stmt
) != GIMPLE_LABEL
);
1598 if (!gsi_end_p (*gsi
)
1599 && gimple_has_mem_ops (vec_stmt
))
1601 gimple
*at_stmt
= gsi_stmt (*gsi
);
1602 tree vuse
= gimple_vuse (at_stmt
);
1603 if (vuse
&& TREE_CODE (vuse
) == SSA_NAME
)
1605 tree vdef
= gimple_vdef (at_stmt
);
1606 gimple_set_vuse (vec_stmt
, gimple_vuse (at_stmt
));
1607 /* If we have an SSA vuse and insert a store, update virtual
1608 SSA form to avoid triggering the renamer. Do so only
1609 if we can easily see all uses - which is what almost always
1610 happens with the way vectorized stmts are inserted. */
1611 if ((vdef
&& TREE_CODE (vdef
) == SSA_NAME
)
1612 && ((is_gimple_assign (vec_stmt
)
1613 && !is_gimple_reg (gimple_assign_lhs (vec_stmt
)))
1614 || (is_gimple_call (vec_stmt
)
1615 && !(gimple_call_flags (vec_stmt
)
1616 & (ECF_CONST
|ECF_PURE
|ECF_NOVOPS
)))))
1618 tree new_vdef
= copy_ssa_name (vuse
, vec_stmt
);
1619 gimple_set_vdef (vec_stmt
, new_vdef
);
1620 SET_USE (gimple_vuse_op (at_stmt
), new_vdef
);
1624 gsi_insert_before (gsi
, vec_stmt
, GSI_SAME_STMT
);
1626 set_vinfo_for_stmt (vec_stmt
, new_stmt_vec_info (vec_stmt
, vinfo
));
1628 if (dump_enabled_p ())
1630 dump_printf_loc (MSG_NOTE
, vect_location
, "add new stmt: ");
1631 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, vec_stmt
, 0);
1634 gimple_set_location (vec_stmt
, gimple_location (stmt
));
1636 /* While EH edges will generally prevent vectorization, stmt might
1637 e.g. be in a must-not-throw region. Ensure newly created stmts
1638 that could throw are part of the same region. */
1639 int lp_nr
= lookup_stmt_eh_lp (stmt
);
1640 if (lp_nr
!= 0 && stmt_could_throw_p (vec_stmt
))
1641 add_stmt_to_eh_lp (vec_stmt
, lp_nr
);
1644 /* Checks if CALL can be vectorized in type VECTYPE. Returns
1645 a function declaration if the target has a vectorized version
1646 of the function, or NULL_TREE if the function cannot be vectorized. */
1649 vectorizable_function (gcall
*call
, tree vectype_out
, tree vectype_in
)
1651 tree fndecl
= gimple_call_fndecl (call
);
1653 /* We only handle functions that do not read or clobber memory -- i.e.
1654 const or novops ones. */
1655 if (!(gimple_call_flags (call
) & (ECF_CONST
| ECF_NOVOPS
)))
1659 || TREE_CODE (fndecl
) != FUNCTION_DECL
1660 || !DECL_BUILT_IN (fndecl
))
1663 return targetm
.vectorize
.builtin_vectorized_function (fndecl
, vectype_out
,
1668 static tree
permute_vec_elements (tree
, tree
, tree
, gimple
*,
1669 gimple_stmt_iterator
*);
1672 /* Function vectorizable_mask_load_store.
1674 Check if STMT performs a conditional load or store that can be vectorized.
1675 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1676 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
1677 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1680 vectorizable_mask_load_store (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
1681 gimple
**vec_stmt
, slp_tree slp_node
)
1683 tree vec_dest
= NULL
;
1684 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1685 stmt_vec_info prev_stmt_info
;
1686 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1687 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
1688 bool nested_in_vect_loop
= nested_in_vect_loop_p (loop
, stmt
);
1689 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
);
1690 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1691 tree rhs_vectype
= NULL_TREE
;
1696 tree dataref_ptr
= NULL_TREE
;
1698 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
1702 tree gather_base
= NULL_TREE
, gather_off
= NULL_TREE
;
1703 tree gather_off_vectype
= NULL_TREE
, gather_decl
= NULL_TREE
;
1704 int gather_scale
= 1;
1705 enum vect_def_type gather_dt
= vect_unknown_def_type
;
1709 enum vect_def_type dt
;
1711 if (slp_node
!= NULL
)
1714 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
1715 gcc_assert (ncopies
>= 1);
1717 is_store
= gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
;
1718 mask
= gimple_call_arg (stmt
, 2);
1720 if (TREE_CODE (TREE_TYPE (mask
)) != BOOLEAN_TYPE
)
1723 /* FORNOW. This restriction should be relaxed. */
1724 if (nested_in_vect_loop
&& ncopies
> 1)
1726 if (dump_enabled_p ())
1727 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1728 "multiple types in nested loop.");
1732 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
1735 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
1738 if (!STMT_VINFO_DATA_REF (stmt_info
))
1741 elem_type
= TREE_TYPE (vectype
);
1743 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
1746 if (STMT_VINFO_STRIDED_P (stmt_info
))
1749 if (TREE_CODE (mask
) != SSA_NAME
)
1752 if (!vect_is_simple_use (mask
, loop_vinfo
, &def_stmt
, &dt
, &mask_vectype
))
1756 mask_vectype
= get_mask_type_for_scalar_type (TREE_TYPE (vectype
));
1763 tree rhs
= gimple_call_arg (stmt
, 3);
1764 if (!vect_is_simple_use (rhs
, loop_vinfo
, &def_stmt
, &dt
, &rhs_vectype
))
1768 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
1771 gather_decl
= vect_check_gather_scatter (stmt
, loop_vinfo
, &gather_base
,
1772 &gather_off
, &gather_scale
);
1773 gcc_assert (gather_decl
);
1774 if (!vect_is_simple_use (gather_off
, loop_vinfo
, &def_stmt
, &gather_dt
,
1775 &gather_off_vectype
))
1777 if (dump_enabled_p ())
1778 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1779 "gather index use not simple.");
1783 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gather_decl
));
1785 = TREE_VALUE (TREE_CHAIN (TREE_CHAIN (TREE_CHAIN (arglist
))));
1786 if (TREE_CODE (masktype
) == INTEGER_TYPE
)
1788 if (dump_enabled_p ())
1789 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
1790 "masked gather with integer mask not supported.");
1794 else if (tree_int_cst_compare (nested_in_vect_loop
1795 ? STMT_VINFO_DR_STEP (stmt_info
)
1796 : DR_STEP (dr
), size_zero_node
) <= 0)
1798 else if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
1799 || !can_vec_mask_load_store_p (TYPE_MODE (vectype
),
1800 TYPE_MODE (mask_vectype
),
1803 && !useless_type_conversion_p (vectype
, rhs_vectype
)))
1806 if (!vec_stmt
) /* transformation not required. */
1808 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
1810 vect_model_store_cost (stmt_info
, ncopies
, false, dt
,
1813 vect_model_load_cost (stmt_info
, ncopies
, false, NULL
, NULL
, NULL
);
1819 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
1821 tree vec_oprnd0
= NULL_TREE
, op
;
1822 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gather_decl
));
1823 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
1824 tree ptr
, vec_mask
= NULL_TREE
, mask_op
= NULL_TREE
, var
, scale
;
1825 tree perm_mask
= NULL_TREE
, prev_res
= NULL_TREE
;
1826 tree mask_perm_mask
= NULL_TREE
;
1827 edge pe
= loop_preheader_edge (loop
);
1830 enum { NARROW
, NONE
, WIDEN
} modifier
;
1831 int gather_off_nunits
= TYPE_VECTOR_SUBPARTS (gather_off_vectype
);
1833 rettype
= TREE_TYPE (TREE_TYPE (gather_decl
));
1834 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1835 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1836 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1837 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
1838 scaletype
= TREE_VALUE (arglist
);
1839 gcc_checking_assert (types_compatible_p (srctype
, rettype
)
1840 && types_compatible_p (srctype
, masktype
));
1842 if (nunits
== gather_off_nunits
)
1844 else if (nunits
== gather_off_nunits
/ 2)
1846 unsigned char *sel
= XALLOCAVEC (unsigned char, gather_off_nunits
);
1849 for (i
= 0; i
< gather_off_nunits
; ++i
)
1850 sel
[i
] = i
| nunits
;
1852 perm_mask
= vect_gen_perm_mask_checked (gather_off_vectype
, sel
);
1854 else if (nunits
== gather_off_nunits
* 2)
1856 unsigned char *sel
= XALLOCAVEC (unsigned char, nunits
);
1859 for (i
= 0; i
< nunits
; ++i
)
1860 sel
[i
] = i
< gather_off_nunits
1861 ? i
: i
+ nunits
- gather_off_nunits
;
1863 perm_mask
= vect_gen_perm_mask_checked (vectype
, sel
);
1865 for (i
= 0; i
< nunits
; ++i
)
1866 sel
[i
] = i
| gather_off_nunits
;
1867 mask_perm_mask
= vect_gen_perm_mask_checked (masktype
, sel
);
1872 vec_dest
= vect_create_destination_var (gimple_call_lhs (stmt
), vectype
);
1874 ptr
= fold_convert (ptrtype
, gather_base
);
1875 if (!is_gimple_min_invariant (ptr
))
1877 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
1878 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
1879 gcc_assert (!new_bb
);
1882 scale
= build_int_cst (scaletype
, gather_scale
);
1884 prev_stmt_info
= NULL
;
1885 for (j
= 0; j
< ncopies
; ++j
)
1887 if (modifier
== WIDEN
&& (j
& 1))
1888 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
,
1889 perm_mask
, stmt
, gsi
);
1892 = vect_get_vec_def_for_operand (gather_off
, stmt
);
1895 = vect_get_vec_def_for_stmt_copy (gather_dt
, vec_oprnd0
);
1897 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
1899 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
1900 == TYPE_VECTOR_SUBPARTS (idxtype
));
1901 var
= vect_get_new_ssa_name (idxtype
, vect_simple_var
);
1902 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
1904 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
1905 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1909 if (mask_perm_mask
&& (j
& 1))
1910 mask_op
= permute_vec_elements (mask_op
, mask_op
,
1911 mask_perm_mask
, stmt
, gsi
);
1915 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
);
1918 vect_is_simple_use (vec_mask
, loop_vinfo
, &def_stmt
, &dt
);
1919 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
1923 if (!useless_type_conversion_p (masktype
, TREE_TYPE (vec_mask
)))
1925 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (mask_op
))
1926 == TYPE_VECTOR_SUBPARTS (masktype
));
1927 var
= vect_get_new_ssa_name (masktype
, vect_simple_var
);
1928 mask_op
= build1 (VIEW_CONVERT_EXPR
, masktype
, mask_op
);
1930 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, mask_op
);
1931 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1937 = gimple_build_call (gather_decl
, 5, mask_op
, ptr
, op
, mask_op
,
1940 if (!useless_type_conversion_p (vectype
, rettype
))
1942 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype
)
1943 == TYPE_VECTOR_SUBPARTS (rettype
));
1944 op
= vect_get_new_ssa_name (rettype
, vect_simple_var
);
1945 gimple_call_set_lhs (new_stmt
, op
);
1946 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1947 var
= make_ssa_name (vec_dest
);
1948 op
= build1 (VIEW_CONVERT_EXPR
, vectype
, op
);
1949 new_stmt
= gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
1953 var
= make_ssa_name (vec_dest
, new_stmt
);
1954 gimple_call_set_lhs (new_stmt
, var
);
1957 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1959 if (modifier
== NARROW
)
1966 var
= permute_vec_elements (prev_res
, var
,
1967 perm_mask
, stmt
, gsi
);
1968 new_stmt
= SSA_NAME_DEF_STMT (var
);
1971 if (prev_stmt_info
== NULL
)
1972 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
1974 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
1975 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
1978 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
1980 if (STMT_VINFO_RELATED_STMT (stmt_info
))
1982 stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
1983 stmt_info
= vinfo_for_stmt (stmt
);
1985 tree lhs
= gimple_call_lhs (stmt
);
1986 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
1987 set_vinfo_for_stmt (new_stmt
, stmt_info
);
1988 set_vinfo_for_stmt (stmt
, NULL
);
1989 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
1990 gsi_replace (gsi
, new_stmt
, true);
1995 tree vec_rhs
= NULL_TREE
, vec_mask
= NULL_TREE
;
1996 prev_stmt_info
= NULL
;
1997 for (i
= 0; i
< ncopies
; i
++)
1999 unsigned align
, misalign
;
2003 tree rhs
= gimple_call_arg (stmt
, 3);
2004 vec_rhs
= vect_get_vec_def_for_operand (rhs
, stmt
);
2005 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
);
2006 /* We should have catched mismatched types earlier. */
2007 gcc_assert (useless_type_conversion_p (vectype
,
2008 TREE_TYPE (vec_rhs
)));
2009 dataref_ptr
= vect_create_data_ref_ptr (stmt
, vectype
, NULL
,
2010 NULL_TREE
, &dummy
, gsi
,
2011 &ptr_incr
, false, &inv_p
);
2012 gcc_assert (!inv_p
);
2016 vect_is_simple_use (vec_rhs
, loop_vinfo
, &def_stmt
, &dt
);
2017 vec_rhs
= vect_get_vec_def_for_stmt_copy (dt
, vec_rhs
);
2018 vect_is_simple_use (vec_mask
, loop_vinfo
, &def_stmt
, &dt
);
2019 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2020 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
2021 TYPE_SIZE_UNIT (vectype
));
2024 align
= TYPE_ALIGN_UNIT (vectype
);
2025 if (aligned_access_p (dr
))
2027 else if (DR_MISALIGNMENT (dr
) == -1)
2029 align
= TYPE_ALIGN_UNIT (elem_type
);
2033 misalign
= DR_MISALIGNMENT (dr
);
2034 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
2037 = gimple_build_call_internal (IFN_MASK_STORE
, 4, dataref_ptr
,
2038 gimple_call_arg (stmt
, 1),
2040 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2042 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2044 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2045 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2050 tree vec_mask
= NULL_TREE
;
2051 prev_stmt_info
= NULL
;
2052 vec_dest
= vect_create_destination_var (gimple_call_lhs (stmt
), vectype
);
2053 for (i
= 0; i
< ncopies
; i
++)
2055 unsigned align
, misalign
;
2059 vec_mask
= vect_get_vec_def_for_operand (mask
, stmt
);
2060 dataref_ptr
= vect_create_data_ref_ptr (stmt
, vectype
, NULL
,
2061 NULL_TREE
, &dummy
, gsi
,
2062 &ptr_incr
, false, &inv_p
);
2063 gcc_assert (!inv_p
);
2067 vect_is_simple_use (vec_mask
, loop_vinfo
, &def_stmt
, &dt
);
2068 vec_mask
= vect_get_vec_def_for_stmt_copy (dt
, vec_mask
);
2069 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
2070 TYPE_SIZE_UNIT (vectype
));
2073 align
= TYPE_ALIGN_UNIT (vectype
);
2074 if (aligned_access_p (dr
))
2076 else if (DR_MISALIGNMENT (dr
) == -1)
2078 align
= TYPE_ALIGN_UNIT (elem_type
);
2082 misalign
= DR_MISALIGNMENT (dr
);
2083 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
2086 = gimple_build_call_internal (IFN_MASK_LOAD
, 3, dataref_ptr
,
2087 gimple_call_arg (stmt
, 1),
2089 gimple_call_set_lhs (new_stmt
, make_ssa_name (vec_dest
));
2090 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2092 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2094 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2095 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2101 /* Ensure that even with -fno-tree-dce the scalar MASK_LOAD is removed
2103 if (STMT_VINFO_RELATED_STMT (stmt_info
))
2105 stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
2106 stmt_info
= vinfo_for_stmt (stmt
);
2108 tree lhs
= gimple_call_lhs (stmt
);
2109 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (TREE_TYPE (lhs
)));
2110 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2111 set_vinfo_for_stmt (stmt
, NULL
);
2112 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2113 gsi_replace (gsi
, new_stmt
, true);
2120 /* Function vectorizable_call.
2122 Check if GS performs a function call that can be vectorized.
2123 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2124 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2125 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2128 vectorizable_call (gimple
*gs
, gimple_stmt_iterator
*gsi
, gimple
**vec_stmt
,
2135 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
2136 stmt_vec_info stmt_info
= vinfo_for_stmt (gs
), prev_stmt_info
;
2137 tree vectype_out
, vectype_in
;
2140 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2141 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
2142 vec_info
*vinfo
= stmt_info
->vinfo
;
2143 tree fndecl
, new_temp
, rhs_type
;
2145 enum vect_def_type dt
[3]
2146 = {vect_unknown_def_type
, vect_unknown_def_type
, vect_unknown_def_type
};
2147 gimple
*new_stmt
= NULL
;
2149 vec
<tree
> vargs
= vNULL
;
2150 enum { NARROW
, NONE
, WIDEN
} modifier
;
2154 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
2157 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
2160 /* Is GS a vectorizable call? */
2161 stmt
= dyn_cast
<gcall
*> (gs
);
2165 if (gimple_call_internal_p (stmt
)
2166 && (gimple_call_internal_fn (stmt
) == IFN_MASK_LOAD
2167 || gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
))
2168 return vectorizable_mask_load_store (stmt
, gsi
, vec_stmt
,
2171 if (gimple_call_lhs (stmt
) == NULL_TREE
2172 || TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
2175 gcc_checking_assert (!stmt_can_throw_internal (stmt
));
2177 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
2179 /* Process function arguments. */
2180 rhs_type
= NULL_TREE
;
2181 vectype_in
= NULL_TREE
;
2182 nargs
= gimple_call_num_args (stmt
);
2184 /* Bail out if the function has more than three arguments, we do not have
2185 interesting builtin functions to vectorize with more than two arguments
2186 except for fma. No arguments is also not good. */
2187 if (nargs
== 0 || nargs
> 3)
2190 /* Ignore the argument of IFN_GOMP_SIMD_LANE, it is magic. */
2191 if (gimple_call_internal_p (stmt
)
2192 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
)
2195 rhs_type
= unsigned_type_node
;
2198 for (i
= 0; i
< nargs
; i
++)
2202 op
= gimple_call_arg (stmt
, i
);
2204 /* We can only handle calls with arguments of the same type. */
2206 && !types_compatible_p (rhs_type
, TREE_TYPE (op
)))
2208 if (dump_enabled_p ())
2209 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2210 "argument types differ.\n");
2214 rhs_type
= TREE_TYPE (op
);
2216 if (!vect_is_simple_use (op
, vinfo
, &def_stmt
, &dt
[i
], &opvectype
))
2218 if (dump_enabled_p ())
2219 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2220 "use not simple.\n");
2225 vectype_in
= opvectype
;
2227 && opvectype
!= vectype_in
)
2229 if (dump_enabled_p ())
2230 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2231 "argument vector types differ.\n");
2235 /* If all arguments are external or constant defs use a vector type with
2236 the same size as the output vector type. */
2238 vectype_in
= get_same_sized_vectype (rhs_type
, vectype_out
);
2240 gcc_assert (vectype_in
);
2243 if (dump_enabled_p ())
2245 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2246 "no vectype for scalar type ");
2247 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
2248 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
2255 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
2256 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
2257 if (nunits_in
== nunits_out
/ 2)
2259 else if (nunits_out
== nunits_in
)
2261 else if (nunits_out
== nunits_in
/ 2)
2266 /* For now, we only vectorize functions if a target specific builtin
2267 is available. TODO -- in some cases, it might be profitable to
2268 insert the calls for pieces of the vector, in order to be able
2269 to vectorize other operations in the loop. */
2270 fndecl
= vectorizable_function (stmt
, vectype_out
, vectype_in
);
2271 if (fndecl
== NULL_TREE
)
2273 if (gimple_call_internal_p (stmt
)
2274 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
2277 && LOOP_VINFO_LOOP (loop_vinfo
)->simduid
2278 && TREE_CODE (gimple_call_arg (stmt
, 0)) == SSA_NAME
2279 && LOOP_VINFO_LOOP (loop_vinfo
)->simduid
2280 == SSA_NAME_VAR (gimple_call_arg (stmt
, 0)))
2282 /* We can handle IFN_GOMP_SIMD_LANE by returning a
2283 { 0, 1, 2, ... vf - 1 } vector. */
2284 gcc_assert (nargs
== 0);
2288 if (dump_enabled_p ())
2289 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2290 "function is not vectorizable.\n");
2295 gcc_assert (!gimple_vuse (stmt
));
2297 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
2299 else if (modifier
== NARROW
)
2300 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_out
;
2302 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
2304 /* Sanity check: make sure that at least one copy of the vectorized stmt
2305 needs to be generated. */
2306 gcc_assert (ncopies
>= 1);
2308 if (!vec_stmt
) /* transformation not required. */
2310 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
2311 if (dump_enabled_p ())
2312 dump_printf_loc (MSG_NOTE
, vect_location
, "=== vectorizable_call ==="
2314 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
2320 if (dump_enabled_p ())
2321 dump_printf_loc (MSG_NOTE
, vect_location
, "transform call.\n");
2324 scalar_dest
= gimple_call_lhs (stmt
);
2325 vec_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
2327 prev_stmt_info
= NULL
;
2331 for (j
= 0; j
< ncopies
; ++j
)
2333 /* Build argument list for the vectorized call. */
2335 vargs
.create (nargs
);
2341 auto_vec
<vec
<tree
> > vec_defs (nargs
);
2342 vec
<tree
> vec_oprnds0
;
2344 for (i
= 0; i
< nargs
; i
++)
2345 vargs
.quick_push (gimple_call_arg (stmt
, i
));
2346 vect_get_slp_defs (vargs
, slp_node
, &vec_defs
, -1);
2347 vec_oprnds0
= vec_defs
[0];
2349 /* Arguments are ready. Create the new vector stmt. */
2350 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_oprnd0
)
2353 for (k
= 0; k
< nargs
; k
++)
2355 vec
<tree
> vec_oprndsk
= vec_defs
[k
];
2356 vargs
[k
] = vec_oprndsk
[i
];
2358 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2359 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2360 gimple_call_set_lhs (new_stmt
, new_temp
);
2361 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2362 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
2365 for (i
= 0; i
< nargs
; i
++)
2367 vec
<tree
> vec_oprndsi
= vec_defs
[i
];
2368 vec_oprndsi
.release ();
2373 for (i
= 0; i
< nargs
; i
++)
2375 op
= gimple_call_arg (stmt
, i
);
2378 = vect_get_vec_def_for_operand (op
, stmt
);
2381 vec_oprnd0
= gimple_call_arg (new_stmt
, i
);
2383 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2386 vargs
.quick_push (vec_oprnd0
);
2389 if (gimple_call_internal_p (stmt
)
2390 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
)
2392 tree
*v
= XALLOCAVEC (tree
, nunits_out
);
2394 for (k
= 0; k
< nunits_out
; ++k
)
2395 v
[k
] = build_int_cst (unsigned_type_node
, j
* nunits_out
+ k
);
2396 tree cst
= build_vector (vectype_out
, v
);
2398 = vect_get_new_ssa_name (vectype_out
, vect_simple_var
, "cst_");
2399 gimple
*init_stmt
= gimple_build_assign (new_var
, cst
);
2400 vect_init_vector_1 (stmt
, init_stmt
, NULL
);
2401 new_temp
= make_ssa_name (vec_dest
);
2402 new_stmt
= gimple_build_assign (new_temp
, new_var
);
2406 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2407 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2408 gimple_call_set_lhs (new_stmt
, new_temp
);
2410 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2413 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2415 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2417 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2423 for (j
= 0; j
< ncopies
; ++j
)
2425 /* Build argument list for the vectorized call. */
2427 vargs
.create (nargs
* 2);
2433 auto_vec
<vec
<tree
> > vec_defs (nargs
);
2434 vec
<tree
> vec_oprnds0
;
2436 for (i
= 0; i
< nargs
; i
++)
2437 vargs
.quick_push (gimple_call_arg (stmt
, i
));
2438 vect_get_slp_defs (vargs
, slp_node
, &vec_defs
, -1);
2439 vec_oprnds0
= vec_defs
[0];
2441 /* Arguments are ready. Create the new vector stmt. */
2442 for (i
= 0; vec_oprnds0
.iterate (i
, &vec_oprnd0
); i
+= 2)
2446 for (k
= 0; k
< nargs
; k
++)
2448 vec
<tree
> vec_oprndsk
= vec_defs
[k
];
2449 vargs
.quick_push (vec_oprndsk
[i
]);
2450 vargs
.quick_push (vec_oprndsk
[i
+ 1]);
2452 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2453 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2454 gimple_call_set_lhs (new_stmt
, new_temp
);
2455 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2456 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
2459 for (i
= 0; i
< nargs
; i
++)
2461 vec
<tree
> vec_oprndsi
= vec_defs
[i
];
2462 vec_oprndsi
.release ();
2467 for (i
= 0; i
< nargs
; i
++)
2469 op
= gimple_call_arg (stmt
, i
);
2473 = vect_get_vec_def_for_operand (op
, stmt
);
2475 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2479 vec_oprnd1
= gimple_call_arg (new_stmt
, 2*i
+ 1);
2481 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd1
);
2483 = vect_get_vec_def_for_stmt_copy (dt
[i
], vec_oprnd0
);
2486 vargs
.quick_push (vec_oprnd0
);
2487 vargs
.quick_push (vec_oprnd1
);
2490 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
2491 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2492 gimple_call_set_lhs (new_stmt
, new_temp
);
2493 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2496 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
2498 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2500 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2503 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
2508 /* No current target implements this case. */
2514 /* The call in STMT might prevent it from being removed in dce.
2515 We however cannot remove it here, due to the way the ssa name
2516 it defines is mapped to the new definition. So just replace
2517 rhs of the statement with something harmless. */
2522 type
= TREE_TYPE (scalar_dest
);
2523 if (is_pattern_stmt_p (stmt_info
))
2524 lhs
= gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info
));
2526 lhs
= gimple_call_lhs (stmt
);
2528 if (gimple_call_internal_p (stmt
)
2529 && gimple_call_internal_fn (stmt
) == IFN_GOMP_SIMD_LANE
)
2531 /* Replace uses of the lhs of GOMP_SIMD_LANE call outside the loop
2532 with vf - 1 rather than 0, that is the last iteration of the
2534 imm_use_iterator iter
;
2535 use_operand_p use_p
;
2537 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, lhs
)
2539 basic_block use_bb
= gimple_bb (use_stmt
);
2541 && !flow_bb_inside_loop_p (LOOP_VINFO_LOOP (loop_vinfo
), use_bb
))
2543 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
2544 SET_USE (use_p
, build_int_cst (TREE_TYPE (lhs
),
2545 ncopies
* nunits_out
- 1));
2546 update_stmt (use_stmt
);
2551 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (type
));
2552 set_vinfo_for_stmt (new_stmt
, stmt_info
);
2553 set_vinfo_for_stmt (stmt
, NULL
);
2554 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
2555 gsi_replace (gsi
, new_stmt
, false);
2561 struct simd_call_arg_info
2565 enum vect_def_type dt
;
2566 HOST_WIDE_INT linear_step
;
2568 bool simd_lane_linear
;
2571 /* Helper function of vectorizable_simd_clone_call. If OP, an SSA_NAME,
2572 is linear within simd lane (but not within whole loop), note it in
2576 vect_simd_lane_linear (tree op
, struct loop
*loop
,
2577 struct simd_call_arg_info
*arginfo
)
2579 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
2581 if (!is_gimple_assign (def_stmt
)
2582 || gimple_assign_rhs_code (def_stmt
) != POINTER_PLUS_EXPR
2583 || !is_gimple_min_invariant (gimple_assign_rhs1 (def_stmt
)))
2586 tree base
= gimple_assign_rhs1 (def_stmt
);
2587 HOST_WIDE_INT linear_step
= 0;
2588 tree v
= gimple_assign_rhs2 (def_stmt
);
2589 while (TREE_CODE (v
) == SSA_NAME
)
2592 def_stmt
= SSA_NAME_DEF_STMT (v
);
2593 if (is_gimple_assign (def_stmt
))
2594 switch (gimple_assign_rhs_code (def_stmt
))
2597 t
= gimple_assign_rhs2 (def_stmt
);
2598 if (linear_step
|| TREE_CODE (t
) != INTEGER_CST
)
2600 base
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (base
), base
, t
);
2601 v
= gimple_assign_rhs1 (def_stmt
);
2604 t
= gimple_assign_rhs2 (def_stmt
);
2605 if (linear_step
|| !tree_fits_shwi_p (t
) || integer_zerop (t
))
2607 linear_step
= tree_to_shwi (t
);
2608 v
= gimple_assign_rhs1 (def_stmt
);
2611 t
= gimple_assign_rhs1 (def_stmt
);
2612 if (TREE_CODE (TREE_TYPE (t
)) != INTEGER_TYPE
2613 || (TYPE_PRECISION (TREE_TYPE (v
))
2614 < TYPE_PRECISION (TREE_TYPE (t
))))
2623 else if (is_gimple_call (def_stmt
)
2624 && gimple_call_internal_p (def_stmt
)
2625 && gimple_call_internal_fn (def_stmt
) == IFN_GOMP_SIMD_LANE
2627 && TREE_CODE (gimple_call_arg (def_stmt
, 0)) == SSA_NAME
2628 && (SSA_NAME_VAR (gimple_call_arg (def_stmt
, 0))
2633 arginfo
->linear_step
= linear_step
;
2635 arginfo
->simd_lane_linear
= true;
2641 /* Function vectorizable_simd_clone_call.
2643 Check if STMT performs a function call that can be vectorized
2644 by calling a simd clone of the function.
2645 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2646 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2647 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2650 vectorizable_simd_clone_call (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
2651 gimple
**vec_stmt
, slp_tree slp_node
)
2656 tree vec_oprnd0
= NULL_TREE
;
2657 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
), prev_stmt_info
;
2659 unsigned int nunits
;
2660 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2661 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
2662 vec_info
*vinfo
= stmt_info
->vinfo
;
2663 struct loop
*loop
= loop_vinfo
? LOOP_VINFO_LOOP (loop_vinfo
) : NULL
;
2664 tree fndecl
, new_temp
;
2666 gimple
*new_stmt
= NULL
;
2668 vec
<simd_call_arg_info
> arginfo
= vNULL
;
2669 vec
<tree
> vargs
= vNULL
;
2671 tree lhs
, rtype
, ratype
;
2672 vec
<constructor_elt
, va_gc
> *ret_ctor_elts
;
2674 /* Is STMT a vectorizable call? */
2675 if (!is_gimple_call (stmt
))
2678 fndecl
= gimple_call_fndecl (stmt
);
2679 if (fndecl
== NULL_TREE
)
2682 struct cgraph_node
*node
= cgraph_node::get (fndecl
);
2683 if (node
== NULL
|| node
->simd_clones
== NULL
)
2686 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
2689 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
2692 if (gimple_call_lhs (stmt
)
2693 && TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
2696 gcc_checking_assert (!stmt_can_throw_internal (stmt
));
2698 vectype
= STMT_VINFO_VECTYPE (stmt_info
);
2700 if (loop_vinfo
&& nested_in_vect_loop_p (loop
, stmt
))
2704 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
2707 /* Process function arguments. */
2708 nargs
= gimple_call_num_args (stmt
);
2710 /* Bail out if the function has zero arguments. */
2714 arginfo
.create (nargs
);
2716 for (i
= 0; i
< nargs
; i
++)
2718 simd_call_arg_info thisarginfo
;
2721 thisarginfo
.linear_step
= 0;
2722 thisarginfo
.align
= 0;
2723 thisarginfo
.op
= NULL_TREE
;
2724 thisarginfo
.simd_lane_linear
= false;
2726 op
= gimple_call_arg (stmt
, i
);
2727 if (!vect_is_simple_use (op
, vinfo
, &def_stmt
, &thisarginfo
.dt
,
2728 &thisarginfo
.vectype
)
2729 || thisarginfo
.dt
== vect_uninitialized_def
)
2731 if (dump_enabled_p ())
2732 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
2733 "use not simple.\n");
2738 if (thisarginfo
.dt
== vect_constant_def
2739 || thisarginfo
.dt
== vect_external_def
)
2740 gcc_assert (thisarginfo
.vectype
== NULL_TREE
);
2742 gcc_assert (thisarginfo
.vectype
!= NULL_TREE
);
2744 /* For linear arguments, the analyze phase should have saved
2745 the base and step in STMT_VINFO_SIMD_CLONE_INFO. */
2746 if (i
* 3 + 4 <= STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).length ()
2747 && STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 2])
2749 gcc_assert (vec_stmt
);
2750 thisarginfo
.linear_step
2751 = tree_to_shwi (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 2]);
2753 = STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 1];
2754 thisarginfo
.simd_lane_linear
2755 = (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 3]
2756 == boolean_true_node
);
2757 /* If loop has been peeled for alignment, we need to adjust it. */
2758 tree n1
= LOOP_VINFO_NITERS_UNCHANGED (loop_vinfo
);
2759 tree n2
= LOOP_VINFO_NITERS (loop_vinfo
);
2760 if (n1
!= n2
&& !thisarginfo
.simd_lane_linear
)
2762 tree bias
= fold_build2 (MINUS_EXPR
, TREE_TYPE (n1
), n1
, n2
);
2763 tree step
= STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[i
* 3 + 2];
2764 tree opt
= TREE_TYPE (thisarginfo
.op
);
2765 bias
= fold_convert (TREE_TYPE (step
), bias
);
2766 bias
= fold_build2 (MULT_EXPR
, TREE_TYPE (step
), bias
, step
);
2768 = fold_build2 (POINTER_TYPE_P (opt
)
2769 ? POINTER_PLUS_EXPR
: PLUS_EXPR
, opt
,
2770 thisarginfo
.op
, bias
);
2774 && thisarginfo
.dt
!= vect_constant_def
2775 && thisarginfo
.dt
!= vect_external_def
2777 && TREE_CODE (op
) == SSA_NAME
2778 && simple_iv (loop
, loop_containing_stmt (stmt
), op
,
2780 && tree_fits_shwi_p (iv
.step
))
2782 thisarginfo
.linear_step
= tree_to_shwi (iv
.step
);
2783 thisarginfo
.op
= iv
.base
;
2785 else if ((thisarginfo
.dt
== vect_constant_def
2786 || thisarginfo
.dt
== vect_external_def
)
2787 && POINTER_TYPE_P (TREE_TYPE (op
)))
2788 thisarginfo
.align
= get_pointer_alignment (op
) / BITS_PER_UNIT
;
2789 /* Addresses of array elements indexed by GOMP_SIMD_LANE are
2791 if (POINTER_TYPE_P (TREE_TYPE (op
))
2792 && !thisarginfo
.linear_step
2794 && thisarginfo
.dt
!= vect_constant_def
2795 && thisarginfo
.dt
!= vect_external_def
2798 && TREE_CODE (op
) == SSA_NAME
)
2799 vect_simd_lane_linear (op
, loop
, &thisarginfo
);
2801 arginfo
.quick_push (thisarginfo
);
2804 unsigned int badness
= 0;
2805 struct cgraph_node
*bestn
= NULL
;
2806 if (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).exists ())
2807 bestn
= cgraph_node::get (STMT_VINFO_SIMD_CLONE_INFO (stmt_info
)[0]);
2809 for (struct cgraph_node
*n
= node
->simd_clones
; n
!= NULL
;
2810 n
= n
->simdclone
->next_clone
)
2812 unsigned int this_badness
= 0;
2813 if (n
->simdclone
->simdlen
2814 > (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
2815 || n
->simdclone
->nargs
!= nargs
)
2817 if (n
->simdclone
->simdlen
2818 < (unsigned) LOOP_VINFO_VECT_FACTOR (loop_vinfo
))
2819 this_badness
+= (exact_log2 (LOOP_VINFO_VECT_FACTOR (loop_vinfo
))
2820 - exact_log2 (n
->simdclone
->simdlen
)) * 1024;
2821 if (n
->simdclone
->inbranch
)
2822 this_badness
+= 2048;
2823 int target_badness
= targetm
.simd_clone
.usable (n
);
2824 if (target_badness
< 0)
2826 this_badness
+= target_badness
* 512;
2827 /* FORNOW: Have to add code to add the mask argument. */
2828 if (n
->simdclone
->inbranch
)
2830 for (i
= 0; i
< nargs
; i
++)
2832 switch (n
->simdclone
->args
[i
].arg_type
)
2834 case SIMD_CLONE_ARG_TYPE_VECTOR
:
2835 if (!useless_type_conversion_p
2836 (n
->simdclone
->args
[i
].orig_type
,
2837 TREE_TYPE (gimple_call_arg (stmt
, i
))))
2839 else if (arginfo
[i
].dt
== vect_constant_def
2840 || arginfo
[i
].dt
== vect_external_def
2841 || arginfo
[i
].linear_step
)
2844 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
2845 if (arginfo
[i
].dt
!= vect_constant_def
2846 && arginfo
[i
].dt
!= vect_external_def
)
2849 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
2850 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
2851 if (arginfo
[i
].dt
== vect_constant_def
2852 || arginfo
[i
].dt
== vect_external_def
2853 || (arginfo
[i
].linear_step
2854 != n
->simdclone
->args
[i
].linear_step
))
2857 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
2858 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
2859 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
2860 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
2861 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
2862 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
2866 case SIMD_CLONE_ARG_TYPE_MASK
:
2869 if (i
== (size_t) -1)
2871 if (n
->simdclone
->args
[i
].alignment
> arginfo
[i
].align
)
2876 if (arginfo
[i
].align
)
2877 this_badness
+= (exact_log2 (arginfo
[i
].align
)
2878 - exact_log2 (n
->simdclone
->args
[i
].alignment
));
2880 if (i
== (size_t) -1)
2882 if (bestn
== NULL
|| this_badness
< badness
)
2885 badness
= this_badness
;
2895 for (i
= 0; i
< nargs
; i
++)
2896 if ((arginfo
[i
].dt
== vect_constant_def
2897 || arginfo
[i
].dt
== vect_external_def
)
2898 && bestn
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
2901 = get_vectype_for_scalar_type (TREE_TYPE (gimple_call_arg (stmt
,
2903 if (arginfo
[i
].vectype
== NULL
2904 || (TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
)
2905 > bestn
->simdclone
->simdlen
))
2912 fndecl
= bestn
->decl
;
2913 nunits
= bestn
->simdclone
->simdlen
;
2914 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
2916 /* If the function isn't const, only allow it in simd loops where user
2917 has asserted that at least nunits consecutive iterations can be
2918 performed using SIMD instructions. */
2919 if ((loop
== NULL
|| (unsigned) loop
->safelen
< nunits
)
2920 && gimple_vuse (stmt
))
2926 /* Sanity check: make sure that at least one copy of the vectorized stmt
2927 needs to be generated. */
2928 gcc_assert (ncopies
>= 1);
2930 if (!vec_stmt
) /* transformation not required. */
2932 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (bestn
->decl
);
2933 for (i
= 0; i
< nargs
; i
++)
2934 if (bestn
->simdclone
->args
[i
].arg_type
2935 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
2937 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_grow_cleared (i
* 3
2939 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (arginfo
[i
].op
);
2940 tree lst
= POINTER_TYPE_P (TREE_TYPE (arginfo
[i
].op
))
2941 ? size_type_node
: TREE_TYPE (arginfo
[i
].op
);
2942 tree ls
= build_int_cst (lst
, arginfo
[i
].linear_step
);
2943 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (ls
);
2944 tree sll
= arginfo
[i
].simd_lane_linear
2945 ? boolean_true_node
: boolean_false_node
;
2946 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).safe_push (sll
);
2948 STMT_VINFO_TYPE (stmt_info
) = call_simd_clone_vec_info_type
;
2949 if (dump_enabled_p ())
2950 dump_printf_loc (MSG_NOTE
, vect_location
,
2951 "=== vectorizable_simd_clone_call ===\n");
2952 /* vect_model_simple_cost (stmt_info, ncopies, dt, NULL, NULL); */
2959 if (dump_enabled_p ())
2960 dump_printf_loc (MSG_NOTE
, vect_location
, "transform call.\n");
2963 scalar_dest
= gimple_call_lhs (stmt
);
2964 vec_dest
= NULL_TREE
;
2969 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
2970 rtype
= TREE_TYPE (TREE_TYPE (fndecl
));
2971 if (TREE_CODE (rtype
) == ARRAY_TYPE
)
2974 rtype
= TREE_TYPE (ratype
);
2978 prev_stmt_info
= NULL
;
2979 for (j
= 0; j
< ncopies
; ++j
)
2981 /* Build argument list for the vectorized call. */
2983 vargs
.create (nargs
);
2987 for (i
= 0; i
< nargs
; i
++)
2989 unsigned int k
, l
, m
, o
;
2991 op
= gimple_call_arg (stmt
, i
);
2992 switch (bestn
->simdclone
->args
[i
].arg_type
)
2994 case SIMD_CLONE_ARG_TYPE_VECTOR
:
2995 atype
= bestn
->simdclone
->args
[i
].vector_type
;
2996 o
= nunits
/ TYPE_VECTOR_SUBPARTS (atype
);
2997 for (m
= j
* o
; m
< (j
+ 1) * o
; m
++)
2999 if (TYPE_VECTOR_SUBPARTS (atype
)
3000 < TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
))
3002 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (atype
));
3003 k
= (TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
)
3004 / TYPE_VECTOR_SUBPARTS (atype
));
3005 gcc_assert ((k
& (k
- 1)) == 0);
3008 = vect_get_vec_def_for_operand (op
, stmt
);
3011 vec_oprnd0
= arginfo
[i
].op
;
3012 if ((m
& (k
- 1)) == 0)
3014 = vect_get_vec_def_for_stmt_copy (arginfo
[i
].dt
,
3017 arginfo
[i
].op
= vec_oprnd0
;
3019 = build3 (BIT_FIELD_REF
, atype
, vec_oprnd0
,
3021 bitsize_int ((m
& (k
- 1)) * prec
));
3023 = gimple_build_assign (make_ssa_name (atype
),
3025 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3026 vargs
.safe_push (gimple_assign_lhs (new_stmt
));
3030 k
= (TYPE_VECTOR_SUBPARTS (atype
)
3031 / TYPE_VECTOR_SUBPARTS (arginfo
[i
].vectype
));
3032 gcc_assert ((k
& (k
- 1)) == 0);
3033 vec
<constructor_elt
, va_gc
> *ctor_elts
;
3035 vec_alloc (ctor_elts
, k
);
3038 for (l
= 0; l
< k
; l
++)
3040 if (m
== 0 && l
== 0)
3042 = vect_get_vec_def_for_operand (op
, stmt
);
3045 = vect_get_vec_def_for_stmt_copy (arginfo
[i
].dt
,
3047 arginfo
[i
].op
= vec_oprnd0
;
3050 CONSTRUCTOR_APPEND_ELT (ctor_elts
, NULL_TREE
,
3054 vargs
.safe_push (vec_oprnd0
);
3057 vec_oprnd0
= build_constructor (atype
, ctor_elts
);
3059 = gimple_build_assign (make_ssa_name (atype
),
3061 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3062 vargs
.safe_push (gimple_assign_lhs (new_stmt
));
3067 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
3068 vargs
.safe_push (op
);
3070 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
3075 = force_gimple_operand (arginfo
[i
].op
, &stmts
, true,
3080 edge pe
= loop_preheader_edge (loop
);
3081 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, stmts
);
3082 gcc_assert (!new_bb
);
3084 if (arginfo
[i
].simd_lane_linear
)
3086 vargs
.safe_push (arginfo
[i
].op
);
3089 tree phi_res
= copy_ssa_name (op
);
3090 gphi
*new_phi
= create_phi_node (phi_res
, loop
->header
);
3091 set_vinfo_for_stmt (new_phi
,
3092 new_stmt_vec_info (new_phi
, loop_vinfo
));
3093 add_phi_arg (new_phi
, arginfo
[i
].op
,
3094 loop_preheader_edge (loop
), UNKNOWN_LOCATION
);
3096 = POINTER_TYPE_P (TREE_TYPE (op
))
3097 ? POINTER_PLUS_EXPR
: PLUS_EXPR
;
3098 tree type
= POINTER_TYPE_P (TREE_TYPE (op
))
3099 ? sizetype
: TREE_TYPE (op
);
3101 = wi::mul (bestn
->simdclone
->args
[i
].linear_step
,
3103 tree tcst
= wide_int_to_tree (type
, cst
);
3104 tree phi_arg
= copy_ssa_name (op
);
3106 = gimple_build_assign (phi_arg
, code
, phi_res
, tcst
);
3107 gimple_stmt_iterator si
= gsi_after_labels (loop
->header
);
3108 gsi_insert_after (&si
, new_stmt
, GSI_NEW_STMT
);
3109 set_vinfo_for_stmt (new_stmt
,
3110 new_stmt_vec_info (new_stmt
, loop_vinfo
));
3111 add_phi_arg (new_phi
, phi_arg
, loop_latch_edge (loop
),
3113 arginfo
[i
].op
= phi_res
;
3114 vargs
.safe_push (phi_res
);
3119 = POINTER_TYPE_P (TREE_TYPE (op
))
3120 ? POINTER_PLUS_EXPR
: PLUS_EXPR
;
3121 tree type
= POINTER_TYPE_P (TREE_TYPE (op
))
3122 ? sizetype
: TREE_TYPE (op
);
3124 = wi::mul (bestn
->simdclone
->args
[i
].linear_step
,
3126 tree tcst
= wide_int_to_tree (type
, cst
);
3127 new_temp
= make_ssa_name (TREE_TYPE (op
));
3128 new_stmt
= gimple_build_assign (new_temp
, code
,
3129 arginfo
[i
].op
, tcst
);
3130 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3131 vargs
.safe_push (new_temp
);
3134 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
3135 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
3136 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
3137 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
3143 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
3146 gcc_assert (ratype
|| TYPE_VECTOR_SUBPARTS (rtype
) == nunits
);
3148 new_temp
= create_tmp_var (ratype
);
3149 else if (TYPE_VECTOR_SUBPARTS (vectype
)
3150 == TYPE_VECTOR_SUBPARTS (rtype
))
3151 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3153 new_temp
= make_ssa_name (rtype
, new_stmt
);
3154 gimple_call_set_lhs (new_stmt
, new_temp
);
3156 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3160 if (TYPE_VECTOR_SUBPARTS (vectype
) < nunits
)
3163 unsigned int prec
= GET_MODE_BITSIZE (TYPE_MODE (vectype
));
3164 k
= nunits
/ TYPE_VECTOR_SUBPARTS (vectype
);
3165 gcc_assert ((k
& (k
- 1)) == 0);
3166 for (l
= 0; l
< k
; l
++)
3171 t
= build_fold_addr_expr (new_temp
);
3172 t
= build2 (MEM_REF
, vectype
, t
,
3173 build_int_cst (TREE_TYPE (t
),
3174 l
* prec
/ BITS_PER_UNIT
));
3177 t
= build3 (BIT_FIELD_REF
, vectype
, new_temp
,
3178 size_int (prec
), bitsize_int (l
* prec
));
3180 = gimple_build_assign (make_ssa_name (vectype
), t
);
3181 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3182 if (j
== 0 && l
== 0)
3183 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3185 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3187 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3192 tree clobber
= build_constructor (ratype
, NULL
);
3193 TREE_THIS_VOLATILE (clobber
) = 1;
3194 new_stmt
= gimple_build_assign (new_temp
, clobber
);
3195 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3199 else if (TYPE_VECTOR_SUBPARTS (vectype
) > nunits
)
3201 unsigned int k
= (TYPE_VECTOR_SUBPARTS (vectype
)
3202 / TYPE_VECTOR_SUBPARTS (rtype
));
3203 gcc_assert ((k
& (k
- 1)) == 0);
3204 if ((j
& (k
- 1)) == 0)
3205 vec_alloc (ret_ctor_elts
, k
);
3208 unsigned int m
, o
= nunits
/ TYPE_VECTOR_SUBPARTS (rtype
);
3209 for (m
= 0; m
< o
; m
++)
3211 tree tem
= build4 (ARRAY_REF
, rtype
, new_temp
,
3212 size_int (m
), NULL_TREE
, NULL_TREE
);
3214 = gimple_build_assign (make_ssa_name (rtype
), tem
);
3215 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3216 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts
, NULL_TREE
,
3217 gimple_assign_lhs (new_stmt
));
3219 tree clobber
= build_constructor (ratype
, NULL
);
3220 TREE_THIS_VOLATILE (clobber
) = 1;
3221 new_stmt
= gimple_build_assign (new_temp
, clobber
);
3222 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3225 CONSTRUCTOR_APPEND_ELT (ret_ctor_elts
, NULL_TREE
, new_temp
);
3226 if ((j
& (k
- 1)) != k
- 1)
3228 vec_oprnd0
= build_constructor (vectype
, ret_ctor_elts
);
3230 = gimple_build_assign (make_ssa_name (vec_dest
), vec_oprnd0
);
3231 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3233 if ((unsigned) j
== k
- 1)
3234 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3236 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3238 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3243 tree t
= build_fold_addr_expr (new_temp
);
3244 t
= build2 (MEM_REF
, vectype
, t
,
3245 build_int_cst (TREE_TYPE (t
), 0));
3247 = gimple_build_assign (make_ssa_name (vec_dest
), t
);
3248 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3249 tree clobber
= build_constructor (ratype
, NULL
);
3250 TREE_THIS_VOLATILE (clobber
) = 1;
3251 vect_finish_stmt_generation (stmt
,
3252 gimple_build_assign (new_temp
,
3258 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3260 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3262 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3267 /* The call in STMT might prevent it from being removed in dce.
3268 We however cannot remove it here, due to the way the ssa name
3269 it defines is mapped to the new definition. So just replace
3270 rhs of the statement with something harmless. */
3277 type
= TREE_TYPE (scalar_dest
);
3278 if (is_pattern_stmt_p (stmt_info
))
3279 lhs
= gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info
));
3281 lhs
= gimple_call_lhs (stmt
);
3282 new_stmt
= gimple_build_assign (lhs
, build_zero_cst (type
));
3285 new_stmt
= gimple_build_nop ();
3286 set_vinfo_for_stmt (new_stmt
, stmt_info
);
3287 set_vinfo_for_stmt (stmt
, NULL
);
3288 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
3289 gsi_replace (gsi
, new_stmt
, true);
3290 unlink_stmt_vdef (stmt
);
3296 /* Function vect_gen_widened_results_half
3298 Create a vector stmt whose code, type, number of arguments, and result
3299 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
3300 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at BSI.
3301 In the case that CODE is a CALL_EXPR, this means that a call to DECL
3302 needs to be created (DECL is a function-decl of a target-builtin).
3303 STMT is the original scalar stmt that we are vectorizing. */
3306 vect_gen_widened_results_half (enum tree_code code
,
3308 tree vec_oprnd0
, tree vec_oprnd1
, int op_type
,
3309 tree vec_dest
, gimple_stmt_iterator
*gsi
,
3315 /* Generate half of the widened result: */
3316 if (code
== CALL_EXPR
)
3318 /* Target specific support */
3319 if (op_type
== binary_op
)
3320 new_stmt
= gimple_build_call (decl
, 2, vec_oprnd0
, vec_oprnd1
);
3322 new_stmt
= gimple_build_call (decl
, 1, vec_oprnd0
);
3323 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3324 gimple_call_set_lhs (new_stmt
, new_temp
);
3328 /* Generic support */
3329 gcc_assert (op_type
== TREE_CODE_LENGTH (code
));
3330 if (op_type
!= binary_op
)
3332 new_stmt
= gimple_build_assign (vec_dest
, code
, vec_oprnd0
, vec_oprnd1
);
3333 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3334 gimple_assign_set_lhs (new_stmt
, new_temp
);
3336 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3342 /* Get vectorized definitions for loop-based vectorization. For the first
3343 operand we call vect_get_vec_def_for_operand() (with OPRND containing
3344 scalar operand), and for the rest we get a copy with
3345 vect_get_vec_def_for_stmt_copy() using the previous vector definition
3346 (stored in OPRND). See vect_get_vec_def_for_stmt_copy() for details.
3347 The vectors are collected into VEC_OPRNDS. */
3350 vect_get_loop_based_defs (tree
*oprnd
, gimple
*stmt
, enum vect_def_type dt
,
3351 vec
<tree
> *vec_oprnds
, int multi_step_cvt
)
3355 /* Get first vector operand. */
3356 /* All the vector operands except the very first one (that is scalar oprnd)
3358 if (TREE_CODE (TREE_TYPE (*oprnd
)) != VECTOR_TYPE
)
3359 vec_oprnd
= vect_get_vec_def_for_operand (*oprnd
, stmt
);
3361 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, *oprnd
);
3363 vec_oprnds
->quick_push (vec_oprnd
);
3365 /* Get second vector operand. */
3366 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, vec_oprnd
);
3367 vec_oprnds
->quick_push (vec_oprnd
);
3371 /* For conversion in multiple steps, continue to get operands
3374 vect_get_loop_based_defs (oprnd
, stmt
, dt
, vec_oprnds
, multi_step_cvt
- 1);
3378 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
3379 For multi-step conversions store the resulting vectors and call the function
3383 vect_create_vectorized_demotion_stmts (vec
<tree
> *vec_oprnds
,
3384 int multi_step_cvt
, gimple
*stmt
,
3386 gimple_stmt_iterator
*gsi
,
3387 slp_tree slp_node
, enum tree_code code
,
3388 stmt_vec_info
*prev_stmt_info
)
3391 tree vop0
, vop1
, new_tmp
, vec_dest
;
3393 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3395 vec_dest
= vec_dsts
.pop ();
3397 for (i
= 0; i
< vec_oprnds
->length (); i
+= 2)
3399 /* Create demotion operation. */
3400 vop0
= (*vec_oprnds
)[i
];
3401 vop1
= (*vec_oprnds
)[i
+ 1];
3402 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
);
3403 new_tmp
= make_ssa_name (vec_dest
, new_stmt
);
3404 gimple_assign_set_lhs (new_stmt
, new_tmp
);
3405 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3408 /* Store the resulting vector for next recursive call. */
3409 (*vec_oprnds
)[i
/2] = new_tmp
;
3412 /* This is the last step of the conversion sequence. Store the
3413 vectors in SLP_NODE or in vector info of the scalar statement
3414 (or in STMT_VINFO_RELATED_STMT chain). */
3416 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3419 if (!*prev_stmt_info
)
3420 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
3422 STMT_VINFO_RELATED_STMT (*prev_stmt_info
) = new_stmt
;
3424 *prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3429 /* For multi-step demotion operations we first generate demotion operations
3430 from the source type to the intermediate types, and then combine the
3431 results (stored in VEC_OPRNDS) in demotion operation to the destination
3435 /* At each level of recursion we have half of the operands we had at the
3437 vec_oprnds
->truncate ((i
+1)/2);
3438 vect_create_vectorized_demotion_stmts (vec_oprnds
, multi_step_cvt
- 1,
3439 stmt
, vec_dsts
, gsi
, slp_node
,
3440 VEC_PACK_TRUNC_EXPR
,
3444 vec_dsts
.quick_push (vec_dest
);
3448 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
3449 and VEC_OPRNDS1 (for binary operations). For multi-step conversions store
3450 the resulting vectors and call the function recursively. */
3453 vect_create_vectorized_promotion_stmts (vec
<tree
> *vec_oprnds0
,
3454 vec
<tree
> *vec_oprnds1
,
3455 gimple
*stmt
, tree vec_dest
,
3456 gimple_stmt_iterator
*gsi
,
3457 enum tree_code code1
,
3458 enum tree_code code2
, tree decl1
,
3459 tree decl2
, int op_type
)
3462 tree vop0
, vop1
, new_tmp1
, new_tmp2
;
3463 gimple
*new_stmt1
, *new_stmt2
;
3464 vec
<tree
> vec_tmp
= vNULL
;
3466 vec_tmp
.create (vec_oprnds0
->length () * 2);
3467 FOR_EACH_VEC_ELT (*vec_oprnds0
, i
, vop0
)
3469 if (op_type
== binary_op
)
3470 vop1
= (*vec_oprnds1
)[i
];
3474 /* Generate the two halves of promotion operation. */
3475 new_stmt1
= vect_gen_widened_results_half (code1
, decl1
, vop0
, vop1
,
3476 op_type
, vec_dest
, gsi
, stmt
);
3477 new_stmt2
= vect_gen_widened_results_half (code2
, decl2
, vop0
, vop1
,
3478 op_type
, vec_dest
, gsi
, stmt
);
3479 if (is_gimple_call (new_stmt1
))
3481 new_tmp1
= gimple_call_lhs (new_stmt1
);
3482 new_tmp2
= gimple_call_lhs (new_stmt2
);
3486 new_tmp1
= gimple_assign_lhs (new_stmt1
);
3487 new_tmp2
= gimple_assign_lhs (new_stmt2
);
3490 /* Store the results for the next step. */
3491 vec_tmp
.quick_push (new_tmp1
);
3492 vec_tmp
.quick_push (new_tmp2
);
3495 vec_oprnds0
->release ();
3496 *vec_oprnds0
= vec_tmp
;
3500 /* Check if STMT performs a conversion operation, that can be vectorized.
3501 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3502 stmt to replace it, put it in VEC_STMT, and insert it at GSI.
3503 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3506 vectorizable_conversion (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
3507 gimple
**vec_stmt
, slp_tree slp_node
)
3511 tree op0
, op1
= NULL_TREE
;
3512 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
3513 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3514 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
3515 enum tree_code code
, code1
= ERROR_MARK
, code2
= ERROR_MARK
;
3516 enum tree_code codecvt1
= ERROR_MARK
, codecvt2
= ERROR_MARK
;
3517 tree decl1
= NULL_TREE
, decl2
= NULL_TREE
;
3520 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
3521 gimple
*new_stmt
= NULL
;
3522 stmt_vec_info prev_stmt_info
;
3525 tree vectype_out
, vectype_in
;
3527 tree lhs_type
, rhs_type
;
3528 enum { NARROW
, NONE
, WIDEN
} modifier
;
3529 vec
<tree
> vec_oprnds0
= vNULL
;
3530 vec
<tree
> vec_oprnds1
= vNULL
;
3532 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
3533 vec_info
*vinfo
= stmt_info
->vinfo
;
3534 int multi_step_cvt
= 0;
3535 vec
<tree
> vec_dsts
= vNULL
;
3536 vec
<tree
> interm_types
= vNULL
;
3537 tree last_oprnd
, intermediate_type
, cvt_type
= NULL_TREE
;
3539 machine_mode rhs_mode
;
3540 unsigned short fltsz
;
3542 /* Is STMT a vectorizable conversion? */
3544 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
3547 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
3550 if (!is_gimple_assign (stmt
))
3553 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
3556 code
= gimple_assign_rhs_code (stmt
);
3557 if (!CONVERT_EXPR_CODE_P (code
)
3558 && code
!= FIX_TRUNC_EXPR
3559 && code
!= FLOAT_EXPR
3560 && code
!= WIDEN_MULT_EXPR
3561 && code
!= WIDEN_LSHIFT_EXPR
)
3564 op_type
= TREE_CODE_LENGTH (code
);
3566 /* Check types of lhs and rhs. */
3567 scalar_dest
= gimple_assign_lhs (stmt
);
3568 lhs_type
= TREE_TYPE (scalar_dest
);
3569 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
3571 op0
= gimple_assign_rhs1 (stmt
);
3572 rhs_type
= TREE_TYPE (op0
);
3574 if ((code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
3575 && !((INTEGRAL_TYPE_P (lhs_type
)
3576 && INTEGRAL_TYPE_P (rhs_type
))
3577 || (SCALAR_FLOAT_TYPE_P (lhs_type
)
3578 && SCALAR_FLOAT_TYPE_P (rhs_type
))))
3581 if (!VECTOR_BOOLEAN_TYPE_P (vectype_out
)
3582 && ((INTEGRAL_TYPE_P (lhs_type
)
3583 && (TYPE_PRECISION (lhs_type
)
3584 != GET_MODE_PRECISION (TYPE_MODE (lhs_type
))))
3585 || (INTEGRAL_TYPE_P (rhs_type
)
3586 && (TYPE_PRECISION (rhs_type
)
3587 != GET_MODE_PRECISION (TYPE_MODE (rhs_type
))))))
3589 if (dump_enabled_p ())
3590 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3591 "type conversion to/from bit-precision unsupported."
3596 /* Check the operands of the operation. */
3597 if (!vect_is_simple_use (op0
, vinfo
, &def_stmt
, &dt
[0], &vectype_in
))
3599 if (dump_enabled_p ())
3600 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3601 "use not simple.\n");
3604 if (op_type
== binary_op
)
3608 op1
= gimple_assign_rhs2 (stmt
);
3609 gcc_assert (code
== WIDEN_MULT_EXPR
|| code
== WIDEN_LSHIFT_EXPR
);
3610 /* For WIDEN_MULT_EXPR, if OP0 is a constant, use the type of
3612 if (CONSTANT_CLASS_P (op0
))
3613 ok
= vect_is_simple_use (op1
, vinfo
, &def_stmt
, &dt
[1], &vectype_in
);
3615 ok
= vect_is_simple_use (op1
, vinfo
, &def_stmt
, &dt
[1]);
3619 if (dump_enabled_p ())
3620 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3621 "use not simple.\n");
3626 /* If op0 is an external or constant defs use a vector type of
3627 the same size as the output vector type. */
3629 vectype_in
= get_same_sized_vectype (rhs_type
, vectype_out
);
3631 gcc_assert (vectype_in
);
3634 if (dump_enabled_p ())
3636 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3637 "no vectype for scalar type ");
3638 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
3639 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
3645 if (VECTOR_BOOLEAN_TYPE_P (vectype_out
)
3646 && !VECTOR_BOOLEAN_TYPE_P (vectype_in
))
3648 if (dump_enabled_p ())
3650 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3651 "can't convert between boolean and non "
3653 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, rhs_type
);
3654 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
3660 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
3661 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
3662 if (nunits_in
< nunits_out
)
3664 else if (nunits_out
== nunits_in
)
3669 /* Multiple types in SLP are handled by creating the appropriate number of
3670 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3672 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
3674 else if (modifier
== NARROW
)
3675 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_out
;
3677 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
3679 /* Sanity check: make sure that at least one copy of the vectorized stmt
3680 needs to be generated. */
3681 gcc_assert (ncopies
>= 1);
3683 /* Supportable by target? */
3687 if (code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
3689 if (supportable_convert_operation (code
, vectype_out
, vectype_in
,
3694 if (dump_enabled_p ())
3695 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
3696 "conversion not supported by target.\n");
3700 if (supportable_widening_operation (code
, stmt
, vectype_out
, vectype_in
,
3701 &code1
, &code2
, &multi_step_cvt
,
3704 /* Binary widening operation can only be supported directly by the
3706 gcc_assert (!(multi_step_cvt
&& op_type
== binary_op
));
3710 if (code
!= FLOAT_EXPR
3711 || (GET_MODE_SIZE (TYPE_MODE (lhs_type
))
3712 <= GET_MODE_SIZE (TYPE_MODE (rhs_type
))))
3715 rhs_mode
= TYPE_MODE (rhs_type
);
3716 fltsz
= GET_MODE_SIZE (TYPE_MODE (lhs_type
));
3717 for (rhs_mode
= GET_MODE_2XWIDER_MODE (TYPE_MODE (rhs_type
));
3718 rhs_mode
!= VOIDmode
&& GET_MODE_SIZE (rhs_mode
) <= fltsz
;
3719 rhs_mode
= GET_MODE_2XWIDER_MODE (rhs_mode
))
3722 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode
), 0);
3723 cvt_type
= get_same_sized_vectype (cvt_type
, vectype_in
);
3724 if (cvt_type
== NULL_TREE
)
3727 if (GET_MODE_SIZE (rhs_mode
) == fltsz
)
3729 if (!supportable_convert_operation (code
, vectype_out
,
3730 cvt_type
, &decl1
, &codecvt1
))
3733 else if (!supportable_widening_operation (code
, stmt
, vectype_out
,
3734 cvt_type
, &codecvt1
,
3735 &codecvt2
, &multi_step_cvt
,
3739 gcc_assert (multi_step_cvt
== 0);
3741 if (supportable_widening_operation (NOP_EXPR
, stmt
, cvt_type
,
3742 vectype_in
, &code1
, &code2
,
3743 &multi_step_cvt
, &interm_types
))
3747 if (rhs_mode
== VOIDmode
|| GET_MODE_SIZE (rhs_mode
) > fltsz
)
3750 if (GET_MODE_SIZE (rhs_mode
) == fltsz
)
3751 codecvt2
= ERROR_MARK
;
3755 interm_types
.safe_push (cvt_type
);
3756 cvt_type
= NULL_TREE
;
3761 gcc_assert (op_type
== unary_op
);
3762 if (supportable_narrowing_operation (code
, vectype_out
, vectype_in
,
3763 &code1
, &multi_step_cvt
,
3767 if (code
!= FIX_TRUNC_EXPR
3768 || (GET_MODE_SIZE (TYPE_MODE (lhs_type
))
3769 >= GET_MODE_SIZE (TYPE_MODE (rhs_type
))))
3772 rhs_mode
= TYPE_MODE (rhs_type
);
3774 = build_nonstandard_integer_type (GET_MODE_BITSIZE (rhs_mode
), 0);
3775 cvt_type
= get_same_sized_vectype (cvt_type
, vectype_in
);
3776 if (cvt_type
== NULL_TREE
)
3778 if (!supportable_convert_operation (code
, cvt_type
, vectype_in
,
3781 if (supportable_narrowing_operation (NOP_EXPR
, vectype_out
, cvt_type
,
3782 &code1
, &multi_step_cvt
,
3791 if (!vec_stmt
) /* transformation not required. */
3793 if (dump_enabled_p ())
3794 dump_printf_loc (MSG_NOTE
, vect_location
,
3795 "=== vectorizable_conversion ===\n");
3796 if (code
== FIX_TRUNC_EXPR
|| code
== FLOAT_EXPR
)
3798 STMT_VINFO_TYPE (stmt_info
) = type_conversion_vec_info_type
;
3799 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
3801 else if (modifier
== NARROW
)
3803 STMT_VINFO_TYPE (stmt_info
) = type_demotion_vec_info_type
;
3804 vect_model_promotion_demotion_cost (stmt_info
, dt
, multi_step_cvt
);
3808 STMT_VINFO_TYPE (stmt_info
) = type_promotion_vec_info_type
;
3809 vect_model_promotion_demotion_cost (stmt_info
, dt
, multi_step_cvt
);
3811 interm_types
.release ();
3816 if (dump_enabled_p ())
3817 dump_printf_loc (MSG_NOTE
, vect_location
,
3818 "transform conversion. ncopies = %d.\n", ncopies
);
3820 if (op_type
== binary_op
)
3822 if (CONSTANT_CLASS_P (op0
))
3823 op0
= fold_convert (TREE_TYPE (op1
), op0
);
3824 else if (CONSTANT_CLASS_P (op1
))
3825 op1
= fold_convert (TREE_TYPE (op0
), op1
);
3828 /* In case of multi-step conversion, we first generate conversion operations
3829 to the intermediate types, and then from that types to the final one.
3830 We create vector destinations for the intermediate type (TYPES) received
3831 from supportable_*_operation, and store them in the correct order
3832 for future use in vect_create_vectorized_*_stmts (). */
3833 vec_dsts
.create (multi_step_cvt
+ 1);
3834 vec_dest
= vect_create_destination_var (scalar_dest
,
3835 (cvt_type
&& modifier
== WIDEN
)
3836 ? cvt_type
: vectype_out
);
3837 vec_dsts
.quick_push (vec_dest
);
3841 for (i
= interm_types
.length () - 1;
3842 interm_types
.iterate (i
, &intermediate_type
); i
--)
3844 vec_dest
= vect_create_destination_var (scalar_dest
,
3846 vec_dsts
.quick_push (vec_dest
);
3851 vec_dest
= vect_create_destination_var (scalar_dest
,
3853 ? vectype_out
: cvt_type
);
3857 if (modifier
== WIDEN
)
3859 vec_oprnds0
.create (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1);
3860 if (op_type
== binary_op
)
3861 vec_oprnds1
.create (1);
3863 else if (modifier
== NARROW
)
3864 vec_oprnds0
.create (
3865 2 * (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1));
3867 else if (code
== WIDEN_LSHIFT_EXPR
)
3868 vec_oprnds1
.create (slp_node
->vec_stmts_size
);
3871 prev_stmt_info
= NULL
;
3875 for (j
= 0; j
< ncopies
; j
++)
3878 vect_get_vec_defs (op0
, NULL
, stmt
, &vec_oprnds0
, NULL
, slp_node
,
3881 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, NULL
);
3883 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
3885 /* Arguments are ready, create the new vector stmt. */
3886 if (code1
== CALL_EXPR
)
3888 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
3889 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3890 gimple_call_set_lhs (new_stmt
, new_temp
);
3894 gcc_assert (TREE_CODE_LENGTH (code1
) == unary_op
);
3895 new_stmt
= gimple_build_assign (vec_dest
, code1
, vop0
);
3896 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3897 gimple_assign_set_lhs (new_stmt
, new_temp
);
3900 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3902 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
3905 if (!prev_stmt_info
)
3906 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3908 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3909 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3916 /* In case the vectorization factor (VF) is bigger than the number
3917 of elements that we can fit in a vectype (nunits), we have to
3918 generate more than one vector stmt - i.e - we need to "unroll"
3919 the vector stmt by a factor VF/nunits. */
3920 for (j
= 0; j
< ncopies
; j
++)
3927 if (code
== WIDEN_LSHIFT_EXPR
)
3932 /* Store vec_oprnd1 for every vector stmt to be created
3933 for SLP_NODE. We check during the analysis that all
3934 the shift arguments are the same. */
3935 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
3936 vec_oprnds1
.quick_push (vec_oprnd1
);
3938 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
3942 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
,
3943 &vec_oprnds1
, slp_node
, -1);
3947 vec_oprnd0
= vect_get_vec_def_for_operand (op0
, stmt
);
3948 vec_oprnds0
.quick_push (vec_oprnd0
);
3949 if (op_type
== binary_op
)
3951 if (code
== WIDEN_LSHIFT_EXPR
)
3954 vec_oprnd1
= vect_get_vec_def_for_operand (op1
, stmt
);
3955 vec_oprnds1
.quick_push (vec_oprnd1
);
3961 vec_oprnd0
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd0
);
3962 vec_oprnds0
.truncate (0);
3963 vec_oprnds0
.quick_push (vec_oprnd0
);
3964 if (op_type
== binary_op
)
3966 if (code
== WIDEN_LSHIFT_EXPR
)
3969 vec_oprnd1
= vect_get_vec_def_for_stmt_copy (dt
[1],
3971 vec_oprnds1
.truncate (0);
3972 vec_oprnds1
.quick_push (vec_oprnd1
);
3976 /* Arguments are ready. Create the new vector stmts. */
3977 for (i
= multi_step_cvt
; i
>= 0; i
--)
3979 tree this_dest
= vec_dsts
[i
];
3980 enum tree_code c1
= code1
, c2
= code2
;
3981 if (i
== 0 && codecvt2
!= ERROR_MARK
)
3986 vect_create_vectorized_promotion_stmts (&vec_oprnds0
,
3988 stmt
, this_dest
, gsi
,
3989 c1
, c2
, decl1
, decl2
,
3993 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
3997 if (codecvt1
== CALL_EXPR
)
3999 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
4000 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4001 gimple_call_set_lhs (new_stmt
, new_temp
);
4005 gcc_assert (TREE_CODE_LENGTH (codecvt1
) == unary_op
);
4006 new_temp
= make_ssa_name (vec_dest
);
4007 new_stmt
= gimple_build_assign (new_temp
, codecvt1
,
4011 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4014 new_stmt
= SSA_NAME_DEF_STMT (vop0
);
4017 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4020 if (!prev_stmt_info
)
4021 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
4023 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4024 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4029 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
4033 /* In case the vectorization factor (VF) is bigger than the number
4034 of elements that we can fit in a vectype (nunits), we have to
4035 generate more than one vector stmt - i.e - we need to "unroll"
4036 the vector stmt by a factor VF/nunits. */
4037 for (j
= 0; j
< ncopies
; j
++)
4041 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
4045 vec_oprnds0
.truncate (0);
4046 vect_get_loop_based_defs (&last_oprnd
, stmt
, dt
[0], &vec_oprnds0
,
4047 vect_pow2 (multi_step_cvt
) - 1);
4050 /* Arguments are ready. Create the new vector stmts. */
4052 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4054 if (codecvt1
== CALL_EXPR
)
4056 new_stmt
= gimple_build_call (decl1
, 1, vop0
);
4057 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4058 gimple_call_set_lhs (new_stmt
, new_temp
);
4062 gcc_assert (TREE_CODE_LENGTH (codecvt1
) == unary_op
);
4063 new_temp
= make_ssa_name (vec_dest
);
4064 new_stmt
= gimple_build_assign (new_temp
, codecvt1
,
4068 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4069 vec_oprnds0
[i
] = new_temp
;
4072 vect_create_vectorized_demotion_stmts (&vec_oprnds0
, multi_step_cvt
,
4073 stmt
, vec_dsts
, gsi
,
4078 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
4082 vec_oprnds0
.release ();
4083 vec_oprnds1
.release ();
4084 vec_dsts
.release ();
4085 interm_types
.release ();
4091 /* Function vectorizable_assignment.
4093 Check if STMT performs an assignment (copy) that can be vectorized.
4094 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4095 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4096 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4099 vectorizable_assignment (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
4100 gimple
**vec_stmt
, slp_tree slp_node
)
4105 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4106 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4109 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
4112 vec
<tree
> vec_oprnds
= vNULL
;
4114 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4115 vec_info
*vinfo
= stmt_info
->vinfo
;
4116 gimple
*new_stmt
= NULL
;
4117 stmt_vec_info prev_stmt_info
= NULL
;
4118 enum tree_code code
;
4121 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4124 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
4127 /* Is vectorizable assignment? */
4128 if (!is_gimple_assign (stmt
))
4131 scalar_dest
= gimple_assign_lhs (stmt
);
4132 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
4135 code
= gimple_assign_rhs_code (stmt
);
4136 if (gimple_assign_single_p (stmt
)
4137 || code
== PAREN_EXPR
4138 || CONVERT_EXPR_CODE_P (code
))
4139 op
= gimple_assign_rhs1 (stmt
);
4143 if (code
== VIEW_CONVERT_EXPR
)
4144 op
= TREE_OPERAND (op
, 0);
4146 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
4147 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
4149 /* Multiple types in SLP are handled by creating the appropriate number of
4150 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4152 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
4155 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
4157 gcc_assert (ncopies
>= 1);
4159 if (!vect_is_simple_use (op
, vinfo
, &def_stmt
, &dt
[0], &vectype_in
))
4161 if (dump_enabled_p ())
4162 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4163 "use not simple.\n");
4167 /* We can handle NOP_EXPR conversions that do not change the number
4168 of elements or the vector size. */
4169 if ((CONVERT_EXPR_CODE_P (code
)
4170 || code
== VIEW_CONVERT_EXPR
)
4172 || TYPE_VECTOR_SUBPARTS (vectype_in
) != nunits
4173 || (GET_MODE_SIZE (TYPE_MODE (vectype
))
4174 != GET_MODE_SIZE (TYPE_MODE (vectype_in
)))))
4177 /* We do not handle bit-precision changes. */
4178 if ((CONVERT_EXPR_CODE_P (code
)
4179 || code
== VIEW_CONVERT_EXPR
)
4180 && INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest
))
4181 && ((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4182 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4183 || ((TYPE_PRECISION (TREE_TYPE (op
))
4184 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (op
))))))
4185 /* But a conversion that does not change the bit-pattern is ok. */
4186 && !((TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4187 > TYPE_PRECISION (TREE_TYPE (op
)))
4188 && TYPE_UNSIGNED (TREE_TYPE (op
))))
4190 if (dump_enabled_p ())
4191 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4192 "type conversion to/from bit-precision "
4197 if (!vec_stmt
) /* transformation not required. */
4199 STMT_VINFO_TYPE (stmt_info
) = assignment_vec_info_type
;
4200 if (dump_enabled_p ())
4201 dump_printf_loc (MSG_NOTE
, vect_location
,
4202 "=== vectorizable_assignment ===\n");
4203 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4208 if (dump_enabled_p ())
4209 dump_printf_loc (MSG_NOTE
, vect_location
, "transform assignment.\n");
4212 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4215 for (j
= 0; j
< ncopies
; j
++)
4219 vect_get_vec_defs (op
, NULL
, stmt
, &vec_oprnds
, NULL
, slp_node
, -1);
4221 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds
, NULL
);
4223 /* Arguments are ready. create the new vector stmt. */
4224 FOR_EACH_VEC_ELT (vec_oprnds
, i
, vop
)
4226 if (CONVERT_EXPR_CODE_P (code
)
4227 || code
== VIEW_CONVERT_EXPR
)
4228 vop
= build1 (VIEW_CONVERT_EXPR
, vectype
, vop
);
4229 new_stmt
= gimple_build_assign (vec_dest
, vop
);
4230 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4231 gimple_assign_set_lhs (new_stmt
, new_temp
);
4232 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4234 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4241 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4243 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4245 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4248 vec_oprnds
.release ();
4253 /* Return TRUE if CODE (a shift operation) is supported for SCALAR_TYPE
4254 either as shift by a scalar or by a vector. */
4257 vect_supportable_shift (enum tree_code code
, tree scalar_type
)
4260 machine_mode vec_mode
;
4265 vectype
= get_vectype_for_scalar_type (scalar_type
);
4269 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
4271 || optab_handler (optab
, TYPE_MODE (vectype
)) == CODE_FOR_nothing
)
4273 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4275 || (optab_handler (optab
, TYPE_MODE (vectype
))
4276 == CODE_FOR_nothing
))
4280 vec_mode
= TYPE_MODE (vectype
);
4281 icode
= (int) optab_handler (optab
, vec_mode
);
4282 if (icode
== CODE_FOR_nothing
)
4289 /* Function vectorizable_shift.
4291 Check if STMT performs a shift operation that can be vectorized.
4292 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4293 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4294 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4297 vectorizable_shift (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
4298 gimple
**vec_stmt
, slp_tree slp_node
)
4302 tree op0
, op1
= NULL
;
4303 tree vec_oprnd1
= NULL_TREE
;
4304 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4306 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4307 enum tree_code code
;
4308 machine_mode vec_mode
;
4312 machine_mode optab_op2_mode
;
4314 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
4315 gimple
*new_stmt
= NULL
;
4316 stmt_vec_info prev_stmt_info
;
4323 vec
<tree
> vec_oprnds0
= vNULL
;
4324 vec
<tree
> vec_oprnds1
= vNULL
;
4327 bool scalar_shift_arg
= true;
4328 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4329 vec_info
*vinfo
= stmt_info
->vinfo
;
4332 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4335 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
4338 /* Is STMT a vectorizable binary/unary operation? */
4339 if (!is_gimple_assign (stmt
))
4342 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
4345 code
= gimple_assign_rhs_code (stmt
);
4347 if (!(code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
4348 || code
== RROTATE_EXPR
))
4351 scalar_dest
= gimple_assign_lhs (stmt
);
4352 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
4353 if (TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4354 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4356 if (dump_enabled_p ())
4357 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4358 "bit-precision shifts not supported.\n");
4362 op0
= gimple_assign_rhs1 (stmt
);
4363 if (!vect_is_simple_use (op0
, vinfo
, &def_stmt
, &dt
[0], &vectype
))
4365 if (dump_enabled_p ())
4366 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4367 "use not simple.\n");
4370 /* If op0 is an external or constant def use a vector type with
4371 the same size as the output vector type. */
4373 vectype
= get_same_sized_vectype (TREE_TYPE (op0
), vectype_out
);
4375 gcc_assert (vectype
);
4378 if (dump_enabled_p ())
4379 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4380 "no vectype for scalar type\n");
4384 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
4385 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
4386 if (nunits_out
!= nunits_in
)
4389 op1
= gimple_assign_rhs2 (stmt
);
4390 if (!vect_is_simple_use (op1
, vinfo
, &def_stmt
, &dt
[1], &op1_vectype
))
4392 if (dump_enabled_p ())
4393 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4394 "use not simple.\n");
4399 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
4403 /* Multiple types in SLP are handled by creating the appropriate number of
4404 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4406 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
4409 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
4411 gcc_assert (ncopies
>= 1);
4413 /* Determine whether the shift amount is a vector, or scalar. If the
4414 shift/rotate amount is a vector, use the vector/vector shift optabs. */
4416 if ((dt
[1] == vect_internal_def
4417 || dt
[1] == vect_induction_def
)
4419 scalar_shift_arg
= false;
4420 else if (dt
[1] == vect_constant_def
4421 || dt
[1] == vect_external_def
4422 || dt
[1] == vect_internal_def
)
4424 /* In SLP, need to check whether the shift count is the same,
4425 in loops if it is a constant or invariant, it is always
4429 vec
<gimple
*> stmts
= SLP_TREE_SCALAR_STMTS (slp_node
);
4432 FOR_EACH_VEC_ELT (stmts
, k
, slpstmt
)
4433 if (!operand_equal_p (gimple_assign_rhs2 (slpstmt
), op1
, 0))
4434 scalar_shift_arg
= false;
4439 if (dump_enabled_p ())
4440 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4441 "operand mode requires invariant argument.\n");
4445 /* Vector shifted by vector. */
4446 if (!scalar_shift_arg
)
4448 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4449 if (dump_enabled_p ())
4450 dump_printf_loc (MSG_NOTE
, vect_location
,
4451 "vector/vector shift/rotate found.\n");
4454 op1_vectype
= get_same_sized_vectype (TREE_TYPE (op1
), vectype_out
);
4455 if (op1_vectype
== NULL_TREE
4456 || TYPE_MODE (op1_vectype
) != TYPE_MODE (vectype
))
4458 if (dump_enabled_p ())
4459 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4460 "unusable type for last operand in"
4461 " vector/vector shift/rotate.\n");
4465 /* See if the machine has a vector shifted by scalar insn and if not
4466 then see if it has a vector shifted by vector insn. */
4469 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
4471 && optab_handler (optab
, TYPE_MODE (vectype
)) != CODE_FOR_nothing
)
4473 if (dump_enabled_p ())
4474 dump_printf_loc (MSG_NOTE
, vect_location
,
4475 "vector/scalar shift/rotate found.\n");
4479 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
4481 && (optab_handler (optab
, TYPE_MODE (vectype
))
4482 != CODE_FOR_nothing
))
4484 scalar_shift_arg
= false;
4486 if (dump_enabled_p ())
4487 dump_printf_loc (MSG_NOTE
, vect_location
,
4488 "vector/vector shift/rotate found.\n");
4490 /* Unlike the other binary operators, shifts/rotates have
4491 the rhs being int, instead of the same type as the lhs,
4492 so make sure the scalar is the right type if we are
4493 dealing with vectors of long long/long/short/char. */
4494 if (dt
[1] == vect_constant_def
)
4495 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
4496 else if (!useless_type_conversion_p (TREE_TYPE (vectype
),
4500 && TYPE_MODE (TREE_TYPE (vectype
))
4501 != TYPE_MODE (TREE_TYPE (op1
)))
4503 if (dump_enabled_p ())
4504 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4505 "unusable type for last operand in"
4506 " vector/vector shift/rotate.\n");
4509 if (vec_stmt
&& !slp_node
)
4511 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
4512 op1
= vect_init_vector (stmt
, op1
,
4513 TREE_TYPE (vectype
), NULL
);
4520 /* Supportable by target? */
4523 if (dump_enabled_p ())
4524 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4528 vec_mode
= TYPE_MODE (vectype
);
4529 icode
= (int) optab_handler (optab
, vec_mode
);
4530 if (icode
== CODE_FOR_nothing
)
4532 if (dump_enabled_p ())
4533 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4534 "op not supported by target.\n");
4535 /* Check only during analysis. */
4536 if (GET_MODE_SIZE (vec_mode
) != UNITS_PER_WORD
4537 || (vf
< vect_min_worthwhile_factor (code
)
4540 if (dump_enabled_p ())
4541 dump_printf_loc (MSG_NOTE
, vect_location
,
4542 "proceeding using word mode.\n");
4545 /* Worthwhile without SIMD support? Check only during analysis. */
4546 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
4547 && vf
< vect_min_worthwhile_factor (code
)
4550 if (dump_enabled_p ())
4551 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4552 "not worthwhile without SIMD support.\n");
4556 if (!vec_stmt
) /* transformation not required. */
4558 STMT_VINFO_TYPE (stmt_info
) = shift_vec_info_type
;
4559 if (dump_enabled_p ())
4560 dump_printf_loc (MSG_NOTE
, vect_location
,
4561 "=== vectorizable_shift ===\n");
4562 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4568 if (dump_enabled_p ())
4569 dump_printf_loc (MSG_NOTE
, vect_location
,
4570 "transform binary/unary operation.\n");
4573 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4575 prev_stmt_info
= NULL
;
4576 for (j
= 0; j
< ncopies
; j
++)
4581 if (scalar_shift_arg
)
4583 /* Vector shl and shr insn patterns can be defined with scalar
4584 operand 2 (shift operand). In this case, use constant or loop
4585 invariant op1 directly, without extending it to vector mode
4587 optab_op2_mode
= insn_data
[icode
].operand
[2].mode
;
4588 if (!VECTOR_MODE_P (optab_op2_mode
))
4590 if (dump_enabled_p ())
4591 dump_printf_loc (MSG_NOTE
, vect_location
,
4592 "operand 1 using scalar mode.\n");
4594 vec_oprnds1
.create (slp_node
? slp_node
->vec_stmts_size
: 1);
4595 vec_oprnds1
.quick_push (vec_oprnd1
);
4598 /* Store vec_oprnd1 for every vector stmt to be created
4599 for SLP_NODE. We check during the analysis that all
4600 the shift arguments are the same.
4601 TODO: Allow different constants for different vector
4602 stmts generated for an SLP instance. */
4603 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
4604 vec_oprnds1
.quick_push (vec_oprnd1
);
4609 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
4610 (a special case for certain kind of vector shifts); otherwise,
4611 operand 1 should be of a vector type (the usual case). */
4613 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
4616 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
, &vec_oprnds1
,
4620 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, &vec_oprnds1
);
4622 /* Arguments are ready. Create the new vector stmt. */
4623 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4625 vop1
= vec_oprnds1
[i
];
4626 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
);
4627 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4628 gimple_assign_set_lhs (new_stmt
, new_temp
);
4629 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4631 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
4638 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
4640 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
4641 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
4644 vec_oprnds0
.release ();
4645 vec_oprnds1
.release ();
4651 /* Function vectorizable_operation.
4653 Check if STMT performs a binary, unary or ternary operation that can
4655 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
4656 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
4657 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
4660 vectorizable_operation (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
4661 gimple
**vec_stmt
, slp_tree slp_node
)
4665 tree op0
, op1
= NULL_TREE
, op2
= NULL_TREE
;
4666 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4668 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4669 enum tree_code code
;
4670 machine_mode vec_mode
;
4674 bool target_support_p
;
4676 enum vect_def_type dt
[3]
4677 = {vect_unknown_def_type
, vect_unknown_def_type
, vect_unknown_def_type
};
4678 gimple
*new_stmt
= NULL
;
4679 stmt_vec_info prev_stmt_info
;
4685 vec
<tree
> vec_oprnds0
= vNULL
;
4686 vec
<tree
> vec_oprnds1
= vNULL
;
4687 vec
<tree
> vec_oprnds2
= vNULL
;
4688 tree vop0
, vop1
, vop2
;
4689 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
4690 vec_info
*vinfo
= stmt_info
->vinfo
;
4693 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
4696 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
4699 /* Is STMT a vectorizable binary/unary operation? */
4700 if (!is_gimple_assign (stmt
))
4703 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
4706 code
= gimple_assign_rhs_code (stmt
);
4708 /* For pointer addition, we should use the normal plus for
4709 the vector addition. */
4710 if (code
== POINTER_PLUS_EXPR
)
4713 /* Support only unary or binary operations. */
4714 op_type
= TREE_CODE_LENGTH (code
);
4715 if (op_type
!= unary_op
&& op_type
!= binary_op
&& op_type
!= ternary_op
)
4717 if (dump_enabled_p ())
4718 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4719 "num. args = %d (not unary/binary/ternary op).\n",
4724 scalar_dest
= gimple_assign_lhs (stmt
);
4725 vectype_out
= STMT_VINFO_VECTYPE (stmt_info
);
4727 /* Most operations cannot handle bit-precision types without extra
4729 if (!VECTOR_BOOLEAN_TYPE_P (vectype_out
)
4730 && (TYPE_PRECISION (TREE_TYPE (scalar_dest
))
4731 != GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (scalar_dest
))))
4732 /* Exception are bitwise binary operations. */
4733 && code
!= BIT_IOR_EXPR
4734 && code
!= BIT_XOR_EXPR
4735 && code
!= BIT_AND_EXPR
)
4737 if (dump_enabled_p ())
4738 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4739 "bit-precision arithmetic not supported.\n");
4743 op0
= gimple_assign_rhs1 (stmt
);
4744 if (!vect_is_simple_use (op0
, vinfo
, &def_stmt
, &dt
[0], &vectype
))
4746 if (dump_enabled_p ())
4747 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4748 "use not simple.\n");
4751 /* If op0 is an external or constant def use a vector type with
4752 the same size as the output vector type. */
4755 /* For boolean type we cannot determine vectype by
4756 invariant value (don't know whether it is a vector
4757 of booleans or vector of integers). We use output
4758 vectype because operations on boolean don't change
4760 if (TREE_CODE (TREE_TYPE (op0
)) == BOOLEAN_TYPE
)
4762 if (TREE_CODE (TREE_TYPE (scalar_dest
)) != BOOLEAN_TYPE
)
4764 if (dump_enabled_p ())
4765 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4766 "not supported operation on bool value.\n");
4769 vectype
= vectype_out
;
4772 vectype
= get_same_sized_vectype (TREE_TYPE (op0
), vectype_out
);
4775 gcc_assert (vectype
);
4778 if (dump_enabled_p ())
4780 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4781 "no vectype for scalar type ");
4782 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
4784 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
4790 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
4791 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
4792 if (nunits_out
!= nunits_in
)
4795 if (op_type
== binary_op
|| op_type
== ternary_op
)
4797 op1
= gimple_assign_rhs2 (stmt
);
4798 if (!vect_is_simple_use (op1
, vinfo
, &def_stmt
, &dt
[1]))
4800 if (dump_enabled_p ())
4801 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4802 "use not simple.\n");
4806 if (op_type
== ternary_op
)
4808 op2
= gimple_assign_rhs3 (stmt
);
4809 if (!vect_is_simple_use (op2
, vinfo
, &def_stmt
, &dt
[2]))
4811 if (dump_enabled_p ())
4812 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4813 "use not simple.\n");
4819 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
4823 /* Multiple types in SLP are handled by creating the appropriate number of
4824 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
4826 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
4829 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
4831 gcc_assert (ncopies
>= 1);
4833 /* Shifts are handled in vectorizable_shift (). */
4834 if (code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
4835 || code
== RROTATE_EXPR
)
4838 /* Supportable by target? */
4840 vec_mode
= TYPE_MODE (vectype
);
4841 if (code
== MULT_HIGHPART_EXPR
)
4842 target_support_p
= can_mult_highpart_p (vec_mode
, TYPE_UNSIGNED (vectype
));
4845 optab
= optab_for_tree_code (code
, vectype
, optab_default
);
4848 if (dump_enabled_p ())
4849 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4853 target_support_p
= (optab_handler (optab
, vec_mode
)
4854 != CODE_FOR_nothing
);
4857 if (!target_support_p
)
4859 if (dump_enabled_p ())
4860 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4861 "op not supported by target.\n");
4862 /* Check only during analysis. */
4863 if (GET_MODE_SIZE (vec_mode
) != UNITS_PER_WORD
4864 || (!vec_stmt
&& vf
< vect_min_worthwhile_factor (code
)))
4866 if (dump_enabled_p ())
4867 dump_printf_loc (MSG_NOTE
, vect_location
,
4868 "proceeding using word mode.\n");
4871 /* Worthwhile without SIMD support? Check only during analysis. */
4872 if (!VECTOR_MODE_P (vec_mode
)
4874 && vf
< vect_min_worthwhile_factor (code
))
4876 if (dump_enabled_p ())
4877 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
4878 "not worthwhile without SIMD support.\n");
4882 if (!vec_stmt
) /* transformation not required. */
4884 STMT_VINFO_TYPE (stmt_info
) = op_vec_info_type
;
4885 if (dump_enabled_p ())
4886 dump_printf_loc (MSG_NOTE
, vect_location
,
4887 "=== vectorizable_operation ===\n");
4888 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
, NULL
);
4894 if (dump_enabled_p ())
4895 dump_printf_loc (MSG_NOTE
, vect_location
,
4896 "transform binary/unary operation.\n");
4899 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
4901 /* In case the vectorization factor (VF) is bigger than the number
4902 of elements that we can fit in a vectype (nunits), we have to generate
4903 more than one vector stmt - i.e - we need to "unroll" the
4904 vector stmt by a factor VF/nunits. In doing so, we record a pointer
4905 from one copy of the vector stmt to the next, in the field
4906 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
4907 stages to find the correct vector defs to be used when vectorizing
4908 stmts that use the defs of the current stmt. The example below
4909 illustrates the vectorization process when VF=16 and nunits=4 (i.e.,
4910 we need to create 4 vectorized stmts):
4912 before vectorization:
4913 RELATED_STMT VEC_STMT
4917 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
4919 RELATED_STMT VEC_STMT
4920 VS1_0: vx0 = memref0 VS1_1 -
4921 VS1_1: vx1 = memref1 VS1_2 -
4922 VS1_2: vx2 = memref2 VS1_3 -
4923 VS1_3: vx3 = memref3 - -
4924 S1: x = load - VS1_0
4927 step2: vectorize stmt S2 (done here):
4928 To vectorize stmt S2 we first need to find the relevant vector
4929 def for the first operand 'x'. This is, as usual, obtained from
4930 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
4931 that defines 'x' (S1). This way we find the stmt VS1_0, and the
4932 relevant vector def 'vx0'. Having found 'vx0' we can generate
4933 the vector stmt VS2_0, and as usual, record it in the
4934 STMT_VINFO_VEC_STMT of stmt S2.
4935 When creating the second copy (VS2_1), we obtain the relevant vector
4936 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
4937 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
4938 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
4939 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
4940 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
4941 chain of stmts and pointers:
4942 RELATED_STMT VEC_STMT
4943 VS1_0: vx0 = memref0 VS1_1 -
4944 VS1_1: vx1 = memref1 VS1_2 -
4945 VS1_2: vx2 = memref2 VS1_3 -
4946 VS1_3: vx3 = memref3 - -
4947 S1: x = load - VS1_0
4948 VS2_0: vz0 = vx0 + v1 VS2_1 -
4949 VS2_1: vz1 = vx1 + v1 VS2_2 -
4950 VS2_2: vz2 = vx2 + v1 VS2_3 -
4951 VS2_3: vz3 = vx3 + v1 - -
4952 S2: z = x + 1 - VS2_0 */
4954 prev_stmt_info
= NULL
;
4955 for (j
= 0; j
< ncopies
; j
++)
4960 if (op_type
== binary_op
|| op_type
== ternary_op
)
4961 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
, &vec_oprnds1
,
4964 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
4966 if (op_type
== ternary_op
)
4968 vec_oprnds2
.create (1);
4969 vec_oprnds2
.quick_push (vect_get_vec_def_for_operand (op2
,
4975 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, &vec_oprnds1
);
4976 if (op_type
== ternary_op
)
4978 tree vec_oprnd
= vec_oprnds2
.pop ();
4979 vec_oprnds2
.quick_push (vect_get_vec_def_for_stmt_copy (dt
[2],
4984 /* Arguments are ready. Create the new vector stmt. */
4985 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vop0
)
4987 vop1
= ((op_type
== binary_op
|| op_type
== ternary_op
)
4988 ? vec_oprnds1
[i
] : NULL_TREE
);
4989 vop2
= ((op_type
== ternary_op
)
4990 ? vec_oprnds2
[i
] : NULL_TREE
);
4991 new_stmt
= gimple_build_assign (vec_dest
, code
, vop0
, vop1
, vop2
);
4992 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
4993 gimple_assign_set_lhs (new_stmt
, new_temp
);
4994 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
4996 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
5003 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
5005 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
5006 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
5009 vec_oprnds0
.release ();
5010 vec_oprnds1
.release ();
5011 vec_oprnds2
.release ();
5016 /* A helper function to ensure data reference DR's base alignment
5020 ensure_base_align (stmt_vec_info stmt_info
, struct data_reference
*dr
)
5025 if (DR_VECT_AUX (dr
)->base_misaligned
)
5027 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
5028 tree base_decl
= DR_VECT_AUX (dr
)->base_decl
;
5030 if (decl_in_symtab_p (base_decl
))
5031 symtab_node::get (base_decl
)->increase_alignment (TYPE_ALIGN (vectype
));
5034 DECL_ALIGN (base_decl
) = TYPE_ALIGN (vectype
);
5035 DECL_USER_ALIGN (base_decl
) = 1;
5037 DR_VECT_AUX (dr
)->base_misaligned
= false;
5042 /* Given a vector type VECTYPE returns the VECTOR_CST mask that implements
5043 reversal of the vector elements. If that is impossible to do,
5047 perm_mask_for_reverse (tree vectype
)
5052 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5053 sel
= XALLOCAVEC (unsigned char, nunits
);
5055 for (i
= 0; i
< nunits
; ++i
)
5056 sel
[i
] = nunits
- 1 - i
;
5058 if (!can_vec_perm_p (TYPE_MODE (vectype
), false, sel
))
5060 return vect_gen_perm_mask_checked (vectype
, sel
);
5063 /* Function vectorizable_store.
5065 Check if STMT defines a non scalar data-ref (array/pointer/structure) that
5067 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
5068 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
5069 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
5072 vectorizable_store (gimple
*stmt
, gimple_stmt_iterator
*gsi
, gimple
**vec_stmt
,
5078 tree vec_oprnd
= NULL_TREE
;
5079 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
5080 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
= NULL
;
5082 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
5083 struct loop
*loop
= NULL
;
5084 machine_mode vec_mode
;
5086 enum dr_alignment_support alignment_support_scheme
;
5088 enum vect_def_type dt
;
5089 stmt_vec_info prev_stmt_info
= NULL
;
5090 tree dataref_ptr
= NULL_TREE
;
5091 tree dataref_offset
= NULL_TREE
;
5092 gimple
*ptr_incr
= NULL
;
5095 gimple
*next_stmt
, *first_stmt
= NULL
;
5096 bool grouped_store
= false;
5097 bool store_lanes_p
= false;
5098 unsigned int group_size
, i
;
5099 vec
<tree
> dr_chain
= vNULL
;
5100 vec
<tree
> oprnds
= vNULL
;
5101 vec
<tree
> result_chain
= vNULL
;
5103 bool negative
= false;
5104 tree offset
= NULL_TREE
;
5105 vec
<tree
> vec_oprnds
= vNULL
;
5106 bool slp
= (slp_node
!= NULL
);
5107 unsigned int vec_num
;
5108 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
5109 vec_info
*vinfo
= stmt_info
->vinfo
;
5111 tree scatter_base
= NULL_TREE
, scatter_off
= NULL_TREE
;
5112 tree scatter_off_vectype
= NULL_TREE
, scatter_decl
= NULL_TREE
;
5113 int scatter_scale
= 1;
5114 enum vect_def_type scatter_idx_dt
= vect_unknown_def_type
;
5115 enum vect_def_type scatter_src_dt
= vect_unknown_def_type
;
5118 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
5121 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
5124 /* Is vectorizable store? */
5126 if (!is_gimple_assign (stmt
))
5129 scalar_dest
= gimple_assign_lhs (stmt
);
5130 if (TREE_CODE (scalar_dest
) == VIEW_CONVERT_EXPR
5131 && is_pattern_stmt_p (stmt_info
))
5132 scalar_dest
= TREE_OPERAND (scalar_dest
, 0);
5133 if (TREE_CODE (scalar_dest
) != ARRAY_REF
5134 && TREE_CODE (scalar_dest
) != BIT_FIELD_REF
5135 && TREE_CODE (scalar_dest
) != INDIRECT_REF
5136 && TREE_CODE (scalar_dest
) != COMPONENT_REF
5137 && TREE_CODE (scalar_dest
) != IMAGPART_EXPR
5138 && TREE_CODE (scalar_dest
) != REALPART_EXPR
5139 && TREE_CODE (scalar_dest
) != MEM_REF
)
5142 gcc_assert (gimple_assign_single_p (stmt
));
5144 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
5145 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5148 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
5150 /* Multiple types in SLP are handled by creating the appropriate number of
5151 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
5153 if (slp
|| PURE_SLP_STMT (stmt_info
))
5156 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
5158 gcc_assert (ncopies
>= 1);
5160 /* FORNOW. This restriction should be relaxed. */
5161 if (loop
&& nested_in_vect_loop_p (loop
, stmt
) && ncopies
> 1)
5163 if (dump_enabled_p ())
5164 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5165 "multiple types in nested loop.\n");
5169 op
= gimple_assign_rhs1 (stmt
);
5170 if (!vect_is_simple_use (op
, vinfo
, &def_stmt
, &dt
))
5172 if (dump_enabled_p ())
5173 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5174 "use not simple.\n");
5178 elem_type
= TREE_TYPE (vectype
);
5179 vec_mode
= TYPE_MODE (vectype
);
5181 /* FORNOW. In some cases can vectorize even if data-type not supported
5182 (e.g. - array initialization with 0). */
5183 if (optab_handler (mov_optab
, vec_mode
) == CODE_FOR_nothing
)
5186 if (!STMT_VINFO_DATA_REF (stmt_info
))
5189 if (!STMT_VINFO_STRIDED_P (stmt_info
))
5192 tree_int_cst_compare (loop
&& nested_in_vect_loop_p (loop
, stmt
)
5193 ? STMT_VINFO_DR_STEP (stmt_info
) : DR_STEP (dr
),
5194 size_zero_node
) < 0;
5195 if (negative
&& ncopies
> 1)
5197 if (dump_enabled_p ())
5198 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5199 "multiple types with negative step.\n");
5204 gcc_assert (!grouped_store
);
5205 alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
5206 if (alignment_support_scheme
!= dr_aligned
5207 && alignment_support_scheme
!= dr_unaligned_supported
)
5209 if (dump_enabled_p ())
5210 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5211 "negative step but alignment required.\n");
5214 if (dt
!= vect_constant_def
5215 && dt
!= vect_external_def
5216 && !perm_mask_for_reverse (vectype
))
5218 if (dump_enabled_p ())
5219 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5220 "negative step and reversing not supported.\n");
5226 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
5228 grouped_store
= true;
5229 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
5230 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
5232 && !PURE_SLP_STMT (stmt_info
)
5233 && !STMT_VINFO_STRIDED_P (stmt_info
))
5235 if (vect_store_lanes_supported (vectype
, group_size
))
5236 store_lanes_p
= true;
5237 else if (!vect_grouped_store_supported (vectype
, group_size
))
5241 if (STMT_VINFO_STRIDED_P (stmt_info
)
5242 && (slp
|| PURE_SLP_STMT (stmt_info
))
5243 && (group_size
> nunits
5244 || nunits
% group_size
!= 0))
5246 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5247 "unhandled strided group store\n");
5251 if (first_stmt
== stmt
)
5253 /* STMT is the leader of the group. Check the operands of all the
5254 stmts of the group. */
5255 next_stmt
= GROUP_NEXT_ELEMENT (stmt_info
);
5258 gcc_assert (gimple_assign_single_p (next_stmt
));
5259 op
= gimple_assign_rhs1 (next_stmt
);
5260 if (!vect_is_simple_use (op
, vinfo
, &def_stmt
, &dt
))
5262 if (dump_enabled_p ())
5263 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5264 "use not simple.\n");
5267 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5272 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
5275 scatter_decl
= vect_check_gather_scatter (stmt
, loop_vinfo
, &scatter_base
,
5276 &scatter_off
, &scatter_scale
);
5277 gcc_assert (scatter_decl
);
5278 if (!vect_is_simple_use (scatter_off
, vinfo
, &def_stmt
, &scatter_idx_dt
,
5279 &scatter_off_vectype
))
5281 if (dump_enabled_p ())
5282 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
5283 "scatter index use not simple.");
5288 if (!vec_stmt
) /* transformation not required. */
5290 STMT_VINFO_TYPE (stmt_info
) = store_vec_info_type
;
5291 /* The SLP costs are calculated during SLP analysis. */
5292 if (!PURE_SLP_STMT (stmt_info
))
5293 vect_model_store_cost (stmt_info
, ncopies
, store_lanes_p
, dt
,
5300 ensure_base_align (stmt_info
, dr
);
5302 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
5304 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
, op
, src
;
5305 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (scatter_decl
));
5306 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
5307 tree ptr
, mask
, var
, scale
, perm_mask
= NULL_TREE
;
5308 edge pe
= loop_preheader_edge (loop
);
5311 enum { NARROW
, NONE
, WIDEN
} modifier
;
5312 int scatter_off_nunits
= TYPE_VECTOR_SUBPARTS (scatter_off_vectype
);
5314 if (nunits
== (unsigned int) scatter_off_nunits
)
5316 else if (nunits
== (unsigned int) scatter_off_nunits
/ 2)
5318 unsigned char *sel
= XALLOCAVEC (unsigned char, scatter_off_nunits
);
5321 for (i
= 0; i
< (unsigned int) scatter_off_nunits
; ++i
)
5322 sel
[i
] = i
| nunits
;
5324 perm_mask
= vect_gen_perm_mask_checked (scatter_off_vectype
, sel
);
5325 gcc_assert (perm_mask
!= NULL_TREE
);
5327 else if (nunits
== (unsigned int) scatter_off_nunits
* 2)
5329 unsigned char *sel
= XALLOCAVEC (unsigned char, nunits
);
5332 for (i
= 0; i
< (unsigned int) nunits
; ++i
)
5333 sel
[i
] = i
| scatter_off_nunits
;
5335 perm_mask
= vect_gen_perm_mask_checked (vectype
, sel
);
5336 gcc_assert (perm_mask
!= NULL_TREE
);
5342 rettype
= TREE_TYPE (TREE_TYPE (scatter_decl
));
5343 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
5344 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
5345 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
5346 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
5347 scaletype
= TREE_VALUE (arglist
);
5349 gcc_checking_assert (TREE_CODE (masktype
) == INTEGER_TYPE
5350 && TREE_CODE (rettype
) == VOID_TYPE
);
5352 ptr
= fold_convert (ptrtype
, scatter_base
);
5353 if (!is_gimple_min_invariant (ptr
))
5355 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
5356 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
5357 gcc_assert (!new_bb
);
5360 /* Currently we support only unconditional scatter stores,
5361 so mask should be all ones. */
5362 mask
= build_int_cst (masktype
, -1);
5363 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
5365 scale
= build_int_cst (scaletype
, scatter_scale
);
5367 prev_stmt_info
= NULL
;
5368 for (j
= 0; j
< ncopies
; ++j
)
5373 = vect_get_vec_def_for_operand (gimple_assign_rhs1 (stmt
), stmt
);
5375 = vect_get_vec_def_for_operand (scatter_off
, stmt
);
5377 else if (modifier
!= NONE
&& (j
& 1))
5379 if (modifier
== WIDEN
)
5382 = vect_get_vec_def_for_stmt_copy (scatter_src_dt
, vec_oprnd1
);
5383 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
, perm_mask
,
5386 else if (modifier
== NARROW
)
5388 src
= permute_vec_elements (vec_oprnd1
, vec_oprnd1
, perm_mask
,
5391 = vect_get_vec_def_for_stmt_copy (scatter_idx_dt
, vec_oprnd0
);
5399 = vect_get_vec_def_for_stmt_copy (scatter_src_dt
, vec_oprnd1
);
5401 = vect_get_vec_def_for_stmt_copy (scatter_idx_dt
, vec_oprnd0
);
5404 if (!useless_type_conversion_p (srctype
, TREE_TYPE (src
)))
5406 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (src
))
5407 == TYPE_VECTOR_SUBPARTS (srctype
));
5408 var
= vect_get_new_ssa_name (srctype
, vect_simple_var
);
5409 src
= build1 (VIEW_CONVERT_EXPR
, srctype
, src
);
5410 new_stmt
= gimple_build_assign (var
, VIEW_CONVERT_EXPR
, src
);
5411 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5415 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
5417 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
5418 == TYPE_VECTOR_SUBPARTS (idxtype
));
5419 var
= vect_get_new_ssa_name (idxtype
, vect_simple_var
);
5420 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
5421 new_stmt
= gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
5422 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5427 = gimple_build_call (scatter_decl
, 5, ptr
, mask
, op
, src
, scale
);
5429 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5431 if (prev_stmt_info
== NULL
)
5432 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
5434 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
5435 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
5442 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
5443 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
5445 GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt
))++;
5448 gcc_assert (!loop
|| !nested_in_vect_loop_p (loop
, stmt
));
5450 /* We vectorize all the stmts of the interleaving group when we
5451 reach the last stmt in the group. */
5452 if (GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt
))
5453 < GROUP_SIZE (vinfo_for_stmt (first_stmt
))
5462 grouped_store
= false;
5463 /* VEC_NUM is the number of vect stmts to be created for this
5465 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
5466 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
5467 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
5468 op
= gimple_assign_rhs1 (first_stmt
);
5471 /* VEC_NUM is the number of vect stmts to be created for this
5473 vec_num
= group_size
;
5479 group_size
= vec_num
= 1;
5482 if (dump_enabled_p ())
5483 dump_printf_loc (MSG_NOTE
, vect_location
,
5484 "transform store. ncopies = %d\n", ncopies
);
5486 if (STMT_VINFO_STRIDED_P (stmt_info
))
5488 gimple_stmt_iterator incr_gsi
;
5494 gimple_seq stmts
= NULL
;
5495 tree stride_base
, stride_step
, alias_off
;
5499 gcc_assert (!nested_in_vect_loop_p (loop
, stmt
));
5502 = fold_build_pointer_plus
5503 (unshare_expr (DR_BASE_ADDRESS (first_dr
)),
5504 size_binop (PLUS_EXPR
,
5505 convert_to_ptrofftype (unshare_expr (DR_OFFSET (first_dr
))),
5506 convert_to_ptrofftype (DR_INIT(first_dr
))));
5507 stride_step
= fold_convert (sizetype
, unshare_expr (DR_STEP (first_dr
)));
5509 /* For a store with loop-invariant (but other than power-of-2)
5510 stride (i.e. not a grouped access) like so:
5512 for (i = 0; i < n; i += stride)
5515 we generate a new induction variable and new stores from
5516 the components of the (vectorized) rhs:
5518 for (j = 0; ; j += VF*stride)
5523 array[j + stride] = tmp2;
5527 unsigned nstores
= nunits
;
5528 tree ltype
= elem_type
;
5531 nstores
= nunits
/ group_size
;
5532 if (group_size
< nunits
)
5533 ltype
= build_vector_type (elem_type
, group_size
);
5536 ltype
= build_aligned_type (ltype
, TYPE_ALIGN (elem_type
));
5537 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
5541 ivstep
= stride_step
;
5542 ivstep
= fold_build2 (MULT_EXPR
, TREE_TYPE (ivstep
), ivstep
,
5543 build_int_cst (TREE_TYPE (ivstep
),
5544 ncopies
* nstores
));
5546 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
5548 create_iv (stride_base
, ivstep
, NULL
,
5549 loop
, &incr_gsi
, insert_after
,
5551 incr
= gsi_stmt (incr_gsi
);
5552 set_vinfo_for_stmt (incr
, new_stmt_vec_info (incr
, loop_vinfo
));
5554 stride_step
= force_gimple_operand (stride_step
, &stmts
, true, NULL_TREE
);
5556 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
5558 prev_stmt_info
= NULL
;
5559 alias_off
= build_int_cst (reference_alias_ptr_type (DR_REF (first_dr
)), 0);
5560 next_stmt
= first_stmt
;
5561 for (g
= 0; g
< group_size
; g
++)
5563 running_off
= offvar
;
5566 tree size
= TYPE_SIZE_UNIT (ltype
);
5567 tree pos
= fold_build2 (MULT_EXPR
, sizetype
, size_int (g
),
5569 tree newoff
= copy_ssa_name (running_off
, NULL
);
5570 incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
5572 vect_finish_stmt_generation (stmt
, incr
, gsi
);
5573 running_off
= newoff
;
5575 for (j
= 0; j
< ncopies
; j
++)
5577 /* We've set op and dt above, from gimple_assign_rhs1(stmt),
5578 and first_stmt == stmt. */
5583 vect_get_vec_defs (op
, NULL_TREE
, stmt
, &vec_oprnds
, NULL
,
5585 vec_oprnd
= vec_oprnds
[0];
5589 gcc_assert (gimple_assign_single_p (next_stmt
));
5590 op
= gimple_assign_rhs1 (next_stmt
);
5591 vec_oprnd
= vect_get_vec_def_for_operand (op
, next_stmt
);
5597 vec_oprnd
= vec_oprnds
[j
];
5600 vect_is_simple_use (vec_oprnd
, vinfo
, &def_stmt
, &dt
);
5601 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, vec_oprnd
);
5605 for (i
= 0; i
< nstores
; i
++)
5607 tree newref
, newoff
;
5608 gimple
*incr
, *assign
;
5609 tree size
= TYPE_SIZE (ltype
);
5610 /* Extract the i'th component. */
5611 tree pos
= fold_build2 (MULT_EXPR
, bitsizetype
,
5612 bitsize_int (i
), size
);
5613 tree elem
= fold_build3 (BIT_FIELD_REF
, ltype
, vec_oprnd
,
5616 elem
= force_gimple_operand_gsi (gsi
, elem
, true,
5620 newref
= build2 (MEM_REF
, ltype
,
5621 running_off
, alias_off
);
5623 /* And store it to *running_off. */
5624 assign
= gimple_build_assign (newref
, elem
);
5625 vect_finish_stmt_generation (stmt
, assign
, gsi
);
5627 newoff
= copy_ssa_name (running_off
, NULL
);
5628 incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
5629 running_off
, stride_step
);
5630 vect_finish_stmt_generation (stmt
, incr
, gsi
);
5632 running_off
= newoff
;
5633 if (g
== group_size
- 1
5636 if (j
== 0 && i
== 0)
5637 STMT_VINFO_VEC_STMT (stmt_info
)
5638 = *vec_stmt
= assign
;
5640 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = assign
;
5641 prev_stmt_info
= vinfo_for_stmt (assign
);
5645 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5650 dr_chain
.create (group_size
);
5651 oprnds
.create (group_size
);
5653 alignment_support_scheme
= vect_supportable_dr_alignment (first_dr
, false);
5654 gcc_assert (alignment_support_scheme
);
5655 /* Targets with store-lane instructions must not require explicit
5657 gcc_assert (!store_lanes_p
5658 || alignment_support_scheme
== dr_aligned
5659 || alignment_support_scheme
== dr_unaligned_supported
);
5662 offset
= size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1);
5665 aggr_type
= build_array_type_nelts (elem_type
, vec_num
* nunits
);
5667 aggr_type
= vectype
;
5669 /* In case the vectorization factor (VF) is bigger than the number
5670 of elements that we can fit in a vectype (nunits), we have to generate
5671 more than one vector stmt - i.e - we need to "unroll" the
5672 vector stmt by a factor VF/nunits. For more details see documentation in
5673 vect_get_vec_def_for_copy_stmt. */
5675 /* In case of interleaving (non-unit grouped access):
5682 We create vectorized stores starting from base address (the access of the
5683 first stmt in the chain (S2 in the above example), when the last store stmt
5684 of the chain (S4) is reached:
5687 VS2: &base + vec_size*1 = vx0
5688 VS3: &base + vec_size*2 = vx1
5689 VS4: &base + vec_size*3 = vx3
5691 Then permutation statements are generated:
5693 VS5: vx5 = VEC_PERM_EXPR < vx0, vx3, {0, 8, 1, 9, 2, 10, 3, 11} >
5694 VS6: vx6 = VEC_PERM_EXPR < vx0, vx3, {4, 12, 5, 13, 6, 14, 7, 15} >
5697 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
5698 (the order of the data-refs in the output of vect_permute_store_chain
5699 corresponds to the order of scalar stmts in the interleaving chain - see
5700 the documentation of vect_permute_store_chain()).
5702 In case of both multiple types and interleaving, above vector stores and
5703 permutation stmts are created for every copy. The result vector stmts are
5704 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
5705 STMT_VINFO_RELATED_STMT for the next copies.
5708 prev_stmt_info
= NULL
;
5709 for (j
= 0; j
< ncopies
; j
++)
5716 /* Get vectorized arguments for SLP_NODE. */
5717 vect_get_vec_defs (op
, NULL_TREE
, stmt
, &vec_oprnds
,
5718 NULL
, slp_node
, -1);
5720 vec_oprnd
= vec_oprnds
[0];
5724 /* For interleaved stores we collect vectorized defs for all the
5725 stores in the group in DR_CHAIN and OPRNDS. DR_CHAIN is then
5726 used as an input to vect_permute_store_chain(), and OPRNDS as
5727 an input to vect_get_vec_def_for_stmt_copy() for the next copy.
5729 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
5730 OPRNDS are of size 1. */
5731 next_stmt
= first_stmt
;
5732 for (i
= 0; i
< group_size
; i
++)
5734 /* Since gaps are not supported for interleaved stores,
5735 GROUP_SIZE is the exact number of stmts in the chain.
5736 Therefore, NEXT_STMT can't be NULL_TREE. In case that
5737 there is no interleaving, GROUP_SIZE is 1, and only one
5738 iteration of the loop will be executed. */
5739 gcc_assert (next_stmt
5740 && gimple_assign_single_p (next_stmt
));
5741 op
= gimple_assign_rhs1 (next_stmt
);
5743 vec_oprnd
= vect_get_vec_def_for_operand (op
, next_stmt
);
5744 dr_chain
.quick_push (vec_oprnd
);
5745 oprnds
.quick_push (vec_oprnd
);
5746 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5750 /* We should have catched mismatched types earlier. */
5751 gcc_assert (useless_type_conversion_p (vectype
,
5752 TREE_TYPE (vec_oprnd
)));
5753 bool simd_lane_access_p
5754 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info
);
5755 if (simd_lane_access_p
5756 && TREE_CODE (DR_BASE_ADDRESS (first_dr
)) == ADDR_EXPR
5757 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr
), 0))
5758 && integer_zerop (DR_OFFSET (first_dr
))
5759 && integer_zerop (DR_INIT (first_dr
))
5760 && alias_sets_conflict_p (get_alias_set (aggr_type
),
5761 get_alias_set (DR_REF (first_dr
))))
5763 dataref_ptr
= unshare_expr (DR_BASE_ADDRESS (first_dr
));
5764 dataref_offset
= build_int_cst (reference_alias_ptr_type
5765 (DR_REF (first_dr
)), 0);
5770 = vect_create_data_ref_ptr (first_stmt
, aggr_type
,
5771 simd_lane_access_p
? loop
: NULL
,
5772 offset
, &dummy
, gsi
, &ptr_incr
,
5773 simd_lane_access_p
, &inv_p
);
5774 gcc_assert (bb_vinfo
|| !inv_p
);
5778 /* For interleaved stores we created vectorized defs for all the
5779 defs stored in OPRNDS in the previous iteration (previous copy).
5780 DR_CHAIN is then used as an input to vect_permute_store_chain(),
5781 and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
5783 If the store is not grouped, GROUP_SIZE is 1, and DR_CHAIN and
5784 OPRNDS are of size 1. */
5785 for (i
= 0; i
< group_size
; i
++)
5788 vect_is_simple_use (op
, vinfo
, &def_stmt
, &dt
);
5789 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, op
);
5790 dr_chain
[i
] = vec_oprnd
;
5791 oprnds
[i
] = vec_oprnd
;
5795 = int_const_binop (PLUS_EXPR
, dataref_offset
,
5796 TYPE_SIZE_UNIT (aggr_type
));
5798 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
5799 TYPE_SIZE_UNIT (aggr_type
));
5806 /* Combine all the vectors into an array. */
5807 vec_array
= create_vector_array (vectype
, vec_num
);
5808 for (i
= 0; i
< vec_num
; i
++)
5810 vec_oprnd
= dr_chain
[i
];
5811 write_vector_array (stmt
, gsi
, vec_oprnd
, vec_array
, i
);
5815 MEM_REF[...all elements...] = STORE_LANES (VEC_ARRAY). */
5816 data_ref
= create_array_ref (aggr_type
, dataref_ptr
, first_dr
);
5817 new_stmt
= gimple_build_call_internal (IFN_STORE_LANES
, 1, vec_array
);
5818 gimple_call_set_lhs (new_stmt
, data_ref
);
5819 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5827 result_chain
.create (group_size
);
5829 vect_permute_store_chain (dr_chain
, group_size
, stmt
, gsi
,
5833 next_stmt
= first_stmt
;
5834 for (i
= 0; i
< vec_num
; i
++)
5836 unsigned align
, misalign
;
5839 /* Bump the vector pointer. */
5840 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
5844 vec_oprnd
= vec_oprnds
[i
];
5845 else if (grouped_store
)
5846 /* For grouped stores vectorized defs are interleaved in
5847 vect_permute_store_chain(). */
5848 vec_oprnd
= result_chain
[i
];
5850 data_ref
= fold_build2 (MEM_REF
, TREE_TYPE (vec_oprnd
),
5854 : build_int_cst (reference_alias_ptr_type
5855 (DR_REF (first_dr
)), 0));
5856 align
= TYPE_ALIGN_UNIT (vectype
);
5857 if (aligned_access_p (first_dr
))
5859 else if (DR_MISALIGNMENT (first_dr
) == -1)
5861 if (DR_VECT_AUX (first_dr
)->base_element_aligned
)
5862 align
= TYPE_ALIGN_UNIT (elem_type
);
5864 align
= get_object_alignment (DR_REF (first_dr
))
5867 TREE_TYPE (data_ref
)
5868 = build_aligned_type (TREE_TYPE (data_ref
),
5869 align
* BITS_PER_UNIT
);
5873 TREE_TYPE (data_ref
)
5874 = build_aligned_type (TREE_TYPE (data_ref
),
5875 TYPE_ALIGN (elem_type
));
5876 misalign
= DR_MISALIGNMENT (first_dr
);
5878 if (dataref_offset
== NULL_TREE
5879 && TREE_CODE (dataref_ptr
) == SSA_NAME
)
5880 set_ptr_info_alignment (get_ptr_info (dataref_ptr
), align
,
5884 && dt
!= vect_constant_def
5885 && dt
!= vect_external_def
)
5887 tree perm_mask
= perm_mask_for_reverse (vectype
);
5889 = vect_create_destination_var (gimple_assign_rhs1 (stmt
),
5891 tree new_temp
= make_ssa_name (perm_dest
);
5893 /* Generate the permute statement. */
5895 = gimple_build_assign (new_temp
, VEC_PERM_EXPR
, vec_oprnd
,
5896 vec_oprnd
, perm_mask
);
5897 vect_finish_stmt_generation (stmt
, perm_stmt
, gsi
);
5899 perm_stmt
= SSA_NAME_DEF_STMT (new_temp
);
5900 vec_oprnd
= new_temp
;
5903 /* Arguments are ready. Create the new vector stmt. */
5904 new_stmt
= gimple_build_assign (data_ref
, vec_oprnd
);
5905 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
5910 next_stmt
= GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt
));
5918 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
5920 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
5921 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
5925 dr_chain
.release ();
5927 result_chain
.release ();
5928 vec_oprnds
.release ();
5933 /* Given a vector type VECTYPE, turns permutation SEL into the equivalent
5934 VECTOR_CST mask. No checks are made that the target platform supports the
5935 mask, so callers may wish to test can_vec_perm_p separately, or use
5936 vect_gen_perm_mask_checked. */
5939 vect_gen_perm_mask_any (tree vectype
, const unsigned char *sel
)
5941 tree mask_elt_type
, mask_type
, mask_vec
, *mask_elts
;
5944 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
5946 mask_elt_type
= lang_hooks
.types
.type_for_mode
5947 (int_mode_for_mode (TYPE_MODE (TREE_TYPE (vectype
))), 1);
5948 mask_type
= get_vectype_for_scalar_type (mask_elt_type
);
5950 mask_elts
= XALLOCAVEC (tree
, nunits
);
5951 for (i
= nunits
- 1; i
>= 0; i
--)
5952 mask_elts
[i
] = build_int_cst (mask_elt_type
, sel
[i
]);
5953 mask_vec
= build_vector (mask_type
, mask_elts
);
5958 /* Checked version of vect_gen_perm_mask_any. Asserts can_vec_perm_p,
5959 i.e. that the target supports the pattern _for arbitrary input vectors_. */
5962 vect_gen_perm_mask_checked (tree vectype
, const unsigned char *sel
)
5964 gcc_assert (can_vec_perm_p (TYPE_MODE (vectype
), false, sel
));
5965 return vect_gen_perm_mask_any (vectype
, sel
);
5968 /* Given a vector variable X and Y, that was generated for the scalar
5969 STMT, generate instructions to permute the vector elements of X and Y
5970 using permutation mask MASK_VEC, insert them at *GSI and return the
5971 permuted vector variable. */
5974 permute_vec_elements (tree x
, tree y
, tree mask_vec
, gimple
*stmt
,
5975 gimple_stmt_iterator
*gsi
)
5977 tree vectype
= TREE_TYPE (x
);
5978 tree perm_dest
, data_ref
;
5981 perm_dest
= vect_create_destination_var (gimple_get_lhs (stmt
), vectype
);
5982 data_ref
= make_ssa_name (perm_dest
);
5984 /* Generate the permute statement. */
5985 perm_stmt
= gimple_build_assign (data_ref
, VEC_PERM_EXPR
, x
, y
, mask_vec
);
5986 vect_finish_stmt_generation (stmt
, perm_stmt
, gsi
);
5991 /* Hoist the definitions of all SSA uses on STMT out of the loop LOOP,
5992 inserting them on the loops preheader edge. Returns true if we
5993 were successful in doing so (and thus STMT can be moved then),
5994 otherwise returns false. */
5997 hoist_defs_of_uses (gimple
*stmt
, struct loop
*loop
)
6003 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_USE
)
6005 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
6006 if (!gimple_nop_p (def_stmt
)
6007 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
)))
6009 /* Make sure we don't need to recurse. While we could do
6010 so in simple cases when there are more complex use webs
6011 we don't have an easy way to preserve stmt order to fulfil
6012 dependencies within them. */
6015 if (gimple_code (def_stmt
) == GIMPLE_PHI
)
6017 FOR_EACH_SSA_TREE_OPERAND (op2
, def_stmt
, i2
, SSA_OP_USE
)
6019 gimple
*def_stmt2
= SSA_NAME_DEF_STMT (op2
);
6020 if (!gimple_nop_p (def_stmt2
)
6021 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt2
)))
6031 FOR_EACH_SSA_TREE_OPERAND (op
, stmt
, i
, SSA_OP_USE
)
6033 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
6034 if (!gimple_nop_p (def_stmt
)
6035 && flow_bb_inside_loop_p (loop
, gimple_bb (def_stmt
)))
6037 gimple_stmt_iterator gsi
= gsi_for_stmt (def_stmt
);
6038 gsi_remove (&gsi
, false);
6039 gsi_insert_on_edge_immediate (loop_preheader_edge (loop
), def_stmt
);
6046 /* vectorizable_load.
6048 Check if STMT reads a non scalar data-ref (array/pointer/structure) that
6050 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
6051 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
6052 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
6055 vectorizable_load (gimple
*stmt
, gimple_stmt_iterator
*gsi
, gimple
**vec_stmt
,
6056 slp_tree slp_node
, slp_instance slp_node_instance
)
6059 tree vec_dest
= NULL
;
6060 tree data_ref
= NULL
;
6061 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
6062 stmt_vec_info prev_stmt_info
;
6063 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
6064 struct loop
*loop
= NULL
;
6065 struct loop
*containing_loop
= (gimple_bb (stmt
))->loop_father
;
6066 bool nested_in_vect_loop
= false;
6067 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
= NULL
;
6071 gimple
*new_stmt
= NULL
;
6073 enum dr_alignment_support alignment_support_scheme
;
6074 tree dataref_ptr
= NULL_TREE
;
6075 tree dataref_offset
= NULL_TREE
;
6076 gimple
*ptr_incr
= NULL
;
6078 int i
, j
, group_size
= -1, group_gap_adj
;
6079 tree msq
= NULL_TREE
, lsq
;
6080 tree offset
= NULL_TREE
;
6081 tree byte_offset
= NULL_TREE
;
6082 tree realignment_token
= NULL_TREE
;
6084 vec
<tree
> dr_chain
= vNULL
;
6085 bool grouped_load
= false;
6086 bool load_lanes_p
= false;
6089 bool negative
= false;
6090 bool compute_in_loop
= false;
6091 struct loop
*at_loop
;
6093 bool slp
= (slp_node
!= NULL
);
6094 bool slp_perm
= false;
6095 enum tree_code code
;
6096 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
6099 tree gather_base
= NULL_TREE
, gather_off
= NULL_TREE
;
6100 tree gather_off_vectype
= NULL_TREE
, gather_decl
= NULL_TREE
;
6101 int gather_scale
= 1;
6102 enum vect_def_type gather_dt
= vect_unknown_def_type
;
6103 vec_info
*vinfo
= stmt_info
->vinfo
;
6105 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
6108 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
6111 /* Is vectorizable load? */
6112 if (!is_gimple_assign (stmt
))
6115 scalar_dest
= gimple_assign_lhs (stmt
);
6116 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
6119 code
= gimple_assign_rhs_code (stmt
);
6120 if (code
!= ARRAY_REF
6121 && code
!= BIT_FIELD_REF
6122 && code
!= INDIRECT_REF
6123 && code
!= COMPONENT_REF
6124 && code
!= IMAGPART_EXPR
6125 && code
!= REALPART_EXPR
6127 && TREE_CODE_CLASS (code
) != tcc_declaration
)
6130 if (!STMT_VINFO_DATA_REF (stmt_info
))
6133 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
6134 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
6138 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
6139 nested_in_vect_loop
= nested_in_vect_loop_p (loop
, stmt
);
6140 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
6145 /* Multiple types in SLP are handled by creating the appropriate number of
6146 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
6148 if (slp
|| PURE_SLP_STMT (stmt_info
))
6151 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
6153 gcc_assert (ncopies
>= 1);
6155 /* FORNOW. This restriction should be relaxed. */
6156 if (nested_in_vect_loop
&& ncopies
> 1)
6158 if (dump_enabled_p ())
6159 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6160 "multiple types in nested loop.\n");
6164 /* Invalidate assumptions made by dependence analysis when vectorization
6165 on the unrolled body effectively re-orders stmts. */
6167 && STMT_VINFO_MIN_NEG_DIST (stmt_info
) != 0
6168 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
6169 > STMT_VINFO_MIN_NEG_DIST (stmt_info
)))
6171 if (dump_enabled_p ())
6172 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6173 "cannot perform implicit CSE when unrolling "
6174 "with negative dependence distance\n");
6178 elem_type
= TREE_TYPE (vectype
);
6179 mode
= TYPE_MODE (vectype
);
6181 /* FORNOW. In some cases can vectorize even if data-type not supported
6182 (e.g. - data copies). */
6183 if (optab_handler (mov_optab
, mode
) == CODE_FOR_nothing
)
6185 if (dump_enabled_p ())
6186 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6187 "Aligned load, but unsupported type.\n");
6191 /* Check if the load is a part of an interleaving chain. */
6192 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
))
6194 grouped_load
= true;
6196 gcc_assert (!nested_in_vect_loop
&& !STMT_VINFO_GATHER_SCATTER_P (stmt_info
));
6198 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
6200 /* If this is single-element interleaving with an element distance
6201 that leaves unused vector loads around punt - we at least create
6202 very sub-optimal code in that case (and blow up memory,
6204 if (first_stmt
== stmt
6205 && !GROUP_NEXT_ELEMENT (stmt_info
)
6206 && GROUP_SIZE (stmt_info
) > TYPE_VECTOR_SUBPARTS (vectype
))
6208 if (dump_enabled_p ())
6209 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6210 "single-element interleaving not supported "
6211 "for not adjacent vector loads\n");
6215 if (slp
&& SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ())
6218 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
6220 && !PURE_SLP_STMT (stmt_info
)
6221 && !STMT_VINFO_STRIDED_P (stmt_info
))
6223 if (vect_load_lanes_supported (vectype
, group_size
))
6224 load_lanes_p
= true;
6225 else if (!vect_grouped_load_supported (vectype
, group_size
))
6229 /* Invalidate assumptions made by dependence analysis when vectorization
6230 on the unrolled body effectively re-orders stmts. */
6231 if (!PURE_SLP_STMT (stmt_info
)
6232 && STMT_VINFO_MIN_NEG_DIST (stmt_info
) != 0
6233 && ((unsigned)LOOP_VINFO_VECT_FACTOR (loop_vinfo
)
6234 > STMT_VINFO_MIN_NEG_DIST (stmt_info
)))
6236 if (dump_enabled_p ())
6237 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6238 "cannot perform implicit CSE when performing "
6239 "group loads with negative dependence distance\n");
6243 /* Similarly when the stmt is a load that is both part of a SLP
6244 instance and a loop vectorized stmt via the same-dr mechanism
6245 we have to give up. */
6246 if (STMT_VINFO_GROUP_SAME_DR_STMT (stmt_info
)
6247 && (STMT_SLP_TYPE (stmt_info
)
6248 != STMT_SLP_TYPE (vinfo_for_stmt
6249 (STMT_VINFO_GROUP_SAME_DR_STMT (stmt_info
)))))
6251 if (dump_enabled_p ())
6252 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6253 "conflicting SLP types for CSEd load\n");
6259 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
6262 gather_decl
= vect_check_gather_scatter (stmt
, loop_vinfo
, &gather_base
,
6263 &gather_off
, &gather_scale
);
6264 gcc_assert (gather_decl
);
6265 if (!vect_is_simple_use (gather_off
, vinfo
, &def_stmt
, &gather_dt
,
6266 &gather_off_vectype
))
6268 if (dump_enabled_p ())
6269 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6270 "gather index use not simple.\n");
6274 else if (STMT_VINFO_STRIDED_P (stmt_info
))
6277 && (slp
|| PURE_SLP_STMT (stmt_info
)))
6278 && (group_size
> nunits
6279 || nunits
% group_size
!= 0))
6281 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6282 "unhandled strided group load\n");
6288 negative
= tree_int_cst_compare (nested_in_vect_loop
6289 ? STMT_VINFO_DR_STEP (stmt_info
)
6291 size_zero_node
) < 0;
6292 if (negative
&& ncopies
> 1)
6294 if (dump_enabled_p ())
6295 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6296 "multiple types with negative step.\n");
6304 if (dump_enabled_p ())
6305 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6306 "negative step for group load not supported"
6310 alignment_support_scheme
= vect_supportable_dr_alignment (dr
, false);
6311 if (alignment_support_scheme
!= dr_aligned
6312 && alignment_support_scheme
!= dr_unaligned_supported
)
6314 if (dump_enabled_p ())
6315 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6316 "negative step but alignment required.\n");
6319 if (!perm_mask_for_reverse (vectype
))
6321 if (dump_enabled_p ())
6322 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
6323 "negative step and reversing not supported."
6330 if (!vec_stmt
) /* transformation not required. */
6332 STMT_VINFO_TYPE (stmt_info
) = load_vec_info_type
;
6333 /* The SLP costs are calculated during SLP analysis. */
6334 if (!PURE_SLP_STMT (stmt_info
))
6335 vect_model_load_cost (stmt_info
, ncopies
, load_lanes_p
,
6340 if (dump_enabled_p ())
6341 dump_printf_loc (MSG_NOTE
, vect_location
,
6342 "transform load. ncopies = %d\n", ncopies
);
6346 ensure_base_align (stmt_info
, dr
);
6348 if (STMT_VINFO_GATHER_SCATTER_P (stmt_info
))
6350 tree vec_oprnd0
= NULL_TREE
, op
;
6351 tree arglist
= TYPE_ARG_TYPES (TREE_TYPE (gather_decl
));
6352 tree rettype
, srctype
, ptrtype
, idxtype
, masktype
, scaletype
;
6353 tree ptr
, mask
, var
, scale
, merge
, perm_mask
= NULL_TREE
, prev_res
= NULL_TREE
;
6354 edge pe
= loop_preheader_edge (loop
);
6357 enum { NARROW
, NONE
, WIDEN
} modifier
;
6358 int gather_off_nunits
= TYPE_VECTOR_SUBPARTS (gather_off_vectype
);
6360 if (nunits
== gather_off_nunits
)
6362 else if (nunits
== gather_off_nunits
/ 2)
6364 unsigned char *sel
= XALLOCAVEC (unsigned char, gather_off_nunits
);
6367 for (i
= 0; i
< gather_off_nunits
; ++i
)
6368 sel
[i
] = i
| nunits
;
6370 perm_mask
= vect_gen_perm_mask_checked (gather_off_vectype
, sel
);
6372 else if (nunits
== gather_off_nunits
* 2)
6374 unsigned char *sel
= XALLOCAVEC (unsigned char, nunits
);
6377 for (i
= 0; i
< nunits
; ++i
)
6378 sel
[i
] = i
< gather_off_nunits
6379 ? i
: i
+ nunits
- gather_off_nunits
;
6381 perm_mask
= vect_gen_perm_mask_checked (vectype
, sel
);
6387 rettype
= TREE_TYPE (TREE_TYPE (gather_decl
));
6388 srctype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6389 ptrtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6390 idxtype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6391 masktype
= TREE_VALUE (arglist
); arglist
= TREE_CHAIN (arglist
);
6392 scaletype
= TREE_VALUE (arglist
);
6393 gcc_checking_assert (types_compatible_p (srctype
, rettype
));
6395 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
6397 ptr
= fold_convert (ptrtype
, gather_base
);
6398 if (!is_gimple_min_invariant (ptr
))
6400 ptr
= force_gimple_operand (ptr
, &seq
, true, NULL_TREE
);
6401 new_bb
= gsi_insert_seq_on_edge_immediate (pe
, seq
);
6402 gcc_assert (!new_bb
);
6405 /* Currently we support only unconditional gather loads,
6406 so mask should be all ones. */
6407 if (TREE_CODE (masktype
) == INTEGER_TYPE
)
6408 mask
= build_int_cst (masktype
, -1);
6409 else if (TREE_CODE (TREE_TYPE (masktype
)) == INTEGER_TYPE
)
6411 mask
= build_int_cst (TREE_TYPE (masktype
), -1);
6412 mask
= build_vector_from_val (masktype
, mask
);
6413 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
6415 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (masktype
)))
6419 for (j
= 0; j
< 6; ++j
)
6421 real_from_target (&r
, tmp
, TYPE_MODE (TREE_TYPE (masktype
)));
6422 mask
= build_real (TREE_TYPE (masktype
), r
);
6423 mask
= build_vector_from_val (masktype
, mask
);
6424 mask
= vect_init_vector (stmt
, mask
, masktype
, NULL
);
6429 scale
= build_int_cst (scaletype
, gather_scale
);
6431 if (TREE_CODE (TREE_TYPE (rettype
)) == INTEGER_TYPE
)
6432 merge
= build_int_cst (TREE_TYPE (rettype
), 0);
6433 else if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (rettype
)))
6437 for (j
= 0; j
< 6; ++j
)
6439 real_from_target (&r
, tmp
, TYPE_MODE (TREE_TYPE (rettype
)));
6440 merge
= build_real (TREE_TYPE (rettype
), r
);
6444 merge
= build_vector_from_val (rettype
, merge
);
6445 merge
= vect_init_vector (stmt
, merge
, rettype
, NULL
);
6447 prev_stmt_info
= NULL
;
6448 for (j
= 0; j
< ncopies
; ++j
)
6450 if (modifier
== WIDEN
&& (j
& 1))
6451 op
= permute_vec_elements (vec_oprnd0
, vec_oprnd0
,
6452 perm_mask
, stmt
, gsi
);
6455 = vect_get_vec_def_for_operand (gather_off
, stmt
);
6458 = vect_get_vec_def_for_stmt_copy (gather_dt
, vec_oprnd0
);
6460 if (!useless_type_conversion_p (idxtype
, TREE_TYPE (op
)))
6462 gcc_assert (TYPE_VECTOR_SUBPARTS (TREE_TYPE (op
))
6463 == TYPE_VECTOR_SUBPARTS (idxtype
));
6464 var
= vect_get_new_ssa_name (idxtype
, vect_simple_var
);
6465 op
= build1 (VIEW_CONVERT_EXPR
, idxtype
, op
);
6467 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
6468 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6473 = gimple_build_call (gather_decl
, 5, merge
, ptr
, op
, mask
, scale
);
6475 if (!useless_type_conversion_p (vectype
, rettype
))
6477 gcc_assert (TYPE_VECTOR_SUBPARTS (vectype
)
6478 == TYPE_VECTOR_SUBPARTS (rettype
));
6479 op
= vect_get_new_ssa_name (rettype
, vect_simple_var
);
6480 gimple_call_set_lhs (new_stmt
, op
);
6481 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6482 var
= make_ssa_name (vec_dest
);
6483 op
= build1 (VIEW_CONVERT_EXPR
, vectype
, op
);
6485 = gimple_build_assign (var
, VIEW_CONVERT_EXPR
, op
);
6489 var
= make_ssa_name (vec_dest
, new_stmt
);
6490 gimple_call_set_lhs (new_stmt
, var
);
6493 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6495 if (modifier
== NARROW
)
6502 var
= permute_vec_elements (prev_res
, var
,
6503 perm_mask
, stmt
, gsi
);
6504 new_stmt
= SSA_NAME_DEF_STMT (var
);
6507 if (prev_stmt_info
== NULL
)
6508 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6510 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6511 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6515 else if (STMT_VINFO_STRIDED_P (stmt_info
))
6517 gimple_stmt_iterator incr_gsi
;
6523 vec
<constructor_elt
, va_gc
> *v
= NULL
;
6524 gimple_seq stmts
= NULL
;
6525 tree stride_base
, stride_step
, alias_off
;
6527 gcc_assert (!nested_in_vect_loop
);
6529 if (slp
&& grouped_load
)
6530 first_dr
= STMT_VINFO_DATA_REF
6531 (vinfo_for_stmt (GROUP_FIRST_ELEMENT (stmt_info
)));
6536 = fold_build_pointer_plus
6537 (DR_BASE_ADDRESS (first_dr
),
6538 size_binop (PLUS_EXPR
,
6539 convert_to_ptrofftype (DR_OFFSET (first_dr
)),
6540 convert_to_ptrofftype (DR_INIT (first_dr
))));
6541 stride_step
= fold_convert (sizetype
, DR_STEP (first_dr
));
6543 /* For a load with loop-invariant (but other than power-of-2)
6544 stride (i.e. not a grouped access) like so:
6546 for (i = 0; i < n; i += stride)
6549 we generate a new induction variable and new accesses to
6550 form a new vector (or vectors, depending on ncopies):
6552 for (j = 0; ; j += VF*stride)
6554 tmp2 = array[j + stride];
6556 vectemp = {tmp1, tmp2, ...}
6559 ivstep
= fold_build2 (MULT_EXPR
, TREE_TYPE (stride_step
), stride_step
,
6560 build_int_cst (TREE_TYPE (stride_step
), vf
));
6562 standard_iv_increment_position (loop
, &incr_gsi
, &insert_after
);
6564 create_iv (unshare_expr (stride_base
), unshare_expr (ivstep
), NULL
,
6565 loop
, &incr_gsi
, insert_after
,
6567 incr
= gsi_stmt (incr_gsi
);
6568 set_vinfo_for_stmt (incr
, new_stmt_vec_info (incr
, loop_vinfo
));
6570 stride_step
= force_gimple_operand (unshare_expr (stride_step
),
6571 &stmts
, true, NULL_TREE
);
6573 gsi_insert_seq_on_edge_immediate (loop_preheader_edge (loop
), stmts
);
6575 prev_stmt_info
= NULL
;
6576 running_off
= offvar
;
6577 alias_off
= build_int_cst (reference_alias_ptr_type (DR_REF (first_dr
)), 0);
6578 int nloads
= nunits
;
6579 tree ltype
= TREE_TYPE (vectype
);
6580 auto_vec
<tree
> dr_chain
;
6583 nloads
= nunits
/ group_size
;
6584 if (group_size
< nunits
)
6585 ltype
= build_vector_type (TREE_TYPE (vectype
), group_size
);
6588 ltype
= build_aligned_type (ltype
, TYPE_ALIGN (TREE_TYPE (vectype
)));
6589 ncopies
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6591 dr_chain
.create (ncopies
);
6593 for (j
= 0; j
< ncopies
; j
++)
6599 vec_alloc (v
, nloads
);
6600 for (i
= 0; i
< nloads
; i
++)
6602 tree newref
, newoff
;
6604 newref
= build2 (MEM_REF
, ltype
, running_off
, alias_off
);
6606 newref
= force_gimple_operand_gsi (gsi
, newref
, true,
6609 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
, newref
);
6610 newoff
= copy_ssa_name (running_off
);
6611 incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
6612 running_off
, stride_step
);
6613 vect_finish_stmt_generation (stmt
, incr
, gsi
);
6615 running_off
= newoff
;
6618 vec_inv
= build_constructor (vectype
, v
);
6619 new_temp
= vect_init_vector (stmt
, vec_inv
, vectype
, gsi
);
6620 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
6624 new_stmt
= gimple_build_assign (make_ssa_name (ltype
),
6625 build2 (MEM_REF
, ltype
,
6626 running_off
, alias_off
));
6627 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6629 tree newoff
= copy_ssa_name (running_off
);
6630 gimple
*incr
= gimple_build_assign (newoff
, POINTER_PLUS_EXPR
,
6631 running_off
, stride_step
);
6632 vect_finish_stmt_generation (stmt
, incr
, gsi
);
6634 running_off
= newoff
;
6639 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
6641 dr_chain
.quick_push (gimple_assign_lhs (new_stmt
));
6646 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
6648 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
6649 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
6653 vect_transform_slp_perm_load (slp_node
, dr_chain
, gsi
, vf
,
6654 slp_node_instance
, false);
6660 first_stmt
= GROUP_FIRST_ELEMENT (stmt_info
);
6662 && !SLP_TREE_LOAD_PERMUTATION (slp_node
).exists ()
6663 && first_stmt
!= SLP_TREE_SCALAR_STMTS (slp_node
)[0])
6664 first_stmt
= SLP_TREE_SCALAR_STMTS (slp_node
)[0];
6666 /* Check if the chain of loads is already vectorized. */
6667 if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt
))
6668 /* For SLP we would need to copy over SLP_TREE_VEC_STMTS.
6669 ??? But we can only do so if there is exactly one
6670 as we have no way to get at the rest. Leave the CSE
6672 ??? With the group load eventually participating
6673 in multiple different permutations (having multiple
6674 slp nodes which refer to the same group) the CSE
6675 is even wrong code. See PR56270. */
6678 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
6681 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
6682 group_size
= GROUP_SIZE (vinfo_for_stmt (first_stmt
));
6685 /* VEC_NUM is the number of vect stmts to be created for this group. */
6688 grouped_load
= false;
6689 /* For SLP permutation support we need to load the whole group,
6690 not only the number of vector stmts the permutation result
6693 vec_num
= (group_size
* vf
+ nunits
- 1) / nunits
;
6695 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
6696 group_gap_adj
= vf
* group_size
- nunits
* vec_num
;
6699 vec_num
= group_size
;
6705 group_size
= vec_num
= 1;
6709 alignment_support_scheme
= vect_supportable_dr_alignment (first_dr
, false);
6710 gcc_assert (alignment_support_scheme
);
6711 /* Targets with load-lane instructions must not require explicit
6713 gcc_assert (!load_lanes_p
6714 || alignment_support_scheme
== dr_aligned
6715 || alignment_support_scheme
== dr_unaligned_supported
);
6717 /* In case the vectorization factor (VF) is bigger than the number
6718 of elements that we can fit in a vectype (nunits), we have to generate
6719 more than one vector stmt - i.e - we need to "unroll" the
6720 vector stmt by a factor VF/nunits. In doing so, we record a pointer
6721 from one copy of the vector stmt to the next, in the field
6722 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
6723 stages to find the correct vector defs to be used when vectorizing
6724 stmts that use the defs of the current stmt. The example below
6725 illustrates the vectorization process when VF=16 and nunits=4 (i.e., we
6726 need to create 4 vectorized stmts):
6728 before vectorization:
6729 RELATED_STMT VEC_STMT
6733 step 1: vectorize stmt S1:
6734 We first create the vector stmt VS1_0, and, as usual, record a
6735 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
6736 Next, we create the vector stmt VS1_1, and record a pointer to
6737 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
6738 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
6740 RELATED_STMT VEC_STMT
6741 VS1_0: vx0 = memref0 VS1_1 -
6742 VS1_1: vx1 = memref1 VS1_2 -
6743 VS1_2: vx2 = memref2 VS1_3 -
6744 VS1_3: vx3 = memref3 - -
6745 S1: x = load - VS1_0
6748 See in documentation in vect_get_vec_def_for_stmt_copy for how the
6749 information we recorded in RELATED_STMT field is used to vectorize
6752 /* In case of interleaving (non-unit grouped access):
6759 Vectorized loads are created in the order of memory accesses
6760 starting from the access of the first stmt of the chain:
6763 VS2: vx1 = &base + vec_size*1
6764 VS3: vx3 = &base + vec_size*2
6765 VS4: vx4 = &base + vec_size*3
6767 Then permutation statements are generated:
6769 VS5: vx5 = VEC_PERM_EXPR < vx0, vx1, { 0, 2, ..., i*2 } >
6770 VS6: vx6 = VEC_PERM_EXPR < vx0, vx1, { 1, 3, ..., i*2+1 } >
6773 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
6774 (the order of the data-refs in the output of vect_permute_load_chain
6775 corresponds to the order of scalar stmts in the interleaving chain - see
6776 the documentation of vect_permute_load_chain()).
6777 The generation of permutation stmts and recording them in
6778 STMT_VINFO_VEC_STMT is done in vect_transform_grouped_load().
6780 In case of both multiple types and interleaving, the vector loads and
6781 permutation stmts above are created for every copy. The result vector
6782 stmts are put in STMT_VINFO_VEC_STMT for the first copy and in the
6783 corresponding STMT_VINFO_RELATED_STMT for the next copies. */
6785 /* If the data reference is aligned (dr_aligned) or potentially unaligned
6786 on a target that supports unaligned accesses (dr_unaligned_supported)
6787 we generate the following code:
6791 p = p + indx * vectype_size;
6796 Otherwise, the data reference is potentially unaligned on a target that
6797 does not support unaligned accesses (dr_explicit_realign_optimized) -
6798 then generate the following code, in which the data in each iteration is
6799 obtained by two vector loads, one from the previous iteration, and one
6800 from the current iteration:
6802 msq_init = *(floor(p1))
6803 p2 = initial_addr + VS - 1;
6804 realignment_token = call target_builtin;
6807 p2 = p2 + indx * vectype_size
6809 vec_dest = realign_load (msq, lsq, realignment_token)
6814 /* If the misalignment remains the same throughout the execution of the
6815 loop, we can create the init_addr and permutation mask at the loop
6816 preheader. Otherwise, it needs to be created inside the loop.
6817 This can only occur when vectorizing memory accesses in the inner-loop
6818 nested within an outer-loop that is being vectorized. */
6820 if (nested_in_vect_loop
6821 && (TREE_INT_CST_LOW (DR_STEP (dr
))
6822 % GET_MODE_SIZE (TYPE_MODE (vectype
)) != 0))
6824 gcc_assert (alignment_support_scheme
!= dr_explicit_realign_optimized
);
6825 compute_in_loop
= true;
6828 if ((alignment_support_scheme
== dr_explicit_realign_optimized
6829 || alignment_support_scheme
== dr_explicit_realign
)
6830 && !compute_in_loop
)
6832 msq
= vect_setup_realignment (first_stmt
, gsi
, &realignment_token
,
6833 alignment_support_scheme
, NULL_TREE
,
6835 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
6837 phi
= as_a
<gphi
*> (SSA_NAME_DEF_STMT (msq
));
6838 byte_offset
= size_binop (MINUS_EXPR
, TYPE_SIZE_UNIT (vectype
),
6846 offset
= size_int (-TYPE_VECTOR_SUBPARTS (vectype
) + 1);
6849 aggr_type
= build_array_type_nelts (elem_type
, vec_num
* nunits
);
6851 aggr_type
= vectype
;
6853 prev_stmt_info
= NULL
;
6854 for (j
= 0; j
< ncopies
; j
++)
6856 /* 1. Create the vector or array pointer update chain. */
6859 bool simd_lane_access_p
6860 = STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info
);
6861 if (simd_lane_access_p
6862 && TREE_CODE (DR_BASE_ADDRESS (first_dr
)) == ADDR_EXPR
6863 && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr
), 0))
6864 && integer_zerop (DR_OFFSET (first_dr
))
6865 && integer_zerop (DR_INIT (first_dr
))
6866 && alias_sets_conflict_p (get_alias_set (aggr_type
),
6867 get_alias_set (DR_REF (first_dr
)))
6868 && (alignment_support_scheme
== dr_aligned
6869 || alignment_support_scheme
== dr_unaligned_supported
))
6871 dataref_ptr
= unshare_expr (DR_BASE_ADDRESS (first_dr
));
6872 dataref_offset
= build_int_cst (reference_alias_ptr_type
6873 (DR_REF (first_dr
)), 0);
6878 = vect_create_data_ref_ptr (first_stmt
, aggr_type
, at_loop
,
6879 offset
, &dummy
, gsi
, &ptr_incr
,
6880 simd_lane_access_p
, &inv_p
,
6883 else if (dataref_offset
)
6884 dataref_offset
= int_const_binop (PLUS_EXPR
, dataref_offset
,
6885 TYPE_SIZE_UNIT (aggr_type
));
6887 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
6888 TYPE_SIZE_UNIT (aggr_type
));
6890 if (grouped_load
|| slp_perm
)
6891 dr_chain
.create (vec_num
);
6897 vec_array
= create_vector_array (vectype
, vec_num
);
6900 VEC_ARRAY = LOAD_LANES (MEM_REF[...all elements...]). */
6901 data_ref
= create_array_ref (aggr_type
, dataref_ptr
, first_dr
);
6902 new_stmt
= gimple_build_call_internal (IFN_LOAD_LANES
, 1, data_ref
);
6903 gimple_call_set_lhs (new_stmt
, vec_array
);
6904 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6906 /* Extract each vector into an SSA_NAME. */
6907 for (i
= 0; i
< vec_num
; i
++)
6909 new_temp
= read_vector_array (stmt
, gsi
, scalar_dest
,
6911 dr_chain
.quick_push (new_temp
);
6914 /* Record the mapping between SSA_NAMEs and statements. */
6915 vect_record_grouped_load_vectors (stmt
, dr_chain
);
6919 for (i
= 0; i
< vec_num
; i
++)
6922 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
6925 /* 2. Create the vector-load in the loop. */
6926 switch (alignment_support_scheme
)
6929 case dr_unaligned_supported
:
6931 unsigned int align
, misalign
;
6934 = fold_build2 (MEM_REF
, vectype
, dataref_ptr
,
6937 : build_int_cst (reference_alias_ptr_type
6938 (DR_REF (first_dr
)), 0));
6939 align
= TYPE_ALIGN_UNIT (vectype
);
6940 if (alignment_support_scheme
== dr_aligned
)
6942 gcc_assert (aligned_access_p (first_dr
));
6945 else if (DR_MISALIGNMENT (first_dr
) == -1)
6947 if (DR_VECT_AUX (first_dr
)->base_element_aligned
)
6948 align
= TYPE_ALIGN_UNIT (elem_type
);
6950 align
= (get_object_alignment (DR_REF (first_dr
))
6953 TREE_TYPE (data_ref
)
6954 = build_aligned_type (TREE_TYPE (data_ref
),
6955 align
* BITS_PER_UNIT
);
6959 TREE_TYPE (data_ref
)
6960 = build_aligned_type (TREE_TYPE (data_ref
),
6961 TYPE_ALIGN (elem_type
));
6962 misalign
= DR_MISALIGNMENT (first_dr
);
6964 if (dataref_offset
== NULL_TREE
6965 && TREE_CODE (dataref_ptr
) == SSA_NAME
)
6966 set_ptr_info_alignment (get_ptr_info (dataref_ptr
),
6970 case dr_explicit_realign
:
6974 tree vs
= size_int (TYPE_VECTOR_SUBPARTS (vectype
));
6976 if (compute_in_loop
)
6977 msq
= vect_setup_realignment (first_stmt
, gsi
,
6979 dr_explicit_realign
,
6982 if (TREE_CODE (dataref_ptr
) == SSA_NAME
)
6983 ptr
= copy_ssa_name (dataref_ptr
);
6985 ptr
= make_ssa_name (TREE_TYPE (dataref_ptr
));
6986 new_stmt
= gimple_build_assign
6987 (ptr
, BIT_AND_EXPR
, dataref_ptr
,
6989 (TREE_TYPE (dataref_ptr
),
6990 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
6991 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
6993 = build2 (MEM_REF
, vectype
, ptr
,
6994 build_int_cst (reference_alias_ptr_type
6995 (DR_REF (first_dr
)), 0));
6996 vec_dest
= vect_create_destination_var (scalar_dest
,
6998 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
6999 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
7000 gimple_assign_set_lhs (new_stmt
, new_temp
);
7001 gimple_set_vdef (new_stmt
, gimple_vdef (stmt
));
7002 gimple_set_vuse (new_stmt
, gimple_vuse (stmt
));
7003 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7006 bump
= size_binop (MULT_EXPR
, vs
,
7007 TYPE_SIZE_UNIT (elem_type
));
7008 bump
= size_binop (MINUS_EXPR
, bump
, size_one_node
);
7009 ptr
= bump_vector_ptr (dataref_ptr
, NULL
, gsi
, stmt
, bump
);
7010 new_stmt
= gimple_build_assign
7011 (NULL_TREE
, BIT_AND_EXPR
, ptr
,
7014 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
7015 ptr
= copy_ssa_name (ptr
, new_stmt
);
7016 gimple_assign_set_lhs (new_stmt
, ptr
);
7017 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7019 = build2 (MEM_REF
, vectype
, ptr
,
7020 build_int_cst (reference_alias_ptr_type
7021 (DR_REF (first_dr
)), 0));
7024 case dr_explicit_realign_optimized
:
7025 if (TREE_CODE (dataref_ptr
) == SSA_NAME
)
7026 new_temp
= copy_ssa_name (dataref_ptr
);
7028 new_temp
= make_ssa_name (TREE_TYPE (dataref_ptr
));
7029 new_stmt
= gimple_build_assign
7030 (new_temp
, BIT_AND_EXPR
, dataref_ptr
,
7032 (TREE_TYPE (dataref_ptr
),
7033 -(HOST_WIDE_INT
)TYPE_ALIGN_UNIT (vectype
)));
7034 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7036 = build2 (MEM_REF
, vectype
, new_temp
,
7037 build_int_cst (reference_alias_ptr_type
7038 (DR_REF (first_dr
)), 0));
7043 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
7044 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
7045 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
7046 gimple_assign_set_lhs (new_stmt
, new_temp
);
7047 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7049 /* 3. Handle explicit realignment if necessary/supported.
7051 vec_dest = realign_load (msq, lsq, realignment_token) */
7052 if (alignment_support_scheme
== dr_explicit_realign_optimized
7053 || alignment_support_scheme
== dr_explicit_realign
)
7055 lsq
= gimple_assign_lhs (new_stmt
);
7056 if (!realignment_token
)
7057 realignment_token
= dataref_ptr
;
7058 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
7059 new_stmt
= gimple_build_assign (vec_dest
, REALIGN_LOAD_EXPR
,
7060 msq
, lsq
, realignment_token
);
7061 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
7062 gimple_assign_set_lhs (new_stmt
, new_temp
);
7063 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7065 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
7068 if (i
== vec_num
- 1 && j
== ncopies
- 1)
7069 add_phi_arg (phi
, lsq
,
7070 loop_latch_edge (containing_loop
),
7076 /* 4. Handle invariant-load. */
7077 if (inv_p
&& !bb_vinfo
)
7079 gcc_assert (!grouped_load
);
7080 /* If we have versioned for aliasing or the loop doesn't
7081 have any data dependencies that would preclude this,
7082 then we are sure this is a loop invariant load and
7083 thus we can insert it on the preheader edge. */
7084 if (LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo
)
7085 && !nested_in_vect_loop
7086 && hoist_defs_of_uses (stmt
, loop
))
7088 if (dump_enabled_p ())
7090 dump_printf_loc (MSG_NOTE
, vect_location
,
7091 "hoisting out of the vectorized "
7093 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7095 tree tem
= copy_ssa_name (scalar_dest
);
7096 gsi_insert_on_edge_immediate
7097 (loop_preheader_edge (loop
),
7098 gimple_build_assign (tem
,
7100 (gimple_assign_rhs1 (stmt
))));
7101 new_temp
= vect_init_vector (stmt
, tem
, vectype
, NULL
);
7105 gimple_stmt_iterator gsi2
= *gsi
;
7107 new_temp
= vect_init_vector (stmt
, scalar_dest
,
7110 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
7111 set_vinfo_for_stmt (new_stmt
,
7112 new_stmt_vec_info (new_stmt
, vinfo
));
7117 tree perm_mask
= perm_mask_for_reverse (vectype
);
7118 new_temp
= permute_vec_elements (new_temp
, new_temp
,
7119 perm_mask
, stmt
, gsi
);
7120 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
7123 /* Collect vector loads and later create their permutation in
7124 vect_transform_grouped_load (). */
7125 if (grouped_load
|| slp_perm
)
7126 dr_chain
.quick_push (new_temp
);
7128 /* Store vector loads in the corresponding SLP_NODE. */
7129 if (slp
&& !slp_perm
)
7130 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
7132 /* Bump the vector pointer to account for a gap or for excess
7133 elements loaded for a permuted SLP load. */
7134 if (group_gap_adj
!= 0)
7138 = wide_int_to_tree (sizetype
,
7139 wi::smul (TYPE_SIZE_UNIT (elem_type
),
7140 group_gap_adj
, &ovf
));
7141 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
,
7146 if (slp
&& !slp_perm
)
7151 if (!vect_transform_slp_perm_load (slp_node
, dr_chain
, gsi
, vf
,
7152 slp_node_instance
, false))
7154 dr_chain
.release ();
7163 vect_transform_grouped_load (stmt
, dr_chain
, group_size
, gsi
);
7164 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
7169 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
7171 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
7172 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
7175 dr_chain
.release ();
7181 /* Function vect_is_simple_cond.
7184 LOOP - the loop that is being vectorized.
7185 COND - Condition that is checked for simple use.
7188 *COMP_VECTYPE - the vector type for the comparison.
7190 Returns whether a COND can be vectorized. Checks whether
7191 condition operands are supportable using vec_is_simple_use. */
7194 vect_is_simple_cond (tree cond
, vec_info
*vinfo
, tree
*comp_vectype
)
7197 enum vect_def_type dt
;
7198 tree vectype1
= NULL_TREE
, vectype2
= NULL_TREE
;
7201 if (TREE_CODE (cond
) == SSA_NAME
7202 && TREE_CODE (TREE_TYPE (cond
)) == BOOLEAN_TYPE
)
7204 gimple
*lhs_def_stmt
= SSA_NAME_DEF_STMT (cond
);
7205 if (!vect_is_simple_use (cond
, vinfo
, &lhs_def_stmt
,
7208 || !VECTOR_BOOLEAN_TYPE_P (*comp_vectype
))
7213 if (!COMPARISON_CLASS_P (cond
))
7216 lhs
= TREE_OPERAND (cond
, 0);
7217 rhs
= TREE_OPERAND (cond
, 1);
7219 if (TREE_CODE (lhs
) == SSA_NAME
)
7221 gimple
*lhs_def_stmt
= SSA_NAME_DEF_STMT (lhs
);
7222 if (!vect_is_simple_use (lhs
, vinfo
, &lhs_def_stmt
, &dt
, &vectype1
))
7225 else if (TREE_CODE (lhs
) != INTEGER_CST
&& TREE_CODE (lhs
) != REAL_CST
7226 && TREE_CODE (lhs
) != FIXED_CST
)
7229 if (TREE_CODE (rhs
) == SSA_NAME
)
7231 gimple
*rhs_def_stmt
= SSA_NAME_DEF_STMT (rhs
);
7232 if (!vect_is_simple_use (rhs
, vinfo
, &rhs_def_stmt
, &dt
, &vectype2
))
7235 else if (TREE_CODE (rhs
) != INTEGER_CST
&& TREE_CODE (rhs
) != REAL_CST
7236 && TREE_CODE (rhs
) != FIXED_CST
)
7239 *comp_vectype
= vectype1
? vectype1
: vectype2
;
7243 /* vectorizable_condition.
7245 Check if STMT is conditional modify expression that can be vectorized.
7246 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
7247 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
7250 When STMT is vectorized as nested cycle, REDUC_DEF is the vector variable
7251 to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in
7252 else clause if it is 2).
7254 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
7257 vectorizable_condition (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
7258 gimple
**vec_stmt
, tree reduc_def
, int reduc_index
,
7261 tree scalar_dest
= NULL_TREE
;
7262 tree vec_dest
= NULL_TREE
;
7263 tree cond_expr
, then_clause
, else_clause
;
7264 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7265 tree comp_vectype
= NULL_TREE
;
7266 tree vec_cond_lhs
= NULL_TREE
, vec_cond_rhs
= NULL_TREE
;
7267 tree vec_then_clause
= NULL_TREE
, vec_else_clause
= NULL_TREE
;
7268 tree vec_compare
, vec_cond_expr
;
7270 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
7271 enum vect_def_type dt
, dts
[4];
7273 enum tree_code code
;
7274 stmt_vec_info prev_stmt_info
= NULL
;
7276 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
7277 vec
<tree
> vec_oprnds0
= vNULL
;
7278 vec
<tree
> vec_oprnds1
= vNULL
;
7279 vec
<tree
> vec_oprnds2
= vNULL
;
7280 vec
<tree
> vec_oprnds3
= vNULL
;
7282 bool masked
= false;
7284 if (reduc_index
&& STMT_SLP_TYPE (stmt_info
))
7287 if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info
) == TREE_CODE_REDUCTION
)
7289 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
7292 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
7293 && !(STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
7297 /* FORNOW: not yet supported. */
7298 if (STMT_VINFO_LIVE_P (stmt_info
))
7300 if (dump_enabled_p ())
7301 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7302 "value used after loop.\n");
7307 /* Is vectorizable conditional operation? */
7308 if (!is_gimple_assign (stmt
))
7311 code
= gimple_assign_rhs_code (stmt
);
7313 if (code
!= COND_EXPR
)
7316 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
7317 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
7319 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
7322 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
7324 gcc_assert (ncopies
>= 1);
7325 if (reduc_index
&& ncopies
> 1)
7326 return false; /* FORNOW */
7328 cond_expr
= gimple_assign_rhs1 (stmt
);
7329 then_clause
= gimple_assign_rhs2 (stmt
);
7330 else_clause
= gimple_assign_rhs3 (stmt
);
7332 if (!vect_is_simple_cond (cond_expr
, stmt_info
->vinfo
, &comp_vectype
)
7337 if (!vect_is_simple_use (then_clause
, stmt_info
->vinfo
, &def_stmt
, &dt
))
7339 if (!vect_is_simple_use (else_clause
, stmt_info
->vinfo
, &def_stmt
, &dt
))
7342 if (VECTOR_BOOLEAN_TYPE_P (comp_vectype
))
7344 vec_cmp_type
= comp_vectype
;
7348 vec_cmp_type
= build_same_sized_truth_vector_type (comp_vectype
);
7349 if (vec_cmp_type
== NULL_TREE
)
7354 STMT_VINFO_TYPE (stmt_info
) = condition_vec_info_type
;
7355 return expand_vec_cond_expr_p (vectype
, comp_vectype
);
7362 vec_oprnds0
.create (1);
7363 vec_oprnds1
.create (1);
7364 vec_oprnds2
.create (1);
7365 vec_oprnds3
.create (1);
7369 scalar_dest
= gimple_assign_lhs (stmt
);
7370 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
7372 /* Handle cond expr. */
7373 for (j
= 0; j
< ncopies
; j
++)
7375 gassign
*new_stmt
= NULL
;
7380 auto_vec
<tree
, 4> ops
;
7381 auto_vec
<vec
<tree
>, 4> vec_defs
;
7384 ops
.safe_push (cond_expr
);
7387 ops
.safe_push (TREE_OPERAND (cond_expr
, 0));
7388 ops
.safe_push (TREE_OPERAND (cond_expr
, 1));
7390 ops
.safe_push (then_clause
);
7391 ops
.safe_push (else_clause
);
7392 vect_get_slp_defs (ops
, slp_node
, &vec_defs
, -1);
7393 vec_oprnds3
= vec_defs
.pop ();
7394 vec_oprnds2
= vec_defs
.pop ();
7396 vec_oprnds1
= vec_defs
.pop ();
7397 vec_oprnds0
= vec_defs
.pop ();
7400 vec_defs
.release ();
7408 = vect_get_vec_def_for_operand (cond_expr
, stmt
,
7410 vect_is_simple_use (cond_expr
, stmt_info
->vinfo
,
7416 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr
, 0),
7417 stmt
, comp_vectype
);
7418 vect_is_simple_use (TREE_OPERAND (cond_expr
, 0),
7419 loop_vinfo
, >emp
, &dts
[0]);
7422 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr
, 1),
7423 stmt
, comp_vectype
);
7424 vect_is_simple_use (TREE_OPERAND (cond_expr
, 1),
7425 loop_vinfo
, >emp
, &dts
[1]);
7427 if (reduc_index
== 1)
7428 vec_then_clause
= reduc_def
;
7431 vec_then_clause
= vect_get_vec_def_for_operand (then_clause
,
7433 vect_is_simple_use (then_clause
, loop_vinfo
,
7436 if (reduc_index
== 2)
7437 vec_else_clause
= reduc_def
;
7440 vec_else_clause
= vect_get_vec_def_for_operand (else_clause
,
7442 vect_is_simple_use (else_clause
, loop_vinfo
, >emp
, &dts
[3]);
7449 = vect_get_vec_def_for_stmt_copy (dts
[0],
7450 vec_oprnds0
.pop ());
7453 = vect_get_vec_def_for_stmt_copy (dts
[1],
7454 vec_oprnds1
.pop ());
7456 vec_then_clause
= vect_get_vec_def_for_stmt_copy (dts
[2],
7457 vec_oprnds2
.pop ());
7458 vec_else_clause
= vect_get_vec_def_for_stmt_copy (dts
[3],
7459 vec_oprnds3
.pop ());
7464 vec_oprnds0
.quick_push (vec_cond_lhs
);
7466 vec_oprnds1
.quick_push (vec_cond_rhs
);
7467 vec_oprnds2
.quick_push (vec_then_clause
);
7468 vec_oprnds3
.quick_push (vec_else_clause
);
7471 /* Arguments are ready. Create the new vector stmt. */
7472 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_cond_lhs
)
7474 vec_then_clause
= vec_oprnds2
[i
];
7475 vec_else_clause
= vec_oprnds3
[i
];
7478 vec_compare
= vec_cond_lhs
;
7481 vec_cond_rhs
= vec_oprnds1
[i
];
7482 vec_compare
= build2 (TREE_CODE (cond_expr
), vec_cmp_type
,
7483 vec_cond_lhs
, vec_cond_rhs
);
7485 vec_cond_expr
= build3 (VEC_COND_EXPR
, vectype
,
7486 vec_compare
, vec_then_clause
, vec_else_clause
);
7488 new_stmt
= gimple_build_assign (vec_dest
, vec_cond_expr
);
7489 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
7490 gimple_assign_set_lhs (new_stmt
, new_temp
);
7491 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7493 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
7500 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
7502 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
7504 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
7507 vec_oprnds0
.release ();
7508 vec_oprnds1
.release ();
7509 vec_oprnds2
.release ();
7510 vec_oprnds3
.release ();
7515 /* vectorizable_comparison.
7517 Check if STMT is comparison expression that can be vectorized.
7518 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
7519 comparison, put it in VEC_STMT, and insert it at GSI.
7521 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
7524 vectorizable_comparison (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
7525 gimple
**vec_stmt
, tree reduc_def
,
7528 tree lhs
, rhs1
, rhs2
;
7529 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7530 tree vectype1
= NULL_TREE
, vectype2
= NULL_TREE
;
7531 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
7532 tree vec_rhs1
= NULL_TREE
, vec_rhs2
= NULL_TREE
;
7534 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
7535 enum vect_def_type dts
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
7538 enum tree_code code
;
7539 stmt_vec_info prev_stmt_info
= NULL
;
7541 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
7542 vec
<tree
> vec_oprnds0
= vNULL
;
7543 vec
<tree
> vec_oprnds1
= vNULL
;
7548 if (!VECTOR_BOOLEAN_TYPE_P (vectype
))
7551 mask_type
= vectype
;
7552 nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
7554 if (slp_node
|| PURE_SLP_STMT (stmt_info
))
7557 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
7559 gcc_assert (ncopies
>= 1);
7560 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
7563 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
7564 && !(STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
7568 if (STMT_VINFO_LIVE_P (stmt_info
))
7570 if (dump_enabled_p ())
7571 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7572 "value used after loop.\n");
7576 if (!is_gimple_assign (stmt
))
7579 code
= gimple_assign_rhs_code (stmt
);
7581 if (TREE_CODE_CLASS (code
) != tcc_comparison
)
7584 rhs1
= gimple_assign_rhs1 (stmt
);
7585 rhs2
= gimple_assign_rhs2 (stmt
);
7587 if (!vect_is_simple_use (rhs1
, stmt_info
->vinfo
, &def_stmt
,
7588 &dts
[0], &vectype1
))
7591 if (!vect_is_simple_use (rhs2
, stmt_info
->vinfo
, &def_stmt
,
7592 &dts
[1], &vectype2
))
7595 if (vectype1
&& vectype2
7596 && TYPE_VECTOR_SUBPARTS (vectype1
) != TYPE_VECTOR_SUBPARTS (vectype2
))
7599 vectype
= vectype1
? vectype1
: vectype2
;
7601 /* Invariant comparison. */
7604 vectype
= build_vector_type (TREE_TYPE (rhs1
), nunits
);
7605 if (tree_to_shwi (TYPE_SIZE_UNIT (vectype
)) != current_vector_size
)
7608 else if (nunits
!= TYPE_VECTOR_SUBPARTS (vectype
))
7613 STMT_VINFO_TYPE (stmt_info
) = comparison_vec_info_type
;
7614 vect_model_simple_cost (stmt_info
, ncopies
, dts
, NULL
, NULL
);
7615 return expand_vec_cmp_expr_p (vectype
, mask_type
);
7621 vec_oprnds0
.create (1);
7622 vec_oprnds1
.create (1);
7626 lhs
= gimple_assign_lhs (stmt
);
7627 mask
= vect_create_destination_var (lhs
, mask_type
);
7629 /* Handle cmp expr. */
7630 for (j
= 0; j
< ncopies
; j
++)
7632 gassign
*new_stmt
= NULL
;
7637 auto_vec
<tree
, 2> ops
;
7638 auto_vec
<vec
<tree
>, 2> vec_defs
;
7640 ops
.safe_push (rhs1
);
7641 ops
.safe_push (rhs2
);
7642 vect_get_slp_defs (ops
, slp_node
, &vec_defs
, -1);
7643 vec_oprnds1
= vec_defs
.pop ();
7644 vec_oprnds0
= vec_defs
.pop ();
7648 vec_rhs1
= vect_get_vec_def_for_operand (rhs1
, stmt
, NULL
);
7649 vec_rhs2
= vect_get_vec_def_for_operand (rhs2
, stmt
, NULL
);
7654 vec_rhs1
= vect_get_vec_def_for_stmt_copy (dts
[0],
7655 vec_oprnds0
.pop ());
7656 vec_rhs2
= vect_get_vec_def_for_stmt_copy (dts
[1],
7657 vec_oprnds1
.pop ());
7662 vec_oprnds0
.quick_push (vec_rhs1
);
7663 vec_oprnds1
.quick_push (vec_rhs2
);
7666 /* Arguments are ready. Create the new vector stmt. */
7667 FOR_EACH_VEC_ELT (vec_oprnds0
, i
, vec_rhs1
)
7669 vec_rhs2
= vec_oprnds1
[i
];
7671 new_temp
= make_ssa_name (mask
);
7672 new_stmt
= gimple_build_assign (new_temp
, code
, vec_rhs1
, vec_rhs2
);
7673 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
7675 SLP_TREE_VEC_STMTS (slp_node
).quick_push (new_stmt
);
7682 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
7684 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
7686 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
7689 vec_oprnds0
.release ();
7690 vec_oprnds1
.release ();
7695 /* Make sure the statement is vectorizable. */
7698 vect_analyze_stmt (gimple
*stmt
, bool *need_to_vectorize
, slp_tree node
)
7700 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7701 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
7702 enum vect_relevant relevance
= STMT_VINFO_RELEVANT (stmt_info
);
7704 tree scalar_type
, vectype
;
7705 gimple
*pattern_stmt
;
7706 gimple_seq pattern_def_seq
;
7708 if (dump_enabled_p ())
7710 dump_printf_loc (MSG_NOTE
, vect_location
, "==> examining statement: ");
7711 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7714 if (gimple_has_volatile_ops (stmt
))
7716 if (dump_enabled_p ())
7717 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7718 "not vectorized: stmt has volatile operands\n");
7723 /* Skip stmts that do not need to be vectorized. In loops this is expected
7725 - the COND_EXPR which is the loop exit condition
7726 - any LABEL_EXPRs in the loop
7727 - computations that are used only for array indexing or loop control.
7728 In basic blocks we only analyze statements that are a part of some SLP
7729 instance, therefore, all the statements are relevant.
7731 Pattern statement needs to be analyzed instead of the original statement
7732 if the original statement is not relevant. Otherwise, we analyze both
7733 statements. In basic blocks we are called from some SLP instance
7734 traversal, don't analyze pattern stmts instead, the pattern stmts
7735 already will be part of SLP instance. */
7737 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
7738 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
7739 && !STMT_VINFO_LIVE_P (stmt_info
))
7741 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
7743 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
7744 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
7746 /* Analyze PATTERN_STMT instead of the original stmt. */
7747 stmt
= pattern_stmt
;
7748 stmt_info
= vinfo_for_stmt (pattern_stmt
);
7749 if (dump_enabled_p ())
7751 dump_printf_loc (MSG_NOTE
, vect_location
,
7752 "==> examining pattern statement: ");
7753 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7758 if (dump_enabled_p ())
7759 dump_printf_loc (MSG_NOTE
, vect_location
, "irrelevant.\n");
7764 else if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
7767 && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt
))
7768 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt
))))
7770 /* Analyze PATTERN_STMT too. */
7771 if (dump_enabled_p ())
7773 dump_printf_loc (MSG_NOTE
, vect_location
,
7774 "==> examining pattern statement: ");
7775 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, stmt
, 0);
7778 if (!vect_analyze_stmt (pattern_stmt
, need_to_vectorize
, node
))
7782 if (is_pattern_stmt_p (stmt_info
)
7784 && (pattern_def_seq
= STMT_VINFO_PATTERN_DEF_SEQ (stmt_info
)))
7786 gimple_stmt_iterator si
;
7788 for (si
= gsi_start (pattern_def_seq
); !gsi_end_p (si
); gsi_next (&si
))
7790 gimple
*pattern_def_stmt
= gsi_stmt (si
);
7791 if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt
))
7792 || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt
)))
7794 /* Analyze def stmt of STMT if it's a pattern stmt. */
7795 if (dump_enabled_p ())
7797 dump_printf_loc (MSG_NOTE
, vect_location
,
7798 "==> examining pattern def statement: ");
7799 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, pattern_def_stmt
, 0);
7802 if (!vect_analyze_stmt (pattern_def_stmt
,
7803 need_to_vectorize
, node
))
7809 switch (STMT_VINFO_DEF_TYPE (stmt_info
))
7811 case vect_internal_def
:
7814 case vect_reduction_def
:
7815 case vect_nested_cycle
:
7816 gcc_assert (!bb_vinfo
7817 && (relevance
== vect_used_in_outer
7818 || relevance
== vect_used_in_outer_by_reduction
7819 || relevance
== vect_used_by_reduction
7820 || relevance
== vect_unused_in_scope
));
7823 case vect_induction_def
:
7824 case vect_constant_def
:
7825 case vect_external_def
:
7826 case vect_unknown_def_type
:
7833 gcc_assert (PURE_SLP_STMT (stmt_info
));
7835 scalar_type
= TREE_TYPE (gimple_get_lhs (stmt
));
7836 if (dump_enabled_p ())
7838 dump_printf_loc (MSG_NOTE
, vect_location
,
7839 "get vectype for scalar type: ");
7840 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, scalar_type
);
7841 dump_printf (MSG_NOTE
, "\n");
7844 vectype
= get_vectype_for_scalar_type (scalar_type
);
7847 if (dump_enabled_p ())
7849 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7850 "not SLPed: unsupported data-type ");
7851 dump_generic_expr (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
,
7853 dump_printf (MSG_MISSED_OPTIMIZATION
, "\n");
7858 if (dump_enabled_p ())
7860 dump_printf_loc (MSG_NOTE
, vect_location
, "vectype: ");
7861 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, vectype
);
7862 dump_printf (MSG_NOTE
, "\n");
7865 STMT_VINFO_VECTYPE (stmt_info
) = vectype
;
7868 if (STMT_VINFO_RELEVANT_P (stmt_info
))
7870 gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt
))));
7871 gcc_assert (STMT_VINFO_VECTYPE (stmt_info
)
7872 || (is_gimple_call (stmt
)
7873 && gimple_call_lhs (stmt
) == NULL_TREE
));
7874 *need_to_vectorize
= true;
7877 if (PURE_SLP_STMT (stmt_info
) && !node
)
7879 dump_printf_loc (MSG_NOTE
, vect_location
,
7880 "handled only by SLP analysis\n");
7886 && (STMT_VINFO_RELEVANT_P (stmt_info
)
7887 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
))
7888 ok
= (vectorizable_simd_clone_call (stmt
, NULL
, NULL
, node
)
7889 || vectorizable_conversion (stmt
, NULL
, NULL
, node
)
7890 || vectorizable_shift (stmt
, NULL
, NULL
, node
)
7891 || vectorizable_operation (stmt
, NULL
, NULL
, node
)
7892 || vectorizable_assignment (stmt
, NULL
, NULL
, node
)
7893 || vectorizable_load (stmt
, NULL
, NULL
, node
, NULL
)
7894 || vectorizable_call (stmt
, NULL
, NULL
, node
)
7895 || vectorizable_store (stmt
, NULL
, NULL
, node
)
7896 || vectorizable_reduction (stmt
, NULL
, NULL
, node
)
7897 || vectorizable_condition (stmt
, NULL
, NULL
, NULL
, 0, node
)
7898 || vectorizable_comparison (stmt
, NULL
, NULL
, NULL
, node
));
7902 ok
= (vectorizable_simd_clone_call (stmt
, NULL
, NULL
, node
)
7903 || vectorizable_conversion (stmt
, NULL
, NULL
, node
)
7904 || vectorizable_shift (stmt
, NULL
, NULL
, node
)
7905 || vectorizable_operation (stmt
, NULL
, NULL
, node
)
7906 || vectorizable_assignment (stmt
, NULL
, NULL
, node
)
7907 || vectorizable_load (stmt
, NULL
, NULL
, node
, NULL
)
7908 || vectorizable_call (stmt
, NULL
, NULL
, node
)
7909 || vectorizable_store (stmt
, NULL
, NULL
, node
)
7910 || vectorizable_condition (stmt
, NULL
, NULL
, NULL
, 0, node
)
7911 || vectorizable_comparison (stmt
, NULL
, NULL
, NULL
, node
));
7916 if (dump_enabled_p ())
7918 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7919 "not vectorized: relevant stmt not ");
7920 dump_printf (MSG_MISSED_OPTIMIZATION
, "supported: ");
7921 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
7930 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
7931 need extra handling, except for vectorizable reductions. */
7932 if (STMT_VINFO_LIVE_P (stmt_info
)
7933 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
)
7934 ok
= vectorizable_live_operation (stmt
, NULL
, NULL
);
7938 if (dump_enabled_p ())
7940 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
7941 "not vectorized: live stmt not ");
7942 dump_printf (MSG_MISSED_OPTIMIZATION
, "supported: ");
7943 dump_gimple_stmt (MSG_MISSED_OPTIMIZATION
, TDF_SLIM
, stmt
, 0);
7953 /* Function vect_transform_stmt.
7955 Create a vectorized stmt to replace STMT, and insert it at BSI. */
7958 vect_transform_stmt (gimple
*stmt
, gimple_stmt_iterator
*gsi
,
7959 bool *grouped_store
, slp_tree slp_node
,
7960 slp_instance slp_node_instance
)
7962 bool is_store
= false;
7963 gimple
*vec_stmt
= NULL
;
7964 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
7967 gimple
*old_vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
7969 switch (STMT_VINFO_TYPE (stmt_info
))
7971 case type_demotion_vec_info_type
:
7972 case type_promotion_vec_info_type
:
7973 case type_conversion_vec_info_type
:
7974 done
= vectorizable_conversion (stmt
, gsi
, &vec_stmt
, slp_node
);
7978 case induc_vec_info_type
:
7979 gcc_assert (!slp_node
);
7980 done
= vectorizable_induction (stmt
, gsi
, &vec_stmt
);
7984 case shift_vec_info_type
:
7985 done
= vectorizable_shift (stmt
, gsi
, &vec_stmt
, slp_node
);
7989 case op_vec_info_type
:
7990 done
= vectorizable_operation (stmt
, gsi
, &vec_stmt
, slp_node
);
7994 case assignment_vec_info_type
:
7995 done
= vectorizable_assignment (stmt
, gsi
, &vec_stmt
, slp_node
);
7999 case load_vec_info_type
:
8000 done
= vectorizable_load (stmt
, gsi
, &vec_stmt
, slp_node
,
8005 case store_vec_info_type
:
8006 done
= vectorizable_store (stmt
, gsi
, &vec_stmt
, slp_node
);
8008 if (STMT_VINFO_GROUPED_ACCESS (stmt_info
) && !slp_node
)
8010 /* In case of interleaving, the whole chain is vectorized when the
8011 last store in the chain is reached. Store stmts before the last
8012 one are skipped, and there vec_stmt_info shouldn't be freed
8014 *grouped_store
= true;
8015 if (STMT_VINFO_VEC_STMT (stmt_info
))
8022 case condition_vec_info_type
:
8023 done
= vectorizable_condition (stmt
, gsi
, &vec_stmt
, NULL
, 0, slp_node
);
8027 case comparison_vec_info_type
:
8028 done
= vectorizable_comparison (stmt
, gsi
, &vec_stmt
, NULL
, slp_node
);
8032 case call_vec_info_type
:
8033 done
= vectorizable_call (stmt
, gsi
, &vec_stmt
, slp_node
);
8034 stmt
= gsi_stmt (*gsi
);
8035 if (is_gimple_call (stmt
)
8036 && gimple_call_internal_p (stmt
)
8037 && gimple_call_internal_fn (stmt
) == IFN_MASK_STORE
)
8041 case call_simd_clone_vec_info_type
:
8042 done
= vectorizable_simd_clone_call (stmt
, gsi
, &vec_stmt
, slp_node
);
8043 stmt
= gsi_stmt (*gsi
);
8046 case reduc_vec_info_type
:
8047 done
= vectorizable_reduction (stmt
, gsi
, &vec_stmt
, slp_node
);
8052 if (!STMT_VINFO_LIVE_P (stmt_info
))
8054 if (dump_enabled_p ())
8055 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8056 "stmt not supported.\n");
8061 /* Verify SLP vectorization doesn't mess with STMT_VINFO_VEC_STMT.
8062 This would break hybrid SLP vectorization. */
8064 gcc_assert (!vec_stmt
8065 && STMT_VINFO_VEC_STMT (stmt_info
) == old_vec_stmt
);
8067 /* Handle inner-loop stmts whose DEF is used in the loop-nest that
8068 is being vectorized, but outside the immediately enclosing loop. */
8070 && STMT_VINFO_LOOP_VINFO (stmt_info
)
8071 && nested_in_vect_loop_p (LOOP_VINFO_LOOP (
8072 STMT_VINFO_LOOP_VINFO (stmt_info
)), stmt
)
8073 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
8074 && (STMT_VINFO_RELEVANT (stmt_info
) == vect_used_in_outer
8075 || STMT_VINFO_RELEVANT (stmt_info
) ==
8076 vect_used_in_outer_by_reduction
))
8078 struct loop
*innerloop
= LOOP_VINFO_LOOP (
8079 STMT_VINFO_LOOP_VINFO (stmt_info
))->inner
;
8080 imm_use_iterator imm_iter
;
8081 use_operand_p use_p
;
8085 if (dump_enabled_p ())
8086 dump_printf_loc (MSG_NOTE
, vect_location
,
8087 "Record the vdef for outer-loop vectorization.\n");
8089 /* Find the relevant loop-exit phi-node, and reord the vec_stmt there
8090 (to be used when vectorizing outer-loop stmts that use the DEF of
8092 if (gimple_code (stmt
) == GIMPLE_PHI
)
8093 scalar_dest
= PHI_RESULT (stmt
);
8095 scalar_dest
= gimple_assign_lhs (stmt
);
8097 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, scalar_dest
)
8099 if (!flow_bb_inside_loop_p (innerloop
, gimple_bb (USE_STMT (use_p
))))
8101 exit_phi
= USE_STMT (use_p
);
8102 STMT_VINFO_VEC_STMT (vinfo_for_stmt (exit_phi
)) = vec_stmt
;
8107 /* Handle stmts whose DEF is used outside the loop-nest that is
8108 being vectorized. */
8109 if (STMT_VINFO_LIVE_P (stmt_info
)
8110 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
)
8112 done
= vectorizable_live_operation (stmt
, gsi
, &vec_stmt
);
8117 STMT_VINFO_VEC_STMT (stmt_info
) = vec_stmt
;
8123 /* Remove a group of stores (for SLP or interleaving), free their
8127 vect_remove_stores (gimple
*first_stmt
)
8129 gimple
*next
= first_stmt
;
8131 gimple_stmt_iterator next_si
;
8135 stmt_vec_info stmt_info
= vinfo_for_stmt (next
);
8137 tmp
= GROUP_NEXT_ELEMENT (stmt_info
);
8138 if (is_pattern_stmt_p (stmt_info
))
8139 next
= STMT_VINFO_RELATED_STMT (stmt_info
);
8140 /* Free the attached stmt_vec_info and remove the stmt. */
8141 next_si
= gsi_for_stmt (next
);
8142 unlink_stmt_vdef (next
);
8143 gsi_remove (&next_si
, true);
8144 release_defs (next
);
8145 free_stmt_vec_info (next
);
8151 /* Function new_stmt_vec_info.
8153 Create and initialize a new stmt_vec_info struct for STMT. */
8156 new_stmt_vec_info (gimple
*stmt
, vec_info
*vinfo
)
8159 res
= (stmt_vec_info
) xcalloc (1, sizeof (struct _stmt_vec_info
));
8161 STMT_VINFO_TYPE (res
) = undef_vec_info_type
;
8162 STMT_VINFO_STMT (res
) = stmt
;
8164 STMT_VINFO_RELEVANT (res
) = vect_unused_in_scope
;
8165 STMT_VINFO_LIVE_P (res
) = false;
8166 STMT_VINFO_VECTYPE (res
) = NULL
;
8167 STMT_VINFO_VEC_STMT (res
) = NULL
;
8168 STMT_VINFO_VECTORIZABLE (res
) = true;
8169 STMT_VINFO_IN_PATTERN_P (res
) = false;
8170 STMT_VINFO_RELATED_STMT (res
) = NULL
;
8171 STMT_VINFO_PATTERN_DEF_SEQ (res
) = NULL
;
8172 STMT_VINFO_DATA_REF (res
) = NULL
;
8173 STMT_VINFO_VEC_REDUCTION_TYPE (res
) = TREE_CODE_REDUCTION
;
8175 STMT_VINFO_DR_BASE_ADDRESS (res
) = NULL
;
8176 STMT_VINFO_DR_OFFSET (res
) = NULL
;
8177 STMT_VINFO_DR_INIT (res
) = NULL
;
8178 STMT_VINFO_DR_STEP (res
) = NULL
;
8179 STMT_VINFO_DR_ALIGNED_TO (res
) = NULL
;
8181 if (gimple_code (stmt
) == GIMPLE_PHI
8182 && is_loop_header_bb_p (gimple_bb (stmt
)))
8183 STMT_VINFO_DEF_TYPE (res
) = vect_unknown_def_type
;
8185 STMT_VINFO_DEF_TYPE (res
) = vect_internal_def
;
8187 STMT_VINFO_SAME_ALIGN_REFS (res
).create (0);
8188 STMT_SLP_TYPE (res
) = loop_vect
;
8189 GROUP_FIRST_ELEMENT (res
) = NULL
;
8190 GROUP_NEXT_ELEMENT (res
) = NULL
;
8191 GROUP_SIZE (res
) = 0;
8192 GROUP_STORE_COUNT (res
) = 0;
8193 GROUP_GAP (res
) = 0;
8194 GROUP_SAME_DR_STMT (res
) = NULL
;
8200 /* Create a hash table for stmt_vec_info. */
8203 init_stmt_vec_info_vec (void)
8205 gcc_assert (!stmt_vec_info_vec
.exists ());
8206 stmt_vec_info_vec
.create (50);
8210 /* Free hash table for stmt_vec_info. */
8213 free_stmt_vec_info_vec (void)
8217 FOR_EACH_VEC_ELT (stmt_vec_info_vec
, i
, info
)
8219 free_stmt_vec_info (STMT_VINFO_STMT (info
));
8220 gcc_assert (stmt_vec_info_vec
.exists ());
8221 stmt_vec_info_vec
.release ();
8225 /* Free stmt vectorization related info. */
8228 free_stmt_vec_info (gimple
*stmt
)
8230 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
8235 /* Check if this statement has a related "pattern stmt"
8236 (introduced by the vectorizer during the pattern recognition
8237 pass). Free pattern's stmt_vec_info and def stmt's stmt_vec_info
8239 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
8241 stmt_vec_info patt_info
8242 = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info
));
8245 gimple_seq seq
= STMT_VINFO_PATTERN_DEF_SEQ (patt_info
);
8246 gimple
*patt_stmt
= STMT_VINFO_STMT (patt_info
);
8247 gimple_set_bb (patt_stmt
, NULL
);
8248 tree lhs
= gimple_get_lhs (patt_stmt
);
8249 if (lhs
&& TREE_CODE (lhs
) == SSA_NAME
)
8250 release_ssa_name (lhs
);
8253 gimple_stmt_iterator si
;
8254 for (si
= gsi_start (seq
); !gsi_end_p (si
); gsi_next (&si
))
8256 gimple
*seq_stmt
= gsi_stmt (si
);
8257 gimple_set_bb (seq_stmt
, NULL
);
8258 lhs
= gimple_get_lhs (seq_stmt
);
8259 if (lhs
&& TREE_CODE (lhs
) == SSA_NAME
)
8260 release_ssa_name (lhs
);
8261 free_stmt_vec_info (seq_stmt
);
8264 free_stmt_vec_info (patt_stmt
);
8268 STMT_VINFO_SAME_ALIGN_REFS (stmt_info
).release ();
8269 STMT_VINFO_SIMD_CLONE_INFO (stmt_info
).release ();
8270 set_vinfo_for_stmt (stmt
, NULL
);
8275 /* Function get_vectype_for_scalar_type_and_size.
8277 Returns the vector type corresponding to SCALAR_TYPE and SIZE as supported
8281 get_vectype_for_scalar_type_and_size (tree scalar_type
, unsigned size
)
8283 machine_mode inner_mode
= TYPE_MODE (scalar_type
);
8284 machine_mode simd_mode
;
8285 unsigned int nbytes
= GET_MODE_SIZE (inner_mode
);
8292 if (GET_MODE_CLASS (inner_mode
) != MODE_INT
8293 && GET_MODE_CLASS (inner_mode
) != MODE_FLOAT
)
8296 /* For vector types of elements whose mode precision doesn't
8297 match their types precision we use a element type of mode
8298 precision. The vectorization routines will have to make sure
8299 they support the proper result truncation/extension.
8300 We also make sure to build vector types with INTEGER_TYPE
8301 component type only. */
8302 if (INTEGRAL_TYPE_P (scalar_type
)
8303 && (GET_MODE_BITSIZE (inner_mode
) != TYPE_PRECISION (scalar_type
)
8304 || TREE_CODE (scalar_type
) != INTEGER_TYPE
))
8305 scalar_type
= build_nonstandard_integer_type (GET_MODE_BITSIZE (inner_mode
),
8306 TYPE_UNSIGNED (scalar_type
));
8308 /* We shouldn't end up building VECTOR_TYPEs of non-scalar components.
8309 When the component mode passes the above test simply use a type
8310 corresponding to that mode. The theory is that any use that
8311 would cause problems with this will disable vectorization anyway. */
8312 else if (!SCALAR_FLOAT_TYPE_P (scalar_type
)
8313 && !INTEGRAL_TYPE_P (scalar_type
))
8314 scalar_type
= lang_hooks
.types
.type_for_mode (inner_mode
, 1);
8316 /* We can't build a vector type of elements with alignment bigger than
8318 else if (nbytes
< TYPE_ALIGN_UNIT (scalar_type
))
8319 scalar_type
= lang_hooks
.types
.type_for_mode (inner_mode
,
8320 TYPE_UNSIGNED (scalar_type
));
8322 /* If we felt back to using the mode fail if there was
8323 no scalar type for it. */
8324 if (scalar_type
== NULL_TREE
)
8327 /* If no size was supplied use the mode the target prefers. Otherwise
8328 lookup a vector mode of the specified size. */
8330 simd_mode
= targetm
.vectorize
.preferred_simd_mode (inner_mode
);
8332 simd_mode
= mode_for_vector (inner_mode
, size
/ nbytes
);
8333 nunits
= GET_MODE_SIZE (simd_mode
) / nbytes
;
8337 vectype
= build_vector_type (scalar_type
, nunits
);
8339 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
8340 && !INTEGRAL_MODE_P (TYPE_MODE (vectype
)))
8346 unsigned int current_vector_size
;
8348 /* Function get_vectype_for_scalar_type.
8350 Returns the vector type corresponding to SCALAR_TYPE as supported
8354 get_vectype_for_scalar_type (tree scalar_type
)
8357 vectype
= get_vectype_for_scalar_type_and_size (scalar_type
,
8358 current_vector_size
);
8360 && current_vector_size
== 0)
8361 current_vector_size
= GET_MODE_SIZE (TYPE_MODE (vectype
));
8365 /* Function get_mask_type_for_scalar_type.
8367 Returns the mask type corresponding to a result of comparison
8368 of vectors of specified SCALAR_TYPE as supported by target. */
8371 get_mask_type_for_scalar_type (tree scalar_type
)
8373 tree vectype
= get_vectype_for_scalar_type (scalar_type
);
8378 return build_truth_vector_type (TYPE_VECTOR_SUBPARTS (vectype
),
8379 current_vector_size
);
8382 /* Function get_same_sized_vectype
8384 Returns a vector type corresponding to SCALAR_TYPE of size
8385 VECTOR_TYPE if supported by the target. */
8388 get_same_sized_vectype (tree scalar_type
, tree vector_type
)
8390 if (TREE_CODE (scalar_type
) == BOOLEAN_TYPE
)
8391 return build_same_sized_truth_vector_type (vector_type
);
8393 return get_vectype_for_scalar_type_and_size
8394 (scalar_type
, GET_MODE_SIZE (TYPE_MODE (vector_type
)));
8397 /* Function vect_is_simple_use.
8400 VINFO - the vect info of the loop or basic block that is being vectorized.
8401 OPERAND - operand in the loop or bb.
8403 DEF_STMT - the defining stmt in case OPERAND is an SSA_NAME.
8404 DT - the type of definition
8406 Returns whether a stmt with OPERAND can be vectorized.
8407 For loops, supportable operands are constants, loop invariants, and operands
8408 that are defined by the current iteration of the loop. Unsupportable
8409 operands are those that are defined by a previous iteration of the loop (as
8410 is the case in reduction/induction computations).
8411 For basic blocks, supportable operands are constants and bb invariants.
8412 For now, operands defined outside the basic block are not supported. */
8415 vect_is_simple_use (tree operand
, vec_info
*vinfo
,
8416 gimple
**def_stmt
, enum vect_def_type
*dt
)
8419 *dt
= vect_unknown_def_type
;
8421 if (dump_enabled_p ())
8423 dump_printf_loc (MSG_NOTE
, vect_location
,
8424 "vect_is_simple_use: operand ");
8425 dump_generic_expr (MSG_NOTE
, TDF_SLIM
, operand
);
8426 dump_printf (MSG_NOTE
, "\n");
8429 if (CONSTANT_CLASS_P (operand
))
8431 *dt
= vect_constant_def
;
8435 if (is_gimple_min_invariant (operand
))
8437 *dt
= vect_external_def
;
8441 if (TREE_CODE (operand
) != SSA_NAME
)
8443 if (dump_enabled_p ())
8444 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8449 if (SSA_NAME_IS_DEFAULT_DEF (operand
))
8451 *dt
= vect_external_def
;
8455 *def_stmt
= SSA_NAME_DEF_STMT (operand
);
8456 if (dump_enabled_p ())
8458 dump_printf_loc (MSG_NOTE
, vect_location
, "def_stmt: ");
8459 dump_gimple_stmt (MSG_NOTE
, TDF_SLIM
, *def_stmt
, 0);
8462 if (! vect_stmt_in_region_p (vinfo
, *def_stmt
))
8463 *dt
= vect_external_def
;
8466 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (*def_stmt
);
8467 if (is_a
<bb_vec_info
> (vinfo
) && !STMT_VINFO_VECTORIZABLE (stmt_vinfo
))
8468 *dt
= vect_external_def
;
8470 *dt
= STMT_VINFO_DEF_TYPE (stmt_vinfo
);
8473 if (dump_enabled_p ())
8475 dump_printf_loc (MSG_NOTE
, vect_location
, "type of def: ");
8478 case vect_uninitialized_def
:
8479 dump_printf (MSG_NOTE
, "uninitialized\n");
8481 case vect_constant_def
:
8482 dump_printf (MSG_NOTE
, "constant\n");
8484 case vect_external_def
:
8485 dump_printf (MSG_NOTE
, "external\n");
8487 case vect_internal_def
:
8488 dump_printf (MSG_NOTE
, "internal\n");
8490 case vect_induction_def
:
8491 dump_printf (MSG_NOTE
, "induction\n");
8493 case vect_reduction_def
:
8494 dump_printf (MSG_NOTE
, "reduction\n");
8496 case vect_double_reduction_def
:
8497 dump_printf (MSG_NOTE
, "double reduction\n");
8499 case vect_nested_cycle
:
8500 dump_printf (MSG_NOTE
, "nested cycle\n");
8502 case vect_unknown_def_type
:
8503 dump_printf (MSG_NOTE
, "unknown\n");
8508 if (*dt
== vect_unknown_def_type
)
8510 if (dump_enabled_p ())
8511 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8512 "Unsupported pattern.\n");
8516 switch (gimple_code (*def_stmt
))
8523 if (dump_enabled_p ())
8524 dump_printf_loc (MSG_MISSED_OPTIMIZATION
, vect_location
,
8525 "unsupported defining stmt:\n");
8532 /* Function vect_is_simple_use.
8534 Same as vect_is_simple_use but also determines the vector operand
8535 type of OPERAND and stores it to *VECTYPE. If the definition of
8536 OPERAND is vect_uninitialized_def, vect_constant_def or
8537 vect_external_def *VECTYPE will be set to NULL_TREE and the caller
8538 is responsible to compute the best suited vector type for the
8542 vect_is_simple_use (tree operand
, vec_info
*vinfo
,
8543 gimple
**def_stmt
, enum vect_def_type
*dt
, tree
*vectype
)
8545 if (!vect_is_simple_use (operand
, vinfo
, def_stmt
, dt
))
8548 /* Now get a vector type if the def is internal, otherwise supply
8549 NULL_TREE and leave it up to the caller to figure out a proper
8550 type for the use stmt. */
8551 if (*dt
== vect_internal_def
8552 || *dt
== vect_induction_def
8553 || *dt
== vect_reduction_def
8554 || *dt
== vect_double_reduction_def
8555 || *dt
== vect_nested_cycle
)
8557 stmt_vec_info stmt_info
= vinfo_for_stmt (*def_stmt
);
8559 if (STMT_VINFO_IN_PATTERN_P (stmt_info
)
8560 && !STMT_VINFO_RELEVANT (stmt_info
)
8561 && !STMT_VINFO_LIVE_P (stmt_info
))
8562 stmt_info
= vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info
));
8564 *vectype
= STMT_VINFO_VECTYPE (stmt_info
);
8565 gcc_assert (*vectype
!= NULL_TREE
);
8567 else if (*dt
== vect_uninitialized_def
8568 || *dt
== vect_constant_def
8569 || *dt
== vect_external_def
)
8570 *vectype
= NULL_TREE
;
8578 /* Function supportable_widening_operation
8580 Check whether an operation represented by the code CODE is a
8581 widening operation that is supported by the target platform in
8582 vector form (i.e., when operating on arguments of type VECTYPE_IN
8583 producing a result of type VECTYPE_OUT).
8585 Widening operations we currently support are NOP (CONVERT), FLOAT
8586 and WIDEN_MULT. This function checks if these operations are supported
8587 by the target platform either directly (via vector tree-codes), or via
8591 - CODE1 and CODE2 are codes of vector operations to be used when
8592 vectorizing the operation, if available.
8593 - MULTI_STEP_CVT determines the number of required intermediate steps in
8594 case of multi-step conversion (like char->short->int - in that case
8595 MULTI_STEP_CVT will be 1).
8596 - INTERM_TYPES contains the intermediate type required to perform the
8597 widening operation (short in the above example). */
8600 supportable_widening_operation (enum tree_code code
, gimple
*stmt
,
8601 tree vectype_out
, tree vectype_in
,
8602 enum tree_code
*code1
, enum tree_code
*code2
,
8603 int *multi_step_cvt
,
8604 vec
<tree
> *interm_types
)
8606 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
8607 loop_vec_info loop_info
= STMT_VINFO_LOOP_VINFO (stmt_info
);
8608 struct loop
*vect_loop
= NULL
;
8609 machine_mode vec_mode
;
8610 enum insn_code icode1
, icode2
;
8611 optab optab1
, optab2
;
8612 tree vectype
= vectype_in
;
8613 tree wide_vectype
= vectype_out
;
8614 enum tree_code c1
, c2
;
8616 tree prev_type
, intermediate_type
;
8617 machine_mode intermediate_mode
, prev_mode
;
8618 optab optab3
, optab4
;
8620 *multi_step_cvt
= 0;
8622 vect_loop
= LOOP_VINFO_LOOP (loop_info
);
8626 case WIDEN_MULT_EXPR
:
8627 /* The result of a vectorized widening operation usually requires
8628 two vectors (because the widened results do not fit into one vector).
8629 The generated vector results would normally be expected to be
8630 generated in the same order as in the original scalar computation,
8631 i.e. if 8 results are generated in each vector iteration, they are
8632 to be organized as follows:
8633 vect1: [res1,res2,res3,res4],
8634 vect2: [res5,res6,res7,res8].
8636 However, in the special case that the result of the widening
8637 operation is used in a reduction computation only, the order doesn't
8638 matter (because when vectorizing a reduction we change the order of
8639 the computation). Some targets can take advantage of this and
8640 generate more efficient code. For example, targets like Altivec,
8641 that support widen_mult using a sequence of {mult_even,mult_odd}
8642 generate the following vectors:
8643 vect1: [res1,res3,res5,res7],
8644 vect2: [res2,res4,res6,res8].
8646 When vectorizing outer-loops, we execute the inner-loop sequentially
8647 (each vectorized inner-loop iteration contributes to VF outer-loop
8648 iterations in parallel). We therefore don't allow to change the
8649 order of the computation in the inner-loop during outer-loop
8651 /* TODO: Another case in which order doesn't *really* matter is when we
8652 widen and then contract again, e.g. (short)((int)x * y >> 8).
8653 Normally, pack_trunc performs an even/odd permute, whereas the
8654 repack from an even/odd expansion would be an interleave, which
8655 would be significantly simpler for e.g. AVX2. */
8656 /* In any case, in order to avoid duplicating the code below, recurse
8657 on VEC_WIDEN_MULT_EVEN_EXPR. If it succeeds, all the return values
8658 are properly set up for the caller. If we fail, we'll continue with
8659 a VEC_WIDEN_MULT_LO/HI_EXPR check. */
8661 && STMT_VINFO_RELEVANT (stmt_info
) == vect_used_by_reduction
8662 && !nested_in_vect_loop_p (vect_loop
, stmt
)
8663 && supportable_widening_operation (VEC_WIDEN_MULT_EVEN_EXPR
,
8664 stmt
, vectype_out
, vectype_in
,
8665 code1
, code2
, multi_step_cvt
,
8668 /* Elements in a vector with vect_used_by_reduction property cannot
8669 be reordered if the use chain with this property does not have the
8670 same operation. One such an example is s += a * b, where elements
8671 in a and b cannot be reordered. Here we check if the vector defined
8672 by STMT is only directly used in the reduction statement. */
8673 tree lhs
= gimple_assign_lhs (stmt
);
8674 use_operand_p dummy
;
8676 stmt_vec_info use_stmt_info
= NULL
;
8677 if (single_imm_use (lhs
, &dummy
, &use_stmt
)
8678 && (use_stmt_info
= vinfo_for_stmt (use_stmt
))
8679 && STMT_VINFO_DEF_TYPE (use_stmt_info
) == vect_reduction_def
)
8682 c1
= VEC_WIDEN_MULT_LO_EXPR
;
8683 c2
= VEC_WIDEN_MULT_HI_EXPR
;
8696 case VEC_WIDEN_MULT_EVEN_EXPR
:
8697 /* Support the recursion induced just above. */
8698 c1
= VEC_WIDEN_MULT_EVEN_EXPR
;
8699 c2
= VEC_WIDEN_MULT_ODD_EXPR
;
8702 case WIDEN_LSHIFT_EXPR
:
8703 c1
= VEC_WIDEN_LSHIFT_LO_EXPR
;
8704 c2
= VEC_WIDEN_LSHIFT_HI_EXPR
;
8708 c1
= VEC_UNPACK_LO_EXPR
;
8709 c2
= VEC_UNPACK_HI_EXPR
;
8713 c1
= VEC_UNPACK_FLOAT_LO_EXPR
;
8714 c2
= VEC_UNPACK_FLOAT_HI_EXPR
;
8717 case FIX_TRUNC_EXPR
:
8718 /* ??? Not yet implemented due to missing VEC_UNPACK_FIX_TRUNC_HI_EXPR/
8719 VEC_UNPACK_FIX_TRUNC_LO_EXPR tree codes and optabs used for
8720 computing the operation. */
8727 if (BYTES_BIG_ENDIAN
&& c1
!= VEC_WIDEN_MULT_EVEN_EXPR
)
8730 if (code
== FIX_TRUNC_EXPR
)
8732 /* The signedness is determined from output operand. */
8733 optab1
= optab_for_tree_code (c1
, vectype_out
, optab_default
);
8734 optab2
= optab_for_tree_code (c2
, vectype_out
, optab_default
);
8738 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
8739 optab2
= optab_for_tree_code (c2
, vectype
, optab_default
);
8742 if (!optab1
|| !optab2
)
8745 vec_mode
= TYPE_MODE (vectype
);
8746 if ((icode1
= optab_handler (optab1
, vec_mode
)) == CODE_FOR_nothing
8747 || (icode2
= optab_handler (optab2
, vec_mode
)) == CODE_FOR_nothing
)
8753 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
8754 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
8757 /* Check if it's a multi-step conversion that can be done using intermediate
8760 prev_type
= vectype
;
8761 prev_mode
= vec_mode
;
8763 if (!CONVERT_EXPR_CODE_P (code
))
8766 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
8767 intermediate steps in promotion sequence. We try
8768 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do
8770 interm_types
->create (MAX_INTERM_CVT_STEPS
);
8771 for (i
= 0; i
< MAX_INTERM_CVT_STEPS
; i
++)
8773 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
8775 = lang_hooks
.types
.type_for_mode (intermediate_mode
,
8776 TYPE_UNSIGNED (prev_type
));
8777 optab3
= optab_for_tree_code (c1
, intermediate_type
, optab_default
);
8778 optab4
= optab_for_tree_code (c2
, intermediate_type
, optab_default
);
8780 if (!optab3
|| !optab4
8781 || (icode1
= optab_handler (optab1
, prev_mode
)) == CODE_FOR_nothing
8782 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
8783 || (icode2
= optab_handler (optab2
, prev_mode
)) == CODE_FOR_nothing
8784 || insn_data
[icode2
].operand
[0].mode
!= intermediate_mode
8785 || ((icode1
= optab_handler (optab3
, intermediate_mode
))
8786 == CODE_FOR_nothing
)
8787 || ((icode2
= optab_handler (optab4
, intermediate_mode
))
8788 == CODE_FOR_nothing
))
8791 interm_types
->quick_push (intermediate_type
);
8792 (*multi_step_cvt
)++;
8794 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
8795 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
8798 prev_type
= intermediate_type
;
8799 prev_mode
= intermediate_mode
;
8802 interm_types
->release ();
8807 /* Function supportable_narrowing_operation
8809 Check whether an operation represented by the code CODE is a
8810 narrowing operation that is supported by the target platform in
8811 vector form (i.e., when operating on arguments of type VECTYPE_IN
8812 and producing a result of type VECTYPE_OUT).
8814 Narrowing operations we currently support are NOP (CONVERT) and
8815 FIX_TRUNC. This function checks if these operations are supported by
8816 the target platform directly via vector tree-codes.
8819 - CODE1 is the code of a vector operation to be used when
8820 vectorizing the operation, if available.
8821 - MULTI_STEP_CVT determines the number of required intermediate steps in
8822 case of multi-step conversion (like int->short->char - in that case
8823 MULTI_STEP_CVT will be 1).
8824 - INTERM_TYPES contains the intermediate type required to perform the
8825 narrowing operation (short in the above example). */
8828 supportable_narrowing_operation (enum tree_code code
,
8829 tree vectype_out
, tree vectype_in
,
8830 enum tree_code
*code1
, int *multi_step_cvt
,
8831 vec
<tree
> *interm_types
)
8833 machine_mode vec_mode
;
8834 enum insn_code icode1
;
8835 optab optab1
, interm_optab
;
8836 tree vectype
= vectype_in
;
8837 tree narrow_vectype
= vectype_out
;
8839 tree intermediate_type
;
8840 machine_mode intermediate_mode
, prev_mode
;
8844 *multi_step_cvt
= 0;
8848 c1
= VEC_PACK_TRUNC_EXPR
;
8851 case FIX_TRUNC_EXPR
:
8852 c1
= VEC_PACK_FIX_TRUNC_EXPR
;
8856 /* ??? Not yet implemented due to missing VEC_PACK_FLOAT_EXPR
8857 tree code and optabs used for computing the operation. */
8864 if (code
== FIX_TRUNC_EXPR
)
8865 /* The signedness is determined from output operand. */
8866 optab1
= optab_for_tree_code (c1
, vectype_out
, optab_default
);
8868 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
8873 vec_mode
= TYPE_MODE (vectype
);
8874 if ((icode1
= optab_handler (optab1
, vec_mode
)) == CODE_FOR_nothing
)
8879 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
8882 /* Check if it's a multi-step conversion that can be done using intermediate
8884 prev_mode
= vec_mode
;
8885 if (code
== FIX_TRUNC_EXPR
)
8886 uns
= TYPE_UNSIGNED (vectype_out
);
8888 uns
= TYPE_UNSIGNED (vectype
);
8890 /* For multi-step FIX_TRUNC_EXPR prefer signed floating to integer
8891 conversion over unsigned, as unsigned FIX_TRUNC_EXPR is often more
8892 costly than signed. */
8893 if (code
== FIX_TRUNC_EXPR
&& uns
)
8895 enum insn_code icode2
;
8898 = lang_hooks
.types
.type_for_mode (TYPE_MODE (vectype_out
), 0);
8900 = optab_for_tree_code (c1
, intermediate_type
, optab_default
);
8901 if (interm_optab
!= unknown_optab
8902 && (icode2
= optab_handler (optab1
, vec_mode
)) != CODE_FOR_nothing
8903 && insn_data
[icode1
].operand
[0].mode
8904 == insn_data
[icode2
].operand
[0].mode
)
8907 optab1
= interm_optab
;
8912 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
8913 intermediate steps in promotion sequence. We try
8914 MAX_INTERM_CVT_STEPS to get to NARROW_VECTYPE, and fail if we do not. */
8915 interm_types
->create (MAX_INTERM_CVT_STEPS
);
8916 for (i
= 0; i
< MAX_INTERM_CVT_STEPS
; i
++)
8918 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
8920 = lang_hooks
.types
.type_for_mode (intermediate_mode
, uns
);
8922 = optab_for_tree_code (VEC_PACK_TRUNC_EXPR
, intermediate_type
,
8925 || ((icode1
= optab_handler (optab1
, prev_mode
)) == CODE_FOR_nothing
)
8926 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
8927 || ((icode1
= optab_handler (interm_optab
, intermediate_mode
))
8928 == CODE_FOR_nothing
))
8931 interm_types
->quick_push (intermediate_type
);
8932 (*multi_step_cvt
)++;
8934 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
8937 prev_mode
= intermediate_mode
;
8938 optab1
= interm_optab
;
8941 interm_types
->release ();