1 /* Statement Analysis and Transformation for Vectorization
2 Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software
4 Contributed by Dorit Naishlos <dorit@il.ibm.com>
5 and Ira Rosen <irar@il.ibm.com>
7 This file is part of GCC.
9 GCC is free software; you can redistribute it and/or modify it under
10 the terms of the GNU General Public License as published by the Free
11 Software Foundation; either version 3, or (at your option) any later
14 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
15 WARRANTY; without even the implied warranty of MERCHANTABILITY or
16 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19 You should have received a copy of the GNU General Public License
20 along with GCC; see the file COPYING3. If not see
21 <http://www.gnu.org/licenses/>. */
25 #include "coretypes.h"
30 #include "basic-block.h"
31 #include "diagnostic.h"
32 #include "tree-flow.h"
33 #include "tree-dump.h"
35 #include "cfglayout.h"
40 #include "tree-vectorizer.h"
41 #include "langhooks.h"
44 /* Utility functions used by vect_mark_stmts_to_be_vectorized. */
46 /* Function vect_mark_relevant.
48 Mark STMT as "relevant for vectorization" and add it to WORKLIST. */
51 vect_mark_relevant (VEC(gimple
,heap
) **worklist
, gimple stmt
,
52 enum vect_relevant relevant
, bool live_p
)
54 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
55 enum vect_relevant save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
56 bool save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
58 if (vect_print_dump_info (REPORT_DETAILS
))
59 fprintf (vect_dump
, "mark relevant %d, live %d.", relevant
, live_p
);
61 if (STMT_VINFO_IN_PATTERN_P (stmt_info
))
65 /* This is the last stmt in a sequence that was detected as a
66 pattern that can potentially be vectorized. Don't mark the stmt
67 as relevant/live because it's not going to be vectorized.
68 Instead mark the pattern-stmt that replaces it. */
70 pattern_stmt
= STMT_VINFO_RELATED_STMT (stmt_info
);
72 if (vect_print_dump_info (REPORT_DETAILS
))
73 fprintf (vect_dump
, "last stmt in pattern. don't mark relevant/live.");
74 stmt_info
= vinfo_for_stmt (pattern_stmt
);
75 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info
) == stmt
);
76 save_relevant
= STMT_VINFO_RELEVANT (stmt_info
);
77 save_live_p
= STMT_VINFO_LIVE_P (stmt_info
);
81 STMT_VINFO_LIVE_P (stmt_info
) |= live_p
;
82 if (relevant
> STMT_VINFO_RELEVANT (stmt_info
))
83 STMT_VINFO_RELEVANT (stmt_info
) = relevant
;
85 if (STMT_VINFO_RELEVANT (stmt_info
) == save_relevant
86 && STMT_VINFO_LIVE_P (stmt_info
) == save_live_p
)
88 if (vect_print_dump_info (REPORT_DETAILS
))
89 fprintf (vect_dump
, "already marked relevant/live.");
93 VEC_safe_push (gimple
, heap
, *worklist
, stmt
);
97 /* Function vect_stmt_relevant_p.
99 Return true if STMT in loop that is represented by LOOP_VINFO is
100 "relevant for vectorization".
102 A stmt is considered "relevant for vectorization" if:
103 - it has uses outside the loop.
104 - it has vdefs (it alters memory).
105 - control stmts in the loop (except for the exit condition).
107 CHECKME: what other side effects would the vectorizer allow? */
110 vect_stmt_relevant_p (gimple stmt
, loop_vec_info loop_vinfo
,
111 enum vect_relevant
*relevant
, bool *live_p
)
113 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
115 imm_use_iterator imm_iter
;
119 *relevant
= vect_unused_in_scope
;
122 /* cond stmt other than loop exit cond. */
123 if (is_ctrl_stmt (stmt
)
124 && STMT_VINFO_TYPE (vinfo_for_stmt (stmt
))
125 != loop_exit_ctrl_vec_info_type
)
126 *relevant
= vect_used_in_scope
;
128 /* changing memory. */
129 if (gimple_code (stmt
) != GIMPLE_PHI
)
130 if (gimple_vdef (stmt
))
132 if (vect_print_dump_info (REPORT_DETAILS
))
133 fprintf (vect_dump
, "vec_stmt_relevant_p: stmt has vdefs.");
134 *relevant
= vect_used_in_scope
;
137 /* uses outside the loop. */
138 FOR_EACH_PHI_OR_STMT_DEF (def_p
, stmt
, op_iter
, SSA_OP_DEF
)
140 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, DEF_FROM_PTR (def_p
))
142 basic_block bb
= gimple_bb (USE_STMT (use_p
));
143 if (!flow_bb_inside_loop_p (loop
, bb
))
145 if (vect_print_dump_info (REPORT_DETAILS
))
146 fprintf (vect_dump
, "vec_stmt_relevant_p: used out of loop.");
148 /* We expect all such uses to be in the loop exit phis
149 (because of loop closed form) */
150 gcc_assert (gimple_code (USE_STMT (use_p
)) == GIMPLE_PHI
);
151 gcc_assert (bb
== single_exit (loop
)->dest
);
158 return (*live_p
|| *relevant
);
162 /* Function exist_non_indexing_operands_for_use_p
164 USE is one of the uses attached to STMT. Check if USE is
165 used in STMT for anything other than indexing an array. */
168 exist_non_indexing_operands_for_use_p (tree use
, gimple stmt
)
171 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
173 /* USE corresponds to some operand in STMT. If there is no data
174 reference in STMT, then any operand that corresponds to USE
175 is not indexing an array. */
176 if (!STMT_VINFO_DATA_REF (stmt_info
))
179 /* STMT has a data_ref. FORNOW this means that its of one of
183 (This should have been verified in analyze_data_refs).
185 'var' in the second case corresponds to a def, not a use,
186 so USE cannot correspond to any operands that are not used
189 Therefore, all we need to check is if STMT falls into the
190 first case, and whether var corresponds to USE. */
192 if (TREE_CODE (gimple_assign_lhs (stmt
)) == SSA_NAME
)
195 if (!gimple_assign_copy_p (stmt
))
197 operand
= gimple_assign_rhs1 (stmt
);
199 if (TREE_CODE (operand
) != SSA_NAME
)
210 Function process_use.
213 - a USE in STMT in a loop represented by LOOP_VINFO
214 - LIVE_P, RELEVANT - enum values to be set in the STMT_VINFO of the stmt
215 that defined USE. This is done by calling mark_relevant and passing it
216 the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant).
219 Generally, LIVE_P and RELEVANT are used to define the liveness and
220 relevance info of the DEF_STMT of this USE:
221 STMT_VINFO_LIVE_P (DEF_STMT_info) <-- live_p
222 STMT_VINFO_RELEVANT (DEF_STMT_info) <-- relevant
224 - case 1: If USE is used only for address computations (e.g. array indexing),
225 which does not need to be directly vectorized, then the liveness/relevance
226 of the respective DEF_STMT is left unchanged.
227 - case 2: If STMT is a reduction phi and DEF_STMT is a reduction stmt, we
228 skip DEF_STMT cause it had already been processed.
229 - case 3: If DEF_STMT and STMT are in different nests, then "relevant" will
230 be modified accordingly.
232 Return true if everything is as expected. Return false otherwise. */
235 process_use (gimple stmt
, tree use
, loop_vec_info loop_vinfo
, bool live_p
,
236 enum vect_relevant relevant
, VEC(gimple
,heap
) **worklist
)
238 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
239 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
240 stmt_vec_info dstmt_vinfo
;
241 basic_block bb
, def_bb
;
244 enum vect_def_type dt
;
246 /* case 1: we are only interested in uses that need to be vectorized. Uses
247 that are used for address computation are not considered relevant. */
248 if (!exist_non_indexing_operands_for_use_p (use
, stmt
))
251 if (!vect_is_simple_use (use
, loop_vinfo
, NULL
, &def_stmt
, &def
, &dt
))
253 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS
))
254 fprintf (vect_dump
, "not vectorized: unsupported use in stmt.");
258 if (!def_stmt
|| gimple_nop_p (def_stmt
))
261 def_bb
= gimple_bb (def_stmt
);
262 if (!flow_bb_inside_loop_p (loop
, def_bb
))
264 if (vect_print_dump_info (REPORT_DETAILS
))
265 fprintf (vect_dump
, "def_stmt is out of loop.");
269 /* case 2: A reduction phi (STMT) defined by a reduction stmt (DEF_STMT).
270 DEF_STMT must have already been processed, because this should be the
271 only way that STMT, which is a reduction-phi, was put in the worklist,
272 as there should be no other uses for DEF_STMT in the loop. So we just
273 check that everything is as expected, and we are done. */
274 dstmt_vinfo
= vinfo_for_stmt (def_stmt
);
275 bb
= gimple_bb (stmt
);
276 if (gimple_code (stmt
) == GIMPLE_PHI
277 && STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
278 && gimple_code (def_stmt
) != GIMPLE_PHI
279 && STMT_VINFO_DEF_TYPE (dstmt_vinfo
) == vect_reduction_def
280 && bb
->loop_father
== def_bb
->loop_father
)
282 if (vect_print_dump_info (REPORT_DETAILS
))
283 fprintf (vect_dump
, "reduc-stmt defining reduc-phi in the same nest.");
284 if (STMT_VINFO_IN_PATTERN_P (dstmt_vinfo
))
285 dstmt_vinfo
= vinfo_for_stmt (STMT_VINFO_RELATED_STMT (dstmt_vinfo
));
286 gcc_assert (STMT_VINFO_RELEVANT (dstmt_vinfo
) < vect_used_by_reduction
);
287 gcc_assert (STMT_VINFO_LIVE_P (dstmt_vinfo
)
288 || STMT_VINFO_RELEVANT (dstmt_vinfo
) > vect_unused_in_scope
);
292 /* case 3a: outer-loop stmt defining an inner-loop stmt:
293 outer-loop-header-bb:
299 if (flow_loop_nested_p (def_bb
->loop_father
, bb
->loop_father
))
301 if (vect_print_dump_info (REPORT_DETAILS
))
302 fprintf (vect_dump
, "outer-loop def-stmt defining inner-loop stmt.");
306 case vect_unused_in_scope
:
307 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_nested_cycle
) ?
308 vect_used_in_scope
: vect_unused_in_scope
;
311 case vect_used_in_outer_by_reduction
:
312 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
313 relevant
= vect_used_by_reduction
;
316 case vect_used_in_outer
:
317 gcc_assert (STMT_VINFO_DEF_TYPE (stmt_vinfo
) != vect_reduction_def
);
318 relevant
= vect_used_in_scope
;
321 case vect_used_in_scope
:
329 /* case 3b: inner-loop stmt defining an outer-loop stmt:
330 outer-loop-header-bb:
334 outer-loop-tail-bb (or outer-loop-exit-bb in double reduction):
336 else if (flow_loop_nested_p (bb
->loop_father
, def_bb
->loop_father
))
338 if (vect_print_dump_info (REPORT_DETAILS
))
339 fprintf (vect_dump
, "inner-loop def-stmt defining outer-loop stmt.");
343 case vect_unused_in_scope
:
344 relevant
= (STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_reduction_def
345 || STMT_VINFO_DEF_TYPE (stmt_vinfo
) == vect_double_reduction_def
) ?
346 vect_used_in_outer_by_reduction
: vect_unused_in_scope
;
349 case vect_used_by_reduction
:
350 relevant
= vect_used_in_outer_by_reduction
;
353 case vect_used_in_scope
:
354 relevant
= vect_used_in_outer
;
362 vect_mark_relevant (worklist
, def_stmt
, relevant
, live_p
);
367 /* Function vect_mark_stmts_to_be_vectorized.
369 Not all stmts in the loop need to be vectorized. For example:
378 Stmt 1 and 3 do not need to be vectorized, because loop control and
379 addressing of vectorized data-refs are handled differently.
381 This pass detects such stmts. */
384 vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo
)
386 VEC(gimple
,heap
) *worklist
;
387 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
388 basic_block
*bbs
= LOOP_VINFO_BBS (loop_vinfo
);
389 unsigned int nbbs
= loop
->num_nodes
;
390 gimple_stmt_iterator si
;
393 stmt_vec_info stmt_vinfo
;
397 enum vect_relevant relevant
, tmp_relevant
;
398 enum vect_def_type def_type
;
400 if (vect_print_dump_info (REPORT_DETAILS
))
401 fprintf (vect_dump
, "=== vect_mark_stmts_to_be_vectorized ===");
403 worklist
= VEC_alloc (gimple
, heap
, 64);
405 /* 1. Init worklist. */
406 for (i
= 0; i
< nbbs
; i
++)
409 for (si
= gsi_start_phis (bb
); !gsi_end_p (si
); gsi_next (&si
))
412 if (vect_print_dump_info (REPORT_DETAILS
))
414 fprintf (vect_dump
, "init: phi relevant? ");
415 print_gimple_stmt (vect_dump
, phi
, 0, TDF_SLIM
);
418 if (vect_stmt_relevant_p (phi
, loop_vinfo
, &relevant
, &live_p
))
419 vect_mark_relevant (&worklist
, phi
, relevant
, live_p
);
421 for (si
= gsi_start_bb (bb
); !gsi_end_p (si
); gsi_next (&si
))
423 stmt
= gsi_stmt (si
);
424 if (vect_print_dump_info (REPORT_DETAILS
))
426 fprintf (vect_dump
, "init: stmt relevant? ");
427 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
430 if (vect_stmt_relevant_p (stmt
, loop_vinfo
, &relevant
, &live_p
))
431 vect_mark_relevant (&worklist
, stmt
, relevant
, live_p
);
435 /* 2. Process_worklist */
436 while (VEC_length (gimple
, worklist
) > 0)
441 stmt
= VEC_pop (gimple
, worklist
);
442 if (vect_print_dump_info (REPORT_DETAILS
))
444 fprintf (vect_dump
, "worklist: examine stmt: ");
445 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
448 /* Examine the USEs of STMT. For each USE, mark the stmt that defines it
449 (DEF_STMT) as relevant/irrelevant and live/dead according to the
450 liveness and relevance properties of STMT. */
451 stmt_vinfo
= vinfo_for_stmt (stmt
);
452 relevant
= STMT_VINFO_RELEVANT (stmt_vinfo
);
453 live_p
= STMT_VINFO_LIVE_P (stmt_vinfo
);
455 /* Generally, the liveness and relevance properties of STMT are
456 propagated as is to the DEF_STMTs of its USEs:
457 live_p <-- STMT_VINFO_LIVE_P (STMT_VINFO)
458 relevant <-- STMT_VINFO_RELEVANT (STMT_VINFO)
460 One exception is when STMT has been identified as defining a reduction
461 variable; in this case we set the liveness/relevance as follows:
463 relevant = vect_used_by_reduction
464 This is because we distinguish between two kinds of relevant stmts -
465 those that are used by a reduction computation, and those that are
466 (also) used by a regular computation. This allows us later on to
467 identify stmts that are used solely by a reduction, and therefore the
468 order of the results that they produce does not have to be kept. */
470 def_type
= STMT_VINFO_DEF_TYPE (stmt_vinfo
);
471 tmp_relevant
= relevant
;
474 case vect_reduction_def
:
475 switch (tmp_relevant
)
477 case vect_unused_in_scope
:
478 relevant
= vect_used_by_reduction
;
481 case vect_used_by_reduction
:
482 if (gimple_code (stmt
) == GIMPLE_PHI
)
487 if (vect_print_dump_info (REPORT_DETAILS
))
488 fprintf (vect_dump
, "unsupported use of reduction.");
490 VEC_free (gimple
, heap
, worklist
);
497 case vect_nested_cycle
:
498 if (tmp_relevant
!= vect_unused_in_scope
499 && tmp_relevant
!= vect_used_in_outer_by_reduction
500 && tmp_relevant
!= vect_used_in_outer
)
502 if (vect_print_dump_info (REPORT_DETAILS
))
503 fprintf (vect_dump
, "unsupported use of nested cycle.");
505 VEC_free (gimple
, heap
, worklist
);
512 case vect_double_reduction_def
:
513 if (tmp_relevant
!= vect_unused_in_scope
514 && tmp_relevant
!= vect_used_by_reduction
)
516 if (vect_print_dump_info (REPORT_DETAILS
))
517 fprintf (vect_dump
, "unsupported use of double reduction.");
519 VEC_free (gimple
, heap
, worklist
);
530 FOR_EACH_PHI_OR_STMT_USE (use_p
, stmt
, iter
, SSA_OP_USE
)
532 tree op
= USE_FROM_PTR (use_p
);
533 if (!process_use (stmt
, op
, loop_vinfo
, live_p
, relevant
, &worklist
))
535 VEC_free (gimple
, heap
, worklist
);
539 } /* while worklist */
541 VEC_free (gimple
, heap
, worklist
);
547 cost_for_stmt (gimple stmt
)
549 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
551 switch (STMT_VINFO_TYPE (stmt_info
))
553 case load_vec_info_type
:
554 return TARG_SCALAR_LOAD_COST
;
555 case store_vec_info_type
:
556 return TARG_SCALAR_STORE_COST
;
557 case op_vec_info_type
:
558 case condition_vec_info_type
:
559 case assignment_vec_info_type
:
560 case reduc_vec_info_type
:
561 case induc_vec_info_type
:
562 case type_promotion_vec_info_type
:
563 case type_demotion_vec_info_type
:
564 case type_conversion_vec_info_type
:
565 case call_vec_info_type
:
566 return TARG_SCALAR_STMT_COST
;
567 case undef_vec_info_type
:
573 /* Function vect_model_simple_cost.
575 Models cost for simple operations, i.e. those that only emit ncopies of a
576 single op. Right now, this does not account for multiple insns that could
577 be generated for the single vector op. We will handle that shortly. */
580 vect_model_simple_cost (stmt_vec_info stmt_info
, int ncopies
,
581 enum vect_def_type
*dt
, slp_tree slp_node
)
584 int inside_cost
= 0, outside_cost
= 0;
586 /* The SLP costs were already calculated during SLP tree build. */
587 if (PURE_SLP_STMT (stmt_info
))
590 inside_cost
= ncopies
* TARG_VEC_STMT_COST
;
592 /* FORNOW: Assuming maximum 2 args per stmts. */
593 for (i
= 0; i
< 2; i
++)
595 if (dt
[i
] == vect_constant_def
|| dt
[i
] == vect_external_def
)
596 outside_cost
+= TARG_SCALAR_TO_VEC_COST
;
599 if (vect_print_dump_info (REPORT_COST
))
600 fprintf (vect_dump
, "vect_model_simple_cost: inside_cost = %d, "
601 "outside_cost = %d .", inside_cost
, outside_cost
);
603 /* Set the costs either in STMT_INFO or SLP_NODE (if exists). */
604 stmt_vinfo_set_inside_of_loop_cost (stmt_info
, slp_node
, inside_cost
);
605 stmt_vinfo_set_outside_of_loop_cost (stmt_info
, slp_node
, outside_cost
);
609 /* Function vect_cost_strided_group_size
611 For strided load or store, return the group_size only if it is the first
612 load or store of a group, else return 1. This ensures that group size is
613 only returned once per group. */
616 vect_cost_strided_group_size (stmt_vec_info stmt_info
)
618 gimple first_stmt
= DR_GROUP_FIRST_DR (stmt_info
);
620 if (first_stmt
== STMT_VINFO_STMT (stmt_info
))
621 return DR_GROUP_SIZE (stmt_info
);
627 /* Function vect_model_store_cost
629 Models cost for stores. In the case of strided accesses, one access
630 has the overhead of the strided access attributed to it. */
633 vect_model_store_cost (stmt_vec_info stmt_info
, int ncopies
,
634 enum vect_def_type dt
, slp_tree slp_node
)
637 int inside_cost
= 0, outside_cost
= 0;
639 /* The SLP costs were already calculated during SLP tree build. */
640 if (PURE_SLP_STMT (stmt_info
))
643 if (dt
== vect_constant_def
|| dt
== vect_external_def
)
644 outside_cost
= TARG_SCALAR_TO_VEC_COST
;
646 /* Strided access? */
647 if (DR_GROUP_FIRST_DR (stmt_info
) && !slp_node
)
648 group_size
= vect_cost_strided_group_size (stmt_info
);
649 /* Not a strided access. */
653 /* Is this an access in a group of stores, which provide strided access?
654 If so, add in the cost of the permutes. */
657 /* Uses a high and low interleave operation for each needed permute. */
658 inside_cost
= ncopies
* exact_log2(group_size
) * group_size
659 * TARG_VEC_STMT_COST
;
661 if (vect_print_dump_info (REPORT_COST
))
662 fprintf (vect_dump
, "vect_model_store_cost: strided group_size = %d .",
667 /* Costs of the stores. */
668 inside_cost
+= ncopies
* TARG_VEC_STORE_COST
;
670 if (vect_print_dump_info (REPORT_COST
))
671 fprintf (vect_dump
, "vect_model_store_cost: inside_cost = %d, "
672 "outside_cost = %d .", inside_cost
, outside_cost
);
674 /* Set the costs either in STMT_INFO or SLP_NODE (if exists). */
675 stmt_vinfo_set_inside_of_loop_cost (stmt_info
, slp_node
, inside_cost
);
676 stmt_vinfo_set_outside_of_loop_cost (stmt_info
, slp_node
, outside_cost
);
680 /* Function vect_model_load_cost
682 Models cost for loads. In the case of strided accesses, the last access
683 has the overhead of the strided access attributed to it. Since unaligned
684 accesses are supported for loads, we also account for the costs of the
685 access scheme chosen. */
688 vect_model_load_cost (stmt_vec_info stmt_info
, int ncopies
, slp_tree slp_node
)
692 int alignment_support_cheme
;
694 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
;
695 int inside_cost
= 0, outside_cost
= 0;
697 /* The SLP costs were already calculated during SLP tree build. */
698 if (PURE_SLP_STMT (stmt_info
))
701 /* Strided accesses? */
702 first_stmt
= DR_GROUP_FIRST_DR (stmt_info
);
703 if (first_stmt
&& !slp_node
)
705 group_size
= vect_cost_strided_group_size (stmt_info
);
706 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
708 /* Not a strided access. */
715 alignment_support_cheme
= vect_supportable_dr_alignment (first_dr
);
717 /* Is this an access in a group of loads providing strided access?
718 If so, add in the cost of the permutes. */
721 /* Uses an even and odd extract operations for each needed permute. */
722 inside_cost
= ncopies
* exact_log2(group_size
) * group_size
723 * TARG_VEC_STMT_COST
;
725 if (vect_print_dump_info (REPORT_COST
))
726 fprintf (vect_dump
, "vect_model_load_cost: strided group_size = %d .",
731 /* The loads themselves. */
732 switch (alignment_support_cheme
)
736 inside_cost
+= ncopies
* TARG_VEC_LOAD_COST
;
738 if (vect_print_dump_info (REPORT_COST
))
739 fprintf (vect_dump
, "vect_model_load_cost: aligned.");
743 case dr_unaligned_supported
:
745 /* Here, we assign an additional cost for the unaligned load. */
746 inside_cost
+= ncopies
* TARG_VEC_UNALIGNED_LOAD_COST
;
748 if (vect_print_dump_info (REPORT_COST
))
749 fprintf (vect_dump
, "vect_model_load_cost: unaligned supported by "
754 case dr_explicit_realign
:
756 inside_cost
+= ncopies
* (2*TARG_VEC_LOAD_COST
+ TARG_VEC_STMT_COST
);
758 /* FIXME: If the misalignment remains fixed across the iterations of
759 the containing loop, the following cost should be added to the
761 if (targetm
.vectorize
.builtin_mask_for_load
)
762 inside_cost
+= TARG_VEC_STMT_COST
;
766 case dr_explicit_realign_optimized
:
768 if (vect_print_dump_info (REPORT_COST
))
769 fprintf (vect_dump
, "vect_model_load_cost: unaligned software "
772 /* Unaligned software pipeline has a load of an address, an initial
773 load, and possibly a mask operation to "prime" the loop. However,
774 if this is an access in a group of loads, which provide strided
775 access, then the above cost should only be considered for one
776 access in the group. Inside the loop, there is a load op
777 and a realignment op. */
779 if ((!DR_GROUP_FIRST_DR (stmt_info
)) || group_size
> 1 || slp_node
)
781 outside_cost
= 2*TARG_VEC_STMT_COST
;
782 if (targetm
.vectorize
.builtin_mask_for_load
)
783 outside_cost
+= TARG_VEC_STMT_COST
;
786 inside_cost
+= ncopies
* (TARG_VEC_LOAD_COST
+ TARG_VEC_STMT_COST
);
795 if (vect_print_dump_info (REPORT_COST
))
796 fprintf (vect_dump
, "vect_model_load_cost: inside_cost = %d, "
797 "outside_cost = %d .", inside_cost
, outside_cost
);
799 /* Set the costs either in STMT_INFO or SLP_NODE (if exists). */
800 stmt_vinfo_set_inside_of_loop_cost (stmt_info
, slp_node
, inside_cost
);
801 stmt_vinfo_set_outside_of_loop_cost (stmt_info
, slp_node
, outside_cost
);
805 /* Function vect_init_vector.
807 Insert a new stmt (INIT_STMT) that initializes a new vector variable with
808 the vector elements of VECTOR_VAR. Place the initialization at BSI if it
809 is not NULL. Otherwise, place the initialization at the loop preheader.
810 Return the DEF of INIT_STMT.
811 It will be used in the vectorization of STMT. */
814 vect_init_vector (gimple stmt
, tree vector_var
, tree vector_type
,
815 gimple_stmt_iterator
*gsi
)
817 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
825 new_var
= vect_get_new_vect_var (vector_type
, vect_simple_var
, "cst_");
826 add_referenced_var (new_var
);
827 init_stmt
= gimple_build_assign (new_var
, vector_var
);
828 new_temp
= make_ssa_name (new_var
, init_stmt
);
829 gimple_assign_set_lhs (init_stmt
, new_temp
);
832 vect_finish_stmt_generation (stmt
, init_stmt
, gsi
);
835 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
839 struct loop
*loop
= LOOP_VINFO_LOOP (loop_vinfo
);
841 if (nested_in_vect_loop_p (loop
, stmt
))
844 pe
= loop_preheader_edge (loop
);
845 new_bb
= gsi_insert_on_edge_immediate (pe
, init_stmt
);
846 gcc_assert (!new_bb
);
850 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_vinfo
);
852 gimple_stmt_iterator gsi_bb_start
;
854 gcc_assert (bb_vinfo
);
855 bb
= BB_VINFO_BB (bb_vinfo
);
856 gsi_bb_start
= gsi_after_labels (bb
);
857 gsi_insert_before (&gsi_bb_start
, init_stmt
, GSI_SAME_STMT
);
861 if (vect_print_dump_info (REPORT_DETAILS
))
863 fprintf (vect_dump
, "created new init_stmt: ");
864 print_gimple_stmt (vect_dump
, init_stmt
, 0, TDF_SLIM
);
867 vec_oprnd
= gimple_assign_lhs (init_stmt
);
872 /* Function vect_get_vec_def_for_operand.
874 OP is an operand in STMT. This function returns a (vector) def that will be
875 used in the vectorized stmt for STMT.
877 In the case that OP is an SSA_NAME which is defined in the loop, then
878 STMT_VINFO_VEC_STMT of the defining stmt holds the relevant def.
880 In case OP is an invariant or constant, a new stmt that creates a vector def
881 needs to be introduced. */
884 vect_get_vec_def_for_operand (tree op
, gimple stmt
, tree
*scalar_def
)
889 stmt_vec_info def_stmt_info
= NULL
;
890 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (stmt
);
891 tree vectype
= STMT_VINFO_VECTYPE (stmt_vinfo
);
892 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
893 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_vinfo
);
899 enum vect_def_type dt
;
903 if (vect_print_dump_info (REPORT_DETAILS
))
905 fprintf (vect_dump
, "vect_get_vec_def_for_operand: ");
906 print_generic_expr (vect_dump
, op
, TDF_SLIM
);
909 is_simple_use
= vect_is_simple_use (op
, loop_vinfo
, NULL
, &def_stmt
, &def
,
911 gcc_assert (is_simple_use
);
912 if (vect_print_dump_info (REPORT_DETAILS
))
916 fprintf (vect_dump
, "def = ");
917 print_generic_expr (vect_dump
, def
, TDF_SLIM
);
921 fprintf (vect_dump
, " def_stmt = ");
922 print_gimple_stmt (vect_dump
, def_stmt
, 0, TDF_SLIM
);
928 /* Case 1: operand is a constant. */
929 case vect_constant_def
:
931 vector_type
= get_vectype_for_scalar_type (TREE_TYPE (op
));
932 gcc_assert (vector_type
);
937 /* Create 'vect_cst_ = {cst,cst,...,cst}' */
938 if (vect_print_dump_info (REPORT_DETAILS
))
939 fprintf (vect_dump
, "Create vector_cst. nunits = %d", nunits
);
941 for (i
= nunits
- 1; i
>= 0; --i
)
943 t
= tree_cons (NULL_TREE
, op
, t
);
945 vec_cst
= build_vector (vector_type
, t
);
946 return vect_init_vector (stmt
, vec_cst
, vector_type
, NULL
);
949 /* Case 2: operand is defined outside the loop - loop invariant. */
950 case vect_external_def
:
952 vector_type
= get_vectype_for_scalar_type (TREE_TYPE (def
));
953 gcc_assert (vector_type
);
954 nunits
= TYPE_VECTOR_SUBPARTS (vector_type
);
959 /* Create 'vec_inv = {inv,inv,..,inv}' */
960 if (vect_print_dump_info (REPORT_DETAILS
))
961 fprintf (vect_dump
, "Create vector_inv.");
963 for (i
= nunits
- 1; i
>= 0; --i
)
965 t
= tree_cons (NULL_TREE
, def
, t
);
968 /* FIXME: use build_constructor directly. */
969 vec_inv
= build_constructor_from_list (vector_type
, t
);
970 return vect_init_vector (stmt
, vec_inv
, vector_type
, NULL
);
973 /* Case 3: operand is defined inside the loop. */
974 case vect_internal_def
:
977 *scalar_def
= NULL
/* FIXME tuples: def_stmt*/;
979 /* Get the def from the vectorized stmt. */
980 def_stmt_info
= vinfo_for_stmt (def_stmt
);
981 vec_stmt
= STMT_VINFO_VEC_STMT (def_stmt_info
);
982 gcc_assert (vec_stmt
);
983 if (gimple_code (vec_stmt
) == GIMPLE_PHI
)
984 vec_oprnd
= PHI_RESULT (vec_stmt
);
985 else if (is_gimple_call (vec_stmt
))
986 vec_oprnd
= gimple_call_lhs (vec_stmt
);
988 vec_oprnd
= gimple_assign_lhs (vec_stmt
);
992 /* Case 4: operand is defined by a loop header phi - reduction */
993 case vect_reduction_def
:
994 case vect_double_reduction_def
:
995 case vect_nested_cycle
:
999 gcc_assert (gimple_code (def_stmt
) == GIMPLE_PHI
);
1000 loop
= (gimple_bb (def_stmt
))->loop_father
;
1002 /* Get the def before the loop */
1003 op
= PHI_ARG_DEF_FROM_EDGE (def_stmt
, loop_preheader_edge (loop
));
1004 return get_initial_def_for_reduction (stmt
, op
, scalar_def
);
1007 /* Case 5: operand is defined by loop-header phi - induction. */
1008 case vect_induction_def
:
1010 gcc_assert (gimple_code (def_stmt
) == GIMPLE_PHI
);
1012 /* Get the def from the vectorized stmt. */
1013 def_stmt_info
= vinfo_for_stmt (def_stmt
);
1014 vec_stmt
= STMT_VINFO_VEC_STMT (def_stmt_info
);
1015 gcc_assert (vec_stmt
&& gimple_code (vec_stmt
) == GIMPLE_PHI
);
1016 vec_oprnd
= PHI_RESULT (vec_stmt
);
1026 /* Function vect_get_vec_def_for_stmt_copy
1028 Return a vector-def for an operand. This function is used when the
1029 vectorized stmt to be created (by the caller to this function) is a "copy"
1030 created in case the vectorized result cannot fit in one vector, and several
1031 copies of the vector-stmt are required. In this case the vector-def is
1032 retrieved from the vector stmt recorded in the STMT_VINFO_RELATED_STMT field
1033 of the stmt that defines VEC_OPRND.
1034 DT is the type of the vector def VEC_OPRND.
1037 In case the vectorization factor (VF) is bigger than the number
1038 of elements that can fit in a vectype (nunits), we have to generate
1039 more than one vector stmt to vectorize the scalar stmt. This situation
1040 arises when there are multiple data-types operated upon in the loop; the
1041 smallest data-type determines the VF, and as a result, when vectorizing
1042 stmts operating on wider types we need to create 'VF/nunits' "copies" of the
1043 vector stmt (each computing a vector of 'nunits' results, and together
1044 computing 'VF' results in each iteration). This function is called when
1045 vectorizing such a stmt (e.g. vectorizing S2 in the illustration below, in
1046 which VF=16 and nunits=4, so the number of copies required is 4):
1048 scalar stmt: vectorized into: STMT_VINFO_RELATED_STMT
1050 S1: x = load VS1.0: vx.0 = memref0 VS1.1
1051 VS1.1: vx.1 = memref1 VS1.2
1052 VS1.2: vx.2 = memref2 VS1.3
1053 VS1.3: vx.3 = memref3
1055 S2: z = x + ... VSnew.0: vz0 = vx.0 + ... VSnew.1
1056 VSnew.1: vz1 = vx.1 + ... VSnew.2
1057 VSnew.2: vz2 = vx.2 + ... VSnew.3
1058 VSnew.3: vz3 = vx.3 + ...
1060 The vectorization of S1 is explained in vectorizable_load.
1061 The vectorization of S2:
1062 To create the first vector-stmt out of the 4 copies - VSnew.0 -
1063 the function 'vect_get_vec_def_for_operand' is called to
1064 get the relevant vector-def for each operand of S2. For operand x it
1065 returns the vector-def 'vx.0'.
1067 To create the remaining copies of the vector-stmt (VSnew.j), this
1068 function is called to get the relevant vector-def for each operand. It is
1069 obtained from the respective VS1.j stmt, which is recorded in the
1070 STMT_VINFO_RELATED_STMT field of the stmt that defines VEC_OPRND.
1072 For example, to obtain the vector-def 'vx.1' in order to create the
1073 vector stmt 'VSnew.1', this function is called with VEC_OPRND='vx.0'.
1074 Given 'vx0' we obtain the stmt that defines it ('VS1.0'); from the
1075 STMT_VINFO_RELATED_STMT field of 'VS1.0' we obtain the next copy - 'VS1.1',
1076 and return its def ('vx.1').
1077 Overall, to create the above sequence this function will be called 3 times:
1078 vx.1 = vect_get_vec_def_for_stmt_copy (dt, vx.0);
1079 vx.2 = vect_get_vec_def_for_stmt_copy (dt, vx.1);
1080 vx.3 = vect_get_vec_def_for_stmt_copy (dt, vx.2); */
1083 vect_get_vec_def_for_stmt_copy (enum vect_def_type dt
, tree vec_oprnd
)
1085 gimple vec_stmt_for_operand
;
1086 stmt_vec_info def_stmt_info
;
1088 /* Do nothing; can reuse same def. */
1089 if (dt
== vect_external_def
|| dt
== vect_constant_def
)
1092 vec_stmt_for_operand
= SSA_NAME_DEF_STMT (vec_oprnd
);
1093 def_stmt_info
= vinfo_for_stmt (vec_stmt_for_operand
);
1094 gcc_assert (def_stmt_info
);
1095 vec_stmt_for_operand
= STMT_VINFO_RELATED_STMT (def_stmt_info
);
1096 gcc_assert (vec_stmt_for_operand
);
1097 vec_oprnd
= gimple_get_lhs (vec_stmt_for_operand
);
1098 if (gimple_code (vec_stmt_for_operand
) == GIMPLE_PHI
)
1099 vec_oprnd
= PHI_RESULT (vec_stmt_for_operand
);
1101 vec_oprnd
= gimple_get_lhs (vec_stmt_for_operand
);
1106 /* Get vectorized definitions for the operands to create a copy of an original
1107 stmt. See vect_get_vec_def_for_stmt_copy() for details. */
1110 vect_get_vec_defs_for_stmt_copy (enum vect_def_type
*dt
,
1111 VEC(tree
,heap
) **vec_oprnds0
,
1112 VEC(tree
,heap
) **vec_oprnds1
)
1114 tree vec_oprnd
= VEC_pop (tree
, *vec_oprnds0
);
1116 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd
);
1117 VEC_quick_push (tree
, *vec_oprnds0
, vec_oprnd
);
1119 if (vec_oprnds1
&& *vec_oprnds1
)
1121 vec_oprnd
= VEC_pop (tree
, *vec_oprnds1
);
1122 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
[1], vec_oprnd
);
1123 VEC_quick_push (tree
, *vec_oprnds1
, vec_oprnd
);
1128 /* Get vectorized definitions for OP0 and OP1, or SLP_NODE if it is not NULL. */
1131 vect_get_vec_defs (tree op0
, tree op1
, gimple stmt
,
1132 VEC(tree
,heap
) **vec_oprnds0
, VEC(tree
,heap
) **vec_oprnds1
,
1136 vect_get_slp_defs (slp_node
, vec_oprnds0
, vec_oprnds1
);
1141 *vec_oprnds0
= VEC_alloc (tree
, heap
, 1);
1142 vec_oprnd
= vect_get_vec_def_for_operand (op0
, stmt
, NULL
);
1143 VEC_quick_push (tree
, *vec_oprnds0
, vec_oprnd
);
1147 *vec_oprnds1
= VEC_alloc (tree
, heap
, 1);
1148 vec_oprnd
= vect_get_vec_def_for_operand (op1
, stmt
, NULL
);
1149 VEC_quick_push (tree
, *vec_oprnds1
, vec_oprnd
);
1155 /* Function vect_finish_stmt_generation.
1157 Insert a new stmt. */
1160 vect_finish_stmt_generation (gimple stmt
, gimple vec_stmt
,
1161 gimple_stmt_iterator
*gsi
)
1163 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1164 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1165 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
1167 gcc_assert (gimple_code (stmt
) != GIMPLE_LABEL
);
1169 gsi_insert_before (gsi
, vec_stmt
, GSI_SAME_STMT
);
1171 set_vinfo_for_stmt (vec_stmt
, new_stmt_vec_info (vec_stmt
, loop_vinfo
,
1174 if (vect_print_dump_info (REPORT_DETAILS
))
1176 fprintf (vect_dump
, "add new stmt: ");
1177 print_gimple_stmt (vect_dump
, vec_stmt
, 0, TDF_SLIM
);
1180 gimple_set_location (vec_stmt
, gimple_location (gsi_stmt (*gsi
)));
1183 /* Checks if CALL can be vectorized in type VECTYPE. Returns
1184 a function declaration if the target has a vectorized version
1185 of the function, or NULL_TREE if the function cannot be vectorized. */
1188 vectorizable_function (gimple call
, tree vectype_out
, tree vectype_in
)
1190 tree fndecl
= gimple_call_fndecl (call
);
1191 enum built_in_function code
;
1193 /* We only handle functions that do not read or clobber memory -- i.e.
1194 const or novops ones. */
1195 if (!(gimple_call_flags (call
) & (ECF_CONST
| ECF_NOVOPS
)))
1199 || TREE_CODE (fndecl
) != FUNCTION_DECL
1200 || !DECL_BUILT_IN (fndecl
))
1203 code
= DECL_FUNCTION_CODE (fndecl
);
1204 return targetm
.vectorize
.builtin_vectorized_function (code
, vectype_out
,
1208 /* Function vectorizable_call.
1210 Check if STMT performs a function call that can be vectorized.
1211 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1212 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
1213 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1216 vectorizable_call (gimple stmt
, gimple_stmt_iterator
*gsi
, gimple
*vec_stmt
)
1221 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
1222 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
), prev_stmt_info
;
1223 tree vectype_out
, vectype_in
;
1226 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1227 tree fndecl
, new_temp
, def
, rhs_type
, lhs_type
;
1229 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
1232 VEC(tree
, heap
) *vargs
= NULL
;
1233 enum { NARROW
, NONE
, WIDEN
} modifier
;
1236 /* FORNOW: unsupported in basic block SLP. */
1237 gcc_assert (loop_vinfo
);
1239 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
1242 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
1245 /* FORNOW: SLP not supported. */
1246 if (STMT_SLP_TYPE (stmt_info
))
1249 /* Is STMT a vectorizable call? */
1250 if (!is_gimple_call (stmt
))
1253 if (TREE_CODE (gimple_call_lhs (stmt
)) != SSA_NAME
)
1256 /* Process function arguments. */
1257 rhs_type
= NULL_TREE
;
1258 nargs
= gimple_call_num_args (stmt
);
1260 /* Bail out if the function has more than two arguments, we
1261 do not have interesting builtin functions to vectorize with
1262 more than two arguments. No arguments is also not good. */
1263 if (nargs
== 0 || nargs
> 2)
1266 for (i
= 0; i
< nargs
; i
++)
1268 op
= gimple_call_arg (stmt
, i
);
1270 /* We can only handle calls with arguments of the same type. */
1272 && rhs_type
!= TREE_TYPE (op
))
1274 if (vect_print_dump_info (REPORT_DETAILS
))
1275 fprintf (vect_dump
, "argument types differ.");
1278 rhs_type
= TREE_TYPE (op
);
1280 if (!vect_is_simple_use (op
, loop_vinfo
, NULL
, &def_stmt
, &def
, &dt
[i
]))
1282 if (vect_print_dump_info (REPORT_DETAILS
))
1283 fprintf (vect_dump
, "use not simple.");
1288 vectype_in
= get_vectype_for_scalar_type (rhs_type
);
1291 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
1293 lhs_type
= TREE_TYPE (gimple_call_lhs (stmt
));
1294 vectype_out
= get_vectype_for_scalar_type (lhs_type
);
1297 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
1300 if (nunits_in
== nunits_out
/ 2)
1302 else if (nunits_out
== nunits_in
)
1304 else if (nunits_out
== nunits_in
/ 2)
1309 /* For now, we only vectorize functions if a target specific builtin
1310 is available. TODO -- in some cases, it might be profitable to
1311 insert the calls for pieces of the vector, in order to be able
1312 to vectorize other operations in the loop. */
1313 fndecl
= vectorizable_function (stmt
, vectype_out
, vectype_in
);
1314 if (fndecl
== NULL_TREE
)
1316 if (vect_print_dump_info (REPORT_DETAILS
))
1317 fprintf (vect_dump
, "function is not vectorizable.");
1322 gcc_assert (!gimple_vuse (stmt
));
1324 if (modifier
== NARROW
)
1325 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_out
;
1327 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
1329 /* Sanity check: make sure that at least one copy of the vectorized stmt
1330 needs to be generated. */
1331 gcc_assert (ncopies
>= 1);
1333 if (!vec_stmt
) /* transformation not required. */
1335 STMT_VINFO_TYPE (stmt_info
) = call_vec_info_type
;
1336 if (vect_print_dump_info (REPORT_DETAILS
))
1337 fprintf (vect_dump
, "=== vectorizable_call ===");
1338 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
);
1344 if (vect_print_dump_info (REPORT_DETAILS
))
1345 fprintf (vect_dump
, "transform operation.");
1348 scalar_dest
= gimple_call_lhs (stmt
);
1349 vec_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
1351 prev_stmt_info
= NULL
;
1355 for (j
= 0; j
< ncopies
; ++j
)
1357 /* Build argument list for the vectorized call. */
1359 vargs
= VEC_alloc (tree
, heap
, nargs
);
1361 VEC_truncate (tree
, vargs
, 0);
1363 for (i
= 0; i
< nargs
; i
++)
1365 op
= gimple_call_arg (stmt
, i
);
1368 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
1371 = vect_get_vec_def_for_stmt_copy (dt
[nargs
], vec_oprnd0
);
1373 VEC_quick_push (tree
, vargs
, vec_oprnd0
);
1376 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
1377 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
1378 gimple_call_set_lhs (new_stmt
, new_temp
);
1380 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1383 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
1385 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
1387 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
1393 for (j
= 0; j
< ncopies
; ++j
)
1395 /* Build argument list for the vectorized call. */
1397 vargs
= VEC_alloc (tree
, heap
, nargs
* 2);
1399 VEC_truncate (tree
, vargs
, 0);
1401 for (i
= 0; i
< nargs
; i
++)
1403 op
= gimple_call_arg (stmt
, i
);
1407 = vect_get_vec_def_for_operand (op
, stmt
, NULL
);
1409 = vect_get_vec_def_for_stmt_copy (dt
[nargs
], vec_oprnd0
);
1414 = vect_get_vec_def_for_stmt_copy (dt
[nargs
], vec_oprnd1
);
1416 = vect_get_vec_def_for_stmt_copy (dt
[nargs
], vec_oprnd0
);
1419 VEC_quick_push (tree
, vargs
, vec_oprnd0
);
1420 VEC_quick_push (tree
, vargs
, vec_oprnd1
);
1423 new_stmt
= gimple_build_call_vec (fndecl
, vargs
);
1424 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
1425 gimple_call_set_lhs (new_stmt
, new_temp
);
1427 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1430 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
1432 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
1434 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
1437 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
1442 /* No current target implements this case. */
1446 VEC_free (tree
, heap
, vargs
);
1448 /* Update the exception handling table with the vector stmt if necessary. */
1449 if (maybe_clean_or_replace_eh_stmt (stmt
, *vec_stmt
))
1450 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
1452 /* The call in STMT might prevent it from being removed in dce.
1453 We however cannot remove it here, due to the way the ssa name
1454 it defines is mapped to the new definition. So just replace
1455 rhs of the statement with something harmless. */
1457 type
= TREE_TYPE (scalar_dest
);
1458 new_stmt
= gimple_build_assign (gimple_call_lhs (stmt
),
1459 fold_convert (type
, integer_zero_node
));
1460 set_vinfo_for_stmt (new_stmt
, stmt_info
);
1461 set_vinfo_for_stmt (stmt
, NULL
);
1462 STMT_VINFO_STMT (stmt_info
) = new_stmt
;
1463 gsi_replace (gsi
, new_stmt
, false);
1464 SSA_NAME_DEF_STMT (gimple_assign_lhs (new_stmt
)) = new_stmt
;
1470 /* Function vect_gen_widened_results_half
1472 Create a vector stmt whose code, type, number of arguments, and result
1473 variable are CODE, OP_TYPE, and VEC_DEST, and its arguments are
1474 VEC_OPRND0 and VEC_OPRND1. The new vector stmt is to be inserted at BSI.
1475 In the case that CODE is a CALL_EXPR, this means that a call to DECL
1476 needs to be created (DECL is a function-decl of a target-builtin).
1477 STMT is the original scalar stmt that we are vectorizing. */
1480 vect_gen_widened_results_half (enum tree_code code
,
1482 tree vec_oprnd0
, tree vec_oprnd1
, int op_type
,
1483 tree vec_dest
, gimple_stmt_iterator
*gsi
,
1489 /* Generate half of the widened result: */
1490 if (code
== CALL_EXPR
)
1492 /* Target specific support */
1493 if (op_type
== binary_op
)
1494 new_stmt
= gimple_build_call (decl
, 2, vec_oprnd0
, vec_oprnd1
);
1496 new_stmt
= gimple_build_call (decl
, 1, vec_oprnd0
);
1497 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
1498 gimple_call_set_lhs (new_stmt
, new_temp
);
1502 /* Generic support */
1503 gcc_assert (op_type
== TREE_CODE_LENGTH (code
));
1504 if (op_type
!= binary_op
)
1506 new_stmt
= gimple_build_assign_with_ops (code
, vec_dest
, vec_oprnd0
,
1508 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
1509 gimple_assign_set_lhs (new_stmt
, new_temp
);
1511 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1517 /* Check if STMT performs a conversion operation, that can be vectorized.
1518 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1519 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
1520 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1523 vectorizable_conversion (gimple stmt
, gimple_stmt_iterator
*gsi
,
1524 gimple
*vec_stmt
, slp_tree slp_node
)
1529 tree vec_oprnd0
= NULL_TREE
, vec_oprnd1
= NULL_TREE
;
1530 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1531 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1532 enum tree_code code
, code1
= ERROR_MARK
, code2
= ERROR_MARK
;
1533 tree decl1
= NULL_TREE
, decl2
= NULL_TREE
;
1537 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
1538 gimple new_stmt
= NULL
;
1539 stmt_vec_info prev_stmt_info
;
1542 tree vectype_out
, vectype_in
;
1545 tree rhs_type
, lhs_type
;
1547 enum { NARROW
, NONE
, WIDEN
} modifier
;
1549 VEC(tree
,heap
) *vec_oprnds0
= NULL
;
1552 VEC(tree
,heap
) *dummy
= NULL
;
1555 /* Is STMT a vectorizable conversion? */
1557 /* FORNOW: unsupported in basic block SLP. */
1558 gcc_assert (loop_vinfo
);
1560 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
1563 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
1566 if (!is_gimple_assign (stmt
))
1569 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
1572 code
= gimple_assign_rhs_code (stmt
);
1573 if (code
!= FIX_TRUNC_EXPR
&& code
!= FLOAT_EXPR
)
1576 /* Check types of lhs and rhs. */
1577 op0
= gimple_assign_rhs1 (stmt
);
1578 rhs_type
= TREE_TYPE (op0
);
1579 vectype_in
= get_vectype_for_scalar_type (rhs_type
);
1582 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
1584 scalar_dest
= gimple_assign_lhs (stmt
);
1585 lhs_type
= TREE_TYPE (scalar_dest
);
1586 vectype_out
= get_vectype_for_scalar_type (lhs_type
);
1589 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
1592 if (nunits_in
== nunits_out
/ 2)
1594 else if (nunits_out
== nunits_in
)
1596 else if (nunits_out
== nunits_in
/ 2)
1601 if (modifier
== NONE
)
1602 gcc_assert (STMT_VINFO_VECTYPE (stmt_info
) == vectype_out
);
1604 /* Bail out if the types are both integral or non-integral. */
1605 if ((INTEGRAL_TYPE_P (rhs_type
) && INTEGRAL_TYPE_P (lhs_type
))
1606 || (!INTEGRAL_TYPE_P (rhs_type
) && !INTEGRAL_TYPE_P (lhs_type
)))
1609 integral_type
= INTEGRAL_TYPE_P (rhs_type
) ? vectype_in
: vectype_out
;
1611 if (modifier
== NARROW
)
1612 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_out
;
1614 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
1616 /* FORNOW: SLP with multiple types is not supported. The SLP analysis verifies
1617 this, so we can safely override NCOPIES with 1 here. */
1621 /* Sanity check: make sure that at least one copy of the vectorized stmt
1622 needs to be generated. */
1623 gcc_assert (ncopies
>= 1);
1625 /* Check the operands of the operation. */
1626 if (!vect_is_simple_use (op0
, loop_vinfo
, NULL
, &def_stmt
, &def
, &dt
[0]))
1628 if (vect_print_dump_info (REPORT_DETAILS
))
1629 fprintf (vect_dump
, "use not simple.");
1633 /* Supportable by target? */
1634 if ((modifier
== NONE
1635 && !targetm
.vectorize
.builtin_conversion (code
, integral_type
))
1636 || (modifier
== WIDEN
1637 && !supportable_widening_operation (code
, stmt
, vectype_in
,
1640 &dummy_int
, &dummy
))
1641 || (modifier
== NARROW
1642 && !supportable_narrowing_operation (code
, stmt
, vectype_in
,
1643 &code1
, &dummy_int
, &dummy
)))
1645 if (vect_print_dump_info (REPORT_DETAILS
))
1646 fprintf (vect_dump
, "conversion not supported by target.");
1650 if (modifier
!= NONE
)
1652 STMT_VINFO_VECTYPE (stmt_info
) = vectype_in
;
1653 /* FORNOW: SLP not supported. */
1654 if (STMT_SLP_TYPE (stmt_info
))
1658 if (!vec_stmt
) /* transformation not required. */
1660 STMT_VINFO_TYPE (stmt_info
) = type_conversion_vec_info_type
;
1665 if (vect_print_dump_info (REPORT_DETAILS
))
1666 fprintf (vect_dump
, "transform conversion.");
1669 vec_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
1671 if (modifier
== NONE
&& !slp_node
)
1672 vec_oprnds0
= VEC_alloc (tree
, heap
, 1);
1674 prev_stmt_info
= NULL
;
1678 for (j
= 0; j
< ncopies
; j
++)
1681 vect_get_vec_defs (op0
, NULL
, stmt
, &vec_oprnds0
, NULL
, slp_node
);
1683 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, NULL
);
1686 targetm
.vectorize
.builtin_conversion (code
, integral_type
);
1687 for (i
= 0; VEC_iterate (tree
, vec_oprnds0
, i
, vop0
); i
++)
1689 /* Arguments are ready. create the new vector stmt. */
1690 new_stmt
= gimple_build_call (builtin_decl
, 1, vop0
);
1691 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
1692 gimple_call_set_lhs (new_stmt
, new_temp
);
1693 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1695 VEC_quick_push (gimple
, SLP_TREE_VEC_STMTS (slp_node
), new_stmt
);
1699 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
1701 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
1702 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
1707 /* In case the vectorization factor (VF) is bigger than the number
1708 of elements that we can fit in a vectype (nunits), we have to
1709 generate more than one vector stmt - i.e - we need to "unroll"
1710 the vector stmt by a factor VF/nunits. */
1711 for (j
= 0; j
< ncopies
; j
++)
1714 vec_oprnd0
= vect_get_vec_def_for_operand (op0
, stmt
, NULL
);
1716 vec_oprnd0
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd0
);
1718 STMT_VINFO_VECTYPE (stmt_info
) = vectype_in
;
1720 /* Generate first half of the widened result: */
1722 = vect_gen_widened_results_half (code1
, decl1
,
1723 vec_oprnd0
, vec_oprnd1
,
1724 unary_op
, vec_dest
, gsi
, stmt
);
1726 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
1728 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
1729 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
1731 /* Generate second half of the widened result: */
1733 = vect_gen_widened_results_half (code2
, decl2
,
1734 vec_oprnd0
, vec_oprnd1
,
1735 unary_op
, vec_dest
, gsi
, stmt
);
1736 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
1737 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
1742 /* In case the vectorization factor (VF) is bigger than the number
1743 of elements that we can fit in a vectype (nunits), we have to
1744 generate more than one vector stmt - i.e - we need to "unroll"
1745 the vector stmt by a factor VF/nunits. */
1746 for (j
= 0; j
< ncopies
; j
++)
1751 vec_oprnd0
= vect_get_vec_def_for_operand (op0
, stmt
, NULL
);
1752 vec_oprnd1
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd0
);
1756 vec_oprnd0
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd1
);
1757 vec_oprnd1
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd0
);
1760 /* Arguments are ready. Create the new vector stmt. */
1761 expr
= build2 (code1
, vectype_out
, vec_oprnd0
, vec_oprnd1
);
1762 new_stmt
= gimple_build_assign_with_ops (code1
, vec_dest
, vec_oprnd0
,
1764 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
1765 gimple_assign_set_lhs (new_stmt
, new_temp
);
1766 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
1769 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
1771 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
1773 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
1776 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
1780 VEC_free (tree
, heap
, vec_oprnds0
);
1784 /* Function vectorizable_assignment.
1786 Check if STMT performs an assignment (copy) that can be vectorized.
1787 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1788 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
1789 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1792 vectorizable_assignment (gimple stmt
, gimple_stmt_iterator
*gsi
,
1793 gimple
*vec_stmt
, slp_tree slp_node
)
1798 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1799 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1800 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1804 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
1805 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
1808 VEC(tree
,heap
) *vec_oprnds
= NULL
;
1810 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
1812 /* Multiple types in SLP are handled by creating the appropriate number of
1813 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
1818 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
1820 gcc_assert (ncopies
>= 1);
1822 return false; /* FORNOW */
1824 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
1827 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
1830 /* Is vectorizable assignment? */
1831 if (!is_gimple_assign (stmt
))
1834 scalar_dest
= gimple_assign_lhs (stmt
);
1835 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
1838 if (gimple_assign_single_p (stmt
)
1839 || gimple_assign_rhs_code (stmt
) == PAREN_EXPR
)
1840 op
= gimple_assign_rhs1 (stmt
);
1844 if (!vect_is_simple_use (op
, loop_vinfo
, bb_vinfo
, &def_stmt
, &def
, &dt
[0]))
1846 if (vect_print_dump_info (REPORT_DETAILS
))
1847 fprintf (vect_dump
, "use not simple.");
1851 if (!vec_stmt
) /* transformation not required. */
1853 STMT_VINFO_TYPE (stmt_info
) = assignment_vec_info_type
;
1854 if (vect_print_dump_info (REPORT_DETAILS
))
1855 fprintf (vect_dump
, "=== vectorizable_assignment ===");
1856 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
);
1861 if (vect_print_dump_info (REPORT_DETAILS
))
1862 fprintf (vect_dump
, "transform assignment.");
1865 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
1868 vect_get_vec_defs (op
, NULL
, stmt
, &vec_oprnds
, NULL
, slp_node
);
1870 /* Arguments are ready. create the new vector stmt. */
1871 for (i
= 0; VEC_iterate (tree
, vec_oprnds
, i
, vop
); i
++)
1873 *vec_stmt
= gimple_build_assign (vec_dest
, vop
);
1874 new_temp
= make_ssa_name (vec_dest
, *vec_stmt
);
1875 gimple_assign_set_lhs (*vec_stmt
, new_temp
);
1876 vect_finish_stmt_generation (stmt
, *vec_stmt
, gsi
);
1877 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
;
1880 VEC_quick_push (gimple
, SLP_TREE_VEC_STMTS (slp_node
), *vec_stmt
);
1883 VEC_free (tree
, heap
, vec_oprnds
);
1887 /* Function vectorizable_operation.
1889 Check if STMT performs a binary or unary operation that can be vectorized.
1890 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
1891 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
1892 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
1895 vectorizable_operation (gimple stmt
, gimple_stmt_iterator
*gsi
,
1896 gimple
*vec_stmt
, slp_tree slp_node
)
1900 tree op0
, op1
= NULL
;
1901 tree vec_oprnd1
= NULL_TREE
;
1902 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
1903 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
1904 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
1905 enum tree_code code
;
1906 enum machine_mode vec_mode
;
1911 enum machine_mode optab_op2_mode
;
1914 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
1915 gimple new_stmt
= NULL
;
1916 stmt_vec_info prev_stmt_info
;
1917 int nunits_in
= TYPE_VECTOR_SUBPARTS (vectype
);
1922 VEC(tree
,heap
) *vec_oprnds0
= NULL
, *vec_oprnds1
= NULL
;
1925 bool shift_p
= false;
1926 bool scalar_shift_arg
= false;
1927 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
1931 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
1933 /* FORNOW: multiple types are not supported in basic block SLP. */
1936 /* Multiple types in SLP are handled by creating the appropriate number of
1937 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
1942 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
1944 gcc_assert (ncopies
>= 1);
1946 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
1949 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
1952 /* Is STMT a vectorizable binary/unary operation? */
1953 if (!is_gimple_assign (stmt
))
1956 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
1959 scalar_dest
= gimple_assign_lhs (stmt
);
1960 vectype_out
= get_vectype_for_scalar_type (TREE_TYPE (scalar_dest
));
1963 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
1964 if (nunits_out
!= nunits_in
)
1967 code
= gimple_assign_rhs_code (stmt
);
1969 /* For pointer addition, we should use the normal plus for
1970 the vector addition. */
1971 if (code
== POINTER_PLUS_EXPR
)
1974 /* Support only unary or binary operations. */
1975 op_type
= TREE_CODE_LENGTH (code
);
1976 if (op_type
!= unary_op
&& op_type
!= binary_op
)
1978 if (vect_print_dump_info (REPORT_DETAILS
))
1979 fprintf (vect_dump
, "num. args = %d (not unary/binary op).", op_type
);
1983 op0
= gimple_assign_rhs1 (stmt
);
1984 if (!vect_is_simple_use (op0
, loop_vinfo
, bb_vinfo
, &def_stmt
, &def
, &dt
[0]))
1986 if (vect_print_dump_info (REPORT_DETAILS
))
1987 fprintf (vect_dump
, "use not simple.");
1991 if (op_type
== binary_op
)
1993 op1
= gimple_assign_rhs2 (stmt
);
1994 if (!vect_is_simple_use (op1
, loop_vinfo
, bb_vinfo
, &def_stmt
, &def
,
1997 if (vect_print_dump_info (REPORT_DETAILS
))
1998 fprintf (vect_dump
, "use not simple.");
2003 /* If this is a shift/rotate, determine whether the shift amount is a vector,
2004 or scalar. If the shift/rotate amount is a vector, use the vector/vector
2006 if (code
== LSHIFT_EXPR
|| code
== RSHIFT_EXPR
|| code
== LROTATE_EXPR
2007 || code
== RROTATE_EXPR
)
2011 /* vector shifted by vector */
2012 if (dt
[1] == vect_internal_def
)
2014 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
2015 if (vect_print_dump_info (REPORT_DETAILS
))
2016 fprintf (vect_dump
, "vector/vector shift/rotate found.");
2019 /* See if the machine has a vector shifted by scalar insn and if not
2020 then see if it has a vector shifted by vector insn */
2021 else if (dt
[1] == vect_constant_def
|| dt
[1] == vect_external_def
)
2023 optab
= optab_for_tree_code (code
, vectype
, optab_scalar
);
2025 && (optab_handler (optab
, TYPE_MODE (vectype
))->insn_code
2026 != CODE_FOR_nothing
))
2028 scalar_shift_arg
= true;
2029 if (vect_print_dump_info (REPORT_DETAILS
))
2030 fprintf (vect_dump
, "vector/scalar shift/rotate found.");
2034 optab
= optab_for_tree_code (code
, vectype
, optab_vector
);
2036 && (optab_handler (optab
, TYPE_MODE (vectype
))->insn_code
2037 != CODE_FOR_nothing
))
2039 if (vect_print_dump_info (REPORT_DETAILS
))
2040 fprintf (vect_dump
, "vector/vector shift/rotate found.");
2042 /* Unlike the other binary operators, shifts/rotates have
2043 the rhs being int, instead of the same type as the lhs,
2044 so make sure the scalar is the right type if we are
2045 dealing with vectors of short/char. */
2046 if (dt
[1] == vect_constant_def
)
2047 op1
= fold_convert (TREE_TYPE (vectype
), op1
);
2054 if (vect_print_dump_info (REPORT_DETAILS
))
2055 fprintf (vect_dump
, "operand mode requires invariant argument.");
2060 optab
= optab_for_tree_code (code
, vectype
, optab_default
);
2062 /* Supportable by target? */
2065 if (vect_print_dump_info (REPORT_DETAILS
))
2066 fprintf (vect_dump
, "no optab.");
2069 vec_mode
= TYPE_MODE (vectype
);
2070 icode
= (int) optab_handler (optab
, vec_mode
)->insn_code
;
2071 if (icode
== CODE_FOR_nothing
)
2073 if (vect_print_dump_info (REPORT_DETAILS
))
2074 fprintf (vect_dump
, "op not supported by target.");
2075 /* Check only during analysis. */
2076 if (GET_MODE_SIZE (vec_mode
) != UNITS_PER_WORD
2077 || (vf
< vect_min_worthwhile_factor (code
)
2080 if (vect_print_dump_info (REPORT_DETAILS
))
2081 fprintf (vect_dump
, "proceeding using word mode.");
2084 /* Worthwhile without SIMD support? Check only during analysis. */
2085 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
2086 && vf
< vect_min_worthwhile_factor (code
)
2089 if (vect_print_dump_info (REPORT_DETAILS
))
2090 fprintf (vect_dump
, "not worthwhile without SIMD support.");
2094 if (!vec_stmt
) /* transformation not required. */
2096 STMT_VINFO_TYPE (stmt_info
) = op_vec_info_type
;
2097 if (vect_print_dump_info (REPORT_DETAILS
))
2098 fprintf (vect_dump
, "=== vectorizable_operation ===");
2099 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
);
2105 if (vect_print_dump_info (REPORT_DETAILS
))
2106 fprintf (vect_dump
, "transform binary/unary operation.");
2109 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
2111 /* Allocate VECs for vector operands. In case of SLP, vector operands are
2112 created in the previous stages of the recursion, so no allocation is
2113 needed, except for the case of shift with scalar shift argument. In that
2114 case we store the scalar operand in VEC_OPRNDS1 for every vector stmt to
2115 be created to vectorize the SLP group, i.e., SLP_NODE->VEC_STMTS_SIZE.
2116 In case of loop-based vectorization we allocate VECs of size 1. We
2117 allocate VEC_OPRNDS1 only in case of binary operation. */
2120 vec_oprnds0
= VEC_alloc (tree
, heap
, 1);
2121 if (op_type
== binary_op
)
2122 vec_oprnds1
= VEC_alloc (tree
, heap
, 1);
2124 else if (scalar_shift_arg
)
2125 vec_oprnds1
= VEC_alloc (tree
, heap
, slp_node
->vec_stmts_size
);
2127 /* In case the vectorization factor (VF) is bigger than the number
2128 of elements that we can fit in a vectype (nunits), we have to generate
2129 more than one vector stmt - i.e - we need to "unroll" the
2130 vector stmt by a factor VF/nunits. In doing so, we record a pointer
2131 from one copy of the vector stmt to the next, in the field
2132 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
2133 stages to find the correct vector defs to be used when vectorizing
2134 stmts that use the defs of the current stmt. The example below illustrates
2135 the vectorization process when VF=16 and nunits=4 (i.e - we need to create
2136 4 vectorized stmts):
2138 before vectorization:
2139 RELATED_STMT VEC_STMT
2143 step 1: vectorize stmt S1 (done in vectorizable_load. See more details
2145 RELATED_STMT VEC_STMT
2146 VS1_0: vx0 = memref0 VS1_1 -
2147 VS1_1: vx1 = memref1 VS1_2 -
2148 VS1_2: vx2 = memref2 VS1_3 -
2149 VS1_3: vx3 = memref3 - -
2150 S1: x = load - VS1_0
2153 step2: vectorize stmt S2 (done here):
2154 To vectorize stmt S2 we first need to find the relevant vector
2155 def for the first operand 'x'. This is, as usual, obtained from
2156 the vector stmt recorded in the STMT_VINFO_VEC_STMT of the stmt
2157 that defines 'x' (S1). This way we find the stmt VS1_0, and the
2158 relevant vector def 'vx0'. Having found 'vx0' we can generate
2159 the vector stmt VS2_0, and as usual, record it in the
2160 STMT_VINFO_VEC_STMT of stmt S2.
2161 When creating the second copy (VS2_1), we obtain the relevant vector
2162 def from the vector stmt recorded in the STMT_VINFO_RELATED_STMT of
2163 stmt VS1_0. This way we find the stmt VS1_1 and the relevant
2164 vector def 'vx1'. Using 'vx1' we create stmt VS2_1 and record a
2165 pointer to it in the STMT_VINFO_RELATED_STMT of the vector stmt VS2_0.
2166 Similarly when creating stmts VS2_2 and VS2_3. This is the resulting
2167 chain of stmts and pointers:
2168 RELATED_STMT VEC_STMT
2169 VS1_0: vx0 = memref0 VS1_1 -
2170 VS1_1: vx1 = memref1 VS1_2 -
2171 VS1_2: vx2 = memref2 VS1_3 -
2172 VS1_3: vx3 = memref3 - -
2173 S1: x = load - VS1_0
2174 VS2_0: vz0 = vx0 + v1 VS2_1 -
2175 VS2_1: vz1 = vx1 + v1 VS2_2 -
2176 VS2_2: vz2 = vx2 + v1 VS2_3 -
2177 VS2_3: vz3 = vx3 + v1 - -
2178 S2: z = x + 1 - VS2_0 */
2180 prev_stmt_info
= NULL
;
2181 for (j
= 0; j
< ncopies
; j
++)
2186 if (op_type
== binary_op
&& scalar_shift_arg
)
2188 /* Vector shl and shr insn patterns can be defined with scalar
2189 operand 2 (shift operand). In this case, use constant or loop
2190 invariant op1 directly, without extending it to vector mode
2192 optab_op2_mode
= insn_data
[icode
].operand
[2].mode
;
2193 if (!VECTOR_MODE_P (optab_op2_mode
))
2195 if (vect_print_dump_info (REPORT_DETAILS
))
2196 fprintf (vect_dump
, "operand 1 using scalar mode.");
2198 VEC_quick_push (tree
, vec_oprnds1
, vec_oprnd1
);
2201 /* Store vec_oprnd1 for every vector stmt to be created
2202 for SLP_NODE. We check during the analysis that all the
2203 shift arguments are the same.
2204 TODO: Allow different constants for different vector
2205 stmts generated for an SLP instance. */
2206 for (k
= 0; k
< slp_node
->vec_stmts_size
- 1; k
++)
2207 VEC_quick_push (tree
, vec_oprnds1
, vec_oprnd1
);
2212 /* vec_oprnd1 is available if operand 1 should be of a scalar-type
2213 (a special case for certain kind of vector shifts); otherwise,
2214 operand 1 should be of a vector type (the usual case). */
2215 if (op_type
== binary_op
&& !vec_oprnd1
)
2216 vect_get_vec_defs (op0
, op1
, stmt
, &vec_oprnds0
, &vec_oprnds1
,
2219 vect_get_vec_defs (op0
, NULL_TREE
, stmt
, &vec_oprnds0
, NULL
,
2223 vect_get_vec_defs_for_stmt_copy (dt
, &vec_oprnds0
, &vec_oprnds1
);
2225 /* Arguments are ready. Create the new vector stmt. */
2226 for (i
= 0; VEC_iterate (tree
, vec_oprnds0
, i
, vop0
); i
++)
2228 vop1
= ((op_type
== binary_op
)
2229 ? VEC_index (tree
, vec_oprnds1
, i
) : NULL
);
2230 new_stmt
= gimple_build_assign_with_ops (code
, vec_dest
, vop0
, vop1
);
2231 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
2232 gimple_assign_set_lhs (new_stmt
, new_temp
);
2233 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2235 VEC_quick_push (gimple
, SLP_TREE_VEC_STMTS (slp_node
), new_stmt
);
2242 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
2244 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
2245 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2248 VEC_free (tree
, heap
, vec_oprnds0
);
2250 VEC_free (tree
, heap
, vec_oprnds1
);
2256 /* Get vectorized definitions for loop-based vectorization. For the first
2257 operand we call vect_get_vec_def_for_operand() (with OPRND containing
2258 scalar operand), and for the rest we get a copy with
2259 vect_get_vec_def_for_stmt_copy() using the previous vector definition
2260 (stored in OPRND). See vect_get_vec_def_for_stmt_copy() for details.
2261 The vectors are collected into VEC_OPRNDS. */
2264 vect_get_loop_based_defs (tree
*oprnd
, gimple stmt
, enum vect_def_type dt
,
2265 VEC (tree
, heap
) **vec_oprnds
, int multi_step_cvt
)
2269 /* Get first vector operand. */
2270 /* All the vector operands except the very first one (that is scalar oprnd)
2272 if (TREE_CODE (TREE_TYPE (*oprnd
)) != VECTOR_TYPE
)
2273 vec_oprnd
= vect_get_vec_def_for_operand (*oprnd
, stmt
, NULL
);
2275 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, *oprnd
);
2277 VEC_quick_push (tree
, *vec_oprnds
, vec_oprnd
);
2279 /* Get second vector operand. */
2280 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, vec_oprnd
);
2281 VEC_quick_push (tree
, *vec_oprnds
, vec_oprnd
);
2285 /* For conversion in multiple steps, continue to get operands
2288 vect_get_loop_based_defs (oprnd
, stmt
, dt
, vec_oprnds
, multi_step_cvt
- 1);
2292 /* Create vectorized demotion statements for vector operands from VEC_OPRNDS.
2293 For multi-step conversions store the resulting vectors and call the function
2297 vect_create_vectorized_demotion_stmts (VEC (tree
, heap
) **vec_oprnds
,
2298 int multi_step_cvt
, gimple stmt
,
2299 VEC (tree
, heap
) *vec_dsts
,
2300 gimple_stmt_iterator
*gsi
,
2301 slp_tree slp_node
, enum tree_code code
,
2302 stmt_vec_info
*prev_stmt_info
)
2305 tree vop0
, vop1
, new_tmp
, vec_dest
;
2307 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
2309 vec_dest
= VEC_pop (tree
, vec_dsts
);
2311 for (i
= 0; i
< VEC_length (tree
, *vec_oprnds
); i
+= 2)
2313 /* Create demotion operation. */
2314 vop0
= VEC_index (tree
, *vec_oprnds
, i
);
2315 vop1
= VEC_index (tree
, *vec_oprnds
, i
+ 1);
2316 new_stmt
= gimple_build_assign_with_ops (code
, vec_dest
, vop0
, vop1
);
2317 new_tmp
= make_ssa_name (vec_dest
, new_stmt
);
2318 gimple_assign_set_lhs (new_stmt
, new_tmp
);
2319 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
2322 /* Store the resulting vector for next recursive call. */
2323 VEC_replace (tree
, *vec_oprnds
, i
/2, new_tmp
);
2326 /* This is the last step of the conversion sequence. Store the
2327 vectors in SLP_NODE or in vector info of the scalar statement
2328 (or in STMT_VINFO_RELATED_STMT chain). */
2330 VEC_quick_push (gimple
, SLP_TREE_VEC_STMTS (slp_node
), new_stmt
);
2333 if (!*prev_stmt_info
)
2334 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt
;
2336 STMT_VINFO_RELATED_STMT (*prev_stmt_info
) = new_stmt
;
2338 *prev_stmt_info
= vinfo_for_stmt (new_stmt
);
2343 /* For multi-step demotion operations we first generate demotion operations
2344 from the source type to the intermediate types, and then combine the
2345 results (stored in VEC_OPRNDS) in demotion operation to the destination
2349 /* At each level of recursion we have have of the operands we had at the
2351 VEC_truncate (tree
, *vec_oprnds
, (i
+1)/2);
2352 vect_create_vectorized_demotion_stmts (vec_oprnds
, multi_step_cvt
- 1,
2353 stmt
, vec_dsts
, gsi
, slp_node
,
2354 code
, prev_stmt_info
);
2359 /* Function vectorizable_type_demotion
2361 Check if STMT performs a binary or unary operation that involves
2362 type demotion, and if it can be vectorized.
2363 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2364 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2365 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2368 vectorizable_type_demotion (gimple stmt
, gimple_stmt_iterator
*gsi
,
2369 gimple
*vec_stmt
, slp_tree slp_node
)
2374 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
2375 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2376 enum tree_code code
, code1
= ERROR_MARK
;
2379 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
2380 stmt_vec_info prev_stmt_info
;
2387 int multi_step_cvt
= 0;
2388 VEC (tree
, heap
) *vec_oprnds0
= NULL
;
2389 VEC (tree
, heap
) *vec_dsts
= NULL
, *interm_types
= NULL
, *tmp_vec_dsts
= NULL
;
2390 tree last_oprnd
, intermediate_type
;
2392 /* FORNOW: not supported by basic block SLP vectorization. */
2393 gcc_assert (loop_vinfo
);
2395 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
2398 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
2401 /* Is STMT a vectorizable type-demotion operation? */
2402 if (!is_gimple_assign (stmt
))
2405 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
2408 code
= gimple_assign_rhs_code (stmt
);
2409 if (!CONVERT_EXPR_CODE_P (code
))
2412 op0
= gimple_assign_rhs1 (stmt
);
2413 vectype_in
= get_vectype_for_scalar_type (TREE_TYPE (op0
));
2416 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
2418 scalar_dest
= gimple_assign_lhs (stmt
);
2419 vectype_out
= get_vectype_for_scalar_type (TREE_TYPE (scalar_dest
));
2422 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
2423 if (nunits_in
>= nunits_out
)
2426 /* Multiple types in SLP are handled by creating the appropriate number of
2427 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
2432 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_out
;
2433 gcc_assert (ncopies
>= 1);
2435 if (! ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest
))
2436 && INTEGRAL_TYPE_P (TREE_TYPE (op0
)))
2437 || (SCALAR_FLOAT_TYPE_P (TREE_TYPE (scalar_dest
))
2438 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (op0
))
2439 && CONVERT_EXPR_CODE_P (code
))))
2442 /* Check the operands of the operation. */
2443 if (!vect_is_simple_use (op0
, loop_vinfo
, NULL
, &def_stmt
, &def
, &dt
[0]))
2445 if (vect_print_dump_info (REPORT_DETAILS
))
2446 fprintf (vect_dump
, "use not simple.");
2450 /* Supportable by target? */
2451 if (!supportable_narrowing_operation (code
, stmt
, vectype_in
, &code1
,
2452 &multi_step_cvt
, &interm_types
))
2455 STMT_VINFO_VECTYPE (stmt_info
) = vectype_in
;
2457 if (!vec_stmt
) /* transformation not required. */
2459 STMT_VINFO_TYPE (stmt_info
) = type_demotion_vec_info_type
;
2460 if (vect_print_dump_info (REPORT_DETAILS
))
2461 fprintf (vect_dump
, "=== vectorizable_demotion ===");
2462 vect_model_simple_cost (stmt_info
, ncopies
, dt
, NULL
);
2467 if (vect_print_dump_info (REPORT_DETAILS
))
2468 fprintf (vect_dump
, "transform type demotion operation. ncopies = %d.",
2471 /* In case of multi-step demotion, we first generate demotion operations to
2472 the intermediate types, and then from that types to the final one.
2473 We create vector destinations for the intermediate type (TYPES) received
2474 from supportable_narrowing_operation, and store them in the correct order
2475 for future use in vect_create_vectorized_demotion_stmts(). */
2477 vec_dsts
= VEC_alloc (tree
, heap
, multi_step_cvt
+ 1);
2479 vec_dsts
= VEC_alloc (tree
, heap
, 1);
2481 vec_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
2482 VEC_quick_push (tree
, vec_dsts
, vec_dest
);
2486 for (i
= VEC_length (tree
, interm_types
) - 1;
2487 VEC_iterate (tree
, interm_types
, i
, intermediate_type
); i
--)
2489 vec_dest
= vect_create_destination_var (scalar_dest
,
2491 VEC_quick_push (tree
, vec_dsts
, vec_dest
);
2495 /* In case the vectorization factor (VF) is bigger than the number
2496 of elements that we can fit in a vectype (nunits), we have to generate
2497 more than one vector stmt - i.e - we need to "unroll" the
2498 vector stmt by a factor VF/nunits. */
2500 prev_stmt_info
= NULL
;
2501 for (j
= 0; j
< ncopies
; j
++)
2505 vect_get_slp_defs (slp_node
, &vec_oprnds0
, NULL
);
2508 VEC_free (tree
, heap
, vec_oprnds0
);
2509 vec_oprnds0
= VEC_alloc (tree
, heap
,
2510 (multi_step_cvt
? vect_pow2 (multi_step_cvt
) * 2 : 2));
2511 vect_get_loop_based_defs (&last_oprnd
, stmt
, dt
[0], &vec_oprnds0
,
2512 vect_pow2 (multi_step_cvt
) - 1);
2515 /* Arguments are ready. Create the new vector stmts. */
2516 tmp_vec_dsts
= VEC_copy (tree
, heap
, vec_dsts
);
2517 vect_create_vectorized_demotion_stmts (&vec_oprnds0
,
2518 multi_step_cvt
, stmt
, tmp_vec_dsts
,
2519 gsi
, slp_node
, code1
,
2523 VEC_free (tree
, heap
, vec_oprnds0
);
2524 VEC_free (tree
, heap
, vec_dsts
);
2525 VEC_free (tree
, heap
, tmp_vec_dsts
);
2526 VEC_free (tree
, heap
, interm_types
);
2528 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
2533 /* Create vectorized promotion statements for vector operands from VEC_OPRNDS0
2534 and VEC_OPRNDS1 (for binary operations). For multi-step conversions store
2535 the resulting vectors and call the function recursively. */
2538 vect_create_vectorized_promotion_stmts (VEC (tree
, heap
) **vec_oprnds0
,
2539 VEC (tree
, heap
) **vec_oprnds1
,
2540 int multi_step_cvt
, gimple stmt
,
2541 VEC (tree
, heap
) *vec_dsts
,
2542 gimple_stmt_iterator
*gsi
,
2543 slp_tree slp_node
, enum tree_code code1
,
2544 enum tree_code code2
, tree decl1
,
2545 tree decl2
, int op_type
,
2546 stmt_vec_info
*prev_stmt_info
)
2549 tree vop0
, vop1
, new_tmp1
, new_tmp2
, vec_dest
;
2550 gimple new_stmt1
, new_stmt2
;
2551 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
2552 VEC (tree
, heap
) *vec_tmp
;
2554 vec_dest
= VEC_pop (tree
, vec_dsts
);
2555 vec_tmp
= VEC_alloc (tree
, heap
, VEC_length (tree
, *vec_oprnds0
) * 2);
2557 for (i
= 0; VEC_iterate (tree
, *vec_oprnds0
, i
, vop0
); i
++)
2559 if (op_type
== binary_op
)
2560 vop1
= VEC_index (tree
, *vec_oprnds1
, i
);
2564 /* Generate the two halves of promotion operation. */
2565 new_stmt1
= vect_gen_widened_results_half (code1
, decl1
, vop0
, vop1
,
2566 op_type
, vec_dest
, gsi
, stmt
);
2567 new_stmt2
= vect_gen_widened_results_half (code2
, decl2
, vop0
, vop1
,
2568 op_type
, vec_dest
, gsi
, stmt
);
2569 if (is_gimple_call (new_stmt1
))
2571 new_tmp1
= gimple_call_lhs (new_stmt1
);
2572 new_tmp2
= gimple_call_lhs (new_stmt2
);
2576 new_tmp1
= gimple_assign_lhs (new_stmt1
);
2577 new_tmp2
= gimple_assign_lhs (new_stmt2
);
2582 /* Store the results for the recursive call. */
2583 VEC_quick_push (tree
, vec_tmp
, new_tmp1
);
2584 VEC_quick_push (tree
, vec_tmp
, new_tmp2
);
2588 /* Last step of promotion sequience - store the results. */
2591 VEC_quick_push (gimple
, SLP_TREE_VEC_STMTS (slp_node
), new_stmt1
);
2592 VEC_quick_push (gimple
, SLP_TREE_VEC_STMTS (slp_node
), new_stmt2
);
2596 if (!*prev_stmt_info
)
2597 STMT_VINFO_VEC_STMT (stmt_info
) = new_stmt1
;
2599 STMT_VINFO_RELATED_STMT (*prev_stmt_info
) = new_stmt1
;
2601 *prev_stmt_info
= vinfo_for_stmt (new_stmt1
);
2602 STMT_VINFO_RELATED_STMT (*prev_stmt_info
) = new_stmt2
;
2603 *prev_stmt_info
= vinfo_for_stmt (new_stmt2
);
2610 /* For multi-step promotion operation we first generate we call the
2611 function recurcively for every stage. We start from the input type,
2612 create promotion operations to the intermediate types, and then
2613 create promotions to the output type. */
2614 *vec_oprnds0
= VEC_copy (tree
, heap
, vec_tmp
);
2615 VEC_free (tree
, heap
, vec_tmp
);
2616 vect_create_vectorized_promotion_stmts (vec_oprnds0
, vec_oprnds1
,
2617 multi_step_cvt
- 1, stmt
,
2618 vec_dsts
, gsi
, slp_node
, code1
,
2619 code2
, decl2
, decl2
, op_type
,
2625 /* Function vectorizable_type_promotion
2627 Check if STMT performs a binary or unary operation that involves
2628 type promotion, and if it can be vectorized.
2629 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2630 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2631 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2634 vectorizable_type_promotion (gimple stmt
, gimple_stmt_iterator
*gsi
,
2635 gimple
*vec_stmt
, slp_tree slp_node
)
2639 tree op0
, op1
= NULL
;
2640 tree vec_oprnd0
=NULL
, vec_oprnd1
=NULL
;
2641 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
2642 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2643 enum tree_code code
, code1
= ERROR_MARK
, code2
= ERROR_MARK
;
2644 tree decl1
= NULL_TREE
, decl2
= NULL_TREE
;
2648 enum vect_def_type dt
[2] = {vect_unknown_def_type
, vect_unknown_def_type
};
2649 stmt_vec_info prev_stmt_info
;
2656 tree intermediate_type
= NULL_TREE
;
2657 int multi_step_cvt
= 0;
2658 VEC (tree
, heap
) *vec_oprnds0
= NULL
, *vec_oprnds1
= NULL
;
2659 VEC (tree
, heap
) *vec_dsts
= NULL
, *interm_types
= NULL
, *tmp_vec_dsts
= NULL
;
2661 /* FORNOW: not supported by basic block SLP vectorization. */
2662 gcc_assert (loop_vinfo
);
2664 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
2667 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
2670 /* Is STMT a vectorizable type-promotion operation? */
2671 if (!is_gimple_assign (stmt
))
2674 if (TREE_CODE (gimple_assign_lhs (stmt
)) != SSA_NAME
)
2677 code
= gimple_assign_rhs_code (stmt
);
2678 if (!CONVERT_EXPR_CODE_P (code
)
2679 && code
!= WIDEN_MULT_EXPR
)
2682 op0
= gimple_assign_rhs1 (stmt
);
2683 vectype_in
= get_vectype_for_scalar_type (TREE_TYPE (op0
));
2686 nunits_in
= TYPE_VECTOR_SUBPARTS (vectype_in
);
2688 scalar_dest
= gimple_assign_lhs (stmt
);
2689 vectype_out
= get_vectype_for_scalar_type (TREE_TYPE (scalar_dest
));
2692 nunits_out
= TYPE_VECTOR_SUBPARTS (vectype_out
);
2693 if (nunits_in
<= nunits_out
)
2696 /* Multiple types in SLP are handled by creating the appropriate number of
2697 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
2702 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits_in
;
2704 gcc_assert (ncopies
>= 1);
2706 if (! ((INTEGRAL_TYPE_P (TREE_TYPE (scalar_dest
))
2707 && INTEGRAL_TYPE_P (TREE_TYPE (op0
)))
2708 || (SCALAR_FLOAT_TYPE_P (TREE_TYPE (scalar_dest
))
2709 && SCALAR_FLOAT_TYPE_P (TREE_TYPE (op0
))
2710 && CONVERT_EXPR_CODE_P (code
))))
2713 /* Check the operands of the operation. */
2714 if (!vect_is_simple_use (op0
, loop_vinfo
, NULL
, &def_stmt
, &def
, &dt
[0]))
2716 if (vect_print_dump_info (REPORT_DETAILS
))
2717 fprintf (vect_dump
, "use not simple.");
2721 op_type
= TREE_CODE_LENGTH (code
);
2722 if (op_type
== binary_op
)
2724 op1
= gimple_assign_rhs2 (stmt
);
2725 if (!vect_is_simple_use (op1
, loop_vinfo
, NULL
, &def_stmt
, &def
, &dt
[1]))
2727 if (vect_print_dump_info (REPORT_DETAILS
))
2728 fprintf (vect_dump
, "use not simple.");
2733 /* Supportable by target? */
2734 if (!supportable_widening_operation (code
, stmt
, vectype_in
,
2735 &decl1
, &decl2
, &code1
, &code2
,
2736 &multi_step_cvt
, &interm_types
))
2739 /* Binary widening operation can only be supported directly by the
2741 gcc_assert (!(multi_step_cvt
&& op_type
== binary_op
));
2743 STMT_VINFO_VECTYPE (stmt_info
) = vectype_in
;
2745 if (!vec_stmt
) /* transformation not required. */
2747 STMT_VINFO_TYPE (stmt_info
) = type_promotion_vec_info_type
;
2748 if (vect_print_dump_info (REPORT_DETAILS
))
2749 fprintf (vect_dump
, "=== vectorizable_promotion ===");
2750 vect_model_simple_cost (stmt_info
, 2*ncopies
, dt
, NULL
);
2756 if (vect_print_dump_info (REPORT_DETAILS
))
2757 fprintf (vect_dump
, "transform type promotion operation. ncopies = %d.",
2761 /* In case of multi-step promotion, we first generate promotion operations
2762 to the intermediate types, and then from that types to the final one.
2763 We store vector destination in VEC_DSTS in the correct order for
2764 recursive creation of promotion operations in
2765 vect_create_vectorized_promotion_stmts(). Vector destinations are created
2766 according to TYPES recieved from supportable_widening_operation(). */
2768 vec_dsts
= VEC_alloc (tree
, heap
, multi_step_cvt
+ 1);
2770 vec_dsts
= VEC_alloc (tree
, heap
, 1);
2772 vec_dest
= vect_create_destination_var (scalar_dest
, vectype_out
);
2773 VEC_quick_push (tree
, vec_dsts
, vec_dest
);
2777 for (i
= VEC_length (tree
, interm_types
) - 1;
2778 VEC_iterate (tree
, interm_types
, i
, intermediate_type
); i
--)
2780 vec_dest
= vect_create_destination_var (scalar_dest
,
2782 VEC_quick_push (tree
, vec_dsts
, vec_dest
);
2788 vec_oprnds0
= VEC_alloc (tree
, heap
,
2789 (multi_step_cvt
? vect_pow2 (multi_step_cvt
) : 1));
2790 if (op_type
== binary_op
)
2791 vec_oprnds1
= VEC_alloc (tree
, heap
, 1);
2794 /* In case the vectorization factor (VF) is bigger than the number
2795 of elements that we can fit in a vectype (nunits), we have to generate
2796 more than one vector stmt - i.e - we need to "unroll" the
2797 vector stmt by a factor VF/nunits. */
2799 prev_stmt_info
= NULL
;
2800 for (j
= 0; j
< ncopies
; j
++)
2806 vect_get_slp_defs (slp_node
, &vec_oprnds0
, &vec_oprnds1
);
2809 vec_oprnd0
= vect_get_vec_def_for_operand (op0
, stmt
, NULL
);
2810 VEC_quick_push (tree
, vec_oprnds0
, vec_oprnd0
);
2811 if (op_type
== binary_op
)
2813 vec_oprnd1
= vect_get_vec_def_for_operand (op1
, stmt
, NULL
);
2814 VEC_quick_push (tree
, vec_oprnds1
, vec_oprnd1
);
2820 vec_oprnd0
= vect_get_vec_def_for_stmt_copy (dt
[0], vec_oprnd0
);
2821 VEC_replace (tree
, vec_oprnds0
, 0, vec_oprnd0
);
2822 if (op_type
== binary_op
)
2824 vec_oprnd1
= vect_get_vec_def_for_stmt_copy (dt
[1], vec_oprnd1
);
2825 VEC_replace (tree
, vec_oprnds1
, 0, vec_oprnd1
);
2829 /* Arguments are ready. Create the new vector stmts. */
2830 tmp_vec_dsts
= VEC_copy (tree
, heap
, vec_dsts
);
2831 vect_create_vectorized_promotion_stmts (&vec_oprnds0
, &vec_oprnds1
,
2832 multi_step_cvt
, stmt
,
2834 gsi
, slp_node
, code1
, code2
,
2835 decl1
, decl2
, op_type
,
2839 VEC_free (tree
, heap
, vec_dsts
);
2840 VEC_free (tree
, heap
, tmp_vec_dsts
);
2841 VEC_free (tree
, heap
, interm_types
);
2842 VEC_free (tree
, heap
, vec_oprnds0
);
2843 VEC_free (tree
, heap
, vec_oprnds1
);
2845 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
2850 /* Function vectorizable_store.
2852 Check if STMT defines a non scalar data-ref (array/pointer/structure) that
2854 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
2855 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
2856 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
2859 vectorizable_store (gimple stmt
, gimple_stmt_iterator
*gsi
, gimple
*vec_stmt
,
2865 tree vec_oprnd
= NULL_TREE
;
2866 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
2867 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
= NULL
;
2868 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
2869 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
2870 struct loop
*loop
= NULL
;
2871 enum machine_mode vec_mode
;
2873 enum dr_alignment_support alignment_support_scheme
;
2876 enum vect_def_type dt
;
2877 stmt_vec_info prev_stmt_info
= NULL
;
2878 tree dataref_ptr
= NULL_TREE
;
2879 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
2882 gimple next_stmt
, first_stmt
= NULL
;
2883 bool strided_store
= false;
2884 unsigned int group_size
, i
;
2885 VEC(tree
,heap
) *dr_chain
= NULL
, *oprnds
= NULL
, *result_chain
= NULL
;
2887 VEC(tree
,heap
) *vec_oprnds
= NULL
;
2888 bool slp
= (slp_node
!= NULL
);
2889 stmt_vec_info first_stmt_vinfo
;
2890 unsigned int vec_num
;
2891 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
2894 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
2896 /* Multiple types in SLP are handled by creating the appropriate number of
2897 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
2902 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
2904 gcc_assert (ncopies
>= 1);
2906 /* FORNOW. This restriction should be relaxed. */
2907 if (loop
&& nested_in_vect_loop_p (loop
, stmt
) && ncopies
> 1)
2909 if (vect_print_dump_info (REPORT_DETAILS
))
2910 fprintf (vect_dump
, "multiple types in nested loop.");
2914 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
2917 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
2920 /* Is vectorizable store? */
2922 if (!is_gimple_assign (stmt
))
2925 scalar_dest
= gimple_assign_lhs (stmt
);
2926 if (TREE_CODE (scalar_dest
) != ARRAY_REF
2927 && TREE_CODE (scalar_dest
) != INDIRECT_REF
2928 && TREE_CODE (scalar_dest
) != COMPONENT_REF
2929 && TREE_CODE (scalar_dest
) != IMAGPART_EXPR
2930 && TREE_CODE (scalar_dest
) != REALPART_EXPR
)
2933 gcc_assert (gimple_assign_single_p (stmt
));
2934 op
= gimple_assign_rhs1 (stmt
);
2935 if (!vect_is_simple_use (op
, loop_vinfo
, bb_vinfo
, &def_stmt
, &def
, &dt
))
2937 if (vect_print_dump_info (REPORT_DETAILS
))
2938 fprintf (vect_dump
, "use not simple.");
2942 /* The scalar rhs type needs to be trivially convertible to the vector
2943 component type. This should always be the case. */
2944 if (!useless_type_conversion_p (TREE_TYPE (vectype
), TREE_TYPE (op
)))
2946 if (vect_print_dump_info (REPORT_DETAILS
))
2947 fprintf (vect_dump
, "??? operands of different types");
2951 vec_mode
= TYPE_MODE (vectype
);
2952 /* FORNOW. In some cases can vectorize even if data-type not supported
2953 (e.g. - array initialization with 0). */
2954 if (optab_handler (mov_optab
, (int)vec_mode
)->insn_code
== CODE_FOR_nothing
)
2957 if (!STMT_VINFO_DATA_REF (stmt_info
))
2960 if (STMT_VINFO_STRIDED_ACCESS (stmt_info
))
2962 strided_store
= true;
2963 first_stmt
= DR_GROUP_FIRST_DR (stmt_info
);
2964 if (!vect_strided_store_supported (vectype
)
2965 && !PURE_SLP_STMT (stmt_info
) && !slp
)
2968 if (first_stmt
== stmt
)
2970 /* STMT is the leader of the group. Check the operands of all the
2971 stmts of the group. */
2972 next_stmt
= DR_GROUP_NEXT_DR (stmt_info
);
2975 gcc_assert (gimple_assign_single_p (next_stmt
));
2976 op
= gimple_assign_rhs1 (next_stmt
);
2977 if (!vect_is_simple_use (op
, loop_vinfo
, bb_vinfo
, &def_stmt
,
2980 if (vect_print_dump_info (REPORT_DETAILS
))
2981 fprintf (vect_dump
, "use not simple.");
2984 next_stmt
= DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt
));
2989 if (!vec_stmt
) /* transformation not required. */
2991 STMT_VINFO_TYPE (stmt_info
) = store_vec_info_type
;
2992 vect_model_store_cost (stmt_info
, ncopies
, dt
, NULL
);
3000 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
3001 group_size
= DR_GROUP_SIZE (vinfo_for_stmt (first_stmt
));
3003 DR_GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt
))++;
3006 gcc_assert (!loop
|| !nested_in_vect_loop_p (loop
, stmt
));
3008 /* We vectorize all the stmts of the interleaving group when we
3009 reach the last stmt in the group. */
3010 if (DR_GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt
))
3011 < DR_GROUP_SIZE (vinfo_for_stmt (first_stmt
))
3019 strided_store
= false;
3021 /* VEC_NUM is the number of vect stmts to be created for this group. */
3023 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
3025 vec_num
= group_size
;
3031 group_size
= vec_num
= 1;
3032 first_stmt_vinfo
= stmt_info
;
3035 if (vect_print_dump_info (REPORT_DETAILS
))
3036 fprintf (vect_dump
, "transform store. ncopies = %d",ncopies
);
3038 dr_chain
= VEC_alloc (tree
, heap
, group_size
);
3039 oprnds
= VEC_alloc (tree
, heap
, group_size
);
3041 alignment_support_scheme
= vect_supportable_dr_alignment (first_dr
);
3042 gcc_assert (alignment_support_scheme
);
3044 /* In case the vectorization factor (VF) is bigger than the number
3045 of elements that we can fit in a vectype (nunits), we have to generate
3046 more than one vector stmt - i.e - we need to "unroll" the
3047 vector stmt by a factor VF/nunits. For more details see documentation in
3048 vect_get_vec_def_for_copy_stmt. */
3050 /* In case of interleaving (non-unit strided access):
3057 We create vectorized stores starting from base address (the access of the
3058 first stmt in the chain (S2 in the above example), when the last store stmt
3059 of the chain (S4) is reached:
3062 VS2: &base + vec_size*1 = vx0
3063 VS3: &base + vec_size*2 = vx1
3064 VS4: &base + vec_size*3 = vx3
3066 Then permutation statements are generated:
3068 VS5: vx5 = VEC_INTERLEAVE_HIGH_EXPR < vx0, vx3 >
3069 VS6: vx6 = VEC_INTERLEAVE_LOW_EXPR < vx0, vx3 >
3072 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
3073 (the order of the data-refs in the output of vect_permute_store_chain
3074 corresponds to the order of scalar stmts in the interleaving chain - see
3075 the documentation of vect_permute_store_chain()).
3077 In case of both multiple types and interleaving, above vector stores and
3078 permutation stmts are created for every copy. The result vector stmts are
3079 put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
3080 STMT_VINFO_RELATED_STMT for the next copies.
3083 prev_stmt_info
= NULL
;
3084 for (j
= 0; j
< ncopies
; j
++)
3093 /* Get vectorized arguments for SLP_NODE. */
3094 vect_get_slp_defs (slp_node
, &vec_oprnds
, NULL
);
3096 vec_oprnd
= VEC_index (tree
, vec_oprnds
, 0);
3100 /* For interleaved stores we collect vectorized defs for all the
3101 stores in the group in DR_CHAIN and OPRNDS. DR_CHAIN is then
3102 used as an input to vect_permute_store_chain(), and OPRNDS as
3103 an input to vect_get_vec_def_for_stmt_copy() for the next copy.
3105 If the store is not strided, GROUP_SIZE is 1, and DR_CHAIN and
3106 OPRNDS are of size 1. */
3107 next_stmt
= first_stmt
;
3108 for (i
= 0; i
< group_size
; i
++)
3110 /* Since gaps are not supported for interleaved stores,
3111 GROUP_SIZE is the exact number of stmts in the chain.
3112 Therefore, NEXT_STMT can't be NULL_TREE. In case that
3113 there is no interleaving, GROUP_SIZE is 1, and only one
3114 iteration of the loop will be executed. */
3115 gcc_assert (next_stmt
3116 && gimple_assign_single_p (next_stmt
));
3117 op
= gimple_assign_rhs1 (next_stmt
);
3119 vec_oprnd
= vect_get_vec_def_for_operand (op
, next_stmt
,
3121 VEC_quick_push(tree
, dr_chain
, vec_oprnd
);
3122 VEC_quick_push(tree
, oprnds
, vec_oprnd
);
3123 next_stmt
= DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt
));
3127 /* We should have catched mismatched types earlier. */
3128 gcc_assert (useless_type_conversion_p (vectype
,
3129 TREE_TYPE (vec_oprnd
)));
3130 dataref_ptr
= vect_create_data_ref_ptr (first_stmt
, NULL
, NULL_TREE
,
3131 &dummy
, &ptr_incr
, false,
3133 gcc_assert (bb_vinfo
|| !inv_p
);
3137 /* For interleaved stores we created vectorized defs for all the
3138 defs stored in OPRNDS in the previous iteration (previous copy).
3139 DR_CHAIN is then used as an input to vect_permute_store_chain(),
3140 and OPRNDS as an input to vect_get_vec_def_for_stmt_copy() for the
3142 If the store is not strided, GROUP_SIZE is 1, and DR_CHAIN and
3143 OPRNDS are of size 1. */
3144 for (i
= 0; i
< group_size
; i
++)
3146 op
= VEC_index (tree
, oprnds
, i
);
3147 vect_is_simple_use (op
, loop_vinfo
, bb_vinfo
, &def_stmt
, &def
,
3149 vec_oprnd
= vect_get_vec_def_for_stmt_copy (dt
, op
);
3150 VEC_replace(tree
, dr_chain
, i
, vec_oprnd
);
3151 VEC_replace(tree
, oprnds
, i
, vec_oprnd
);
3154 bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
, NULL_TREE
);
3159 result_chain
= VEC_alloc (tree
, heap
, group_size
);
3161 if (!vect_permute_store_chain (dr_chain
, group_size
, stmt
, gsi
,
3166 next_stmt
= first_stmt
;
3167 for (i
= 0; i
< vec_num
; i
++)
3170 /* Bump the vector pointer. */
3171 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
3175 vec_oprnd
= VEC_index (tree
, vec_oprnds
, i
);
3176 else if (strided_store
)
3177 /* For strided stores vectorized defs are interleaved in
3178 vect_permute_store_chain(). */
3179 vec_oprnd
= VEC_index (tree
, result_chain
, i
);
3181 if (aligned_access_p (first_dr
))
3182 data_ref
= build_fold_indirect_ref (dataref_ptr
);
3185 int mis
= DR_MISALIGNMENT (first_dr
);
3186 tree tmis
= (mis
== -1 ? size_zero_node
: size_int (mis
));
3187 tmis
= size_binop (MULT_EXPR
, tmis
, size_int (BITS_PER_UNIT
));
3188 data_ref
= build2 (MISALIGNED_INDIRECT_REF
, vectype
, dataref_ptr
, tmis
);
3191 /* If accesses through a pointer to vectype do not alias the original
3192 memory reference we have a problem. This should never happen. */
3193 gcc_assert (alias_sets_conflict_p (get_alias_set (data_ref
),
3194 get_alias_set (gimple_assign_lhs (stmt
))));
3196 /* Arguments are ready. Create the new vector stmt. */
3197 new_stmt
= gimple_build_assign (data_ref
, vec_oprnd
);
3198 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3199 mark_symbols_for_renaming (new_stmt
);
3205 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3207 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3209 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3210 next_stmt
= DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt
));
3216 VEC_free (tree
, heap
, dr_chain
);
3217 VEC_free (tree
, heap
, oprnds
);
3219 VEC_free (tree
, heap
, result_chain
);
3224 /* vectorizable_load.
3226 Check if STMT reads a non scalar data-ref (array/pointer/structure) that
3228 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3229 stmt to replace it, put it in VEC_STMT, and insert it at BSI.
3230 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3233 vectorizable_load (gimple stmt
, gimple_stmt_iterator
*gsi
, gimple
*vec_stmt
,
3234 slp_tree slp_node
, slp_instance slp_node_instance
)
3237 tree vec_dest
= NULL
;
3238 tree data_ref
= NULL
;
3239 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3240 stmt_vec_info prev_stmt_info
;
3241 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
3242 struct loop
*loop
= NULL
;
3243 struct loop
*containing_loop
= (gimple_bb (stmt
))->loop_father
;
3244 bool nested_in_vect_loop
= false;
3245 struct data_reference
*dr
= STMT_VINFO_DATA_REF (stmt_info
), *first_dr
;
3246 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
3249 gimple new_stmt
= NULL
;
3251 enum dr_alignment_support alignment_support_scheme
;
3252 tree dataref_ptr
= NULL_TREE
;
3254 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
3256 int i
, j
, group_size
;
3257 tree msq
= NULL_TREE
, lsq
;
3258 tree offset
= NULL_TREE
;
3259 tree realignment_token
= NULL_TREE
;
3261 VEC(tree
,heap
) *dr_chain
= NULL
;
3262 bool strided_load
= false;
3266 bool compute_in_loop
= false;
3267 struct loop
*at_loop
;
3269 bool slp
= (slp_node
!= NULL
);
3270 bool slp_perm
= false;
3271 enum tree_code code
;
3272 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
3277 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
3278 nested_in_vect_loop
= nested_in_vect_loop_p (loop
, stmt
);
3279 vf
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
);
3282 /* FORNOW: multiple types are not supported in basic block SLP. */
3285 /* Multiple types in SLP are handled by creating the appropriate number of
3286 vectorized stmts for each SLP node. Hence, NCOPIES is always 1 in
3291 ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
3293 gcc_assert (ncopies
>= 1);
3295 /* FORNOW. This restriction should be relaxed. */
3296 if (nested_in_vect_loop
&& ncopies
> 1)
3298 if (vect_print_dump_info (REPORT_DETAILS
))
3299 fprintf (vect_dump
, "multiple types in nested loop.");
3303 if (!STMT_VINFO_RELEVANT_P (stmt_info
) && !bb_vinfo
)
3306 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
)
3309 /* Is vectorizable load? */
3310 if (!is_gimple_assign (stmt
))
3313 scalar_dest
= gimple_assign_lhs (stmt
);
3314 if (TREE_CODE (scalar_dest
) != SSA_NAME
)
3317 code
= gimple_assign_rhs_code (stmt
);
3318 if (code
!= ARRAY_REF
3319 && code
!= INDIRECT_REF
3320 && code
!= COMPONENT_REF
3321 && code
!= IMAGPART_EXPR
3322 && code
!= REALPART_EXPR
)
3325 if (!STMT_VINFO_DATA_REF (stmt_info
))
3328 scalar_type
= TREE_TYPE (DR_REF (dr
));
3329 mode
= (int) TYPE_MODE (vectype
);
3331 /* FORNOW. In some cases can vectorize even if data-type not supported
3332 (e.g. - data copies). */
3333 if (optab_handler (mov_optab
, mode
)->insn_code
== CODE_FOR_nothing
)
3335 if (vect_print_dump_info (REPORT_DETAILS
))
3336 fprintf (vect_dump
, "Aligned load, but unsupported type.");
3340 /* The vector component type needs to be trivially convertible to the
3341 scalar lhs. This should always be the case. */
3342 if (!useless_type_conversion_p (TREE_TYPE (scalar_dest
), TREE_TYPE (vectype
)))
3344 if (vect_print_dump_info (REPORT_DETAILS
))
3345 fprintf (vect_dump
, "??? operands of different types");
3349 /* Check if the load is a part of an interleaving chain. */
3350 if (STMT_VINFO_STRIDED_ACCESS (stmt_info
))
3352 strided_load
= true;
3354 gcc_assert (! nested_in_vect_loop
);
3356 /* Check if interleaving is supported. */
3357 if (!vect_strided_load_supported (vectype
)
3358 && !PURE_SLP_STMT (stmt_info
) && !slp
)
3362 if (!vec_stmt
) /* transformation not required. */
3364 STMT_VINFO_TYPE (stmt_info
) = load_vec_info_type
;
3365 vect_model_load_cost (stmt_info
, ncopies
, NULL
);
3369 if (vect_print_dump_info (REPORT_DETAILS
))
3370 fprintf (vect_dump
, "transform load.");
3376 first_stmt
= DR_GROUP_FIRST_DR (stmt_info
);
3377 /* Check if the chain of loads is already vectorized. */
3378 if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt
)))
3380 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
3383 first_dr
= STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt
));
3384 group_size
= DR_GROUP_SIZE (vinfo_for_stmt (first_stmt
));
3386 /* VEC_NUM is the number of vect stmts to be created for this group. */
3389 strided_load
= false;
3390 vec_num
= SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node
);
3391 if (SLP_INSTANCE_LOAD_PERMUTATION (slp_node_instance
))
3395 vec_num
= group_size
;
3397 dr_chain
= VEC_alloc (tree
, heap
, vec_num
);
3403 group_size
= vec_num
= 1;
3406 alignment_support_scheme
= vect_supportable_dr_alignment (first_dr
);
3407 gcc_assert (alignment_support_scheme
);
3409 /* In case the vectorization factor (VF) is bigger than the number
3410 of elements that we can fit in a vectype (nunits), we have to generate
3411 more than one vector stmt - i.e - we need to "unroll" the
3412 vector stmt by a factor VF/nunits. In doing so, we record a pointer
3413 from one copy of the vector stmt to the next, in the field
3414 STMT_VINFO_RELATED_STMT. This is necessary in order to allow following
3415 stages to find the correct vector defs to be used when vectorizing
3416 stmts that use the defs of the current stmt. The example below illustrates
3417 the vectorization process when VF=16 and nunits=4 (i.e - we need to create
3418 4 vectorized stmts):
3420 before vectorization:
3421 RELATED_STMT VEC_STMT
3425 step 1: vectorize stmt S1:
3426 We first create the vector stmt VS1_0, and, as usual, record a
3427 pointer to it in the STMT_VINFO_VEC_STMT of the scalar stmt S1.
3428 Next, we create the vector stmt VS1_1, and record a pointer to
3429 it in the STMT_VINFO_RELATED_STMT of the vector stmt VS1_0.
3430 Similarly, for VS1_2 and VS1_3. This is the resulting chain of
3432 RELATED_STMT VEC_STMT
3433 VS1_0: vx0 = memref0 VS1_1 -
3434 VS1_1: vx1 = memref1 VS1_2 -
3435 VS1_2: vx2 = memref2 VS1_3 -
3436 VS1_3: vx3 = memref3 - -
3437 S1: x = load - VS1_0
3440 See in documentation in vect_get_vec_def_for_stmt_copy for how the
3441 information we recorded in RELATED_STMT field is used to vectorize
3444 /* In case of interleaving (non-unit strided access):
3451 Vectorized loads are created in the order of memory accesses
3452 starting from the access of the first stmt of the chain:
3455 VS2: vx1 = &base + vec_size*1
3456 VS3: vx3 = &base + vec_size*2
3457 VS4: vx4 = &base + vec_size*3
3459 Then permutation statements are generated:
3461 VS5: vx5 = VEC_EXTRACT_EVEN_EXPR < vx0, vx1 >
3462 VS6: vx6 = VEC_EXTRACT_ODD_EXPR < vx0, vx1 >
3465 And they are put in STMT_VINFO_VEC_STMT of the corresponding scalar stmts
3466 (the order of the data-refs in the output of vect_permute_load_chain
3467 corresponds to the order of scalar stmts in the interleaving chain - see
3468 the documentation of vect_permute_load_chain()).
3469 The generation of permutation stmts and recording them in
3470 STMT_VINFO_VEC_STMT is done in vect_transform_strided_load().
3472 In case of both multiple types and interleaving, the vector loads and
3473 permutation stmts above are created for every copy. The result vector stmts
3474 are put in STMT_VINFO_VEC_STMT for the first copy and in the corresponding
3475 STMT_VINFO_RELATED_STMT for the next copies. */
3477 /* If the data reference is aligned (dr_aligned) or potentially unaligned
3478 on a target that supports unaligned accesses (dr_unaligned_supported)
3479 we generate the following code:
3483 p = p + indx * vectype_size;
3488 Otherwise, the data reference is potentially unaligned on a target that
3489 does not support unaligned accesses (dr_explicit_realign_optimized) -
3490 then generate the following code, in which the data in each iteration is
3491 obtained by two vector loads, one from the previous iteration, and one
3492 from the current iteration:
3494 msq_init = *(floor(p1))
3495 p2 = initial_addr + VS - 1;
3496 realignment_token = call target_builtin;
3499 p2 = p2 + indx * vectype_size
3501 vec_dest = realign_load (msq, lsq, realignment_token)
3506 /* If the misalignment remains the same throughout the execution of the
3507 loop, we can create the init_addr and permutation mask at the loop
3508 preheader. Otherwise, it needs to be created inside the loop.
3509 This can only occur when vectorizing memory accesses in the inner-loop
3510 nested within an outer-loop that is being vectorized. */
3512 if (loop
&& nested_in_vect_loop_p (loop
, stmt
)
3513 && (TREE_INT_CST_LOW (DR_STEP (dr
))
3514 % GET_MODE_SIZE (TYPE_MODE (vectype
)) != 0))
3516 gcc_assert (alignment_support_scheme
!= dr_explicit_realign_optimized
);
3517 compute_in_loop
= true;
3520 if ((alignment_support_scheme
== dr_explicit_realign_optimized
3521 || alignment_support_scheme
== dr_explicit_realign
)
3522 && !compute_in_loop
)
3524 msq
= vect_setup_realignment (first_stmt
, gsi
, &realignment_token
,
3525 alignment_support_scheme
, NULL_TREE
,
3527 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
3529 phi
= SSA_NAME_DEF_STMT (msq
);
3530 offset
= size_int (TYPE_VECTOR_SUBPARTS (vectype
) - 1);
3536 prev_stmt_info
= NULL
;
3537 for (j
= 0; j
< ncopies
; j
++)
3539 /* 1. Create the vector pointer update chain. */
3541 dataref_ptr
= vect_create_data_ref_ptr (first_stmt
,
3543 &dummy
, &ptr_incr
, false,
3547 bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
, NULL_TREE
);
3549 for (i
= 0; i
< vec_num
; i
++)
3552 dataref_ptr
= bump_vector_ptr (dataref_ptr
, ptr_incr
, gsi
, stmt
,
3555 /* 2. Create the vector-load in the loop. */
3556 switch (alignment_support_scheme
)
3559 gcc_assert (aligned_access_p (first_dr
));
3560 data_ref
= build_fold_indirect_ref (dataref_ptr
);
3562 case dr_unaligned_supported
:
3564 int mis
= DR_MISALIGNMENT (first_dr
);
3565 tree tmis
= (mis
== -1 ? size_zero_node
: size_int (mis
));
3567 tmis
= size_binop (MULT_EXPR
, tmis
, size_int(BITS_PER_UNIT
));
3569 build2 (MISALIGNED_INDIRECT_REF
, vectype
, dataref_ptr
, tmis
);
3572 case dr_explicit_realign
:
3575 tree vs_minus_1
= size_int (TYPE_VECTOR_SUBPARTS (vectype
) - 1);
3577 if (compute_in_loop
)
3578 msq
= vect_setup_realignment (first_stmt
, gsi
,
3580 dr_explicit_realign
,
3583 data_ref
= build1 (ALIGN_INDIRECT_REF
, vectype
, dataref_ptr
);
3584 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
3585 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
3586 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3587 gimple_assign_set_lhs (new_stmt
, new_temp
);
3588 gimple_set_vdef (new_stmt
, gimple_vdef (stmt
));
3589 gimple_set_vuse (new_stmt
, gimple_vuse (stmt
));
3590 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3593 bump
= size_binop (MULT_EXPR
, vs_minus_1
,
3594 TYPE_SIZE_UNIT (scalar_type
));
3595 ptr
= bump_vector_ptr (dataref_ptr
, NULL
, gsi
, stmt
, bump
);
3596 data_ref
= build1 (ALIGN_INDIRECT_REF
, vectype
, ptr
);
3599 case dr_explicit_realign_optimized
:
3600 data_ref
= build1 (ALIGN_INDIRECT_REF
, vectype
, dataref_ptr
);
3605 /* If accesses through a pointer to vectype do not alias the original
3606 memory reference we have a problem. This should never happen. */
3607 gcc_assert (alias_sets_conflict_p (get_alias_set (data_ref
),
3608 get_alias_set (gimple_assign_rhs1 (stmt
))));
3609 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
3610 new_stmt
= gimple_build_assign (vec_dest
, data_ref
);
3611 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3612 gimple_assign_set_lhs (new_stmt
, new_temp
);
3613 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3614 mark_symbols_for_renaming (new_stmt
);
3616 /* 3. Handle explicit realignment if necessary/supported. Create in
3617 loop: vec_dest = realign_load (msq, lsq, realignment_token) */
3618 if (alignment_support_scheme
== dr_explicit_realign_optimized
3619 || alignment_support_scheme
== dr_explicit_realign
)
3623 lsq
= gimple_assign_lhs (new_stmt
);
3624 if (!realignment_token
)
3625 realignment_token
= dataref_ptr
;
3626 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
3627 tmp
= build3 (REALIGN_LOAD_EXPR
, vectype
, msq
, lsq
,
3629 new_stmt
= gimple_build_assign (vec_dest
, tmp
);
3630 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3631 gimple_assign_set_lhs (new_stmt
, new_temp
);
3632 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3634 if (alignment_support_scheme
== dr_explicit_realign_optimized
)
3637 if (i
== vec_num
- 1 && j
== ncopies
- 1)
3638 add_phi_arg (phi
, lsq
, loop_latch_edge (containing_loop
));
3643 /* 4. Handle invariant-load. */
3644 if (inv_p
&& !bb_vinfo
)
3646 gcc_assert (!strided_load
);
3647 gcc_assert (nested_in_vect_loop_p (loop
, stmt
));
3652 tree vec_inv
, bitpos
, bitsize
= TYPE_SIZE (scalar_type
);
3654 /* CHECKME: bitpos depends on endianess? */
3655 bitpos
= bitsize_zero_node
;
3656 vec_inv
= build3 (BIT_FIELD_REF
, scalar_type
, new_temp
,
3659 vect_create_destination_var (scalar_dest
, NULL_TREE
);
3660 new_stmt
= gimple_build_assign (vec_dest
, vec_inv
);
3661 new_temp
= make_ssa_name (vec_dest
, new_stmt
);
3662 gimple_assign_set_lhs (new_stmt
, new_temp
);
3663 vect_finish_stmt_generation (stmt
, new_stmt
, gsi
);
3665 for (k
= nunits
- 1; k
>= 0; --k
)
3666 t
= tree_cons (NULL_TREE
, new_temp
, t
);
3667 /* FIXME: use build_constructor directly. */
3668 vec_inv
= build_constructor_from_list (vectype
, t
);
3669 new_temp
= vect_init_vector (stmt
, vec_inv
, vectype
, gsi
);
3670 new_stmt
= SSA_NAME_DEF_STMT (new_temp
);
3673 gcc_unreachable (); /* FORNOW. */
3676 /* Collect vector loads and later create their permutation in
3677 vect_transform_strided_load (). */
3678 if (strided_load
|| slp_perm
)
3679 VEC_quick_push (tree
, dr_chain
, new_temp
);
3681 /* Store vector loads in the corresponding SLP_NODE. */
3682 if (slp
&& !slp_perm
)
3683 VEC_quick_push (gimple
, SLP_TREE_VEC_STMTS (slp_node
), new_stmt
);
3686 if (slp
&& !slp_perm
)
3691 if (!vect_transform_slp_perm_load (stmt
, dr_chain
, gsi
, vf
,
3692 slp_node_instance
, false))
3694 VEC_free (tree
, heap
, dr_chain
);
3702 if (!vect_transform_strided_load (stmt
, dr_chain
, group_size
, gsi
))
3705 *vec_stmt
= STMT_VINFO_VEC_STMT (stmt_info
);
3706 VEC_free (tree
, heap
, dr_chain
);
3707 dr_chain
= VEC_alloc (tree
, heap
, group_size
);
3712 STMT_VINFO_VEC_STMT (stmt_info
) = *vec_stmt
= new_stmt
;
3714 STMT_VINFO_RELATED_STMT (prev_stmt_info
) = new_stmt
;
3715 prev_stmt_info
= vinfo_for_stmt (new_stmt
);
3721 VEC_free (tree
, heap
, dr_chain
);
3726 /* Function vect_is_simple_cond.
3729 LOOP - the loop that is being vectorized.
3730 COND - Condition that is checked for simple use.
3732 Returns whether a COND can be vectorized. Checks whether
3733 condition operands are supportable using vec_is_simple_use. */
3736 vect_is_simple_cond (tree cond
, loop_vec_info loop_vinfo
)
3740 enum vect_def_type dt
;
3742 if (!COMPARISON_CLASS_P (cond
))
3745 lhs
= TREE_OPERAND (cond
, 0);
3746 rhs
= TREE_OPERAND (cond
, 1);
3748 if (TREE_CODE (lhs
) == SSA_NAME
)
3750 gimple lhs_def_stmt
= SSA_NAME_DEF_STMT (lhs
);
3751 if (!vect_is_simple_use (lhs
, loop_vinfo
, NULL
, &lhs_def_stmt
, &def
,
3755 else if (TREE_CODE (lhs
) != INTEGER_CST
&& TREE_CODE (lhs
) != REAL_CST
3756 && TREE_CODE (lhs
) != FIXED_CST
)
3759 if (TREE_CODE (rhs
) == SSA_NAME
)
3761 gimple rhs_def_stmt
= SSA_NAME_DEF_STMT (rhs
);
3762 if (!vect_is_simple_use (rhs
, loop_vinfo
, NULL
, &rhs_def_stmt
, &def
,
3766 else if (TREE_CODE (rhs
) != INTEGER_CST
&& TREE_CODE (rhs
) != REAL_CST
3767 && TREE_CODE (rhs
) != FIXED_CST
)
3773 /* vectorizable_condition.
3775 Check if STMT is conditional modify expression that can be vectorized.
3776 If VEC_STMT is also passed, vectorize the STMT: create a vectorized
3777 stmt using VEC_COND_EXPR to replace it, put it in VEC_STMT, and insert it
3780 When STMT is vectorized as nested cycle, REDUC_DEF is the vector variable
3781 to be used at REDUC_INDEX (in then clause if REDUC_INDEX is 1, and in
3782 else caluse if it is 2).
3784 Return FALSE if not a vectorizable STMT, TRUE otherwise. */
3787 vectorizable_condition (gimple stmt
, gimple_stmt_iterator
*gsi
,
3788 gimple
*vec_stmt
, tree reduc_def
, int reduc_index
)
3790 tree scalar_dest
= NULL_TREE
;
3791 tree vec_dest
= NULL_TREE
;
3792 tree op
= NULL_TREE
;
3793 tree cond_expr
, then_clause
, else_clause
;
3794 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3795 tree vectype
= STMT_VINFO_VECTYPE (stmt_info
);
3796 tree vec_cond_lhs
, vec_cond_rhs
, vec_then_clause
, vec_else_clause
;
3797 tree vec_compare
, vec_cond_expr
;
3799 loop_vec_info loop_vinfo
= STMT_VINFO_LOOP_VINFO (stmt_info
);
3800 enum machine_mode vec_mode
;
3802 enum vect_def_type dt
;
3803 int nunits
= TYPE_VECTOR_SUBPARTS (vectype
);
3804 int ncopies
= LOOP_VINFO_VECT_FACTOR (loop_vinfo
) / nunits
;
3805 enum tree_code code
;
3807 /* FORNOW: unsupported in basic block SLP. */
3808 gcc_assert (loop_vinfo
);
3810 gcc_assert (ncopies
>= 1);
3812 return false; /* FORNOW */
3814 if (!STMT_VINFO_RELEVANT_P (stmt_info
))
3817 if (STMT_VINFO_DEF_TYPE (stmt_info
) != vect_internal_def
3818 && !(STMT_VINFO_DEF_TYPE (stmt_info
) == vect_nested_cycle
3822 /* FORNOW: SLP not supported. */
3823 if (STMT_SLP_TYPE (stmt_info
))
3826 /* FORNOW: not yet supported. */
3827 if (STMT_VINFO_LIVE_P (stmt_info
))
3829 if (vect_print_dump_info (REPORT_DETAILS
))
3830 fprintf (vect_dump
, "value used after loop.");
3834 /* Is vectorizable conditional operation? */
3835 if (!is_gimple_assign (stmt
))
3838 code
= gimple_assign_rhs_code (stmt
);
3840 if (code
!= COND_EXPR
)
3843 gcc_assert (gimple_assign_single_p (stmt
));
3844 op
= gimple_assign_rhs1 (stmt
);
3845 cond_expr
= TREE_OPERAND (op
, 0);
3846 then_clause
= TREE_OPERAND (op
, 1);
3847 else_clause
= TREE_OPERAND (op
, 2);
3849 if (!vect_is_simple_cond (cond_expr
, loop_vinfo
))
3852 /* We do not handle two different vector types for the condition
3854 if (TREE_TYPE (TREE_OPERAND (cond_expr
, 0)) != TREE_TYPE (vectype
))
3857 if (TREE_CODE (then_clause
) == SSA_NAME
)
3859 gimple then_def_stmt
= SSA_NAME_DEF_STMT (then_clause
);
3860 if (!vect_is_simple_use (then_clause
, loop_vinfo
, NULL
,
3861 &then_def_stmt
, &def
, &dt
))
3864 else if (TREE_CODE (then_clause
) != INTEGER_CST
3865 && TREE_CODE (then_clause
) != REAL_CST
3866 && TREE_CODE (then_clause
) != FIXED_CST
)
3869 if (TREE_CODE (else_clause
) == SSA_NAME
)
3871 gimple else_def_stmt
= SSA_NAME_DEF_STMT (else_clause
);
3872 if (!vect_is_simple_use (else_clause
, loop_vinfo
, NULL
,
3873 &else_def_stmt
, &def
, &dt
))
3876 else if (TREE_CODE (else_clause
) != INTEGER_CST
3877 && TREE_CODE (else_clause
) != REAL_CST
3878 && TREE_CODE (else_clause
) != FIXED_CST
)
3882 vec_mode
= TYPE_MODE (vectype
);
3886 STMT_VINFO_TYPE (stmt_info
) = condition_vec_info_type
;
3887 return expand_vec_cond_expr_p (op
, vec_mode
);
3893 scalar_dest
= gimple_assign_lhs (stmt
);
3894 vec_dest
= vect_create_destination_var (scalar_dest
, vectype
);
3896 /* Handle cond expr. */
3898 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr
, 0), stmt
, NULL
);
3900 vect_get_vec_def_for_operand (TREE_OPERAND (cond_expr
, 1), stmt
, NULL
);
3901 if (reduc_index
== 1)
3902 vec_then_clause
= reduc_def
;
3904 vec_then_clause
= vect_get_vec_def_for_operand (then_clause
, stmt
, NULL
);
3905 if (reduc_index
== 2)
3906 vec_else_clause
= reduc_def
;
3908 vec_else_clause
= vect_get_vec_def_for_operand (else_clause
, stmt
, NULL
);
3910 /* Arguments are ready. Create the new vector stmt. */
3911 vec_compare
= build2 (TREE_CODE (cond_expr
), vectype
,
3912 vec_cond_lhs
, vec_cond_rhs
);
3913 vec_cond_expr
= build3 (VEC_COND_EXPR
, vectype
,
3914 vec_compare
, vec_then_clause
, vec_else_clause
);
3916 *vec_stmt
= gimple_build_assign (vec_dest
, vec_cond_expr
);
3917 new_temp
= make_ssa_name (vec_dest
, *vec_stmt
);
3918 gimple_assign_set_lhs (*vec_stmt
, new_temp
);
3919 vect_finish_stmt_generation (stmt
, *vec_stmt
, gsi
);
3925 /* Make sure the statement is vectorizable. */
3928 vect_analyze_stmt (gimple stmt
, bool *need_to_vectorize
, slp_tree node
)
3930 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
3931 bb_vec_info bb_vinfo
= STMT_VINFO_BB_VINFO (stmt_info
);
3932 enum vect_relevant relevance
= STMT_VINFO_RELEVANT (stmt_info
);
3934 HOST_WIDE_INT dummy
;
3935 tree scalar_type
, vectype
;
3937 if (vect_print_dump_info (REPORT_DETAILS
))
3939 fprintf (vect_dump
, "==> examining statement: ");
3940 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
3943 if (gimple_has_volatile_ops (stmt
))
3945 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS
))
3946 fprintf (vect_dump
, "not vectorized: stmt has volatile operands");
3951 /* Skip stmts that do not need to be vectorized. In loops this is expected
3953 - the COND_EXPR which is the loop exit condition
3954 - any LABEL_EXPRs in the loop
3955 - computations that are used only for array indexing or loop control.
3956 In basic blocks we only analyze statements that are a part of some SLP
3957 instance, therefore, all the statements are relevant. */
3959 if (!STMT_VINFO_RELEVANT_P (stmt_info
)
3960 && !STMT_VINFO_LIVE_P (stmt_info
))
3962 if (vect_print_dump_info (REPORT_DETAILS
))
3963 fprintf (vect_dump
, "irrelevant.");
3968 switch (STMT_VINFO_DEF_TYPE (stmt_info
))
3970 case vect_internal_def
:
3973 case vect_reduction_def
:
3974 case vect_nested_cycle
:
3975 gcc_assert (!bb_vinfo
&& (relevance
== vect_used_in_outer
3976 || relevance
== vect_used_in_outer_by_reduction
3977 || relevance
== vect_unused_in_scope
));
3980 case vect_induction_def
:
3981 case vect_constant_def
:
3982 case vect_external_def
:
3983 case vect_unknown_def_type
:
3990 gcc_assert (PURE_SLP_STMT (stmt_info
));
3992 scalar_type
= vect_get_smallest_scalar_type (stmt
, &dummy
, &dummy
);
3993 if (vect_print_dump_info (REPORT_DETAILS
))
3995 fprintf (vect_dump
, "get vectype for scalar type: ");
3996 print_generic_expr (vect_dump
, scalar_type
, TDF_SLIM
);
3999 vectype
= get_vectype_for_scalar_type (scalar_type
);
4002 if (vect_print_dump_info (REPORT_DETAILS
))
4004 fprintf (vect_dump
, "not SLPed: unsupported data-type ");
4005 print_generic_expr (vect_dump
, scalar_type
, TDF_SLIM
);
4010 if (vect_print_dump_info (REPORT_DETAILS
))
4012 fprintf (vect_dump
, "vectype: ");
4013 print_generic_expr (vect_dump
, vectype
, TDF_SLIM
);
4016 STMT_VINFO_VECTYPE (stmt_info
) = vectype
;
4019 if (STMT_VINFO_RELEVANT_P (stmt_info
))
4021 gcc_assert (!VECTOR_MODE_P (TYPE_MODE (gimple_expr_type (stmt
))));
4022 gcc_assert (STMT_VINFO_VECTYPE (stmt_info
));
4023 *need_to_vectorize
= true;
4028 && (STMT_VINFO_RELEVANT_P (stmt_info
)
4029 || STMT_VINFO_DEF_TYPE (stmt_info
) == vect_reduction_def
))
4030 ok
= (vectorizable_type_promotion (stmt
, NULL
, NULL
, NULL
)
4031 || vectorizable_type_demotion (stmt
, NULL
, NULL
, NULL
)
4032 || vectorizable_conversion (stmt
, NULL
, NULL
, NULL
)
4033 || vectorizable_operation (stmt
, NULL
, NULL
, NULL
)
4034 || vectorizable_assignment (stmt
, NULL
, NULL
, NULL
)
4035 || vectorizable_load (stmt
, NULL
, NULL
, NULL
, NULL
)
4036 || vectorizable_call (stmt
, NULL
, NULL
)
4037 || vectorizable_store (stmt
, NULL
, NULL
, NULL
)
4038 || vectorizable_reduction (stmt
, NULL
, NULL
)
4039 || vectorizable_condition (stmt
, NULL
, NULL
, NULL
, 0));
4043 ok
= (vectorizable_operation (stmt
, NULL
, NULL
, node
)
4044 || vectorizable_assignment (stmt
, NULL
, NULL
, node
)
4045 || vectorizable_load (stmt
, NULL
, NULL
, node
, NULL
)
4046 || vectorizable_store (stmt
, NULL
, NULL
, node
));
4051 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS
))
4053 fprintf (vect_dump
, "not vectorized: relevant stmt not ");
4054 fprintf (vect_dump
, "supported: ");
4055 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
4064 /* Stmts that are (also) "live" (i.e. - that are used out of the loop)
4065 need extra handling, except for vectorizable reductions. */
4066 if (STMT_VINFO_LIVE_P (stmt_info
)
4067 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
)
4068 ok
= vectorizable_live_operation (stmt
, NULL
, NULL
);
4072 if (vect_print_dump_info (REPORT_UNVECTORIZED_LOCATIONS
))
4074 fprintf (vect_dump
, "not vectorized: live stmt not ");
4075 fprintf (vect_dump
, "supported: ");
4076 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
4082 if (!PURE_SLP_STMT (stmt_info
))
4084 /* Groups of strided accesses whose size is not a power of 2 are not
4085 vectorizable yet using loop-vectorization. Therefore, if this stmt
4086 feeds non-SLP-able stmts (i.e., this stmt has to be both SLPed and
4087 loop-based vectorized), the loop cannot be vectorized. */
4088 if (STMT_VINFO_STRIDED_ACCESS (stmt_info
)
4089 && exact_log2 (DR_GROUP_SIZE (vinfo_for_stmt (
4090 DR_GROUP_FIRST_DR (stmt_info
)))) == -1)
4092 if (vect_print_dump_info (REPORT_DETAILS
))
4094 fprintf (vect_dump
, "not vectorized: the size of group "
4095 "of strided accesses is not a power of 2");
4096 print_gimple_stmt (vect_dump
, stmt
, 0, TDF_SLIM
);
4107 /* Function vect_transform_stmt.
4109 Create a vectorized stmt to replace STMT, and insert it at BSI. */
4112 vect_transform_stmt (gimple stmt
, gimple_stmt_iterator
*gsi
,
4113 bool *strided_store
, slp_tree slp_node
,
4114 slp_instance slp_node_instance
)
4116 bool is_store
= false;
4117 gimple vec_stmt
= NULL
;
4118 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4119 gimple orig_stmt_in_pattern
;
4122 switch (STMT_VINFO_TYPE (stmt_info
))
4124 case type_demotion_vec_info_type
:
4125 done
= vectorizable_type_demotion (stmt
, gsi
, &vec_stmt
, slp_node
);
4129 case type_promotion_vec_info_type
:
4130 done
= vectorizable_type_promotion (stmt
, gsi
, &vec_stmt
, slp_node
);
4134 case type_conversion_vec_info_type
:
4135 done
= vectorizable_conversion (stmt
, gsi
, &vec_stmt
, slp_node
);
4139 case induc_vec_info_type
:
4140 gcc_assert (!slp_node
);
4141 done
= vectorizable_induction (stmt
, gsi
, &vec_stmt
);
4145 case op_vec_info_type
:
4146 done
= vectorizable_operation (stmt
, gsi
, &vec_stmt
, slp_node
);
4150 case assignment_vec_info_type
:
4151 done
= vectorizable_assignment (stmt
, gsi
, &vec_stmt
, slp_node
);
4155 case load_vec_info_type
:
4156 done
= vectorizable_load (stmt
, gsi
, &vec_stmt
, slp_node
,
4161 case store_vec_info_type
:
4162 done
= vectorizable_store (stmt
, gsi
, &vec_stmt
, slp_node
);
4164 if (STMT_VINFO_STRIDED_ACCESS (stmt_info
) && !slp_node
)
4166 /* In case of interleaving, the whole chain is vectorized when the
4167 last store in the chain is reached. Store stmts before the last
4168 one are skipped, and there vec_stmt_info shouldn't be freed
4170 *strided_store
= true;
4171 if (STMT_VINFO_VEC_STMT (stmt_info
))
4178 case condition_vec_info_type
:
4179 gcc_assert (!slp_node
);
4180 done
= vectorizable_condition (stmt
, gsi
, &vec_stmt
, NULL
, 0);
4184 case call_vec_info_type
:
4185 gcc_assert (!slp_node
);
4186 done
= vectorizable_call (stmt
, gsi
, &vec_stmt
);
4189 case reduc_vec_info_type
:
4190 gcc_assert (!slp_node
);
4191 done
= vectorizable_reduction (stmt
, gsi
, &vec_stmt
);
4196 if (!STMT_VINFO_LIVE_P (stmt_info
))
4198 if (vect_print_dump_info (REPORT_DETAILS
))
4199 fprintf (vect_dump
, "stmt not supported.");
4204 /* Handle inner-loop stmts whose DEF is used in the loop-nest that
4205 is being vectorized, but outside the immediately enclosing loop. */
4207 && STMT_VINFO_LOOP_VINFO (stmt_info
)
4208 && nested_in_vect_loop_p (LOOP_VINFO_LOOP (
4209 STMT_VINFO_LOOP_VINFO (stmt_info
)), stmt
)
4210 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
4211 && (STMT_VINFO_RELEVANT (stmt_info
) == vect_used_in_outer
4212 || STMT_VINFO_RELEVANT (stmt_info
) ==
4213 vect_used_in_outer_by_reduction
))
4215 struct loop
*innerloop
= LOOP_VINFO_LOOP (
4216 STMT_VINFO_LOOP_VINFO (stmt_info
))->inner
;
4217 imm_use_iterator imm_iter
;
4218 use_operand_p use_p
;
4222 if (vect_print_dump_info (REPORT_DETAILS
))
4223 fprintf (vect_dump
, "Record the vdef for outer-loop vectorization.");
4225 /* Find the relevant loop-exit phi-node, and reord the vec_stmt there
4226 (to be used when vectorizing outer-loop stmts that use the DEF of
4228 if (gimple_code (stmt
) == GIMPLE_PHI
)
4229 scalar_dest
= PHI_RESULT (stmt
);
4231 scalar_dest
= gimple_assign_lhs (stmt
);
4233 FOR_EACH_IMM_USE_FAST (use_p
, imm_iter
, scalar_dest
)
4235 if (!flow_bb_inside_loop_p (innerloop
, gimple_bb (USE_STMT (use_p
))))
4237 exit_phi
= USE_STMT (use_p
);
4238 STMT_VINFO_VEC_STMT (vinfo_for_stmt (exit_phi
)) = vec_stmt
;
4243 /* Handle stmts whose DEF is used outside the loop-nest that is
4244 being vectorized. */
4245 if (STMT_VINFO_LIVE_P (stmt_info
)
4246 && STMT_VINFO_TYPE (stmt_info
) != reduc_vec_info_type
)
4248 done
= vectorizable_live_operation (stmt
, gsi
, &vec_stmt
);
4254 STMT_VINFO_VEC_STMT (stmt_info
) = vec_stmt
;
4255 orig_stmt_in_pattern
= STMT_VINFO_RELATED_STMT (stmt_info
);
4256 if (orig_stmt_in_pattern
)
4258 stmt_vec_info stmt_vinfo
= vinfo_for_stmt (orig_stmt_in_pattern
);
4259 /* STMT was inserted by the vectorizer to replace a computation idiom.
4260 ORIG_STMT_IN_PATTERN is a stmt in the original sequence that
4261 computed this idiom. We need to record a pointer to VEC_STMT in
4262 the stmt_info of ORIG_STMT_IN_PATTERN. See more details in the
4263 documentation of vect_pattern_recog. */
4264 if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo
))
4266 gcc_assert (STMT_VINFO_RELATED_STMT (stmt_vinfo
) == stmt
);
4267 STMT_VINFO_VEC_STMT (stmt_vinfo
) = vec_stmt
;
4276 /* Remove a group of stores (for SLP or interleaving), free their
4280 vect_remove_stores (gimple first_stmt
)
4282 gimple next
= first_stmt
;
4284 gimple_stmt_iterator next_si
;
4288 /* Free the attached stmt_vec_info and remove the stmt. */
4289 next_si
= gsi_for_stmt (next
);
4290 gsi_remove (&next_si
, true);
4291 tmp
= DR_GROUP_NEXT_DR (vinfo_for_stmt (next
));
4292 free_stmt_vec_info (next
);
4298 /* Function new_stmt_vec_info.
4300 Create and initialize a new stmt_vec_info struct for STMT. */
4303 new_stmt_vec_info (gimple stmt
, loop_vec_info loop_vinfo
,
4304 bb_vec_info bb_vinfo
)
4307 res
= (stmt_vec_info
) xcalloc (1, sizeof (struct _stmt_vec_info
));
4309 STMT_VINFO_TYPE (res
) = undef_vec_info_type
;
4310 STMT_VINFO_STMT (res
) = stmt
;
4311 STMT_VINFO_LOOP_VINFO (res
) = loop_vinfo
;
4312 STMT_VINFO_BB_VINFO (res
) = bb_vinfo
;
4313 STMT_VINFO_RELEVANT (res
) = vect_unused_in_scope
;
4314 STMT_VINFO_LIVE_P (res
) = false;
4315 STMT_VINFO_VECTYPE (res
) = NULL
;
4316 STMT_VINFO_VEC_STMT (res
) = NULL
;
4317 STMT_VINFO_IN_PATTERN_P (res
) = false;
4318 STMT_VINFO_RELATED_STMT (res
) = NULL
;
4319 STMT_VINFO_DATA_REF (res
) = NULL
;
4321 STMT_VINFO_DR_BASE_ADDRESS (res
) = NULL
;
4322 STMT_VINFO_DR_OFFSET (res
) = NULL
;
4323 STMT_VINFO_DR_INIT (res
) = NULL
;
4324 STMT_VINFO_DR_STEP (res
) = NULL
;
4325 STMT_VINFO_DR_ALIGNED_TO (res
) = NULL
;
4327 if (gimple_code (stmt
) == GIMPLE_PHI
4328 && is_loop_header_bb_p (gimple_bb (stmt
)))
4329 STMT_VINFO_DEF_TYPE (res
) = vect_unknown_def_type
;
4331 STMT_VINFO_DEF_TYPE (res
) = vect_internal_def
;
4333 STMT_VINFO_SAME_ALIGN_REFS (res
) = VEC_alloc (dr_p
, heap
, 5);
4334 STMT_VINFO_INSIDE_OF_LOOP_COST (res
) = 0;
4335 STMT_VINFO_OUTSIDE_OF_LOOP_COST (res
) = 0;
4336 STMT_SLP_TYPE (res
) = loop_vect
;
4337 DR_GROUP_FIRST_DR (res
) = NULL
;
4338 DR_GROUP_NEXT_DR (res
) = NULL
;
4339 DR_GROUP_SIZE (res
) = 0;
4340 DR_GROUP_STORE_COUNT (res
) = 0;
4341 DR_GROUP_GAP (res
) = 0;
4342 DR_GROUP_SAME_DR_STMT (res
) = NULL
;
4343 DR_GROUP_READ_WRITE_DEPENDENCE (res
) = false;
4349 /* Create a hash table for stmt_vec_info. */
4352 init_stmt_vec_info_vec (void)
4354 gcc_assert (!stmt_vec_info_vec
);
4355 stmt_vec_info_vec
= VEC_alloc (vec_void_p
, heap
, 50);
4359 /* Free hash table for stmt_vec_info. */
4362 free_stmt_vec_info_vec (void)
4364 gcc_assert (stmt_vec_info_vec
);
4365 VEC_free (vec_void_p
, heap
, stmt_vec_info_vec
);
4369 /* Free stmt vectorization related info. */
4372 free_stmt_vec_info (gimple stmt
)
4374 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4379 VEC_free (dr_p
, heap
, STMT_VINFO_SAME_ALIGN_REFS (stmt_info
));
4380 set_vinfo_for_stmt (stmt
, NULL
);
4385 /* Function get_vectype_for_scalar_type.
4387 Returns the vector type corresponding to SCALAR_TYPE as supported
4391 get_vectype_for_scalar_type (tree scalar_type
)
4393 enum machine_mode inner_mode
= TYPE_MODE (scalar_type
);
4394 int nbytes
= GET_MODE_SIZE (inner_mode
);
4398 if (nbytes
== 0 || nbytes
>= UNITS_PER_SIMD_WORD (inner_mode
))
4401 /* FORNOW: Only a single vector size per mode (UNITS_PER_SIMD_WORD)
4403 nunits
= UNITS_PER_SIMD_WORD (inner_mode
) / nbytes
;
4405 vectype
= build_vector_type (scalar_type
, nunits
);
4406 if (vect_print_dump_info (REPORT_DETAILS
))
4408 fprintf (vect_dump
, "get vectype with %d units of type ", nunits
);
4409 print_generic_expr (vect_dump
, scalar_type
, TDF_SLIM
);
4415 if (vect_print_dump_info (REPORT_DETAILS
))
4417 fprintf (vect_dump
, "vectype: ");
4418 print_generic_expr (vect_dump
, vectype
, TDF_SLIM
);
4421 if (!VECTOR_MODE_P (TYPE_MODE (vectype
))
4422 && !INTEGRAL_MODE_P (TYPE_MODE (vectype
)))
4424 if (vect_print_dump_info (REPORT_DETAILS
))
4425 fprintf (vect_dump
, "mode not supported by target.");
4432 /* Function vect_is_simple_use.
4435 LOOP_VINFO - the vect info of the loop that is being vectorized.
4436 BB_VINFO - the vect info of the basic block that is being vectorized.
4437 OPERAND - operand of a stmt in the loop or bb.
4438 DEF - the defining stmt in case OPERAND is an SSA_NAME.
4440 Returns whether a stmt with OPERAND can be vectorized.
4441 For loops, supportable operands are constants, loop invariants, and operands
4442 that are defined by the current iteration of the loop. Unsupportable
4443 operands are those that are defined by a previous iteration of the loop (as
4444 is the case in reduction/induction computations).
4445 For basic blocks, supportable operands are constants and bb invariants.
4446 For now, operands defined outside the basic block are not supported. */
4449 vect_is_simple_use (tree operand
, loop_vec_info loop_vinfo
,
4450 bb_vec_info bb_vinfo
, gimple
*def_stmt
,
4451 tree
*def
, enum vect_def_type
*dt
)
4454 stmt_vec_info stmt_vinfo
;
4455 struct loop
*loop
= NULL
;
4458 loop
= LOOP_VINFO_LOOP (loop_vinfo
);
4463 if (vect_print_dump_info (REPORT_DETAILS
))
4465 fprintf (vect_dump
, "vect_is_simple_use: operand ");
4466 print_generic_expr (vect_dump
, operand
, TDF_SLIM
);
4469 if (TREE_CODE (operand
) == INTEGER_CST
|| TREE_CODE (operand
) == REAL_CST
)
4471 *dt
= vect_constant_def
;
4475 if (is_gimple_min_invariant (operand
))
4478 *dt
= vect_external_def
;
4482 if (TREE_CODE (operand
) == PAREN_EXPR
)
4484 if (vect_print_dump_info (REPORT_DETAILS
))
4485 fprintf (vect_dump
, "non-associatable copy.");
4486 operand
= TREE_OPERAND (operand
, 0);
4489 if (TREE_CODE (operand
) != SSA_NAME
)
4491 if (vect_print_dump_info (REPORT_DETAILS
))
4492 fprintf (vect_dump
, "not ssa-name.");
4496 *def_stmt
= SSA_NAME_DEF_STMT (operand
);
4497 if (*def_stmt
== NULL
)
4499 if (vect_print_dump_info (REPORT_DETAILS
))
4500 fprintf (vect_dump
, "no def_stmt.");
4504 if (vect_print_dump_info (REPORT_DETAILS
))
4506 fprintf (vect_dump
, "def_stmt: ");
4507 print_gimple_stmt (vect_dump
, *def_stmt
, 0, TDF_SLIM
);
4510 /* Empty stmt is expected only in case of a function argument.
4511 (Otherwise - we expect a phi_node or a GIMPLE_ASSIGN). */
4512 if (gimple_nop_p (*def_stmt
))
4515 *dt
= vect_external_def
;
4519 bb
= gimple_bb (*def_stmt
);
4521 if ((loop
&& !flow_bb_inside_loop_p (loop
, bb
))
4522 || (!loop
&& bb
!= BB_VINFO_BB (bb_vinfo
))
4523 || (!loop
&& gimple_code (*def_stmt
) == GIMPLE_PHI
))
4524 *dt
= vect_external_def
;
4527 stmt_vinfo
= vinfo_for_stmt (*def_stmt
);
4528 *dt
= STMT_VINFO_DEF_TYPE (stmt_vinfo
);
4531 if (*dt
== vect_unknown_def_type
)
4533 if (vect_print_dump_info (REPORT_DETAILS
))
4534 fprintf (vect_dump
, "Unsupported pattern.");
4538 if (vect_print_dump_info (REPORT_DETAILS
))
4539 fprintf (vect_dump
, "type of def: %d.",*dt
);
4541 switch (gimple_code (*def_stmt
))
4544 *def
= gimple_phi_result (*def_stmt
);
4548 *def
= gimple_assign_lhs (*def_stmt
);
4552 *def
= gimple_call_lhs (*def_stmt
);
4557 if (vect_print_dump_info (REPORT_DETAILS
))
4558 fprintf (vect_dump
, "unsupported defining stmt: ");
4566 /* Function supportable_widening_operation
4568 Check whether an operation represented by the code CODE is a
4569 widening operation that is supported by the target platform in
4570 vector form (i.e., when operating on arguments of type VECTYPE).
4572 Widening operations we currently support are NOP (CONVERT), FLOAT
4573 and WIDEN_MULT. This function checks if these operations are supported
4574 by the target platform either directly (via vector tree-codes), or via
4578 - CODE1 and CODE2 are codes of vector operations to be used when
4579 vectorizing the operation, if available.
4580 - DECL1 and DECL2 are decls of target builtin functions to be used
4581 when vectorizing the operation, if available. In this case,
4582 CODE1 and CODE2 are CALL_EXPR.
4583 - MULTI_STEP_CVT determines the number of required intermediate steps in
4584 case of multi-step conversion (like char->short->int - in that case
4585 MULTI_STEP_CVT will be 1).
4586 - INTERM_TYPES contains the intermediate type required to perform the
4587 widening operation (short in the above example). */
4590 supportable_widening_operation (enum tree_code code
, gimple stmt
, tree vectype
,
4591 tree
*decl1
, tree
*decl2
,
4592 enum tree_code
*code1
, enum tree_code
*code2
,
4593 int *multi_step_cvt
,
4594 VEC (tree
, heap
) **interm_types
)
4596 stmt_vec_info stmt_info
= vinfo_for_stmt (stmt
);
4597 loop_vec_info loop_info
= STMT_VINFO_LOOP_VINFO (stmt_info
);
4598 struct loop
*vect_loop
= LOOP_VINFO_LOOP (loop_info
);
4600 enum machine_mode vec_mode
;
4601 enum insn_code icode1
, icode2
;
4602 optab optab1
, optab2
;
4603 tree type
= gimple_expr_type (stmt
);
4604 tree wide_vectype
= get_vectype_for_scalar_type (type
);
4605 enum tree_code c1
, c2
;
4607 /* The result of a vectorized widening operation usually requires two vectors
4608 (because the widened results do not fit int one vector). The generated
4609 vector results would normally be expected to be generated in the same
4610 order as in the original scalar computation, i.e. if 8 results are
4611 generated in each vector iteration, they are to be organized as follows:
4612 vect1: [res1,res2,res3,res4], vect2: [res5,res6,res7,res8].
4614 However, in the special case that the result of the widening operation is
4615 used in a reduction computation only, the order doesn't matter (because
4616 when vectorizing a reduction we change the order of the computation).
4617 Some targets can take advantage of this and generate more efficient code.
4618 For example, targets like Altivec, that support widen_mult using a sequence
4619 of {mult_even,mult_odd} generate the following vectors:
4620 vect1: [res1,res3,res5,res7], vect2: [res2,res4,res6,res8].
4622 When vectorizing outer-loops, we execute the inner-loop sequentially
4623 (each vectorized inner-loop iteration contributes to VF outer-loop
4624 iterations in parallel). We therefore don't allow to change the order
4625 of the computation in the inner-loop during outer-loop vectorization. */
4627 if (STMT_VINFO_RELEVANT (stmt_info
) == vect_used_by_reduction
4628 && !nested_in_vect_loop_p (vect_loop
, stmt
))
4634 && code
== WIDEN_MULT_EXPR
4635 && targetm
.vectorize
.builtin_mul_widen_even
4636 && targetm
.vectorize
.builtin_mul_widen_even (vectype
)
4637 && targetm
.vectorize
.builtin_mul_widen_odd
4638 && targetm
.vectorize
.builtin_mul_widen_odd (vectype
))
4640 if (vect_print_dump_info (REPORT_DETAILS
))
4641 fprintf (vect_dump
, "Unordered widening operation detected.");
4643 *code1
= *code2
= CALL_EXPR
;
4644 *decl1
= targetm
.vectorize
.builtin_mul_widen_even (vectype
);
4645 *decl2
= targetm
.vectorize
.builtin_mul_widen_odd (vectype
);
4651 case WIDEN_MULT_EXPR
:
4652 if (BYTES_BIG_ENDIAN
)
4654 c1
= VEC_WIDEN_MULT_HI_EXPR
;
4655 c2
= VEC_WIDEN_MULT_LO_EXPR
;
4659 c2
= VEC_WIDEN_MULT_HI_EXPR
;
4660 c1
= VEC_WIDEN_MULT_LO_EXPR
;
4665 if (BYTES_BIG_ENDIAN
)
4667 c1
= VEC_UNPACK_HI_EXPR
;
4668 c2
= VEC_UNPACK_LO_EXPR
;
4672 c2
= VEC_UNPACK_HI_EXPR
;
4673 c1
= VEC_UNPACK_LO_EXPR
;
4678 if (BYTES_BIG_ENDIAN
)
4680 c1
= VEC_UNPACK_FLOAT_HI_EXPR
;
4681 c2
= VEC_UNPACK_FLOAT_LO_EXPR
;
4685 c2
= VEC_UNPACK_FLOAT_HI_EXPR
;
4686 c1
= VEC_UNPACK_FLOAT_LO_EXPR
;
4690 case FIX_TRUNC_EXPR
:
4691 /* ??? Not yet implemented due to missing VEC_UNPACK_FIX_TRUNC_HI_EXPR/
4692 VEC_UNPACK_FIX_TRUNC_LO_EXPR tree codes and optabs used for
4693 computing the operation. */
4700 if (code
== FIX_TRUNC_EXPR
)
4702 /* The signedness is determined from output operand. */
4703 optab1
= optab_for_tree_code (c1
, type
, optab_default
);
4704 optab2
= optab_for_tree_code (c2
, type
, optab_default
);
4708 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
4709 optab2
= optab_for_tree_code (c2
, vectype
, optab_default
);
4712 if (!optab1
|| !optab2
)
4715 vec_mode
= TYPE_MODE (vectype
);
4716 if ((icode1
= optab_handler (optab1
, vec_mode
)->insn_code
) == CODE_FOR_nothing
4717 || (icode2
= optab_handler (optab2
, vec_mode
)->insn_code
)
4718 == CODE_FOR_nothing
)
4721 /* Check if it's a multi-step conversion that can be done using intermediate
4723 if (insn_data
[icode1
].operand
[0].mode
!= TYPE_MODE (wide_vectype
)
4724 || insn_data
[icode2
].operand
[0].mode
!= TYPE_MODE (wide_vectype
))
4727 tree prev_type
= vectype
, intermediate_type
;
4728 enum machine_mode intermediate_mode
, prev_mode
= vec_mode
;
4729 optab optab3
, optab4
;
4731 if (!CONVERT_EXPR_CODE_P (code
))
4737 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
4738 intermediate steps in promotion sequence. We try MAX_INTERM_CVT_STEPS
4739 to get to NARROW_VECTYPE, and fail if we do not. */
4740 *interm_types
= VEC_alloc (tree
, heap
, MAX_INTERM_CVT_STEPS
);
4741 for (i
= 0; i
< 3; i
++)
4743 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
4744 intermediate_type
= lang_hooks
.types
.type_for_mode (intermediate_mode
,
4745 TYPE_UNSIGNED (prev_type
));
4746 optab3
= optab_for_tree_code (c1
, intermediate_type
, optab_default
);
4747 optab4
= optab_for_tree_code (c2
, intermediate_type
, optab_default
);
4749 if (!optab3
|| !optab4
4750 || (icode1
= optab1
->handlers
[(int) prev_mode
].insn_code
)
4752 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
4753 || (icode2
= optab2
->handlers
[(int) prev_mode
].insn_code
)
4755 || insn_data
[icode2
].operand
[0].mode
!= intermediate_mode
4756 || (icode1
= optab3
->handlers
[(int) intermediate_mode
].insn_code
)
4758 || (icode2
= optab4
->handlers
[(int) intermediate_mode
].insn_code
)
4759 == CODE_FOR_nothing
)
4762 VEC_quick_push (tree
, *interm_types
, intermediate_type
);
4763 (*multi_step_cvt
)++;
4765 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (wide_vectype
)
4766 && insn_data
[icode2
].operand
[0].mode
== TYPE_MODE (wide_vectype
))
4769 prev_type
= intermediate_type
;
4770 prev_mode
= intermediate_mode
;
4782 /* Function supportable_narrowing_operation
4784 Check whether an operation represented by the code CODE is a
4785 narrowing operation that is supported by the target platform in
4786 vector form (i.e., when operating on arguments of type VECTYPE).
4788 Narrowing operations we currently support are NOP (CONVERT) and
4789 FIX_TRUNC. This function checks if these operations are supported by
4790 the target platform directly via vector tree-codes.
4793 - CODE1 is the code of a vector operation to be used when
4794 vectorizing the operation, if available.
4795 - MULTI_STEP_CVT determines the number of required intermediate steps in
4796 case of multi-step conversion (like int->short->char - in that case
4797 MULTI_STEP_CVT will be 1).
4798 - INTERM_TYPES contains the intermediate type required to perform the
4799 narrowing operation (short in the above example). */
4802 supportable_narrowing_operation (enum tree_code code
,
4803 const_gimple stmt
, tree vectype
,
4804 enum tree_code
*code1
, int *multi_step_cvt
,
4805 VEC (tree
, heap
) **interm_types
)
4807 enum machine_mode vec_mode
;
4808 enum insn_code icode1
;
4809 optab optab1
, interm_optab
;
4810 tree type
= gimple_expr_type (stmt
);
4811 tree narrow_vectype
= get_vectype_for_scalar_type (type
);
4813 tree intermediate_type
, prev_type
;
4819 c1
= VEC_PACK_TRUNC_EXPR
;
4822 case FIX_TRUNC_EXPR
:
4823 c1
= VEC_PACK_FIX_TRUNC_EXPR
;
4827 /* ??? Not yet implemented due to missing VEC_PACK_FLOAT_EXPR
4828 tree code and optabs used for computing the operation. */
4835 if (code
== FIX_TRUNC_EXPR
)
4836 /* The signedness is determined from output operand. */
4837 optab1
= optab_for_tree_code (c1
, type
, optab_default
);
4839 optab1
= optab_for_tree_code (c1
, vectype
, optab_default
);
4844 vec_mode
= TYPE_MODE (vectype
);
4845 if ((icode1
= optab_handler (optab1
, vec_mode
)->insn_code
)
4846 == CODE_FOR_nothing
)
4849 /* Check if it's a multi-step conversion that can be done using intermediate
4851 if (insn_data
[icode1
].operand
[0].mode
!= TYPE_MODE (narrow_vectype
))
4853 enum machine_mode intermediate_mode
, prev_mode
= vec_mode
;
4856 prev_type
= vectype
;
4857 /* We assume here that there will not be more than MAX_INTERM_CVT_STEPS
4858 intermediate steps in promotion sequence. We try MAX_INTERM_CVT_STEPS
4859 to get to NARROW_VECTYPE, and fail if we do not. */
4860 *interm_types
= VEC_alloc (tree
, heap
, MAX_INTERM_CVT_STEPS
);
4861 for (i
= 0; i
< 3; i
++)
4863 intermediate_mode
= insn_data
[icode1
].operand
[0].mode
;
4864 intermediate_type
= lang_hooks
.types
.type_for_mode (intermediate_mode
,
4865 TYPE_UNSIGNED (prev_type
));
4866 interm_optab
= optab_for_tree_code (c1
, intermediate_type
,
4869 || (icode1
= optab1
->handlers
[(int) prev_mode
].insn_code
)
4871 || insn_data
[icode1
].operand
[0].mode
!= intermediate_mode
4873 = interm_optab
->handlers
[(int) intermediate_mode
].insn_code
)
4874 == CODE_FOR_nothing
)
4877 VEC_quick_push (tree
, *interm_types
, intermediate_type
);
4878 (*multi_step_cvt
)++;
4880 if (insn_data
[icode1
].operand
[0].mode
== TYPE_MODE (narrow_vectype
))
4883 prev_type
= intermediate_type
;
4884 prev_mode
= intermediate_mode
;