1 /* Lower vector operations to scalar operations.
2 Copyright (C) 2004-2015 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it
7 under the terms of the GNU General Public License as published by the
8 Free Software Foundation; either version 3, or (at your option) any
11 GCC is distributed in the hope that it will be useful, but WITHOUT
12 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
27 #include "tree-pass.h"
30 #include "optabs-tree.h"
31 #include "diagnostic.h"
32 #include "fold-const.h"
33 #include "stor-layout.h"
34 #include "langhooks.h"
36 #include "gimple-iterator.h"
37 #include "gimplify-me.h"
41 static void expand_vector_operations_1 (gimple_stmt_iterator
*);
44 /* Build a constant of type TYPE, made of VALUE's bits replicated
45 every TYPE_SIZE (INNER_TYPE) bits to fit TYPE's precision. */
47 build_replicated_const (tree type
, tree inner_type
, HOST_WIDE_INT value
)
49 int width
= tree_to_uhwi (TYPE_SIZE (inner_type
));
50 int n
= (TYPE_PRECISION (type
) + HOST_BITS_PER_WIDE_INT
- 1)
51 / HOST_BITS_PER_WIDE_INT
;
52 unsigned HOST_WIDE_INT low
, mask
;
53 HOST_WIDE_INT a
[WIDE_INT_MAX_ELTS
];
56 gcc_assert (n
&& n
<= WIDE_INT_MAX_ELTS
);
58 if (width
== HOST_BITS_PER_WIDE_INT
)
62 mask
= ((HOST_WIDE_INT
)1 << width
) - 1;
63 low
= (unsigned HOST_WIDE_INT
) ~0 / mask
* (value
& mask
);
66 for (i
= 0; i
< n
; i
++)
69 gcc_assert (TYPE_PRECISION (type
) <= MAX_BITSIZE_MODE_ANY_INT
);
70 return wide_int_to_tree
71 (type
, wide_int::from_array (a
, n
, TYPE_PRECISION (type
)));
74 static GTY(()) tree vector_inner_type
;
75 static GTY(()) tree vector_last_type
;
76 static GTY(()) int vector_last_nunits
;
78 /* Return a suitable vector types made of SUBPARTS units each of mode
79 "word_mode" (the global variable). */
81 build_word_mode_vector_type (int nunits
)
83 if (!vector_inner_type
)
84 vector_inner_type
= lang_hooks
.types
.type_for_mode (word_mode
, 1);
85 else if (vector_last_nunits
== nunits
)
87 gcc_assert (TREE_CODE (vector_last_type
) == VECTOR_TYPE
);
88 return vector_last_type
;
91 /* We build a new type, but we canonicalize it nevertheless,
92 because it still saves some memory. */
93 vector_last_nunits
= nunits
;
94 vector_last_type
= type_hash_canon (nunits
,
95 build_vector_type (vector_inner_type
,
97 return vector_last_type
;
100 typedef tree (*elem_op_func
) (gimple_stmt_iterator
*,
101 tree
, tree
, tree
, tree
, tree
, enum tree_code
,
105 tree_vec_extract (gimple_stmt_iterator
*gsi
, tree type
,
106 tree t
, tree bitsize
, tree bitpos
)
110 if (TREE_CODE (type
) == BOOLEAN_TYPE
)
113 = build_nonstandard_integer_type (tree_to_uhwi (bitsize
), 0);
114 tree field
= gimplify_build3 (gsi
, BIT_FIELD_REF
, itype
, t
,
116 return gimplify_build2 (gsi
, NE_EXPR
, type
, field
,
117 build_zero_cst (itype
));
120 return gimplify_build3 (gsi
, BIT_FIELD_REF
, type
, t
, bitsize
, bitpos
);
123 return gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, type
, t
);
127 do_unop (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
,
128 tree b ATTRIBUTE_UNUSED
, tree bitpos
, tree bitsize
,
129 enum tree_code code
, tree type ATTRIBUTE_UNUSED
)
131 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
132 return gimplify_build1 (gsi
, code
, inner_type
, a
);
136 do_binop (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
137 tree bitpos
, tree bitsize
, enum tree_code code
,
138 tree type ATTRIBUTE_UNUSED
)
140 if (TREE_CODE (TREE_TYPE (a
)) == VECTOR_TYPE
)
141 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
142 if (TREE_CODE (TREE_TYPE (b
)) == VECTOR_TYPE
)
143 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
144 return gimplify_build2 (gsi
, code
, inner_type
, a
, b
);
147 /* Construct expression (A[BITPOS] code B[BITPOS]) ? -1 : 0
149 INNER_TYPE is the type of A and B elements
151 returned expression is of signed integer type with the
152 size equal to the size of INNER_TYPE. */
154 do_compare (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
155 tree bitpos
, tree bitsize
, enum tree_code code
, tree type
)
157 tree stype
= TREE_TYPE (type
);
158 tree cst_false
= build_zero_cst (stype
);
159 tree cst_true
= build_all_ones_cst (stype
);
162 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
163 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
165 cmp
= build2 (code
, boolean_type_node
, a
, b
);
166 return gimplify_build3 (gsi
, COND_EXPR
, stype
, cmp
, cst_true
, cst_false
);
169 /* Expand vector addition to scalars. This does bit twiddling
170 in order to increase parallelism:
172 a + b = (((int) a & 0x7f7f7f7f) + ((int) b & 0x7f7f7f7f)) ^
175 a - b = (((int) a | 0x80808080) - ((int) b & 0x7f7f7f7f)) ^
176 (a ^ ~b) & 0x80808080
178 -b = (0x80808080 - ((int) b & 0x7f7f7f7f)) ^ (~b & 0x80808080)
180 This optimization should be done only if 4 vector items or more
183 do_plus_minus (gimple_stmt_iterator
*gsi
, tree word_type
, tree a
, tree b
,
184 tree bitpos ATTRIBUTE_UNUSED
, tree bitsize ATTRIBUTE_UNUSED
,
185 enum tree_code code
, tree type ATTRIBUTE_UNUSED
)
187 tree inner_type
= TREE_TYPE (TREE_TYPE (a
));
188 unsigned HOST_WIDE_INT max
;
189 tree low_bits
, high_bits
, a_low
, b_low
, result_low
, signs
;
191 max
= GET_MODE_MASK (TYPE_MODE (inner_type
));
192 low_bits
= build_replicated_const (word_type
, inner_type
, max
>> 1);
193 high_bits
= build_replicated_const (word_type
, inner_type
, max
& ~(max
>> 1));
195 a
= tree_vec_extract (gsi
, word_type
, a
, bitsize
, bitpos
);
196 b
= tree_vec_extract (gsi
, word_type
, b
, bitsize
, bitpos
);
198 signs
= gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, a
, b
);
199 b_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, b
, low_bits
);
200 if (code
== PLUS_EXPR
)
201 a_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, a
, low_bits
);
204 a_low
= gimplify_build2 (gsi
, BIT_IOR_EXPR
, word_type
, a
, high_bits
);
205 signs
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, word_type
, signs
);
208 signs
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, signs
, high_bits
);
209 result_low
= gimplify_build2 (gsi
, code
, word_type
, a_low
, b_low
);
210 return gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, result_low
, signs
);
214 do_negate (gimple_stmt_iterator
*gsi
, tree word_type
, tree b
,
215 tree unused ATTRIBUTE_UNUSED
, tree bitpos ATTRIBUTE_UNUSED
,
216 tree bitsize ATTRIBUTE_UNUSED
,
217 enum tree_code code ATTRIBUTE_UNUSED
,
218 tree type ATTRIBUTE_UNUSED
)
220 tree inner_type
= TREE_TYPE (TREE_TYPE (b
));
222 tree low_bits
, high_bits
, b_low
, result_low
, signs
;
224 max
= GET_MODE_MASK (TYPE_MODE (inner_type
));
225 low_bits
= build_replicated_const (word_type
, inner_type
, max
>> 1);
226 high_bits
= build_replicated_const (word_type
, inner_type
, max
& ~(max
>> 1));
228 b
= tree_vec_extract (gsi
, word_type
, b
, bitsize
, bitpos
);
230 b_low
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, b
, low_bits
);
231 signs
= gimplify_build1 (gsi
, BIT_NOT_EXPR
, word_type
, b
);
232 signs
= gimplify_build2 (gsi
, BIT_AND_EXPR
, word_type
, signs
, high_bits
);
233 result_low
= gimplify_build2 (gsi
, MINUS_EXPR
, word_type
, high_bits
, b_low
);
234 return gimplify_build2 (gsi
, BIT_XOR_EXPR
, word_type
, result_low
, signs
);
237 /* Expand a vector operation to scalars, by using many operations
238 whose type is the vector type's inner type. */
240 expand_vector_piecewise (gimple_stmt_iterator
*gsi
, elem_op_func f
,
241 tree type
, tree inner_type
,
242 tree a
, tree b
, enum tree_code code
)
244 vec
<constructor_elt
, va_gc
> *v
;
245 tree part_width
= TYPE_SIZE (inner_type
);
246 tree index
= bitsize_int (0);
247 int nunits
= TYPE_VECTOR_SUBPARTS (type
);
248 int delta
= tree_to_uhwi (part_width
)
249 / tree_to_uhwi (TYPE_SIZE (TREE_TYPE (type
)));
251 location_t loc
= gimple_location (gsi_stmt (*gsi
));
253 if (types_compatible_p (gimple_expr_type (gsi_stmt (*gsi
)), type
))
254 warning_at (loc
, OPT_Wvector_operation_performance
,
255 "vector operation will be expanded piecewise");
257 warning_at (loc
, OPT_Wvector_operation_performance
,
258 "vector operation will be expanded in parallel");
260 vec_alloc (v
, (nunits
+ delta
- 1) / delta
);
261 for (i
= 0; i
< nunits
;
262 i
+= delta
, index
= int_const_binop (PLUS_EXPR
, index
, part_width
))
264 tree result
= f (gsi
, inner_type
, a
, b
, index
, part_width
, code
, type
);
265 constructor_elt ce
= {NULL_TREE
, result
};
269 return build_constructor (type
, v
);
272 /* Expand a vector operation to scalars with the freedom to use
273 a scalar integer type, or to use a different size for the items
274 in the vector type. */
276 expand_vector_parallel (gimple_stmt_iterator
*gsi
, elem_op_func f
, tree type
,
280 tree result
, compute_type
;
282 int n_words
= tree_to_uhwi (TYPE_SIZE_UNIT (type
)) / UNITS_PER_WORD
;
283 location_t loc
= gimple_location (gsi_stmt (*gsi
));
285 /* We have three strategies. If the type is already correct, just do
286 the operation an element at a time. Else, if the vector is wider than
287 one word, do it a word at a time; finally, if the vector is smaller
288 than one word, do it as a scalar. */
289 if (TYPE_MODE (TREE_TYPE (type
)) == word_mode
)
290 return expand_vector_piecewise (gsi
, f
,
291 type
, TREE_TYPE (type
),
293 else if (n_words
> 1)
295 tree word_type
= build_word_mode_vector_type (n_words
);
296 result
= expand_vector_piecewise (gsi
, f
,
297 word_type
, TREE_TYPE (word_type
),
299 result
= force_gimple_operand_gsi (gsi
, result
, true, NULL
, true,
304 /* Use a single scalar operation with a mode no wider than word_mode. */
305 mode
= mode_for_size (tree_to_uhwi (TYPE_SIZE (type
)), MODE_INT
, 0);
306 compute_type
= lang_hooks
.types
.type_for_mode (mode
, 1);
307 result
= f (gsi
, compute_type
, a
, b
, NULL_TREE
, NULL_TREE
, code
, type
);
308 warning_at (loc
, OPT_Wvector_operation_performance
,
309 "vector operation will be expanded with a "
310 "single scalar operation");
316 /* Expand a vector operation to scalars; for integer types we can use
317 special bit twiddling tricks to do the sums a word at a time, using
318 function F_PARALLEL instead of F. These tricks are done only if
319 they can process at least four items, that is, only if the vector
320 holds at least four items and if a word can hold four items. */
322 expand_vector_addition (gimple_stmt_iterator
*gsi
,
323 elem_op_func f
, elem_op_func f_parallel
,
324 tree type
, tree a
, tree b
, enum tree_code code
)
326 int parts_per_word
= UNITS_PER_WORD
327 / tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (type
)));
329 if (INTEGRAL_TYPE_P (TREE_TYPE (type
))
330 && parts_per_word
>= 4
331 && TYPE_VECTOR_SUBPARTS (type
) >= 4)
332 return expand_vector_parallel (gsi
, f_parallel
,
335 return expand_vector_piecewise (gsi
, f
,
336 type
, TREE_TYPE (type
),
340 /* Try to expand vector comparison expression OP0 CODE OP1 by
341 querying optab if the following expression:
342 VEC_COND_EXPR< OP0 CODE OP1, {-1,...}, {0,...}>
345 expand_vector_comparison (gimple_stmt_iterator
*gsi
, tree type
, tree op0
,
346 tree op1
, enum tree_code code
)
349 if (!expand_vec_cmp_expr_p (TREE_TYPE (op0
), type
)
350 && !expand_vec_cond_expr_p (type
, TREE_TYPE (op0
)))
351 t
= expand_vector_piecewise (gsi
, do_compare
, type
,
352 TREE_TYPE (TREE_TYPE (op0
)), op0
, op1
, code
);
359 /* Helper function of expand_vector_divmod. Gimplify a RSHIFT_EXPR in type
360 of OP0 with shift counts in SHIFTCNTS array and return the temporary holding
361 the result if successful, otherwise return NULL_TREE. */
363 add_rshift (gimple_stmt_iterator
*gsi
, tree type
, tree op0
, int *shiftcnts
)
366 unsigned int i
, nunits
= TYPE_VECTOR_SUBPARTS (type
);
367 bool scalar_shift
= true;
369 for (i
= 1; i
< nunits
; i
++)
371 if (shiftcnts
[i
] != shiftcnts
[0])
372 scalar_shift
= false;
375 if (scalar_shift
&& shiftcnts
[0] == 0)
380 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_scalar
);
381 if (op
!= unknown_optab
382 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
383 return gimplify_build2 (gsi
, RSHIFT_EXPR
, type
, op0
,
384 build_int_cst (NULL_TREE
, shiftcnts
[0]));
387 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_vector
);
388 if (op
!= unknown_optab
389 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
391 tree
*vec
= XALLOCAVEC (tree
, nunits
);
392 for (i
= 0; i
< nunits
; i
++)
393 vec
[i
] = build_int_cst (TREE_TYPE (type
), shiftcnts
[i
]);
394 return gimplify_build2 (gsi
, RSHIFT_EXPR
, type
, op0
,
395 build_vector (type
, vec
));
401 /* Try to expand integer vector division by constant using
402 widening multiply, shifts and additions. */
404 expand_vector_divmod (gimple_stmt_iterator
*gsi
, tree type
, tree op0
,
405 tree op1
, enum tree_code code
)
407 bool use_pow2
= true;
408 bool has_vector_shift
= true;
409 int mode
= -1, this_mode
;
410 int pre_shift
= -1, post_shift
;
411 unsigned int nunits
= TYPE_VECTOR_SUBPARTS (type
);
412 int *shifts
= XALLOCAVEC (int, nunits
* 4);
413 int *pre_shifts
= shifts
+ nunits
;
414 int *post_shifts
= pre_shifts
+ nunits
;
415 int *shift_temps
= post_shifts
+ nunits
;
416 unsigned HOST_WIDE_INT
*mulc
= XALLOCAVEC (unsigned HOST_WIDE_INT
, nunits
);
417 int prec
= TYPE_PRECISION (TREE_TYPE (type
));
420 signop sign_p
= TYPE_SIGN (TREE_TYPE (type
));
421 unsigned HOST_WIDE_INT mask
= GET_MODE_MASK (TYPE_MODE (TREE_TYPE (type
)));
423 tree cur_op
, mulcst
, tem
;
426 if (prec
> HOST_BITS_PER_WIDE_INT
)
429 op
= optab_for_tree_code (RSHIFT_EXPR
, type
, optab_vector
);
430 if (op
== unknown_optab
431 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
432 has_vector_shift
= false;
434 /* Analysis phase. Determine if all op1 elements are either power
435 of two and it is possible to expand it using shifts (or for remainder
436 using masking). Additionally compute the multiplicative constants
437 and pre and post shifts if the division is to be expanded using
438 widening or high part multiplication plus shifts. */
439 for (i
= 0; i
< nunits
; i
++)
441 tree cst
= VECTOR_CST_ELT (op1
, i
);
442 unsigned HOST_WIDE_INT ml
;
444 if (TREE_CODE (cst
) != INTEGER_CST
|| integer_zerop (cst
))
450 && (!integer_pow2p (cst
) || tree_int_cst_sgn (cst
) != 1))
454 shifts
[i
] = tree_log2 (cst
);
455 if (shifts
[i
] != shifts
[0]
456 && code
== TRUNC_DIV_EXPR
457 && !has_vector_shift
)
462 if (sign_p
== UNSIGNED
)
464 unsigned HOST_WIDE_INT mh
;
465 unsigned HOST_WIDE_INT d
= TREE_INT_CST_LOW (cst
) & mask
;
467 if (d
>= ((unsigned HOST_WIDE_INT
) 1 << (prec
- 1)))
468 /* FIXME: Can transform this into op0 >= op1 ? 1 : 0. */
477 /* Find a suitable multiplier and right shift count
478 instead of multiplying with D. */
479 mh
= choose_multiplier (d
, prec
, prec
, &ml
, &post_shift
, &dummy_int
);
481 /* If the suggested multiplier is more than SIZE bits, we can
482 do better for even divisors, using an initial right shift. */
483 if ((mh
!= 0 && (d
& 1) == 0)
484 || (!has_vector_shift
&& pre_shift
!= -1))
486 if (has_vector_shift
)
487 pre_shift
= floor_log2 (d
& -d
);
488 else if (pre_shift
== -1)
491 for (j
= 0; j
< nunits
; j
++)
493 tree cst2
= VECTOR_CST_ELT (op1
, j
);
494 unsigned HOST_WIDE_INT d2
;
497 if (!tree_fits_uhwi_p (cst2
))
499 d2
= tree_to_uhwi (cst2
) & mask
;
502 this_pre_shift
= floor_log2 (d2
& -d2
);
503 if (pre_shift
== -1 || this_pre_shift
< pre_shift
)
504 pre_shift
= this_pre_shift
;
506 if (i
!= 0 && pre_shift
!= 0)
516 if ((d
>> pre_shift
) <= 1)
521 mh
= choose_multiplier (d
>> pre_shift
, prec
,
523 &ml
, &post_shift
, &dummy_int
);
525 pre_shifts
[i
] = pre_shift
;
535 HOST_WIDE_INT d
= TREE_INT_CST_LOW (cst
);
536 unsigned HOST_WIDE_INT abs_d
;
541 /* Since d might be INT_MIN, we have to cast to
542 unsigned HOST_WIDE_INT before negating to avoid
543 undefined signed overflow. */
545 ? (unsigned HOST_WIDE_INT
) d
546 : - (unsigned HOST_WIDE_INT
) d
);
548 /* n rem d = n rem -d */
549 if (code
== TRUNC_MOD_EXPR
&& d
< 0)
551 else if (abs_d
== (unsigned HOST_WIDE_INT
) 1 << (prec
- 1))
553 /* This case is not handled correctly below. */
563 choose_multiplier (abs_d
, prec
, prec
- 1, &ml
,
564 &post_shift
, &dummy_int
);
565 if (ml
>= (unsigned HOST_WIDE_INT
) 1 << (prec
- 1))
567 this_mode
= 4 + (d
< 0);
568 ml
|= (~(unsigned HOST_WIDE_INT
) 0) << (prec
- 1);
571 this_mode
= 2 + (d
< 0);
574 post_shifts
[i
] = post_shift
;
575 if ((i
&& !has_vector_shift
&& post_shifts
[0] != post_shift
)
576 || post_shift
>= prec
577 || pre_shifts
[i
] >= prec
)
582 else if (mode
!= this_mode
)
586 vec
= XALLOCAVEC (tree
, nunits
);
590 tree addend
= NULL_TREE
;
591 if (sign_p
== SIGNED
)
595 /* Both division and remainder sequences need
596 op0 < 0 ? mask : 0 computed. It can be either computed as
597 (type) (((uns_type) (op0 >> (prec - 1))) >> (prec - shifts[i]))
598 if none of the shifts is 0, or as the conditional. */
599 for (i
= 0; i
< nunits
; i
++)
603 = build_vector_type (build_nonstandard_integer_type (prec
, 1),
605 if (i
== nunits
&& TYPE_MODE (uns_type
) == TYPE_MODE (type
))
607 for (i
= 0; i
< nunits
; i
++)
608 shift_temps
[i
] = prec
- 1;
609 cur_op
= add_rshift (gsi
, type
, op0
, shift_temps
);
610 if (cur_op
!= NULL_TREE
)
612 cur_op
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
,
614 for (i
= 0; i
< nunits
; i
++)
615 shift_temps
[i
] = prec
- shifts
[i
];
616 cur_op
= add_rshift (gsi
, uns_type
, cur_op
, shift_temps
);
617 if (cur_op
!= NULL_TREE
)
618 addend
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
,
622 if (addend
== NULL_TREE
623 && expand_vec_cond_expr_p (type
, type
))
625 tree zero
, cst
, cond
, mask_type
;
628 mask_type
= build_same_sized_truth_vector_type (type
);
629 zero
= build_zero_cst (type
);
630 cond
= build2 (LT_EXPR
, mask_type
, op0
, zero
);
631 for (i
= 0; i
< nunits
; i
++)
632 vec
[i
] = build_int_cst (TREE_TYPE (type
),
633 ((unsigned HOST_WIDE_INT
) 1
635 cst
= build_vector (type
, vec
);
636 addend
= make_ssa_name (type
);
637 stmt
= gimple_build_assign (addend
, VEC_COND_EXPR
, cond
,
639 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
642 if (code
== TRUNC_DIV_EXPR
)
644 if (sign_p
== UNSIGNED
)
646 /* q = op0 >> shift; */
647 cur_op
= add_rshift (gsi
, type
, op0
, shifts
);
648 if (cur_op
!= NULL_TREE
)
651 else if (addend
!= NULL_TREE
)
653 /* t1 = op0 + addend;
655 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
656 if (op
!= unknown_optab
657 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
659 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, op0
, addend
);
660 cur_op
= add_rshift (gsi
, type
, cur_op
, shifts
);
661 if (cur_op
!= NULL_TREE
)
669 for (i
= 0; i
< nunits
; i
++)
670 vec
[i
] = build_int_cst (TREE_TYPE (type
),
671 ((unsigned HOST_WIDE_INT
) 1
673 mask
= build_vector (type
, vec
);
674 op
= optab_for_tree_code (BIT_AND_EXPR
, type
, optab_default
);
675 if (op
!= unknown_optab
676 && optab_handler (op
, TYPE_MODE (type
)) != CODE_FOR_nothing
)
678 if (sign_p
== UNSIGNED
)
679 /* r = op0 & mask; */
680 return gimplify_build2 (gsi
, BIT_AND_EXPR
, type
, op0
, mask
);
681 else if (addend
!= NULL_TREE
)
683 /* t1 = op0 + addend;
686 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
687 if (op
!= unknown_optab
688 && optab_handler (op
, TYPE_MODE (type
))
691 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, op0
,
693 cur_op
= gimplify_build2 (gsi
, BIT_AND_EXPR
, type
,
695 op
= optab_for_tree_code (MINUS_EXPR
, type
,
697 if (op
!= unknown_optab
698 && optab_handler (op
, TYPE_MODE (type
))
700 return gimplify_build2 (gsi
, MINUS_EXPR
, type
,
708 if (mode
== -2 || BYTES_BIG_ENDIAN
!= WORDS_BIG_ENDIAN
)
711 if (!can_mult_highpart_p (TYPE_MODE (type
), TYPE_UNSIGNED (type
)))
719 gcc_assert (sign_p
== UNSIGNED
);
720 /* t1 = oprnd0 >> pre_shift;
722 q = t2 >> post_shift; */
723 cur_op
= add_rshift (gsi
, type
, cur_op
, pre_shifts
);
724 if (cur_op
== NULL_TREE
)
728 gcc_assert (sign_p
== UNSIGNED
);
729 for (i
= 0; i
< nunits
; i
++)
739 gcc_assert (sign_p
== SIGNED
);
740 for (i
= 0; i
< nunits
; i
++)
741 shift_temps
[i
] = prec
- 1;
747 for (i
= 0; i
< nunits
; i
++)
748 vec
[i
] = build_int_cst (TREE_TYPE (type
), mulc
[i
]);
749 mulcst
= build_vector (type
, vec
);
751 cur_op
= gimplify_build2 (gsi
, MULT_HIGHPART_EXPR
, type
, cur_op
, mulcst
);
756 /* t1 = oprnd0 >> pre_shift;
758 q = t2 >> post_shift; */
759 cur_op
= add_rshift (gsi
, type
, cur_op
, post_shifts
);
762 /* t1 = oprnd0 h* ml;
766 q = t4 >> (post_shift - 1); */
767 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
768 if (op
== unknown_optab
769 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
771 tem
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, op0
, cur_op
);
772 tem
= add_rshift (gsi
, type
, tem
, shift_temps
);
773 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
774 if (op
== unknown_optab
775 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
777 tem
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, cur_op
, tem
);
778 cur_op
= add_rshift (gsi
, type
, tem
, post_shifts
);
779 if (cur_op
== NULL_TREE
)
786 /* t1 = oprnd0 h* ml;
787 t2 = t1; [ iff (mode & 2) != 0 ]
788 t2 = t1 + oprnd0; [ iff (mode & 2) == 0 ]
789 t3 = t2 >> post_shift;
790 t4 = oprnd0 >> (prec - 1);
791 q = t3 - t4; [ iff (mode & 1) == 0 ]
792 q = t4 - t3; [ iff (mode & 1) != 0 ] */
795 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
796 if (op
== unknown_optab
797 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
799 cur_op
= gimplify_build2 (gsi
, PLUS_EXPR
, type
, cur_op
, op0
);
801 cur_op
= add_rshift (gsi
, type
, cur_op
, post_shifts
);
802 if (cur_op
== NULL_TREE
)
804 tem
= add_rshift (gsi
, type
, op0
, shift_temps
);
805 if (tem
== NULL_TREE
)
807 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
808 if (op
== unknown_optab
809 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
812 cur_op
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, cur_op
, tem
);
814 cur_op
= gimplify_build2 (gsi
, MINUS_EXPR
, type
, tem
, cur_op
);
820 if (code
== TRUNC_DIV_EXPR
)
823 /* We divided. Now finish by:
826 op
= optab_for_tree_code (MULT_EXPR
, type
, optab_default
);
827 if (op
== unknown_optab
828 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
830 tem
= gimplify_build2 (gsi
, MULT_EXPR
, type
, cur_op
, op1
);
831 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
832 if (op
== unknown_optab
833 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
835 return gimplify_build2 (gsi
, MINUS_EXPR
, type
, op0
, tem
);
838 /* Expand a vector condition to scalars, by using many conditions
839 on the vector's elements. */
841 expand_vector_condition (gimple_stmt_iterator
*gsi
)
843 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
844 tree type
= gimple_expr_type (stmt
);
845 tree a
= gimple_assign_rhs1 (stmt
);
848 bool a_is_comparison
= false;
849 tree b
= gimple_assign_rhs2 (stmt
);
850 tree c
= gimple_assign_rhs3 (stmt
);
851 vec
<constructor_elt
, va_gc
> *v
;
853 tree inner_type
= TREE_TYPE (type
);
854 tree cond_type
= TREE_TYPE (TREE_TYPE (a
));
855 tree comp_inner_type
= cond_type
;
856 tree width
= TYPE_SIZE (inner_type
);
857 tree index
= bitsize_int (0);
858 int nunits
= TYPE_VECTOR_SUBPARTS (type
);
860 location_t loc
= gimple_location (gsi_stmt (*gsi
));
862 if (!is_gimple_val (a
))
864 gcc_assert (COMPARISON_CLASS_P (a
));
865 a_is_comparison
= true;
866 a1
= TREE_OPERAND (a
, 0);
867 a2
= TREE_OPERAND (a
, 1);
868 comp_inner_type
= TREE_TYPE (TREE_TYPE (a1
));
871 if (expand_vec_cond_expr_p (type
, TREE_TYPE (a1
)))
874 /* TODO: try and find a smaller vector type. */
876 warning_at (loc
, OPT_Wvector_operation_performance
,
877 "vector condition will be expanded piecewise");
879 vec_alloc (v
, nunits
);
880 for (i
= 0; i
< nunits
;
881 i
++, index
= int_const_binop (PLUS_EXPR
, index
, width
))
884 tree bb
= tree_vec_extract (gsi
, inner_type
, b
, width
, index
);
885 tree cc
= tree_vec_extract (gsi
, inner_type
, c
, width
, index
);
888 tree aa1
= tree_vec_extract (gsi
, comp_inner_type
, a1
, width
, index
);
889 tree aa2
= tree_vec_extract (gsi
, comp_inner_type
, a2
, width
, index
);
890 aa
= build2 (TREE_CODE (a
), cond_type
, aa1
, aa2
);
893 aa
= tree_vec_extract (gsi
, cond_type
, a
, width
, index
);
894 result
= gimplify_build3 (gsi
, COND_EXPR
, inner_type
, aa
, bb
, cc
);
895 constructor_elt ce
= {NULL_TREE
, result
};
899 constr
= build_constructor (type
, v
);
900 gimple_assign_set_rhs_from_tree (gsi
, constr
);
901 update_stmt (gsi_stmt (*gsi
));
905 expand_vector_operation (gimple_stmt_iterator
*gsi
, tree type
, tree compute_type
,
906 gassign
*assign
, enum tree_code code
)
908 machine_mode compute_mode
= TYPE_MODE (compute_type
);
910 /* If the compute mode is not a vector mode (hence we are not decomposing
911 a BLKmode vector to smaller, hardware-supported vectors), we may want
912 to expand the operations in parallel. */
913 if (GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_INT
914 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_FLOAT
915 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_FRACT
916 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_UFRACT
917 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_ACCUM
918 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_UACCUM
)
923 if (ANY_INTEGRAL_TYPE_P (type
) && !TYPE_OVERFLOW_TRAPS (type
))
924 return expand_vector_addition (gsi
, do_binop
, do_plus_minus
, type
,
925 gimple_assign_rhs1 (assign
),
926 gimple_assign_rhs2 (assign
), code
);
930 if (ANY_INTEGRAL_TYPE_P (type
) && !TYPE_OVERFLOW_TRAPS (type
))
931 return expand_vector_addition (gsi
, do_unop
, do_negate
, type
,
932 gimple_assign_rhs1 (assign
),
939 return expand_vector_parallel (gsi
, do_binop
, type
,
940 gimple_assign_rhs1 (assign
),
941 gimple_assign_rhs2 (assign
), code
);
944 return expand_vector_parallel (gsi
, do_unop
, type
,
945 gimple_assign_rhs1 (assign
),
962 tree rhs1
= gimple_assign_rhs1 (assign
);
963 tree rhs2
= gimple_assign_rhs2 (assign
);
965 return expand_vector_comparison (gsi
, type
, rhs1
, rhs2
, code
);
971 tree rhs1
= gimple_assign_rhs1 (assign
);
972 tree rhs2
= gimple_assign_rhs2 (assign
);
976 || !VECTOR_INTEGER_TYPE_P (type
)
977 || TREE_CODE (rhs2
) != VECTOR_CST
978 || !VECTOR_MODE_P (TYPE_MODE (type
)))
981 ret
= expand_vector_divmod (gsi
, type
, rhs1
, rhs2
, code
);
982 if (ret
!= NULL_TREE
)
991 if (TREE_CODE_CLASS (code
) == tcc_unary
)
992 return expand_vector_piecewise (gsi
, do_unop
, type
, compute_type
,
993 gimple_assign_rhs1 (assign
),
996 return expand_vector_piecewise (gsi
, do_binop
, type
, compute_type
,
997 gimple_assign_rhs1 (assign
),
998 gimple_assign_rhs2 (assign
), code
);
1002 a_5 = { b_7, b_7 + 3, b_7 + 6, b_7 + 9 };
1004 _9 = { b_7, b_7, b_7, b_7 };
1005 a_5 = _9 + { 0, 3, 6, 9 };
1006 because vector splat operation is usually more efficient
1007 than piecewise initialization of the vector. */
1010 optimize_vector_constructor (gimple_stmt_iterator
*gsi
)
1012 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1013 tree lhs
= gimple_assign_lhs (stmt
);
1014 tree rhs
= gimple_assign_rhs1 (stmt
);
1015 tree type
= TREE_TYPE (rhs
);
1016 unsigned int i
, j
, nelts
= TYPE_VECTOR_SUBPARTS (type
);
1017 bool all_same
= true;
1018 constructor_elt
*elt
;
1021 tree base
= NULL_TREE
;
1024 if (nelts
<= 2 || CONSTRUCTOR_NELTS (rhs
) != nelts
)
1026 op
= optab_for_tree_code (PLUS_EXPR
, type
, optab_default
);
1027 if (op
== unknown_optab
1028 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
)
1030 FOR_EACH_VEC_SAFE_ELT (CONSTRUCTOR_ELTS (rhs
), i
, elt
)
1031 if (TREE_CODE (elt
->value
) != SSA_NAME
1032 || TREE_CODE (TREE_TYPE (elt
->value
)) == VECTOR_TYPE
)
1036 tree this_base
= elt
->value
;
1037 if (this_base
!= CONSTRUCTOR_ELT (rhs
, 0)->value
)
1039 for (j
= 0; j
< nelts
+ 1; j
++)
1041 g
= SSA_NAME_DEF_STMT (this_base
);
1042 if (is_gimple_assign (g
)
1043 && gimple_assign_rhs_code (g
) == PLUS_EXPR
1044 && TREE_CODE (gimple_assign_rhs2 (g
)) == INTEGER_CST
1045 && TREE_CODE (gimple_assign_rhs1 (g
)) == SSA_NAME
1046 && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (gimple_assign_rhs1 (g
)))
1047 this_base
= gimple_assign_rhs1 (g
);
1053 else if (this_base
!= base
)
1058 cst
= XALLOCAVEC (tree
, nelts
);
1059 for (i
= 0; i
< nelts
; i
++)
1061 tree this_base
= CONSTRUCTOR_ELT (rhs
, i
)->value
;;
1062 cst
[i
] = build_zero_cst (TREE_TYPE (base
));
1063 while (this_base
!= base
)
1065 g
= SSA_NAME_DEF_STMT (this_base
);
1066 cst
[i
] = fold_binary (PLUS_EXPR
, TREE_TYPE (base
),
1067 cst
[i
], gimple_assign_rhs2 (g
));
1068 if (cst
[i
] == NULL_TREE
1069 || TREE_CODE (cst
[i
]) != INTEGER_CST
1070 || TREE_OVERFLOW (cst
[i
]))
1072 this_base
= gimple_assign_rhs1 (g
);
1075 for (i
= 0; i
< nelts
; i
++)
1076 CONSTRUCTOR_ELT (rhs
, i
)->value
= base
;
1077 g
= gimple_build_assign (make_ssa_name (type
), rhs
);
1078 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
1079 g
= gimple_build_assign (lhs
, PLUS_EXPR
, gimple_assign_lhs (g
),
1080 build_vector (type
, cst
));
1081 gsi_replace (gsi
, g
, false);
1084 /* Return a type for the widest vector mode whose components are of type
1085 TYPE, or NULL_TREE if none is found. */
1088 type_for_widest_vector_mode (tree type
, optab op
)
1090 machine_mode inner_mode
= TYPE_MODE (type
);
1091 machine_mode best_mode
= VOIDmode
, mode
;
1092 int best_nunits
= 0;
1094 if (SCALAR_FLOAT_MODE_P (inner_mode
))
1095 mode
= MIN_MODE_VECTOR_FLOAT
;
1096 else if (SCALAR_FRACT_MODE_P (inner_mode
))
1097 mode
= MIN_MODE_VECTOR_FRACT
;
1098 else if (SCALAR_UFRACT_MODE_P (inner_mode
))
1099 mode
= MIN_MODE_VECTOR_UFRACT
;
1100 else if (SCALAR_ACCUM_MODE_P (inner_mode
))
1101 mode
= MIN_MODE_VECTOR_ACCUM
;
1102 else if (SCALAR_UACCUM_MODE_P (inner_mode
))
1103 mode
= MIN_MODE_VECTOR_UACCUM
;
1105 mode
= MIN_MODE_VECTOR_INT
;
1107 for (; mode
!= VOIDmode
; mode
= GET_MODE_WIDER_MODE (mode
))
1108 if (GET_MODE_INNER (mode
) == inner_mode
1109 && GET_MODE_NUNITS (mode
) > best_nunits
1110 && optab_handler (op
, mode
) != CODE_FOR_nothing
)
1111 best_mode
= mode
, best_nunits
= GET_MODE_NUNITS (mode
);
1113 if (best_mode
== VOIDmode
)
1116 return build_vector_type_for_mode (type
, best_mode
);
1120 /* Build a reference to the element of the vector VECT. Function
1121 returns either the element itself, either BIT_FIELD_REF, or an
1122 ARRAY_REF expression.
1124 GSI is required to insert temporary variables while building a
1125 refernece to the element of the vector VECT.
1127 PTMPVEC is a pointer to the temporary variable for caching
1128 purposes. In case when PTMPVEC is NULL new temporary variable
1131 vector_element (gimple_stmt_iterator
*gsi
, tree vect
, tree idx
, tree
*ptmpvec
)
1133 tree vect_type
, vect_elt_type
;
1137 bool need_asgn
= true;
1138 unsigned int elements
;
1140 vect_type
= TREE_TYPE (vect
);
1141 vect_elt_type
= TREE_TYPE (vect_type
);
1142 elements
= TYPE_VECTOR_SUBPARTS (vect_type
);
1144 if (TREE_CODE (idx
) == INTEGER_CST
)
1146 unsigned HOST_WIDE_INT index
;
1148 /* Given that we're about to compute a binary modulus,
1149 we don't care about the high bits of the value. */
1150 index
= TREE_INT_CST_LOW (idx
);
1151 if (!tree_fits_uhwi_p (idx
) || index
>= elements
)
1153 index
&= elements
- 1;
1154 idx
= build_int_cst (TREE_TYPE (idx
), index
);
1157 /* When lowering a vector statement sequence do some easy
1158 simplification by looking through intermediate vector results. */
1159 if (TREE_CODE (vect
) == SSA_NAME
)
1161 gimple
*def_stmt
= SSA_NAME_DEF_STMT (vect
);
1162 if (is_gimple_assign (def_stmt
)
1163 && (gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
1164 || gimple_assign_rhs_code (def_stmt
) == CONSTRUCTOR
))
1165 vect
= gimple_assign_rhs1 (def_stmt
);
1168 if (TREE_CODE (vect
) == VECTOR_CST
)
1169 return VECTOR_CST_ELT (vect
, index
);
1170 else if (TREE_CODE (vect
) == CONSTRUCTOR
1171 && (CONSTRUCTOR_NELTS (vect
) == 0
1172 || TREE_CODE (TREE_TYPE (CONSTRUCTOR_ELT (vect
, 0)->value
))
1175 if (index
< CONSTRUCTOR_NELTS (vect
))
1176 return CONSTRUCTOR_ELT (vect
, index
)->value
;
1177 return build_zero_cst (vect_elt_type
);
1181 tree size
= TYPE_SIZE (vect_elt_type
);
1182 tree pos
= fold_build2 (MULT_EXPR
, bitsizetype
, bitsize_int (index
),
1184 return fold_build3 (BIT_FIELD_REF
, vect_elt_type
, vect
, size
, pos
);
1189 tmpvec
= create_tmp_var (vect_type
, "vectmp");
1191 tmpvec
= *ptmpvec
= create_tmp_var (vect_type
, "vectmp");
1200 TREE_ADDRESSABLE (tmpvec
) = 1;
1201 asgn
= gimple_build_assign (tmpvec
, vect
);
1202 gsi_insert_before (gsi
, asgn
, GSI_SAME_STMT
);
1205 arraytype
= build_array_type_nelts (vect_elt_type
, elements
);
1206 return build4 (ARRAY_REF
, vect_elt_type
,
1207 build1 (VIEW_CONVERT_EXPR
, arraytype
, tmpvec
),
1208 idx
, NULL_TREE
, NULL_TREE
);
1211 /* Check if VEC_PERM_EXPR within the given setting is supported
1212 by hardware, or lower it piecewise.
1214 When VEC_PERM_EXPR has the same first and second operands:
1215 VEC_PERM_EXPR <v0, v0, mask> the lowered version would be
1216 {v0[mask[0]], v0[mask[1]], ...}
1217 MASK and V0 must have the same number of elements.
1219 Otherwise VEC_PERM_EXPR <v0, v1, mask> is lowered to
1220 {mask[0] < len(v0) ? v0[mask[0]] : v1[mask[0]], ...}
1221 V0 and V1 must have the same type. MASK, V0, V1 must have the
1222 same number of arguments. */
1225 lower_vec_perm (gimple_stmt_iterator
*gsi
)
1227 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1228 tree mask
= gimple_assign_rhs3 (stmt
);
1229 tree vec0
= gimple_assign_rhs1 (stmt
);
1230 tree vec1
= gimple_assign_rhs2 (stmt
);
1231 tree vect_type
= TREE_TYPE (vec0
);
1232 tree mask_type
= TREE_TYPE (mask
);
1233 tree vect_elt_type
= TREE_TYPE (vect_type
);
1234 tree mask_elt_type
= TREE_TYPE (mask_type
);
1235 unsigned int elements
= TYPE_VECTOR_SUBPARTS (vect_type
);
1236 vec
<constructor_elt
, va_gc
> *v
;
1237 tree constr
, t
, si
, i_val
;
1238 tree vec0tmp
= NULL_TREE
, vec1tmp
= NULL_TREE
, masktmp
= NULL_TREE
;
1239 bool two_operand_p
= !operand_equal_p (vec0
, vec1
, 0);
1240 location_t loc
= gimple_location (gsi_stmt (*gsi
));
1243 if (TREE_CODE (mask
) == SSA_NAME
)
1245 gimple
*def_stmt
= SSA_NAME_DEF_STMT (mask
);
1246 if (is_gimple_assign (def_stmt
)
1247 && gimple_assign_rhs_code (def_stmt
) == VECTOR_CST
)
1248 mask
= gimple_assign_rhs1 (def_stmt
);
1251 if (TREE_CODE (mask
) == VECTOR_CST
)
1253 unsigned char *sel_int
= XALLOCAVEC (unsigned char, elements
);
1255 for (i
= 0; i
< elements
; ++i
)
1256 sel_int
[i
] = (TREE_INT_CST_LOW (VECTOR_CST_ELT (mask
, i
))
1257 & (2 * elements
- 1));
1259 if (can_vec_perm_p (TYPE_MODE (vect_type
), false, sel_int
))
1261 gimple_assign_set_rhs3 (stmt
, mask
);
1266 else if (can_vec_perm_p (TYPE_MODE (vect_type
), true, NULL
))
1269 warning_at (loc
, OPT_Wvector_operation_performance
,
1270 "vector shuffling operation will be expanded piecewise");
1272 vec_alloc (v
, elements
);
1273 for (i
= 0; i
< elements
; i
++)
1276 i_val
= vector_element (gsi
, mask
, si
, &masktmp
);
1278 if (TREE_CODE (i_val
) == INTEGER_CST
)
1280 unsigned HOST_WIDE_INT index
;
1282 index
= TREE_INT_CST_LOW (i_val
);
1283 if (!tree_fits_uhwi_p (i_val
) || index
>= elements
)
1284 i_val
= build_int_cst (mask_elt_type
, index
& (elements
- 1));
1286 if (two_operand_p
&& (index
& elements
) != 0)
1287 t
= vector_element (gsi
, vec1
, i_val
, &vec1tmp
);
1289 t
= vector_element (gsi
, vec0
, i_val
, &vec0tmp
);
1291 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
1292 true, GSI_SAME_STMT
);
1296 tree cond
= NULL_TREE
, v0_val
;
1300 cond
= fold_build2 (BIT_AND_EXPR
, mask_elt_type
, i_val
,
1301 build_int_cst (mask_elt_type
, elements
));
1302 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
,
1303 true, GSI_SAME_STMT
);
1306 i_val
= fold_build2 (BIT_AND_EXPR
, mask_elt_type
, i_val
,
1307 build_int_cst (mask_elt_type
, elements
- 1));
1308 i_val
= force_gimple_operand_gsi (gsi
, i_val
, true, NULL_TREE
,
1309 true, GSI_SAME_STMT
);
1311 v0_val
= vector_element (gsi
, vec0
, i_val
, &vec0tmp
);
1312 v0_val
= force_gimple_operand_gsi (gsi
, v0_val
, true, NULL_TREE
,
1313 true, GSI_SAME_STMT
);
1319 v1_val
= vector_element (gsi
, vec1
, i_val
, &vec1tmp
);
1320 v1_val
= force_gimple_operand_gsi (gsi
, v1_val
, true, NULL_TREE
,
1321 true, GSI_SAME_STMT
);
1323 cond
= fold_build2 (EQ_EXPR
, boolean_type_node
,
1324 cond
, build_zero_cst (mask_elt_type
));
1325 cond
= fold_build3 (COND_EXPR
, vect_elt_type
,
1326 cond
, v0_val
, v1_val
);
1327 t
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
,
1328 true, GSI_SAME_STMT
);
1334 CONSTRUCTOR_APPEND_ELT (v
, NULL_TREE
, t
);
1337 constr
= build_constructor (vect_type
, v
);
1338 gimple_assign_set_rhs_from_tree (gsi
, constr
);
1339 update_stmt (gsi_stmt (*gsi
));
1342 /* If OP is a uniform vector return the element it is a splat from. */
1345 ssa_uniform_vector_p (tree op
)
1347 if (TREE_CODE (op
) == VECTOR_CST
1348 || TREE_CODE (op
) == CONSTRUCTOR
)
1349 return uniform_vector_p (op
);
1350 if (TREE_CODE (op
) == SSA_NAME
)
1352 gimple
*def_stmt
= SSA_NAME_DEF_STMT (op
);
1353 if (gimple_assign_single_p (def_stmt
))
1354 return uniform_vector_p (gimple_assign_rhs1 (def_stmt
));
1359 /* Return type in which CODE operation with optab OP can be
1363 get_compute_type (enum tree_code code
, optab op
, tree type
)
1365 /* For very wide vectors, try using a smaller vector mode. */
1366 tree compute_type
= type
;
1368 && (!VECTOR_MODE_P (TYPE_MODE (type
))
1369 || optab_handler (op
, TYPE_MODE (type
)) == CODE_FOR_nothing
))
1371 tree vector_compute_type
1372 = type_for_widest_vector_mode (TREE_TYPE (type
), op
);
1373 if (vector_compute_type
!= NULL_TREE
1374 && (TYPE_VECTOR_SUBPARTS (vector_compute_type
)
1375 < TYPE_VECTOR_SUBPARTS (compute_type
))
1376 && (optab_handler (op
, TYPE_MODE (vector_compute_type
))
1377 != CODE_FOR_nothing
))
1378 compute_type
= vector_compute_type
;
1381 /* If we are breaking a BLKmode vector into smaller pieces,
1382 type_for_widest_vector_mode has already looked into the optab,
1383 so skip these checks. */
1384 if (compute_type
== type
)
1386 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1387 if (VECTOR_MODE_P (compute_mode
))
1389 if (op
&& optab_handler (op
, compute_mode
) != CODE_FOR_nothing
)
1390 return compute_type
;
1391 if (code
== MULT_HIGHPART_EXPR
1392 && can_mult_highpart_p (compute_mode
,
1393 TYPE_UNSIGNED (compute_type
)))
1394 return compute_type
;
1396 /* There is no operation in hardware, so fall back to scalars. */
1397 compute_type
= TREE_TYPE (type
);
1400 return compute_type
;
1403 /* Helper function of expand_vector_operations_1. Return number of
1404 vector elements for vector types or 1 for other types. */
1407 count_type_subparts (tree type
)
1409 return VECTOR_TYPE_P (type
) ? TYPE_VECTOR_SUBPARTS (type
) : 1;
1413 do_cond (gimple_stmt_iterator
*gsi
, tree inner_type
, tree a
, tree b
,
1414 tree bitpos
, tree bitsize
, enum tree_code code
,
1415 tree type ATTRIBUTE_UNUSED
)
1417 if (TREE_CODE (TREE_TYPE (a
)) == VECTOR_TYPE
)
1418 a
= tree_vec_extract (gsi
, inner_type
, a
, bitsize
, bitpos
);
1419 if (TREE_CODE (TREE_TYPE (b
)) == VECTOR_TYPE
)
1420 b
= tree_vec_extract (gsi
, inner_type
, b
, bitsize
, bitpos
);
1421 tree cond
= gimple_assign_rhs1 (gsi_stmt (*gsi
));
1422 return gimplify_build3 (gsi
, code
, inner_type
, cond
, a
, b
);
1425 /* Expand a vector COND_EXPR to scalars, piecewise. */
1427 expand_vector_scalar_condition (gimple_stmt_iterator
*gsi
)
1429 gassign
*stmt
= as_a
<gassign
*> (gsi_stmt (*gsi
));
1430 tree type
= gimple_expr_type (stmt
);
1431 tree compute_type
= get_compute_type (COND_EXPR
, mov_optab
, type
);
1432 machine_mode compute_mode
= TYPE_MODE (compute_type
);
1433 gcc_assert (compute_mode
!= BLKmode
);
1434 tree lhs
= gimple_assign_lhs (stmt
);
1435 tree rhs2
= gimple_assign_rhs2 (stmt
);
1436 tree rhs3
= gimple_assign_rhs3 (stmt
);
1439 /* If the compute mode is not a vector mode (hence we are not decomposing
1440 a BLKmode vector to smaller, hardware-supported vectors), we may want
1441 to expand the operations in parallel. */
1442 if (GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_INT
1443 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_FLOAT
1444 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_FRACT
1445 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_UFRACT
1446 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_ACCUM
1447 && GET_MODE_CLASS (compute_mode
) != MODE_VECTOR_UACCUM
)
1448 new_rhs
= expand_vector_parallel (gsi
, do_cond
, type
, rhs2
, rhs3
,
1451 new_rhs
= expand_vector_piecewise (gsi
, do_cond
, type
, compute_type
,
1452 rhs2
, rhs3
, COND_EXPR
);
1453 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (new_rhs
)))
1454 new_rhs
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
),
1457 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
1458 way to do it is change expand_vector_operation and its callees to
1459 return a tree_code, RHS1 and RHS2 instead of a tree. */
1460 gimple_assign_set_rhs_from_tree (gsi
, new_rhs
);
1461 update_stmt (gsi_stmt (*gsi
));
1464 /* Process one statement. If we identify a vector operation, expand it. */
1467 expand_vector_operations_1 (gimple_stmt_iterator
*gsi
)
1469 tree lhs
, rhs1
, rhs2
= NULL
, type
, compute_type
= NULL_TREE
;
1470 enum tree_code code
;
1471 optab op
= unknown_optab
;
1472 enum gimple_rhs_class rhs_class
;
1475 /* Only consider code == GIMPLE_ASSIGN. */
1476 gassign
*stmt
= dyn_cast
<gassign
*> (gsi_stmt (*gsi
));
1480 code
= gimple_assign_rhs_code (stmt
);
1481 rhs_class
= get_gimple_rhs_class (code
);
1482 lhs
= gimple_assign_lhs (stmt
);
1484 if (code
== VEC_PERM_EXPR
)
1486 lower_vec_perm (gsi
);
1490 if (code
== VEC_COND_EXPR
)
1492 expand_vector_condition (gsi
);
1496 if (code
== COND_EXPR
1497 && TREE_CODE (TREE_TYPE (gimple_assign_lhs (stmt
))) == VECTOR_TYPE
1498 && TYPE_MODE (TREE_TYPE (gimple_assign_lhs (stmt
))) == BLKmode
)
1500 expand_vector_scalar_condition (gsi
);
1504 if (code
== CONSTRUCTOR
1505 && TREE_CODE (lhs
) == SSA_NAME
1506 && VECTOR_MODE_P (TYPE_MODE (TREE_TYPE (lhs
)))
1507 && !gimple_clobber_p (stmt
)
1510 optimize_vector_constructor (gsi
);
1514 if (rhs_class
!= GIMPLE_UNARY_RHS
&& rhs_class
!= GIMPLE_BINARY_RHS
)
1517 rhs1
= gimple_assign_rhs1 (stmt
);
1518 type
= gimple_expr_type (stmt
);
1519 if (rhs_class
== GIMPLE_BINARY_RHS
)
1520 rhs2
= gimple_assign_rhs2 (stmt
);
1522 if (TREE_CODE (type
) != VECTOR_TYPE
)
1525 /* If the vector operation is operating on all same vector elements
1526 implement it with a scalar operation and a splat if the target
1527 supports the scalar operation. */
1528 tree srhs1
, srhs2
= NULL_TREE
;
1529 if ((srhs1
= ssa_uniform_vector_p (rhs1
)) != NULL_TREE
1530 && (rhs2
== NULL_TREE
1531 || (! VECTOR_TYPE_P (TREE_TYPE (rhs2
))
1533 || (srhs2
= ssa_uniform_vector_p (rhs2
)) != NULL_TREE
)
1534 /* As we query direct optabs restrict to non-convert operations. */
1535 && TYPE_MODE (TREE_TYPE (type
)) == TYPE_MODE (TREE_TYPE (srhs1
)))
1537 op
= optab_for_tree_code (code
, TREE_TYPE (type
), optab_scalar
);
1538 if (op
>= FIRST_NORM_OPTAB
&& op
<= LAST_NORM_OPTAB
1539 && optab_handler (op
, TYPE_MODE (TREE_TYPE (type
))) != CODE_FOR_nothing
)
1541 tree slhs
= make_ssa_name (TREE_TYPE (srhs1
));
1542 gimple
*repl
= gimple_build_assign (slhs
, code
, srhs1
, srhs2
);
1543 gsi_insert_before (gsi
, repl
, GSI_SAME_STMT
);
1544 gimple_assign_set_rhs_from_tree (gsi
,
1545 build_vector_from_val (type
, slhs
));
1551 /* A scalar operation pretending to be a vector one. */
1552 if (VECTOR_BOOLEAN_TYPE_P (type
)
1553 && !VECTOR_MODE_P (TYPE_MODE (type
))
1554 && TYPE_MODE (type
) != BLKmode
)
1557 if (CONVERT_EXPR_CODE_P (code
)
1558 || code
== FLOAT_EXPR
1559 || code
== FIX_TRUNC_EXPR
1560 || code
== VIEW_CONVERT_EXPR
)
1563 /* The signedness is determined from input argument. */
1564 if (code
== VEC_UNPACK_FLOAT_HI_EXPR
1565 || code
== VEC_UNPACK_FLOAT_LO_EXPR
)
1566 type
= TREE_TYPE (rhs1
);
1568 /* For widening/narrowing vector operations, the relevant type is of the
1569 arguments, not the widened result. VEC_UNPACK_FLOAT_*_EXPR is
1570 calculated in the same way above. */
1571 if (code
== WIDEN_SUM_EXPR
1572 || code
== VEC_WIDEN_MULT_HI_EXPR
1573 || code
== VEC_WIDEN_MULT_LO_EXPR
1574 || code
== VEC_WIDEN_MULT_EVEN_EXPR
1575 || code
== VEC_WIDEN_MULT_ODD_EXPR
1576 || code
== VEC_UNPACK_HI_EXPR
1577 || code
== VEC_UNPACK_LO_EXPR
1578 || code
== VEC_PACK_TRUNC_EXPR
1579 || code
== VEC_PACK_SAT_EXPR
1580 || code
== VEC_PACK_FIX_TRUNC_EXPR
1581 || code
== VEC_WIDEN_LSHIFT_HI_EXPR
1582 || code
== VEC_WIDEN_LSHIFT_LO_EXPR
)
1583 type
= TREE_TYPE (rhs1
);
1585 /* Choose between vector shift/rotate by vector and vector shift/rotate by
1587 if (code
== LSHIFT_EXPR
1588 || code
== RSHIFT_EXPR
1589 || code
== LROTATE_EXPR
1590 || code
== RROTATE_EXPR
)
1594 /* Check whether we have vector <op> {x,x,x,x} where x
1595 could be a scalar variable or a constant. Transform
1596 vector <op> {x,x,x,x} ==> vector <op> scalar. */
1597 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
1601 if ((first
= ssa_uniform_vector_p (rhs2
)) != NULL_TREE
)
1603 gimple_assign_set_rhs2 (stmt
, first
);
1609 opv
= optab_for_tree_code (code
, type
, optab_vector
);
1610 if (VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
1614 op
= optab_for_tree_code (code
, type
, optab_scalar
);
1616 compute_type
= get_compute_type (code
, op
, type
);
1617 if (compute_type
== type
)
1619 /* The rtl expander will expand vector/scalar as vector/vector
1620 if necessary. Pick one with wider vector type. */
1621 tree compute_vtype
= get_compute_type (code
, opv
, type
);
1622 if (count_type_subparts (compute_vtype
)
1623 > count_type_subparts (compute_type
))
1625 compute_type
= compute_vtype
;
1630 if (code
== LROTATE_EXPR
|| code
== RROTATE_EXPR
)
1632 if (compute_type
== NULL_TREE
)
1633 compute_type
= get_compute_type (code
, op
, type
);
1634 if (compute_type
== type
)
1636 /* Before splitting vector rotates into scalar rotates,
1637 see if we can't use vector shifts and BIT_IOR_EXPR
1638 instead. For vector by vector rotates we'd also
1639 need to check BIT_AND_EXPR and NEGATE_EXPR, punt there
1640 for now, fold doesn't seem to create such rotates anyway. */
1641 if (compute_type
== TREE_TYPE (type
)
1642 && !VECTOR_INTEGER_TYPE_P (TREE_TYPE (rhs2
)))
1644 optab oplv
= vashl_optab
, opl
= ashl_optab
;
1645 optab oprv
= vlshr_optab
, opr
= lshr_optab
, opo
= ior_optab
;
1646 tree compute_lvtype
= get_compute_type (LSHIFT_EXPR
, oplv
, type
);
1647 tree compute_rvtype
= get_compute_type (RSHIFT_EXPR
, oprv
, type
);
1648 tree compute_otype
= get_compute_type (BIT_IOR_EXPR
, opo
, type
);
1649 tree compute_ltype
= get_compute_type (LSHIFT_EXPR
, opl
, type
);
1650 tree compute_rtype
= get_compute_type (RSHIFT_EXPR
, opr
, type
);
1651 /* The rtl expander will expand vector/scalar as vector/vector
1652 if necessary. Pick one with wider vector type. */
1653 if (count_type_subparts (compute_lvtype
)
1654 > count_type_subparts (compute_ltype
))
1656 compute_ltype
= compute_lvtype
;
1659 if (count_type_subparts (compute_rvtype
)
1660 > count_type_subparts (compute_rtype
))
1662 compute_rtype
= compute_rvtype
;
1665 /* Pick the narrowest type from LSHIFT_EXPR, RSHIFT_EXPR and
1667 compute_type
= compute_ltype
;
1668 if (count_type_subparts (compute_type
)
1669 > count_type_subparts (compute_rtype
))
1670 compute_type
= compute_rtype
;
1671 if (count_type_subparts (compute_type
)
1672 > count_type_subparts (compute_otype
))
1673 compute_type
= compute_otype
;
1674 /* Verify all 3 operations can be performed in that type. */
1675 if (compute_type
!= TREE_TYPE (type
))
1677 if (optab_handler (opl
, TYPE_MODE (compute_type
))
1679 || optab_handler (opr
, TYPE_MODE (compute_type
))
1681 || optab_handler (opo
, TYPE_MODE (compute_type
))
1682 == CODE_FOR_nothing
)
1683 compute_type
= TREE_TYPE (type
);
1689 op
= optab_for_tree_code (code
, type
, optab_default
);
1691 /* Optabs will try converting a negation into a subtraction, so
1692 look for it as well. TODO: negation of floating-point vectors
1693 might be turned into an exclusive OR toggling the sign bit. */
1694 if (op
== unknown_optab
1695 && code
== NEGATE_EXPR
1696 && INTEGRAL_TYPE_P (TREE_TYPE (type
)))
1697 op
= optab_for_tree_code (MINUS_EXPR
, type
, optab_default
);
1699 if (compute_type
== NULL_TREE
)
1700 compute_type
= get_compute_type (code
, op
, type
);
1701 if (compute_type
== type
)
1704 new_rhs
= expand_vector_operation (gsi
, type
, compute_type
, stmt
, code
);
1706 /* Leave expression untouched for later expansion. */
1707 if (new_rhs
== NULL_TREE
)
1710 if (!useless_type_conversion_p (TREE_TYPE (lhs
), TREE_TYPE (new_rhs
)))
1711 new_rhs
= gimplify_build1 (gsi
, VIEW_CONVERT_EXPR
, TREE_TYPE (lhs
),
1714 /* NOTE: We should avoid using gimple_assign_set_rhs_from_tree. One
1715 way to do it is change expand_vector_operation and its callees to
1716 return a tree_code, RHS1 and RHS2 instead of a tree. */
1717 gimple_assign_set_rhs_from_tree (gsi
, new_rhs
);
1718 update_stmt (gsi_stmt (*gsi
));
1721 /* Use this to lower vector operations introduced by the vectorizer,
1722 if it may need the bit-twiddling tricks implemented in this file. */
1725 expand_vector_operations (void)
1727 gimple_stmt_iterator gsi
;
1729 bool cfg_changed
= false;
1731 FOR_EACH_BB_FN (bb
, cfun
)
1733 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1735 expand_vector_operations_1 (&gsi
);
1736 /* ??? If we do not cleanup EH then we will ICE in
1737 verification. But in reality we have created wrong-code
1738 as we did not properly transition EH info and edges to
1739 the piecewise computations. */
1740 if (maybe_clean_eh_stmt (gsi_stmt (gsi
))
1741 && gimple_purge_dead_eh_edges (bb
))
1746 return cfg_changed
? TODO_cleanup_cfg
: 0;
1751 const pass_data pass_data_lower_vector
=
1753 GIMPLE_PASS
, /* type */
1754 "veclower", /* name */
1755 OPTGROUP_VEC
, /* optinfo_flags */
1756 TV_NONE
, /* tv_id */
1757 PROP_cfg
, /* properties_required */
1758 PROP_gimple_lvec
, /* properties_provided */
1759 0, /* properties_destroyed */
1760 0, /* todo_flags_start */
1761 TODO_update_ssa
, /* todo_flags_finish */
1764 class pass_lower_vector
: public gimple_opt_pass
1767 pass_lower_vector (gcc::context
*ctxt
)
1768 : gimple_opt_pass (pass_data_lower_vector
, ctxt
)
1771 /* opt_pass methods: */
1772 virtual bool gate (function
*fun
)
1774 return !(fun
->curr_properties
& PROP_gimple_lvec
);
1777 virtual unsigned int execute (function
*)
1779 return expand_vector_operations ();
1782 }; // class pass_lower_vector
1787 make_pass_lower_vector (gcc::context
*ctxt
)
1789 return new pass_lower_vector (ctxt
);
1794 const pass_data pass_data_lower_vector_ssa
=
1796 GIMPLE_PASS
, /* type */
1797 "veclower2", /* name */
1798 OPTGROUP_VEC
, /* optinfo_flags */
1799 TV_NONE
, /* tv_id */
1800 PROP_cfg
, /* properties_required */
1801 PROP_gimple_lvec
, /* properties_provided */
1802 0, /* properties_destroyed */
1803 0, /* todo_flags_start */
1805 | TODO_cleanup_cfg
), /* todo_flags_finish */
1808 class pass_lower_vector_ssa
: public gimple_opt_pass
1811 pass_lower_vector_ssa (gcc::context
*ctxt
)
1812 : gimple_opt_pass (pass_data_lower_vector_ssa
, ctxt
)
1815 /* opt_pass methods: */
1816 opt_pass
* clone () { return new pass_lower_vector_ssa (m_ctxt
); }
1817 virtual unsigned int execute (function
*)
1819 return expand_vector_operations ();
1822 }; // class pass_lower_vector_ssa
1827 make_pass_lower_vector_ssa (gcc::context
*ctxt
)
1829 return new pass_lower_vector_ssa (ctxt
);
1832 #include "gt-tree-vect-generic.h"