1 /* OMP constructs' SIMD clone supporting code.
3 Copyright (C) 2005-2020 Free Software Foundation, Inc.
5 This file is part of GCC.
7 GCC is free software; you can redistribute it and/or modify it under
8 the terms of the GNU General Public License as published by the Free
9 Software Foundation; either version 3, or (at your option) any later
12 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
17 You should have received a copy of the GNU General Public License
18 along with GCC; see the file COPYING3. If not see
19 <http://www.gnu.org/licenses/>. */
23 #include "coretypes.h"
29 #include "alloc-pool.h"
30 #include "tree-pass.h"
33 #include "pretty-print.h"
34 #include "diagnostic-core.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
39 #include "gimple-iterator.h"
40 #include "gimplify-me.h"
41 #include "gimple-walk.h"
42 #include "langhooks.h"
44 #include "tree-into-ssa.h"
47 #include "symbol-summary.h"
48 #include "ipa-param-manipulation.h"
51 #include "stringpool.h"
53 #include "omp-simd-clone.h"
55 /* Return the number of elements in vector type VECTYPE, which is associated
56 with a SIMD clone. At present these always have a constant length. */
58 static unsigned HOST_WIDE_INT
59 simd_clone_subparts (tree vectype
)
61 return TYPE_VECTOR_SUBPARTS (vectype
).to_constant ();
64 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
65 of arguments to reserve space for. */
67 static struct cgraph_simd_clone
*
68 simd_clone_struct_alloc (int nargs
)
70 struct cgraph_simd_clone
*clone_info
;
71 size_t len
= (sizeof (struct cgraph_simd_clone
)
72 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
73 clone_info
= (struct cgraph_simd_clone
*)
74 ggc_internal_cleared_alloc (len
);
78 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
81 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
82 struct cgraph_simd_clone
*from
)
84 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
85 + ((from
->nargs
- from
->inbranch
)
86 * sizeof (struct cgraph_simd_clone_arg
))));
89 /* Fill an empty vector ARGS with parameter types of function FNDECL. This
90 uses TYPE_ARG_TYPES if available, otherwise falls back to types of
91 DECL_ARGUMENTS types. */
94 simd_clone_vector_of_formal_parm_types (vec
<tree
> *args
, tree fndecl
)
96 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
98 push_function_arg_types (args
, TREE_TYPE (fndecl
));
101 push_function_arg_decls (args
, fndecl
);
104 FOR_EACH_VEC_ELT (*args
, i
, arg
)
105 (*args
)[i
] = TREE_TYPE ((*args
)[i
]);
108 /* Given a simd function in NODE, extract the simd specific
109 information from the OMP clauses passed in CLAUSES, and return
110 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
111 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
112 otherwise set to FALSE. */
114 static struct cgraph_simd_clone
*
115 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
116 bool *inbranch_specified
)
119 simd_clone_vector_of_formal_parm_types (&args
, node
->decl
);
122 *inbranch_specified
= false;
125 if (n
> 0 && args
.last () == void_type_node
)
128 /* Allocate one more than needed just in case this is an in-branch
129 clone which will require a mask argument. */
130 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
131 clone_info
->nargs
= n
;
136 clauses
= TREE_VALUE (clauses
);
137 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
140 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
142 switch (OMP_CLAUSE_CODE (t
))
144 case OMP_CLAUSE_INBRANCH
:
145 clone_info
->inbranch
= 1;
146 *inbranch_specified
= true;
148 case OMP_CLAUSE_NOTINBRANCH
:
149 clone_info
->inbranch
= 0;
150 *inbranch_specified
= true;
152 case OMP_CLAUSE_SIMDLEN
:
154 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
156 case OMP_CLAUSE_LINEAR
:
158 tree decl
= OMP_CLAUSE_DECL (t
);
159 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
160 int argno
= TREE_INT_CST_LOW (decl
);
161 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
163 enum cgraph_simd_clone_arg_type arg_type
;
164 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
165 switch (OMP_CLAUSE_LINEAR_KIND (t
))
167 case OMP_CLAUSE_LINEAR_REF
:
169 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
;
171 case OMP_CLAUSE_LINEAR_UVAL
:
173 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
;
175 case OMP_CLAUSE_LINEAR_VAL
:
176 case OMP_CLAUSE_LINEAR_DEFAULT
:
178 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
;
184 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
185 clone_info
->args
[argno
].arg_type
= arg_type
;
186 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
187 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
188 && clone_info
->args
[argno
].linear_step
< n
);
192 if (POINTER_TYPE_P (args
[argno
]))
193 step
= fold_convert (ssizetype
, step
);
194 if (!tree_fits_shwi_p (step
))
196 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
197 "ignoring large linear step");
200 else if (integer_zerop (step
))
202 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
203 "ignoring zero linear step");
208 enum cgraph_simd_clone_arg_type arg_type
;
209 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
210 switch (OMP_CLAUSE_LINEAR_KIND (t
))
212 case OMP_CLAUSE_LINEAR_REF
:
214 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
;
216 case OMP_CLAUSE_LINEAR_UVAL
:
218 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
;
220 case OMP_CLAUSE_LINEAR_VAL
:
221 case OMP_CLAUSE_LINEAR_DEFAULT
:
223 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
;
229 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
230 clone_info
->args
[argno
].arg_type
= arg_type
;
231 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
236 case OMP_CLAUSE_UNIFORM
:
238 tree decl
= OMP_CLAUSE_DECL (t
);
239 int argno
= tree_to_uhwi (decl
);
240 clone_info
->args
[argno
].arg_type
241 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
244 case OMP_CLAUSE_ALIGNED
:
246 /* Ignore aligned (x) for declare simd, for the ABI we really
247 need an alignment specified. */
248 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
) == NULL_TREE
)
250 tree decl
= OMP_CLAUSE_DECL (t
);
251 int argno
= tree_to_uhwi (decl
);
252 clone_info
->args
[argno
].alignment
253 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
262 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node
->decl
))))
264 warning_at (DECL_SOURCE_LOCATION (node
->decl
), 0,
265 "ignoring %<#pragma omp declare simd%> on function "
266 "with %<_Atomic%> qualified return type");
270 for (unsigned int argno
= 0; argno
< clone_info
->nargs
; argno
++)
271 if (TYPE_ATOMIC (args
[argno
])
272 && clone_info
->args
[argno
].arg_type
!= SIMD_CLONE_ARG_TYPE_UNIFORM
)
274 warning_at (DECL_SOURCE_LOCATION (node
->decl
), 0,
275 "ignoring %<#pragma omp declare simd%> on function "
276 "with %<_Atomic%> qualified non-%<uniform%> argument");
284 /* Given a SIMD clone in NODE, calculate the characteristic data
285 type and return the coresponding type. The characteristic data
286 type is computed as described in the Intel Vector ABI. */
289 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
290 struct cgraph_simd_clone
*clone_info
)
292 tree type
= integer_type_node
;
293 tree fndecl
= node
->decl
;
295 /* a) For non-void function, the characteristic data type is the
297 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
298 type
= TREE_TYPE (TREE_TYPE (fndecl
));
300 /* b) If the function has any non-uniform, non-linear parameters,
301 then the characteristic data type is the type of the first
306 simd_clone_vector_of_formal_parm_types (&map
, fndecl
);
307 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
308 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
315 /* c) If the characteristic data type determined by a) or b) above
316 is struct, union, or class type which is pass-by-value (except
317 for the type that maps to the built-in complex data type), the
318 characteristic data type is int. */
319 if (RECORD_OR_UNION_TYPE_P (type
)
320 && !aggregate_value_p (type
, NULL
)
321 && TREE_CODE (type
) != COMPLEX_TYPE
)
322 return integer_type_node
;
324 /* d) If none of the above three classes is applicable, the
325 characteristic data type is int. */
329 /* e) For Intel Xeon Phi native and offload compilation, if the
330 resulting characteristic data type is 8-bit or 16-bit integer
331 data type, the characteristic data type is int. */
332 /* Well, we don't handle Xeon Phi yet. */
336 simd_clone_mangle (struct cgraph_node
*node
,
337 struct cgraph_simd_clone
*clone_info
)
339 char vecsize_mangle
= clone_info
->vecsize_mangle
;
340 char mask
= clone_info
->inbranch
? 'M' : 'N';
341 unsigned int simdlen
= clone_info
->simdlen
;
345 gcc_assert (vecsize_mangle
&& simdlen
);
347 pp_string (&pp
, "_ZGV");
348 pp_character (&pp
, vecsize_mangle
);
349 pp_character (&pp
, mask
);
350 pp_decimal_int (&pp
, simdlen
);
352 for (n
= 0; n
< clone_info
->nargs
; ++n
)
354 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
356 switch (arg
.arg_type
)
358 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
359 pp_character (&pp
, 'u');
361 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
362 pp_character (&pp
, 'l');
364 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
365 pp_character (&pp
, 'R');
367 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
368 pp_character (&pp
, 'L');
370 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
371 pp_character (&pp
, 'U');
374 gcc_assert (arg
.linear_step
!= 0);
375 if (arg
.linear_step
> 1)
376 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
377 else if (arg
.linear_step
< 0)
379 pp_character (&pp
, 'n');
380 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
384 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
385 pp_string (&pp
, "ls");
386 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
388 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
389 pp_string (&pp
, "Rs");
390 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
392 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
393 pp_string (&pp
, "Ls");
394 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
396 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
397 pp_string (&pp
, "Us");
398 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
401 pp_character (&pp
, 'v');
405 pp_character (&pp
, 'a');
406 pp_decimal_int (&pp
, arg
.alignment
);
411 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
414 pp_string (&pp
, str
);
415 str
= pp_formatted_text (&pp
);
417 /* If there already is a SIMD clone with the same mangled name, don't
418 add another one. This can happen e.g. for
419 #pragma omp declare simd
420 #pragma omp declare simd simdlen(8)
422 if the simdlen is assumed to be 8 for the first one, etc. */
423 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
424 clone
= clone
->simdclone
->next_clone
)
425 if (id_equal (DECL_ASSEMBLER_NAME (clone
->decl
), str
))
428 return get_identifier (str
);
431 /* Create a simd clone of OLD_NODE and return it. */
433 static struct cgraph_node
*
434 simd_clone_create (struct cgraph_node
*old_node
)
436 struct cgraph_node
*new_node
;
437 if (old_node
->definition
)
439 if (!old_node
->has_gimple_body_p ())
441 old_node
->get_body ();
442 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
448 tree old_decl
= old_node
->decl
;
449 tree new_decl
= copy_node (old_node
->decl
);
450 DECL_NAME (new_decl
) = clone_function_name_numbered (old_decl
,
452 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
453 SET_DECL_RTL (new_decl
, NULL
);
454 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
455 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
456 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
457 if (old_node
->in_other_partition
)
458 new_node
->in_other_partition
= 1;
460 if (new_node
== NULL
)
463 set_decl_built_in_function (new_node
->decl
, NOT_BUILT_IN
, 0);
464 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
465 DECL_COMDAT (new_node
->decl
) = DECL_COMDAT (old_node
->decl
);
466 DECL_WEAK (new_node
->decl
) = DECL_WEAK (old_node
->decl
);
467 DECL_EXTERNAL (new_node
->decl
) = DECL_EXTERNAL (old_node
->decl
);
468 DECL_VISIBILITY_SPECIFIED (new_node
->decl
)
469 = DECL_VISIBILITY_SPECIFIED (old_node
->decl
);
470 DECL_VISIBILITY (new_node
->decl
) = DECL_VISIBILITY (old_node
->decl
);
471 DECL_DLLIMPORT_P (new_node
->decl
) = DECL_DLLIMPORT_P (old_node
->decl
);
472 if (DECL_ONE_ONLY (old_node
->decl
))
473 make_decl_one_only (new_node
->decl
, DECL_ASSEMBLER_NAME (new_node
->decl
));
475 /* The method cgraph_version_clone_with_body () will force the new
476 symbol local. Undo this, and inherit external visibility from
478 new_node
->local
= old_node
->local
;
479 new_node
->externally_visible
= old_node
->externally_visible
;
484 /* Adjust the return type of the given function to its appropriate
485 vector counterpart. Returns a simd array to be used throughout the
486 function as a return value. */
489 simd_clone_adjust_return_type (struct cgraph_node
*node
)
491 tree fndecl
= node
->decl
;
492 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
496 /* Adjust the function return type. */
497 if (orig_rettype
== void_type_node
)
499 t
= TREE_TYPE (TREE_TYPE (fndecl
));
500 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
501 veclen
= node
->simdclone
->vecsize_int
;
503 veclen
= node
->simdclone
->vecsize_float
;
504 veclen
/= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (t
));
505 if (veclen
> node
->simdclone
->simdlen
)
506 veclen
= node
->simdclone
->simdlen
;
507 if (POINTER_TYPE_P (t
))
508 t
= pointer_sized_int_node
;
509 if (veclen
== node
->simdclone
->simdlen
)
510 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
513 t
= build_vector_type (t
, veclen
);
514 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
516 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
517 if (!node
->definition
)
520 t
= DECL_RESULT (fndecl
);
521 /* Adjust the DECL_RESULT. */
522 gcc_assert (TREE_TYPE (t
) != void_type_node
);
523 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
526 tree atype
= build_array_type_nelts (orig_rettype
,
527 node
->simdclone
->simdlen
);
528 if (veclen
!= node
->simdclone
->simdlen
)
529 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
531 /* Set up a SIMD array to use as the return value. */
532 tree retval
= create_tmp_var_raw (atype
, "retval");
533 gimple_add_tmp_var (retval
);
537 /* Each vector argument has a corresponding array to be used locally
538 as part of the eventual loop. Create such temporary array and
541 PREFIX is the prefix to be used for the temporary.
543 TYPE is the inner element type.
545 SIMDLEN is the number of elements. */
548 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
550 tree atype
= build_array_type_nelts (type
, simdlen
);
551 tree avar
= create_tmp_var_raw (atype
, prefix
);
552 gimple_add_tmp_var (avar
);
556 /* Modify the function argument types to their corresponding vector
557 counterparts if appropriate. Also, create one array for each simd
558 argument to be used locally when using the function arguments as
561 NODE is the function whose arguments are to be adjusted.
563 If NODE does not represent function definition, returns NULL. Otherwise
564 returns an adjustment class that will be filled describing how the argument
565 declarations will be remapped. New arguments which are not to be remapped
566 are marked with USER_FLAG. */
568 static ipa_param_body_adjustments
*
569 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
573 if (node
->definition
)
574 push_function_arg_decls (&args
, node
->decl
);
576 simd_clone_vector_of_formal_parm_types (&args
, node
->decl
);
577 struct cgraph_simd_clone
*sc
= node
->simdclone
;
578 vec
<ipa_adjusted_param
, va_gc
> *new_params
= NULL
;
579 vec_safe_reserve (new_params
, sc
->nargs
);
580 unsigned i
, j
, veclen
;
582 for (i
= 0; i
< sc
->nargs
; ++i
)
584 ipa_adjusted_param adj
;
585 memset (&adj
, 0, sizeof (adj
));
587 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
589 adj
.prev_clone_index
= i
;
591 sc
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
592 sc
->args
[i
].orig_type
= parm_type
;
594 switch (sc
->args
[i
].arg_type
)
597 /* No adjustment necessary for scalar arguments. */
598 adj
.op
= IPA_PARAM_OP_COPY
;
600 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
601 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
602 if (node
->definition
)
603 sc
->args
[i
].simd_array
604 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
605 TREE_TYPE (parm_type
),
607 adj
.op
= IPA_PARAM_OP_COPY
;
609 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
610 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
611 case SIMD_CLONE_ARG_TYPE_VECTOR
:
612 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
613 veclen
= sc
->vecsize_int
;
615 veclen
= sc
->vecsize_float
;
616 veclen
/= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (parm_type
));
617 if (veclen
> sc
->simdlen
)
618 veclen
= sc
->simdlen
;
619 adj
.op
= IPA_PARAM_OP_NEW
;
620 adj
.param_prefix_index
= IPA_PARAM_PREFIX_SIMD
;
621 if (POINTER_TYPE_P (parm_type
))
622 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
624 adj
.type
= build_vector_type (parm_type
, veclen
);
625 sc
->args
[i
].vector_type
= adj
.type
;
626 for (j
= veclen
; j
< sc
->simdlen
; j
+= veclen
)
628 vec_safe_push (new_params
, adj
);
631 memset (&adj
, 0, sizeof (adj
));
632 adj
.op
= IPA_PARAM_OP_NEW
;
634 adj
.param_prefix_index
= IPA_PARAM_PREFIX_SIMD
;
636 adj
.prev_clone_index
= i
;
637 adj
.type
= sc
->args
[i
].vector_type
;
641 if (node
->definition
)
642 sc
->args
[i
].simd_array
643 = create_tmp_simd_array (DECL_NAME (parm
)
644 ? IDENTIFIER_POINTER (DECL_NAME (parm
))
645 : NULL
, parm_type
, sc
->simdlen
);
647 vec_safe_push (new_params
, adj
);
652 tree base_type
= simd_clone_compute_base_data_type (sc
->origin
, sc
);
653 ipa_adjusted_param adj
;
654 memset (&adj
, 0, sizeof (adj
));
655 adj
.op
= IPA_PARAM_OP_NEW
;
657 adj
.param_prefix_index
= IPA_PARAM_PREFIX_MASK
;
660 adj
.prev_clone_index
= i
;
661 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
662 veclen
= sc
->vecsize_int
;
664 veclen
= sc
->vecsize_float
;
665 veclen
/= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type
));
666 if (veclen
> sc
->simdlen
)
667 veclen
= sc
->simdlen
;
668 if (sc
->mask_mode
!= VOIDmode
)
670 = lang_hooks
.types
.type_for_mode (sc
->mask_mode
, 1);
671 else if (POINTER_TYPE_P (base_type
))
672 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
674 adj
.type
= build_vector_type (base_type
, veclen
);
675 vec_safe_push (new_params
, adj
);
677 for (j
= veclen
; j
< sc
->simdlen
; j
+= veclen
)
678 vec_safe_push (new_params
, adj
);
680 /* We have previously allocated one extra entry for the mask. Use
683 if (sc
->mask_mode
!= VOIDmode
)
684 base_type
= boolean_type_node
;
685 if (node
->definition
)
688 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
689 if (sc
->mask_mode
== VOIDmode
)
690 sc
->args
[i
].simd_array
691 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
692 else if (veclen
< sc
->simdlen
)
693 sc
->args
[i
].simd_array
694 = create_tmp_simd_array ("mask", adj
.type
, sc
->simdlen
/ veclen
);
696 sc
->args
[i
].simd_array
= NULL_TREE
;
698 sc
->args
[i
].orig_type
= base_type
;
699 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
702 if (node
->definition
)
704 ipa_param_body_adjustments
*adjustments
705 = new ipa_param_body_adjustments (new_params
, node
->decl
);
707 adjustments
->modify_formal_parameters ();
712 tree new_arg_types
= NULL_TREE
, new_reversed
;
713 bool last_parm_void
= false;
714 if (args
.length () > 0 && args
.last () == void_type_node
)
715 last_parm_void
= true;
717 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
718 j
= vec_safe_length (new_params
);
719 for (i
= 0; i
< j
; i
++)
721 struct ipa_adjusted_param
*adj
= &(*new_params
)[i
];
723 if (adj
->op
== IPA_PARAM_OP_COPY
)
724 ptype
= args
[adj
->base_index
];
727 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
729 new_reversed
= nreverse (new_arg_types
);
733 TREE_CHAIN (new_arg_types
) = void_list_node
;
735 new_reversed
= void_list_node
;
737 TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) = new_reversed
;
742 /* Initialize and copy the function arguments in NODE to their
743 corresponding local simd arrays. Returns a fresh gimple_seq with
744 the instruction sequence generated. */
747 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
748 ipa_param_body_adjustments
*adjustments
)
750 gimple_seq seq
= NULL
;
751 unsigned i
= 0, j
= 0, k
;
753 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
755 arg
= DECL_CHAIN (arg
), i
++, j
++)
757 if ((*adjustments
->m_adj_params
)[j
].op
== IPA_PARAM_OP_COPY
758 || POINTER_TYPE_P (TREE_TYPE (arg
)))
761 node
->simdclone
->args
[i
].vector_arg
= arg
;
763 tree array
= node
->simdclone
->args
[i
].simd_array
;
764 if (node
->simdclone
->mask_mode
!= VOIDmode
765 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_MASK
)
767 if (array
== NULL_TREE
)
770 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array
))));
771 for (k
= 0; k
<= l
; k
++)
775 arg
= DECL_CHAIN (arg
);
778 tree t
= build4 (ARRAY_REF
, TREE_TYPE (TREE_TYPE (array
)),
779 array
, size_int (k
), NULL
, NULL
);
780 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
781 gimplify_and_add (t
, &seq
);
785 if (simd_clone_subparts (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
787 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
788 tree ptr
= build_fold_addr_expr (array
);
789 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
790 build_int_cst (ptype
, 0));
791 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
792 gimplify_and_add (t
, &seq
);
796 unsigned int simdlen
= simd_clone_subparts (TREE_TYPE (arg
));
797 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
798 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
800 tree ptr
= build_fold_addr_expr (array
);
804 arg
= DECL_CHAIN (arg
);
807 tree elemtype
= TREE_TYPE (TREE_TYPE (arg
));
808 elemsize
= GET_MODE_SIZE (SCALAR_TYPE_MODE (elemtype
));
809 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
810 build_int_cst (ptype
, k
* elemsize
));
811 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
812 gimplify_and_add (t
, &seq
);
819 /* Callback info for ipa_simd_modify_stmt_ops below. */
821 struct modify_stmt_info
{
822 ipa_param_body_adjustments
*adjustments
;
824 /* True if the parent statement was modified by
825 ipa_simd_modify_stmt_ops. */
829 /* Callback for walk_gimple_op.
831 Adjust operands from a given statement as specified in the
832 adjustments vector in the callback data. */
835 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
837 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
838 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
840 if (TREE_CODE (*tp
) == ADDR_EXPR
)
841 tp
= &TREE_OPERAND (*tp
, 0);
843 if (TREE_CODE (*tp
) == BIT_FIELD_REF
844 || TREE_CODE (*tp
) == IMAGPART_EXPR
845 || TREE_CODE (*tp
) == REALPART_EXPR
)
846 tp
= &TREE_OPERAND (*tp
, 0);
848 tree repl
= NULL_TREE
;
849 ipa_param_body_replacement
*pbr
= NULL
;
851 if (TREE_CODE (*tp
) == PARM_DECL
)
853 pbr
= info
->adjustments
->get_expr_replacement (*tp
, true);
857 else if (TYPE_P (*tp
))
861 repl
= unshare_expr (repl
);
867 bool modified
= info
->modified
;
868 info
->modified
= false;
869 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
872 info
->modified
= modified
;
875 info
->modified
= modified
;
884 if (gimple_code (info
->stmt
) == GIMPLE_PHI
886 && TREE_CODE (*orig_tp
) == ADDR_EXPR
887 && TREE_CODE (TREE_OPERAND (*orig_tp
, 0)) == PARM_DECL
890 gcc_assert (TREE_CODE (pbr
->dummy
) == SSA_NAME
);
891 *orig_tp
= pbr
->dummy
;
892 info
->modified
= true;
896 repl
= build_fold_addr_expr (repl
);
898 if (is_gimple_debug (info
->stmt
))
900 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
901 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
902 DECL_ARTIFICIAL (vexpr
) = 1;
903 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
904 SET_DECL_MODE (vexpr
, TYPE_MODE (TREE_TYPE (repl
)));
909 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
910 repl
= gimple_assign_lhs (stmt
);
912 gimple_stmt_iterator gsi
;
913 if (gimple_code (info
->stmt
) == GIMPLE_PHI
)
915 gsi
= gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
)));
916 /* Cache SSA_NAME for next time. */
918 && TREE_CODE (*orig_tp
) == ADDR_EXPR
919 && TREE_CODE (TREE_OPERAND (*orig_tp
, 0)) == PARM_DECL
)
921 gcc_assert (!pbr
->dummy
);
926 gsi
= gsi_for_stmt (info
->stmt
);
927 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
930 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
932 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
938 info
->modified
= true;
942 /* Traverse the function body and perform all modifications as
943 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
944 modified such that the replacement/reduction value will now be an
945 offset into the corresponding simd_array.
947 This function will replace all function argument uses with their
948 corresponding simd array elements, and ajust the return values
952 ipa_simd_modify_function_body (struct cgraph_node
*node
,
953 ipa_param_body_adjustments
*adjustments
,
954 tree retval_array
, tree iter
)
960 /* Register replacements for every function argument use to an offset into
961 the corresponding simd_array. */
962 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
964 if (!node
->simdclone
->args
[i
].vector_arg
965 || (*adjustments
->m_adj_params
)[j
].user_flag
)
968 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
969 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
970 tree r
= build4 (ARRAY_REF
, basetype
, node
->simdclone
->args
[i
].simd_array
,
971 iter
, NULL_TREE
, NULL_TREE
);
972 adjustments
->register_replacement (&(*adjustments
->m_adj_params
)[j
], r
);
974 if (simd_clone_subparts (vectype
) < node
->simdclone
->simdlen
)
975 j
+= node
->simdclone
->simdlen
/ simd_clone_subparts (vectype
) - 1;
979 FOR_EACH_SSA_NAME (i
, name
, cfun
)
982 if (SSA_NAME_VAR (name
)
983 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
985 = adjustments
->get_replacement_ssa_base (SSA_NAME_VAR (name
))))
987 if (SSA_NAME_IS_DEFAULT_DEF (name
))
989 tree old_decl
= SSA_NAME_VAR (name
);
990 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
991 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
992 tree repl
= adjustments
->lookup_replacement (old_decl
, 0);
993 gcc_checking_assert (repl
);
994 repl
= unshare_expr (repl
);
995 set_ssa_default_def (cfun
, old_decl
, NULL_TREE
);
996 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
997 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
998 gimple
*stmt
= gimple_build_assign (name
, repl
);
999 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
1002 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
1006 struct modify_stmt_info info
;
1007 info
.adjustments
= adjustments
;
1009 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
1011 gimple_stmt_iterator gsi
;
1013 for (gsi
= gsi_start_phis (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1015 gphi
*phi
= as_a
<gphi
*> (gsi_stmt (gsi
));
1016 int i
, n
= gimple_phi_num_args (phi
);
1018 struct walk_stmt_info wi
;
1019 memset (&wi
, 0, sizeof (wi
));
1020 info
.modified
= false;
1022 for (i
= 0; i
< n
; ++i
)
1024 int walk_subtrees
= 1;
1025 tree arg
= gimple_phi_arg_def (phi
, i
);
1027 ipa_simd_modify_stmt_ops (&op
, &walk_subtrees
, &wi
);
1030 SET_PHI_ARG_DEF (phi
, i
, op
);
1031 gcc_assert (TREE_CODE (op
) == SSA_NAME
);
1032 if (gimple_phi_arg_edge (phi
, i
)->flags
& EDGE_ABNORMAL
)
1033 SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op
) = 1;
1038 gsi
= gsi_start_bb (bb
);
1039 while (!gsi_end_p (gsi
))
1041 gimple
*stmt
= gsi_stmt (gsi
);
1043 struct walk_stmt_info wi
;
1045 memset (&wi
, 0, sizeof (wi
));
1046 info
.modified
= false;
1048 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
1050 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
1052 tree retval
= gimple_return_retval (return_stmt
);
1053 edge e
= find_edge (bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
));
1054 e
->flags
|= EDGE_FALLTHRU
;
1057 gsi_remove (&gsi
, true);
1061 /* Replace `return foo' with `retval_array[iter] = foo'. */
1062 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
1063 retval_array
, iter
, NULL
, NULL
);
1064 stmt
= gimple_build_assign (ref
, retval
);
1065 gsi_replace (&gsi
, stmt
, true);
1066 info
.modified
= true;
1072 /* If the above changed the var of a debug bind into something
1073 different, remove the debug stmt. We could also for all the
1074 replaced parameters add VAR_DECLs for debug info purposes,
1075 add debug stmts for those to be the simd array accesses and
1076 replace debug stmt var operand with that var. Debugging of
1077 vectorized loops doesn't work too well, so don't bother for
1079 if ((gimple_debug_bind_p (stmt
)
1080 && !DECL_P (gimple_debug_bind_get_var (stmt
)))
1081 || (gimple_debug_source_bind_p (stmt
)
1082 && !DECL_P (gimple_debug_source_bind_get_var (stmt
))))
1084 gsi_remove (&gsi
, true);
1087 if (maybe_clean_eh_stmt (stmt
))
1088 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
1095 /* Helper function of simd_clone_adjust, return linear step addend
1099 simd_clone_linear_addend (struct cgraph_node
*node
, unsigned int i
,
1100 tree addtype
, basic_block entry_bb
)
1102 tree ptype
= NULL_TREE
;
1103 switch (node
->simdclone
->args
[i
].arg_type
)
1105 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
1106 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
1107 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
1108 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
1109 return build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
1110 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
1111 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
1112 ptype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
1114 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
1115 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
1116 ptype
= TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
));
1122 unsigned int idx
= node
->simdclone
->args
[i
].linear_step
;
1123 tree arg
= node
->simdclone
->args
[idx
].orig_arg
;
1124 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg
)));
1125 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
1128 if (is_gimple_reg (arg
))
1129 ret
= get_or_create_ssa_default_def (cfun
, arg
);
1132 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (arg
)), arg
);
1133 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1134 ret
= gimple_assign_lhs (g
);
1136 if (TREE_CODE (TREE_TYPE (arg
)) == REFERENCE_TYPE
)
1138 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg
))),
1139 build_simple_mem_ref (ret
));
1140 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1141 ret
= gimple_assign_lhs (g
);
1143 if (!useless_type_conversion_p (addtype
, TREE_TYPE (ret
)))
1145 g
= gimple_build_assign (make_ssa_name (addtype
), NOP_EXPR
, ret
);
1146 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1147 ret
= gimple_assign_lhs (g
);
1149 if (POINTER_TYPE_P (ptype
))
1151 tree size
= TYPE_SIZE_UNIT (TREE_TYPE (ptype
));
1152 if (size
&& TREE_CODE (size
) == INTEGER_CST
)
1154 g
= gimple_build_assign (make_ssa_name (addtype
), MULT_EXPR
,
1155 ret
, fold_convert (addtype
, size
));
1156 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1157 ret
= gimple_assign_lhs (g
);
1163 /* Adjust the argument types in NODE to their appropriate vector
1167 simd_clone_adjust (struct cgraph_node
*node
)
1169 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
1171 TREE_TYPE (node
->decl
) = build_distinct_type_copy (TREE_TYPE (node
->decl
));
1172 targetm
.simd_clone
.adjust (node
);
1174 tree retval
= simd_clone_adjust_return_type (node
);
1175 ipa_param_body_adjustments
*adjustments
1176 = simd_clone_adjust_argument_types (node
);
1177 gcc_assert (adjustments
);
1179 push_gimplify_context ();
1181 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
1183 /* Adjust all uses of vector arguments accordingly. Adjust all
1184 return values accordingly. */
1185 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
1186 tree iter1
= make_ssa_name (iter
);
1187 tree iter2
= NULL_TREE
;
1188 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
1191 /* Initialize the iteration variable. */
1192 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1193 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
1194 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
1195 /* Insert the SIMD array and iv initialization at function
1197 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
1199 pop_gimplify_context (NULL
);
1202 basic_block incr_bb
= NULL
;
1203 class loop
*loop
= NULL
;
1205 /* Create a new BB right before the original exit BB, to hold the
1206 iteration increment and the condition/branch. */
1207 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
))
1209 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
1210 incr_bb
= create_empty_bb (orig_exit
);
1211 incr_bb
->count
= profile_count::zero ();
1212 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
1213 while (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
))
1215 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
1216 redirect_edge_succ (e
, incr_bb
);
1217 incr_bb
->count
+= e
->count ();
1220 else if (node
->simdclone
->inbranch
)
1222 incr_bb
= create_empty_bb (entry_bb
);
1223 incr_bb
->count
= profile_count::zero ();
1224 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
1229 make_single_succ_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
1230 gsi
= gsi_last_bb (incr_bb
);
1231 iter2
= make_ssa_name (iter
);
1232 g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
1233 build_int_cst (unsigned_type_node
, 1));
1234 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1236 /* Mostly annotate the loop for the vectorizer (the rest is done
1238 loop
= alloc_loop ();
1239 cfun
->has_force_vectorize_loops
= true;
1240 loop
->safelen
= node
->simdclone
->simdlen
;
1241 loop
->force_vectorize
= true;
1242 loop
->header
= body_bb
;
1245 /* Branch around the body if the mask applies. */
1246 if (node
->simdclone
->inbranch
)
1248 gsi
= gsi_last_bb (loop
->header
);
1250 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
1252 if (node
->simdclone
->mask_mode
!= VOIDmode
)
1255 if (mask_array
== NULL_TREE
)
1257 tree arg
= node
->simdclone
->args
[node
->simdclone
->nargs
1259 mask
= get_or_create_ssa_default_def (cfun
, arg
);
1264 tree maskt
= TREE_TYPE (mask_array
);
1265 int c
= tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt
)));
1266 c
= node
->simdclone
->simdlen
/ (c
+ 1);
1267 int s
= exact_log2 (c
);
1270 tree idx
= make_ssa_name (TREE_TYPE (iter1
));
1271 g
= gimple_build_assign (idx
, RSHIFT_EXPR
, iter1
,
1272 build_int_cst (NULL_TREE
, s
));
1273 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1274 mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
1275 tree aref
= build4 (ARRAY_REF
,
1276 TREE_TYPE (TREE_TYPE (mask_array
)),
1277 mask_array
, idx
, NULL
, NULL
);
1278 g
= gimple_build_assign (mask
, aref
);
1279 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1280 shift_cnt
= make_ssa_name (TREE_TYPE (iter1
));
1281 g
= gimple_build_assign (shift_cnt
, BIT_AND_EXPR
, iter1
,
1282 build_int_cst (TREE_TYPE (iter1
), c
));
1283 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1285 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (mask
)),
1286 RSHIFT_EXPR
, mask
, shift_cnt
);
1287 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1288 mask
= gimple_assign_lhs (g
);
1289 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (mask
)),
1291 build_int_cst (TREE_TYPE (mask
), 1));
1292 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1293 mask
= gimple_assign_lhs (g
);
1297 mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
1298 tree aref
= build4 (ARRAY_REF
,
1299 TREE_TYPE (TREE_TYPE (mask_array
)),
1300 mask_array
, iter1
, NULL
, NULL
);
1301 g
= gimple_build_assign (mask
, aref
);
1302 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1303 int bitsize
= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (aref
)));
1304 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
1306 aref
= build1 (VIEW_CONVERT_EXPR
,
1307 build_nonstandard_integer_type (bitsize
, 0),
1309 mask
= make_ssa_name (TREE_TYPE (aref
));
1310 g
= gimple_build_assign (mask
, aref
);
1311 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1315 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
1317 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1318 edge e
= make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
1319 e
->probability
= profile_probability::unlikely ().guessed ();
1320 incr_bb
->count
+= e
->count ();
1321 edge fallthru
= FALLTHRU_EDGE (loop
->header
);
1322 fallthru
->flags
= EDGE_FALSE_VALUE
;
1323 fallthru
->probability
= profile_probability::likely ().guessed ();
1326 basic_block latch_bb
= NULL
;
1327 basic_block new_exit_bb
= NULL
;
1329 /* Generate the condition. */
1332 gsi
= gsi_last_bb (incr_bb
);
1333 g
= gimple_build_cond (LT_EXPR
, iter2
,
1334 build_int_cst (unsigned_type_node
,
1335 node
->simdclone
->simdlen
),
1337 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1338 edge e
= split_block (incr_bb
, gsi_stmt (gsi
));
1340 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
1341 loop
->latch
= latch_bb
;
1343 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
1345 edge new_e
= make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
1347 /* FIXME: Do we need to distribute probabilities for the conditional? */
1348 new_e
->probability
= profile_probability::guessed_never ();
1349 /* The successor of incr_bb is already pointing to latch_bb; just
1351 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
1352 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
1355 gphi
*phi
= create_phi_node (iter1
, body_bb
);
1356 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
1357 edge latch_edge
= NULL
;
1358 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
1362 latch_edge
= single_succ_edge (latch_bb
);
1363 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1365 /* Generate the new return. */
1366 gsi
= gsi_last_bb (new_exit_bb
);
1368 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
1369 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
1370 retval
= TREE_OPERAND (retval
, 0);
1373 retval
= build1 (VIEW_CONVERT_EXPR
,
1374 TREE_TYPE (TREE_TYPE (node
->decl
)),
1376 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
1377 false, GSI_CONTINUE_LINKING
);
1379 g
= gimple_build_return (retval
);
1380 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
1383 /* Handle aligned clauses by replacing default defs of the aligned
1384 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
1385 lhs. Handle linear by adding PHIs. */
1386 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
1387 if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
1388 && (TREE_ADDRESSABLE (node
->simdclone
->args
[i
].orig_arg
)
1389 || !is_gimple_reg_type
1390 (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
1392 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1393 if (is_gimple_reg_type (TREE_TYPE (orig_arg
)))
1394 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
1397 iter1
= create_tmp_var_raw (TREE_TYPE (orig_arg
));
1398 gimple_add_tmp_var (iter1
);
1400 gsi
= gsi_after_labels (entry_bb
);
1401 g
= gimple_build_assign (iter1
, orig_arg
);
1402 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1403 gsi
= gsi_after_labels (body_bb
);
1404 g
= gimple_build_assign (orig_arg
, iter1
);
1405 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1407 else if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
1408 && DECL_BY_REFERENCE (node
->simdclone
->args
[i
].orig_arg
)
1409 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
1412 (TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
1414 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1415 tree def
= ssa_default_def (cfun
, orig_arg
);
1416 if (def
&& !has_zero_uses (def
))
1418 iter1
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg
)));
1419 gimple_add_tmp_var (iter1
);
1420 gsi
= gsi_after_labels (entry_bb
);
1421 g
= gimple_build_assign (iter1
, build_simple_mem_ref (def
));
1422 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1423 gsi
= gsi_after_labels (body_bb
);
1424 g
= gimple_build_assign (build_simple_mem_ref (def
), iter1
);
1425 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1428 else if (node
->simdclone
->args
[i
].alignment
1429 && node
->simdclone
->args
[i
].arg_type
1430 == SIMD_CLONE_ARG_TYPE_UNIFORM
1431 && (node
->simdclone
->args
[i
].alignment
1432 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
1433 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
1436 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
1437 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1438 tree def
= ssa_default_def (cfun
, orig_arg
);
1439 if (def
&& !has_zero_uses (def
))
1441 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
1442 gimple_seq seq
= NULL
;
1443 bool need_cvt
= false;
1445 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
1447 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
1450 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
1451 gimple_call_set_lhs (g
, t
);
1452 gimple_seq_add_stmt_without_update (&seq
, g
);
1455 t
= make_ssa_name (orig_arg
);
1456 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
1457 gimple_seq_add_stmt_without_update (&seq
, g
);
1459 gsi_insert_seq_on_edge_immediate
1460 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
1462 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1463 node
->create_edge (cgraph_node::get_create (fn
),
1464 call
, entry_bb
->count
);
1466 imm_use_iterator iter
;
1467 use_operand_p use_p
;
1469 tree repl
= gimple_get_lhs (g
);
1470 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1471 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
1474 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1475 SET_USE (use_p
, repl
);
1478 else if ((node
->simdclone
->args
[i
].arg_type
1479 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
1480 || (node
->simdclone
->args
[i
].arg_type
1481 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
)
1482 || (node
->simdclone
->args
[i
].arg_type
1483 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
1484 || (node
->simdclone
->args
[i
].arg_type
1485 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
))
1487 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1488 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1489 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
1490 tree def
= NULL_TREE
;
1491 if (TREE_ADDRESSABLE (orig_arg
))
1493 def
= make_ssa_name (TREE_TYPE (orig_arg
));
1494 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
1496 iter2
= make_ssa_name (TREE_TYPE (orig_arg
));
1497 gsi
= gsi_after_labels (entry_bb
);
1498 g
= gimple_build_assign (def
, orig_arg
);
1499 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1503 def
= ssa_default_def (cfun
, orig_arg
);
1504 if (!def
|| has_zero_uses (def
))
1508 iter1
= make_ssa_name (orig_arg
);
1510 iter2
= make_ssa_name (orig_arg
);
1515 phi
= create_phi_node (iter1
, body_bb
);
1516 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
1519 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1520 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1521 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
1522 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
1523 ? TREE_TYPE (orig_arg
) : sizetype
;
1524 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
1526 gsi
= gsi_last_bb (incr_bb
);
1527 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
1528 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1531 imm_use_iterator iter
;
1532 use_operand_p use_p
;
1534 if (TREE_ADDRESSABLE (orig_arg
))
1536 gsi
= gsi_after_labels (body_bb
);
1537 g
= gimple_build_assign (orig_arg
, iter1
);
1538 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
1541 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1542 if (use_stmt
== phi
)
1545 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1546 SET_USE (use_p
, iter1
);
1549 else if (node
->simdclone
->args
[i
].arg_type
1550 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
1551 || (node
->simdclone
->args
[i
].arg_type
1552 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
))
1554 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
1555 tree def
= ssa_default_def (cfun
, orig_arg
);
1556 gcc_assert (!TREE_ADDRESSABLE (orig_arg
)
1557 && TREE_CODE (TREE_TYPE (orig_arg
)) == REFERENCE_TYPE
);
1558 if (def
&& !has_zero_uses (def
))
1560 tree rtype
= TREE_TYPE (TREE_TYPE (orig_arg
));
1561 iter1
= make_ssa_name (orig_arg
);
1563 iter2
= make_ssa_name (orig_arg
);
1564 tree iter3
= make_ssa_name (rtype
);
1565 tree iter4
= make_ssa_name (rtype
);
1566 tree iter5
= incr_bb
? make_ssa_name (rtype
) : NULL_TREE
;
1567 gsi
= gsi_after_labels (entry_bb
);
1569 = gimple_build_assign (iter3
, build_simple_mem_ref (def
));
1570 gsi_insert_before (&gsi
, load
, GSI_NEW_STMT
);
1572 tree array
= node
->simdclone
->args
[i
].simd_array
;
1573 TREE_ADDRESSABLE (array
) = 1;
1574 tree ptr
= build_fold_addr_expr (array
);
1575 phi
= create_phi_node (iter1
, body_bb
);
1576 add_phi_arg (phi
, ptr
, preheader_edge
, UNKNOWN_LOCATION
);
1579 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
1580 g
= gimple_build_assign (iter2
, POINTER_PLUS_EXPR
, iter1
,
1581 TYPE_SIZE_UNIT (TREE_TYPE (iter3
)));
1582 gsi
= gsi_last_bb (incr_bb
);
1583 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1586 phi
= create_phi_node (iter4
, body_bb
);
1587 add_phi_arg (phi
, iter3
, preheader_edge
, UNKNOWN_LOCATION
);
1590 add_phi_arg (phi
, iter5
, latch_edge
, UNKNOWN_LOCATION
);
1591 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
1592 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
1593 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
1594 ? TREE_TYPE (iter3
) : sizetype
;
1595 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
1597 g
= gimple_build_assign (iter5
, code
, iter4
, addcst
);
1598 gsi
= gsi_last_bb (incr_bb
);
1599 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1602 g
= gimple_build_assign (build_simple_mem_ref (iter1
), iter4
);
1603 gsi
= gsi_after_labels (body_bb
);
1604 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1606 imm_use_iterator iter
;
1607 use_operand_p use_p
;
1609 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
1610 if (use_stmt
== load
)
1613 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
1614 SET_USE (use_p
, iter1
);
1616 if (!TYPE_READONLY (rtype
) && incr_bb
)
1618 tree v
= make_ssa_name (rtype
);
1619 tree aref
= build4 (ARRAY_REF
, rtype
, array
,
1620 size_zero_node
, NULL_TREE
,
1622 gsi
= gsi_after_labels (new_exit_bb
);
1623 g
= gimple_build_assign (v
, aref
);
1624 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1625 g
= gimple_build_assign (build_simple_mem_ref (def
), v
);
1626 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
1631 calculate_dominance_info (CDI_DOMINATORS
);
1633 add_loop (loop
, loop
->header
->loop_father
);
1634 update_ssa (TODO_update_ssa
);
1639 /* If the function in NODE is tagged as an elemental SIMD function,
1640 create the appropriate SIMD clones. */
1643 expand_simd_clones (struct cgraph_node
*node
)
1645 tree attr
= lookup_attribute ("omp declare simd",
1646 DECL_ATTRIBUTES (node
->decl
));
1647 if (attr
== NULL_TREE
1649 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
1653 #pragma omp declare simd
1655 in C, there we don't know the argument types at all. */
1656 if (!node
->definition
1657 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
1660 /* Call this before creating clone_info, as it might ggc_collect. */
1661 if (node
->definition
&& node
->has_gimple_body_p ())
1666 /* Start with parsing the "omp declare simd" attribute(s). */
1667 bool inbranch_clause_specified
;
1668 struct cgraph_simd_clone
*clone_info
1669 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
1670 &inbranch_clause_specified
);
1671 if (clone_info
== NULL
)
1674 int orig_simdlen
= clone_info
->simdlen
;
1675 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
1676 /* The target can return 0 (no simd clones should be created),
1677 1 (just one ISA of simd clones should be created) or higher
1678 count of ISA variants. In that case, clone_info is initialized
1679 for the first ISA variant. */
1681 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
1686 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
1687 also create one inbranch and one !inbranch clone of it. */
1688 for (int i
= 0; i
< count
* 2; i
++)
1690 struct cgraph_simd_clone
*clone
= clone_info
;
1691 if (inbranch_clause_specified
&& (i
& 1) != 0)
1696 clone
= simd_clone_struct_alloc (clone_info
->nargs
1698 simd_clone_struct_copy (clone
, clone_info
);
1699 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
1700 and simd_clone_adjust_argument_types did to the first
1702 clone
->nargs
-= clone_info
->inbranch
;
1703 clone
->simdlen
= orig_simdlen
;
1704 /* And call the target hook again to get the right ISA. */
1705 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
1709 clone
->inbranch
= 1;
1712 /* simd_clone_mangle might fail if such a clone has been created
1714 tree id
= simd_clone_mangle (node
, clone
);
1715 if (id
== NULL_TREE
)
1718 clone
->nargs
+= clone
->inbranch
;
1722 /* Only when we are sure we want to create the clone actually
1723 clone the function (or definitions) or create another
1724 extern FUNCTION_DECL (for prototypes without definitions). */
1725 struct cgraph_node
*n
= simd_clone_create (node
);
1729 clone
->nargs
+= clone
->inbranch
;
1733 n
->simdclone
= clone
;
1734 clone
->origin
= node
;
1735 clone
->next_clone
= NULL
;
1736 if (node
->simd_clones
== NULL
)
1738 clone
->prev_clone
= n
;
1739 node
->simd_clones
= n
;
1743 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
1744 clone
->prev_clone
->simdclone
->next_clone
= n
;
1745 node
->simd_clones
->simdclone
->prev_clone
= n
;
1747 symtab
->change_decl_assembler_name (n
->decl
, id
);
1748 /* And finally adjust the return type, parameters and for
1749 definitions also function body. */
1750 if (node
->definition
)
1751 simd_clone_adjust (n
);
1755 = build_distinct_type_copy (TREE_TYPE (n
->decl
));
1756 targetm
.simd_clone
.adjust (n
);
1757 simd_clone_adjust_return_type (n
);
1758 simd_clone_adjust_argument_types (n
);
1762 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
1765 /* Entry point for IPA simd clone creation pass. */
1768 ipa_omp_simd_clone (void)
1770 struct cgraph_node
*node
;
1771 FOR_EACH_FUNCTION (node
)
1772 expand_simd_clones (node
);
1778 const pass_data pass_data_omp_simd_clone
=
1780 SIMPLE_IPA_PASS
, /* type */
1781 "simdclone", /* name */
1782 OPTGROUP_OMP
, /* optinfo_flags */
1783 TV_NONE
, /* tv_id */
1784 ( PROP_ssa
| PROP_cfg
), /* properties_required */
1785 0, /* properties_provided */
1786 0, /* properties_destroyed */
1787 0, /* todo_flags_start */
1788 0, /* todo_flags_finish */
1791 class pass_omp_simd_clone
: public simple_ipa_opt_pass
1794 pass_omp_simd_clone(gcc::context
*ctxt
)
1795 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
1798 /* opt_pass methods: */
1799 virtual bool gate (function
*);
1800 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
1804 pass_omp_simd_clone::gate (function
*)
1806 return targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
;
1811 simple_ipa_opt_pass
*
1812 make_pass_omp_simd_clone (gcc::context
*ctxt
)
1814 return new pass_omp_simd_clone (ctxt
);