1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2015 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
34 #include "alloc-pool.h"
35 #include "tree-pass.h"
41 #include "pretty-print.h"
42 #include "diagnostic-core.h"
44 #include "fold-const.h"
45 #include "stor-layout.h"
47 #include "internal-fn.h"
48 #include "gimple-fold.h"
50 #include "gimple-iterator.h"
51 #include "gimplify-me.h"
52 #include "gimple-walk.h"
53 #include "tree-iterator.h"
54 #include "tree-inline.h"
55 #include "langhooks.h"
57 #include "tree-into-ssa.h"
68 #include "splay-tree.h"
70 #include "common/common-target.h"
72 #include "gimple-low.h"
73 #include "tree-cfgcleanup.h"
74 #include "symbol-summary.h"
76 #include "tree-nested.h"
80 #include "lto-section-names.h"
81 #include "gomp-constants.h"
82 #include "gimple-pretty-print.h"
84 /* Lowering of OMP parallel and workshare constructs proceeds in two
85 phases. The first phase scans the function looking for OMP statements
86 and then for variables that must be replaced to satisfy data sharing
87 clauses. The second phase expands code for the constructs, as well as
88 re-gimplifying things when variables have been replaced with complex
91 Final code generation is done by pass_expand_omp. The flowgraph is
92 scanned for regions which are then moved to a new
93 function, to be invoked by the thread library, or offloaded. */
95 /* OMP region information. Every parallel and workshare
96 directive is enclosed between two markers, the OMP_* directive
97 and a corresponding GIMPLE_OMP_RETURN statement. */
101 /* The enclosing region. */
102 struct omp_region
*outer
;
104 /* First child region. */
105 struct omp_region
*inner
;
107 /* Next peer region. */
108 struct omp_region
*next
;
110 /* Block containing the omp directive as its last stmt. */
113 /* Block containing the GIMPLE_OMP_RETURN as its last stmt. */
116 /* Block containing the GIMPLE_OMP_CONTINUE as its last stmt. */
119 /* If this is a combined parallel+workshare region, this is a list
120 of additional arguments needed by the combined parallel+workshare
122 vec
<tree
, va_gc
> *ws_args
;
124 /* The code for the omp directive of this region. */
125 enum gimple_code type
;
127 /* Schedule kind, only used for GIMPLE_OMP_FOR type regions. */
128 enum omp_clause_schedule_kind sched_kind
;
130 /* Schedule modifiers. */
131 unsigned char sched_modifiers
;
133 /* True if this is a combined parallel+workshare region. */
134 bool is_combined_parallel
;
136 /* The ordered stmt if type is GIMPLE_OMP_ORDERED and it has
138 gomp_ordered
*ord_stmt
;
141 /* Context structure. Used to store information about each parallel
142 directive in the code. */
146 /* This field must be at the beginning, as we do "inheritance": Some
147 callback functions for tree-inline.c (e.g., omp_copy_decl)
148 receive a copy_body_data pointer that is up-casted to an
149 omp_context pointer. */
152 /* The tree of contexts corresponding to the encountered constructs. */
153 struct omp_context
*outer
;
156 /* Map variables to fields in a structure that allows communication
157 between sending and receiving threads. */
158 splay_tree field_map
;
163 /* These are used just by task contexts, if task firstprivate fn is
164 needed. srecord_type is used to communicate from the thread
165 that encountered the task construct to task firstprivate fn,
166 record_type is allocated by GOMP_task, initialized by task firstprivate
167 fn and passed to the task body fn. */
168 splay_tree sfield_map
;
171 /* A chain of variables to add to the top-level block surrounding the
172 construct. In the case of a parallel, this is in the child function. */
175 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
176 barriers should jump to during omplower pass. */
179 /* What to do with variables with implicitly determined sharing
181 enum omp_clause_default_kind default_kind
;
183 /* Nesting depth of this context. Used to beautify error messages re
184 invalid gotos. The outermost ctx is depth 1, with depth 0 being
185 reserved for the main body of the function. */
188 /* True if this parallel directive is nested within another. */
191 /* True if this construct can be cancelled. */
195 /* A structure holding the elements of:
196 for (V = N1; V cond N2; V += STEP) [...] */
198 struct omp_for_data_loop
200 tree v
, n1
, n2
, step
;
201 enum tree_code cond_code
;
204 /* A structure describing the main elements of a parallel loop. */
208 struct omp_for_data_loop loop
;
214 bool have_nowait
, have_ordered
, simd_schedule
;
215 unsigned char sched_modifiers
;
216 enum omp_clause_schedule_kind sched_kind
;
217 struct omp_for_data_loop
*loops
;
220 /* Describe the OpenACC looping structure of a function. The entire
221 function is held in a 'NULL' loop. */
225 oacc_loop
*parent
; /* Containing loop. */
227 oacc_loop
*child
; /* First inner loop. */
229 oacc_loop
*sibling
; /* Next loop within same parent. */
231 location_t loc
; /* Location of the loop start. */
233 gcall
*marker
; /* Initial head marker. */
235 gcall
*heads
[GOMP_DIM_MAX
]; /* Head marker functions. */
236 gcall
*tails
[GOMP_DIM_MAX
]; /* Tail marker functions. */
238 tree routine
; /* Pseudo-loop enclosing a routine. */
240 unsigned mask
; /* Partitioning mask. */
241 unsigned flags
; /* Partitioning flags. */
242 tree chunk_size
; /* Chunk size. */
243 gcall
*head_end
; /* Final marker of head sequence. */
246 /* Flags for an OpenACC loop. */
248 enum oacc_loop_flags
{
249 OLF_SEQ
= 1u << 0, /* Explicitly sequential */
250 OLF_AUTO
= 1u << 1, /* Compiler chooses axes. */
251 OLF_INDEPENDENT
= 1u << 2, /* Iterations are known independent. */
252 OLF_GANG_STATIC
= 1u << 3, /* Gang partitioning is static (has op). */
254 /* Explicitly specified loop axes. */
256 OLF_DIM_GANG
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_GANG
),
257 OLF_DIM_WORKER
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_WORKER
),
258 OLF_DIM_VECTOR
= 1u << (OLF_DIM_BASE
+ GOMP_DIM_VECTOR
),
260 OLF_MAX
= OLF_DIM_BASE
+ GOMP_DIM_MAX
264 static splay_tree all_contexts
;
265 static int taskreg_nesting_level
;
266 static int target_nesting_level
;
267 static struct omp_region
*root_omp_region
;
268 static bitmap task_shared_vars
;
269 static vec
<omp_context
*> taskreg_contexts
;
270 static bool omp_any_child_fn_dumped
;
272 static void scan_omp (gimple_seq
*, omp_context
*);
273 static tree
scan_omp_1_op (tree
*, int *, void *);
274 static gphi
*find_phi_with_arg_on_edge (tree
, edge
);
276 #define WALK_SUBSTMTS \
280 case GIMPLE_EH_FILTER: \
281 case GIMPLE_TRANSACTION: \
282 /* The sub-statements for these should be walked. */ \
283 *handled_ops_p = false; \
286 /* Return true if CTX corresponds to an oacc parallel region. */
289 is_oacc_parallel (omp_context
*ctx
)
291 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
292 return ((outer_type
== GIMPLE_OMP_TARGET
)
293 && (gimple_omp_target_kind (ctx
->stmt
)
294 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
297 /* Return true if CTX corresponds to an oacc kernels region. */
300 is_oacc_kernels (omp_context
*ctx
)
302 enum gimple_code outer_type
= gimple_code (ctx
->stmt
);
303 return ((outer_type
== GIMPLE_OMP_TARGET
)
304 && (gimple_omp_target_kind (ctx
->stmt
)
305 == GF_OMP_TARGET_KIND_OACC_KERNELS
));
308 /* If DECL is the artificial dummy VAR_DECL created for non-static
309 data member privatization, return the underlying "this" parameter,
310 otherwise return NULL. */
313 omp_member_access_dummy_var (tree decl
)
316 || !DECL_ARTIFICIAL (decl
)
317 || !DECL_IGNORED_P (decl
)
318 || !DECL_HAS_VALUE_EXPR_P (decl
)
319 || !lang_hooks
.decls
.omp_disregard_value_expr (decl
, false))
322 tree v
= DECL_VALUE_EXPR (decl
);
323 if (TREE_CODE (v
) != COMPONENT_REF
)
327 switch (TREE_CODE (v
))
333 case POINTER_PLUS_EXPR
:
334 v
= TREE_OPERAND (v
, 0);
337 if (DECL_CONTEXT (v
) == current_function_decl
338 && DECL_ARTIFICIAL (v
)
339 && TREE_CODE (TREE_TYPE (v
)) == POINTER_TYPE
)
347 /* Helper for unshare_and_remap, called through walk_tree. */
350 unshare_and_remap_1 (tree
*tp
, int *walk_subtrees
, void *data
)
352 tree
*pair
= (tree
*) data
;
355 *tp
= unshare_expr (pair
[1]);
358 else if (IS_TYPE_OR_DECL_P (*tp
))
363 /* Return unshare_expr (X) with all occurrences of FROM
367 unshare_and_remap (tree x
, tree from
, tree to
)
369 tree pair
[2] = { from
, to
};
370 x
= unshare_expr (x
);
371 walk_tree (&x
, unshare_and_remap_1
, pair
, NULL
);
375 /* Holds offload tables with decls. */
376 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
378 /* Convenience function for calling scan_omp_1_op on tree operands. */
381 scan_omp_op (tree
*tp
, omp_context
*ctx
)
383 struct walk_stmt_info wi
;
385 memset (&wi
, 0, sizeof (wi
));
387 wi
.want_locations
= true;
389 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
392 static void lower_omp (gimple_seq
*, omp_context
*);
393 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
394 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
396 /* Find an OMP clause of type KIND within CLAUSES. */
399 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
401 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
402 if (OMP_CLAUSE_CODE (clauses
) == kind
)
408 /* Return true if CTX is for an omp parallel. */
411 is_parallel_ctx (omp_context
*ctx
)
413 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
417 /* Return true if CTX is for an omp task. */
420 is_task_ctx (omp_context
*ctx
)
422 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
426 /* Return true if CTX is for an omp taskloop. */
429 is_taskloop_ctx (omp_context
*ctx
)
431 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
432 && gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
;
436 /* Return true if CTX is for an omp parallel or omp task. */
439 is_taskreg_ctx (omp_context
*ctx
)
441 return is_parallel_ctx (ctx
) || is_task_ctx (ctx
);
445 /* Return true if REGION is a combined parallel+workshare region. */
448 is_combined_parallel (struct omp_region
*region
)
450 return region
->is_combined_parallel
;
454 /* Extract the header elements of parallel loop FOR_STMT and store
458 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
459 struct omp_for_data_loop
*loops
)
461 tree t
, var
, *collapse_iter
, *collapse_count
;
462 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
463 struct omp_for_data_loop
*loop
;
465 struct omp_for_data_loop dummy_loop
;
466 location_t loc
= gimple_location (for_stmt
);
467 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
468 bool distribute
= gimple_omp_for_kind (for_stmt
)
469 == GF_OMP_FOR_KIND_DISTRIBUTE
;
470 bool taskloop
= gimple_omp_for_kind (for_stmt
)
471 == GF_OMP_FOR_KIND_TASKLOOP
;
474 fd
->for_stmt
= for_stmt
;
476 if (gimple_omp_for_collapse (for_stmt
) > 1)
479 fd
->loops
= &fd
->loop
;
481 fd
->have_nowait
= distribute
|| simd
;
482 fd
->have_ordered
= false;
485 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
486 fd
->sched_modifiers
= 0;
487 fd
->chunk_size
= NULL_TREE
;
488 fd
->simd_schedule
= false;
489 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
490 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
491 collapse_iter
= NULL
;
492 collapse_count
= NULL
;
494 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
495 switch (OMP_CLAUSE_CODE (t
))
497 case OMP_CLAUSE_NOWAIT
:
498 fd
->have_nowait
= true;
500 case OMP_CLAUSE_ORDERED
:
501 fd
->have_ordered
= true;
502 if (OMP_CLAUSE_ORDERED_EXPR (t
))
503 fd
->ordered
= tree_to_shwi (OMP_CLAUSE_ORDERED_EXPR (t
));
505 case OMP_CLAUSE_SCHEDULE
:
506 gcc_assert (!distribute
&& !taskloop
);
508 = (enum omp_clause_schedule_kind
)
509 (OMP_CLAUSE_SCHEDULE_KIND (t
) & OMP_CLAUSE_SCHEDULE_MASK
);
510 fd
->sched_modifiers
= (OMP_CLAUSE_SCHEDULE_KIND (t
)
511 & ~OMP_CLAUSE_SCHEDULE_MASK
);
512 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
513 fd
->simd_schedule
= OMP_CLAUSE_SCHEDULE_SIMD (t
);
515 case OMP_CLAUSE_DIST_SCHEDULE
:
516 gcc_assert (distribute
);
517 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
519 case OMP_CLAUSE_COLLAPSE
:
520 fd
->collapse
= tree_to_shwi (OMP_CLAUSE_COLLAPSE_EXPR (t
));
521 if (fd
->collapse
> 1)
523 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
524 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
530 if (fd
->ordered
&& fd
->collapse
== 1 && loops
!= NULL
)
535 collapse_iter
= &iterv
;
536 collapse_count
= &countv
;
539 /* FIXME: for now map schedule(auto) to schedule(static).
540 There should be analysis to determine whether all iterations
541 are approximately the same amount of work (then schedule(static)
542 is best) or if it varies (then schedule(dynamic,N) is better). */
543 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
545 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
546 gcc_assert (fd
->chunk_size
== NULL
);
548 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
550 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_RUNTIME
;
551 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
552 gcc_assert (fd
->chunk_size
== NULL
);
553 else if (fd
->chunk_size
== NULL
)
555 /* We only need to compute a default chunk size for ordered
556 static loops and dynamic loops. */
557 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
559 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
560 ? integer_zero_node
: integer_one_node
;
563 int cnt
= fd
->ordered
? fd
->ordered
: fd
->collapse
;
564 for (i
= 0; i
< cnt
; i
++)
566 if (i
== 0 && fd
->collapse
== 1 && (fd
->ordered
== 0 || loops
== NULL
))
568 else if (loops
!= NULL
)
573 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
574 gcc_assert (SSA_VAR_P (loop
->v
));
575 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
576 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
577 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
578 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
580 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
581 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
582 switch (loop
->cond_code
)
588 gcc_assert (gimple_omp_for_kind (for_stmt
)
589 == GF_OMP_FOR_KIND_CILKSIMD
590 || (gimple_omp_for_kind (for_stmt
)
591 == GF_OMP_FOR_KIND_CILKFOR
));
594 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
595 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
597 loop
->n2
= fold_build2_loc (loc
,
598 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
599 build_int_cst (TREE_TYPE (loop
->n2
), 1));
600 loop
->cond_code
= LT_EXPR
;
603 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
604 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
606 loop
->n2
= fold_build2_loc (loc
,
607 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
608 build_int_cst (TREE_TYPE (loop
->n2
), 1));
609 loop
->cond_code
= GT_EXPR
;
615 t
= gimple_omp_for_incr (for_stmt
, i
);
616 gcc_assert (TREE_OPERAND (t
, 0) == var
);
617 switch (TREE_CODE (t
))
620 loop
->step
= TREE_OPERAND (t
, 1);
622 case POINTER_PLUS_EXPR
:
623 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
626 loop
->step
= TREE_OPERAND (t
, 1);
627 loop
->step
= fold_build1_loc (loc
,
628 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
636 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
637 && !fd
->have_ordered
))
639 if (fd
->collapse
== 1)
640 iter_type
= TREE_TYPE (loop
->v
);
642 || TYPE_PRECISION (iter_type
)
643 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
645 = build_nonstandard_integer_type
646 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
648 else if (iter_type
!= long_long_unsigned_type_node
)
650 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
651 iter_type
= long_long_unsigned_type_node
;
652 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
653 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
654 >= TYPE_PRECISION (iter_type
))
658 if (loop
->cond_code
== LT_EXPR
)
659 n
= fold_build2_loc (loc
,
660 PLUS_EXPR
, TREE_TYPE (loop
->v
),
661 loop
->n2
, loop
->step
);
664 if (TREE_CODE (n
) != INTEGER_CST
665 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
666 iter_type
= long_long_unsigned_type_node
;
668 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
669 > TYPE_PRECISION (iter_type
))
673 if (loop
->cond_code
== LT_EXPR
)
676 n2
= fold_build2_loc (loc
,
677 PLUS_EXPR
, TREE_TYPE (loop
->v
),
678 loop
->n2
, loop
->step
);
682 n1
= fold_build2_loc (loc
,
683 MINUS_EXPR
, TREE_TYPE (loop
->v
),
684 loop
->n2
, loop
->step
);
687 if (TREE_CODE (n1
) != INTEGER_CST
688 || TREE_CODE (n2
) != INTEGER_CST
689 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
690 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
691 iter_type
= long_long_unsigned_type_node
;
695 if (i
>= fd
->collapse
)
698 if (collapse_count
&& *collapse_count
== NULL
)
700 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
701 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
702 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
703 if (t
&& integer_zerop (t
))
704 count
= build_zero_cst (long_long_unsigned_type_node
);
705 else if ((i
== 0 || count
!= NULL_TREE
)
706 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
707 && TREE_CONSTANT (loop
->n1
)
708 && TREE_CONSTANT (loop
->n2
)
709 && TREE_CODE (loop
->step
) == INTEGER_CST
)
711 tree itype
= TREE_TYPE (loop
->v
);
713 if (POINTER_TYPE_P (itype
))
714 itype
= signed_type_for (itype
);
715 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
716 t
= fold_build2_loc (loc
,
718 fold_convert_loc (loc
, itype
, loop
->step
), t
);
719 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
720 fold_convert_loc (loc
, itype
, loop
->n2
));
721 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
722 fold_convert_loc (loc
, itype
, loop
->n1
));
723 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
724 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
725 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
726 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
727 fold_convert_loc (loc
, itype
,
730 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
731 fold_convert_loc (loc
, itype
, loop
->step
));
732 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
733 if (count
!= NULL_TREE
)
734 count
= fold_build2_loc (loc
,
735 MULT_EXPR
, long_long_unsigned_type_node
,
739 if (TREE_CODE (count
) != INTEGER_CST
)
742 else if (count
&& !integer_zerop (count
))
749 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
750 || fd
->have_ordered
))
752 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
753 iter_type
= long_long_unsigned_type_node
;
755 iter_type
= long_integer_type_node
;
757 else if (collapse_iter
&& *collapse_iter
!= NULL
)
758 iter_type
= TREE_TYPE (*collapse_iter
);
759 fd
->iter_type
= iter_type
;
760 if (collapse_iter
&& *collapse_iter
== NULL
)
761 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
762 if (collapse_count
&& *collapse_count
== NULL
)
765 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
767 *collapse_count
= create_tmp_var (iter_type
, ".count");
770 if (fd
->collapse
> 1 || (fd
->ordered
&& loops
))
772 fd
->loop
.v
= *collapse_iter
;
773 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
774 fd
->loop
.n2
= *collapse_count
;
775 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
776 fd
->loop
.cond_code
= LT_EXPR
;
783 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
784 is the immediate dominator of PAR_ENTRY_BB, return true if there
785 are no data dependencies that would prevent expanding the parallel
786 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
788 When expanding a combined parallel+workshare region, the call to
789 the child function may need additional arguments in the case of
790 GIMPLE_OMP_FOR regions. In some cases, these arguments are
791 computed out of variables passed in from the parent to the child
792 via 'struct .omp_data_s'. For instance:
794 #pragma omp parallel for schedule (guided, i * 4)
799 # BLOCK 2 (PAR_ENTRY_BB)
801 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
803 # BLOCK 3 (WS_ENTRY_BB)
804 .omp_data_i = &.omp_data_o;
805 D.1667 = .omp_data_i->i;
807 #pragma omp for schedule (guided, D.1598)
809 When we outline the parallel region, the call to the child function
810 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
811 that value is computed *after* the call site. So, in principle we
812 cannot do the transformation.
814 To see whether the code in WS_ENTRY_BB blocks the combined
815 parallel+workshare call, we collect all the variables used in the
816 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
817 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
820 FIXME. If we had the SSA form built at this point, we could merely
821 hoist the code in block 3 into block 2 and be done with it. But at
822 this point we don't have dataflow information and though we could
823 hack something up here, it is really not worth the aggravation. */
826 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
828 struct omp_for_data fd
;
829 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
831 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
834 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
836 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
838 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
840 if (fd
.iter_type
!= long_integer_type_node
)
843 /* FIXME. We give up too easily here. If any of these arguments
844 are not constants, they will likely involve variables that have
845 been mapped into fields of .omp_data_s for sharing with the child
846 function. With appropriate data flow, it would be possible to
848 if (!is_gimple_min_invariant (fd
.loop
.n1
)
849 || !is_gimple_min_invariant (fd
.loop
.n2
)
850 || !is_gimple_min_invariant (fd
.loop
.step
)
851 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
858 static int omp_max_vf (void);
860 /* Adjust CHUNK_SIZE from SCHEDULE clause, depending on simd modifier
861 presence (SIMD_SCHEDULE). */
864 omp_adjust_chunk_size (tree chunk_size
, bool simd_schedule
)
869 int vf
= omp_max_vf ();
873 tree type
= TREE_TYPE (chunk_size
);
874 chunk_size
= fold_build2 (PLUS_EXPR
, type
, chunk_size
,
875 build_int_cst (type
, vf
- 1));
876 return fold_build2 (BIT_AND_EXPR
, type
, chunk_size
,
877 build_int_cst (type
, -vf
));
881 /* Collect additional arguments needed to emit a combined
882 parallel+workshare call. WS_STMT is the workshare directive being
885 static vec
<tree
, va_gc
> *
886 get_ws_args_for (gimple
*par_stmt
, gimple
*ws_stmt
)
889 location_t loc
= gimple_location (ws_stmt
);
890 vec
<tree
, va_gc
> *ws_args
;
892 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
894 struct omp_for_data fd
;
897 extract_omp_for_data (for_stmt
, &fd
, NULL
);
901 if (gimple_omp_for_combined_into_p (for_stmt
))
904 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
905 OMP_CLAUSE__LOOPTEMP_
);
907 n1
= OMP_CLAUSE_DECL (innerc
);
908 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
909 OMP_CLAUSE__LOOPTEMP_
);
911 n2
= OMP_CLAUSE_DECL (innerc
);
914 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
916 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
917 ws_args
->quick_push (t
);
919 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
920 ws_args
->quick_push (t
);
922 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
923 ws_args
->quick_push (t
);
927 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
928 t
= omp_adjust_chunk_size (t
, fd
.simd_schedule
);
929 ws_args
->quick_push (t
);
934 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
936 /* Number of sections is equal to the number of edges from the
937 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
938 the exit of the sections region. */
939 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
940 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
941 vec_alloc (ws_args
, 1);
942 ws_args
->quick_push (t
);
950 /* Discover whether REGION is a combined parallel+workshare region. */
953 determine_parallel_type (struct omp_region
*region
)
955 basic_block par_entry_bb
, par_exit_bb
;
956 basic_block ws_entry_bb
, ws_exit_bb
;
958 if (region
== NULL
|| region
->inner
== NULL
959 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
960 || region
->inner
->cont
== NULL
)
963 /* We only support parallel+for and parallel+sections. */
964 if (region
->type
!= GIMPLE_OMP_PARALLEL
965 || (region
->inner
->type
!= GIMPLE_OMP_FOR
966 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
969 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
970 WS_EXIT_BB -> PAR_EXIT_BB. */
971 par_entry_bb
= region
->entry
;
972 par_exit_bb
= region
->exit
;
973 ws_entry_bb
= region
->inner
->entry
;
974 ws_exit_bb
= region
->inner
->exit
;
976 if (single_succ (par_entry_bb
) == ws_entry_bb
977 && single_succ (ws_exit_bb
) == par_exit_bb
978 && workshare_safe_to_combine_p (ws_entry_bb
)
979 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
980 || (last_and_only_stmt (ws_entry_bb
)
981 && last_and_only_stmt (par_exit_bb
))))
983 gimple
*par_stmt
= last_stmt (par_entry_bb
);
984 gimple
*ws_stmt
= last_stmt (ws_entry_bb
);
986 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
988 /* If this is a combined parallel loop, we need to determine
989 whether or not to use the combined library calls. There
990 are two cases where we do not apply the transformation:
991 static loops and any kind of ordered loop. In the first
992 case, we already open code the loop so there is no need
993 to do anything else. In the latter case, the combined
994 parallel loop call would still need extra synchronization
995 to implement ordered semantics, so there would not be any
996 gain in using the combined call. */
997 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
998 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
1000 || ((OMP_CLAUSE_SCHEDULE_KIND (c
) & OMP_CLAUSE_SCHEDULE_MASK
)
1001 == OMP_CLAUSE_SCHEDULE_STATIC
)
1002 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
1004 region
->is_combined_parallel
= false;
1005 region
->inner
->is_combined_parallel
= false;
1010 region
->is_combined_parallel
= true;
1011 region
->inner
->is_combined_parallel
= true;
1012 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
1017 /* Return true if EXPR is variable sized. */
1020 is_variable_sized (const_tree expr
)
1022 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
1025 /* Return true if DECL is a reference type. */
1028 is_reference (tree decl
)
1030 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
1033 /* Return the type of a decl. If the decl is reference type,
1034 return its base type. */
1036 get_base_type (tree decl
)
1038 tree type
= TREE_TYPE (decl
);
1039 if (is_reference (decl
))
1040 type
= TREE_TYPE (type
);
1044 /* Lookup variables. The "maybe" form
1045 allows for the variable form to not have been entered, otherwise we
1046 assert that the variable must have been entered. */
1049 lookup_decl (tree var
, omp_context
*ctx
)
1051 tree
*n
= ctx
->cb
.decl_map
->get (var
);
1056 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
1058 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
1059 return n
? *n
: NULL_TREE
;
1063 lookup_field (tree var
, omp_context
*ctx
)
1066 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
1067 return (tree
) n
->value
;
1071 lookup_sfield (splay_tree_key key
, omp_context
*ctx
)
1074 n
= splay_tree_lookup (ctx
->sfield_map
1075 ? ctx
->sfield_map
: ctx
->field_map
, key
);
1076 return (tree
) n
->value
;
1080 lookup_sfield (tree var
, omp_context
*ctx
)
1082 return lookup_sfield ((splay_tree_key
) var
, ctx
);
1086 maybe_lookup_field (splay_tree_key key
, omp_context
*ctx
)
1089 n
= splay_tree_lookup (ctx
->field_map
, key
);
1090 return n
? (tree
) n
->value
: NULL_TREE
;
1094 maybe_lookup_field (tree var
, omp_context
*ctx
)
1096 return maybe_lookup_field ((splay_tree_key
) var
, ctx
);
1099 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1100 the parallel context if DECL is to be shared. */
1103 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1105 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1108 /* We can only use copy-in/copy-out semantics for shared variables
1109 when we know the value is not accessible from an outer scope. */
1112 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1114 /* ??? Trivially accessible from anywhere. But why would we even
1115 be passing an address in this case? Should we simply assert
1116 this to be false, or should we have a cleanup pass that removes
1117 these from the list of mappings? */
1118 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1121 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1122 without analyzing the expression whether or not its location
1123 is accessible to anyone else. In the case of nested parallel
1124 regions it certainly may be. */
1125 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1128 /* Do not use copy-in/copy-out for variables that have their
1130 if (TREE_ADDRESSABLE (decl
))
1133 /* lower_send_shared_vars only uses copy-in, but not copy-out
1135 if (TREE_READONLY (decl
)
1136 || ((TREE_CODE (decl
) == RESULT_DECL
1137 || TREE_CODE (decl
) == PARM_DECL
)
1138 && DECL_BY_REFERENCE (decl
)))
1141 /* Disallow copy-in/out in nested parallel if
1142 decl is shared in outer parallel, otherwise
1143 each thread could store the shared variable
1144 in its own copy-in location, making the
1145 variable no longer really shared. */
1146 if (shared_ctx
->is_nested
)
1150 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1151 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1158 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1159 c
; c
= OMP_CLAUSE_CHAIN (c
))
1160 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1161 && OMP_CLAUSE_DECL (c
) == decl
)
1165 goto maybe_mark_addressable_and_ret
;
1169 /* For tasks avoid using copy-in/out. As tasks can be
1170 deferred or executed in different thread, when GOMP_task
1171 returns, the task hasn't necessarily terminated. */
1172 if (is_task_ctx (shared_ctx
))
1175 maybe_mark_addressable_and_ret
:
1176 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1177 if (is_gimple_reg (outer
) && !omp_member_access_dummy_var (outer
))
1179 /* Taking address of OUTER in lower_send_shared_vars
1180 might need regimplification of everything that uses the
1182 if (!task_shared_vars
)
1183 task_shared_vars
= BITMAP_ALLOC (NULL
);
1184 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1185 TREE_ADDRESSABLE (outer
) = 1;
1194 /* Construct a new automatic decl similar to VAR. */
1197 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1199 tree copy
= copy_var_decl (var
, name
, type
);
1201 DECL_CONTEXT (copy
) = current_function_decl
;
1202 DECL_CHAIN (copy
) = ctx
->block_vars
;
1203 /* If VAR is listed in task_shared_vars, it means it wasn't
1204 originally addressable and is just because task needs to take
1205 it's address. But we don't need to take address of privatizations
1207 if (TREE_ADDRESSABLE (var
)
1209 && bitmap_bit_p (task_shared_vars
, DECL_UID (var
)))
1210 TREE_ADDRESSABLE (copy
) = 0;
1211 ctx
->block_vars
= copy
;
1217 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1219 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1222 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1225 omp_build_component_ref (tree obj
, tree field
)
1227 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1228 if (TREE_THIS_VOLATILE (field
))
1229 TREE_THIS_VOLATILE (ret
) |= 1;
1230 if (TREE_READONLY (field
))
1231 TREE_READONLY (ret
) |= 1;
1235 /* Build tree nodes to access the field for VAR on the receiver side. */
1238 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1240 tree x
, field
= lookup_field (var
, ctx
);
1242 /* If the receiver record type was remapped in the child function,
1243 remap the field into the new record type. */
1244 x
= maybe_lookup_field (field
, ctx
);
1248 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1249 TREE_THIS_NOTRAP (x
) = 1;
1250 x
= omp_build_component_ref (x
, field
);
1252 x
= build_simple_mem_ref (x
);
1257 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1258 of a parallel, this is a component reference; for workshare constructs
1259 this is some variable. */
1262 build_outer_var_ref (tree var
, omp_context
*ctx
, bool lastprivate
= false)
1266 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1268 else if (is_variable_sized (var
))
1270 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1271 x
= build_outer_var_ref (x
, ctx
, lastprivate
);
1272 x
= build_simple_mem_ref (x
);
1274 else if (is_taskreg_ctx (ctx
))
1276 bool by_ref
= use_pointer_for_field (var
, NULL
);
1277 x
= build_receiver_ref (var
, by_ref
, ctx
);
1279 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1280 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1282 /* #pragma omp simd isn't a worksharing construct, and can reference even
1283 private vars in its linear etc. clauses. */
1285 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1286 x
= lookup_decl (var
, ctx
->outer
);
1287 else if (ctx
->outer
)
1288 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1292 else if (lastprivate
&& is_taskloop_ctx (ctx
))
1294 gcc_assert (ctx
->outer
);
1296 = splay_tree_lookup (ctx
->outer
->field_map
,
1297 (splay_tree_key
) &DECL_UID (var
));
1300 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
->outer
)))
1303 x
= lookup_decl (var
, ctx
->outer
);
1307 tree field
= (tree
) n
->value
;
1308 /* If the receiver record type was remapped in the child function,
1309 remap the field into the new record type. */
1310 x
= maybe_lookup_field (field
, ctx
->outer
);
1314 x
= build_simple_mem_ref (ctx
->outer
->receiver_decl
);
1315 x
= omp_build_component_ref (x
, field
);
1316 if (use_pointer_for_field (var
, ctx
->outer
))
1317 x
= build_simple_mem_ref (x
);
1320 else if (ctx
->outer
)
1321 x
= lookup_decl (var
, ctx
->outer
);
1322 else if (is_reference (var
))
1323 /* This can happen with orphaned constructs. If var is reference, it is
1324 possible it is shared and as such valid. */
1326 else if (omp_member_access_dummy_var (var
))
1333 tree t
= omp_member_access_dummy_var (var
);
1336 x
= DECL_VALUE_EXPR (var
);
1337 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
1339 x
= unshare_and_remap (x
, t
, o
);
1341 x
= unshare_expr (x
);
1345 if (is_reference (var
))
1346 x
= build_simple_mem_ref (x
);
1351 /* Build tree nodes to access the field for VAR on the sender side. */
1354 build_sender_ref (splay_tree_key key
, omp_context
*ctx
)
1356 tree field
= lookup_sfield (key
, ctx
);
1357 return omp_build_component_ref (ctx
->sender_decl
, field
);
1361 build_sender_ref (tree var
, omp_context
*ctx
)
1363 return build_sender_ref ((splay_tree_key
) var
, ctx
);
1366 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1369 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1371 tree field
, type
, sfield
= NULL_TREE
;
1372 splay_tree_key key
= (splay_tree_key
) var
;
1374 if ((mask
& 8) != 0)
1376 key
= (splay_tree_key
) &DECL_UID (var
);
1377 gcc_checking_assert (key
!= (splay_tree_key
) var
);
1379 gcc_assert ((mask
& 1) == 0
1380 || !splay_tree_lookup (ctx
->field_map
, key
));
1381 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1382 || !splay_tree_lookup (ctx
->sfield_map
, key
));
1383 gcc_assert ((mask
& 3) == 3
1384 || !is_gimple_omp_oacc (ctx
->stmt
));
1386 type
= TREE_TYPE (var
);
1389 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1390 type
= build_pointer_type (build_pointer_type (type
));
1393 type
= build_pointer_type (type
);
1394 else if ((mask
& 3) == 1 && is_reference (var
))
1395 type
= TREE_TYPE (type
);
1397 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1398 FIELD_DECL
, DECL_NAME (var
), type
);
1400 /* Remember what variable this field was created for. This does have a
1401 side effect of making dwarf2out ignore this member, so for helpful
1402 debugging we clear it later in delete_omp_context. */
1403 DECL_ABSTRACT_ORIGIN (field
) = var
;
1404 if (type
== TREE_TYPE (var
))
1406 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1407 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1408 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1411 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1413 if ((mask
& 3) == 3)
1415 insert_field_into_struct (ctx
->record_type
, field
);
1416 if (ctx
->srecord_type
)
1418 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1419 FIELD_DECL
, DECL_NAME (var
), type
);
1420 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1421 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1422 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1423 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1424 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1429 if (ctx
->srecord_type
== NULL_TREE
)
1433 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1434 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1435 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1437 sfield
= build_decl (DECL_SOURCE_LOCATION (t
),
1438 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1439 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1440 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1441 splay_tree_insert (ctx
->sfield_map
,
1442 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1443 (splay_tree_value
) sfield
);
1447 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1448 : ctx
->srecord_type
, field
);
1452 splay_tree_insert (ctx
->field_map
, key
, (splay_tree_value
) field
);
1453 if ((mask
& 2) && ctx
->sfield_map
)
1454 splay_tree_insert (ctx
->sfield_map
, key
, (splay_tree_value
) sfield
);
1458 install_var_local (tree var
, omp_context
*ctx
)
1460 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1461 insert_decl_map (&ctx
->cb
, var
, new_var
);
1465 /* Adjust the replacement for DECL in CTX for the new context. This means
1466 copying the DECL_VALUE_EXPR, and fixing up the type. */
1469 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1471 tree new_decl
, size
;
1473 new_decl
= lookup_decl (decl
, ctx
);
1475 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1477 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1478 && DECL_HAS_VALUE_EXPR_P (decl
))
1480 tree ve
= DECL_VALUE_EXPR (decl
);
1481 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1482 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1483 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1486 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1488 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1489 if (size
== error_mark_node
)
1490 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1491 DECL_SIZE (new_decl
) = size
;
1493 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1494 if (size
== error_mark_node
)
1495 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1496 DECL_SIZE_UNIT (new_decl
) = size
;
1500 /* The callback for remap_decl. Search all containing contexts for a
1501 mapping of the variable; this avoids having to duplicate the splay
1502 tree ahead of time. We know a mapping doesn't already exist in the
1503 given context. Create new mappings to implement default semantics. */
1506 omp_copy_decl (tree var
, copy_body_data
*cb
)
1508 omp_context
*ctx
= (omp_context
*) cb
;
1511 if (TREE_CODE (var
) == LABEL_DECL
)
1513 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1514 DECL_CONTEXT (new_var
) = current_function_decl
;
1515 insert_decl_map (&ctx
->cb
, var
, new_var
);
1519 while (!is_taskreg_ctx (ctx
))
1524 new_var
= maybe_lookup_decl (var
, ctx
);
1529 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1532 return error_mark_node
;
1536 /* Debugging dumps for parallel regions. */
1537 void dump_omp_region (FILE *, struct omp_region
*, int);
1538 void debug_omp_region (struct omp_region
*);
1539 void debug_all_omp_regions (void);
1541 /* Dump the parallel region tree rooted at REGION. */
1544 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1546 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1547 gimple_code_name
[region
->type
]);
1550 dump_omp_region (file
, region
->inner
, indent
+ 4);
1554 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1555 region
->cont
->index
);
1559 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1560 region
->exit
->index
);
1562 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1565 dump_omp_region (file
, region
->next
, indent
);
1569 debug_omp_region (struct omp_region
*region
)
1571 dump_omp_region (stderr
, region
, 0);
1575 debug_all_omp_regions (void)
1577 dump_omp_region (stderr
, root_omp_region
, 0);
1581 /* Create a new parallel region starting at STMT inside region PARENT. */
1583 static struct omp_region
*
1584 new_omp_region (basic_block bb
, enum gimple_code type
,
1585 struct omp_region
*parent
)
1587 struct omp_region
*region
= XCNEW (struct omp_region
);
1589 region
->outer
= parent
;
1591 region
->type
= type
;
1595 /* This is a nested region. Add it to the list of inner
1596 regions in PARENT. */
1597 region
->next
= parent
->inner
;
1598 parent
->inner
= region
;
1602 /* This is a toplevel region. Add it to the list of toplevel
1603 regions in ROOT_OMP_REGION. */
1604 region
->next
= root_omp_region
;
1605 root_omp_region
= region
;
1611 /* Release the memory associated with the region tree rooted at REGION. */
1614 free_omp_region_1 (struct omp_region
*region
)
1616 struct omp_region
*i
, *n
;
1618 for (i
= region
->inner
; i
; i
= n
)
1621 free_omp_region_1 (i
);
1627 /* Release the memory for the entire omp region tree. */
1630 free_omp_regions (void)
1632 struct omp_region
*r
, *n
;
1633 for (r
= root_omp_region
; r
; r
= n
)
1636 free_omp_region_1 (r
);
1638 root_omp_region
= NULL
;
1642 /* Create a new context, with OUTER_CTX being the surrounding context. */
1644 static omp_context
*
1645 new_omp_context (gimple
*stmt
, omp_context
*outer_ctx
)
1647 omp_context
*ctx
= XCNEW (omp_context
);
1649 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1650 (splay_tree_value
) ctx
);
1655 ctx
->outer
= outer_ctx
;
1656 ctx
->cb
= outer_ctx
->cb
;
1657 ctx
->cb
.block
= NULL
;
1658 ctx
->depth
= outer_ctx
->depth
+ 1;
1662 ctx
->cb
.src_fn
= current_function_decl
;
1663 ctx
->cb
.dst_fn
= current_function_decl
;
1664 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1665 gcc_checking_assert (ctx
->cb
.src_node
);
1666 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1667 ctx
->cb
.src_cfun
= cfun
;
1668 ctx
->cb
.copy_decl
= omp_copy_decl
;
1669 ctx
->cb
.eh_lp_nr
= 0;
1670 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1674 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1679 static gimple_seq
maybe_catch_exception (gimple_seq
);
1681 /* Finalize task copyfn. */
1684 finalize_task_copyfn (gomp_task
*task_stmt
)
1686 struct function
*child_cfun
;
1688 gimple_seq seq
= NULL
, new_seq
;
1691 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1692 if (child_fn
== NULL_TREE
)
1695 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1696 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1698 push_cfun (child_cfun
);
1699 bind
= gimplify_body (child_fn
, false);
1700 gimple_seq_add_stmt (&seq
, bind
);
1701 new_seq
= maybe_catch_exception (seq
);
1704 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1706 gimple_seq_add_stmt (&seq
, bind
);
1708 gimple_set_body (child_fn
, seq
);
1711 /* Inform the callgraph about the new function. */
1712 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
1713 node
->parallelized_function
= 1;
1714 cgraph_node::add_new_function (child_fn
, false);
1717 /* Destroy a omp_context data structures. Called through the splay tree
1718 value delete callback. */
1721 delete_omp_context (splay_tree_value value
)
1723 omp_context
*ctx
= (omp_context
*) value
;
1725 delete ctx
->cb
.decl_map
;
1728 splay_tree_delete (ctx
->field_map
);
1729 if (ctx
->sfield_map
)
1730 splay_tree_delete (ctx
->sfield_map
);
1732 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1733 it produces corrupt debug information. */
1734 if (ctx
->record_type
)
1737 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1738 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1740 if (ctx
->srecord_type
)
1743 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1744 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1747 if (is_task_ctx (ctx
))
1748 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1753 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1757 fixup_child_record_type (omp_context
*ctx
)
1759 tree f
, type
= ctx
->record_type
;
1761 /* ??? It isn't sufficient to just call remap_type here, because
1762 variably_modified_type_p doesn't work the way we expect for
1763 record types. Testing each field for whether it needs remapping
1764 and creating a new record by hand works, however. */
1765 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1766 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1770 tree name
, new_fields
= NULL
;
1772 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1773 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1774 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1775 TYPE_DECL
, name
, type
);
1776 TYPE_NAME (type
) = name
;
1778 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1780 tree new_f
= copy_node (f
);
1781 DECL_CONTEXT (new_f
) = type
;
1782 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1783 DECL_CHAIN (new_f
) = new_fields
;
1784 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1785 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1787 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1791 /* Arrange to be able to look up the receiver field
1792 given the sender field. */
1793 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1794 (splay_tree_value
) new_f
);
1796 TYPE_FIELDS (type
) = nreverse (new_fields
);
1800 /* In a target region we never modify any of the pointers in *.omp_data_i,
1801 so attempt to help the optimizers. */
1802 if (is_gimple_omp_offloaded (ctx
->stmt
))
1803 type
= build_qualified_type (type
, TYPE_QUAL_CONST
);
1805 TREE_TYPE (ctx
->receiver_decl
)
1806 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1809 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1810 specified by CLAUSES. */
1813 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1816 bool scan_array_reductions
= false;
1818 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1822 switch (OMP_CLAUSE_CODE (c
))
1824 case OMP_CLAUSE_PRIVATE
:
1825 decl
= OMP_CLAUSE_DECL (c
);
1826 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1828 else if (!is_variable_sized (decl
))
1829 install_var_local (decl
, ctx
);
1832 case OMP_CLAUSE_SHARED
:
1833 decl
= OMP_CLAUSE_DECL (c
);
1834 /* Ignore shared directives in teams construct. */
1835 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1837 /* Global variables don't need to be copied,
1838 the receiver side will use them directly. */
1839 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1840 if (is_global_var (odecl
))
1842 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1845 gcc_assert (is_taskreg_ctx (ctx
));
1846 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1847 || !is_variable_sized (decl
));
1848 /* Global variables don't need to be copied,
1849 the receiver side will use them directly. */
1850 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1852 by_ref
= use_pointer_for_field (decl
, ctx
);
1853 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
1855 if (! TREE_READONLY (decl
)
1856 || TREE_ADDRESSABLE (decl
)
1858 || is_reference (decl
))
1860 install_var_field (decl
, by_ref
, 3, ctx
);
1861 install_var_local (decl
, ctx
);
1864 /* We don't need to copy const scalar vars back. */
1865 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1868 case OMP_CLAUSE_REDUCTION
:
1869 decl
= OMP_CLAUSE_DECL (c
);
1870 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1871 && TREE_CODE (decl
) == MEM_REF
)
1873 tree t
= TREE_OPERAND (decl
, 0);
1874 if (TREE_CODE (t
) == POINTER_PLUS_EXPR
)
1875 t
= TREE_OPERAND (t
, 0);
1876 if (TREE_CODE (t
) == INDIRECT_REF
1877 || TREE_CODE (t
) == ADDR_EXPR
)
1878 t
= TREE_OPERAND (t
, 0);
1879 install_var_local (t
, ctx
);
1880 if (is_taskreg_ctx (ctx
)
1881 && !is_global_var (maybe_lookup_decl_in_outer_ctx (t
, ctx
))
1882 && !is_variable_sized (t
))
1884 by_ref
= use_pointer_for_field (t
, ctx
);
1885 install_var_field (t
, by_ref
, 3, ctx
);
1891 case OMP_CLAUSE_LASTPRIVATE
:
1892 /* Let the corresponding firstprivate clause create
1894 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1898 case OMP_CLAUSE_FIRSTPRIVATE
:
1899 case OMP_CLAUSE_LINEAR
:
1900 decl
= OMP_CLAUSE_DECL (c
);
1902 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
1903 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
1904 && is_gimple_omp_offloaded (ctx
->stmt
))
1906 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
1907 install_var_field (decl
, !is_reference (decl
), 3, ctx
);
1908 else if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1909 install_var_field (decl
, true, 3, ctx
);
1911 install_var_field (decl
, false, 3, ctx
);
1913 if (is_variable_sized (decl
))
1915 if (is_task_ctx (ctx
))
1916 install_var_field (decl
, false, 1, ctx
);
1919 else if (is_taskreg_ctx (ctx
))
1922 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1923 by_ref
= use_pointer_for_field (decl
, NULL
);
1925 if (is_task_ctx (ctx
)
1926 && (global
|| by_ref
|| is_reference (decl
)))
1928 install_var_field (decl
, false, 1, ctx
);
1930 install_var_field (decl
, by_ref
, 2, ctx
);
1933 install_var_field (decl
, by_ref
, 3, ctx
);
1935 install_var_local (decl
, ctx
);
1938 case OMP_CLAUSE_USE_DEVICE_PTR
:
1939 decl
= OMP_CLAUSE_DECL (c
);
1940 if (TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1941 install_var_field (decl
, true, 3, ctx
);
1943 install_var_field (decl
, false, 3, ctx
);
1944 if (DECL_SIZE (decl
)
1945 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1947 tree decl2
= DECL_VALUE_EXPR (decl
);
1948 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1949 decl2
= TREE_OPERAND (decl2
, 0);
1950 gcc_assert (DECL_P (decl2
));
1951 install_var_local (decl2
, ctx
);
1953 install_var_local (decl
, ctx
);
1956 case OMP_CLAUSE_IS_DEVICE_PTR
:
1957 decl
= OMP_CLAUSE_DECL (c
);
1960 case OMP_CLAUSE__LOOPTEMP_
:
1961 gcc_assert (is_taskreg_ctx (ctx
));
1962 decl
= OMP_CLAUSE_DECL (c
);
1963 install_var_field (decl
, false, 3, ctx
);
1964 install_var_local (decl
, ctx
);
1967 case OMP_CLAUSE_COPYPRIVATE
:
1968 case OMP_CLAUSE_COPYIN
:
1969 decl
= OMP_CLAUSE_DECL (c
);
1970 by_ref
= use_pointer_for_field (decl
, NULL
);
1971 install_var_field (decl
, by_ref
, 3, ctx
);
1974 case OMP_CLAUSE_DEFAULT
:
1975 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1978 case OMP_CLAUSE_FINAL
:
1980 case OMP_CLAUSE_NUM_THREADS
:
1981 case OMP_CLAUSE_NUM_TEAMS
:
1982 case OMP_CLAUSE_THREAD_LIMIT
:
1983 case OMP_CLAUSE_DEVICE
:
1984 case OMP_CLAUSE_SCHEDULE
:
1985 case OMP_CLAUSE_DIST_SCHEDULE
:
1986 case OMP_CLAUSE_DEPEND
:
1987 case OMP_CLAUSE_PRIORITY
:
1988 case OMP_CLAUSE_GRAINSIZE
:
1989 case OMP_CLAUSE_NUM_TASKS
:
1990 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1991 case OMP_CLAUSE_NUM_GANGS
:
1992 case OMP_CLAUSE_NUM_WORKERS
:
1993 case OMP_CLAUSE_VECTOR_LENGTH
:
1995 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1999 case OMP_CLAUSE_FROM
:
2000 case OMP_CLAUSE_MAP
:
2002 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
2003 decl
= OMP_CLAUSE_DECL (c
);
2004 /* Global variables with "omp declare target" attribute
2005 don't need to be copied, the receiver side will use them
2007 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2009 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2010 && (OMP_CLAUSE_MAP_KIND (c
)
2011 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2012 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2013 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2014 && varpool_node::get_create (decl
)->offloadable
)
2016 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2017 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
2019 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
2020 not offloaded; there is nothing to map for those. */
2021 if (!is_gimple_omp_offloaded (ctx
->stmt
)
2022 && !POINTER_TYPE_P (TREE_TYPE (decl
))
2023 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
2026 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2027 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
2028 || (OMP_CLAUSE_MAP_KIND (c
)
2029 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
2031 if (TREE_CODE (decl
) == COMPONENT_REF
2032 || (TREE_CODE (decl
) == INDIRECT_REF
2033 && TREE_CODE (TREE_OPERAND (decl
, 0)) == COMPONENT_REF
2034 && (TREE_CODE (TREE_TYPE (TREE_OPERAND (decl
, 0)))
2035 == REFERENCE_TYPE
)))
2037 if (DECL_SIZE (decl
)
2038 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2040 tree decl2
= DECL_VALUE_EXPR (decl
);
2041 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2042 decl2
= TREE_OPERAND (decl2
, 0);
2043 gcc_assert (DECL_P (decl2
));
2044 install_var_local (decl2
, ctx
);
2046 install_var_local (decl
, ctx
);
2051 if (DECL_SIZE (decl
)
2052 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2054 tree decl2
= DECL_VALUE_EXPR (decl
);
2055 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2056 decl2
= TREE_OPERAND (decl2
, 0);
2057 gcc_assert (DECL_P (decl2
));
2058 install_var_field (decl2
, true, 3, ctx
);
2059 install_var_local (decl2
, ctx
);
2060 install_var_local (decl
, ctx
);
2064 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
2065 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2066 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
2067 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2068 install_var_field (decl
, true, 7, ctx
);
2070 install_var_field (decl
, true, 3, ctx
);
2071 if (is_gimple_omp_offloaded (ctx
->stmt
))
2072 install_var_local (decl
, ctx
);
2077 tree base
= get_base_address (decl
);
2078 tree nc
= OMP_CLAUSE_CHAIN (c
);
2081 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
2082 && OMP_CLAUSE_DECL (nc
) == base
2083 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
2084 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
2086 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
2087 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
2093 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
2094 decl
= OMP_CLAUSE_DECL (c
);
2096 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
2097 (splay_tree_key
) decl
));
2099 = build_decl (OMP_CLAUSE_LOCATION (c
),
2100 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
2101 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
2102 insert_field_into_struct (ctx
->record_type
, field
);
2103 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
2104 (splay_tree_value
) field
);
2109 case OMP_CLAUSE_NOWAIT
:
2110 case OMP_CLAUSE_ORDERED
:
2111 case OMP_CLAUSE_COLLAPSE
:
2112 case OMP_CLAUSE_UNTIED
:
2113 case OMP_CLAUSE_MERGEABLE
:
2114 case OMP_CLAUSE_PROC_BIND
:
2115 case OMP_CLAUSE_SAFELEN
:
2116 case OMP_CLAUSE_SIMDLEN
:
2117 case OMP_CLAUSE_THREADS
:
2118 case OMP_CLAUSE_SIMD
:
2119 case OMP_CLAUSE_NOGROUP
:
2120 case OMP_CLAUSE_DEFAULTMAP
:
2121 case OMP_CLAUSE_ASYNC
:
2122 case OMP_CLAUSE_WAIT
:
2123 case OMP_CLAUSE_GANG
:
2124 case OMP_CLAUSE_WORKER
:
2125 case OMP_CLAUSE_VECTOR
:
2126 case OMP_CLAUSE_TILE
:
2129 case OMP_CLAUSE_ALIGNED
:
2130 decl
= OMP_CLAUSE_DECL (c
);
2131 if (is_global_var (decl
)
2132 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2133 install_var_local (decl
, ctx
);
2136 case OMP_CLAUSE_DEVICE_RESIDENT
:
2137 case OMP_CLAUSE_USE_DEVICE
:
2138 case OMP_CLAUSE__CACHE_
:
2139 case OMP_CLAUSE_INDEPENDENT
:
2140 case OMP_CLAUSE_AUTO
:
2141 case OMP_CLAUSE_SEQ
:
2142 sorry ("Clause not supported yet");
2150 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2152 switch (OMP_CLAUSE_CODE (c
))
2154 case OMP_CLAUSE_LASTPRIVATE
:
2155 /* Let the corresponding firstprivate clause create
2157 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2158 scan_array_reductions
= true;
2159 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
2163 case OMP_CLAUSE_FIRSTPRIVATE
:
2164 case OMP_CLAUSE_PRIVATE
:
2165 case OMP_CLAUSE_LINEAR
:
2166 case OMP_CLAUSE_IS_DEVICE_PTR
:
2167 decl
= OMP_CLAUSE_DECL (c
);
2168 if (is_variable_sized (decl
))
2170 if ((OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
2171 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_IS_DEVICE_PTR
)
2172 && is_gimple_omp_offloaded (ctx
->stmt
))
2174 tree decl2
= DECL_VALUE_EXPR (decl
);
2175 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2176 decl2
= TREE_OPERAND (decl2
, 0);
2177 gcc_assert (DECL_P (decl2
));
2178 install_var_local (decl2
, ctx
);
2179 fixup_remapped_decl (decl2
, ctx
, false);
2181 install_var_local (decl
, ctx
);
2183 fixup_remapped_decl (decl
, ctx
,
2184 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
2185 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
2186 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2187 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2188 scan_array_reductions
= true;
2191 case OMP_CLAUSE_REDUCTION
:
2192 decl
= OMP_CLAUSE_DECL (c
);
2193 if (TREE_CODE (decl
) != MEM_REF
)
2195 if (is_variable_sized (decl
))
2196 install_var_local (decl
, ctx
);
2197 fixup_remapped_decl (decl
, ctx
, false);
2199 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2200 scan_array_reductions
= true;
2203 case OMP_CLAUSE_SHARED
:
2204 /* Ignore shared directives in teams construct. */
2205 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2207 decl
= OMP_CLAUSE_DECL (c
);
2208 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2210 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2212 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
,
2215 bool by_ref
= use_pointer_for_field (decl
, ctx
);
2216 install_var_field (decl
, by_ref
, 11, ctx
);
2219 fixup_remapped_decl (decl
, ctx
, false);
2222 case OMP_CLAUSE_MAP
:
2223 if (!is_gimple_omp_offloaded (ctx
->stmt
))
2225 decl
= OMP_CLAUSE_DECL (c
);
2227 && ((OMP_CLAUSE_MAP_KIND (c
) != GOMP_MAP_FIRSTPRIVATE_POINTER
2228 && (OMP_CLAUSE_MAP_KIND (c
)
2229 != GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
2230 || TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
2231 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2232 && varpool_node::get_create (decl
)->offloadable
)
2236 if ((OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2237 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
)
2238 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
2239 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
2241 tree new_decl
= lookup_decl (decl
, ctx
);
2242 TREE_TYPE (new_decl
)
2243 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2245 else if (DECL_SIZE (decl
)
2246 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2248 tree decl2
= DECL_VALUE_EXPR (decl
);
2249 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2250 decl2
= TREE_OPERAND (decl2
, 0);
2251 gcc_assert (DECL_P (decl2
));
2252 fixup_remapped_decl (decl2
, ctx
, false);
2253 fixup_remapped_decl (decl
, ctx
, true);
2256 fixup_remapped_decl (decl
, ctx
, false);
2260 case OMP_CLAUSE_COPYPRIVATE
:
2261 case OMP_CLAUSE_COPYIN
:
2262 case OMP_CLAUSE_DEFAULT
:
2264 case OMP_CLAUSE_NUM_THREADS
:
2265 case OMP_CLAUSE_NUM_TEAMS
:
2266 case OMP_CLAUSE_THREAD_LIMIT
:
2267 case OMP_CLAUSE_DEVICE
:
2268 case OMP_CLAUSE_SCHEDULE
:
2269 case OMP_CLAUSE_DIST_SCHEDULE
:
2270 case OMP_CLAUSE_NOWAIT
:
2271 case OMP_CLAUSE_ORDERED
:
2272 case OMP_CLAUSE_COLLAPSE
:
2273 case OMP_CLAUSE_UNTIED
:
2274 case OMP_CLAUSE_FINAL
:
2275 case OMP_CLAUSE_MERGEABLE
:
2276 case OMP_CLAUSE_PROC_BIND
:
2277 case OMP_CLAUSE_SAFELEN
:
2278 case OMP_CLAUSE_SIMDLEN
:
2279 case OMP_CLAUSE_ALIGNED
:
2280 case OMP_CLAUSE_DEPEND
:
2281 case OMP_CLAUSE__LOOPTEMP_
:
2283 case OMP_CLAUSE_FROM
:
2284 case OMP_CLAUSE_PRIORITY
:
2285 case OMP_CLAUSE_GRAINSIZE
:
2286 case OMP_CLAUSE_NUM_TASKS
:
2287 case OMP_CLAUSE_THREADS
:
2288 case OMP_CLAUSE_SIMD
:
2289 case OMP_CLAUSE_NOGROUP
:
2290 case OMP_CLAUSE_DEFAULTMAP
:
2291 case OMP_CLAUSE_USE_DEVICE_PTR
:
2292 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2293 case OMP_CLAUSE_ASYNC
:
2294 case OMP_CLAUSE_WAIT
:
2295 case OMP_CLAUSE_NUM_GANGS
:
2296 case OMP_CLAUSE_NUM_WORKERS
:
2297 case OMP_CLAUSE_VECTOR_LENGTH
:
2298 case OMP_CLAUSE_GANG
:
2299 case OMP_CLAUSE_WORKER
:
2300 case OMP_CLAUSE_VECTOR
:
2301 case OMP_CLAUSE_TILE
:
2304 case OMP_CLAUSE_DEVICE_RESIDENT
:
2305 case OMP_CLAUSE_USE_DEVICE
:
2306 case OMP_CLAUSE__CACHE_
:
2307 case OMP_CLAUSE_INDEPENDENT
:
2308 case OMP_CLAUSE_AUTO
:
2309 case OMP_CLAUSE_SEQ
:
2310 sorry ("Clause not supported yet");
2318 gcc_checking_assert (!scan_array_reductions
2319 || !is_gimple_omp_oacc (ctx
->stmt
));
2320 if (scan_array_reductions
)
2321 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2322 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2323 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2325 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2326 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2328 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2329 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2330 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2331 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2332 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2333 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2336 /* Create a new name for omp child function. Returns an identifier. If
2337 IS_CILK_FOR is true then the suffix for the child function is
2341 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2344 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2345 return clone_function_name (current_function_decl
,
2346 task_copy
? "_omp_cpyfn" : "_omp_fn");
2349 /* Returns the type of the induction variable for the child function for
2350 _Cilk_for and the types for _high and _low variables based on TYPE. */
2353 cilk_for_check_loop_diff_type (tree type
)
2355 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2357 if (TYPE_UNSIGNED (type
))
2358 return uint32_type_node
;
2360 return integer_type_node
;
2364 if (TYPE_UNSIGNED (type
))
2365 return uint64_type_node
;
2367 return long_long_integer_type_node
;
2371 /* Build a decl for the omp child function. It'll not contain a body
2372 yet, just the bare decl. */
2375 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2377 tree decl
, type
, name
, t
;
2380 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2381 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2382 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2383 tree cilk_var_type
= NULL_TREE
;
2385 name
= create_omp_child_function_name (task_copy
,
2386 cilk_for_count
!= NULL_TREE
);
2388 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2389 ptr_type_node
, NULL_TREE
);
2390 else if (cilk_for_count
)
2392 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2393 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2394 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2395 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2398 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2400 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2402 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2405 ctx
->cb
.dst_fn
= decl
;
2407 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2409 TREE_STATIC (decl
) = 1;
2410 TREE_USED (decl
) = 1;
2411 DECL_ARTIFICIAL (decl
) = 1;
2412 DECL_IGNORED_P (decl
) = 0;
2413 TREE_PUBLIC (decl
) = 0;
2414 DECL_UNINLINABLE (decl
) = 1;
2415 DECL_EXTERNAL (decl
) = 0;
2416 DECL_CONTEXT (decl
) = NULL_TREE
;
2417 DECL_INITIAL (decl
) = make_node (BLOCK
);
2418 if (cgraph_node::get (current_function_decl
)->offloadable
)
2419 cgraph_node::get_create (decl
)->offloadable
= 1;
2423 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2424 if (is_gimple_omp_offloaded (octx
->stmt
))
2426 cgraph_node::get_create (decl
)->offloadable
= 1;
2427 #ifdef ENABLE_OFFLOADING
2428 g
->have_offload
= true;
2434 if (cgraph_node::get_create (decl
)->offloadable
2435 && !lookup_attribute ("omp declare target",
2436 DECL_ATTRIBUTES (current_function_decl
)))
2437 DECL_ATTRIBUTES (decl
)
2438 = tree_cons (get_identifier ("omp target entrypoint"),
2439 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2441 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2442 RESULT_DECL
, NULL_TREE
, void_type_node
);
2443 DECL_ARTIFICIAL (t
) = 1;
2444 DECL_IGNORED_P (t
) = 1;
2445 DECL_CONTEXT (t
) = decl
;
2446 DECL_RESULT (decl
) = t
;
2448 /* _Cilk_for's child function requires two extra parameters called
2449 __low and __high that are set the by Cilk runtime when it calls this
2453 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2454 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2455 DECL_ARTIFICIAL (t
) = 1;
2456 DECL_NAMELESS (t
) = 1;
2457 DECL_ARG_TYPE (t
) = ptr_type_node
;
2458 DECL_CONTEXT (t
) = current_function_decl
;
2460 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2461 DECL_ARGUMENTS (decl
) = t
;
2463 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2464 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2465 DECL_ARTIFICIAL (t
) = 1;
2466 DECL_NAMELESS (t
) = 1;
2467 DECL_ARG_TYPE (t
) = ptr_type_node
;
2468 DECL_CONTEXT (t
) = current_function_decl
;
2470 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2471 DECL_ARGUMENTS (decl
) = t
;
2474 tree data_name
= get_identifier (".omp_data_i");
2475 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2477 DECL_ARTIFICIAL (t
) = 1;
2478 DECL_NAMELESS (t
) = 1;
2479 DECL_ARG_TYPE (t
) = ptr_type_node
;
2480 DECL_CONTEXT (t
) = current_function_decl
;
2482 TREE_READONLY (t
) = 1;
2484 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2485 DECL_ARGUMENTS (decl
) = t
;
2487 ctx
->receiver_decl
= t
;
2490 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2491 PARM_DECL
, get_identifier (".omp_data_o"),
2493 DECL_ARTIFICIAL (t
) = 1;
2494 DECL_NAMELESS (t
) = 1;
2495 DECL_ARG_TYPE (t
) = ptr_type_node
;
2496 DECL_CONTEXT (t
) = current_function_decl
;
2498 TREE_ADDRESSABLE (t
) = 1;
2499 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2500 DECL_ARGUMENTS (decl
) = t
;
2503 /* Allocate memory for the function structure. The call to
2504 allocate_struct_function clobbers CFUN, so we need to restore
2506 push_struct_function (decl
);
2507 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2511 /* Callback for walk_gimple_seq. Check if combined parallel
2512 contains gimple_omp_for_combined_into_p OMP_FOR. */
2515 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2516 bool *handled_ops_p
,
2517 struct walk_stmt_info
*wi
)
2519 gimple
*stmt
= gsi_stmt (*gsi_p
);
2521 *handled_ops_p
= true;
2522 switch (gimple_code (stmt
))
2526 case GIMPLE_OMP_FOR
:
2527 if (gimple_omp_for_combined_into_p (stmt
)
2528 && gimple_omp_for_kind (stmt
)
2529 == *(const enum gf_mask
*) (wi
->info
))
2532 return integer_zero_node
;
2541 /* Add _LOOPTEMP_ clauses on OpenMP parallel or task. */
2544 add_taskreg_looptemp_clauses (enum gf_mask msk
, gimple
*stmt
,
2545 omp_context
*outer_ctx
)
2547 struct walk_stmt_info wi
;
2549 memset (&wi
, 0, sizeof (wi
));
2551 wi
.info
= (void *) &msk
;
2552 walk_gimple_seq (gimple_omp_body (stmt
), find_combined_for
, NULL
, &wi
);
2553 if (wi
.info
!= (void *) &msk
)
2555 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
*) wi
.info
);
2556 struct omp_for_data fd
;
2557 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2558 /* We need two temporaries with fd.loop.v type (istart/iend)
2559 and then (fd.collapse - 1) temporaries with the same
2560 type for count2 ... countN-1 vars if not constant. */
2561 size_t count
= 2, i
;
2562 tree type
= fd
.iter_type
;
2564 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2566 count
+= fd
.collapse
- 1;
2567 /* If there are lastprivate clauses on the inner
2568 GIMPLE_OMP_FOR, add one more temporaries for the total number
2569 of iterations (product of count1 ... countN-1). */
2570 if (find_omp_clause (gimple_omp_for_clauses (for_stmt
),
2571 OMP_CLAUSE_LASTPRIVATE
))
2573 else if (msk
== GF_OMP_FOR_KIND_FOR
2574 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2575 OMP_CLAUSE_LASTPRIVATE
))
2578 for (i
= 0; i
< count
; i
++)
2580 tree temp
= create_tmp_var (type
);
2581 tree c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
2582 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2583 OMP_CLAUSE_DECL (c
) = temp
;
2584 OMP_CLAUSE_CHAIN (c
) = gimple_omp_taskreg_clauses (stmt
);
2585 gimple_omp_taskreg_set_clauses (stmt
, c
);
2590 /* Scan an OpenMP parallel directive. */
2593 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2597 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2599 /* Ignore parallel directives with empty bodies, unless there
2600 are copyin clauses. */
2602 && empty_body_p (gimple_omp_body (stmt
))
2603 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2604 OMP_CLAUSE_COPYIN
) == NULL
)
2606 gsi_replace (gsi
, gimple_build_nop (), false);
2610 if (gimple_omp_parallel_combined_p (stmt
))
2611 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR
, stmt
, outer_ctx
);
2613 ctx
= new_omp_context (stmt
, outer_ctx
);
2614 taskreg_contexts
.safe_push (ctx
);
2615 if (taskreg_nesting_level
> 1)
2616 ctx
->is_nested
= true;
2617 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2618 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2619 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2620 name
= create_tmp_var_name (".omp_data_s");
2621 name
= build_decl (gimple_location (stmt
),
2622 TYPE_DECL
, name
, ctx
->record_type
);
2623 DECL_ARTIFICIAL (name
) = 1;
2624 DECL_NAMELESS (name
) = 1;
2625 TYPE_NAME (ctx
->record_type
) = name
;
2626 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2627 create_omp_child_function (ctx
, false);
2628 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2630 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2631 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2633 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2634 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2637 /* Scan an OpenMP task directive. */
2640 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2644 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2646 /* Ignore task directives with empty bodies. */
2648 && empty_body_p (gimple_omp_body (stmt
)))
2650 gsi_replace (gsi
, gimple_build_nop (), false);
2654 if (gimple_omp_task_taskloop_p (stmt
))
2655 add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP
, stmt
, outer_ctx
);
2657 ctx
= new_omp_context (stmt
, outer_ctx
);
2658 taskreg_contexts
.safe_push (ctx
);
2659 if (taskreg_nesting_level
> 1)
2660 ctx
->is_nested
= true;
2661 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2662 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2663 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2664 name
= create_tmp_var_name (".omp_data_s");
2665 name
= build_decl (gimple_location (stmt
),
2666 TYPE_DECL
, name
, ctx
->record_type
);
2667 DECL_ARTIFICIAL (name
) = 1;
2668 DECL_NAMELESS (name
) = 1;
2669 TYPE_NAME (ctx
->record_type
) = name
;
2670 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2671 create_omp_child_function (ctx
, false);
2672 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2674 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2676 if (ctx
->srecord_type
)
2678 name
= create_tmp_var_name (".omp_data_a");
2679 name
= build_decl (gimple_location (stmt
),
2680 TYPE_DECL
, name
, ctx
->srecord_type
);
2681 DECL_ARTIFICIAL (name
) = 1;
2682 DECL_NAMELESS (name
) = 1;
2683 TYPE_NAME (ctx
->srecord_type
) = name
;
2684 TYPE_ARTIFICIAL (ctx
->srecord_type
) = 1;
2685 create_omp_child_function (ctx
, true);
2688 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2690 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2692 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2693 t
= build_int_cst (long_integer_type_node
, 0);
2694 gimple_omp_task_set_arg_size (stmt
, t
);
2695 t
= build_int_cst (long_integer_type_node
, 1);
2696 gimple_omp_task_set_arg_align (stmt
, t
);
2701 /* If any decls have been made addressable during scan_omp,
2702 adjust their fields if needed, and layout record types
2703 of parallel/task constructs. */
2706 finish_taskreg_scan (omp_context
*ctx
)
2708 if (ctx
->record_type
== NULL_TREE
)
2711 /* If any task_shared_vars were needed, verify all
2712 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2713 statements if use_pointer_for_field hasn't changed
2714 because of that. If it did, update field types now. */
2715 if (task_shared_vars
)
2719 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2720 c
; c
= OMP_CLAUSE_CHAIN (c
))
2721 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
2722 && !OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
2724 tree decl
= OMP_CLAUSE_DECL (c
);
2726 /* Global variables don't need to be copied,
2727 the receiver side will use them directly. */
2728 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2730 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2731 || !use_pointer_for_field (decl
, ctx
))
2733 tree field
= lookup_field (decl
, ctx
);
2734 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2735 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2737 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2738 TREE_THIS_VOLATILE (field
) = 0;
2739 DECL_USER_ALIGN (field
) = 0;
2740 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2741 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2742 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2743 if (ctx
->srecord_type
)
2745 tree sfield
= lookup_sfield (decl
, ctx
);
2746 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2747 TREE_THIS_VOLATILE (sfield
) = 0;
2748 DECL_USER_ALIGN (sfield
) = 0;
2749 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2750 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2751 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2756 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2758 layout_type (ctx
->record_type
);
2759 fixup_child_record_type (ctx
);
2763 location_t loc
= gimple_location (ctx
->stmt
);
2764 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2765 /* Move VLA fields to the end. */
2766 p
= &TYPE_FIELDS (ctx
->record_type
);
2768 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2769 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2772 *p
= TREE_CHAIN (*p
);
2773 TREE_CHAIN (*q
) = NULL_TREE
;
2774 q
= &TREE_CHAIN (*q
);
2777 p
= &DECL_CHAIN (*p
);
2779 if (gimple_omp_task_taskloop_p (ctx
->stmt
))
2781 /* Move fields corresponding to first and second _looptemp_
2782 clause first. There are filled by GOMP_taskloop
2783 and thus need to be in specific positions. */
2784 tree c1
= gimple_omp_task_clauses (ctx
->stmt
);
2785 c1
= find_omp_clause (c1
, OMP_CLAUSE__LOOPTEMP_
);
2786 tree c2
= find_omp_clause (OMP_CLAUSE_CHAIN (c1
),
2787 OMP_CLAUSE__LOOPTEMP_
);
2788 tree f1
= lookup_field (OMP_CLAUSE_DECL (c1
), ctx
);
2789 tree f2
= lookup_field (OMP_CLAUSE_DECL (c2
), ctx
);
2790 p
= &TYPE_FIELDS (ctx
->record_type
);
2792 if (*p
== f1
|| *p
== f2
)
2793 *p
= DECL_CHAIN (*p
);
2795 p
= &DECL_CHAIN (*p
);
2796 DECL_CHAIN (f1
) = f2
;
2797 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->record_type
);
2798 TYPE_FIELDS (ctx
->record_type
) = f1
;
2799 if (ctx
->srecord_type
)
2801 f1
= lookup_sfield (OMP_CLAUSE_DECL (c1
), ctx
);
2802 f2
= lookup_sfield (OMP_CLAUSE_DECL (c2
), ctx
);
2803 p
= &TYPE_FIELDS (ctx
->srecord_type
);
2805 if (*p
== f1
|| *p
== f2
)
2806 *p
= DECL_CHAIN (*p
);
2808 p
= &DECL_CHAIN (*p
);
2809 DECL_CHAIN (f1
) = f2
;
2810 DECL_CHAIN (f2
) = TYPE_FIELDS (ctx
->srecord_type
);
2811 TYPE_FIELDS (ctx
->srecord_type
) = f1
;
2814 layout_type (ctx
->record_type
);
2815 fixup_child_record_type (ctx
);
2816 if (ctx
->srecord_type
)
2817 layout_type (ctx
->srecord_type
);
2818 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2819 TYPE_SIZE_UNIT (ctx
->record_type
));
2820 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2821 t
= build_int_cst (long_integer_type_node
,
2822 TYPE_ALIGN_UNIT (ctx
->record_type
));
2823 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2827 /* Find the enclosing offload context. */
2829 static omp_context
*
2830 enclosing_target_ctx (omp_context
*ctx
)
2832 for (; ctx
; ctx
= ctx
->outer
)
2833 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
)
2839 /* Return true if ctx is part of an oacc kernels region. */
2842 ctx_in_oacc_kernels_region (omp_context
*ctx
)
2844 for (;ctx
!= NULL
; ctx
= ctx
->outer
)
2846 gimple
*stmt
= ctx
->stmt
;
2847 if (gimple_code (stmt
) == GIMPLE_OMP_TARGET
2848 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_OACC_KERNELS
)
2855 /* Check the parallelism clauses inside a kernels regions.
2856 Until kernels handling moves to use the same loop indirection
2857 scheme as parallel, we need to do this checking early. */
2860 check_oacc_kernel_gwv (gomp_for
*stmt
, omp_context
*ctx
)
2862 bool checking
= true;
2863 unsigned outer_mask
= 0;
2864 unsigned this_mask
= 0;
2865 bool has_seq
= false, has_auto
= false;
2868 outer_mask
= check_oacc_kernel_gwv (NULL
, ctx
->outer
);
2872 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
)
2874 stmt
= as_a
<gomp_for
*> (ctx
->stmt
);
2877 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
2879 switch (OMP_CLAUSE_CODE (c
))
2881 case OMP_CLAUSE_GANG
:
2882 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_GANG
);
2884 case OMP_CLAUSE_WORKER
:
2885 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_WORKER
);
2887 case OMP_CLAUSE_VECTOR
:
2888 this_mask
|= GOMP_DIM_MASK (GOMP_DIM_VECTOR
);
2890 case OMP_CLAUSE_SEQ
:
2893 case OMP_CLAUSE_AUTO
:
2903 if (has_seq
&& (this_mask
|| has_auto
))
2904 error_at (gimple_location (stmt
), "%<seq%> overrides other"
2905 " OpenACC loop specifiers");
2906 else if (has_auto
&& this_mask
)
2907 error_at (gimple_location (stmt
), "%<auto%> conflicts with other"
2908 " OpenACC loop specifiers");
2910 if (this_mask
& outer_mask
)
2911 error_at (gimple_location (stmt
), "inner loop uses same"
2912 " OpenACC parallelism as containing loop");
2915 return outer_mask
| this_mask
;
2918 /* Scan a GIMPLE_OMP_FOR. */
2921 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2925 tree clauses
= gimple_omp_for_clauses (stmt
);
2927 ctx
= new_omp_context (stmt
, outer_ctx
);
2929 if (is_gimple_omp_oacc (stmt
))
2931 omp_context
*tgt
= enclosing_target_ctx (outer_ctx
);
2933 if (!tgt
|| is_oacc_parallel (tgt
))
2934 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2936 char const *check
= NULL
;
2938 switch (OMP_CLAUSE_CODE (c
))
2940 case OMP_CLAUSE_GANG
:
2944 case OMP_CLAUSE_WORKER
:
2948 case OMP_CLAUSE_VECTOR
:
2956 if (check
&& OMP_CLAUSE_OPERAND (c
, 0))
2957 error_at (gimple_location (stmt
),
2958 "argument not permitted on %qs clause in"
2959 " OpenACC %<parallel%>", check
);
2962 if (tgt
&& is_oacc_kernels (tgt
))
2964 /* Strip out reductions, as they are not handled yet. */
2965 tree
*prev_ptr
= &clauses
;
2967 while (tree probe
= *prev_ptr
)
2969 tree
*next_ptr
= &OMP_CLAUSE_CHAIN (probe
);
2971 if (OMP_CLAUSE_CODE (probe
) == OMP_CLAUSE_REDUCTION
)
2972 *prev_ptr
= *next_ptr
;
2974 prev_ptr
= next_ptr
;
2977 gimple_omp_for_set_clauses (stmt
, clauses
);
2978 check_oacc_kernel_gwv (stmt
, ctx
);
2982 scan_sharing_clauses (clauses
, ctx
);
2984 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2985 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2987 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2988 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2989 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2990 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2992 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2995 /* Scan an OpenMP sections directive. */
2998 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
3002 ctx
= new_omp_context (stmt
, outer_ctx
);
3003 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
3004 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3007 /* Scan an OpenMP single directive. */
3010 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
3015 ctx
= new_omp_context (stmt
, outer_ctx
);
3016 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3017 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3018 name
= create_tmp_var_name (".omp_copy_s");
3019 name
= build_decl (gimple_location (stmt
),
3020 TYPE_DECL
, name
, ctx
->record_type
);
3021 TYPE_NAME (ctx
->record_type
) = name
;
3023 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
3024 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3026 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3027 ctx
->record_type
= NULL
;
3029 layout_type (ctx
->record_type
);
3032 /* Scan a GIMPLE_OMP_TARGET. */
3035 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
3039 bool offloaded
= is_gimple_omp_offloaded (stmt
);
3040 tree clauses
= gimple_omp_target_clauses (stmt
);
3042 ctx
= new_omp_context (stmt
, outer_ctx
);
3043 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
3044 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
3045 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
3046 name
= create_tmp_var_name (".omp_data_t");
3047 name
= build_decl (gimple_location (stmt
),
3048 TYPE_DECL
, name
, ctx
->record_type
);
3049 DECL_ARTIFICIAL (name
) = 1;
3050 DECL_NAMELESS (name
) = 1;
3051 TYPE_NAME (ctx
->record_type
) = name
;
3052 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
3055 create_omp_child_function (ctx
, false);
3056 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
3059 scan_sharing_clauses (clauses
, ctx
);
3060 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3062 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
3063 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
3066 TYPE_FIELDS (ctx
->record_type
)
3067 = nreverse (TYPE_FIELDS (ctx
->record_type
));
3070 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
3071 for (tree field
= TYPE_FIELDS (ctx
->record_type
);
3073 field
= DECL_CHAIN (field
))
3074 gcc_assert (DECL_ALIGN (field
) == align
);
3076 layout_type (ctx
->record_type
);
3078 fixup_child_record_type (ctx
);
3082 /* Scan an OpenMP teams directive. */
3085 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
3087 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
3088 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
3089 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3092 /* Check nesting restrictions. */
3094 check_omp_nesting_restrictions (gimple
*stmt
, omp_context
*ctx
)
3098 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
3099 inside an OpenACC CTX. */
3100 if (!(is_gimple_omp (stmt
)
3101 && is_gimple_omp_oacc (stmt
)))
3103 for (omp_context
*octx
= ctx
; octx
!= NULL
; octx
= octx
->outer
)
3104 if (is_gimple_omp (octx
->stmt
)
3105 && is_gimple_omp_oacc (octx
->stmt
)
3106 /* Except for atomic codes that we share with OpenMP. */
3107 && ! (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
3108 || gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
))
3110 error_at (gimple_location (stmt
),
3111 "non-OpenACC construct inside of OpenACC region");
3118 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3119 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3122 if (gimple_code (stmt
) == GIMPLE_OMP_ORDERED
)
3124 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3125 if (find_omp_clause (c
, OMP_CLAUSE_SIMD
))
3127 if (find_omp_clause (c
, OMP_CLAUSE_THREADS
)
3128 && (ctx
->outer
== NULL
3129 || !gimple_omp_for_combined_into_p (ctx
->stmt
)
3130 || gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
3131 || (gimple_omp_for_kind (ctx
->outer
->stmt
)
3132 != GF_OMP_FOR_KIND_FOR
)
3133 || !gimple_omp_for_combined_p (ctx
->outer
->stmt
)))
3135 error_at (gimple_location (stmt
),
3136 "%<ordered simd threads%> must be closely "
3137 "nested inside of %<for simd%> region");
3143 error_at (gimple_location (stmt
),
3144 "OpenMP constructs other than %<#pragma omp ordered simd%>"
3145 " may not be nested inside %<simd%> region");
3148 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3150 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
3151 || (gimple_omp_for_kind (stmt
)
3152 != GF_OMP_FOR_KIND_DISTRIBUTE
))
3153 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
3155 error_at (gimple_location (stmt
),
3156 "only %<distribute%> or %<parallel%> regions are "
3157 "allowed to be strictly nested inside %<teams%> "
3163 switch (gimple_code (stmt
))
3165 case GIMPLE_OMP_FOR
:
3166 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
3168 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
3170 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
3172 error_at (gimple_location (stmt
),
3173 "%<distribute%> region must be strictly nested "
3174 "inside %<teams%> construct");
3179 /* We split taskloop into task and nested taskloop in it. */
3180 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3182 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
3187 switch (gimple_code (ctx
->stmt
))
3189 case GIMPLE_OMP_FOR
:
3190 ok
= (gimple_omp_for_kind (ctx
->stmt
)
3191 == GF_OMP_FOR_KIND_OACC_LOOP
);
3194 case GIMPLE_OMP_TARGET
:
3195 switch (gimple_omp_target_kind (ctx
->stmt
))
3197 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3198 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3209 else if (get_oacc_fn_attrib (current_function_decl
))
3213 error_at (gimple_location (stmt
),
3214 "OpenACC loop directive must be associated with"
3215 " an OpenACC compute region");
3221 if (is_gimple_call (stmt
)
3222 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3223 == BUILT_IN_GOMP_CANCEL
3224 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3225 == BUILT_IN_GOMP_CANCELLATION_POINT
))
3227 const char *bad
= NULL
;
3228 const char *kind
= NULL
;
3229 const char *construct
3230 = (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3231 == BUILT_IN_GOMP_CANCEL
)
3232 ? "#pragma omp cancel"
3233 : "#pragma omp cancellation point";
3236 error_at (gimple_location (stmt
), "orphaned %qs construct",
3240 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
3241 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
3245 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
3246 bad
= "#pragma omp parallel";
3247 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3248 == BUILT_IN_GOMP_CANCEL
3249 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3250 ctx
->cancellable
= true;
3254 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3255 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
3256 bad
= "#pragma omp for";
3257 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3258 == BUILT_IN_GOMP_CANCEL
3259 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3261 ctx
->cancellable
= true;
3262 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3264 warning_at (gimple_location (stmt
), 0,
3265 "%<#pragma omp cancel for%> inside "
3266 "%<nowait%> for construct");
3267 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3268 OMP_CLAUSE_ORDERED
))
3269 warning_at (gimple_location (stmt
), 0,
3270 "%<#pragma omp cancel for%> inside "
3271 "%<ordered%> for construct");
3276 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
3277 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
3278 bad
= "#pragma omp sections";
3279 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3280 == BUILT_IN_GOMP_CANCEL
3281 && !integer_zerop (gimple_call_arg (stmt
, 1)))
3283 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
3285 ctx
->cancellable
= true;
3286 if (find_omp_clause (gimple_omp_sections_clauses
3289 warning_at (gimple_location (stmt
), 0,
3290 "%<#pragma omp cancel sections%> inside "
3291 "%<nowait%> sections construct");
3295 gcc_assert (ctx
->outer
3296 && gimple_code (ctx
->outer
->stmt
)
3297 == GIMPLE_OMP_SECTIONS
);
3298 ctx
->outer
->cancellable
= true;
3299 if (find_omp_clause (gimple_omp_sections_clauses
3302 warning_at (gimple_location (stmt
), 0,
3303 "%<#pragma omp cancel sections%> inside "
3304 "%<nowait%> sections construct");
3310 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
3311 bad
= "#pragma omp task";
3314 for (omp_context
*octx
= ctx
->outer
;
3315 octx
; octx
= octx
->outer
)
3317 switch (gimple_code (octx
->stmt
))
3319 case GIMPLE_OMP_TASKGROUP
:
3321 case GIMPLE_OMP_TARGET
:
3322 if (gimple_omp_target_kind (octx
->stmt
)
3323 != GF_OMP_TARGET_KIND_REGION
)
3326 case GIMPLE_OMP_PARALLEL
:
3327 case GIMPLE_OMP_TEAMS
:
3328 error_at (gimple_location (stmt
),
3329 "%<%s taskgroup%> construct not closely "
3330 "nested inside of %<taskgroup%> region",
3338 ctx
->cancellable
= true;
3343 error_at (gimple_location (stmt
), "invalid arguments");
3348 error_at (gimple_location (stmt
),
3349 "%<%s %s%> construct not closely nested inside of %qs",
3350 construct
, kind
, bad
);
3355 case GIMPLE_OMP_SECTIONS
:
3356 case GIMPLE_OMP_SINGLE
:
3357 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3358 switch (gimple_code (ctx
->stmt
))
3360 case GIMPLE_OMP_FOR
:
3361 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3362 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3365 case GIMPLE_OMP_SECTIONS
:
3366 case GIMPLE_OMP_SINGLE
:
3367 case GIMPLE_OMP_ORDERED
:
3368 case GIMPLE_OMP_MASTER
:
3369 case GIMPLE_OMP_TASK
:
3370 case GIMPLE_OMP_CRITICAL
:
3371 if (is_gimple_call (stmt
))
3373 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
3374 != BUILT_IN_GOMP_BARRIER
)
3376 error_at (gimple_location (stmt
),
3377 "barrier region may not be closely nested inside "
3378 "of work-sharing, %<critical%>, %<ordered%>, "
3379 "%<master%>, explicit %<task%> or %<taskloop%> "
3383 error_at (gimple_location (stmt
),
3384 "work-sharing region may not be closely nested inside "
3385 "of work-sharing, %<critical%>, %<ordered%>, "
3386 "%<master%>, explicit %<task%> or %<taskloop%> region");
3388 case GIMPLE_OMP_PARALLEL
:
3389 case GIMPLE_OMP_TEAMS
:
3391 case GIMPLE_OMP_TARGET
:
3392 if (gimple_omp_target_kind (ctx
->stmt
)
3393 == GF_OMP_TARGET_KIND_REGION
)
3400 case GIMPLE_OMP_MASTER
:
3401 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3402 switch (gimple_code (ctx
->stmt
))
3404 case GIMPLE_OMP_FOR
:
3405 if (gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
3406 && gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_TASKLOOP
)
3409 case GIMPLE_OMP_SECTIONS
:
3410 case GIMPLE_OMP_SINGLE
:
3411 case GIMPLE_OMP_TASK
:
3412 error_at (gimple_location (stmt
),
3413 "%<master%> region may not be closely nested inside "
3414 "of work-sharing, explicit %<task%> or %<taskloop%> "
3417 case GIMPLE_OMP_PARALLEL
:
3418 case GIMPLE_OMP_TEAMS
:
3420 case GIMPLE_OMP_TARGET
:
3421 if (gimple_omp_target_kind (ctx
->stmt
)
3422 == GF_OMP_TARGET_KIND_REGION
)
3429 case GIMPLE_OMP_TASK
:
3430 for (c
= gimple_omp_task_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3431 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3432 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3433 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3435 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3436 error_at (OMP_CLAUSE_LOCATION (c
),
3437 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3438 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3442 case GIMPLE_OMP_ORDERED
:
3443 for (c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3444 c
; c
= OMP_CLAUSE_CHAIN (c
))
3446 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
)
3448 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_THREADS
3449 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SIMD
);
3452 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3453 if (kind
== OMP_CLAUSE_DEPEND_SOURCE
3454 || kind
== OMP_CLAUSE_DEPEND_SINK
)
3457 /* Look for containing ordered(N) loop. */
3459 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3461 = find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3462 OMP_CLAUSE_ORDERED
)) == NULL_TREE
)
3464 error_at (OMP_CLAUSE_LOCATION (c
),
3465 "%<ordered%> construct with %<depend%> clause "
3466 "must be closely nested inside an %<ordered%> "
3470 else if (OMP_CLAUSE_ORDERED_EXPR (oclause
) == NULL_TREE
)
3472 error_at (OMP_CLAUSE_LOCATION (c
),
3473 "%<ordered%> construct with %<depend%> clause "
3474 "must be closely nested inside a loop with "
3475 "%<ordered%> clause with a parameter");
3481 error_at (OMP_CLAUSE_LOCATION (c
),
3482 "invalid depend kind in omp %<ordered%> %<depend%>");
3486 c
= gimple_omp_ordered_clauses (as_a
<gomp_ordered
*> (stmt
));
3487 if (find_omp_clause (c
, OMP_CLAUSE_SIMD
))
3489 /* ordered simd must be closely nested inside of simd region,
3490 and simd region must not encounter constructs other than
3491 ordered simd, therefore ordered simd may be either orphaned,
3492 or ctx->stmt must be simd. The latter case is handled already
3496 error_at (gimple_location (stmt
),
3497 "%<ordered%> %<simd%> must be closely nested inside "
3502 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3503 switch (gimple_code (ctx
->stmt
))
3505 case GIMPLE_OMP_CRITICAL
:
3506 case GIMPLE_OMP_TASK
:
3507 case GIMPLE_OMP_ORDERED
:
3508 ordered_in_taskloop
:
3509 error_at (gimple_location (stmt
),
3510 "%<ordered%> region may not be closely nested inside "
3511 "of %<critical%>, %<ordered%>, explicit %<task%> or "
3512 "%<taskloop%> region");
3514 case GIMPLE_OMP_FOR
:
3515 if (gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
3516 goto ordered_in_taskloop
;
3517 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3518 OMP_CLAUSE_ORDERED
) == NULL
)
3520 error_at (gimple_location (stmt
),
3521 "%<ordered%> region must be closely nested inside "
3522 "a loop region with an %<ordered%> clause");
3526 case GIMPLE_OMP_TARGET
:
3527 if (gimple_omp_target_kind (ctx
->stmt
)
3528 != GF_OMP_TARGET_KIND_REGION
)
3531 case GIMPLE_OMP_PARALLEL
:
3532 case GIMPLE_OMP_TEAMS
:
3533 error_at (gimple_location (stmt
),
3534 "%<ordered%> region must be closely nested inside "
3535 "a loop region with an %<ordered%> clause");
3541 case GIMPLE_OMP_CRITICAL
:
3544 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
3545 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3546 if (gomp_critical
*other_crit
3547 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
3548 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
3550 error_at (gimple_location (stmt
),
3551 "%<critical%> region may not be nested inside "
3552 "a %<critical%> region with the same name");
3557 case GIMPLE_OMP_TEAMS
:
3559 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
3560 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
3562 error_at (gimple_location (stmt
),
3563 "%<teams%> construct not closely nested inside of "
3564 "%<target%> construct");
3568 case GIMPLE_OMP_TARGET
:
3569 for (c
= gimple_omp_target_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
3570 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
3571 && (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
3572 || OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
))
3574 enum omp_clause_depend_kind kind
= OMP_CLAUSE_DEPEND_KIND (c
);
3575 error_at (OMP_CLAUSE_LOCATION (c
),
3576 "%<depend(%s)%> is only allowed in %<omp ordered%>",
3577 kind
== OMP_CLAUSE_DEPEND_SOURCE
? "source" : "sink");
3580 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3582 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3584 if (is_gimple_omp (stmt
)
3585 && is_gimple_omp_oacc (stmt
)
3586 && is_gimple_omp (ctx
->stmt
))
3588 error_at (gimple_location (stmt
),
3589 "OpenACC construct inside of non-OpenACC region");
3595 const char *stmt_name
, *ctx_stmt_name
;
3596 switch (gimple_omp_target_kind (stmt
))
3598 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3599 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3600 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3601 case GF_OMP_TARGET_KIND_ENTER_DATA
:
3602 stmt_name
= "target enter data"; break;
3603 case GF_OMP_TARGET_KIND_EXIT_DATA
:
3604 stmt_name
= "target exit data"; break;
3605 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3606 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3607 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3608 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3609 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
3610 stmt_name
= "enter/exit data"; break;
3611 default: gcc_unreachable ();
3613 switch (gimple_omp_target_kind (ctx
->stmt
))
3615 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3616 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3617 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
3618 ctx_stmt_name
= "parallel"; break;
3619 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
3620 ctx_stmt_name
= "kernels"; break;
3621 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3622 default: gcc_unreachable ();
3625 /* OpenACC/OpenMP mismatch? */
3626 if (is_gimple_omp_oacc (stmt
)
3627 != is_gimple_omp_oacc (ctx
->stmt
))
3629 error_at (gimple_location (stmt
),
3630 "%s %qs construct inside of %s %qs region",
3631 (is_gimple_omp_oacc (stmt
)
3632 ? "OpenACC" : "OpenMP"), stmt_name
,
3633 (is_gimple_omp_oacc (ctx
->stmt
)
3634 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3637 if (is_gimple_omp_offloaded (ctx
->stmt
))
3639 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3640 if (is_gimple_omp_oacc (ctx
->stmt
))
3642 error_at (gimple_location (stmt
),
3643 "%qs construct inside of %qs region",
3644 stmt_name
, ctx_stmt_name
);
3649 warning_at (gimple_location (stmt
), 0,
3650 "%qs construct inside of %qs region",
3651 stmt_name
, ctx_stmt_name
);
3663 /* Helper function scan_omp.
3665 Callback for walk_tree or operators in walk_gimple_stmt used to
3666 scan for OMP directives in TP. */
3669 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3671 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3672 omp_context
*ctx
= (omp_context
*) wi
->info
;
3675 switch (TREE_CODE (t
))
3682 *tp
= remap_decl (t
, &ctx
->cb
);
3686 if (ctx
&& TYPE_P (t
))
3687 *tp
= remap_type (t
, &ctx
->cb
);
3688 else if (!DECL_P (t
))
3693 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3694 if (tem
!= TREE_TYPE (t
))
3696 if (TREE_CODE (t
) == INTEGER_CST
)
3697 *tp
= wide_int_to_tree (tem
, t
);
3699 TREE_TYPE (t
) = tem
;
3709 /* Return true if FNDECL is a setjmp or a longjmp. */
3712 setjmp_or_longjmp_p (const_tree fndecl
)
3714 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3715 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3716 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3719 tree declname
= DECL_NAME (fndecl
);
3722 const char *name
= IDENTIFIER_POINTER (declname
);
3723 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3727 /* Helper function for scan_omp.
3729 Callback for walk_gimple_stmt used to scan for OMP directives in
3730 the current statement in GSI. */
3733 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3734 struct walk_stmt_info
*wi
)
3736 gimple
*stmt
= gsi_stmt (*gsi
);
3737 omp_context
*ctx
= (omp_context
*) wi
->info
;
3739 if (gimple_has_location (stmt
))
3740 input_location
= gimple_location (stmt
);
3742 /* Check the nesting restrictions. */
3743 bool remove
= false;
3744 if (is_gimple_omp (stmt
))
3745 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3746 else if (is_gimple_call (stmt
))
3748 tree fndecl
= gimple_call_fndecl (stmt
);
3751 if (setjmp_or_longjmp_p (fndecl
)
3753 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3754 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3757 error_at (gimple_location (stmt
),
3758 "setjmp/longjmp inside simd construct");
3760 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3761 switch (DECL_FUNCTION_CODE (fndecl
))
3763 case BUILT_IN_GOMP_BARRIER
:
3764 case BUILT_IN_GOMP_CANCEL
:
3765 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3766 case BUILT_IN_GOMP_TASKYIELD
:
3767 case BUILT_IN_GOMP_TASKWAIT
:
3768 case BUILT_IN_GOMP_TASKGROUP_START
:
3769 case BUILT_IN_GOMP_TASKGROUP_END
:
3770 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3779 stmt
= gimple_build_nop ();
3780 gsi_replace (gsi
, stmt
, false);
3783 *handled_ops_p
= true;
3785 switch (gimple_code (stmt
))
3787 case GIMPLE_OMP_PARALLEL
:
3788 taskreg_nesting_level
++;
3789 scan_omp_parallel (gsi
, ctx
);
3790 taskreg_nesting_level
--;
3793 case GIMPLE_OMP_TASK
:
3794 taskreg_nesting_level
++;
3795 scan_omp_task (gsi
, ctx
);
3796 taskreg_nesting_level
--;
3799 case GIMPLE_OMP_FOR
:
3800 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3803 case GIMPLE_OMP_SECTIONS
:
3804 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3807 case GIMPLE_OMP_SINGLE
:
3808 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3811 case GIMPLE_OMP_SECTION
:
3812 case GIMPLE_OMP_MASTER
:
3813 case GIMPLE_OMP_TASKGROUP
:
3814 case GIMPLE_OMP_ORDERED
:
3815 case GIMPLE_OMP_CRITICAL
:
3816 ctx
= new_omp_context (stmt
, ctx
);
3817 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3820 case GIMPLE_OMP_TARGET
:
3821 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3824 case GIMPLE_OMP_TEAMS
:
3825 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3832 *handled_ops_p
= false;
3834 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3836 var
= DECL_CHAIN (var
))
3837 insert_decl_map (&ctx
->cb
, var
, var
);
3841 *handled_ops_p
= false;
3849 /* Scan all the statements starting at the current statement. CTX
3850 contains context information about the OMP directives and
3851 clauses found during the scan. */
3854 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
3856 location_t saved_location
;
3857 struct walk_stmt_info wi
;
3859 memset (&wi
, 0, sizeof (wi
));
3861 wi
.want_locations
= true;
3863 saved_location
= input_location
;
3864 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
3865 input_location
= saved_location
;
3868 /* Re-gimplification and code generation routines. */
3870 /* Build a call to GOMP_barrier. */
3873 build_omp_barrier (tree lhs
)
3875 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
3876 : BUILT_IN_GOMP_BARRIER
);
3877 gcall
*g
= gimple_build_call (fndecl
, 0);
3879 gimple_call_set_lhs (g
, lhs
);
3883 /* If a context was created for STMT when it was scanned, return it. */
3885 static omp_context
*
3886 maybe_lookup_ctx (gimple
*stmt
)
3889 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
3890 return n
? (omp_context
*) n
->value
: NULL
;
3894 /* Find the mapping for DECL in CTX or the immediately enclosing
3895 context that has a mapping for DECL.
3897 If CTX is a nested parallel directive, we may have to use the decl
3898 mappings created in CTX's parent context. Suppose that we have the
3899 following parallel nesting (variable UIDs showed for clarity):
3902 #omp parallel shared(iD.1562) -> outer parallel
3903 iD.1562 = iD.1562 + 1;
3905 #omp parallel shared (iD.1562) -> inner parallel
3906 iD.1562 = iD.1562 - 1;
3908 Each parallel structure will create a distinct .omp_data_s structure
3909 for copying iD.1562 in/out of the directive:
3911 outer parallel .omp_data_s.1.i -> iD.1562
3912 inner parallel .omp_data_s.2.i -> iD.1562
3914 A shared variable mapping will produce a copy-out operation before
3915 the parallel directive and a copy-in operation after it. So, in
3916 this case we would have:
3919 .omp_data_o.1.i = iD.1562;
3920 #omp parallel shared(iD.1562) -> outer parallel
3921 .omp_data_i.1 = &.omp_data_o.1
3922 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3924 .omp_data_o.2.i = iD.1562; -> **
3925 #omp parallel shared(iD.1562) -> inner parallel
3926 .omp_data_i.2 = &.omp_data_o.2
3927 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3930 ** This is a problem. The symbol iD.1562 cannot be referenced
3931 inside the body of the outer parallel region. But since we are
3932 emitting this copy operation while expanding the inner parallel
3933 directive, we need to access the CTX structure of the outer
3934 parallel directive to get the correct mapping:
3936 .omp_data_o.2.i = .omp_data_i.1->i
3938 Since there may be other workshare or parallel directives enclosing
3939 the parallel directive, it may be necessary to walk up the context
3940 parent chain. This is not a problem in general because nested
3941 parallelism happens only rarely. */
3944 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3949 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3950 t
= maybe_lookup_decl (decl
, up
);
3952 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
3954 return t
? t
: decl
;
3958 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3959 in outer contexts. */
3962 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3967 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3968 t
= maybe_lookup_decl (decl
, up
);
3970 return t
? t
: decl
;
3974 /* Construct the initialization value for reduction operation OP. */
3977 omp_reduction_init_op (location_t loc
, enum tree_code op
, tree type
)
3986 case TRUTH_ORIF_EXPR
:
3987 case TRUTH_XOR_EXPR
:
3989 return build_zero_cst (type
);
3992 case TRUTH_AND_EXPR
:
3993 case TRUTH_ANDIF_EXPR
:
3995 return fold_convert_loc (loc
, type
, integer_one_node
);
3998 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
4001 if (SCALAR_FLOAT_TYPE_P (type
))
4003 REAL_VALUE_TYPE max
, min
;
4004 if (HONOR_INFINITIES (type
))
4007 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
4010 real_maxval (&min
, 1, TYPE_MODE (type
));
4011 return build_real (type
, min
);
4013 else if (POINTER_TYPE_P (type
))
4016 = wi::min_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4017 return wide_int_to_tree (type
, min
);
4021 gcc_assert (INTEGRAL_TYPE_P (type
));
4022 return TYPE_MIN_VALUE (type
);
4026 if (SCALAR_FLOAT_TYPE_P (type
))
4028 REAL_VALUE_TYPE max
;
4029 if (HONOR_INFINITIES (type
))
4032 real_maxval (&max
, 0, TYPE_MODE (type
));
4033 return build_real (type
, max
);
4035 else if (POINTER_TYPE_P (type
))
4038 = wi::max_value (TYPE_PRECISION (type
), TYPE_SIGN (type
));
4039 return wide_int_to_tree (type
, max
);
4043 gcc_assert (INTEGRAL_TYPE_P (type
));
4044 return TYPE_MAX_VALUE (type
);
4052 /* Construct the initialization value for reduction CLAUSE. */
4055 omp_reduction_init (tree clause
, tree type
)
4057 return omp_reduction_init_op (OMP_CLAUSE_LOCATION (clause
),
4058 OMP_CLAUSE_REDUCTION_CODE (clause
), type
);
4061 /* Return alignment to be assumed for var in CLAUSE, which should be
4062 OMP_CLAUSE_ALIGNED. */
4065 omp_clause_aligned_alignment (tree clause
)
4067 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
4068 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
4070 /* Otherwise return implementation defined alignment. */
4071 unsigned int al
= 1;
4072 machine_mode mode
, vmode
;
4073 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4075 vs
= 1 << floor_log2 (vs
);
4076 static enum mode_class classes
[]
4077 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
4078 for (int i
= 0; i
< 4; i
+= 2)
4079 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
4081 mode
= GET_MODE_WIDER_MODE (mode
))
4083 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
4084 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
4087 && GET_MODE_SIZE (vmode
) < vs
4088 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
4089 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
4091 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
4092 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
4094 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
4095 / GET_MODE_SIZE (mode
));
4096 if (TYPE_MODE (type
) != vmode
)
4098 if (TYPE_ALIGN_UNIT (type
) > al
)
4099 al
= TYPE_ALIGN_UNIT (type
);
4101 return build_int_cst (integer_type_node
, al
);
4104 /* Return maximum possible vectorization factor for the target. */
4111 || !flag_tree_loop_optimize
4112 || (!flag_tree_loop_vectorize
4113 && (global_options_set
.x_flag_tree_loop_vectorize
4114 || global_options_set
.x_flag_tree_vectorize
)))
4117 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
4120 vs
= 1 << floor_log2 (vs
);
4123 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
4124 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
4125 return GET_MODE_NUNITS (vqimode
);
4129 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
4133 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
4134 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
4138 max_vf
= omp_max_vf ();
4141 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4142 OMP_CLAUSE_SAFELEN
);
4143 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
4145 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4147 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
4151 idx
= create_tmp_var (unsigned_type_node
);
4152 lane
= create_tmp_var (unsigned_type_node
);
4158 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
4159 tree avar
= create_tmp_var_raw (atype
);
4160 if (TREE_ADDRESSABLE (new_var
))
4161 TREE_ADDRESSABLE (avar
) = 1;
4162 DECL_ATTRIBUTES (avar
)
4163 = tree_cons (get_identifier ("omp simd array"), NULL
,
4164 DECL_ATTRIBUTES (avar
));
4165 gimple_add_tmp_var (avar
);
4166 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
4167 NULL_TREE
, NULL_TREE
);
4168 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
4169 NULL_TREE
, NULL_TREE
);
4170 if (DECL_P (new_var
))
4172 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4173 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4178 /* Helper function of lower_rec_input_clauses. For a reference
4179 in simd reduction, add an underlying variable it will reference. */
4182 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
4184 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
4185 if (TREE_CONSTANT (z
))
4187 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)),
4188 get_name (new_vard
));
4189 gimple_add_tmp_var (z
);
4190 TREE_ADDRESSABLE (z
) = 1;
4191 z
= build_fold_addr_expr_loc (loc
, z
);
4192 gimplify_assign (new_vard
, z
, ilist
);
4196 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
4197 from the receiver (aka child) side and initializers for REFERENCE_TYPE
4198 private variables. Initialization statements go in ILIST, while calls
4199 to destructors go in DLIST. */
4202 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
4203 omp_context
*ctx
, struct omp_for_data
*fd
)
4205 tree c
, dtor
, copyin_seq
, x
, ptr
;
4206 bool copyin_by_ref
= false;
4207 bool lastprivate_firstprivate
= false;
4208 bool reduction_omp_orig_ref
= false;
4210 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4211 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
4213 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
4214 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
4215 gimple_seq llist
[2] = { NULL
, NULL
};
4219 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
4220 with data sharing clauses referencing variable sized vars. That
4221 is unnecessarily hard to support and very unlikely to result in
4222 vectorized code anyway. */
4224 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4225 switch (OMP_CLAUSE_CODE (c
))
4227 case OMP_CLAUSE_LINEAR
:
4228 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4231 case OMP_CLAUSE_PRIVATE
:
4232 case OMP_CLAUSE_FIRSTPRIVATE
:
4233 case OMP_CLAUSE_LASTPRIVATE
:
4234 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
4237 case OMP_CLAUSE_REDUCTION
:
4238 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
4239 || is_variable_sized (OMP_CLAUSE_DECL (c
)))
4246 /* Do all the fixed sized types in the first pass, and the variable sized
4247 types in the second pass. This makes sure that the scalar arguments to
4248 the variable sized types are processed before we use them in the
4249 variable sized operations. */
4250 for (pass
= 0; pass
< 2; ++pass
)
4252 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4254 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
4257 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4261 case OMP_CLAUSE_PRIVATE
:
4262 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
4265 case OMP_CLAUSE_SHARED
:
4266 /* Ignore shared directives in teams construct. */
4267 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4269 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
4271 gcc_assert (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
)
4272 || is_global_var (OMP_CLAUSE_DECL (c
)));
4275 case OMP_CLAUSE_FIRSTPRIVATE
:
4276 case OMP_CLAUSE_COPYIN
:
4278 case OMP_CLAUSE_LINEAR
:
4279 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
)
4280 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4281 lastprivate_firstprivate
= true;
4283 case OMP_CLAUSE_REDUCTION
:
4284 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4285 reduction_omp_orig_ref
= true;
4287 case OMP_CLAUSE__LOOPTEMP_
:
4288 /* Handle _looptemp_ clauses only on parallel/task. */
4292 case OMP_CLAUSE_LASTPRIVATE
:
4293 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4295 lastprivate_firstprivate
= true;
4296 if (pass
!= 0 || is_taskloop_ctx (ctx
))
4299 /* Even without corresponding firstprivate, if
4300 decl is Fortran allocatable, it needs outer var
4303 && lang_hooks
.decls
.omp_private_outer_ref
4304 (OMP_CLAUSE_DECL (c
)))
4305 lastprivate_firstprivate
= true;
4307 case OMP_CLAUSE_ALIGNED
:
4310 var
= OMP_CLAUSE_DECL (c
);
4311 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
4312 && !is_global_var (var
))
4314 new_var
= maybe_lookup_decl (var
, ctx
);
4315 if (new_var
== NULL_TREE
)
4316 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4317 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4318 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
4319 omp_clause_aligned_alignment (c
));
4320 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4321 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4322 gimplify_and_add (x
, ilist
);
4324 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
4325 && is_global_var (var
))
4327 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
4328 new_var
= lookup_decl (var
, ctx
);
4329 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
4330 t
= build_fold_addr_expr_loc (clause_loc
, t
);
4331 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
4332 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
4333 omp_clause_aligned_alignment (c
));
4334 t
= fold_convert_loc (clause_loc
, ptype
, t
);
4335 x
= create_tmp_var (ptype
);
4336 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
4337 gimplify_and_add (t
, ilist
);
4338 t
= build_simple_mem_ref_loc (clause_loc
, x
);
4339 SET_DECL_VALUE_EXPR (new_var
, t
);
4340 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4347 new_var
= var
= OMP_CLAUSE_DECL (c
);
4348 if (c_kind
== OMP_CLAUSE_REDUCTION
&& TREE_CODE (var
) == MEM_REF
)
4350 var
= TREE_OPERAND (var
, 0);
4351 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
4352 var
= TREE_OPERAND (var
, 0);
4353 if (TREE_CODE (var
) == INDIRECT_REF
4354 || TREE_CODE (var
) == ADDR_EXPR
)
4355 var
= TREE_OPERAND (var
, 0);
4356 if (is_variable_sized (var
))
4358 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
4359 var
= DECL_VALUE_EXPR (var
);
4360 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
4361 var
= TREE_OPERAND (var
, 0);
4362 gcc_assert (DECL_P (var
));
4366 if (c_kind
!= OMP_CLAUSE_COPYIN
)
4367 new_var
= lookup_decl (var
, ctx
);
4369 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
4374 /* C/C++ array section reductions. */
4375 else if (c_kind
== OMP_CLAUSE_REDUCTION
4376 && var
!= OMP_CLAUSE_DECL (c
))
4381 tree bias
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 1);
4382 tree orig_var
= TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0);
4383 if (TREE_CODE (orig_var
) == POINTER_PLUS_EXPR
)
4385 tree b
= TREE_OPERAND (orig_var
, 1);
4386 b
= maybe_lookup_decl (b
, ctx
);
4389 b
= TREE_OPERAND (orig_var
, 1);
4390 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
4392 if (integer_zerop (bias
))
4396 bias
= fold_convert_loc (clause_loc
,
4397 TREE_TYPE (b
), bias
);
4398 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4399 TREE_TYPE (b
), b
, bias
);
4401 orig_var
= TREE_OPERAND (orig_var
, 0);
4403 if (TREE_CODE (orig_var
) == INDIRECT_REF
4404 || TREE_CODE (orig_var
) == ADDR_EXPR
)
4405 orig_var
= TREE_OPERAND (orig_var
, 0);
4406 tree d
= OMP_CLAUSE_DECL (c
);
4407 tree type
= TREE_TYPE (d
);
4408 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
4409 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
4410 const char *name
= get_name (orig_var
);
4411 if (TREE_CONSTANT (v
))
4413 x
= create_tmp_var_raw (type
, name
);
4414 gimple_add_tmp_var (x
);
4415 TREE_ADDRESSABLE (x
) = 1;
4416 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4421 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4422 tree t
= maybe_lookup_decl (v
, ctx
);
4426 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
4427 gimplify_expr (&v
, ilist
, NULL
, is_gimple_val
, fb_rvalue
);
4428 t
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
4430 build_int_cst (TREE_TYPE (v
), 1));
4431 t
= fold_build2_loc (clause_loc
, MULT_EXPR
,
4433 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4434 tree al
= size_int (TYPE_ALIGN (TREE_TYPE (type
)));
4435 x
= build_call_expr_loc (clause_loc
, atmp
, 2, t
, al
);
4438 tree ptype
= build_pointer_type (TREE_TYPE (type
));
4439 x
= fold_convert_loc (clause_loc
, ptype
, x
);
4440 tree y
= create_tmp_var (ptype
, name
);
4441 gimplify_assign (y
, x
, ilist
);
4445 if (!integer_zerop (bias
))
4447 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
4448 bias
= fold_build1_loc (clause_loc
, NEGATE_EXPR
,
4450 x
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
4451 TREE_TYPE (x
), x
, bias
);
4452 yb
= create_tmp_var (ptype
, name
);
4453 gimplify_assign (yb
, x
, ilist
);
4457 d
= TREE_OPERAND (d
, 0);
4458 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
4459 d
= TREE_OPERAND (d
, 0);
4460 if (TREE_CODE (d
) == ADDR_EXPR
)
4462 if (orig_var
!= var
)
4464 gcc_assert (is_variable_sized (orig_var
));
4465 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
),
4467 gimplify_assign (new_var
, x
, ilist
);
4468 tree new_orig_var
= lookup_decl (orig_var
, ctx
);
4469 tree t
= build_fold_indirect_ref (new_var
);
4470 DECL_IGNORED_P (new_var
) = 0;
4471 TREE_THIS_NOTRAP (t
);
4472 SET_DECL_VALUE_EXPR (new_orig_var
, t
);
4473 DECL_HAS_VALUE_EXPR_P (new_orig_var
) = 1;
4477 x
= build2 (MEM_REF
, TREE_TYPE (new_var
), x
,
4478 build_int_cst (ptype
, 0));
4479 SET_DECL_VALUE_EXPR (new_var
, x
);
4480 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4485 gcc_assert (orig_var
== var
);
4486 if (TREE_CODE (d
) == INDIRECT_REF
)
4488 x
= create_tmp_var (ptype
, name
);
4489 TREE_ADDRESSABLE (x
) = 1;
4490 gimplify_assign (x
, yb
, ilist
);
4491 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4493 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4494 gimplify_assign (new_var
, x
, ilist
);
4496 tree y1
= create_tmp_var (ptype
, NULL
);
4497 gimplify_assign (y1
, y
, ilist
);
4498 tree i2
= NULL_TREE
, y2
= NULL_TREE
;
4499 tree body2
= NULL_TREE
, end2
= NULL_TREE
;
4500 tree y3
= NULL_TREE
, y4
= NULL_TREE
;
4501 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) || is_simd
)
4503 y2
= create_tmp_var (ptype
, NULL
);
4504 gimplify_assign (y2
, y
, ilist
);
4505 tree ref
= build_outer_var_ref (var
, ctx
);
4506 /* For ref build_outer_var_ref already performs this. */
4507 if (TREE_CODE (d
) == INDIRECT_REF
)
4508 gcc_assert (is_reference (var
));
4509 else if (TREE_CODE (d
) == ADDR_EXPR
)
4510 ref
= build_fold_addr_expr (ref
);
4511 else if (is_reference (var
))
4512 ref
= build_fold_addr_expr (ref
);
4513 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
4514 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
4515 && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
4517 y3
= create_tmp_var (ptype
, NULL
);
4518 gimplify_assign (y3
, unshare_expr (ref
), ilist
);
4522 y4
= create_tmp_var (ptype
, NULL
);
4523 gimplify_assign (y4
, ref
, dlist
);
4526 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
4527 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), ilist
);
4528 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4529 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4530 gimple_seq_add_stmt (ilist
, gimple_build_label (body
));
4533 i2
= create_tmp_var (TREE_TYPE (v
), NULL
);
4534 gimplify_assign (i2
, build_int_cst (TREE_TYPE (v
), 0), dlist
);
4535 body2
= create_artificial_label (UNKNOWN_LOCATION
);
4536 end2
= create_artificial_label (UNKNOWN_LOCATION
);
4537 gimple_seq_add_stmt (dlist
, gimple_build_label (body2
));
4539 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4541 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4542 tree decl_placeholder
4543 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
4544 SET_DECL_VALUE_EXPR (decl_placeholder
,
4545 build_simple_mem_ref (y1
));
4546 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
4547 SET_DECL_VALUE_EXPR (placeholder
,
4548 y3
? build_simple_mem_ref (y3
)
4550 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4551 x
= lang_hooks
.decls
.omp_clause_default_ctor
4552 (c
, build_simple_mem_ref (y1
),
4553 y3
? build_simple_mem_ref (y3
) : NULL_TREE
);
4555 gimplify_and_add (x
, ilist
);
4556 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4558 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4559 lower_omp (&tseq
, ctx
);
4560 gimple_seq_add_seq (ilist
, tseq
);
4562 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4565 SET_DECL_VALUE_EXPR (decl_placeholder
,
4566 build_simple_mem_ref (y2
));
4567 SET_DECL_VALUE_EXPR (placeholder
,
4568 build_simple_mem_ref (y4
));
4569 gimple_seq tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4570 lower_omp (&tseq
, ctx
);
4571 gimple_seq_add_seq (dlist
, tseq
);
4572 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4574 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4575 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 0;
4576 x
= lang_hooks
.decls
.omp_clause_dtor
4577 (c
, build_simple_mem_ref (y2
));
4580 gimple_seq tseq
= NULL
;
4582 gimplify_stmt (&dtor
, &tseq
);
4583 gimple_seq_add_seq (dlist
, tseq
);
4588 x
= omp_reduction_init (c
, TREE_TYPE (type
));
4589 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4591 /* reduction(-:var) sums up the partial results, so it
4592 acts identically to reduction(+:var). */
4593 if (code
== MINUS_EXPR
)
4596 gimplify_assign (build_simple_mem_ref (y1
), x
, ilist
);
4599 x
= build2 (code
, TREE_TYPE (type
),
4600 build_simple_mem_ref (y4
),
4601 build_simple_mem_ref (y2
));
4602 gimplify_assign (build_simple_mem_ref (y4
), x
, dlist
);
4606 = gimple_build_assign (y1
, POINTER_PLUS_EXPR
, y1
,
4607 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4608 gimple_seq_add_stmt (ilist
, g
);
4611 g
= gimple_build_assign (y3
, POINTER_PLUS_EXPR
, y3
,
4612 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4613 gimple_seq_add_stmt (ilist
, g
);
4615 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
4616 build_int_cst (TREE_TYPE (i
), 1));
4617 gimple_seq_add_stmt (ilist
, g
);
4618 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
4619 gimple_seq_add_stmt (ilist
, g
);
4620 gimple_seq_add_stmt (ilist
, gimple_build_label (end
));
4623 g
= gimple_build_assign (y2
, POINTER_PLUS_EXPR
, y2
,
4624 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4625 gimple_seq_add_stmt (dlist
, g
);
4628 g
= gimple_build_assign
4629 (y4
, POINTER_PLUS_EXPR
, y4
,
4630 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
4631 gimple_seq_add_stmt (dlist
, g
);
4633 g
= gimple_build_assign (i2
, PLUS_EXPR
, i2
,
4634 build_int_cst (TREE_TYPE (i2
), 1));
4635 gimple_seq_add_stmt (dlist
, g
);
4636 g
= gimple_build_cond (LE_EXPR
, i2
, v
, body2
, end2
);
4637 gimple_seq_add_stmt (dlist
, g
);
4638 gimple_seq_add_stmt (dlist
, gimple_build_label (end2
));
4642 else if (is_variable_sized (var
))
4644 /* For variable sized types, we need to allocate the
4645 actual storage here. Call alloca and store the
4646 result in the pointer decl that we created elsewhere. */
4650 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
4655 ptr
= DECL_VALUE_EXPR (new_var
);
4656 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
4657 ptr
= TREE_OPERAND (ptr
, 0);
4658 gcc_assert (DECL_P (ptr
));
4659 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
4661 /* void *tmp = __builtin_alloca */
4662 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4663 stmt
= gimple_build_call (atmp
, 2, x
,
4664 size_int (DECL_ALIGN (var
)));
4665 tmp
= create_tmp_var_raw (ptr_type_node
);
4666 gimple_add_tmp_var (tmp
);
4667 gimple_call_set_lhs (stmt
, tmp
);
4669 gimple_seq_add_stmt (ilist
, stmt
);
4671 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
4672 gimplify_assign (ptr
, x
, ilist
);
4675 else if (is_reference (var
) && !is_oacc_parallel (ctx
))
4677 /* For references that are being privatized for Fortran,
4678 allocate new backing storage for the new pointer
4679 variable. This allows us to avoid changing all the
4680 code that expects a pointer to something that expects
4681 a direct variable. */
4685 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
4686 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
4688 x
= build_receiver_ref (var
, false, ctx
);
4689 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4691 else if (TREE_CONSTANT (x
))
4693 /* For reduction in SIMD loop, defer adding the
4694 initialization of the reference, because if we decide
4695 to use SIMD array for it, the initilization could cause
4697 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
4701 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
4703 gimple_add_tmp_var (x
);
4704 TREE_ADDRESSABLE (x
) = 1;
4705 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4711 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
4712 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
4713 tree al
= size_int (TYPE_ALIGN (rtype
));
4714 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
4719 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
4720 gimplify_assign (new_var
, x
, ilist
);
4723 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4725 else if (c_kind
== OMP_CLAUSE_REDUCTION
4726 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4734 switch (OMP_CLAUSE_CODE (c
))
4736 case OMP_CLAUSE_SHARED
:
4737 /* Ignore shared directives in teams construct. */
4738 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
4740 /* Shared global vars are just accessed directly. */
4741 if (is_global_var (new_var
))
4743 /* For taskloop firstprivate/lastprivate, represented
4744 as firstprivate and shared clause on the task, new_var
4745 is the firstprivate var. */
4746 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
4748 /* Set up the DECL_VALUE_EXPR for shared variables now. This
4749 needs to be delayed until after fixup_child_record_type so
4750 that we get the correct type during the dereference. */
4751 by_ref
= use_pointer_for_field (var
, ctx
);
4752 x
= build_receiver_ref (var
, by_ref
, ctx
);
4753 SET_DECL_VALUE_EXPR (new_var
, x
);
4754 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4756 /* ??? If VAR is not passed by reference, and the variable
4757 hasn't been initialized yet, then we'll get a warning for
4758 the store into the omp_data_s structure. Ideally, we'd be
4759 able to notice this and not store anything at all, but
4760 we're generating code too early. Suppress the warning. */
4762 TREE_NO_WARNING (var
) = 1;
4765 case OMP_CLAUSE_LASTPRIVATE
:
4766 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4770 case OMP_CLAUSE_PRIVATE
:
4771 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
4772 x
= build_outer_var_ref (var
, ctx
);
4773 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4775 if (is_task_ctx (ctx
))
4776 x
= build_receiver_ref (var
, false, ctx
);
4778 x
= build_outer_var_ref (var
, ctx
);
4784 nx
= lang_hooks
.decls
.omp_clause_default_ctor
4785 (c
, unshare_expr (new_var
), x
);
4788 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4789 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
4790 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
4791 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4792 idx
, lane
, ivar
, lvar
))
4795 x
= lang_hooks
.decls
.omp_clause_default_ctor
4796 (c
, unshare_expr (ivar
), x
);
4798 gimplify_and_add (x
, &llist
[0]);
4801 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4804 gimple_seq tseq
= NULL
;
4807 gimplify_stmt (&dtor
, &tseq
);
4808 gimple_seq_add_seq (&llist
[1], tseq
);
4815 gimplify_and_add (nx
, ilist
);
4819 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
4822 gimple_seq tseq
= NULL
;
4825 gimplify_stmt (&dtor
, &tseq
);
4826 gimple_seq_add_seq (dlist
, tseq
);
4830 case OMP_CLAUSE_LINEAR
:
4831 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
4832 goto do_firstprivate
;
4833 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
4836 x
= build_outer_var_ref (var
, ctx
);
4839 case OMP_CLAUSE_FIRSTPRIVATE
:
4840 if (is_task_ctx (ctx
))
4842 if (is_reference (var
) || is_variable_sized (var
))
4844 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
4846 || use_pointer_for_field (var
, NULL
))
4848 x
= build_receiver_ref (var
, false, ctx
);
4849 SET_DECL_VALUE_EXPR (new_var
, x
);
4850 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
4855 x
= build_outer_var_ref (var
, ctx
);
4858 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4859 && gimple_omp_for_combined_into_p (ctx
->stmt
))
4861 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
4862 tree stept
= TREE_TYPE (t
);
4863 tree ct
= find_omp_clause (clauses
,
4864 OMP_CLAUSE__LOOPTEMP_
);
4866 tree l
= OMP_CLAUSE_DECL (ct
);
4867 tree n1
= fd
->loop
.n1
;
4868 tree step
= fd
->loop
.step
;
4869 tree itype
= TREE_TYPE (l
);
4870 if (POINTER_TYPE_P (itype
))
4871 itype
= signed_type_for (itype
);
4872 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
4873 if (TYPE_UNSIGNED (itype
)
4874 && fd
->loop
.cond_code
== GT_EXPR
)
4875 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
4876 fold_build1 (NEGATE_EXPR
, itype
, l
),
4877 fold_build1 (NEGATE_EXPR
,
4880 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
4881 t
= fold_build2 (MULT_EXPR
, stept
,
4882 fold_convert (stept
, l
), t
);
4884 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4886 x
= lang_hooks
.decls
.omp_clause_linear_ctor
4888 gimplify_and_add (x
, ilist
);
4892 if (POINTER_TYPE_P (TREE_TYPE (x
)))
4893 x
= fold_build2 (POINTER_PLUS_EXPR
,
4894 TREE_TYPE (x
), x
, t
);
4896 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
4899 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
4900 || TREE_ADDRESSABLE (new_var
))
4901 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4902 idx
, lane
, ivar
, lvar
))
4904 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
4906 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
4907 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
4908 gimplify_and_add (x
, ilist
);
4909 gimple_stmt_iterator gsi
4910 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4912 = gimple_build_assign (unshare_expr (lvar
), iv
);
4913 gsi_insert_before_without_update (&gsi
, g
,
4915 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
4916 enum tree_code code
= PLUS_EXPR
;
4917 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
4918 code
= POINTER_PLUS_EXPR
;
4919 g
= gimple_build_assign (iv
, code
, iv
, t
);
4920 gsi_insert_before_without_update (&gsi
, g
,
4924 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4925 (c
, unshare_expr (ivar
), x
);
4926 gimplify_and_add (x
, &llist
[0]);
4927 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4930 gimple_seq tseq
= NULL
;
4933 gimplify_stmt (&dtor
, &tseq
);
4934 gimple_seq_add_seq (&llist
[1], tseq
);
4939 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4940 (c
, unshare_expr (new_var
), x
);
4941 gimplify_and_add (x
, ilist
);
4944 case OMP_CLAUSE__LOOPTEMP_
:
4945 gcc_assert (is_taskreg_ctx (ctx
));
4946 x
= build_outer_var_ref (var
, ctx
);
4947 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4948 gimplify_and_add (x
, ilist
);
4951 case OMP_CLAUSE_COPYIN
:
4952 by_ref
= use_pointer_for_field (var
, NULL
);
4953 x
= build_receiver_ref (var
, by_ref
, ctx
);
4954 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
4955 append_to_statement_list (x
, ©in_seq
);
4956 copyin_by_ref
|= by_ref
;
4959 case OMP_CLAUSE_REDUCTION
:
4960 /* OpenACC reductions are initialized using the
4961 GOACC_REDUCTION internal function. */
4962 if (is_gimple_omp_oacc (ctx
->stmt
))
4964 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4966 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4968 x
= build_outer_var_ref (var
, ctx
);
4970 if (is_reference (var
)
4971 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4973 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4974 SET_DECL_VALUE_EXPR (placeholder
, x
);
4975 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4976 tree new_vard
= new_var
;
4977 if (is_reference (var
))
4979 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4980 new_vard
= TREE_OPERAND (new_var
, 0);
4981 gcc_assert (DECL_P (new_vard
));
4984 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4985 idx
, lane
, ivar
, lvar
))
4987 if (new_vard
== new_var
)
4989 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
4990 SET_DECL_VALUE_EXPR (new_var
, ivar
);
4994 SET_DECL_VALUE_EXPR (new_vard
,
4995 build_fold_addr_expr (ivar
));
4996 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4998 x
= lang_hooks
.decls
.omp_clause_default_ctor
4999 (c
, unshare_expr (ivar
),
5000 build_outer_var_ref (var
, ctx
));
5002 gimplify_and_add (x
, &llist
[0]);
5003 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5005 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5006 lower_omp (&tseq
, ctx
);
5007 gimple_seq_add_seq (&llist
[0], tseq
);
5009 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5010 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5011 lower_omp (&tseq
, ctx
);
5012 gimple_seq_add_seq (&llist
[1], tseq
);
5013 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5014 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5015 if (new_vard
== new_var
)
5016 SET_DECL_VALUE_EXPR (new_var
, lvar
);
5018 SET_DECL_VALUE_EXPR (new_vard
,
5019 build_fold_addr_expr (lvar
));
5020 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
5025 gimplify_stmt (&dtor
, &tseq
);
5026 gimple_seq_add_seq (&llist
[1], tseq
);
5030 /* If this is a reference to constant size reduction var
5031 with placeholder, we haven't emitted the initializer
5032 for it because it is undesirable if SIMD arrays are used.
5033 But if they aren't used, we need to emit the deferred
5034 initialization now. */
5035 else if (is_reference (var
) && is_simd
)
5036 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5037 x
= lang_hooks
.decls
.omp_clause_default_ctor
5038 (c
, unshare_expr (new_var
),
5039 build_outer_var_ref (var
, ctx
));
5041 gimplify_and_add (x
, ilist
);
5042 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
5044 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
5045 lower_omp (&tseq
, ctx
);
5046 gimple_seq_add_seq (ilist
, tseq
);
5048 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
5051 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
5052 lower_omp (&tseq
, ctx
);
5053 gimple_seq_add_seq (dlist
, tseq
);
5054 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5056 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
5061 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
5062 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
5063 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5065 /* reduction(-:var) sums up the partial results, so it
5066 acts identically to reduction(+:var). */
5067 if (code
== MINUS_EXPR
)
5070 tree new_vard
= new_var
;
5071 if (is_simd
&& is_reference (var
))
5073 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
5074 new_vard
= TREE_OPERAND (new_var
, 0);
5075 gcc_assert (DECL_P (new_vard
));
5078 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
5079 idx
, lane
, ivar
, lvar
))
5081 tree ref
= build_outer_var_ref (var
, ctx
);
5083 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
5085 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
5086 ref
= build_outer_var_ref (var
, ctx
);
5087 gimplify_assign (ref
, x
, &llist
[1]);
5089 if (new_vard
!= new_var
)
5091 SET_DECL_VALUE_EXPR (new_vard
,
5092 build_fold_addr_expr (lvar
));
5093 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
5098 if (is_reference (var
) && is_simd
)
5099 handle_simd_reference (clause_loc
, new_vard
, ilist
);
5100 gimplify_assign (new_var
, x
, ilist
);
5103 tree ref
= build_outer_var_ref (var
, ctx
);
5105 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5106 ref
= build_outer_var_ref (var
, ctx
);
5107 gimplify_assign (ref
, x
, dlist
);
5121 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
5122 /* Don't want uninit warnings on simduid, it is always uninitialized,
5123 but we use it not for the value, but for the DECL_UID only. */
5124 TREE_NO_WARNING (uid
) = 1;
5126 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
5127 gimple_call_set_lhs (g
, lane
);
5128 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
5129 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
5130 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
5131 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
5132 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5133 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5134 g
= gimple_build_assign (lane
, INTEGER_CST
,
5135 build_int_cst (unsigned_type_node
, 0));
5136 gimple_seq_add_stmt (ilist
, g
);
5137 for (int i
= 0; i
< 2; i
++)
5140 tree vf
= create_tmp_var (unsigned_type_node
);
5141 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
5142 gimple_call_set_lhs (g
, vf
);
5143 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
5144 gimple_seq_add_stmt (seq
, g
);
5145 tree t
= build_int_cst (unsigned_type_node
, 0);
5146 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
5147 gimple_seq_add_stmt (seq
, g
);
5148 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5149 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
5150 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5151 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
5152 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
5153 gimple_seq_add_seq (seq
, llist
[i
]);
5154 t
= build_int_cst (unsigned_type_node
, 1);
5155 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
5156 gimple_seq_add_stmt (seq
, g
);
5157 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
5158 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
5159 gimple_seq_add_stmt (seq
, g
);
5160 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
5164 /* The copyin sequence is not to be executed by the main thread, since
5165 that would result in self-copies. Perhaps not visible to scalars,
5166 but it certainly is to C++ operator=. */
5169 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
5171 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
5172 build_int_cst (TREE_TYPE (x
), 0));
5173 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
5174 gimplify_and_add (x
, ilist
);
5177 /* If any copyin variable is passed by reference, we must ensure the
5178 master thread doesn't modify it before it is copied over in all
5179 threads. Similarly for variables in both firstprivate and
5180 lastprivate clauses we need to ensure the lastprivate copying
5181 happens after firstprivate copying in all threads. And similarly
5182 for UDRs if initializer expression refers to omp_orig. */
5183 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
5185 /* Don't add any barrier for #pragma omp simd or
5186 #pragma omp distribute. */
5187 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
5188 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
5189 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
5192 /* If max_vf is non-zero, then we can use only a vectorization factor
5193 up to the max_vf we chose. So stick it into the safelen clause. */
5196 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
5197 OMP_CLAUSE_SAFELEN
);
5199 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
5200 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
5203 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
5204 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
5206 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
5207 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
5213 /* Generate code to implement the LASTPRIVATE clauses. This is used for
5214 both parallel and workshare constructs. PREDICATE may be NULL if it's
5218 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
5221 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
5222 bool par_clauses
= false;
5223 tree simduid
= NULL
, lastlane
= NULL
;
5225 /* Early exit if there are no lastprivate or linear clauses. */
5226 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
5227 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
5228 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
5229 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
5231 if (clauses
== NULL
)
5233 /* If this was a workshare clause, see if it had been combined
5234 with its parallel. In that case, look for the clauses on the
5235 parallel statement itself. */
5236 if (is_parallel_ctx (ctx
))
5240 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5243 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5244 OMP_CLAUSE_LASTPRIVATE
);
5245 if (clauses
== NULL
)
5253 tree label_true
, arm1
, arm2
;
5255 label
= create_artificial_label (UNKNOWN_LOCATION
);
5256 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
5257 arm1
= TREE_OPERAND (predicate
, 0);
5258 arm2
= TREE_OPERAND (predicate
, 1);
5259 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5260 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
5261 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
5263 gimple_seq_add_stmt (stmt_list
, stmt
);
5264 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
5267 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5268 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5270 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
5272 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
5275 for (c
= clauses
; c
;)
5278 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5280 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5281 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5282 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
5284 var
= OMP_CLAUSE_DECL (c
);
5285 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5286 && OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
)
5287 && is_taskloop_ctx (ctx
))
5289 gcc_checking_assert (ctx
->outer
&& is_task_ctx (ctx
->outer
));
5290 new_var
= lookup_decl (var
, ctx
->outer
);
5293 new_var
= lookup_decl (var
, ctx
);
5295 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
5297 tree val
= DECL_VALUE_EXPR (new_var
);
5298 if (TREE_CODE (val
) == ARRAY_REF
5299 && VAR_P (TREE_OPERAND (val
, 0))
5300 && lookup_attribute ("omp simd array",
5301 DECL_ATTRIBUTES (TREE_OPERAND (val
,
5304 if (lastlane
== NULL
)
5306 lastlane
= create_tmp_var (unsigned_type_node
);
5308 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
5310 TREE_OPERAND (val
, 1));
5311 gimple_call_set_lhs (g
, lastlane
);
5312 gimple_seq_add_stmt (stmt_list
, g
);
5314 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
5315 TREE_OPERAND (val
, 0), lastlane
,
5316 NULL_TREE
, NULL_TREE
);
5320 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5321 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
5323 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
5324 gimple_seq_add_seq (stmt_list
,
5325 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
5326 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
5328 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
5329 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
5331 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
5332 gimple_seq_add_seq (stmt_list
,
5333 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
5334 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
5338 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
5339 && OMP_CLAUSE_LASTPRIVATE_TASKLOOP_IV (c
))
5341 gcc_checking_assert (is_taskloop_ctx (ctx
));
5342 tree ovar
= maybe_lookup_decl_in_outer_ctx (var
,
5344 if (is_global_var (ovar
))
5348 x
= build_outer_var_ref (var
, ctx
, true);
5349 if (is_reference (var
))
5350 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5351 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
5352 gimplify_and_add (x
, stmt_list
);
5354 c
= OMP_CLAUSE_CHAIN (c
);
5355 if (c
== NULL
&& !par_clauses
)
5357 /* If this was a workshare clause, see if it had been combined
5358 with its parallel. In that case, continue looking for the
5359 clauses also on the parallel statement itself. */
5360 if (is_parallel_ctx (ctx
))
5364 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
5367 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
5368 OMP_CLAUSE_LASTPRIVATE
);
5374 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
5377 /* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
5378 (which might be a placeholder). INNER is true if this is an inner
5379 axis of a multi-axis loop. FORK and JOIN are (optional) fork and
5380 join markers. Generate the before-loop forking sequence in
5381 FORK_SEQ and the after-loop joining sequence to JOIN_SEQ. The
5382 general form of these sequences is
5384 GOACC_REDUCTION_SETUP
5386 GOACC_REDUCTION_INIT
5388 GOACC_REDUCTION_FINI
5390 GOACC_REDUCTION_TEARDOWN. */
5393 lower_oacc_reductions (location_t loc
, tree clauses
, tree level
, bool inner
,
5394 gcall
*fork
, gcall
*join
, gimple_seq
*fork_seq
,
5395 gimple_seq
*join_seq
, omp_context
*ctx
)
5397 gimple_seq before_fork
= NULL
;
5398 gimple_seq after_fork
= NULL
;
5399 gimple_seq before_join
= NULL
;
5400 gimple_seq after_join
= NULL
;
5401 tree init_code
= NULL_TREE
, fini_code
= NULL_TREE
,
5402 setup_code
= NULL_TREE
, teardown_code
= NULL_TREE
;
5403 unsigned offset
= 0;
5405 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5406 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5408 tree orig
= OMP_CLAUSE_DECL (c
);
5409 tree var
= maybe_lookup_decl (orig
, ctx
);
5410 tree ref_to_res
= NULL_TREE
;
5411 tree incoming
, outgoing
;
5413 enum tree_code rcode
= OMP_CLAUSE_REDUCTION_CODE (c
);
5414 if (rcode
== MINUS_EXPR
)
5416 else if (rcode
== TRUTH_ANDIF_EXPR
)
5417 rcode
= BIT_AND_EXPR
;
5418 else if (rcode
== TRUTH_ORIF_EXPR
)
5419 rcode
= BIT_IOR_EXPR
;
5420 tree op
= build_int_cst (unsigned_type_node
, rcode
);
5424 gcc_assert (!is_reference (var
));
5426 incoming
= outgoing
= var
;
5430 /* See if an outer construct also reduces this variable. */
5431 omp_context
*outer
= ctx
;
5433 while (omp_context
*probe
= outer
->outer
)
5435 enum gimple_code type
= gimple_code (probe
->stmt
);
5440 case GIMPLE_OMP_FOR
:
5441 cls
= gimple_omp_for_clauses (probe
->stmt
);
5444 case GIMPLE_OMP_TARGET
:
5445 if (gimple_omp_target_kind (probe
->stmt
)
5446 != GF_OMP_TARGET_KIND_OACC_PARALLEL
)
5449 cls
= gimple_omp_target_clauses (probe
->stmt
);
5457 for (; cls
; cls
= OMP_CLAUSE_CHAIN (cls
))
5458 if (OMP_CLAUSE_CODE (cls
) == OMP_CLAUSE_REDUCTION
5459 && orig
== OMP_CLAUSE_DECL (cls
))
5460 goto has_outer_reduction
;
5464 /* This is the outermost construct with this reduction,
5465 see if there's a mapping for it. */
5466 if (gimple_code (outer
->stmt
) == GIMPLE_OMP_TARGET
5467 && maybe_lookup_field (orig
, outer
))
5469 ref_to_res
= build_receiver_ref (orig
, false, outer
);
5470 if (is_reference (orig
))
5471 ref_to_res
= build_simple_mem_ref (ref_to_res
);
5474 incoming
= omp_reduction_init_op (loc
, rcode
, TREE_TYPE (var
));
5477 incoming
= outgoing
= orig
;
5479 has_outer_reduction
:;
5483 ref_to_res
= integer_zero_node
;
5485 /* Determine position in reduction buffer, which may be used
5487 enum machine_mode mode
= TYPE_MODE (TREE_TYPE (var
));
5488 unsigned align
= GET_MODE_ALIGNMENT (mode
) / BITS_PER_UNIT
;
5489 offset
= (offset
+ align
- 1) & ~(align
- 1);
5490 tree off
= build_int_cst (sizetype
, offset
);
5491 offset
+= GET_MODE_SIZE (mode
);
5495 init_code
= build_int_cst (integer_type_node
,
5496 IFN_GOACC_REDUCTION_INIT
);
5497 fini_code
= build_int_cst (integer_type_node
,
5498 IFN_GOACC_REDUCTION_FINI
);
5499 setup_code
= build_int_cst (integer_type_node
,
5500 IFN_GOACC_REDUCTION_SETUP
);
5501 teardown_code
= build_int_cst (integer_type_node
,
5502 IFN_GOACC_REDUCTION_TEARDOWN
);
5506 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5507 TREE_TYPE (var
), 6, setup_code
,
5508 unshare_expr (ref_to_res
),
5509 incoming
, level
, op
, off
);
5511 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5512 TREE_TYPE (var
), 6, init_code
,
5513 unshare_expr (ref_to_res
),
5514 var
, level
, op
, off
);
5516 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5517 TREE_TYPE (var
), 6, fini_code
,
5518 unshare_expr (ref_to_res
),
5519 var
, level
, op
, off
);
5521 = build_call_expr_internal_loc (loc
, IFN_GOACC_REDUCTION
,
5522 TREE_TYPE (var
), 6, teardown_code
,
5523 ref_to_res
, var
, level
, op
, off
);
5525 gimplify_assign (var
, setup_call
, &before_fork
);
5526 gimplify_assign (var
, init_call
, &after_fork
);
5527 gimplify_assign (var
, fini_call
, &before_join
);
5528 gimplify_assign (outgoing
, teardown_call
, &after_join
);
5531 /* Now stitch things together. */
5532 gimple_seq_add_seq (fork_seq
, before_fork
);
5534 gimple_seq_add_stmt (fork_seq
, fork
);
5535 gimple_seq_add_seq (fork_seq
, after_fork
);
5537 gimple_seq_add_seq (join_seq
, before_join
);
5539 gimple_seq_add_stmt (join_seq
, join
);
5540 gimple_seq_add_seq (join_seq
, after_join
);
5543 /* Generate code to implement the REDUCTION clauses. */
5546 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
5548 gimple_seq sub_seq
= NULL
;
5553 /* OpenACC loop reductions are handled elsewhere. */
5554 if (is_gimple_omp_oacc (ctx
->stmt
))
5557 /* SIMD reductions are handled in lower_rec_input_clauses. */
5558 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
5559 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
5562 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
5563 update in that case, otherwise use a lock. */
5564 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
5565 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
5567 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
)
5568 || TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5570 /* Never use OMP_ATOMIC for array reductions or UDRs. */
5580 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5582 tree var
, ref
, new_var
, orig_var
;
5583 enum tree_code code
;
5584 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5586 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
5589 orig_var
= var
= OMP_CLAUSE_DECL (c
);
5590 if (TREE_CODE (var
) == MEM_REF
)
5592 var
= TREE_OPERAND (var
, 0);
5593 if (TREE_CODE (var
) == POINTER_PLUS_EXPR
)
5594 var
= TREE_OPERAND (var
, 0);
5595 if (TREE_CODE (var
) == INDIRECT_REF
5596 || TREE_CODE (var
) == ADDR_EXPR
)
5597 var
= TREE_OPERAND (var
, 0);
5599 if (is_variable_sized (var
))
5601 gcc_assert (DECL_HAS_VALUE_EXPR_P (var
));
5602 var
= DECL_VALUE_EXPR (var
);
5603 gcc_assert (TREE_CODE (var
) == INDIRECT_REF
);
5604 var
= TREE_OPERAND (var
, 0);
5605 gcc_assert (DECL_P (var
));
5608 new_var
= lookup_decl (var
, ctx
);
5609 if (var
== OMP_CLAUSE_DECL (c
) && is_reference (var
))
5610 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5611 ref
= build_outer_var_ref (var
, ctx
);
5612 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
5614 /* reduction(-:var) sums up the partial results, so it acts
5615 identically to reduction(+:var). */
5616 if (code
== MINUS_EXPR
)
5621 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
5623 addr
= save_expr (addr
);
5624 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
5625 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
5626 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
5627 gimplify_and_add (x
, stmt_seqp
);
5630 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == MEM_REF
)
5632 tree d
= OMP_CLAUSE_DECL (c
);
5633 tree type
= TREE_TYPE (d
);
5634 tree v
= TYPE_MAX_VALUE (TYPE_DOMAIN (type
));
5635 tree i
= create_tmp_var (TREE_TYPE (v
), NULL
);
5636 tree ptype
= build_pointer_type (TREE_TYPE (type
));
5637 tree bias
= TREE_OPERAND (d
, 1);
5638 d
= TREE_OPERAND (d
, 0);
5639 if (TREE_CODE (d
) == POINTER_PLUS_EXPR
)
5641 tree b
= TREE_OPERAND (d
, 1);
5642 b
= maybe_lookup_decl (b
, ctx
);
5645 b
= TREE_OPERAND (d
, 1);
5646 b
= maybe_lookup_decl_in_outer_ctx (b
, ctx
);
5648 if (integer_zerop (bias
))
5652 bias
= fold_convert_loc (clause_loc
, TREE_TYPE (b
), bias
);
5653 bias
= fold_build2_loc (clause_loc
, PLUS_EXPR
,
5654 TREE_TYPE (b
), b
, bias
);
5656 d
= TREE_OPERAND (d
, 0);
5658 /* For ref build_outer_var_ref already performs this, so
5659 only new_var needs a dereference. */
5660 if (TREE_CODE (d
) == INDIRECT_REF
)
5662 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5663 gcc_assert (is_reference (var
) && var
== orig_var
);
5665 else if (TREE_CODE (d
) == ADDR_EXPR
)
5667 if (orig_var
== var
)
5669 new_var
= build_fold_addr_expr (new_var
);
5670 ref
= build_fold_addr_expr (ref
);
5675 gcc_assert (orig_var
== var
);
5676 if (is_reference (var
))
5677 ref
= build_fold_addr_expr (ref
);
5681 tree t
= maybe_lookup_decl (v
, ctx
);
5685 v
= maybe_lookup_decl_in_outer_ctx (v
, ctx
);
5686 gimplify_expr (&v
, stmt_seqp
, NULL
, is_gimple_val
, fb_rvalue
);
5688 if (!integer_zerop (bias
))
5690 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
5691 new_var
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5692 TREE_TYPE (new_var
), new_var
,
5693 unshare_expr (bias
));
5694 ref
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
5695 TREE_TYPE (ref
), ref
, bias
);
5697 new_var
= fold_convert_loc (clause_loc
, ptype
, new_var
);
5698 ref
= fold_convert_loc (clause_loc
, ptype
, ref
);
5699 tree m
= create_tmp_var (ptype
, NULL
);
5700 gimplify_assign (m
, new_var
, stmt_seqp
);
5702 m
= create_tmp_var (ptype
, NULL
);
5703 gimplify_assign (m
, ref
, stmt_seqp
);
5705 gimplify_assign (i
, build_int_cst (TREE_TYPE (v
), 0), stmt_seqp
);
5706 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
5707 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
5708 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (body
));
5709 tree priv
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5710 tree out
= build_simple_mem_ref_loc (clause_loc
, ref
);
5711 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5713 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5714 tree decl_placeholder
5715 = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
);
5716 SET_DECL_VALUE_EXPR (placeholder
, out
);
5717 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5718 SET_DECL_VALUE_EXPR (decl_placeholder
, priv
);
5719 DECL_HAS_VALUE_EXPR_P (decl_placeholder
) = 1;
5720 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5721 gimple_seq_add_seq (&sub_seq
,
5722 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5723 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5724 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5725 OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c
) = NULL
;
5729 x
= build2 (code
, TREE_TYPE (out
), out
, priv
);
5730 out
= unshare_expr (out
);
5731 gimplify_assign (out
, x
, &sub_seq
);
5733 gimple
*g
= gimple_build_assign (new_var
, POINTER_PLUS_EXPR
, new_var
,
5734 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5735 gimple_seq_add_stmt (&sub_seq
, g
);
5736 g
= gimple_build_assign (ref
, POINTER_PLUS_EXPR
, ref
,
5737 TYPE_SIZE_UNIT (TREE_TYPE (type
)));
5738 gimple_seq_add_stmt (&sub_seq
, g
);
5739 g
= gimple_build_assign (i
, PLUS_EXPR
, i
,
5740 build_int_cst (TREE_TYPE (i
), 1));
5741 gimple_seq_add_stmt (&sub_seq
, g
);
5742 g
= gimple_build_cond (LE_EXPR
, i
, v
, body
, end
);
5743 gimple_seq_add_stmt (&sub_seq
, g
);
5744 gimple_seq_add_stmt (&sub_seq
, gimple_build_label (end
));
5746 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
5748 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
5750 if (is_reference (var
)
5751 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
5753 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
5754 SET_DECL_VALUE_EXPR (placeholder
, ref
);
5755 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
5756 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
5757 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
5758 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
5759 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
5763 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
5764 ref
= build_outer_var_ref (var
, ctx
);
5765 gimplify_assign (ref
, x
, &sub_seq
);
5769 if (is_gimple_omp_oacc (ctx
->stmt
))
5772 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
5774 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5776 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
5778 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
5780 gimple_seq_add_stmt (stmt_seqp
, stmt
);
5784 /* Generate code to implement the COPYPRIVATE clauses. */
5787 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
5792 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5794 tree var
, new_var
, ref
, x
;
5796 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5798 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
5801 var
= OMP_CLAUSE_DECL (c
);
5802 by_ref
= use_pointer_for_field (var
, NULL
);
5804 ref
= build_sender_ref (var
, ctx
);
5805 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
5808 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
5809 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
5811 gimplify_assign (ref
, x
, slist
);
5813 ref
= build_receiver_ref (var
, false, ctx
);
5816 ref
= fold_convert_loc (clause_loc
,
5817 build_pointer_type (TREE_TYPE (new_var
)),
5819 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
5821 if (is_reference (var
))
5823 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
5824 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
5825 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
5827 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
5828 gimplify_and_add (x
, rlist
);
5833 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
5834 and REDUCTION from the sender (aka parent) side. */
5837 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
5841 int ignored_looptemp
= 0;
5842 bool is_taskloop
= false;
5844 /* For taskloop, ignore first two _looptemp_ clauses, those are initialized
5845 by GOMP_taskloop. */
5846 if (is_task_ctx (ctx
) && gimple_omp_task_taskloop_p (ctx
->stmt
))
5848 ignored_looptemp
= 2;
5852 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
5854 tree val
, ref
, x
, var
;
5855 bool by_ref
, do_in
= false, do_out
= false;
5856 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
5858 switch (OMP_CLAUSE_CODE (c
))
5860 case OMP_CLAUSE_PRIVATE
:
5861 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
5864 case OMP_CLAUSE_FIRSTPRIVATE
:
5865 case OMP_CLAUSE_COPYIN
:
5866 case OMP_CLAUSE_LASTPRIVATE
:
5867 case OMP_CLAUSE_REDUCTION
:
5869 case OMP_CLAUSE_SHARED
:
5870 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
5873 case OMP_CLAUSE__LOOPTEMP_
:
5874 if (ignored_looptemp
)
5884 val
= OMP_CLAUSE_DECL (c
);
5885 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
5886 && TREE_CODE (val
) == MEM_REF
)
5888 val
= TREE_OPERAND (val
, 0);
5889 if (TREE_CODE (val
) == POINTER_PLUS_EXPR
)
5890 val
= TREE_OPERAND (val
, 0);
5891 if (TREE_CODE (val
) == INDIRECT_REF
5892 || TREE_CODE (val
) == ADDR_EXPR
)
5893 val
= TREE_OPERAND (val
, 0);
5894 if (is_variable_sized (val
))
5898 /* For OMP_CLAUSE_SHARED_FIRSTPRIVATE, look beyond the
5899 outer taskloop region. */
5900 omp_context
*ctx_for_o
= ctx
;
5902 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
5903 && OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
5904 ctx_for_o
= ctx
->outer
;
5906 var
= lookup_decl_in_outer_ctx (val
, ctx_for_o
);
5908 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
5909 && is_global_var (var
))
5912 t
= omp_member_access_dummy_var (var
);
5915 var
= DECL_VALUE_EXPR (var
);
5916 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx_for_o
);
5918 var
= unshare_and_remap (var
, t
, o
);
5920 var
= unshare_expr (var
);
5923 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
5925 /* Handle taskloop firstprivate/lastprivate, where the
5926 lastprivate on GIMPLE_OMP_TASK is represented as
5927 OMP_CLAUSE_SHARED_FIRSTPRIVATE. */
5928 tree f
= lookup_sfield ((splay_tree_key
) &DECL_UID (val
), ctx
);
5929 x
= omp_build_component_ref (ctx
->sender_decl
, f
);
5930 if (use_pointer_for_field (val
, ctx
))
5931 var
= build_fold_addr_expr (var
);
5932 gimplify_assign (x
, var
, ilist
);
5933 DECL_ABSTRACT_ORIGIN (f
) = NULL
;
5937 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
5938 || val
== OMP_CLAUSE_DECL (c
))
5939 && is_variable_sized (val
))
5941 by_ref
= use_pointer_for_field (val
, NULL
);
5943 switch (OMP_CLAUSE_CODE (c
))
5945 case OMP_CLAUSE_PRIVATE
:
5946 case OMP_CLAUSE_FIRSTPRIVATE
:
5947 case OMP_CLAUSE_COPYIN
:
5948 case OMP_CLAUSE__LOOPTEMP_
:
5952 case OMP_CLAUSE_LASTPRIVATE
:
5953 if (by_ref
|| is_reference (val
))
5955 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
5962 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
5967 case OMP_CLAUSE_REDUCTION
:
5969 if (val
== OMP_CLAUSE_DECL (c
))
5970 do_out
= !(by_ref
|| is_reference (val
));
5972 by_ref
= TREE_CODE (TREE_TYPE (val
)) == ARRAY_TYPE
;
5981 ref
= build_sender_ref (val
, ctx
);
5982 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
5983 gimplify_assign (ref
, x
, ilist
);
5984 if (is_task_ctx (ctx
))
5985 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
5990 ref
= build_sender_ref (val
, ctx
);
5991 gimplify_assign (var
, ref
, olist
);
5996 /* Generate code to implement SHARED from the sender (aka parent)
5997 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
5998 list things that got automatically shared. */
6001 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
6003 tree var
, ovar
, nvar
, t
, f
, x
, record_type
;
6005 if (ctx
->record_type
== NULL
)
6008 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
6009 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
6011 ovar
= DECL_ABSTRACT_ORIGIN (f
);
6012 if (!ovar
|| TREE_CODE (ovar
) == FIELD_DECL
)
6015 nvar
= maybe_lookup_decl (ovar
, ctx
);
6016 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
6019 /* If CTX is a nested parallel directive. Find the immediately
6020 enclosing parallel or workshare construct that contains a
6021 mapping for OVAR. */
6022 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
6024 t
= omp_member_access_dummy_var (var
);
6027 var
= DECL_VALUE_EXPR (var
);
6028 tree o
= maybe_lookup_decl_in_outer_ctx (t
, ctx
);
6030 var
= unshare_and_remap (var
, t
, o
);
6032 var
= unshare_expr (var
);
6035 if (use_pointer_for_field (ovar
, ctx
))
6037 x
= build_sender_ref (ovar
, ctx
);
6038 var
= build_fold_addr_expr (var
);
6039 gimplify_assign (x
, var
, ilist
);
6043 x
= build_sender_ref (ovar
, ctx
);
6044 gimplify_assign (x
, var
, ilist
);
6046 if (!TREE_READONLY (var
)
6047 /* We don't need to receive a new reference to a result
6048 or parm decl. In fact we may not store to it as we will
6049 invalidate any pending RSO and generate wrong gimple
6051 && !((TREE_CODE (var
) == RESULT_DECL
6052 || TREE_CODE (var
) == PARM_DECL
)
6053 && DECL_BY_REFERENCE (var
)))
6055 x
= build_sender_ref (ovar
, ctx
);
6056 gimplify_assign (var
, x
, olist
);
6062 /* Emit an OpenACC head marker call, encapulating the partitioning and
6063 other information that must be processed by the target compiler.
6064 Return the maximum number of dimensions the associated loop might
6065 be partitioned over. */
6068 lower_oacc_head_mark (location_t loc
, tree ddvar
, tree clauses
,
6069 gimple_seq
*seq
, omp_context
*ctx
)
6071 unsigned levels
= 0;
6073 tree gang_static
= NULL_TREE
;
6074 auto_vec
<tree
, 5> args
;
6076 args
.quick_push (build_int_cst
6077 (integer_type_node
, IFN_UNIQUE_OACC_HEAD_MARK
));
6078 args
.quick_push (ddvar
);
6079 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
6081 switch (OMP_CLAUSE_CODE (c
))
6083 case OMP_CLAUSE_GANG
:
6084 tag
|= OLF_DIM_GANG
;
6085 gang_static
= OMP_CLAUSE_GANG_STATIC_EXPR (c
);
6086 /* static:* is represented by -1, and we can ignore it, as
6087 scheduling is always static. */
6088 if (gang_static
&& integer_minus_onep (gang_static
))
6089 gang_static
= NULL_TREE
;
6093 case OMP_CLAUSE_WORKER
:
6094 tag
|= OLF_DIM_WORKER
;
6098 case OMP_CLAUSE_VECTOR
:
6099 tag
|= OLF_DIM_VECTOR
;
6103 case OMP_CLAUSE_SEQ
:
6107 case OMP_CLAUSE_AUTO
:
6111 case OMP_CLAUSE_INDEPENDENT
:
6112 tag
|= OLF_INDEPENDENT
;
6122 if (DECL_P (gang_static
))
6123 gang_static
= build_outer_var_ref (gang_static
, ctx
);
6124 tag
|= OLF_GANG_STATIC
;
6127 /* In a parallel region, loops are implicitly INDEPENDENT. */
6128 omp_context
*tgt
= enclosing_target_ctx (ctx
);
6129 if (!tgt
|| is_oacc_parallel (tgt
))
6130 tag
|= OLF_INDEPENDENT
;
6132 /* A loop lacking SEQ, GANG, WORKER and/or VECTOR is implicitly AUTO. */
6133 if (!(tag
& (((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
)
6137 /* Ensure at least one level. */
6141 args
.quick_push (build_int_cst (integer_type_node
, levels
));
6142 args
.quick_push (build_int_cst (integer_type_node
, tag
));
6144 args
.quick_push (gang_static
);
6146 gcall
*call
= gimple_build_call_internal_vec (IFN_UNIQUE
, args
);
6147 gimple_set_location (call
, loc
);
6148 gimple_set_lhs (call
, ddvar
);
6149 gimple_seq_add_stmt (seq
, call
);
6154 /* Emit an OpenACC lopp head or tail marker to SEQ. LEVEL is the
6155 partitioning level of the enclosed region. */
6158 lower_oacc_loop_marker (location_t loc
, tree ddvar
, bool head
,
6159 tree tofollow
, gimple_seq
*seq
)
6161 int marker_kind
= (head
? IFN_UNIQUE_OACC_HEAD_MARK
6162 : IFN_UNIQUE_OACC_TAIL_MARK
);
6163 tree marker
= build_int_cst (integer_type_node
, marker_kind
);
6164 int nargs
= 2 + (tofollow
!= NULL_TREE
);
6165 gcall
*call
= gimple_build_call_internal (IFN_UNIQUE
, nargs
,
6166 marker
, ddvar
, tofollow
);
6167 gimple_set_location (call
, loc
);
6168 gimple_set_lhs (call
, ddvar
);
6169 gimple_seq_add_stmt (seq
, call
);
6172 /* Generate the before and after OpenACC loop sequences. CLAUSES are
6173 the loop clauses, from which we extract reductions. Initialize
6177 lower_oacc_head_tail (location_t loc
, tree clauses
,
6178 gimple_seq
*head
, gimple_seq
*tail
, omp_context
*ctx
)
6181 tree ddvar
= create_tmp_var (integer_type_node
, ".data_dep");
6182 gimple_seq_add_stmt (head
, gimple_build_assign (ddvar
, integer_zero_node
));
6184 unsigned count
= lower_oacc_head_mark (loc
, ddvar
, clauses
, head
, ctx
);
6186 lower_oacc_loop_marker (loc
, ddvar
, false, integer_zero_node
, tail
);
6188 tree fork_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_FORK
);
6189 tree join_kind
= build_int_cst (unsigned_type_node
, IFN_UNIQUE_OACC_JOIN
);
6191 for (unsigned done
= 1; count
; count
--, done
++)
6193 gimple_seq fork_seq
= NULL
;
6194 gimple_seq join_seq
= NULL
;
6196 tree place
= build_int_cst (integer_type_node
, -1);
6197 gcall
*fork
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6198 fork_kind
, ddvar
, place
);
6199 gimple_set_location (fork
, loc
);
6200 gimple_set_lhs (fork
, ddvar
);
6202 gcall
*join
= gimple_build_call_internal (IFN_UNIQUE
, 3,
6203 join_kind
, ddvar
, place
);
6204 gimple_set_location (join
, loc
);
6205 gimple_set_lhs (join
, ddvar
);
6207 /* Mark the beginning of this level sequence. */
6209 lower_oacc_loop_marker (loc
, ddvar
, true,
6210 build_int_cst (integer_type_node
, count
),
6212 lower_oacc_loop_marker (loc
, ddvar
, false,
6213 build_int_cst (integer_type_node
, done
),
6216 lower_oacc_reductions (loc
, clauses
, place
, inner
,
6217 fork
, join
, &fork_seq
, &join_seq
, ctx
);
6219 /* Append this level to head. */
6220 gimple_seq_add_seq (head
, fork_seq
);
6221 /* Prepend it to tail. */
6222 gimple_seq_add_seq (&join_seq
, *tail
);
6228 /* Mark the end of the sequence. */
6229 lower_oacc_loop_marker (loc
, ddvar
, true, NULL_TREE
, head
);
6230 lower_oacc_loop_marker (loc
, ddvar
, false, NULL_TREE
, tail
);
6233 /* A convenience function to build an empty GIMPLE_COND with just the
6237 gimple_build_cond_empty (tree cond
)
6239 enum tree_code pred_code
;
6242 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
6243 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
6246 static void expand_omp_build_assign (gimple_stmt_iterator
*, tree
, tree
,
6249 /* Build the function calls to GOMP_parallel_start etc to actually
6250 generate the parallel operation. REGION is the parallel region
6251 being expanded. BB is the block where to insert the code. WS_ARGS
6252 will be set if this is a call to a combined parallel+workshare
6253 construct, it contains the list of additional arguments needed by
6254 the workshare construct. */
6257 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
6258 gomp_parallel
*entry_stmt
,
6259 vec
<tree
, va_gc
> *ws_args
)
6261 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
6262 gimple_stmt_iterator gsi
;
6264 enum built_in_function start_ix
;
6266 location_t clause_loc
;
6267 vec
<tree
, va_gc
> *args
;
6269 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6271 /* Determine what flavor of GOMP_parallel we will be
6273 start_ix
= BUILT_IN_GOMP_PARALLEL
;
6274 if (is_combined_parallel (region
))
6276 switch (region
->inner
->type
)
6278 case GIMPLE_OMP_FOR
:
6279 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6280 switch (region
->inner
->sched_kind
)
6282 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
6285 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
6286 case OMP_CLAUSE_SCHEDULE_GUIDED
:
6287 if (region
->inner
->sched_modifiers
6288 & OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
6290 start_ix2
= 3 + region
->inner
->sched_kind
;
6295 start_ix2
= region
->inner
->sched_kind
;
6298 start_ix2
+= (int) BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
;
6299 start_ix
= (enum built_in_function
) start_ix2
;
6301 case GIMPLE_OMP_SECTIONS
:
6302 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
6309 /* By default, the value of NUM_THREADS is zero (selected at run time)
6310 and there is no conditional. */
6312 val
= build_int_cst (unsigned_type_node
, 0);
6313 flags
= build_int_cst (unsigned_type_node
, 0);
6315 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6317 cond
= OMP_CLAUSE_IF_EXPR (c
);
6319 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
6322 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
6323 clause_loc
= OMP_CLAUSE_LOCATION (c
);
6326 clause_loc
= gimple_location (entry_stmt
);
6328 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
6330 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
6332 /* Ensure 'val' is of the correct type. */
6333 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
6335 /* If we found the clause 'if (cond)', build either
6336 (cond != 0) or (cond ? val : 1u). */
6339 cond
= gimple_boolify (cond
);
6341 if (integer_zerop (val
))
6342 val
= fold_build2_loc (clause_loc
,
6343 EQ_EXPR
, unsigned_type_node
, cond
,
6344 build_int_cst (TREE_TYPE (cond
), 0));
6347 basic_block cond_bb
, then_bb
, else_bb
;
6348 edge e
, e_then
, e_else
;
6349 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
6351 tmp_var
= create_tmp_var (TREE_TYPE (val
));
6352 if (gimple_in_ssa_p (cfun
))
6354 tmp_then
= make_ssa_name (tmp_var
);
6355 tmp_else
= make_ssa_name (tmp_var
);
6356 tmp_join
= make_ssa_name (tmp_var
);
6365 e
= split_block_after_labels (bb
);
6370 then_bb
= create_empty_bb (cond_bb
);
6371 else_bb
= create_empty_bb (then_bb
);
6372 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
6373 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
6375 stmt
= gimple_build_cond_empty (cond
);
6376 gsi
= gsi_start_bb (cond_bb
);
6377 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6379 gsi
= gsi_start_bb (then_bb
);
6380 expand_omp_build_assign (&gsi
, tmp_then
, val
, true);
6382 gsi
= gsi_start_bb (else_bb
);
6383 expand_omp_build_assign (&gsi
, tmp_else
,
6384 build_int_cst (unsigned_type_node
, 1),
6387 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
6388 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
6389 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
6390 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
6391 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
6392 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
6394 if (gimple_in_ssa_p (cfun
))
6396 gphi
*phi
= create_phi_node (tmp_join
, bb
);
6397 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
6398 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
6404 gsi
= gsi_start_bb (bb
);
6405 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
6406 false, GSI_CONTINUE_LINKING
);
6409 gsi
= gsi_last_bb (bb
);
6410 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6412 t1
= null_pointer_node
;
6414 t1
= build_fold_addr_expr (t
);
6415 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
6417 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
6418 args
->quick_push (t2
);
6419 args
->quick_push (t1
);
6420 args
->quick_push (val
);
6422 args
->splice (*ws_args
);
6423 args
->quick_push (flags
);
6425 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
6426 builtin_decl_explicit (start_ix
), args
);
6428 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6429 false, GSI_CONTINUE_LINKING
);
6432 /* Insert a function call whose name is FUNC_NAME with the information from
6433 ENTRY_STMT into the basic_block BB. */
6436 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
6437 vec
<tree
, va_gc
> *ws_args
)
6440 gimple_stmt_iterator gsi
;
6441 vec
<tree
, va_gc
> *args
;
6443 gcc_assert (vec_safe_length (ws_args
) == 2);
6444 tree func_name
= (*ws_args
)[0];
6445 tree grain
= (*ws_args
)[1];
6447 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
6448 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
6449 gcc_assert (count
!= NULL_TREE
);
6450 count
= OMP_CLAUSE_OPERAND (count
, 0);
6452 gsi
= gsi_last_bb (bb
);
6453 t
= gimple_omp_parallel_data_arg (entry_stmt
);
6455 t1
= null_pointer_node
;
6457 t1
= build_fold_addr_expr (t
);
6458 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
6460 vec_alloc (args
, 4);
6461 args
->quick_push (t2
);
6462 args
->quick_push (t1
);
6463 args
->quick_push (count
);
6464 args
->quick_push (grain
);
6465 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
6467 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
6468 GSI_CONTINUE_LINKING
);
6471 /* Build the function call to GOMP_task to actually
6472 generate the task operation. BB is the block where to insert the code. */
6475 expand_task_call (struct omp_region
*region
, basic_block bb
,
6476 gomp_task
*entry_stmt
)
6479 gimple_stmt_iterator gsi
;
6480 location_t loc
= gimple_location (entry_stmt
);
6482 tree clauses
= gimple_omp_task_clauses (entry_stmt
);
6484 tree ifc
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
6485 tree untied
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
6486 tree mergeable
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
6487 tree depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
6488 tree finalc
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
6489 tree priority
= find_omp_clause (clauses
, OMP_CLAUSE_PRIORITY
);
6492 = (untied
? GOMP_TASK_FLAG_UNTIED
: 0)
6493 | (mergeable
? GOMP_TASK_FLAG_MERGEABLE
: 0)
6494 | (depend
? GOMP_TASK_FLAG_DEPEND
: 0);
6496 bool taskloop_p
= gimple_omp_task_taskloop_p (entry_stmt
);
6497 tree startvar
= NULL_TREE
, endvar
= NULL_TREE
, step
= NULL_TREE
;
6498 tree num_tasks
= NULL_TREE
;
6502 gimple
*g
= last_stmt (region
->outer
->entry
);
6503 gcc_assert (gimple_code (g
) == GIMPLE_OMP_FOR
6504 && gimple_omp_for_kind (g
) == GF_OMP_FOR_KIND_TASKLOOP
);
6505 struct omp_for_data fd
;
6506 extract_omp_for_data (as_a
<gomp_for
*> (g
), &fd
, NULL
);
6507 startvar
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6508 endvar
= find_omp_clause (OMP_CLAUSE_CHAIN (startvar
),
6509 OMP_CLAUSE__LOOPTEMP_
);
6510 startvar
= OMP_CLAUSE_DECL (startvar
);
6511 endvar
= OMP_CLAUSE_DECL (endvar
);
6512 step
= fold_convert_loc (loc
, fd
.iter_type
, fd
.loop
.step
);
6513 if (fd
.loop
.cond_code
== LT_EXPR
)
6514 iflags
|= GOMP_TASK_FLAG_UP
;
6515 tree tclauses
= gimple_omp_for_clauses (g
);
6516 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_NUM_TASKS
);
6518 num_tasks
= OMP_CLAUSE_NUM_TASKS_EXPR (num_tasks
);
6521 num_tasks
= find_omp_clause (tclauses
, OMP_CLAUSE_GRAINSIZE
);
6524 iflags
|= GOMP_TASK_FLAG_GRAINSIZE
;
6525 num_tasks
= OMP_CLAUSE_GRAINSIZE_EXPR (num_tasks
);
6528 num_tasks
= integer_zero_node
;
6530 num_tasks
= fold_convert_loc (loc
, long_integer_type_node
, num_tasks
);
6531 if (ifc
== NULL_TREE
)
6532 iflags
|= GOMP_TASK_FLAG_IF
;
6533 if (find_omp_clause (tclauses
, OMP_CLAUSE_NOGROUP
))
6534 iflags
|= GOMP_TASK_FLAG_NOGROUP
;
6535 ull
= fd
.iter_type
== long_long_unsigned_type_node
;
6538 iflags
|= GOMP_TASK_FLAG_PRIORITY
;
6540 tree flags
= build_int_cst (unsigned_type_node
, iflags
);
6542 tree cond
= boolean_true_node
;
6547 tree t
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6548 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6549 build_int_cst (unsigned_type_node
,
6551 build_int_cst (unsigned_type_node
, 0));
6552 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
,
6556 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (ifc
));
6561 tree t
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (finalc
));
6562 t
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, t
,
6563 build_int_cst (unsigned_type_node
,
6564 GOMP_TASK_FLAG_FINAL
),
6565 build_int_cst (unsigned_type_node
, 0));
6566 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, t
);
6569 depend
= OMP_CLAUSE_DECL (depend
);
6571 depend
= build_int_cst (ptr_type_node
, 0);
6573 priority
= fold_convert (integer_type_node
,
6574 OMP_CLAUSE_PRIORITY_EXPR (priority
));
6576 priority
= integer_zero_node
;
6578 gsi
= gsi_last_bb (bb
);
6579 tree t
= gimple_omp_task_data_arg (entry_stmt
);
6581 t2
= null_pointer_node
;
6583 t2
= build_fold_addr_expr_loc (loc
, t
);
6584 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
6585 t
= gimple_omp_task_copy_fn (entry_stmt
);
6587 t3
= null_pointer_node
;
6589 t3
= build_fold_addr_expr_loc (loc
, t
);
6592 t
= build_call_expr (ull
6593 ? builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP_ULL
)
6594 : builtin_decl_explicit (BUILT_IN_GOMP_TASKLOOP
),
6596 gimple_omp_task_arg_size (entry_stmt
),
6597 gimple_omp_task_arg_align (entry_stmt
), flags
,
6598 num_tasks
, priority
, startvar
, endvar
, step
);
6600 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
6602 gimple_omp_task_arg_size (entry_stmt
),
6603 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
6606 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6607 false, GSI_CONTINUE_LINKING
);
6611 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
6612 catch handler and return it. This prevents programs from violating the
6613 structured block semantics with throws. */
6616 maybe_catch_exception (gimple_seq body
)
6621 if (!flag_exceptions
)
6624 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
6625 decl
= lang_hooks
.eh_protect_cleanup_actions ();
6627 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
6629 g
= gimple_build_eh_must_not_throw (decl
);
6630 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
6633 return gimple_seq_alloc_with_stmt (g
);
6636 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
6639 vec2chain (vec
<tree
, va_gc
> *v
)
6641 tree chain
= NULL_TREE
, t
;
6644 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
6646 DECL_CHAIN (t
) = chain
;
6654 /* Remove barriers in REGION->EXIT's block. Note that this is only
6655 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
6656 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
6657 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
6661 remove_exit_barrier (struct omp_region
*region
)
6663 gimple_stmt_iterator gsi
;
6664 basic_block exit_bb
;
6668 int any_addressable_vars
= -1;
6670 exit_bb
= region
->exit
;
6672 /* If the parallel region doesn't return, we don't have REGION->EXIT
6677 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
6678 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
6679 statements that can appear in between are extremely limited -- no
6680 memory operations at all. Here, we allow nothing at all, so the
6681 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
6682 gsi
= gsi_last_bb (exit_bb
);
6683 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
6685 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
6688 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
6690 gsi
= gsi_last_bb (e
->src
);
6691 if (gsi_end_p (gsi
))
6693 stmt
= gsi_stmt (gsi
);
6694 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
6695 && !gimple_omp_return_nowait_p (stmt
))
6697 /* OpenMP 3.0 tasks unfortunately prevent this optimization
6698 in many cases. If there could be tasks queued, the barrier
6699 might be needed to let the tasks run before some local
6700 variable of the parallel that the task uses as shared
6701 runs out of scope. The task can be spawned either
6702 from within current function (this would be easy to check)
6703 or from some function it calls and gets passed an address
6704 of such a variable. */
6705 if (any_addressable_vars
< 0)
6707 gomp_parallel
*parallel_stmt
6708 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
6709 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
6710 tree local_decls
, block
, decl
;
6713 any_addressable_vars
= 0;
6714 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
6715 if (TREE_ADDRESSABLE (decl
))
6717 any_addressable_vars
= 1;
6720 for (block
= gimple_block (stmt
);
6721 !any_addressable_vars
6723 && TREE_CODE (block
) == BLOCK
;
6724 block
= BLOCK_SUPERCONTEXT (block
))
6726 for (local_decls
= BLOCK_VARS (block
);
6728 local_decls
= DECL_CHAIN (local_decls
))
6729 if (TREE_ADDRESSABLE (local_decls
))
6731 any_addressable_vars
= 1;
6734 if (block
== gimple_block (parallel_stmt
))
6738 if (!any_addressable_vars
)
6739 gimple_omp_return_set_nowait (stmt
);
6745 remove_exit_barriers (struct omp_region
*region
)
6747 if (region
->type
== GIMPLE_OMP_PARALLEL
)
6748 remove_exit_barrier (region
);
6752 region
= region
->inner
;
6753 remove_exit_barriers (region
);
6754 while (region
->next
)
6756 region
= region
->next
;
6757 remove_exit_barriers (region
);
6762 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
6763 calls. These can't be declared as const functions, but
6764 within one parallel body they are constant, so they can be
6765 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
6766 which are declared const. Similarly for task body, except
6767 that in untied task omp_get_thread_num () can change at any task
6768 scheduling point. */
6771 optimize_omp_library_calls (gimple
*entry_stmt
)
6774 gimple_stmt_iterator gsi
;
6775 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6776 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
6777 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6778 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
6779 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
6780 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
6781 OMP_CLAUSE_UNTIED
) != NULL
);
6783 FOR_EACH_BB_FN (bb
, cfun
)
6784 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
6786 gimple
*call
= gsi_stmt (gsi
);
6789 if (is_gimple_call (call
)
6790 && (decl
= gimple_call_fndecl (call
))
6791 && DECL_EXTERNAL (decl
)
6792 && TREE_PUBLIC (decl
)
6793 && DECL_INITIAL (decl
) == NULL
)
6797 if (DECL_NAME (decl
) == thr_num_id
)
6799 /* In #pragma omp task untied omp_get_thread_num () can change
6800 during the execution of the task region. */
6803 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6805 else if (DECL_NAME (decl
) == num_thr_id
)
6806 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6810 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
6811 || gimple_call_num_args (call
) != 0)
6814 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
6817 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
6818 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
6819 TREE_TYPE (TREE_TYPE (built_in
))))
6822 gimple_call_set_fndecl (call
, built_in
);
6827 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
6831 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
6835 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
6836 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
6839 if (TREE_CODE (t
) == ADDR_EXPR
)
6840 recompute_tree_invariant_for_addr_expr (t
);
6842 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
6846 /* Prepend or append TO = FROM assignment before or after *GSI_P. */
6849 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
,
6852 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
6853 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
6854 !after
, after
? GSI_CONTINUE_LINKING
6856 gimple
*stmt
= gimple_build_assign (to
, from
);
6858 gsi_insert_after (gsi_p
, stmt
, GSI_CONTINUE_LINKING
);
6860 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
6861 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
6862 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
6864 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
6865 gimple_regimplify_operands (stmt
, &gsi
);
6869 /* Expand the OpenMP parallel or task directive starting at REGION. */
6872 expand_omp_taskreg (struct omp_region
*region
)
6874 basic_block entry_bb
, exit_bb
, new_bb
;
6875 struct function
*child_cfun
;
6876 tree child_fn
, block
, t
;
6877 gimple_stmt_iterator gsi
;
6878 gimple
*entry_stmt
, *stmt
;
6880 vec
<tree
, va_gc
> *ws_args
;
6882 entry_stmt
= last_stmt (region
->entry
);
6883 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
6884 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
6886 entry_bb
= region
->entry
;
6887 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
)
6888 exit_bb
= region
->cont
;
6890 exit_bb
= region
->exit
;
6894 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
6895 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
6896 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
6899 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
6900 and the inner statement contains the name of the built-in function
6902 ws_args
= region
->inner
->ws_args
;
6903 else if (is_combined_parallel (region
))
6904 ws_args
= region
->ws_args
;
6908 if (child_cfun
->cfg
)
6910 /* Due to inlining, it may happen that we have already outlined
6911 the region, in which case all we need to do is make the
6912 sub-graph unreachable and emit the parallel call. */
6913 edge entry_succ_e
, exit_succ_e
;
6915 entry_succ_e
= single_succ_edge (entry_bb
);
6917 gsi
= gsi_last_bb (entry_bb
);
6918 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
6919 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
6920 gsi_remove (&gsi
, true);
6925 exit_succ_e
= single_succ_edge (exit_bb
);
6926 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
6928 remove_edge_and_dominated_blocks (entry_succ_e
);
6932 unsigned srcidx
, dstidx
, num
;
6934 /* If the parallel region needs data sent from the parent
6935 function, then the very first statement (except possible
6936 tree profile counter updates) of the parallel body
6937 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
6938 &.OMP_DATA_O is passed as an argument to the child function,
6939 we need to replace it with the argument as seen by the child
6942 In most cases, this will end up being the identity assignment
6943 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
6944 a function call that has been inlined, the original PARM_DECL
6945 .OMP_DATA_I may have been converted into a different local
6946 variable. In which case, we need to keep the assignment. */
6947 if (gimple_omp_taskreg_data_arg (entry_stmt
))
6949 basic_block entry_succ_bb
6950 = single_succ_p (entry_bb
) ? single_succ (entry_bb
)
6951 : FALLTHRU_EDGE (entry_bb
)->dest
;
6953 gimple
*parcopy_stmt
= NULL
;
6955 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
6959 gcc_assert (!gsi_end_p (gsi
));
6960 stmt
= gsi_stmt (gsi
);
6961 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
6964 if (gimple_num_ops (stmt
) == 2)
6966 tree arg
= gimple_assign_rhs1 (stmt
);
6968 /* We're ignore the subcode because we're
6969 effectively doing a STRIP_NOPS. */
6971 if (TREE_CODE (arg
) == ADDR_EXPR
6972 && TREE_OPERAND (arg
, 0)
6973 == gimple_omp_taskreg_data_arg (entry_stmt
))
6975 parcopy_stmt
= stmt
;
6981 gcc_assert (parcopy_stmt
!= NULL
);
6982 arg
= DECL_ARGUMENTS (child_fn
);
6984 if (!gimple_in_ssa_p (cfun
))
6986 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
6987 gsi_remove (&gsi
, true);
6990 /* ?? Is setting the subcode really necessary ?? */
6991 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
6992 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
6997 tree lhs
= gimple_assign_lhs (parcopy_stmt
);
6998 gcc_assert (SSA_NAME_VAR (lhs
) == arg
);
6999 /* We'd like to set the rhs to the default def in the child_fn,
7000 but it's too early to create ssa names in the child_fn.
7001 Instead, we set the rhs to the parm. In
7002 move_sese_region_to_fn, we introduce a default def for the
7003 parm, map the parm to it's default def, and once we encounter
7004 this stmt, replace the parm with the default def. */
7005 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
7006 update_stmt (parcopy_stmt
);
7010 /* Declare local variables needed in CHILD_CFUN. */
7011 block
= DECL_INITIAL (child_fn
);
7012 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
7013 /* The gimplifier could record temporaries in parallel/task block
7014 rather than in containing function's local_decls chain,
7015 which would mean cgraph missed finalizing them. Do it now. */
7016 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
7017 if (TREE_CODE (t
) == VAR_DECL
7019 && !DECL_EXTERNAL (t
))
7020 varpool_node::finalize_decl (t
);
7021 DECL_SAVED_TREE (child_fn
) = NULL
;
7022 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7023 gimple_set_body (child_fn
, NULL
);
7024 TREE_USED (block
) = 1;
7026 /* Reset DECL_CONTEXT on function arguments. */
7027 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
7028 DECL_CONTEXT (t
) = child_fn
;
7030 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
7031 so that it can be moved to the child function. */
7032 gsi
= gsi_last_bb (entry_bb
);
7033 stmt
= gsi_stmt (gsi
);
7034 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
7035 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
7036 e
= split_block (entry_bb
, stmt
);
7037 gsi_remove (&gsi
, true);
7040 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7041 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7044 e2
= make_edge (e
->src
, BRANCH_EDGE (entry_bb
)->dest
, EDGE_ABNORMAL
);
7045 gcc_assert (e2
->dest
== region
->exit
);
7046 remove_edge (BRANCH_EDGE (entry_bb
));
7047 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e
->src
);
7048 gsi
= gsi_last_bb (region
->exit
);
7049 gcc_assert (!gsi_end_p (gsi
)
7050 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
7051 gsi_remove (&gsi
, true);
7054 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
7057 gsi
= gsi_last_bb (exit_bb
);
7058 gcc_assert (!gsi_end_p (gsi
)
7059 && (gimple_code (gsi_stmt (gsi
))
7060 == (e2
? GIMPLE_OMP_CONTINUE
: GIMPLE_OMP_RETURN
)));
7061 stmt
= gimple_build_return (NULL
);
7062 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
7063 gsi_remove (&gsi
, true);
7066 /* Move the parallel region into CHILD_CFUN. */
7068 if (gimple_in_ssa_p (cfun
))
7070 init_tree_ssa (child_cfun
);
7071 init_ssa_operands (child_cfun
);
7072 child_cfun
->gimple_df
->in_ssa_p
= true;
7076 block
= gimple_block (entry_stmt
);
7078 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7080 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
7083 basic_block dest_bb
= e2
->dest
;
7085 make_edge (new_bb
, dest_bb
, EDGE_FALLTHRU
);
7087 set_immediate_dominator (CDI_DOMINATORS
, dest_bb
, new_bb
);
7089 /* When the OMP expansion process cannot guarantee an up-to-date
7090 loop tree arrange for the child function to fixup loops. */
7091 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7092 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
7094 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
7095 num
= vec_safe_length (child_cfun
->local_decls
);
7096 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
7098 t
= (*child_cfun
->local_decls
)[srcidx
];
7099 if (DECL_CONTEXT (t
) == cfun
->decl
)
7101 if (srcidx
!= dstidx
)
7102 (*child_cfun
->local_decls
)[dstidx
] = t
;
7106 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
7108 /* Inform the callgraph about the new function. */
7109 child_cfun
->curr_properties
= cfun
->curr_properties
;
7110 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
7111 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
7112 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
7113 node
->parallelized_function
= 1;
7114 cgraph_node::add_new_function (child_fn
, true);
7116 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
7117 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
7119 /* Fix the callgraph edges for child_cfun. Those for cfun will be
7120 fixed in a following pass. */
7121 push_cfun (child_cfun
);
7123 assign_assembler_name_if_neeeded (child_fn
);
7126 optimize_omp_library_calls (entry_stmt
);
7127 cgraph_edge::rebuild_edges ();
7129 /* Some EH regions might become dead, see PR34608. If
7130 pass_cleanup_cfg isn't the first pass to happen with the
7131 new child, these dead EH edges might cause problems.
7132 Clean them up now. */
7133 if (flag_exceptions
)
7136 bool changed
= false;
7138 FOR_EACH_BB_FN (bb
, cfun
)
7139 changed
|= gimple_purge_dead_eh_edges (bb
);
7141 cleanup_tree_cfg ();
7143 if (gimple_in_ssa_p (cfun
))
7144 update_ssa (TODO_update_ssa
);
7145 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7146 verify_loop_structure ();
7149 if (dump_file
&& !gimple_in_ssa_p (cfun
))
7151 omp_any_child_fn_dumped
= true;
7152 dump_function_header (dump_file
, child_fn
, dump_flags
);
7153 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
7157 /* Emit a library call to launch the children threads. */
7159 expand_cilk_for_call (new_bb
,
7160 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7161 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
7162 expand_parallel_call (region
, new_bb
,
7163 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
7165 expand_task_call (region
, new_bb
, as_a
<gomp_task
*> (entry_stmt
));
7166 if (gimple_in_ssa_p (cfun
))
7167 update_ssa (TODO_update_ssa_only_virtuals
);
7170 /* Information about members of an OpenACC collapsed loop nest. */
7172 struct oacc_collapse
7174 tree base
; /* Base value. */
7175 tree iters
; /* Number of steps. */
7176 tree step
; /* step size. */
7179 /* Helper for expand_oacc_for. Determine collapsed loop information.
7180 Fill in COUNTS array. Emit any initialization code before GSI.
7181 Return the calculated outer loop bound of BOUND_TYPE. */
7184 expand_oacc_collapse_init (const struct omp_for_data
*fd
,
7185 gimple_stmt_iterator
*gsi
,
7186 oacc_collapse
*counts
, tree bound_type
)
7188 tree total
= build_int_cst (bound_type
, 1);
7191 gcc_assert (integer_onep (fd
->loop
.step
));
7192 gcc_assert (integer_zerop (fd
->loop
.n1
));
7194 for (ix
= 0; ix
!= fd
->collapse
; ix
++)
7196 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7198 tree iter_type
= TREE_TYPE (loop
->v
);
7199 tree diff_type
= iter_type
;
7200 tree plus_type
= iter_type
;
7202 gcc_assert (loop
->cond_code
== fd
->loop
.cond_code
);
7204 if (POINTER_TYPE_P (iter_type
))
7205 plus_type
= sizetype
;
7206 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
7207 diff_type
= signed_type_for (diff_type
);
7211 tree s
= loop
->step
;
7212 bool up
= loop
->cond_code
== LT_EXPR
;
7213 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
7217 b
= force_gimple_operand_gsi (gsi
, b
, true, NULL_TREE
,
7218 true, GSI_SAME_STMT
);
7219 e
= force_gimple_operand_gsi (gsi
, e
, true, NULL_TREE
,
7220 true, GSI_SAME_STMT
);
7222 /* Convert the step, avoiding possible unsigned->signed overflow. */
7223 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
7225 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
7226 s
= fold_convert (diff_type
, s
);
7228 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
7229 s
= force_gimple_operand_gsi (gsi
, s
, true, NULL_TREE
,
7230 true, GSI_SAME_STMT
);
7232 /* Determine the range, avoiding possible unsigned->signed overflow. */
7233 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
7234 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
7235 fold_convert (plus_type
, negating
? b
: e
),
7236 fold_convert (plus_type
, negating
? e
: b
));
7237 expr
= fold_convert (diff_type
, expr
);
7239 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
7240 tree range
= force_gimple_operand_gsi
7241 (gsi
, expr
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7243 /* Determine number of iterations. */
7244 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
7245 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
7246 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
7248 tree iters
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7249 true, GSI_SAME_STMT
);
7251 counts
[ix
].base
= b
;
7252 counts
[ix
].iters
= iters
;
7253 counts
[ix
].step
= s
;
7255 total
= fold_build2 (MULT_EXPR
, bound_type
, total
,
7256 fold_convert (bound_type
, iters
));
7262 /* Emit initializers for collapsed loop members. IVAR is the outer
7263 loop iteration variable, from which collapsed loop iteration values
7264 are calculated. COUNTS array has been initialized by
7265 expand_oacc_collapse_inits. */
7268 expand_oacc_collapse_vars (const struct omp_for_data
*fd
,
7269 gimple_stmt_iterator
*gsi
,
7270 const oacc_collapse
*counts
, tree ivar
)
7272 tree ivar_type
= TREE_TYPE (ivar
);
7274 /* The most rapidly changing iteration variable is the innermost
7276 for (int ix
= fd
->collapse
; ix
--;)
7278 const omp_for_data_loop
*loop
= &fd
->loops
[ix
];
7279 const oacc_collapse
*collapse
= &counts
[ix
];
7280 tree iter_type
= TREE_TYPE (loop
->v
);
7281 tree diff_type
= TREE_TYPE (collapse
->step
);
7282 tree plus_type
= iter_type
;
7283 enum tree_code plus_code
= PLUS_EXPR
;
7286 if (POINTER_TYPE_P (iter_type
))
7288 plus_code
= POINTER_PLUS_EXPR
;
7289 plus_type
= sizetype
;
7292 expr
= fold_build2 (TRUNC_MOD_EXPR
, ivar_type
, ivar
,
7293 fold_convert (ivar_type
, collapse
->iters
));
7294 expr
= fold_build2 (MULT_EXPR
, diff_type
, fold_convert (diff_type
, expr
),
7296 expr
= fold_build2 (plus_code
, iter_type
, collapse
->base
,
7297 fold_convert (plus_type
, expr
));
7298 expr
= force_gimple_operand_gsi (gsi
, expr
, false, NULL_TREE
,
7299 true, GSI_SAME_STMT
);
7300 gassign
*ass
= gimple_build_assign (loop
->v
, expr
);
7301 gsi_insert_before (gsi
, ass
, GSI_SAME_STMT
);
7305 expr
= fold_build2 (TRUNC_DIV_EXPR
, ivar_type
, ivar
,
7306 fold_convert (ivar_type
, collapse
->iters
));
7307 ivar
= force_gimple_operand_gsi (gsi
, expr
, true, NULL_TREE
,
7308 true, GSI_SAME_STMT
);
7314 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
7315 of the combined collapse > 1 loop constructs, generate code like:
7316 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
7321 count3 = (adj + N32 - N31) / STEP3;
7322 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
7327 count2 = (adj + N22 - N21) / STEP2;
7328 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
7333 count1 = (adj + N12 - N11) / STEP1;
7334 count = count1 * count2 * count3;
7335 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
7337 and set ZERO_ITER_BB to that bb. If this isn't the outermost
7338 of the combined loop constructs, just initialize COUNTS array
7339 from the _looptemp_ clauses. */
7341 /* NOTE: It *could* be better to moosh all of the BBs together,
7342 creating one larger BB with all the computation and the unexpected
7343 jump at the end. I.e.
7345 bool zero3, zero2, zero1, zero;
7348 count3 = (N32 - N31) /[cl] STEP3;
7350 count2 = (N22 - N21) /[cl] STEP2;
7352 count1 = (N12 - N11) /[cl] STEP1;
7353 zero = zero3 || zero2 || zero1;
7354 count = count1 * count2 * count3;
7355 if (__builtin_expect(zero, false)) goto zero_iter_bb;
7357 After all, we expect the zero=false, and thus we expect to have to
7358 evaluate all of the comparison expressions, so short-circuiting
7359 oughtn't be a win. Since the condition isn't protecting a
7360 denominator, we're not concerned about divide-by-zero, so we can
7361 fully evaluate count even if a numerator turned out to be wrong.
7363 It seems like putting this all together would create much better
7364 scheduling opportunities, and less pressure on the chip's branch
7368 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7369 basic_block
&entry_bb
, tree
*counts
,
7370 basic_block
&zero_iter1_bb
, int &first_zero_iter1
,
7371 basic_block
&zero_iter2_bb
, int &first_zero_iter2
,
7372 basic_block
&l2_dom_bb
)
7374 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
7378 /* Collapsed loops need work for expansion into SSA form. */
7379 gcc_assert (!gimple_in_ssa_p (cfun
));
7381 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
7382 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
7384 gcc_assert (fd
->ordered
== 0);
7385 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7386 isn't supposed to be handled, as the inner loop doesn't
7388 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7389 OMP_CLAUSE__LOOPTEMP_
);
7390 gcc_assert (innerc
);
7391 for (i
= 0; i
< fd
->collapse
; i
++)
7393 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7394 OMP_CLAUSE__LOOPTEMP_
);
7395 gcc_assert (innerc
);
7397 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
7399 counts
[0] = NULL_TREE
;
7404 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7406 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7407 counts
[i
] = NULL_TREE
;
7408 t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7409 fold_convert (itype
, fd
->loops
[i
].n1
),
7410 fold_convert (itype
, fd
->loops
[i
].n2
));
7411 if (t
&& integer_zerop (t
))
7413 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7414 counts
[i
] = build_int_cst (type
, 0);
7418 for (i
= 0; i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
7420 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7422 if (i
>= fd
->collapse
&& counts
[i
])
7424 if ((SSA_VAR_P (fd
->loop
.n2
) || i
>= fd
->collapse
)
7425 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
7426 fold_convert (itype
, fd
->loops
[i
].n1
),
7427 fold_convert (itype
, fd
->loops
[i
].n2
)))
7428 == NULL_TREE
|| !integer_onep (t
)))
7432 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
7433 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
7434 true, GSI_SAME_STMT
);
7435 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
7436 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
7437 true, GSI_SAME_STMT
);
7438 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
7439 NULL_TREE
, NULL_TREE
);
7440 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
7441 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
7442 expand_omp_regimplify_p
, NULL
, NULL
)
7443 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
7444 expand_omp_regimplify_p
, NULL
, NULL
))
7446 *gsi
= gsi_for_stmt (cond_stmt
);
7447 gimple_regimplify_operands (cond_stmt
, gsi
);
7449 e
= split_block (entry_bb
, cond_stmt
);
7450 basic_block
&zero_iter_bb
7451 = i
< fd
->collapse
? zero_iter1_bb
: zero_iter2_bb
;
7452 int &first_zero_iter
7453 = i
< fd
->collapse
? first_zero_iter1
: first_zero_iter2
;
7454 if (zero_iter_bb
== NULL
)
7456 gassign
*assign_stmt
;
7457 first_zero_iter
= i
;
7458 zero_iter_bb
= create_empty_bb (entry_bb
);
7459 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
7460 *gsi
= gsi_after_labels (zero_iter_bb
);
7461 if (i
< fd
->collapse
)
7462 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
7463 build_zero_cst (type
));
7466 counts
[i
] = create_tmp_reg (type
, ".count");
7468 = gimple_build_assign (counts
[i
], build_zero_cst (type
));
7470 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
7471 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
7474 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
7475 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
7476 e
->flags
= EDGE_TRUE_VALUE
;
7477 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
7478 if (l2_dom_bb
== NULL
)
7479 l2_dom_bb
= entry_bb
;
7481 *gsi
= gsi_last_bb (entry_bb
);
7484 if (POINTER_TYPE_P (itype
))
7485 itype
= signed_type_for (itype
);
7486 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
7488 t
= fold_build2 (PLUS_EXPR
, itype
,
7489 fold_convert (itype
, fd
->loops
[i
].step
), t
);
7490 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
7491 fold_convert (itype
, fd
->loops
[i
].n2
));
7492 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
7493 fold_convert (itype
, fd
->loops
[i
].n1
));
7494 /* ?? We could probably use CEIL_DIV_EXPR instead of
7495 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
7496 generate the same code in the end because generically we
7497 don't know that the values involved must be negative for
7499 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7500 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7501 fold_build1 (NEGATE_EXPR
, itype
, t
),
7502 fold_build1 (NEGATE_EXPR
, itype
,
7503 fold_convert (itype
,
7504 fd
->loops
[i
].step
)));
7506 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
7507 fold_convert (itype
, fd
->loops
[i
].step
));
7508 t
= fold_convert (type
, t
);
7509 if (TREE_CODE (t
) == INTEGER_CST
)
7513 if (i
< fd
->collapse
|| i
!= first_zero_iter2
)
7514 counts
[i
] = create_tmp_reg (type
, ".count");
7515 expand_omp_build_assign (gsi
, counts
[i
], t
);
7517 if (SSA_VAR_P (fd
->loop
.n2
) && i
< fd
->collapse
)
7522 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
7523 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
7529 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
7531 V3 = N31 + (T % count3) * STEP3;
7533 V2 = N21 + (T % count2) * STEP2;
7535 V1 = N11 + T * STEP1;
7536 if this loop doesn't have an inner loop construct combined with it.
7537 If it does have an inner loop construct combined with it and the
7538 iteration count isn't known constant, store values from counts array
7539 into its _looptemp_ temporaries instead. */
7542 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
7543 tree
*counts
, gimple
*inner_stmt
, tree startvar
)
7546 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7548 /* If fd->loop.n2 is constant, then no propagation of the counts
7549 is needed, they are constant. */
7550 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
7553 tree clauses
= gimple_code (inner_stmt
) != GIMPLE_OMP_FOR
7554 ? gimple_omp_taskreg_clauses (inner_stmt
)
7555 : gimple_omp_for_clauses (inner_stmt
);
7556 /* First two _looptemp_ clauses are for istart/iend, counts[0]
7557 isn't supposed to be handled, as the inner loop doesn't
7559 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7560 gcc_assert (innerc
);
7561 for (i
= 0; i
< fd
->collapse
; i
++)
7563 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7564 OMP_CLAUSE__LOOPTEMP_
);
7565 gcc_assert (innerc
);
7568 tree tem
= OMP_CLAUSE_DECL (innerc
);
7569 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
7570 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7571 false, GSI_CONTINUE_LINKING
);
7572 gassign
*stmt
= gimple_build_assign (tem
, t
);
7573 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7579 tree type
= TREE_TYPE (fd
->loop
.v
);
7580 tree tem
= create_tmp_reg (type
, ".tem");
7581 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
7582 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7584 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7586 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
7588 if (POINTER_TYPE_P (vtype
))
7589 itype
= signed_type_for (vtype
);
7591 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
7594 t
= fold_convert (itype
, t
);
7595 t
= fold_build2 (MULT_EXPR
, itype
, t
,
7596 fold_convert (itype
, fd
->loops
[i
].step
));
7597 if (POINTER_TYPE_P (vtype
))
7598 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
7600 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
7601 t
= force_gimple_operand_gsi (gsi
, t
,
7602 DECL_P (fd
->loops
[i
].v
)
7603 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7605 GSI_CONTINUE_LINKING
);
7606 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7607 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7610 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
7611 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
7612 false, GSI_CONTINUE_LINKING
);
7613 stmt
= gimple_build_assign (tem
, t
);
7614 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
7620 /* Helper function for expand_omp_for_*. Generate code like:
7623 if (V3 cond3 N32) goto BODY_BB; else goto L11;
7627 if (V2 cond2 N22) goto BODY_BB; else goto L12;
7634 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
7635 basic_block body_bb
)
7637 basic_block last_bb
, bb
, collapse_bb
= NULL
;
7639 gimple_stmt_iterator gsi
;
7645 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
7647 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
7649 bb
= create_empty_bb (last_bb
);
7650 add_bb_to_loop (bb
, last_bb
->loop_father
);
7651 gsi
= gsi_start_bb (bb
);
7653 if (i
< fd
->collapse
- 1)
7655 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
7656 e
->probability
= REG_BR_PROB_BASE
/ 8;
7658 t
= fd
->loops
[i
+ 1].n1
;
7659 t
= force_gimple_operand_gsi (&gsi
, t
,
7660 DECL_P (fd
->loops
[i
+ 1].v
)
7661 && TREE_ADDRESSABLE (fd
->loops
[i
7664 GSI_CONTINUE_LINKING
);
7665 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
7666 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7671 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
7673 if (POINTER_TYPE_P (vtype
))
7674 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7676 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
7677 t
= force_gimple_operand_gsi (&gsi
, t
,
7678 DECL_P (fd
->loops
[i
].v
)
7679 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
7680 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7681 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
7682 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7686 t
= fd
->loops
[i
].n2
;
7687 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7688 false, GSI_CONTINUE_LINKING
);
7689 tree v
= fd
->loops
[i
].v
;
7690 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
7691 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
7692 false, GSI_CONTINUE_LINKING
);
7693 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
7694 stmt
= gimple_build_cond_empty (t
);
7695 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7696 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
7697 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7700 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
7708 /* Expand #pragma omp ordered depend(source). */
7711 expand_omp_ordered_source (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7712 tree
*counts
, location_t loc
)
7714 enum built_in_function source_ix
7715 = fd
->iter_type
== long_integer_type_node
7716 ? BUILT_IN_GOMP_DOACROSS_POST
: BUILT_IN_GOMP_DOACROSS_ULL_POST
;
7718 = gimple_build_call (builtin_decl_explicit (source_ix
), 1,
7719 build_fold_addr_expr (counts
[fd
->ordered
]));
7720 gimple_set_location (g
, loc
);
7721 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
7724 /* Expand a single depend from #pragma omp ordered depend(sink:...). */
7727 expand_omp_ordered_sink (gimple_stmt_iterator
*gsi
, struct omp_for_data
*fd
,
7728 tree
*counts
, tree c
, location_t loc
)
7730 auto_vec
<tree
, 10> args
;
7731 enum built_in_function sink_ix
7732 = fd
->iter_type
== long_integer_type_node
7733 ? BUILT_IN_GOMP_DOACROSS_WAIT
: BUILT_IN_GOMP_DOACROSS_ULL_WAIT
;
7734 tree t
, off
, coff
= NULL_TREE
, deps
= OMP_CLAUSE_DECL (c
), cond
= NULL_TREE
;
7736 gimple_stmt_iterator gsi2
= *gsi
;
7737 bool warned_step
= false;
7739 for (i
= 0; i
< fd
->ordered
; i
++)
7741 off
= TREE_PURPOSE (deps
);
7742 if (!integer_zerop (off
))
7744 gcc_assert (fd
->loops
[i
].cond_code
== LT_EXPR
7745 || fd
->loops
[i
].cond_code
== GT_EXPR
);
7746 bool forward
= fd
->loops
[i
].cond_code
== LT_EXPR
;
7747 if (forward
^ OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7748 warning_at (loc
, 0, "%<depend(sink)%> clause waiting for "
7749 "lexically later iteration");
7752 deps
= TREE_CHAIN (deps
);
7754 /* If all offsets corresponding to the collapsed loops are zero,
7755 this depend clause can be ignored. FIXME: but there is still a
7756 flush needed. We need to emit one __sync_synchronize () for it
7757 though (perhaps conditionally)? Solve this together with the
7758 conservative dependence folding optimization.
7759 if (i >= fd->collapse)
7762 deps
= OMP_CLAUSE_DECL (c
);
7764 edge e1
= split_block (gsi_bb (gsi2
), gsi_stmt (gsi2
));
7765 edge e2
= split_block_after_labels (e1
->dest
);
7767 *gsi
= gsi_after_labels (e1
->dest
);
7768 for (i
= 0; i
< fd
->ordered
; i
++)
7770 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7771 if (POINTER_TYPE_P (itype
))
7774 deps
= TREE_CHAIN (deps
);
7775 off
= TREE_PURPOSE (deps
);
7776 tree s
= fold_convert_loc (loc
, itype
, fd
->loops
[i
].step
);
7778 if (integer_zerop (off
))
7779 t
= boolean_true_node
;
7783 tree co
= fold_convert_loc (loc
, itype
, off
);
7784 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7786 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7787 co
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, co
);
7788 a
= fold_build2_loc (loc
, POINTER_PLUS_EXPR
,
7789 TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].v
,
7792 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7793 a
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7794 fd
->loops
[i
].v
, co
);
7796 a
= fold_build2_loc (loc
, PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7797 fd
->loops
[i
].v
, co
);
7798 if (fd
->loops
[i
].cond_code
== LT_EXPR
)
7800 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7801 t
= fold_build2_loc (loc
, GE_EXPR
, boolean_type_node
, a
,
7804 t
= fold_build2_loc (loc
, LT_EXPR
, boolean_type_node
, a
,
7807 else if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7808 t
= fold_build2_loc (loc
, GT_EXPR
, boolean_type_node
, a
,
7811 t
= fold_build2_loc (loc
, LE_EXPR
, boolean_type_node
, a
,
7815 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
, cond
, t
);
7819 off
= fold_convert_loc (loc
, itype
, off
);
7821 if (fd
->loops
[i
].cond_code
== LT_EXPR
7822 ? !integer_onep (fd
->loops
[i
].step
)
7823 : !integer_minus_onep (fd
->loops
[i
].step
))
7825 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7826 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
,
7827 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
7830 t
= fold_build2_loc (loc
, TRUNC_MOD_EXPR
, itype
, off
, s
);
7831 t
= fold_build2_loc (loc
, EQ_EXPR
, boolean_type_node
, t
,
7832 build_int_cst (itype
, 0));
7833 if (integer_zerop (t
) && !warned_step
)
7835 warning_at (loc
, 0, "%<depend(sink)%> refers to iteration never "
7836 "in the iteration space");
7839 cond
= fold_build2_loc (loc
, BIT_AND_EXPR
, boolean_type_node
,
7843 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
7849 t
= fold_build2_loc (loc
, MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7850 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
7851 t
= fold_convert_loc (loc
, fd
->iter_type
, t
);
7853 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
7854 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
,
7855 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
7858 off
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, off
, s
);
7859 if (OMP_CLAUSE_DEPEND_SINK_NEGATIVE (deps
))
7860 off
= fold_build1_loc (loc
, NEGATE_EXPR
, itype
, off
);
7861 off
= fold_convert_loc (loc
, fd
->iter_type
, off
);
7862 if (i
<= fd
->collapse
- 1 && fd
->collapse
> 1)
7865 off
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, coff
,
7867 if (i
< fd
->collapse
- 1)
7869 coff
= fold_build2_loc (loc
, MULT_EXPR
, fd
->iter_type
, off
,
7874 off
= unshare_expr (off
);
7875 t
= fold_build2_loc (loc
, PLUS_EXPR
, fd
->iter_type
, t
, off
);
7876 t
= force_gimple_operand_gsi (gsi
, t
, true, NULL_TREE
,
7877 true, GSI_SAME_STMT
);
7880 gimple
*g
= gimple_build_call_vec (builtin_decl_explicit (sink_ix
), args
);
7881 gimple_set_location (g
, loc
);
7882 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
7884 *gsi
= gsi_last_bb (e1
->src
);
7885 cond
= unshare_expr (cond
);
7886 cond
= force_gimple_operand_gsi (gsi
, cond
, true, NULL_TREE
, false,
7887 GSI_CONTINUE_LINKING
);
7888 gsi_insert_after (gsi
, gimple_build_cond_empty (cond
), GSI_NEW_STMT
);
7889 edge e3
= make_edge (e1
->src
, e2
->dest
, EDGE_FALSE_VALUE
);
7890 e3
->probability
= REG_BR_PROB_BASE
/ 8;
7891 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
7892 e1
->flags
= EDGE_TRUE_VALUE
;
7893 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e1
->src
);
7895 *gsi
= gsi_after_labels (e2
->dest
);
7898 /* Expand all #pragma omp ordered depend(source) and
7899 #pragma omp ordered depend(sink:...) constructs in the current
7900 #pragma omp for ordered(n) region. */
7903 expand_omp_ordered_source_sink (struct omp_region
*region
,
7904 struct omp_for_data
*fd
, tree
*counts
,
7905 basic_block cont_bb
)
7907 struct omp_region
*inner
;
7909 for (i
= fd
->collapse
- 1; i
< fd
->ordered
; i
++)
7910 if (i
== fd
->collapse
- 1 && fd
->collapse
> 1)
7911 counts
[i
] = NULL_TREE
;
7912 else if (i
>= fd
->collapse
&& !cont_bb
)
7913 counts
[i
] = build_zero_cst (fd
->iter_type
);
7914 else if (!POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
))
7915 && integer_onep (fd
->loops
[i
].step
))
7916 counts
[i
] = NULL_TREE
;
7918 counts
[i
] = create_tmp_var (fd
->iter_type
, ".orditer");
7920 = build_array_type_nelts (fd
->iter_type
, fd
->ordered
- fd
->collapse
+ 1);
7921 counts
[fd
->ordered
] = create_tmp_var (atype
, ".orditera");
7922 TREE_ADDRESSABLE (counts
[fd
->ordered
]) = 1;
7924 for (inner
= region
->inner
; inner
; inner
= inner
->next
)
7925 if (inner
->type
== GIMPLE_OMP_ORDERED
)
7927 gomp_ordered
*ord_stmt
= inner
->ord_stmt
;
7928 gimple_stmt_iterator gsi
= gsi_for_stmt (ord_stmt
);
7929 location_t loc
= gimple_location (ord_stmt
);
7931 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
7932 c
; c
= OMP_CLAUSE_CHAIN (c
))
7933 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SOURCE
)
7936 expand_omp_ordered_source (&gsi
, fd
, counts
, loc
);
7937 for (c
= gimple_omp_ordered_clauses (ord_stmt
);
7938 c
; c
= OMP_CLAUSE_CHAIN (c
))
7939 if (OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
7940 expand_omp_ordered_sink (&gsi
, fd
, counts
, c
, loc
);
7941 gsi_remove (&gsi
, true);
7945 /* Wrap the body into fd->ordered - fd->collapse loops that aren't
7949 expand_omp_for_ordered_loops (struct omp_for_data
*fd
, tree
*counts
,
7950 basic_block cont_bb
, basic_block body_bb
,
7951 bool ordered_lastprivate
)
7953 if (fd
->ordered
== fd
->collapse
)
7958 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
7959 for (int i
= fd
->collapse
; i
< fd
->ordered
; i
++)
7961 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
7962 tree n1
= fold_convert (type
, fd
->loops
[i
].n1
);
7963 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, n1
);
7964 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
7965 size_int (i
- fd
->collapse
+ 1),
7966 NULL_TREE
, NULL_TREE
);
7967 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
7972 for (int i
= fd
->ordered
- 1; i
>= fd
->collapse
; i
--)
7974 tree t
, type
= TREE_TYPE (fd
->loops
[i
].v
);
7975 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
7976 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
7977 fold_convert (type
, fd
->loops
[i
].n1
));
7979 expand_omp_build_assign (&gsi
, counts
[i
],
7980 build_zero_cst (fd
->iter_type
));
7981 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
7982 size_int (i
- fd
->collapse
+ 1),
7983 NULL_TREE
, NULL_TREE
);
7984 expand_omp_build_assign (&gsi
, aref
, build_zero_cst (fd
->iter_type
));
7985 if (!gsi_end_p (gsi
))
7988 gsi
= gsi_last_bb (body_bb
);
7989 edge e1
= split_block (body_bb
, gsi_stmt (gsi
));
7990 basic_block new_body
= e1
->dest
;
7991 if (body_bb
== cont_bb
)
7994 basic_block new_header
;
7995 if (EDGE_COUNT (cont_bb
->preds
) > 0)
7997 gsi
= gsi_last_bb (cont_bb
);
7998 if (POINTER_TYPE_P (type
))
7999 t
= fold_build_pointer_plus (fd
->loops
[i
].v
,
8000 fold_convert (sizetype
,
8001 fd
->loops
[i
].step
));
8003 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loops
[i
].v
,
8004 fold_convert (type
, fd
->loops
[i
].step
));
8005 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
8008 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[i
],
8009 build_int_cst (fd
->iter_type
, 1));
8010 expand_omp_build_assign (&gsi
, counts
[i
], t
);
8015 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
8016 fd
->loops
[i
].v
, fd
->loops
[i
].n1
);
8017 t
= fold_convert (fd
->iter_type
, t
);
8018 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8019 true, GSI_SAME_STMT
);
8021 aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8022 size_int (i
- fd
->collapse
+ 1),
8023 NULL_TREE
, NULL_TREE
);
8024 expand_omp_build_assign (&gsi
, aref
, t
);
8026 e2
= split_block (cont_bb
, gsi_stmt (gsi
));
8027 new_header
= e2
->dest
;
8030 new_header
= cont_bb
;
8031 gsi
= gsi_after_labels (new_header
);
8032 tree v
= force_gimple_operand_gsi (&gsi
, fd
->loops
[i
].v
, true, NULL_TREE
,
8033 true, GSI_SAME_STMT
);
8035 = force_gimple_operand_gsi (&gsi
, fold_convert (type
, fd
->loops
[i
].n2
),
8036 true, NULL_TREE
, true, GSI_SAME_STMT
);
8037 t
= build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, n2
);
8038 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_NEW_STMT
);
8039 edge e3
= split_block (new_header
, gsi_stmt (gsi
));
8042 make_edge (body_bb
, new_header
, EDGE_FALLTHRU
);
8043 e3
->flags
= EDGE_FALSE_VALUE
;
8044 e3
->probability
= REG_BR_PROB_BASE
/ 8;
8045 e1
= make_edge (new_header
, new_body
, EDGE_TRUE_VALUE
);
8046 e1
->probability
= REG_BR_PROB_BASE
- e3
->probability
;
8048 set_immediate_dominator (CDI_DOMINATORS
, new_header
, body_bb
);
8049 set_immediate_dominator (CDI_DOMINATORS
, new_body
, new_header
);
8053 struct loop
*loop
= alloc_loop ();
8054 loop
->header
= new_header
;
8055 loop
->latch
= e2
->src
;
8056 add_loop (loop
, body_bb
->loop_father
);
8060 /* If there are any lastprivate clauses and it is possible some loops
8061 might have zero iterations, ensure all the decls are initialized,
8062 otherwise we could crash evaluating C++ class iterators with lastprivate
8064 bool need_inits
= false;
8065 for (int i
= fd
->collapse
; ordered_lastprivate
&& i
< fd
->ordered
; i
++)
8068 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8069 gimple_stmt_iterator gsi
= gsi_after_labels (body_bb
);
8070 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
,
8071 fold_convert (type
, fd
->loops
[i
].n1
));
8075 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8076 tree this_cond
= fold_build2 (fd
->loops
[i
].cond_code
,
8078 fold_convert (type
, fd
->loops
[i
].n1
),
8079 fold_convert (type
, fd
->loops
[i
].n2
));
8080 if (!integer_onep (this_cond
))
8088 /* A subroutine of expand_omp_for. Generate code for a parallel
8089 loop with any schedule. Given parameters:
8091 for (V = N1; V cond N2; V += STEP) BODY;
8093 where COND is "<" or ">", we generate pseudocode
8095 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
8096 if (more) goto L0; else goto L3;
8103 if (V cond iend) goto L1; else goto L2;
8105 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8108 If this is a combined omp parallel loop, instead of the call to
8109 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
8110 If this is gimple_omp_for_combined_p loop, then instead of assigning
8111 V and iend in L0 we assign the first two _looptemp_ clause decls of the
8112 inner GIMPLE_OMP_FOR and V += STEP; and
8113 if (V cond iend) goto L1; else goto L2; are removed.
8115 For collapsed loops, given parameters:
8117 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
8118 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
8119 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
8122 we generate pseudocode
8124 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
8129 count3 = (adj + N32 - N31) / STEP3;
8130 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
8135 count2 = (adj + N22 - N21) / STEP2;
8136 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
8141 count1 = (adj + N12 - N11) / STEP1;
8142 count = count1 * count2 * count3;
8147 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
8148 if (more) goto L0; else goto L3;
8152 V3 = N31 + (T % count3) * STEP3;
8154 V2 = N21 + (T % count2) * STEP2;
8156 V1 = N11 + T * STEP1;
8161 if (V < iend) goto L10; else goto L2;
8164 if (V3 cond3 N32) goto L1; else goto L11;
8168 if (V2 cond2 N22) goto L1; else goto L12;
8174 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
8180 expand_omp_for_generic (struct omp_region
*region
,
8181 struct omp_for_data
*fd
,
8182 enum built_in_function start_fn
,
8183 enum built_in_function next_fn
,
8186 tree type
, istart0
, iend0
, iend
;
8187 tree t
, vmain
, vback
, bias
= NULL_TREE
;
8188 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
8189 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
8190 gimple_stmt_iterator gsi
;
8191 gassign
*assign_stmt
;
8192 bool in_combined_parallel
= is_combined_parallel (region
);
8193 bool broken_loop
= region
->cont
== NULL
;
8195 tree
*counts
= NULL
;
8197 bool ordered_lastprivate
= false;
8199 gcc_assert (!broken_loop
|| !in_combined_parallel
);
8200 gcc_assert (fd
->iter_type
== long_integer_type_node
8201 || !in_combined_parallel
);
8203 entry_bb
= region
->entry
;
8204 cont_bb
= region
->cont
;
8206 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
8207 gcc_assert (broken_loop
8208 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
8209 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
8210 l1_bb
= single_succ (l0_bb
);
8213 l2_bb
= create_empty_bb (cont_bb
);
8214 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
8215 || (single_succ_edge (BRANCH_EDGE (cont_bb
)->dest
)->dest
8217 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
8221 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
8222 exit_bb
= region
->exit
;
8224 gsi
= gsi_last_bb (entry_bb
);
8226 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8228 && find_omp_clause (gimple_omp_for_clauses (gsi_stmt (gsi
)),
8229 OMP_CLAUSE_LASTPRIVATE
))
8230 ordered_lastprivate
= false;
8231 if (fd
->collapse
> 1 || fd
->ordered
)
8233 int first_zero_iter1
= -1, first_zero_iter2
= -1;
8234 basic_block zero_iter1_bb
= NULL
, zero_iter2_bb
= NULL
, l2_dom_bb
= NULL
;
8236 counts
= XALLOCAVEC (tree
, fd
->ordered
? fd
->ordered
+ 1 : fd
->collapse
);
8237 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
8238 zero_iter1_bb
, first_zero_iter1
,
8239 zero_iter2_bb
, first_zero_iter2
, l2_dom_bb
);
8243 /* Some counts[i] vars might be uninitialized if
8244 some loop has zero iterations. But the body shouldn't
8245 be executed in that case, so just avoid uninit warnings. */
8246 for (i
= first_zero_iter1
;
8247 i
< (fd
->ordered
? fd
->ordered
: fd
->collapse
); i
++)
8248 if (SSA_VAR_P (counts
[i
]))
8249 TREE_NO_WARNING (counts
[i
]) = 1;
8251 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8253 make_edge (zero_iter1_bb
, entry_bb
, EDGE_FALLTHRU
);
8254 gsi
= gsi_last_bb (entry_bb
);
8255 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8256 get_immediate_dominator (CDI_DOMINATORS
,
8261 /* Some counts[i] vars might be uninitialized if
8262 some loop has zero iterations. But the body shouldn't
8263 be executed in that case, so just avoid uninit warnings. */
8264 for (i
= first_zero_iter2
; i
< fd
->ordered
; i
++)
8265 if (SSA_VAR_P (counts
[i
]))
8266 TREE_NO_WARNING (counts
[i
]) = 1;
8268 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8272 e
= split_block (entry_bb
, gsi_stmt (gsi
));
8274 make_edge (zero_iter2_bb
, entry_bb
, EDGE_FALLTHRU
);
8275 gsi
= gsi_last_bb (entry_bb
);
8276 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
8277 get_immediate_dominator
8278 (CDI_DOMINATORS
, zero_iter2_bb
));
8281 if (fd
->collapse
== 1)
8283 counts
[0] = fd
->loop
.n2
;
8284 fd
->loop
= fd
->loops
[0];
8288 type
= TREE_TYPE (fd
->loop
.v
);
8289 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
8290 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
8291 TREE_ADDRESSABLE (istart0
) = 1;
8292 TREE_ADDRESSABLE (iend0
) = 1;
8294 /* See if we need to bias by LLONG_MIN. */
8295 if (fd
->iter_type
== long_long_unsigned_type_node
8296 && TREE_CODE (type
) == INTEGER_TYPE
8297 && !TYPE_UNSIGNED (type
)
8298 && fd
->ordered
== 0)
8302 if (fd
->loop
.cond_code
== LT_EXPR
)
8305 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8309 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
8312 if (TREE_CODE (n1
) != INTEGER_CST
8313 || TREE_CODE (n2
) != INTEGER_CST
8314 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
8315 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
8318 gimple_stmt_iterator gsif
= gsi
;
8321 tree arr
= NULL_TREE
;
8322 if (in_combined_parallel
)
8324 gcc_assert (fd
->ordered
== 0);
8325 /* In a combined parallel loop, emit a call to
8326 GOMP_loop_foo_next. */
8327 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8328 build_fold_addr_expr (istart0
),
8329 build_fold_addr_expr (iend0
));
8333 tree t0
, t1
, t2
, t3
, t4
;
8334 /* If this is not a combined parallel loop, emit a call to
8335 GOMP_loop_foo_start in ENTRY_BB. */
8336 t4
= build_fold_addr_expr (iend0
);
8337 t3
= build_fold_addr_expr (istart0
);
8340 t0
= build_int_cst (unsigned_type_node
,
8341 fd
->ordered
- fd
->collapse
+ 1);
8342 arr
= create_tmp_var (build_array_type_nelts (fd
->iter_type
,
8344 - fd
->collapse
+ 1),
8346 DECL_NAMELESS (arr
) = 1;
8347 TREE_ADDRESSABLE (arr
) = 1;
8348 TREE_STATIC (arr
) = 1;
8349 vec
<constructor_elt
, va_gc
> *v
;
8350 vec_alloc (v
, fd
->ordered
- fd
->collapse
+ 1);
8353 for (idx
= 0; idx
< fd
->ordered
- fd
->collapse
+ 1; idx
++)
8356 if (idx
== 0 && fd
->collapse
> 1)
8359 c
= counts
[idx
+ fd
->collapse
- 1];
8360 tree purpose
= size_int (idx
);
8361 CONSTRUCTOR_APPEND_ELT (v
, purpose
, c
);
8362 if (TREE_CODE (c
) != INTEGER_CST
)
8363 TREE_STATIC (arr
) = 0;
8366 DECL_INITIAL (arr
) = build_constructor (TREE_TYPE (arr
), v
);
8367 if (!TREE_STATIC (arr
))
8368 force_gimple_operand_gsi (&gsi
, build1 (DECL_EXPR
,
8369 void_type_node
, arr
),
8370 true, NULL_TREE
, true, GSI_SAME_STMT
);
8371 t1
= build_fold_addr_expr (arr
);
8376 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
8379 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8382 = find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
8383 OMP_CLAUSE__LOOPTEMP_
);
8384 gcc_assert (innerc
);
8385 t0
= OMP_CLAUSE_DECL (innerc
);
8386 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8387 OMP_CLAUSE__LOOPTEMP_
);
8388 gcc_assert (innerc
);
8389 t1
= OMP_CLAUSE_DECL (innerc
);
8391 if (POINTER_TYPE_P (TREE_TYPE (t0
))
8392 && TYPE_PRECISION (TREE_TYPE (t0
))
8393 != TYPE_PRECISION (fd
->iter_type
))
8395 /* Avoid casting pointers to integer of a different size. */
8396 tree itype
= signed_type_for (type
);
8397 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
8398 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
8402 t1
= fold_convert (fd
->iter_type
, t1
);
8403 t0
= fold_convert (fd
->iter_type
, t0
);
8407 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
8408 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
8411 if (fd
->iter_type
== long_integer_type_node
|| fd
->ordered
)
8415 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8416 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8418 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8419 5, t0
, t1
, t
, t3
, t4
);
8421 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8422 6, t0
, t1
, t2
, t
, t3
, t4
);
8424 else if (fd
->ordered
)
8425 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8428 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8429 5, t0
, t1
, t2
, t3
, t4
);
8437 /* The GOMP_loop_ull_*start functions have additional boolean
8438 argument, true for < loops and false for > loops.
8439 In Fortran, the C bool type can be different from
8440 boolean_type_node. */
8441 bfn_decl
= builtin_decl_explicit (start_fn
);
8442 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
8443 t5
= build_int_cst (c_bool_type
,
8444 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
8447 tree bfn_decl
= builtin_decl_explicit (start_fn
);
8448 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
8449 t
= omp_adjust_chunk_size (t
, fd
->simd_schedule
);
8450 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
8453 t
= build_call_expr (builtin_decl_explicit (start_fn
),
8454 6, t5
, t0
, t1
, t2
, t3
, t4
);
8457 if (TREE_TYPE (t
) != boolean_type_node
)
8458 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8459 t
, build_int_cst (TREE_TYPE (t
), 0));
8460 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8461 true, GSI_SAME_STMT
);
8462 if (arr
&& !TREE_STATIC (arr
))
8464 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8465 TREE_THIS_VOLATILE (clobber
) = 1;
8466 gsi_insert_before (&gsi
, gimple_build_assign (arr
, clobber
),
8469 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
8471 /* Remove the GIMPLE_OMP_FOR statement. */
8472 gsi_remove (&gsi
, true);
8474 if (gsi_end_p (gsif
))
8475 gsif
= gsi_after_labels (gsi_bb (gsif
));
8478 /* Iteration setup for sequential loop goes in L0_BB. */
8479 tree startvar
= fd
->loop
.v
;
8480 tree endvar
= NULL_TREE
;
8482 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8484 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
8485 && gimple_omp_for_kind (inner_stmt
)
8486 == GF_OMP_FOR_KIND_SIMD
);
8487 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
8488 OMP_CLAUSE__LOOPTEMP_
);
8489 gcc_assert (innerc
);
8490 startvar
= OMP_CLAUSE_DECL (innerc
);
8491 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
8492 OMP_CLAUSE__LOOPTEMP_
);
8493 gcc_assert (innerc
);
8494 endvar
= OMP_CLAUSE_DECL (innerc
);
8497 gsi
= gsi_start_bb (l0_bb
);
8499 if (fd
->ordered
&& fd
->collapse
== 1)
8500 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8501 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8503 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8504 if (fd
->ordered
&& fd
->collapse
== 1)
8506 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8507 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8508 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8511 t
= fold_convert (TREE_TYPE (startvar
), t
);
8512 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8518 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8519 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8520 t
= fold_convert (TREE_TYPE (startvar
), t
);
8522 t
= force_gimple_operand_gsi (&gsi
, t
,
8524 && TREE_ADDRESSABLE (startvar
),
8525 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
8526 assign_stmt
= gimple_build_assign (startvar
, t
);
8527 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8530 if (fd
->ordered
&& fd
->collapse
== 1)
8531 t
= fold_build2 (MULT_EXPR
, fd
->iter_type
, t
,
8532 fold_convert (fd
->iter_type
, fd
->loop
.step
));
8534 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
8535 if (fd
->ordered
&& fd
->collapse
== 1)
8537 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8538 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (startvar
),
8539 fd
->loop
.n1
, fold_convert (sizetype
, t
));
8542 t
= fold_convert (TREE_TYPE (startvar
), t
);
8543 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (startvar
),
8549 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
8550 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
8551 t
= fold_convert (TREE_TYPE (startvar
), t
);
8553 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8554 false, GSI_CONTINUE_LINKING
);
8557 assign_stmt
= gimple_build_assign (endvar
, iend
);
8558 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8559 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
8560 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
8562 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
8563 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8565 /* Handle linear clause adjustments. */
8566 tree itercnt
= NULL_TREE
;
8567 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
8568 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
8569 c
; c
= OMP_CLAUSE_CHAIN (c
))
8570 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
8571 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
8573 tree d
= OMP_CLAUSE_DECL (c
);
8574 bool is_ref
= is_reference (d
);
8575 tree t
= d
, a
, dest
;
8577 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
8578 tree type
= TREE_TYPE (t
);
8579 if (POINTER_TYPE_P (type
))
8581 dest
= unshare_expr (t
);
8582 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
8583 expand_omp_build_assign (&gsif
, v
, t
);
8584 if (itercnt
== NULL_TREE
)
8587 tree n1
= fd
->loop
.n1
;
8588 if (POINTER_TYPE_P (TREE_TYPE (itercnt
)))
8591 = fold_convert (signed_type_for (TREE_TYPE (itercnt
)),
8593 n1
= fold_convert (TREE_TYPE (itercnt
), n1
);
8595 itercnt
= fold_build2 (MINUS_EXPR
, TREE_TYPE (itercnt
),
8597 itercnt
= fold_build2 (EXACT_DIV_EXPR
, TREE_TYPE (itercnt
),
8598 itercnt
, fd
->loop
.step
);
8599 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
8601 GSI_CONTINUE_LINKING
);
8603 a
= fold_build2 (MULT_EXPR
, type
,
8604 fold_convert (type
, itercnt
),
8605 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
8606 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
8607 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
8608 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8609 false, GSI_CONTINUE_LINKING
);
8610 assign_stmt
= gimple_build_assign (dest
, t
);
8611 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
8613 if (fd
->collapse
> 1)
8614 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
8618 /* Until now, counts array contained number of iterations or
8619 variable containing it for ith loop. From now on, we need
8620 those counts only for collapsed loops, and only for the 2nd
8621 till the last collapsed one. Move those one element earlier,
8622 we'll use counts[fd->collapse - 1] for the first source/sink
8623 iteration counter and so on and counts[fd->ordered]
8624 as the array holding the current counter values for
8626 if (fd
->collapse
> 1)
8627 memmove (counts
, counts
+ 1, (fd
->collapse
- 1) * sizeof (counts
[0]));
8631 for (i
= fd
->collapse
; i
< fd
->ordered
; i
++)
8633 tree type
= TREE_TYPE (fd
->loops
[i
].v
);
8635 = fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
8636 fold_convert (type
, fd
->loops
[i
].n1
),
8637 fold_convert (type
, fd
->loops
[i
].n2
));
8638 if (!integer_onep (this_cond
))
8641 if (i
< fd
->ordered
)
8644 = create_empty_bb (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8645 add_bb_to_loop (cont_bb
, l1_bb
->loop_father
);
8646 gimple_stmt_iterator gsi
= gsi_after_labels (cont_bb
);
8647 gimple
*g
= gimple_build_omp_continue (fd
->loop
.v
, fd
->loop
.v
);
8648 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8649 make_edge (cont_bb
, l3_bb
, EDGE_FALLTHRU
);
8650 make_edge (cont_bb
, l1_bb
, 0);
8651 l2_bb
= create_empty_bb (cont_bb
);
8652 broken_loop
= false;
8655 expand_omp_ordered_source_sink (region
, fd
, counts
, cont_bb
);
8656 cont_bb
= expand_omp_for_ordered_loops (fd
, counts
, cont_bb
, l1_bb
,
8657 ordered_lastprivate
);
8658 if (counts
[fd
->collapse
- 1])
8660 gcc_assert (fd
->collapse
== 1);
8661 gsi
= gsi_last_bb (l0_bb
);
8662 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1],
8664 gsi
= gsi_last_bb (cont_bb
);
8665 t
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, counts
[fd
->collapse
- 1],
8666 build_int_cst (fd
->iter_type
, 1));
8667 expand_omp_build_assign (&gsi
, counts
[fd
->collapse
- 1], t
);
8668 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8669 size_zero_node
, NULL_TREE
, NULL_TREE
);
8670 expand_omp_build_assign (&gsi
, aref
, counts
[fd
->collapse
- 1]);
8671 t
= counts
[fd
->collapse
- 1];
8673 else if (fd
->collapse
> 1)
8677 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8678 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8679 t
= fold_convert (fd
->iter_type
, t
);
8681 gsi
= gsi_last_bb (l0_bb
);
8682 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
, counts
[fd
->ordered
],
8683 size_zero_node
, NULL_TREE
, NULL_TREE
);
8684 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8685 false, GSI_CONTINUE_LINKING
);
8686 expand_omp_build_assign (&gsi
, aref
, t
, true);
8691 /* Code to control the increment and predicate for the sequential
8692 loop goes in the CONT_BB. */
8693 gsi
= gsi_last_bb (cont_bb
);
8694 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
8695 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
8696 vmain
= gimple_omp_continue_control_use (cont_stmt
);
8697 vback
= gimple_omp_continue_control_def (cont_stmt
);
8699 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
8701 if (POINTER_TYPE_P (type
))
8702 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
8704 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
8705 t
= force_gimple_operand_gsi (&gsi
, t
,
8707 && TREE_ADDRESSABLE (vback
),
8708 NULL_TREE
, true, GSI_SAME_STMT
);
8709 assign_stmt
= gimple_build_assign (vback
, t
);
8710 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
8712 if (fd
->ordered
&& counts
[fd
->collapse
- 1] == NULL_TREE
)
8714 if (fd
->collapse
> 1)
8718 t
= fold_build2 (MINUS_EXPR
, TREE_TYPE (fd
->loops
[0].v
),
8719 fd
->loops
[0].v
, fd
->loops
[0].n1
);
8720 t
= fold_convert (fd
->iter_type
, t
);
8722 tree aref
= build4 (ARRAY_REF
, fd
->iter_type
,
8723 counts
[fd
->ordered
], size_zero_node
,
8724 NULL_TREE
, NULL_TREE
);
8725 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8726 true, GSI_SAME_STMT
);
8727 expand_omp_build_assign (&gsi
, aref
, t
);
8730 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
8731 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
8733 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8734 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
8737 /* Remove GIMPLE_OMP_CONTINUE. */
8738 gsi_remove (&gsi
, true);
8740 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
8741 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
8743 /* Emit code to get the next parallel iteration in L2_BB. */
8744 gsi
= gsi_start_bb (l2_bb
);
8746 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
8747 build_fold_addr_expr (istart0
),
8748 build_fold_addr_expr (iend0
));
8749 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
8750 false, GSI_CONTINUE_LINKING
);
8751 if (TREE_TYPE (t
) != boolean_type_node
)
8752 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
8753 t
, build_int_cst (TREE_TYPE (t
), 0));
8754 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
8755 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
8758 /* Add the loop cleanup function. */
8759 gsi
= gsi_last_bb (exit_bb
);
8760 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
8761 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
8762 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
8763 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
8765 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
8766 gcall
*call_stmt
= gimple_build_call (t
, 0);
8767 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
8768 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
8769 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
8772 tree arr
= counts
[fd
->ordered
];
8773 tree clobber
= build_constructor (TREE_TYPE (arr
), NULL
);
8774 TREE_THIS_VOLATILE (clobber
) = 1;
8775 gsi_insert_after (&gsi
, gimple_build_assign (arr
, clobber
),
8778 gsi_remove (&gsi
, true);
8780 /* Connect the new blocks. */
8781 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
8782 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
8788 e
= find_edge (cont_bb
, l3_bb
);
8789 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
8791 phis
= phi_nodes (l3_bb
);
8792 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
8794 gimple
*phi
= gsi_stmt (gsi
);
8795 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
8796 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
8800 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
8801 e
= find_edge (cont_bb
, l1_bb
);
8804 e
= BRANCH_EDGE (cont_bb
);
8805 gcc_assert (single_succ (e
->dest
) == l1_bb
);
8807 if (gimple_omp_for_combined_p (fd
->for_stmt
))
8812 else if (fd
->collapse
> 1)
8815 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
8818 e
->flags
= EDGE_TRUE_VALUE
;
8821 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
8822 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
8826 e
= find_edge (cont_bb
, l2_bb
);
8827 e
->flags
= EDGE_FALLTHRU
;
8829 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
8831 if (gimple_in_ssa_p (cfun
))
8833 /* Add phis to the outer loop that connect to the phis in the inner,
8834 original loop, and move the loop entry value of the inner phi to
8835 the loop entry value of the outer phi. */
8837 for (psi
= gsi_start_phis (l3_bb
); !gsi_end_p (psi
); gsi_next (&psi
))
8839 source_location locus
;
8841 gphi
*exit_phi
= psi
.phi ();
8843 edge l2_to_l3
= find_edge (l2_bb
, l3_bb
);
8844 tree exit_res
= PHI_ARG_DEF_FROM_EDGE (exit_phi
, l2_to_l3
);
8846 basic_block latch
= BRANCH_EDGE (cont_bb
)->dest
;
8847 edge latch_to_l1
= find_edge (latch
, l1_bb
);
8849 = find_phi_with_arg_on_edge (exit_res
, latch_to_l1
);
8851 tree t
= gimple_phi_result (exit_phi
);
8852 tree new_res
= copy_ssa_name (t
, NULL
);
8853 nphi
= create_phi_node (new_res
, l0_bb
);
8855 edge l0_to_l1
= find_edge (l0_bb
, l1_bb
);
8856 t
= PHI_ARG_DEF_FROM_EDGE (inner_phi
, l0_to_l1
);
8857 locus
= gimple_phi_arg_location_from_edge (inner_phi
, l0_to_l1
);
8858 edge entry_to_l0
= find_edge (entry_bb
, l0_bb
);
8859 add_phi_arg (nphi
, t
, entry_to_l0
, locus
);
8861 edge l2_to_l0
= find_edge (l2_bb
, l0_bb
);
8862 add_phi_arg (nphi
, exit_res
, l2_to_l0
, UNKNOWN_LOCATION
);
8864 add_phi_arg (inner_phi
, new_res
, l0_to_l1
, UNKNOWN_LOCATION
);
8868 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
8869 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
8870 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
8871 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
8872 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
8873 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
8874 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
8875 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
8877 /* We enter expand_omp_for_generic with a loop. This original loop may
8878 have its own loop struct, or it may be part of an outer loop struct
8879 (which may be the fake loop). */
8880 struct loop
*outer_loop
= entry_bb
->loop_father
;
8881 bool orig_loop_has_loop_struct
= l1_bb
->loop_father
!= outer_loop
;
8883 add_bb_to_loop (l2_bb
, outer_loop
);
8885 /* We've added a new loop around the original loop. Allocate the
8886 corresponding loop struct. */
8887 struct loop
*new_loop
= alloc_loop ();
8888 new_loop
->header
= l0_bb
;
8889 new_loop
->latch
= l2_bb
;
8890 add_loop (new_loop
, outer_loop
);
8892 /* Allocate a loop structure for the original loop unless we already
8894 if (!orig_loop_has_loop_struct
8895 && !gimple_omp_for_combined_p (fd
->for_stmt
))
8897 struct loop
*orig_loop
= alloc_loop ();
8898 orig_loop
->header
= l1_bb
;
8899 /* The loop may have multiple latches. */
8900 add_loop (orig_loop
, new_loop
);
8906 /* A subroutine of expand_omp_for. Generate code for a parallel
8907 loop with static schedule and no specified chunk size. Given
8910 for (V = N1; V cond N2; V += STEP) BODY;
8912 where COND is "<" or ">", we generate pseudocode
8914 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
8919 if ((__typeof (V)) -1 > 0 && cond is >)
8920 n = -(adj + N2 - N1) / -STEP;
8922 n = (adj + N2 - N1) / STEP;
8925 if (threadid < tt) goto L3; else goto L4;
8930 s0 = q * threadid + tt;
8933 if (s0 >= e0) goto L2; else goto L0;
8939 if (V cond e) goto L1;
8944 expand_omp_for_static_nochunk (struct omp_region
*region
,
8945 struct omp_for_data
*fd
,
8948 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
8949 tree type
, itype
, vmain
, vback
;
8950 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
8951 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
8953 gimple_stmt_iterator gsi
;
8955 bool broken_loop
= region
->cont
== NULL
;
8956 tree
*counts
= NULL
;
8959 itype
= type
= TREE_TYPE (fd
->loop
.v
);
8960 if (POINTER_TYPE_P (type
))
8961 itype
= signed_type_for (type
);
8963 entry_bb
= region
->entry
;
8964 cont_bb
= region
->cont
;
8965 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
8966 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
8967 gcc_assert (broken_loop
8968 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
8969 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
8970 body_bb
= single_succ (seq_start_bb
);
8973 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
8974 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
8975 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
8977 exit_bb
= region
->exit
;
8979 /* Iteration space partitioning goes in ENTRY_BB. */
8980 gsi
= gsi_last_bb (entry_bb
);
8981 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
8983 if (fd
->collapse
> 1)
8985 int first_zero_iter
= -1, dummy
= -1;
8986 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
8988 counts
= XALLOCAVEC (tree
, fd
->collapse
);
8989 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
8990 fin_bb
, first_zero_iter
,
8991 dummy_bb
, dummy
, l2_dom_bb
);
8994 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
8995 t
= integer_one_node
;
8997 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
8998 fold_convert (type
, fd
->loop
.n1
),
8999 fold_convert (type
, fd
->loop
.n2
));
9000 if (fd
->collapse
== 1
9001 && TYPE_UNSIGNED (type
)
9002 && (t
== NULL_TREE
|| !integer_onep (t
)))
9004 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9005 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9006 true, GSI_SAME_STMT
);
9007 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9008 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9009 true, GSI_SAME_STMT
);
9010 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9011 NULL_TREE
, NULL_TREE
);
9012 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9013 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9014 expand_omp_regimplify_p
, NULL
, NULL
)
9015 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9016 expand_omp_regimplify_p
, NULL
, NULL
))
9018 gsi
= gsi_for_stmt (cond_stmt
);
9019 gimple_regimplify_operands (cond_stmt
, &gsi
);
9021 ep
= split_block (entry_bb
, cond_stmt
);
9022 ep
->flags
= EDGE_TRUE_VALUE
;
9023 entry_bb
= ep
->dest
;
9024 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9025 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9026 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9027 if (gimple_in_ssa_p (cfun
))
9029 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
9030 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9031 !gsi_end_p (gpi
); gsi_next (&gpi
))
9033 gphi
*phi
= gpi
.phi ();
9034 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9035 ep
, UNKNOWN_LOCATION
);
9038 gsi
= gsi_last_bb (entry_bb
);
9041 switch (gimple_omp_for_kind (fd
->for_stmt
))
9043 case GF_OMP_FOR_KIND_FOR
:
9044 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9045 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9047 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9048 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9049 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9054 nthreads
= build_call_expr (nthreads
, 0);
9055 nthreads
= fold_convert (itype
, nthreads
);
9056 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9057 true, GSI_SAME_STMT
);
9058 threadid
= build_call_expr (threadid
, 0);
9059 threadid
= fold_convert (itype
, threadid
);
9060 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9061 true, GSI_SAME_STMT
);
9065 step
= fd
->loop
.step
;
9066 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9068 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9069 OMP_CLAUSE__LOOPTEMP_
);
9070 gcc_assert (innerc
);
9071 n1
= OMP_CLAUSE_DECL (innerc
);
9072 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9073 OMP_CLAUSE__LOOPTEMP_
);
9074 gcc_assert (innerc
);
9075 n2
= OMP_CLAUSE_DECL (innerc
);
9077 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9078 true, NULL_TREE
, true, GSI_SAME_STMT
);
9079 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9080 true, NULL_TREE
, true, GSI_SAME_STMT
);
9081 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9082 true, NULL_TREE
, true, GSI_SAME_STMT
);
9084 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9085 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9086 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9087 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9088 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9089 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9090 fold_build1 (NEGATE_EXPR
, itype
, t
),
9091 fold_build1 (NEGATE_EXPR
, itype
, step
));
9093 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9094 t
= fold_convert (itype
, t
);
9095 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9097 q
= create_tmp_reg (itype
, "q");
9098 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
9099 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9100 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
9102 tt
= create_tmp_reg (itype
, "tt");
9103 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
9104 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
9105 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
9107 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
9108 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
9109 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9111 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
9112 gsi
= gsi_last_bb (second_bb
);
9113 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9115 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
9117 gassign
*assign_stmt
9118 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
9119 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9121 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
9122 gsi
= gsi_last_bb (third_bb
);
9123 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9125 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
9126 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
9127 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9129 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
9130 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
9132 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
9133 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9135 /* Remove the GIMPLE_OMP_FOR statement. */
9136 gsi_remove (&gsi
, true);
9138 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9139 gsi
= gsi_start_bb (seq_start_bb
);
9141 tree startvar
= fd
->loop
.v
;
9142 tree endvar
= NULL_TREE
;
9144 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9146 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9147 ? gimple_omp_parallel_clauses (inner_stmt
)
9148 : gimple_omp_for_clauses (inner_stmt
);
9149 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9150 gcc_assert (innerc
);
9151 startvar
= OMP_CLAUSE_DECL (innerc
);
9152 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9153 OMP_CLAUSE__LOOPTEMP_
);
9154 gcc_assert (innerc
);
9155 endvar
= OMP_CLAUSE_DECL (innerc
);
9156 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9157 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9160 for (i
= 1; i
< fd
->collapse
; i
++)
9162 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9163 OMP_CLAUSE__LOOPTEMP_
);
9164 gcc_assert (innerc
);
9166 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9167 OMP_CLAUSE__LOOPTEMP_
);
9170 /* If needed (distribute parallel for with lastprivate),
9171 propagate down the total number of iterations. */
9172 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9174 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9175 GSI_CONTINUE_LINKING
);
9176 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9177 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9181 t
= fold_convert (itype
, s0
);
9182 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9183 if (POINTER_TYPE_P (type
))
9184 t
= fold_build_pointer_plus (n1
, t
);
9186 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9187 t
= fold_convert (TREE_TYPE (startvar
), t
);
9188 t
= force_gimple_operand_gsi (&gsi
, t
,
9190 && TREE_ADDRESSABLE (startvar
),
9191 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9192 assign_stmt
= gimple_build_assign (startvar
, t
);
9193 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9195 t
= fold_convert (itype
, e0
);
9196 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9197 if (POINTER_TYPE_P (type
))
9198 t
= fold_build_pointer_plus (n1
, t
);
9200 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9201 t
= fold_convert (TREE_TYPE (startvar
), t
);
9202 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9203 false, GSI_CONTINUE_LINKING
);
9206 assign_stmt
= gimple_build_assign (endvar
, e
);
9207 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9208 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9209 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9211 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9212 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9214 /* Handle linear clause adjustments. */
9215 tree itercnt
= NULL_TREE
;
9216 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9217 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9218 c
; c
= OMP_CLAUSE_CHAIN (c
))
9219 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9220 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9222 tree d
= OMP_CLAUSE_DECL (c
);
9223 bool is_ref
= is_reference (d
);
9224 tree t
= d
, a
, dest
;
9226 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9227 if (itercnt
== NULL_TREE
)
9229 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9231 itercnt
= fold_build2 (MINUS_EXPR
, itype
,
9232 fold_convert (itype
, n1
),
9233 fold_convert (itype
, fd
->loop
.n1
));
9234 itercnt
= fold_build2 (EXACT_DIV_EXPR
, itype
, itercnt
, step
);
9235 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercnt
, s0
);
9236 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9238 GSI_CONTINUE_LINKING
);
9243 tree type
= TREE_TYPE (t
);
9244 if (POINTER_TYPE_P (type
))
9246 a
= fold_build2 (MULT_EXPR
, type
,
9247 fold_convert (type
, itercnt
),
9248 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9249 dest
= unshare_expr (t
);
9250 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9251 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), t
, a
);
9252 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9253 false, GSI_CONTINUE_LINKING
);
9254 assign_stmt
= gimple_build_assign (dest
, t
);
9255 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9257 if (fd
->collapse
> 1)
9258 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9262 /* The code controlling the sequential loop replaces the
9263 GIMPLE_OMP_CONTINUE. */
9264 gsi
= gsi_last_bb (cont_bb
);
9265 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9266 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
9267 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9268 vback
= gimple_omp_continue_control_def (cont_stmt
);
9270 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9272 if (POINTER_TYPE_P (type
))
9273 t
= fold_build_pointer_plus (vmain
, step
);
9275 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9276 t
= force_gimple_operand_gsi (&gsi
, t
,
9278 && TREE_ADDRESSABLE (vback
),
9279 NULL_TREE
, true, GSI_SAME_STMT
);
9280 assign_stmt
= gimple_build_assign (vback
, t
);
9281 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9283 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9284 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9286 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9289 /* Remove the GIMPLE_OMP_CONTINUE statement. */
9290 gsi_remove (&gsi
, true);
9292 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9293 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9296 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
9297 gsi
= gsi_last_bb (exit_bb
);
9298 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9300 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
9301 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
9303 gsi_remove (&gsi
, true);
9305 /* Connect all the blocks. */
9306 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
9307 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
9308 ep
= find_edge (entry_bb
, second_bb
);
9309 ep
->flags
= EDGE_TRUE_VALUE
;
9310 ep
->probability
= REG_BR_PROB_BASE
/ 4;
9311 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
9312 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
9316 ep
= find_edge (cont_bb
, body_bb
);
9319 ep
= BRANCH_EDGE (cont_bb
);
9320 gcc_assert (single_succ (ep
->dest
) == body_bb
);
9322 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9327 else if (fd
->collapse
> 1)
9330 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9333 ep
->flags
= EDGE_TRUE_VALUE
;
9334 find_edge (cont_bb
, fin_bb
)->flags
9335 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
9338 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
9339 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
9340 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
9342 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
9343 recompute_dominator (CDI_DOMINATORS
, body_bb
));
9344 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
9345 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
9347 struct loop
*loop
= body_bb
->loop_father
;
9348 if (loop
!= entry_bb
->loop_father
)
9350 gcc_assert (loop
->header
== body_bb
);
9351 gcc_assert (broken_loop
9352 || loop
->latch
== region
->cont
9353 || single_pred (loop
->latch
) == region
->cont
);
9357 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
9359 loop
= alloc_loop ();
9360 loop
->header
= body_bb
;
9361 if (collapse_bb
== NULL
)
9362 loop
->latch
= cont_bb
;
9363 add_loop (loop
, body_bb
->loop_father
);
9367 /* Return phi in E->DEST with ARG on edge E. */
9370 find_phi_with_arg_on_edge (tree arg
, edge e
)
9372 basic_block bb
= e
->dest
;
9374 for (gphi_iterator gpi
= gsi_start_phis (bb
);
9378 gphi
*phi
= gpi
.phi ();
9379 if (PHI_ARG_DEF_FROM_EDGE (phi
, e
) == arg
)
9386 /* A subroutine of expand_omp_for. Generate code for a parallel
9387 loop with static schedule and a specified chunk size. Given
9390 for (V = N1; V cond N2; V += STEP) BODY;
9392 where COND is "<" or ">", we generate pseudocode
9394 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
9399 if ((__typeof (V)) -1 > 0 && cond is >)
9400 n = -(adj + N2 - N1) / -STEP;
9402 n = (adj + N2 - N1) / STEP;
9404 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
9405 here so that V is defined
9406 if the loop is not entered
9408 s0 = (trip * nthreads + threadid) * CHUNK;
9409 e0 = min(s0 + CHUNK, n);
9410 if (s0 < n) goto L1; else goto L4;
9417 if (V cond e) goto L2; else goto L3;
9425 expand_omp_for_static_chunk (struct omp_region
*region
,
9426 struct omp_for_data
*fd
, gimple
*inner_stmt
)
9428 tree n
, s0
, e0
, e
, t
;
9429 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
9430 tree type
, itype
, vmain
, vback
, vextra
;
9431 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
9432 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
9433 gimple_stmt_iterator gsi
;
9435 bool broken_loop
= region
->cont
== NULL
;
9436 tree
*counts
= NULL
;
9439 itype
= type
= TREE_TYPE (fd
->loop
.v
);
9440 if (POINTER_TYPE_P (type
))
9441 itype
= signed_type_for (type
);
9443 entry_bb
= region
->entry
;
9444 se
= split_block (entry_bb
, last_stmt (entry_bb
));
9446 iter_part_bb
= se
->dest
;
9447 cont_bb
= region
->cont
;
9448 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
9449 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
9450 gcc_assert (broken_loop
9451 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
9452 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
9453 body_bb
= single_succ (seq_start_bb
);
9456 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
9457 || single_succ (BRANCH_EDGE (cont_bb
)->dest
) == body_bb
);
9458 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
9459 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
9461 exit_bb
= region
->exit
;
9463 /* Trip and adjustment setup goes in ENTRY_BB. */
9464 gsi
= gsi_last_bb (entry_bb
);
9465 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
9467 if (fd
->collapse
> 1)
9469 int first_zero_iter
= -1, dummy
= -1;
9470 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
9472 counts
= XALLOCAVEC (tree
, fd
->collapse
);
9473 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
9474 fin_bb
, first_zero_iter
,
9475 dummy_bb
, dummy
, l2_dom_bb
);
9478 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9479 t
= integer_one_node
;
9481 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
9482 fold_convert (type
, fd
->loop
.n1
),
9483 fold_convert (type
, fd
->loop
.n2
));
9484 if (fd
->collapse
== 1
9485 && TYPE_UNSIGNED (type
)
9486 && (t
== NULL_TREE
|| !integer_onep (t
)))
9488 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
9489 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
9490 true, GSI_SAME_STMT
);
9491 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
9492 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
9493 true, GSI_SAME_STMT
);
9494 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
9495 NULL_TREE
, NULL_TREE
);
9496 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
9497 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
9498 expand_omp_regimplify_p
, NULL
, NULL
)
9499 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
9500 expand_omp_regimplify_p
, NULL
, NULL
))
9502 gsi
= gsi_for_stmt (cond_stmt
);
9503 gimple_regimplify_operands (cond_stmt
, &gsi
);
9505 se
= split_block (entry_bb
, cond_stmt
);
9506 se
->flags
= EDGE_TRUE_VALUE
;
9507 entry_bb
= se
->dest
;
9508 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
9509 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
9510 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
9511 if (gimple_in_ssa_p (cfun
))
9513 int dest_idx
= find_edge (iter_part_bb
, fin_bb
)->dest_idx
;
9514 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
9515 !gsi_end_p (gpi
); gsi_next (&gpi
))
9517 gphi
*phi
= gpi
.phi ();
9518 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
9519 se
, UNKNOWN_LOCATION
);
9522 gsi
= gsi_last_bb (entry_bb
);
9525 switch (gimple_omp_for_kind (fd
->for_stmt
))
9527 case GF_OMP_FOR_KIND_FOR
:
9528 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
9529 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9531 case GF_OMP_FOR_KIND_DISTRIBUTE
:
9532 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
9533 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
9538 nthreads
= build_call_expr (nthreads
, 0);
9539 nthreads
= fold_convert (itype
, nthreads
);
9540 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
9541 true, GSI_SAME_STMT
);
9542 threadid
= build_call_expr (threadid
, 0);
9543 threadid
= fold_convert (itype
, threadid
);
9544 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
9545 true, GSI_SAME_STMT
);
9549 step
= fd
->loop
.step
;
9550 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9552 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
9553 OMP_CLAUSE__LOOPTEMP_
);
9554 gcc_assert (innerc
);
9555 n1
= OMP_CLAUSE_DECL (innerc
);
9556 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9557 OMP_CLAUSE__LOOPTEMP_
);
9558 gcc_assert (innerc
);
9559 n2
= OMP_CLAUSE_DECL (innerc
);
9561 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
9562 true, NULL_TREE
, true, GSI_SAME_STMT
);
9563 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
9564 true, NULL_TREE
, true, GSI_SAME_STMT
);
9565 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
9566 true, NULL_TREE
, true, GSI_SAME_STMT
);
9567 tree chunk_size
= fold_convert (itype
, fd
->chunk_size
);
9568 chunk_size
= omp_adjust_chunk_size (chunk_size
, fd
->simd_schedule
);
9570 = force_gimple_operand_gsi (&gsi
, chunk_size
, true, NULL_TREE
, true,
9573 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
9574 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
9575 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
9576 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
9577 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
9578 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
9579 fold_build1 (NEGATE_EXPR
, itype
, t
),
9580 fold_build1 (NEGATE_EXPR
, itype
, step
));
9582 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
9583 t
= fold_convert (itype
, t
);
9584 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9585 true, GSI_SAME_STMT
);
9587 trip_var
= create_tmp_reg (itype
, ".trip");
9588 if (gimple_in_ssa_p (cfun
))
9590 trip_init
= make_ssa_name (trip_var
);
9591 trip_main
= make_ssa_name (trip_var
);
9592 trip_back
= make_ssa_name (trip_var
);
9596 trip_init
= trip_var
;
9597 trip_main
= trip_var
;
9598 trip_back
= trip_var
;
9601 gassign
*assign_stmt
9602 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
9603 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9605 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, chunk_size
);
9606 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9607 if (POINTER_TYPE_P (type
))
9608 t
= fold_build_pointer_plus (n1
, t
);
9610 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9611 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9612 true, GSI_SAME_STMT
);
9614 /* Remove the GIMPLE_OMP_FOR. */
9615 gsi_remove (&gsi
, true);
9617 gimple_stmt_iterator gsif
= gsi
;
9619 /* Iteration space partitioning goes in ITER_PART_BB. */
9620 gsi
= gsi_last_bb (iter_part_bb
);
9622 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
9623 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
9624 t
= fold_build2 (MULT_EXPR
, itype
, t
, chunk_size
);
9625 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9626 false, GSI_CONTINUE_LINKING
);
9628 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, chunk_size
);
9629 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
9630 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9631 false, GSI_CONTINUE_LINKING
);
9633 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
9634 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
9636 /* Setup code for sequential iteration goes in SEQ_START_BB. */
9637 gsi
= gsi_start_bb (seq_start_bb
);
9639 tree startvar
= fd
->loop
.v
;
9640 tree endvar
= NULL_TREE
;
9642 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9644 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
9645 ? gimple_omp_parallel_clauses (inner_stmt
)
9646 : gimple_omp_for_clauses (inner_stmt
);
9647 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
9648 gcc_assert (innerc
);
9649 startvar
= OMP_CLAUSE_DECL (innerc
);
9650 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9651 OMP_CLAUSE__LOOPTEMP_
);
9652 gcc_assert (innerc
);
9653 endvar
= OMP_CLAUSE_DECL (innerc
);
9654 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
9655 && gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
9658 for (i
= 1; i
< fd
->collapse
; i
++)
9660 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9661 OMP_CLAUSE__LOOPTEMP_
);
9662 gcc_assert (innerc
);
9664 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
9665 OMP_CLAUSE__LOOPTEMP_
);
9668 /* If needed (distribute parallel for with lastprivate),
9669 propagate down the total number of iterations. */
9670 tree t
= fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (innerc
)),
9672 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, false,
9673 GSI_CONTINUE_LINKING
);
9674 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
9675 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9680 t
= fold_convert (itype
, s0
);
9681 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9682 if (POINTER_TYPE_P (type
))
9683 t
= fold_build_pointer_plus (n1
, t
);
9685 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9686 t
= fold_convert (TREE_TYPE (startvar
), t
);
9687 t
= force_gimple_operand_gsi (&gsi
, t
,
9689 && TREE_ADDRESSABLE (startvar
),
9690 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
9691 assign_stmt
= gimple_build_assign (startvar
, t
);
9692 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9694 t
= fold_convert (itype
, e0
);
9695 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
9696 if (POINTER_TYPE_P (type
))
9697 t
= fold_build_pointer_plus (n1
, t
);
9699 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
9700 t
= fold_convert (TREE_TYPE (startvar
), t
);
9701 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9702 false, GSI_CONTINUE_LINKING
);
9705 assign_stmt
= gimple_build_assign (endvar
, e
);
9706 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9707 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
9708 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
9710 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
9711 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9713 /* Handle linear clause adjustments. */
9714 tree itercnt
= NULL_TREE
, itercntbias
= NULL_TREE
;
9715 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_FOR
)
9716 for (tree c
= gimple_omp_for_clauses (fd
->for_stmt
);
9717 c
; c
= OMP_CLAUSE_CHAIN (c
))
9718 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
9719 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
9721 tree d
= OMP_CLAUSE_DECL (c
);
9722 bool is_ref
= is_reference (d
);
9723 tree t
= d
, a
, dest
;
9725 t
= build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c
), t
);
9726 tree type
= TREE_TYPE (t
);
9727 if (POINTER_TYPE_P (type
))
9729 dest
= unshare_expr (t
);
9730 tree v
= create_tmp_var (TREE_TYPE (t
), NULL
);
9731 expand_omp_build_assign (&gsif
, v
, t
);
9732 if (itercnt
== NULL_TREE
)
9734 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
9737 = fold_build2 (MINUS_EXPR
, itype
, fold_convert (itype
, n1
),
9738 fold_convert (itype
, fd
->loop
.n1
));
9739 itercntbias
= fold_build2 (EXACT_DIV_EXPR
, itype
,
9742 = force_gimple_operand_gsi (&gsif
, itercntbias
, true,
9745 itercnt
= fold_build2 (PLUS_EXPR
, itype
, itercntbias
, s0
);
9746 itercnt
= force_gimple_operand_gsi (&gsi
, itercnt
, true,
9748 GSI_CONTINUE_LINKING
);
9753 a
= fold_build2 (MULT_EXPR
, type
,
9754 fold_convert (type
, itercnt
),
9755 fold_convert (type
, OMP_CLAUSE_LINEAR_STEP (c
)));
9756 t
= fold_build2 (type
== TREE_TYPE (t
) ? PLUS_EXPR
9757 : POINTER_PLUS_EXPR
, TREE_TYPE (t
), v
, a
);
9758 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9759 false, GSI_CONTINUE_LINKING
);
9760 assign_stmt
= gimple_build_assign (dest
, t
);
9761 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9763 if (fd
->collapse
> 1)
9764 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
9768 /* The code controlling the sequential loop goes in CONT_BB,
9769 replacing the GIMPLE_OMP_CONTINUE. */
9770 gsi
= gsi_last_bb (cont_bb
);
9771 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
9772 vmain
= gimple_omp_continue_control_use (cont_stmt
);
9773 vback
= gimple_omp_continue_control_def (cont_stmt
);
9775 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9777 if (POINTER_TYPE_P (type
))
9778 t
= fold_build_pointer_plus (vmain
, step
);
9780 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
9781 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
9782 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
9783 true, GSI_SAME_STMT
);
9784 assign_stmt
= gimple_build_assign (vback
, t
);
9785 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
9787 if (tree_int_cst_equal (fd
->chunk_size
, integer_one_node
))
9788 t
= build2 (EQ_EXPR
, boolean_type_node
,
9789 build_int_cst (itype
, 0),
9790 build_int_cst (itype
, 1));
9792 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
9793 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
9795 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
9798 /* Remove GIMPLE_OMP_CONTINUE. */
9799 gsi_remove (&gsi
, true);
9801 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
9802 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
9804 /* Trip update code goes into TRIP_UPDATE_BB. */
9805 gsi
= gsi_start_bb (trip_update_bb
);
9807 t
= build_int_cst (itype
, 1);
9808 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
9809 assign_stmt
= gimple_build_assign (trip_back
, t
);
9810 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
9813 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
9814 gsi
= gsi_last_bb (exit_bb
);
9815 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
9817 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
9818 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
9820 gsi_remove (&gsi
, true);
9822 /* Connect the new blocks. */
9823 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
9824 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
9828 se
= find_edge (cont_bb
, body_bb
);
9831 se
= BRANCH_EDGE (cont_bb
);
9832 gcc_assert (single_succ (se
->dest
) == body_bb
);
9834 if (gimple_omp_for_combined_p (fd
->for_stmt
))
9839 else if (fd
->collapse
> 1)
9842 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
9845 se
->flags
= EDGE_TRUE_VALUE
;
9846 find_edge (cont_bb
, trip_update_bb
)->flags
9847 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
9849 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
9852 if (gimple_in_ssa_p (cfun
))
9860 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
9862 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
9863 remove arguments of the phi nodes in fin_bb. We need to create
9864 appropriate phi nodes in iter_part_bb instead. */
9865 se
= find_edge (iter_part_bb
, fin_bb
);
9866 re
= single_succ_edge (trip_update_bb
);
9867 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
9868 ene
= single_succ_edge (entry_bb
);
9870 psi
= gsi_start_phis (fin_bb
);
9871 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
9872 gsi_next (&psi
), ++i
)
9875 source_location locus
;
9878 t
= gimple_phi_result (phi
);
9879 gcc_assert (t
== redirect_edge_var_map_result (vm
));
9881 if (!single_pred_p (fin_bb
))
9882 t
= copy_ssa_name (t
, phi
);
9884 nphi
= create_phi_node (t
, iter_part_bb
);
9886 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
9887 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
9889 /* A special case -- fd->loop.v is not yet computed in
9890 iter_part_bb, we need to use vextra instead. */
9891 if (t
== fd
->loop
.v
)
9893 add_phi_arg (nphi
, t
, ene
, locus
);
9894 locus
= redirect_edge_var_map_location (vm
);
9895 tree back_arg
= redirect_edge_var_map_def (vm
);
9896 add_phi_arg (nphi
, back_arg
, re
, locus
);
9897 edge ce
= find_edge (cont_bb
, body_bb
);
9900 ce
= BRANCH_EDGE (cont_bb
);
9901 gcc_assert (single_succ (ce
->dest
) == body_bb
);
9902 ce
= single_succ_edge (ce
->dest
);
9904 gphi
*inner_loop_phi
= find_phi_with_arg_on_edge (back_arg
, ce
);
9905 gcc_assert (inner_loop_phi
!= NULL
);
9906 add_phi_arg (inner_loop_phi
, gimple_phi_result (nphi
),
9907 find_edge (seq_start_bb
, body_bb
), locus
);
9909 if (!single_pred_p (fin_bb
))
9910 add_phi_arg (phi
, gimple_phi_result (nphi
), se
, locus
);
9912 gcc_assert (gsi_end_p (psi
) && (head
== NULL
|| i
== head
->length ()));
9913 redirect_edge_var_map_clear (re
);
9914 if (single_pred_p (fin_bb
))
9917 psi
= gsi_start_phis (fin_bb
);
9918 if (gsi_end_p (psi
))
9920 remove_phi_node (&psi
, false);
9923 /* Make phi node for trip. */
9924 phi
= create_phi_node (trip_main
, iter_part_bb
);
9925 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
9927 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
9932 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
9933 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
9934 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
9935 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
9936 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
9937 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
9938 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
9939 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
9940 recompute_dominator (CDI_DOMINATORS
, body_bb
));
9944 struct loop
*loop
= body_bb
->loop_father
;
9945 struct loop
*trip_loop
= alloc_loop ();
9946 trip_loop
->header
= iter_part_bb
;
9947 trip_loop
->latch
= trip_update_bb
;
9948 add_loop (trip_loop
, iter_part_bb
->loop_father
);
9950 if (loop
!= entry_bb
->loop_father
)
9952 gcc_assert (loop
->header
== body_bb
);
9953 gcc_assert (loop
->latch
== region
->cont
9954 || single_pred (loop
->latch
) == region
->cont
);
9955 trip_loop
->inner
= loop
;
9959 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
9961 loop
= alloc_loop ();
9962 loop
->header
= body_bb
;
9963 if (collapse_bb
== NULL
)
9964 loop
->latch
= cont_bb
;
9965 add_loop (loop
, trip_loop
);
9970 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
9972 for (V = N1; V cond N2; V += STEP) BODY;
9974 where COND is "<" or ">" or "!=", we generate pseudocode
9976 for (ind_var = low; ind_var < high; ind_var++)
9978 V = n1 + (ind_var * STEP)
9983 In the above pseudocode, low and high are function parameters of the
9984 child function. In the function below, we are inserting a temp.
9985 variable that will be making a call to two OMP functions that will not be
9986 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
9987 with _Cilk_for). These functions are replaced with low and high
9988 by the function that handles taskreg. */
9992 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
9994 bool broken_loop
= region
->cont
== NULL
;
9995 basic_block entry_bb
= region
->entry
;
9996 basic_block cont_bb
= region
->cont
;
9998 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
9999 gcc_assert (broken_loop
10000 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10001 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10002 basic_block l1_bb
, l2_bb
;
10006 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10007 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10008 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10009 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10013 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10014 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10015 l2_bb
= single_succ (l1_bb
);
10017 basic_block exit_bb
= region
->exit
;
10018 basic_block l2_dom_bb
= NULL
;
10020 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
10022 /* Below statements until the "tree high_val = ..." are pseudo statements
10023 used to pass information to be used by expand_omp_taskreg.
10024 low_val and high_val will be replaced by the __low and __high
10025 parameter from the child function.
10027 The call_exprs part is a place-holder, it is mainly used
10028 to distinctly identify to the top-level part that this is
10029 where we should put low and high (reasoning given in header
10033 = gimple_omp_parallel_child_fn (
10034 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
10035 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
10036 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
10038 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
10040 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
10043 gcc_assert (low_val
&& high_val
);
10045 tree type
= TREE_TYPE (low_val
);
10046 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
10047 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10049 /* Not needed in SSA form right now. */
10050 gcc_assert (!gimple_in_ssa_p (cfun
));
10051 if (l2_dom_bb
== NULL
)
10055 tree n2
= high_val
;
10057 gimple
*stmt
= gimple_build_assign (ind_var
, n1
);
10059 /* Replace the GIMPLE_OMP_FOR statement. */
10060 gsi_replace (&gsi
, stmt
, true);
10064 /* Code to control the increment goes in the CONT_BB. */
10065 gsi
= gsi_last_bb (cont_bb
);
10066 stmt
= gsi_stmt (gsi
);
10067 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10068 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
10069 build_one_cst (type
));
10071 /* Replace GIMPLE_OMP_CONTINUE. */
10072 gsi_replace (&gsi
, stmt
, true);
10075 /* Emit the condition in L1_BB. */
10076 gsi
= gsi_after_labels (l1_bb
);
10077 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
10078 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
10080 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
10081 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10082 fd
->loop
.n1
, fold_convert (sizetype
, t
));
10084 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
10085 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
10086 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
10087 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10089 /* The condition is always '<' since the runtime will fill in the low
10090 and high values. */
10091 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
10092 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
10094 /* Remove GIMPLE_OMP_RETURN. */
10095 gsi
= gsi_last_bb (exit_bb
);
10096 gsi_remove (&gsi
, true);
10098 /* Connect the new blocks. */
10099 remove_edge (FALLTHRU_EDGE (entry_bb
));
10104 remove_edge (BRANCH_EDGE (entry_bb
));
10105 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10107 e
= BRANCH_EDGE (l1_bb
);
10108 ne
= FALLTHRU_EDGE (l1_bb
);
10109 e
->flags
= EDGE_TRUE_VALUE
;
10113 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10115 ne
= single_succ_edge (l1_bb
);
10116 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10119 ne
->flags
= EDGE_FALSE_VALUE
;
10120 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10121 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10123 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10124 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10125 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10129 struct loop
*loop
= alloc_loop ();
10130 loop
->header
= l1_bb
;
10131 loop
->latch
= cont_bb
;
10132 add_loop (loop
, l1_bb
->loop_father
);
10133 loop
->safelen
= INT_MAX
;
10136 /* Pick the correct library function based on the precision of the
10137 induction variable type. */
10138 tree lib_fun
= NULL_TREE
;
10139 if (TYPE_PRECISION (type
) == 32)
10140 lib_fun
= cilk_for_32_fndecl
;
10141 else if (TYPE_PRECISION (type
) == 64)
10142 lib_fun
= cilk_for_64_fndecl
;
10144 gcc_unreachable ();
10146 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
10148 /* WS_ARGS contains the library function flavor to call:
10149 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
10150 user-defined grain value. If the user does not define one, then zero
10151 is passed in by the parser. */
10152 vec_alloc (region
->ws_args
, 2);
10153 region
->ws_args
->quick_push (lib_fun
);
10154 region
->ws_args
->quick_push (fd
->chunk_size
);
10157 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
10158 loop. Given parameters:
10160 for (V = N1; V cond N2; V += STEP) BODY;
10162 where COND is "<" or ">", we generate pseudocode
10170 if (V cond N2) goto L0; else goto L2;
10173 For collapsed loops, given parameters:
10175 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
10176 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
10177 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
10180 we generate pseudocode
10186 count3 = (adj + N32 - N31) / STEP3;
10191 count2 = (adj + N22 - N21) / STEP2;
10196 count1 = (adj + N12 - N11) / STEP1;
10197 count = count1 * count2 * count3;
10207 V2 += (V3 cond3 N32) ? 0 : STEP2;
10208 V3 = (V3 cond3 N32) ? V3 : N31;
10209 V1 += (V2 cond2 N22) ? 0 : STEP1;
10210 V2 = (V2 cond2 N22) ? V2 : N21;
10212 if (V < count) goto L0; else goto L2;
10218 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
10221 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
10222 gimple_stmt_iterator gsi
;
10225 bool broken_loop
= region
->cont
== NULL
;
10227 tree
*counts
= NULL
;
10229 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10230 OMP_CLAUSE_SAFELEN
);
10231 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10232 OMP_CLAUSE__SIMDUID_
);
10235 type
= TREE_TYPE (fd
->loop
.v
);
10236 entry_bb
= region
->entry
;
10237 cont_bb
= region
->cont
;
10238 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10239 gcc_assert (broken_loop
10240 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10241 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10244 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
10245 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10246 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
10247 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10251 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
10252 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
10253 l2_bb
= single_succ (l1_bb
);
10255 exit_bb
= region
->exit
;
10258 gsi
= gsi_last_bb (entry_bb
);
10260 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10261 /* Not needed in SSA form right now. */
10262 gcc_assert (!gimple_in_ssa_p (cfun
));
10263 if (fd
->collapse
> 1)
10265 int first_zero_iter
= -1, dummy
= -1;
10266 basic_block zero_iter_bb
= l2_bb
, dummy_bb
= NULL
;
10268 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10269 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10270 zero_iter_bb
, first_zero_iter
,
10271 dummy_bb
, dummy
, l2_dom_bb
);
10273 if (l2_dom_bb
== NULL
)
10278 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
10280 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10281 OMP_CLAUSE__LOOPTEMP_
);
10282 gcc_assert (innerc
);
10283 n1
= OMP_CLAUSE_DECL (innerc
);
10284 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10285 OMP_CLAUSE__LOOPTEMP_
);
10286 gcc_assert (innerc
);
10287 n2
= OMP_CLAUSE_DECL (innerc
);
10288 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10289 fold_convert (type
, n1
));
10290 if (fd
->collapse
> 1)
10293 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
10299 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
10300 fold_convert (type
, fd
->loop
.n1
));
10301 if (fd
->collapse
> 1)
10302 for (i
= 0; i
< fd
->collapse
; i
++)
10304 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10305 if (POINTER_TYPE_P (itype
))
10306 itype
= signed_type_for (itype
);
10307 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
10308 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10312 /* Remove the GIMPLE_OMP_FOR statement. */
10313 gsi_remove (&gsi
, true);
10317 /* Code to control the increment goes in the CONT_BB. */
10318 gsi
= gsi_last_bb (cont_bb
);
10319 stmt
= gsi_stmt (gsi
);
10320 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
10322 if (POINTER_TYPE_P (type
))
10323 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
10325 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
10326 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
10328 if (fd
->collapse
> 1)
10330 i
= fd
->collapse
- 1;
10331 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
10333 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
10334 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
10338 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
10339 fd
->loops
[i
].step
);
10340 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
10341 fd
->loops
[i
].v
, t
);
10343 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10345 for (i
= fd
->collapse
- 1; i
> 0; i
--)
10347 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
10348 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
10349 if (POINTER_TYPE_P (itype2
))
10350 itype2
= signed_type_for (itype2
);
10351 t
= build3 (COND_EXPR
, itype2
,
10352 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10354 fold_convert (itype
, fd
->loops
[i
].n2
)),
10355 build_int_cst (itype2
, 0),
10356 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
10357 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
10358 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
10360 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
10361 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
10363 t
= build3 (COND_EXPR
, itype
,
10364 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
10366 fold_convert (itype
, fd
->loops
[i
].n2
)),
10368 fold_convert (itype
, fd
->loops
[i
].n1
));
10369 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
10373 /* Remove GIMPLE_OMP_CONTINUE. */
10374 gsi_remove (&gsi
, true);
10377 /* Emit the condition in L1_BB. */
10378 gsi
= gsi_start_bb (l1_bb
);
10380 t
= fold_convert (type
, n2
);
10381 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10382 false, GSI_CONTINUE_LINKING
);
10383 tree v
= fd
->loop
.v
;
10384 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
10385 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
10386 false, GSI_CONTINUE_LINKING
);
10387 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, v
, t
);
10388 cond_stmt
= gimple_build_cond_empty (t
);
10389 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
10390 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10392 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
10395 gsi
= gsi_for_stmt (cond_stmt
);
10396 gimple_regimplify_operands (cond_stmt
, &gsi
);
10399 /* Remove GIMPLE_OMP_RETURN. */
10400 gsi
= gsi_last_bb (exit_bb
);
10401 gsi_remove (&gsi
, true);
10403 /* Connect the new blocks. */
10404 remove_edge (FALLTHRU_EDGE (entry_bb
));
10408 remove_edge (BRANCH_EDGE (entry_bb
));
10409 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
10411 e
= BRANCH_EDGE (l1_bb
);
10412 ne
= FALLTHRU_EDGE (l1_bb
);
10413 e
->flags
= EDGE_TRUE_VALUE
;
10417 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
10419 ne
= single_succ_edge (l1_bb
);
10420 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
10423 ne
->flags
= EDGE_FALSE_VALUE
;
10424 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
10425 ne
->probability
= REG_BR_PROB_BASE
/ 8;
10427 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
10428 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
10429 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
10433 struct loop
*loop
= alloc_loop ();
10434 loop
->header
= l1_bb
;
10435 loop
->latch
= cont_bb
;
10436 add_loop (loop
, l1_bb
->loop_father
);
10437 if (safelen
== NULL_TREE
)
10438 loop
->safelen
= INT_MAX
;
10441 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
10442 if (TREE_CODE (safelen
) != INTEGER_CST
)
10444 else if (!tree_fits_uhwi_p (safelen
)
10445 || tree_to_uhwi (safelen
) > INT_MAX
)
10446 loop
->safelen
= INT_MAX
;
10448 loop
->safelen
= tree_to_uhwi (safelen
);
10449 if (loop
->safelen
== 1)
10454 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
10455 cfun
->has_simduid_loops
= true;
10457 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
10459 if ((flag_tree_loop_vectorize
10460 || (!global_options_set
.x_flag_tree_loop_vectorize
10461 && !global_options_set
.x_flag_tree_vectorize
))
10462 && flag_tree_loop_optimize
10463 && loop
->safelen
> 1)
10465 loop
->force_vectorize
= true;
10466 cfun
->has_force_vectorize_loops
= true;
10470 cfun
->has_simduid_loops
= true;
10473 /* Taskloop construct is represented after gimplification with
10474 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10475 in between them. This routine expands the outer GIMPLE_OMP_FOR,
10476 which should just compute all the needed loop temporaries
10477 for GIMPLE_OMP_TASK. */
10480 expand_omp_taskloop_for_outer (struct omp_region
*region
,
10481 struct omp_for_data
*fd
,
10482 gimple
*inner_stmt
)
10484 tree type
, bias
= NULL_TREE
;
10485 basic_block entry_bb
, cont_bb
, exit_bb
;
10486 gimple_stmt_iterator gsi
;
10487 gassign
*assign_stmt
;
10488 tree
*counts
= NULL
;
10491 gcc_assert (inner_stmt
);
10492 gcc_assert (region
->cont
);
10493 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_TASK
10494 && gimple_omp_task_taskloop_p (inner_stmt
));
10495 type
= TREE_TYPE (fd
->loop
.v
);
10497 /* See if we need to bias by LLONG_MIN. */
10498 if (fd
->iter_type
== long_long_unsigned_type_node
10499 && TREE_CODE (type
) == INTEGER_TYPE
10500 && !TYPE_UNSIGNED (type
))
10504 if (fd
->loop
.cond_code
== LT_EXPR
)
10507 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10511 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10514 if (TREE_CODE (n1
) != INTEGER_CST
10515 || TREE_CODE (n2
) != INTEGER_CST
10516 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10517 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10520 entry_bb
= region
->entry
;
10521 cont_bb
= region
->cont
;
10522 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10523 gcc_assert (BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
10524 exit_bb
= region
->exit
;
10526 gsi
= gsi_last_bb (entry_bb
);
10527 gimple
*for_stmt
= gsi_stmt (gsi
);
10528 gcc_assert (gimple_code (for_stmt
) == GIMPLE_OMP_FOR
);
10529 if (fd
->collapse
> 1)
10531 int first_zero_iter
= -1, dummy
= -1;
10532 basic_block zero_iter_bb
= NULL
, dummy_bb
= NULL
, l2_dom_bb
= NULL
;
10534 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10535 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10536 zero_iter_bb
, first_zero_iter
,
10537 dummy_bb
, dummy
, l2_dom_bb
);
10541 /* Some counts[i] vars might be uninitialized if
10542 some loop has zero iterations. But the body shouldn't
10543 be executed in that case, so just avoid uninit warnings. */
10544 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
10545 if (SSA_VAR_P (counts
[i
]))
10546 TREE_NO_WARNING (counts
[i
]) = 1;
10548 edge e
= split_block (entry_bb
, gsi_stmt (gsi
));
10549 entry_bb
= e
->dest
;
10550 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
10551 gsi
= gsi_last_bb (entry_bb
);
10552 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
10553 get_immediate_dominator (CDI_DOMINATORS
,
10561 if (POINTER_TYPE_P (TREE_TYPE (t0
))
10562 && TYPE_PRECISION (TREE_TYPE (t0
))
10563 != TYPE_PRECISION (fd
->iter_type
))
10565 /* Avoid casting pointers to integer of a different size. */
10566 tree itype
= signed_type_for (type
);
10567 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
10568 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
10572 t1
= fold_convert (fd
->iter_type
, t1
);
10573 t0
= fold_convert (fd
->iter_type
, t0
);
10577 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
10578 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
10581 tree innerc
= find_omp_clause (gimple_omp_task_clauses (inner_stmt
),
10582 OMP_CLAUSE__LOOPTEMP_
);
10583 gcc_assert (innerc
);
10584 tree startvar
= OMP_CLAUSE_DECL (innerc
);
10585 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10586 gcc_assert (innerc
);
10587 tree endvar
= OMP_CLAUSE_DECL (innerc
);
10588 if (fd
->collapse
> 1 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
10590 gcc_assert (innerc
);
10591 for (i
= 1; i
< fd
->collapse
; i
++)
10593 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10594 OMP_CLAUSE__LOOPTEMP_
);
10595 gcc_assert (innerc
);
10597 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10598 OMP_CLAUSE__LOOPTEMP_
);
10601 /* If needed (inner taskloop has lastprivate clause), propagate
10602 down the total number of iterations. */
10603 tree t
= force_gimple_operand_gsi (&gsi
, fd
->loop
.n2
, false,
10605 GSI_CONTINUE_LINKING
);
10606 assign_stmt
= gimple_build_assign (OMP_CLAUSE_DECL (innerc
), t
);
10607 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10611 t0
= force_gimple_operand_gsi (&gsi
, t0
, false, NULL_TREE
, false,
10612 GSI_CONTINUE_LINKING
);
10613 assign_stmt
= gimple_build_assign (startvar
, t0
);
10614 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10616 t1
= force_gimple_operand_gsi (&gsi
, t1
, false, NULL_TREE
, false,
10617 GSI_CONTINUE_LINKING
);
10618 assign_stmt
= gimple_build_assign (endvar
, t1
);
10619 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10620 if (fd
->collapse
> 1)
10621 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10623 /* Remove the GIMPLE_OMP_FOR statement. */
10624 gsi
= gsi_for_stmt (for_stmt
);
10625 gsi_remove (&gsi
, true);
10627 gsi
= gsi_last_bb (cont_bb
);
10628 gsi_remove (&gsi
, true);
10630 gsi
= gsi_last_bb (exit_bb
);
10631 gsi_remove (&gsi
, true);
10633 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
10634 remove_edge (BRANCH_EDGE (entry_bb
));
10635 FALLTHRU_EDGE (cont_bb
)->probability
= REG_BR_PROB_BASE
;
10636 remove_edge (BRANCH_EDGE (cont_bb
));
10637 set_immediate_dominator (CDI_DOMINATORS
, exit_bb
, cont_bb
);
10638 set_immediate_dominator (CDI_DOMINATORS
, region
->entry
,
10639 recompute_dominator (CDI_DOMINATORS
, region
->entry
));
10642 /* Taskloop construct is represented after gimplification with
10643 two GIMPLE_OMP_FOR constructs with GIMPLE_OMP_TASK sandwiched
10644 in between them. This routine expands the inner GIMPLE_OMP_FOR.
10645 GOMP_taskloop{,_ull} function arranges for each task to be given just
10646 a single range of iterations. */
10649 expand_omp_taskloop_for_inner (struct omp_region
*region
,
10650 struct omp_for_data
*fd
,
10651 gimple
*inner_stmt
)
10653 tree e
, t
, type
, itype
, vmain
, vback
, bias
= NULL_TREE
;
10654 basic_block entry_bb
, exit_bb
, body_bb
, cont_bb
, collapse_bb
= NULL
;
10655 basic_block fin_bb
;
10656 gimple_stmt_iterator gsi
;
10658 bool broken_loop
= region
->cont
== NULL
;
10659 tree
*counts
= NULL
;
10662 itype
= type
= TREE_TYPE (fd
->loop
.v
);
10663 if (POINTER_TYPE_P (type
))
10664 itype
= signed_type_for (type
);
10666 /* See if we need to bias by LLONG_MIN. */
10667 if (fd
->iter_type
== long_long_unsigned_type_node
10668 && TREE_CODE (type
) == INTEGER_TYPE
10669 && !TYPE_UNSIGNED (type
))
10673 if (fd
->loop
.cond_code
== LT_EXPR
)
10676 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10680 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
10683 if (TREE_CODE (n1
) != INTEGER_CST
10684 || TREE_CODE (n2
) != INTEGER_CST
10685 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
10686 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
10689 entry_bb
= region
->entry
;
10690 cont_bb
= region
->cont
;
10691 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
10692 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
10693 gcc_assert (broken_loop
10694 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
10695 body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10698 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
10699 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
10701 exit_bb
= region
->exit
;
10703 /* Iteration space partitioning goes in ENTRY_BB. */
10704 gsi
= gsi_last_bb (entry_bb
);
10705 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
10707 if (fd
->collapse
> 1)
10709 int first_zero_iter
= -1, dummy
= -1;
10710 basic_block l2_dom_bb
= NULL
, dummy_bb
= NULL
;
10712 counts
= XALLOCAVEC (tree
, fd
->collapse
);
10713 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
10714 fin_bb
, first_zero_iter
,
10715 dummy_bb
, dummy
, l2_dom_bb
);
10719 t
= integer_one_node
;
10721 step
= fd
->loop
.step
;
10722 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
10723 OMP_CLAUSE__LOOPTEMP_
);
10724 gcc_assert (innerc
);
10725 n1
= OMP_CLAUSE_DECL (innerc
);
10726 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
), OMP_CLAUSE__LOOPTEMP_
);
10727 gcc_assert (innerc
);
10728 n2
= OMP_CLAUSE_DECL (innerc
);
10731 n1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n1
, bias
);
10732 n2
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, n2
, bias
);
10734 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
10735 true, NULL_TREE
, true, GSI_SAME_STMT
);
10736 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
10737 true, NULL_TREE
, true, GSI_SAME_STMT
);
10738 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
10739 true, NULL_TREE
, true, GSI_SAME_STMT
);
10741 tree startvar
= fd
->loop
.v
;
10742 tree endvar
= NULL_TREE
;
10744 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10746 tree clauses
= gimple_omp_for_clauses (inner_stmt
);
10747 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
10748 gcc_assert (innerc
);
10749 startvar
= OMP_CLAUSE_DECL (innerc
);
10750 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
10751 OMP_CLAUSE__LOOPTEMP_
);
10752 gcc_assert (innerc
);
10753 endvar
= OMP_CLAUSE_DECL (innerc
);
10755 t
= fold_convert (TREE_TYPE (startvar
), n1
);
10756 t
= force_gimple_operand_gsi (&gsi
, t
,
10758 && TREE_ADDRESSABLE (startvar
),
10759 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
10760 gimple
*assign_stmt
= gimple_build_assign (startvar
, t
);
10761 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10763 t
= fold_convert (TREE_TYPE (startvar
), n2
);
10764 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
10765 false, GSI_CONTINUE_LINKING
);
10768 assign_stmt
= gimple_build_assign (endvar
, e
);
10769 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10770 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
10771 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
10773 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
10774 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
10776 if (fd
->collapse
> 1)
10777 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
10781 /* The code controlling the sequential loop replaces the
10782 GIMPLE_OMP_CONTINUE. */
10783 gsi
= gsi_last_bb (cont_bb
);
10784 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
10785 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
10786 vmain
= gimple_omp_continue_control_use (cont_stmt
);
10787 vback
= gimple_omp_continue_control_def (cont_stmt
);
10789 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
10791 if (POINTER_TYPE_P (type
))
10792 t
= fold_build_pointer_plus (vmain
, step
);
10794 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
10795 t
= force_gimple_operand_gsi (&gsi
, t
,
10797 && TREE_ADDRESSABLE (vback
),
10798 NULL_TREE
, true, GSI_SAME_STMT
);
10799 assign_stmt
= gimple_build_assign (vback
, t
);
10800 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
10802 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
10803 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
10805 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
10808 /* Remove the GIMPLE_OMP_CONTINUE statement. */
10809 gsi_remove (&gsi
, true);
10811 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
10812 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
10815 /* Remove the GIMPLE_OMP_FOR statement. */
10816 gsi
= gsi_for_stmt (fd
->for_stmt
);
10817 gsi_remove (&gsi
, true);
10819 /* Remove the GIMPLE_OMP_RETURN statement. */
10820 gsi
= gsi_last_bb (exit_bb
);
10821 gsi_remove (&gsi
, true);
10823 FALLTHRU_EDGE (entry_bb
)->probability
= REG_BR_PROB_BASE
;
10825 remove_edge (BRANCH_EDGE (entry_bb
));
10828 remove_edge_and_dominated_blocks (BRANCH_EDGE (entry_bb
));
10829 region
->outer
->cont
= NULL
;
10832 /* Connect all the blocks. */
10835 ep
= find_edge (cont_bb
, body_bb
);
10836 if (gimple_omp_for_combined_p (fd
->for_stmt
))
10841 else if (fd
->collapse
> 1)
10844 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
10847 ep
->flags
= EDGE_TRUE_VALUE
;
10848 find_edge (cont_bb
, fin_bb
)->flags
10849 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
10852 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
10853 recompute_dominator (CDI_DOMINATORS
, body_bb
));
10855 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
10856 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
10858 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
10860 struct loop
*loop
= alloc_loop ();
10861 loop
->header
= body_bb
;
10862 if (collapse_bb
== NULL
)
10863 loop
->latch
= cont_bb
;
10864 add_loop (loop
, body_bb
->loop_father
);
10868 /* A subroutine of expand_omp_for. Generate code for an OpenACC
10869 partitioned loop. The lowering here is abstracted, in that the
10870 loop parameters are passed through internal functions, which are
10871 further lowered by oacc_device_lower, once we get to the target
10872 compiler. The loop is of the form:
10874 for (V = B; V LTGT E; V += S) {BODY}
10876 where LTGT is < or >. We may have a specified chunking size, CHUNKING
10877 (constant 0 for no chunking) and we will have a GWV partitioning
10878 mask, specifying dimensions over which the loop is to be
10879 partitioned (see note below). We generate code that looks like:
10881 <entry_bb> [incoming FALL->body, BRANCH->exit]
10882 typedef signedintify (typeof (V)) T; // underlying signed integral type
10885 T DIR = LTGT == '<' ? +1 : -1;
10886 T chunk_max = GOACC_LOOP_CHUNK (dir, range, S, CHUNK_SIZE, GWV);
10887 T step = GOACC_LOOP_STEP (dir, range, S, CHUNK_SIZE, GWV);
10889 <head_bb> [created by splitting end of entry_bb]
10890 T offset = GOACC_LOOP_OFFSET (dir, range, S, CHUNK_SIZE, GWV, chunk_no);
10891 T bound = GOACC_LOOP_BOUND (dir, range, S, CHUNK_SIZE, GWV, offset);
10892 if (!(offset LTGT bound)) goto bottom_bb;
10894 <body_bb> [incoming]
10898 <cont_bb> [incoming, may == body_bb FALL->exit_bb, BRANCH->body_bb]
10900 if (offset LTGT bound) goto body_bb; [*]
10902 <bottom_bb> [created by splitting start of exit_bb] insert BRANCH->head_bb
10904 if (chunk < chunk_max) goto head_bb;
10906 <exit_bb> [incoming]
10907 V = B + ((range -/+ 1) / S +/- 1) * S [*]
10909 [*] Needed if V live at end of loop
10911 Note: CHUNKING & GWV mask are specified explicitly here. This is a
10912 transition, and will be specified by a more general mechanism shortly.
10916 expand_oacc_for (struct omp_region
*region
, struct omp_for_data
*fd
)
10918 tree v
= fd
->loop
.v
;
10919 enum tree_code cond_code
= fd
->loop
.cond_code
;
10920 enum tree_code plus_code
= PLUS_EXPR
;
10922 tree chunk_size
= integer_minus_one_node
;
10923 tree gwv
= integer_zero_node
;
10924 tree iter_type
= TREE_TYPE (v
);
10925 tree diff_type
= iter_type
;
10926 tree plus_type
= iter_type
;
10927 struct oacc_collapse
*counts
= NULL
;
10929 gcc_checking_assert (gimple_omp_for_kind (fd
->for_stmt
)
10930 == GF_OMP_FOR_KIND_OACC_LOOP
);
10931 gcc_assert (!gimple_omp_for_combined_into_p (fd
->for_stmt
));
10932 gcc_assert (cond_code
== LT_EXPR
|| cond_code
== GT_EXPR
);
10934 if (POINTER_TYPE_P (iter_type
))
10936 plus_code
= POINTER_PLUS_EXPR
;
10937 plus_type
= sizetype
;
10939 if (POINTER_TYPE_P (diff_type
) || TYPE_UNSIGNED (diff_type
))
10940 diff_type
= signed_type_for (diff_type
);
10942 basic_block entry_bb
= region
->entry
; /* BB ending in OMP_FOR */
10943 basic_block exit_bb
= region
->exit
; /* BB ending in OMP_RETURN */
10944 basic_block cont_bb
= region
->cont
; /* BB ending in OMP_CONTINUE */
10945 basic_block bottom_bb
= NULL
;
10947 /* entry_bb has two sucessors; the branch edge is to the exit
10948 block, fallthrough edge to body. */
10949 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2
10950 && BRANCH_EDGE (entry_bb
)->dest
== exit_bb
);
10952 /* If cont_bb non-NULL, it has 2 successors. The branch successor is
10953 body_bb, or to a block whose only successor is the body_bb. Its
10954 fallthrough successor is the final block (same as the branch
10955 successor of the entry_bb). */
10958 basic_block body_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
10959 basic_block bed
= BRANCH_EDGE (cont_bb
)->dest
;
10961 gcc_assert (FALLTHRU_EDGE (cont_bb
)->dest
== exit_bb
);
10962 gcc_assert (bed
== body_bb
|| single_succ_edge (bed
)->dest
== body_bb
);
10965 gcc_assert (!gimple_in_ssa_p (cfun
));
10967 /* The exit block only has entry_bb and cont_bb as predecessors. */
10968 gcc_assert (EDGE_COUNT (exit_bb
->preds
) == 1 + (cont_bb
!= NULL
));
10971 tree chunk_max
= NULL_TREE
;
10972 tree bound
, offset
;
10973 tree step
= create_tmp_var (diff_type
, ".step");
10974 bool up
= cond_code
== LT_EXPR
;
10975 tree dir
= build_int_cst (diff_type
, up
? +1 : -1);
10976 bool chunking
= !gimple_in_ssa_p (cfun
);;
10979 /* SSA instances. */
10980 tree offset_incr
= NULL_TREE
;
10981 tree offset_init
= NULL_TREE
;
10983 gimple_stmt_iterator gsi
;
10989 edge split
, be
, fte
;
10991 /* Split the end of entry_bb to create head_bb. */
10992 split
= split_block (entry_bb
, last_stmt (entry_bb
));
10993 basic_block head_bb
= split
->dest
;
10994 entry_bb
= split
->src
;
10996 /* Chunk setup goes at end of entry_bb, replacing the omp_for. */
10997 gsi
= gsi_last_bb (entry_bb
);
10998 gomp_for
*for_stmt
= as_a
<gomp_for
*> (gsi_stmt (gsi
));
10999 loc
= gimple_location (for_stmt
);
11001 if (gimple_in_ssa_p (cfun
))
11003 offset_init
= gimple_omp_for_index (for_stmt
, 0);
11004 gcc_assert (integer_zerop (fd
->loop
.n1
));
11005 /* The SSA parallelizer does gang parallelism. */
11006 gwv
= build_int_cst (integer_type_node
, GOMP_DIM_MASK (GOMP_DIM_GANG
));
11009 if (fd
->collapse
> 1)
11011 counts
= XALLOCAVEC (struct oacc_collapse
, fd
->collapse
);
11012 tree total
= expand_oacc_collapse_init (fd
, &gsi
, counts
,
11013 TREE_TYPE (fd
->loop
.n2
));
11015 if (SSA_VAR_P (fd
->loop
.n2
))
11017 total
= force_gimple_operand_gsi (&gsi
, total
, false, NULL_TREE
,
11018 true, GSI_SAME_STMT
);
11019 ass
= gimple_build_assign (fd
->loop
.n2
, total
);
11020 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11025 tree b
= fd
->loop
.n1
;
11026 tree e
= fd
->loop
.n2
;
11027 tree s
= fd
->loop
.step
;
11029 b
= force_gimple_operand_gsi (&gsi
, b
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11030 e
= force_gimple_operand_gsi (&gsi
, e
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11032 /* Convert the step, avoiding possible unsigned->signed overflow. */
11033 negating
= !up
&& TYPE_UNSIGNED (TREE_TYPE (s
));
11035 s
= fold_build1 (NEGATE_EXPR
, TREE_TYPE (s
), s
);
11036 s
= fold_convert (diff_type
, s
);
11038 s
= fold_build1 (NEGATE_EXPR
, diff_type
, s
);
11039 s
= force_gimple_operand_gsi (&gsi
, s
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11042 chunk_size
= integer_zero_node
;
11043 expr
= fold_convert (diff_type
, chunk_size
);
11044 chunk_size
= force_gimple_operand_gsi (&gsi
, expr
, true,
11045 NULL_TREE
, true, GSI_SAME_STMT
);
11046 /* Determine the range, avoiding possible unsigned->signed overflow. */
11047 negating
= !up
&& TYPE_UNSIGNED (iter_type
);
11048 expr
= fold_build2 (MINUS_EXPR
, plus_type
,
11049 fold_convert (plus_type
, negating
? b
: e
),
11050 fold_convert (plus_type
, negating
? e
: b
));
11051 expr
= fold_convert (diff_type
, expr
);
11053 expr
= fold_build1 (NEGATE_EXPR
, diff_type
, expr
);
11054 tree range
= force_gimple_operand_gsi (&gsi
, expr
, true,
11055 NULL_TREE
, true, GSI_SAME_STMT
);
11057 chunk_no
= build_int_cst (diff_type
, 0);
11060 gcc_assert (!gimple_in_ssa_p (cfun
));
11063 chunk_max
= create_tmp_var (diff_type
, ".chunk_max");
11064 chunk_no
= create_tmp_var (diff_type
, ".chunk_no");
11066 ass
= gimple_build_assign (chunk_no
, expr
);
11067 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11069 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11070 build_int_cst (integer_type_node
,
11071 IFN_GOACC_LOOP_CHUNKS
),
11072 dir
, range
, s
, chunk_size
, gwv
);
11073 gimple_call_set_lhs (call
, chunk_max
);
11074 gimple_set_location (call
, loc
);
11075 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11078 chunk_size
= chunk_no
;
11080 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 6,
11081 build_int_cst (integer_type_node
,
11082 IFN_GOACC_LOOP_STEP
),
11083 dir
, range
, s
, chunk_size
, gwv
);
11084 gimple_call_set_lhs (call
, step
);
11085 gimple_set_location (call
, loc
);
11086 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
11088 /* Remove the GIMPLE_OMP_FOR. */
11089 gsi_remove (&gsi
, true);
11091 /* Fixup edges from head_bb */
11092 be
= BRANCH_EDGE (head_bb
);
11093 fte
= FALLTHRU_EDGE (head_bb
);
11094 be
->flags
|= EDGE_FALSE_VALUE
;
11095 fte
->flags
^= EDGE_FALLTHRU
| EDGE_TRUE_VALUE
;
11097 basic_block body_bb
= fte
->dest
;
11099 if (gimple_in_ssa_p (cfun
))
11101 gsi
= gsi_last_bb (cont_bb
);
11102 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11104 offset
= gimple_omp_continue_control_use (cont_stmt
);
11105 offset_incr
= gimple_omp_continue_control_def (cont_stmt
);
11109 offset
= create_tmp_var (diff_type
, ".offset");
11110 offset_init
= offset_incr
= offset
;
11112 bound
= create_tmp_var (TREE_TYPE (offset
), ".bound");
11114 /* Loop offset & bound go into head_bb. */
11115 gsi
= gsi_start_bb (head_bb
);
11117 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11118 build_int_cst (integer_type_node
,
11119 IFN_GOACC_LOOP_OFFSET
),
11121 chunk_size
, gwv
, chunk_no
);
11122 gimple_call_set_lhs (call
, offset_init
);
11123 gimple_set_location (call
, loc
);
11124 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11126 call
= gimple_build_call_internal (IFN_GOACC_LOOP
, 7,
11127 build_int_cst (integer_type_node
,
11128 IFN_GOACC_LOOP_BOUND
),
11130 chunk_size
, gwv
, offset_init
);
11131 gimple_call_set_lhs (call
, bound
);
11132 gimple_set_location (call
, loc
);
11133 gsi_insert_after (&gsi
, call
, GSI_CONTINUE_LINKING
);
11135 expr
= build2 (cond_code
, boolean_type_node
, offset_init
, bound
);
11136 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11137 GSI_CONTINUE_LINKING
);
11139 /* V assignment goes into body_bb. */
11140 if (!gimple_in_ssa_p (cfun
))
11142 gsi
= gsi_start_bb (body_bb
);
11144 expr
= build2 (plus_code
, iter_type
, b
,
11145 fold_convert (plus_type
, offset
));
11146 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11147 true, GSI_SAME_STMT
);
11148 ass
= gimple_build_assign (v
, expr
);
11149 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11150 if (fd
->collapse
> 1)
11151 expand_oacc_collapse_vars (fd
, &gsi
, counts
, v
);
11154 /* Loop increment goes into cont_bb. If this is not a loop, we
11155 will have spawned threads as if it was, and each one will
11156 execute one iteration. The specification is not explicit about
11157 whether such constructs are ill-formed or not, and they can
11158 occur, especially when noreturn routines are involved. */
11161 gsi
= gsi_last_bb (cont_bb
);
11162 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
11163 loc
= gimple_location (cont_stmt
);
11165 /* Increment offset. */
11166 if (gimple_in_ssa_p (cfun
))
11167 expr
= build2 (plus_code
, iter_type
, offset
,
11168 fold_convert (plus_type
, step
));
11170 expr
= build2 (PLUS_EXPR
, diff_type
, offset
, step
);
11171 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11172 true, GSI_SAME_STMT
);
11173 ass
= gimple_build_assign (offset_incr
, expr
);
11174 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11175 expr
= build2 (cond_code
, boolean_type_node
, offset_incr
, bound
);
11176 gsi_insert_before (&gsi
, gimple_build_cond_empty (expr
), GSI_SAME_STMT
);
11178 /* Remove the GIMPLE_OMP_CONTINUE. */
11179 gsi_remove (&gsi
, true);
11181 /* Fixup edges from cont_bb */
11182 be
= BRANCH_EDGE (cont_bb
);
11183 fte
= FALLTHRU_EDGE (cont_bb
);
11184 be
->flags
|= EDGE_TRUE_VALUE
;
11185 fte
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11189 /* Split the beginning of exit_bb to make bottom_bb. We
11190 need to insert a nop at the start, because splitting is
11191 after a stmt, not before. */
11192 gsi
= gsi_start_bb (exit_bb
);
11193 stmt
= gimple_build_nop ();
11194 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11195 split
= split_block (exit_bb
, stmt
);
11196 bottom_bb
= split
->src
;
11197 exit_bb
= split
->dest
;
11198 gsi
= gsi_last_bb (bottom_bb
);
11200 /* Chunk increment and test goes into bottom_bb. */
11201 expr
= build2 (PLUS_EXPR
, diff_type
, chunk_no
,
11202 build_int_cst (diff_type
, 1));
11203 ass
= gimple_build_assign (chunk_no
, expr
);
11204 gsi_insert_after (&gsi
, ass
, GSI_CONTINUE_LINKING
);
11206 /* Chunk test at end of bottom_bb. */
11207 expr
= build2 (LT_EXPR
, boolean_type_node
, chunk_no
, chunk_max
);
11208 gsi_insert_after (&gsi
, gimple_build_cond_empty (expr
),
11209 GSI_CONTINUE_LINKING
);
11211 /* Fixup edges from bottom_bb. */
11212 split
->flags
^= EDGE_FALLTHRU
| EDGE_FALSE_VALUE
;
11213 make_edge (bottom_bb
, head_bb
, EDGE_TRUE_VALUE
);
11217 gsi
= gsi_last_bb (exit_bb
);
11218 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
11219 loc
= gimple_location (gsi_stmt (gsi
));
11221 if (!gimple_in_ssa_p (cfun
))
11223 /* Insert the final value of V, in case it is live. This is the
11224 value for the only thread that survives past the join. */
11225 expr
= fold_build2 (MINUS_EXPR
, diff_type
, range
, dir
);
11226 expr
= fold_build2 (PLUS_EXPR
, diff_type
, expr
, s
);
11227 expr
= fold_build2 (TRUNC_DIV_EXPR
, diff_type
, expr
, s
);
11228 expr
= fold_build2 (MULT_EXPR
, diff_type
, expr
, s
);
11229 expr
= build2 (plus_code
, iter_type
, b
, fold_convert (plus_type
, expr
));
11230 expr
= force_gimple_operand_gsi (&gsi
, expr
, false, NULL_TREE
,
11231 true, GSI_SAME_STMT
);
11232 ass
= gimple_build_assign (v
, expr
);
11233 gsi_insert_before (&gsi
, ass
, GSI_SAME_STMT
);
11236 /* Remove the OMP_RETURN. */
11237 gsi_remove (&gsi
, true);
11241 /* We now have one or two nested loops. Update the loop
11243 struct loop
*parent
= entry_bb
->loop_father
;
11244 struct loop
*body
= body_bb
->loop_father
;
11248 struct loop
*chunk_loop
= alloc_loop ();
11249 chunk_loop
->header
= head_bb
;
11250 chunk_loop
->latch
= bottom_bb
;
11251 add_loop (chunk_loop
, parent
);
11252 parent
= chunk_loop
;
11254 else if (parent
!= body
)
11256 gcc_assert (body
->header
== body_bb
);
11257 gcc_assert (body
->latch
== cont_bb
11258 || single_pred (body
->latch
) == cont_bb
);
11264 struct loop
*body_loop
= alloc_loop ();
11265 body_loop
->header
= body_bb
;
11266 body_loop
->latch
= cont_bb
;
11267 add_loop (body_loop
, parent
);
11272 /* Expand the OMP loop defined by REGION. */
11275 expand_omp_for (struct omp_region
*region
, gimple
*inner_stmt
)
11277 struct omp_for_data fd
;
11278 struct omp_for_data_loop
*loops
;
11281 = (struct omp_for_data_loop
*)
11282 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
11283 * sizeof (struct omp_for_data_loop
));
11284 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
11286 region
->sched_kind
= fd
.sched_kind
;
11287 region
->sched_modifiers
= fd
.sched_modifiers
;
11289 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
11290 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11291 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
11294 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
11295 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11296 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
11299 /* If there isn't a continue then this is a degerate case where
11300 the introduction of abnormal edges during lowering will prevent
11301 original loops from being detected. Fix that up. */
11302 loops_state_set (LOOPS_NEED_FIXUP
);
11304 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
11305 expand_omp_simd (region
, &fd
);
11306 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
11307 expand_cilk_for (region
, &fd
);
11308 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
11310 gcc_assert (!inner_stmt
);
11311 expand_oacc_for (region
, &fd
);
11313 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_TASKLOOP
)
11315 if (gimple_omp_for_combined_into_p (fd
.for_stmt
))
11316 expand_omp_taskloop_for_inner (region
, &fd
, inner_stmt
);
11318 expand_omp_taskloop_for_outer (region
, &fd
, inner_stmt
);
11320 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
11321 && !fd
.have_ordered
)
11323 if (fd
.chunk_size
== NULL
)
11324 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
11326 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
11330 int fn_index
, start_ix
, next_ix
;
11332 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
11333 == GF_OMP_FOR_KIND_FOR
);
11334 if (fd
.chunk_size
== NULL
11335 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
11336 fd
.chunk_size
= integer_zero_node
;
11337 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
11338 switch (fd
.sched_kind
)
11340 case OMP_CLAUSE_SCHEDULE_RUNTIME
:
11343 case OMP_CLAUSE_SCHEDULE_DYNAMIC
:
11344 case OMP_CLAUSE_SCHEDULE_GUIDED
:
11345 if ((fd
.sched_modifiers
& OMP_CLAUSE_SCHEDULE_NONMONOTONIC
)
11347 && !fd
.have_ordered
)
11349 fn_index
= 3 + fd
.sched_kind
;
11354 fn_index
= fd
.sched_kind
;
11358 fn_index
+= fd
.have_ordered
* 6;
11360 start_ix
= ((int)BUILT_IN_GOMP_LOOP_DOACROSS_STATIC_START
) + fn_index
;
11362 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
11363 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
11364 if (fd
.iter_type
== long_long_unsigned_type_node
)
11366 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
11367 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
11368 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
11369 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
11371 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
11372 (enum built_in_function
) next_ix
, inner_stmt
);
11375 if (gimple_in_ssa_p (cfun
))
11376 update_ssa (TODO_update_ssa_only_virtuals
);
11380 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
11382 v = GOMP_sections_start (n);
11399 v = GOMP_sections_next ();
11404 If this is a combined parallel sections, replace the call to
11405 GOMP_sections_start with call to GOMP_sections_next. */
11408 expand_omp_sections (struct omp_region
*region
)
11410 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
11412 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
11413 gimple_stmt_iterator si
, switch_si
;
11414 gomp_sections
*sections_stmt
;
11416 gomp_continue
*cont
;
11419 struct omp_region
*inner
;
11421 bool exit_reachable
= region
->cont
!= NULL
;
11423 gcc_assert (region
->exit
!= NULL
);
11424 entry_bb
= region
->entry
;
11425 l0_bb
= single_succ (entry_bb
);
11426 l1_bb
= region
->cont
;
11427 l2_bb
= region
->exit
;
11428 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
11429 l2
= gimple_block_label (l2_bb
);
11432 /* This can happen if there are reductions. */
11433 len
= EDGE_COUNT (l0_bb
->succs
);
11434 gcc_assert (len
> 0);
11435 e
= EDGE_SUCC (l0_bb
, len
- 1);
11436 si
= gsi_last_bb (e
->dest
);
11439 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11440 l2
= gimple_block_label (e
->dest
);
11442 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
11444 si
= gsi_last_bb (e
->dest
);
11446 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
11448 l2
= gimple_block_label (e
->dest
);
11453 if (exit_reachable
)
11454 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
11456 default_bb
= create_empty_bb (l0_bb
);
11458 /* We will build a switch() with enough cases for all the
11459 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
11460 and a default case to abort if something goes wrong. */
11461 len
= EDGE_COUNT (l0_bb
->succs
);
11463 /* Use vec::quick_push on label_vec throughout, since we know the size
11465 auto_vec
<tree
> label_vec (len
);
11467 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
11468 GIMPLE_OMP_SECTIONS statement. */
11469 si
= gsi_last_bb (entry_bb
);
11470 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
11471 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
11472 vin
= gimple_omp_sections_control (sections_stmt
);
11473 if (!is_combined_parallel (region
))
11475 /* If we are not inside a combined parallel+sections region,
11476 call GOMP_sections_start. */
11477 t
= build_int_cst (unsigned_type_node
, len
- 1);
11478 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
11479 stmt
= gimple_build_call (u
, 1, t
);
11483 /* Otherwise, call GOMP_sections_next. */
11484 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11485 stmt
= gimple_build_call (u
, 0);
11487 gimple_call_set_lhs (stmt
, vin
);
11488 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11489 gsi_remove (&si
, true);
11491 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
11493 switch_si
= gsi_last_bb (l0_bb
);
11494 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
11495 if (exit_reachable
)
11497 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
11498 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
11499 vmain
= gimple_omp_continue_control_use (cont
);
11500 vnext
= gimple_omp_continue_control_def (cont
);
11508 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
11509 label_vec
.quick_push (t
);
11512 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
11513 for (inner
= region
->inner
, casei
= 1;
11515 inner
= inner
->next
, i
++, casei
++)
11517 basic_block s_entry_bb
, s_exit_bb
;
11519 /* Skip optional reduction region. */
11520 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
11527 s_entry_bb
= inner
->entry
;
11528 s_exit_bb
= inner
->exit
;
11530 t
= gimple_block_label (s_entry_bb
);
11531 u
= build_int_cst (unsigned_type_node
, casei
);
11532 u
= build_case_label (u
, NULL
, t
);
11533 label_vec
.quick_push (u
);
11535 si
= gsi_last_bb (s_entry_bb
);
11536 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
11537 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
11538 gsi_remove (&si
, true);
11539 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
11541 if (s_exit_bb
== NULL
)
11544 si
= gsi_last_bb (s_exit_bb
);
11545 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11546 gsi_remove (&si
, true);
11548 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
11551 /* Error handling code goes in DEFAULT_BB. */
11552 t
= gimple_block_label (default_bb
);
11553 u
= build_case_label (NULL
, NULL
, t
);
11554 make_edge (l0_bb
, default_bb
, 0);
11555 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
11557 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
11558 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
11559 gsi_remove (&switch_si
, true);
11561 si
= gsi_start_bb (default_bb
);
11562 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
11563 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
11565 if (exit_reachable
)
11569 /* Code to get the next section goes in L1_BB. */
11570 si
= gsi_last_bb (l1_bb
);
11571 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
11573 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
11574 stmt
= gimple_build_call (bfn_decl
, 0);
11575 gimple_call_set_lhs (stmt
, vnext
);
11576 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11577 gsi_remove (&si
, true);
11579 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
11582 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
11583 si
= gsi_last_bb (l2_bb
);
11584 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
11585 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
11586 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
11587 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
11589 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
11590 stmt
= gimple_build_call (t
, 0);
11591 if (gimple_omp_return_lhs (gsi_stmt (si
)))
11592 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
11593 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
11594 gsi_remove (&si
, true);
11596 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
11600 /* Expand code for an OpenMP single directive. We've already expanded
11601 much of the code, here we simply place the GOMP_barrier call. */
11604 expand_omp_single (struct omp_region
*region
)
11606 basic_block entry_bb
, exit_bb
;
11607 gimple_stmt_iterator si
;
11609 entry_bb
= region
->entry
;
11610 exit_bb
= region
->exit
;
11612 si
= gsi_last_bb (entry_bb
);
11613 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
11614 gsi_remove (&si
, true);
11615 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11617 si
= gsi_last_bb (exit_bb
);
11618 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
11620 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
11621 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
11623 gsi_remove (&si
, true);
11624 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11628 /* Generic expansion for OpenMP synchronization directives: master,
11629 ordered and critical. All we need to do here is remove the entry
11630 and exit markers for REGION. */
11633 expand_omp_synch (struct omp_region
*region
)
11635 basic_block entry_bb
, exit_bb
;
11636 gimple_stmt_iterator si
;
11638 entry_bb
= region
->entry
;
11639 exit_bb
= region
->exit
;
11641 si
= gsi_last_bb (entry_bb
);
11642 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
11643 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
11644 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
11645 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
11646 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
11647 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
11648 gsi_remove (&si
, true);
11649 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
11653 si
= gsi_last_bb (exit_bb
);
11654 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
11655 gsi_remove (&si
, true);
11656 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
11660 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11661 operation as a normal volatile load. */
11664 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
11665 tree loaded_val
, int index
)
11667 enum built_in_function tmpbase
;
11668 gimple_stmt_iterator gsi
;
11669 basic_block store_bb
;
11672 tree decl
, call
, type
, itype
;
11674 gsi
= gsi_last_bb (load_bb
);
11675 stmt
= gsi_stmt (gsi
);
11676 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11677 loc
= gimple_location (stmt
);
11679 /* ??? If the target does not implement atomic_load_optab[mode], and mode
11680 is smaller than word size, then expand_atomic_load assumes that the load
11681 is atomic. We could avoid the builtin entirely in this case. */
11683 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
11684 decl
= builtin_decl_explicit (tmpbase
);
11685 if (decl
== NULL_TREE
)
11688 type
= TREE_TYPE (loaded_val
);
11689 itype
= TREE_TYPE (TREE_TYPE (decl
));
11691 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
11692 build_int_cst (NULL
,
11693 gimple_omp_atomic_seq_cst_p (stmt
)
11695 : MEMMODEL_RELAXED
));
11696 if (!useless_type_conversion_p (type
, itype
))
11697 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
11698 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
11700 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11701 gsi_remove (&gsi
, true);
11703 store_bb
= single_succ (load_bb
);
11704 gsi
= gsi_last_bb (store_bb
);
11705 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
11706 gsi_remove (&gsi
, true);
11708 if (gimple_in_ssa_p (cfun
))
11709 update_ssa (TODO_update_ssa_no_phi
);
11714 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11715 operation as a normal volatile store. */
11718 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
11719 tree loaded_val
, tree stored_val
, int index
)
11721 enum built_in_function tmpbase
;
11722 gimple_stmt_iterator gsi
;
11723 basic_block store_bb
= single_succ (load_bb
);
11726 tree decl
, call
, type
, itype
;
11727 machine_mode imode
;
11730 gsi
= gsi_last_bb (load_bb
);
11731 stmt
= gsi_stmt (gsi
);
11732 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
11734 /* If the load value is needed, then this isn't a store but an exchange. */
11735 exchange
= gimple_omp_atomic_need_value_p (stmt
);
11737 gsi
= gsi_last_bb (store_bb
);
11738 stmt
= gsi_stmt (gsi
);
11739 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
11740 loc
= gimple_location (stmt
);
11742 /* ??? If the target does not implement atomic_store_optab[mode], and mode
11743 is smaller than word size, then expand_atomic_store assumes that the store
11744 is atomic. We could avoid the builtin entirely in this case. */
11746 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
11747 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
11748 decl
= builtin_decl_explicit (tmpbase
);
11749 if (decl
== NULL_TREE
)
11752 type
= TREE_TYPE (stored_val
);
11754 /* Dig out the type of the function's second argument. */
11755 itype
= TREE_TYPE (decl
);
11756 itype
= TYPE_ARG_TYPES (itype
);
11757 itype
= TREE_CHAIN (itype
);
11758 itype
= TREE_VALUE (itype
);
11759 imode
= TYPE_MODE (itype
);
11761 if (exchange
&& !can_atomic_exchange_p (imode
, true))
11764 if (!useless_type_conversion_p (itype
, type
))
11765 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
11766 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
11767 build_int_cst (NULL
,
11768 gimple_omp_atomic_seq_cst_p (stmt
)
11770 : MEMMODEL_RELAXED
));
11773 if (!useless_type_conversion_p (type
, itype
))
11774 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
11775 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
11778 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11779 gsi_remove (&gsi
, true);
11781 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
11782 gsi
= gsi_last_bb (load_bb
);
11783 gsi_remove (&gsi
, true);
11785 if (gimple_in_ssa_p (cfun
))
11786 update_ssa (TODO_update_ssa_no_phi
);
11791 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
11792 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
11793 size of the data type, and thus usable to find the index of the builtin
11794 decl. Returns false if the expression is not of the proper form. */
11797 expand_omp_atomic_fetch_op (basic_block load_bb
,
11798 tree addr
, tree loaded_val
,
11799 tree stored_val
, int index
)
11801 enum built_in_function oldbase
, newbase
, tmpbase
;
11802 tree decl
, itype
, call
;
11804 basic_block store_bb
= single_succ (load_bb
);
11805 gimple_stmt_iterator gsi
;
11808 enum tree_code code
;
11809 bool need_old
, need_new
;
11810 machine_mode imode
;
11813 /* We expect to find the following sequences:
11816 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
11819 val = tmp OP something; (or: something OP tmp)
11820 GIMPLE_OMP_STORE (val)
11822 ???FIXME: Allow a more flexible sequence.
11823 Perhaps use data flow to pick the statements.
11827 gsi
= gsi_after_labels (store_bb
);
11828 stmt
= gsi_stmt (gsi
);
11829 loc
= gimple_location (stmt
);
11830 if (!is_gimple_assign (stmt
))
11833 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
11835 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
11836 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
11837 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
11838 gcc_checking_assert (!need_old
|| !need_new
);
11840 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
11843 /* Check for one of the supported fetch-op operations. */
11844 code
= gimple_assign_rhs_code (stmt
);
11848 case POINTER_PLUS_EXPR
:
11849 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
11850 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
11853 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
11854 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
11857 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
11858 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
11861 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
11862 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
11865 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
11866 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
11872 /* Make sure the expression is of the proper form. */
11873 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
11874 rhs
= gimple_assign_rhs2 (stmt
);
11875 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
11876 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
11877 rhs
= gimple_assign_rhs1 (stmt
);
11881 tmpbase
= ((enum built_in_function
)
11882 ((need_new
? newbase
: oldbase
) + index
+ 1));
11883 decl
= builtin_decl_explicit (tmpbase
);
11884 if (decl
== NULL_TREE
)
11886 itype
= TREE_TYPE (TREE_TYPE (decl
));
11887 imode
= TYPE_MODE (itype
);
11889 /* We could test all of the various optabs involved, but the fact of the
11890 matter is that (with the exception of i486 vs i586 and xadd) all targets
11891 that support any atomic operaton optab also implements compare-and-swap.
11892 Let optabs.c take care of expanding any compare-and-swap loop. */
11893 if (!can_compare_and_swap_p (imode
, true))
11896 gsi
= gsi_last_bb (load_bb
);
11897 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
11899 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
11900 It only requires that the operation happen atomically. Thus we can
11901 use the RELAXED memory model. */
11902 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
11903 fold_convert_loc (loc
, itype
, rhs
),
11904 build_int_cst (NULL
,
11905 seq_cst
? MEMMODEL_SEQ_CST
11906 : MEMMODEL_RELAXED
));
11908 if (need_old
|| need_new
)
11910 lhs
= need_old
? loaded_val
: stored_val
;
11911 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
11912 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
11915 call
= fold_convert_loc (loc
, void_type_node
, call
);
11916 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
11917 gsi_remove (&gsi
, true);
11919 gsi
= gsi_last_bb (store_bb
);
11920 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
11921 gsi_remove (&gsi
, true);
11922 gsi
= gsi_last_bb (store_bb
);
11923 gsi_remove (&gsi
, true);
11925 if (gimple_in_ssa_p (cfun
))
11926 update_ssa (TODO_update_ssa_no_phi
);
11931 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
11935 newval = rhs; // with oldval replacing *addr in rhs
11936 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
11937 if (oldval != newval)
11940 INDEX is log2 of the size of the data type, and thus usable to find the
11941 index of the builtin decl. */
11944 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
11945 tree addr
, tree loaded_val
, tree stored_val
,
11948 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
11949 tree type
, itype
, cmpxchg
, iaddr
;
11950 gimple_stmt_iterator si
;
11951 basic_block loop_header
= single_succ (load_bb
);
11952 gimple
*phi
, *stmt
;
11954 enum built_in_function fncode
;
11956 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
11957 order to use the RELAXED memory model effectively. */
11958 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
11960 cmpxchg
= builtin_decl_explicit (fncode
);
11961 if (cmpxchg
== NULL_TREE
)
11963 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
11964 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
11966 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
11969 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
11970 si
= gsi_last_bb (load_bb
);
11971 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
11973 /* For floating-point values, we'll need to view-convert them to integers
11974 so that we can perform the atomic compare and swap. Simplify the
11975 following code by always setting up the "i"ntegral variables. */
11976 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
11980 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
11983 = force_gimple_operand_gsi (&si
,
11984 fold_convert (TREE_TYPE (iaddr
), addr
),
11985 false, NULL_TREE
, true, GSI_SAME_STMT
);
11986 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
11987 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
11988 loadedi
= create_tmp_var (itype
);
11989 if (gimple_in_ssa_p (cfun
))
11990 loadedi
= make_ssa_name (loadedi
);
11995 loadedi
= loaded_val
;
11998 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
11999 tree loaddecl
= builtin_decl_explicit (fncode
);
12002 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
12003 build_call_expr (loaddecl
, 2, iaddr
,
12004 build_int_cst (NULL_TREE
,
12005 MEMMODEL_RELAXED
)));
12007 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
12008 build_int_cst (TREE_TYPE (iaddr
), 0));
12011 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
12014 /* Move the value to the LOADEDI temporary. */
12015 if (gimple_in_ssa_p (cfun
))
12017 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
12018 phi
= create_phi_node (loadedi
, loop_header
);
12019 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
12023 gsi_insert_before (&si
,
12024 gimple_build_assign (loadedi
, initial
),
12026 if (loadedi
!= loaded_val
)
12028 gimple_stmt_iterator gsi2
;
12031 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
12032 gsi2
= gsi_start_bb (loop_header
);
12033 if (gimple_in_ssa_p (cfun
))
12036 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
12037 true, GSI_SAME_STMT
);
12038 stmt
= gimple_build_assign (loaded_val
, x
);
12039 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
12043 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
12044 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
12045 true, GSI_SAME_STMT
);
12048 gsi_remove (&si
, true);
12050 si
= gsi_last_bb (store_bb
);
12051 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12054 storedi
= stored_val
;
12057 force_gimple_operand_gsi (&si
,
12058 build1 (VIEW_CONVERT_EXPR
, itype
,
12059 stored_val
), true, NULL_TREE
, true,
12062 /* Build the compare&swap statement. */
12063 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
12064 new_storedi
= force_gimple_operand_gsi (&si
,
12065 fold_convert (TREE_TYPE (loadedi
),
12068 true, GSI_SAME_STMT
);
12070 if (gimple_in_ssa_p (cfun
))
12071 old_vali
= loadedi
;
12074 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
12075 stmt
= gimple_build_assign (old_vali
, loadedi
);
12076 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12078 stmt
= gimple_build_assign (loadedi
, new_storedi
);
12079 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12082 /* Note that we always perform the comparison as an integer, even for
12083 floating point. This allows the atomic operation to properly
12084 succeed even with NaNs and -0.0. */
12085 stmt
= gimple_build_cond_empty
12086 (build2 (NE_EXPR
, boolean_type_node
,
12087 new_storedi
, old_vali
));
12088 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12091 e
= single_succ_edge (store_bb
);
12092 e
->flags
&= ~EDGE_FALLTHRU
;
12093 e
->flags
|= EDGE_FALSE_VALUE
;
12095 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
12097 /* Copy the new value to loadedi (we already did that before the condition
12098 if we are not in SSA). */
12099 if (gimple_in_ssa_p (cfun
))
12101 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
12102 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
12105 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
12106 gsi_remove (&si
, true);
12108 struct loop
*loop
= alloc_loop ();
12109 loop
->header
= loop_header
;
12110 loop
->latch
= store_bb
;
12111 add_loop (loop
, loop_header
->loop_father
);
12113 if (gimple_in_ssa_p (cfun
))
12114 update_ssa (TODO_update_ssa_no_phi
);
12119 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
12121 GOMP_atomic_start ();
12123 GOMP_atomic_end ();
12125 The result is not globally atomic, but works so long as all parallel
12126 references are within #pragma omp atomic directives. According to
12127 responses received from omp@openmp.org, appears to be within spec.
12128 Which makes sense, since that's how several other compilers handle
12129 this situation as well.
12130 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
12131 expanding. STORED_VAL is the operand of the matching
12132 GIMPLE_OMP_ATOMIC_STORE.
12135 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
12136 loaded_val = *addr;
12139 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
12140 *addr = stored_val;
12144 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
12145 tree addr
, tree loaded_val
, tree stored_val
)
12147 gimple_stmt_iterator si
;
12151 si
= gsi_last_bb (load_bb
);
12152 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
12154 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
12155 t
= build_call_expr (t
, 0);
12156 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12158 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
12159 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12160 gsi_remove (&si
, true);
12162 si
= gsi_last_bb (store_bb
);
12163 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
12165 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
12167 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
12169 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
12170 t
= build_call_expr (t
, 0);
12171 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
12172 gsi_remove (&si
, true);
12174 if (gimple_in_ssa_p (cfun
))
12175 update_ssa (TODO_update_ssa_no_phi
);
12179 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
12180 using expand_omp_atomic_fetch_op. If it failed, we try to
12181 call expand_omp_atomic_pipeline, and if it fails too, the
12182 ultimate fallback is wrapping the operation in a mutex
12183 (expand_omp_atomic_mutex). REGION is the atomic region built
12184 by build_omp_regions_1(). */
12187 expand_omp_atomic (struct omp_region
*region
)
12189 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
12190 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
12191 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
12192 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
12193 tree addr
= gimple_omp_atomic_load_rhs (load
);
12194 tree stored_val
= gimple_omp_atomic_store_val (store
);
12195 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
12196 HOST_WIDE_INT index
;
12198 /* Make sure the type is one of the supported sizes. */
12199 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
12200 index
= exact_log2 (index
);
12201 if (index
>= 0 && index
<= 4)
12203 unsigned int align
= TYPE_ALIGN_UNIT (type
);
12205 /* __sync builtins require strict data alignment. */
12206 if (exact_log2 (align
) >= index
)
12209 if (loaded_val
== stored_val
12210 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12211 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12212 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12213 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
12216 /* Atomic store. */
12217 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
12218 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
12219 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
12220 && store_bb
== single_succ (load_bb
)
12221 && first_stmt (store_bb
) == store
12222 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
12223 stored_val
, index
))
12226 /* When possible, use specialized atomic update functions. */
12227 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
12228 && store_bb
== single_succ (load_bb
)
12229 && expand_omp_atomic_fetch_op (load_bb
, addr
,
12230 loaded_val
, stored_val
, index
))
12233 /* If we don't have specialized __sync builtins, try and implement
12234 as a compare and swap loop. */
12235 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
12236 loaded_val
, stored_val
, index
))
12241 /* The ultimate fallback is wrapping the operation in a mutex. */
12242 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
12246 /* Encode an oacc launch argument. This matches the GOMP_LAUNCH_PACK
12247 macro on gomp-constants.h. We do not check for overflow. */
12250 oacc_launch_pack (unsigned code
, tree device
, unsigned op
)
12254 res
= build_int_cst (unsigned_type_node
, GOMP_LAUNCH_PACK (code
, 0, op
));
12257 device
= fold_build2 (LSHIFT_EXPR
, unsigned_type_node
,
12258 device
, build_int_cst (unsigned_type_node
,
12259 GOMP_LAUNCH_DEVICE_SHIFT
));
12260 res
= fold_build2 (BIT_IOR_EXPR
, unsigned_type_node
, res
, device
);
12265 /* Look for compute grid dimension clauses and convert to an attribute
12266 attached to FN. This permits the target-side code to (a) massage
12267 the dimensions, (b) emit that data and (c) optimize. Non-constant
12268 dimensions are pushed onto ARGS.
12270 The attribute value is a TREE_LIST. A set of dimensions is
12271 represented as a list of INTEGER_CST. Those that are runtime
12272 exprs are represented as an INTEGER_CST of zero.
12274 TOOO. Normally the attribute will just contain a single such list. If
12275 however it contains a list of lists, this will represent the use of
12276 device_type. Each member of the outer list is an assoc list of
12277 dimensions, keyed by the device type. The first entry will be the
12278 default. Well, that's the plan. */
12280 #define OACC_FN_ATTRIB "oacc function"
12282 /* Replace any existing oacc fn attribute with updated dimensions. */
12285 replace_oacc_fn_attrib (tree fn
, tree dims
)
12287 tree ident
= get_identifier (OACC_FN_ATTRIB
);
12288 tree attribs
= DECL_ATTRIBUTES (fn
);
12290 /* If we happen to be present as the first attrib, drop it. */
12291 if (attribs
&& TREE_PURPOSE (attribs
) == ident
)
12292 attribs
= TREE_CHAIN (attribs
);
12293 DECL_ATTRIBUTES (fn
) = tree_cons (ident
, dims
, attribs
);
12296 /* Scan CLAUSES for launch dimensions and attach them to the oacc
12297 function attribute. Push any that are non-constant onto the ARGS
12298 list, along with an appropriate GOMP_LAUNCH_DIM tag. */
12301 set_oacc_fn_attrib (tree fn
, tree clauses
, vec
<tree
> *args
)
12303 /* Must match GOMP_DIM ordering. */
12304 static const omp_clause_code ids
[]
12305 = { OMP_CLAUSE_NUM_GANGS
, OMP_CLAUSE_NUM_WORKERS
,
12306 OMP_CLAUSE_VECTOR_LENGTH
};
12308 tree dims
[GOMP_DIM_MAX
];
12309 tree attr
= NULL_TREE
;
12310 unsigned non_const
= 0;
12312 for (ix
= GOMP_DIM_MAX
; ix
--;)
12314 tree clause
= find_omp_clause (clauses
, ids
[ix
]);
12315 tree dim
= NULL_TREE
;
12318 dim
= OMP_CLAUSE_EXPR (clause
, ids
[ix
]);
12320 if (dim
&& TREE_CODE (dim
) != INTEGER_CST
)
12322 dim
= integer_zero_node
;
12323 non_const
|= GOMP_DIM_MASK (ix
);
12325 attr
= tree_cons (NULL_TREE
, dim
, attr
);
12328 replace_oacc_fn_attrib (fn
, attr
);
12332 /* Push a dynamic argument set. */
12333 args
->safe_push (oacc_launch_pack (GOMP_LAUNCH_DIM
,
12334 NULL_TREE
, non_const
));
12335 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
12336 if (non_const
& GOMP_DIM_MASK (ix
))
12337 args
->safe_push (dims
[ix
]);
12341 /* Process the routine's dimension clauess to generate an attribute
12342 value. Issue diagnostics as appropriate. We default to SEQ
12343 (OpenACC 2.5 clarifies this). All dimensions have a size of zero
12344 (dynamic). TREE_PURPOSE is set to indicate whether that dimension
12345 can have a loop partitioned on it. non-zero indicates
12346 yes, zero indicates no. By construction once a non-zero has been
12347 reached, further inner dimensions must also be non-zero. We set
12348 TREE_VALUE to zero for the dimensions that may be partitioned and
12349 1 for the other ones -- if a loop is (erroneously) spawned at
12350 an outer level, we don't want to try and partition it. */
12353 build_oacc_routine_dims (tree clauses
)
12355 /* Must match GOMP_DIM ordering. */
12356 static const omp_clause_code ids
[] =
12357 {OMP_CLAUSE_GANG
, OMP_CLAUSE_WORKER
, OMP_CLAUSE_VECTOR
, OMP_CLAUSE_SEQ
};
12361 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
12362 for (ix
= GOMP_DIM_MAX
+ 1; ix
--;)
12363 if (OMP_CLAUSE_CODE (clauses
) == ids
[ix
])
12366 error_at (OMP_CLAUSE_LOCATION (clauses
),
12367 "multiple loop axes specified for routine");
12372 /* Default to SEQ. */
12374 level
= GOMP_DIM_MAX
;
12376 tree dims
= NULL_TREE
;
12378 for (ix
= GOMP_DIM_MAX
; ix
--;)
12379 dims
= tree_cons (build_int_cst (boolean_type_node
, ix
>= level
),
12380 build_int_cst (integer_type_node
, ix
< level
), dims
);
12385 /* Retrieve the oacc function attrib and return it. Non-oacc
12386 functions will return NULL. */
12389 get_oacc_fn_attrib (tree fn
)
12391 return lookup_attribute (OACC_FN_ATTRIB
, DECL_ATTRIBUTES (fn
));
12394 /* Extract an oacc execution dimension from FN. FN must be an
12395 offloaded function or routine that has already had its execution
12396 dimensions lowered to the target-specific values. */
12399 get_oacc_fn_dim_size (tree fn
, int axis
)
12401 tree attrs
= get_oacc_fn_attrib (fn
);
12403 gcc_assert (axis
< GOMP_DIM_MAX
);
12405 tree dims
= TREE_VALUE (attrs
);
12407 dims
= TREE_CHAIN (dims
);
12409 int size
= TREE_INT_CST_LOW (TREE_VALUE (dims
));
12414 /* Extract the dimension axis from an IFN_GOACC_DIM_POS or
12415 IFN_GOACC_DIM_SIZE call. */
12418 get_oacc_ifn_dim_arg (const gimple
*stmt
)
12420 gcc_checking_assert (gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_SIZE
12421 || gimple_call_internal_fn (stmt
) == IFN_GOACC_DIM_POS
);
12422 tree arg
= gimple_call_arg (stmt
, 0);
12423 HOST_WIDE_INT axis
= TREE_INT_CST_LOW (arg
);
12425 gcc_checking_assert (axis
>= 0 && axis
< GOMP_DIM_MAX
);
12429 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
12432 expand_omp_target (struct omp_region
*region
)
12434 basic_block entry_bb
, exit_bb
, new_bb
;
12435 struct function
*child_cfun
;
12436 tree child_fn
, block
, t
;
12437 gimple_stmt_iterator gsi
;
12438 gomp_target
*entry_stmt
;
12441 bool offloaded
, data_region
;
12443 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
12444 new_bb
= region
->entry
;
12446 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
12447 switch (gimple_omp_target_kind (entry_stmt
))
12449 case GF_OMP_TARGET_KIND_REGION
:
12450 case GF_OMP_TARGET_KIND_UPDATE
:
12451 case GF_OMP_TARGET_KIND_ENTER_DATA
:
12452 case GF_OMP_TARGET_KIND_EXIT_DATA
:
12453 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12454 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12455 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12456 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12457 data_region
= false;
12459 case GF_OMP_TARGET_KIND_DATA
:
12460 case GF_OMP_TARGET_KIND_OACC_DATA
:
12461 data_region
= true;
12464 gcc_unreachable ();
12467 child_fn
= NULL_TREE
;
12471 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
12472 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
12475 /* Supported by expand_omp_taskreg, but not here. */
12476 if (child_cfun
!= NULL
)
12477 gcc_checking_assert (!child_cfun
->cfg
);
12478 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
12480 entry_bb
= region
->entry
;
12481 exit_bb
= region
->exit
;
12485 unsigned srcidx
, dstidx
, num
;
12487 /* If the offloading region needs data sent from the parent
12488 function, then the very first statement (except possible
12489 tree profile counter updates) of the offloading body
12490 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
12491 &.OMP_DATA_O is passed as an argument to the child function,
12492 we need to replace it with the argument as seen by the child
12495 In most cases, this will end up being the identity assignment
12496 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
12497 a function call that has been inlined, the original PARM_DECL
12498 .OMP_DATA_I may have been converted into a different local
12499 variable. In which case, we need to keep the assignment. */
12500 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
12503 basic_block entry_succ_bb
= single_succ (entry_bb
);
12504 gimple_stmt_iterator gsi
;
12506 gimple
*tgtcopy_stmt
= NULL
;
12507 tree sender
= TREE_VEC_ELT (data_arg
, 0);
12509 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
12511 gcc_assert (!gsi_end_p (gsi
));
12512 stmt
= gsi_stmt (gsi
);
12513 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
12516 if (gimple_num_ops (stmt
) == 2)
12518 tree arg
= gimple_assign_rhs1 (stmt
);
12520 /* We're ignoring the subcode because we're
12521 effectively doing a STRIP_NOPS. */
12523 if (TREE_CODE (arg
) == ADDR_EXPR
12524 && TREE_OPERAND (arg
, 0) == sender
)
12526 tgtcopy_stmt
= stmt
;
12532 gcc_assert (tgtcopy_stmt
!= NULL
);
12533 arg
= DECL_ARGUMENTS (child_fn
);
12535 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
12536 gsi_remove (&gsi
, true);
12539 /* Declare local variables needed in CHILD_CFUN. */
12540 block
= DECL_INITIAL (child_fn
);
12541 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
12542 /* The gimplifier could record temporaries in the offloading block
12543 rather than in containing function's local_decls chain,
12544 which would mean cgraph missed finalizing them. Do it now. */
12545 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
12546 if (TREE_CODE (t
) == VAR_DECL
12548 && !DECL_EXTERNAL (t
))
12549 varpool_node::finalize_decl (t
);
12550 DECL_SAVED_TREE (child_fn
) = NULL
;
12551 /* We'll create a CFG for child_fn, so no gimple body is needed. */
12552 gimple_set_body (child_fn
, NULL
);
12553 TREE_USED (block
) = 1;
12555 /* Reset DECL_CONTEXT on function arguments. */
12556 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
12557 DECL_CONTEXT (t
) = child_fn
;
12559 /* Split ENTRY_BB at GIMPLE_*,
12560 so that it can be moved to the child function. */
12561 gsi
= gsi_last_bb (entry_bb
);
12562 stmt
= gsi_stmt (gsi
);
12564 && gimple_code (stmt
) == gimple_code (entry_stmt
));
12565 e
= split_block (entry_bb
, stmt
);
12566 gsi_remove (&gsi
, true);
12567 entry_bb
= e
->dest
;
12568 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
12570 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
12573 gsi
= gsi_last_bb (exit_bb
);
12574 gcc_assert (!gsi_end_p (gsi
)
12575 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
12576 stmt
= gimple_build_return (NULL
);
12577 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
12578 gsi_remove (&gsi
, true);
12581 /* Move the offloading region into CHILD_CFUN. */
12583 block
= gimple_block (entry_stmt
);
12585 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
12587 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
12588 /* When the OMP expansion process cannot guarantee an up-to-date
12589 loop tree arrange for the child function to fixup loops. */
12590 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
12591 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
12593 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
12594 num
= vec_safe_length (child_cfun
->local_decls
);
12595 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
12597 t
= (*child_cfun
->local_decls
)[srcidx
];
12598 if (DECL_CONTEXT (t
) == cfun
->decl
)
12600 if (srcidx
!= dstidx
)
12601 (*child_cfun
->local_decls
)[dstidx
] = t
;
12605 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
12607 /* Inform the callgraph about the new function. */
12608 child_cfun
->curr_properties
= cfun
->curr_properties
;
12609 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
12610 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
12611 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
12612 node
->parallelized_function
= 1;
12613 cgraph_node::add_new_function (child_fn
, true);
12615 #ifdef ENABLE_OFFLOADING
12616 /* Add the new function to the offload table. */
12617 vec_safe_push (offload_funcs
, child_fn
);
12620 bool need_asm
= DECL_ASSEMBLER_NAME_SET_P (current_function_decl
)
12621 && !DECL_ASSEMBLER_NAME_SET_P (child_fn
);
12623 /* Fix the callgraph edges for child_cfun. Those for cfun will be
12624 fixed in a following pass. */
12625 push_cfun (child_cfun
);
12627 assign_assembler_name_if_neeeded (child_fn
);
12628 cgraph_edge::rebuild_edges ();
12630 #ifdef ENABLE_OFFLOADING
12631 /* Prevent IPA from removing child_fn as unreachable, since there are no
12632 refs from the parent function to child_fn in offload LTO mode. */
12633 cgraph_node::get (child_fn
)->mark_force_output ();
12636 /* Some EH regions might become dead, see PR34608. If
12637 pass_cleanup_cfg isn't the first pass to happen with the
12638 new child, these dead EH edges might cause problems.
12639 Clean them up now. */
12640 if (flag_exceptions
)
12643 bool changed
= false;
12645 FOR_EACH_BB_FN (bb
, cfun
)
12646 changed
|= gimple_purge_dead_eh_edges (bb
);
12648 cleanup_tree_cfg ();
12650 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
12651 verify_loop_structure ();
12654 if (dump_file
&& !gimple_in_ssa_p (cfun
))
12656 omp_any_child_fn_dumped
= true;
12657 dump_function_header (dump_file
, child_fn
, dump_flags
);
12658 dump_function_to_file (child_fn
, dump_file
, dump_flags
);
12662 /* Emit a library call to launch the offloading region, or do data
12664 tree t1
, t2
, t3
, t4
, device
, cond
, depend
, c
, clauses
;
12665 enum built_in_function start_ix
;
12666 location_t clause_loc
;
12667 unsigned int flags_i
= 0;
12669 switch (gimple_omp_target_kind (entry_stmt
))
12671 case GF_OMP_TARGET_KIND_REGION
:
12672 start_ix
= BUILT_IN_GOMP_TARGET
;
12674 case GF_OMP_TARGET_KIND_DATA
:
12675 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
12677 case GF_OMP_TARGET_KIND_UPDATE
:
12678 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
12680 case GF_OMP_TARGET_KIND_ENTER_DATA
:
12681 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
12683 case GF_OMP_TARGET_KIND_EXIT_DATA
:
12684 start_ix
= BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
;
12685 flags_i
|= GOMP_TARGET_FLAG_EXIT_DATA
;
12687 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12688 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12689 start_ix
= BUILT_IN_GOACC_PARALLEL
;
12691 case GF_OMP_TARGET_KIND_OACC_DATA
:
12692 start_ix
= BUILT_IN_GOACC_DATA_START
;
12694 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12695 start_ix
= BUILT_IN_GOACC_UPDATE
;
12697 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12698 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
12701 gcc_unreachable ();
12704 clauses
= gimple_omp_target_clauses (entry_stmt
);
12706 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
12707 library choose) and there is no conditional. */
12709 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
12711 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
12713 cond
= OMP_CLAUSE_IF_EXPR (c
);
12715 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
12718 /* Even if we pass it to all library function calls, it is currently only
12719 defined/used for the OpenMP target ones. */
12720 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
12721 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
12722 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
12723 || start_ix
== BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
);
12725 device
= OMP_CLAUSE_DEVICE_ID (c
);
12726 clause_loc
= OMP_CLAUSE_LOCATION (c
);
12729 clause_loc
= gimple_location (entry_stmt
);
12731 c
= find_omp_clause (clauses
, OMP_CLAUSE_NOWAIT
);
12733 flags_i
|= GOMP_TARGET_FLAG_NOWAIT
;
12735 /* Ensure 'device' is of the correct type. */
12736 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
12738 /* If we found the clause 'if (cond)', build
12739 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
12742 cond
= gimple_boolify (cond
);
12744 basic_block cond_bb
, then_bb
, else_bb
;
12748 tmp_var
= create_tmp_var (TREE_TYPE (device
));
12750 e
= split_block_after_labels (new_bb
);
12753 gsi
= gsi_last_bb (new_bb
);
12755 e
= split_block (new_bb
, gsi_stmt (gsi
));
12761 then_bb
= create_empty_bb (cond_bb
);
12762 else_bb
= create_empty_bb (then_bb
);
12763 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
12764 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
12766 stmt
= gimple_build_cond_empty (cond
);
12767 gsi
= gsi_last_bb (cond_bb
);
12768 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
12770 gsi
= gsi_start_bb (then_bb
);
12771 stmt
= gimple_build_assign (tmp_var
, device
);
12772 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
12774 gsi
= gsi_start_bb (else_bb
);
12775 stmt
= gimple_build_assign (tmp_var
,
12776 build_int_cst (integer_type_node
,
12777 GOMP_DEVICE_HOST_FALLBACK
));
12778 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
12780 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
12781 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
12782 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
12783 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
12784 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
12785 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
12790 gsi
= gsi_last_bb (new_bb
);
12791 t
= gimple_omp_target_data_arg (entry_stmt
);
12794 t1
= size_zero_node
;
12795 t2
= build_zero_cst (ptr_type_node
);
12801 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
12802 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
12803 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
12804 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
12805 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
12809 bool tagging
= false;
12810 /* The maximum number used by any start_ix, without varargs. */
12811 auto_vec
<tree
, 11> args
;
12812 args
.quick_push (device
);
12814 args
.quick_push (build_fold_addr_expr (child_fn
));
12815 args
.quick_push (t1
);
12816 args
.quick_push (t2
);
12817 args
.quick_push (t3
);
12818 args
.quick_push (t4
);
12821 case BUILT_IN_GOACC_DATA_START
:
12822 case BUILT_IN_GOMP_TARGET_DATA
:
12824 case BUILT_IN_GOMP_TARGET
:
12825 case BUILT_IN_GOMP_TARGET_UPDATE
:
12826 case BUILT_IN_GOMP_TARGET_ENTER_EXIT_DATA
:
12827 args
.quick_push (build_int_cst (unsigned_type_node
, flags_i
));
12828 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
12830 depend
= OMP_CLAUSE_DECL (c
);
12832 depend
= build_int_cst (ptr_type_node
, 0);
12833 args
.quick_push (depend
);
12834 if (start_ix
== BUILT_IN_GOMP_TARGET
)
12836 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_TEAMS
);
12839 t
= fold_convert (integer_type_node
,
12840 OMP_CLAUSE_NUM_TEAMS_EXPR (c
));
12841 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL
,
12842 true, GSI_SAME_STMT
);
12845 t
= integer_minus_one_node
;
12846 args
.quick_push (t
);
12847 c
= find_omp_clause (clauses
, OMP_CLAUSE_THREAD_LIMIT
);
12850 t
= fold_convert (integer_type_node
,
12851 OMP_CLAUSE_THREAD_LIMIT_EXPR (c
));
12852 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL
,
12853 true, GSI_SAME_STMT
);
12856 t
= integer_minus_one_node
;
12857 args
.quick_push (t
);
12860 case BUILT_IN_GOACC_PARALLEL
:
12862 set_oacc_fn_attrib (child_fn
, clauses
, &args
);
12866 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
12867 case BUILT_IN_GOACC_UPDATE
:
12869 tree t_async
= NULL_TREE
;
12871 /* If present, use the value specified by the respective
12872 clause, making sure that is of the correct type. */
12873 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
12875 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
12877 OMP_CLAUSE_ASYNC_EXPR (c
));
12879 /* Default values for t_async. */
12880 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
12882 build_int_cst (integer_type_node
,
12884 if (tagging
&& t_async
)
12886 unsigned HOST_WIDE_INT i_async
;
12888 if (TREE_CODE (t_async
) == INTEGER_CST
)
12890 /* See if we can pack the async arg in to the tag's
12892 i_async
= TREE_INT_CST_LOW (t_async
);
12894 if (i_async
< GOMP_LAUNCH_OP_MAX
)
12895 t_async
= NULL_TREE
;
12898 i_async
= GOMP_LAUNCH_OP_MAX
;
12899 args
.safe_push (oacc_launch_pack
12900 (GOMP_LAUNCH_ASYNC
, NULL_TREE
, i_async
));
12903 args
.safe_push (t_async
);
12905 /* Save the argument index, and ... */
12906 unsigned t_wait_idx
= args
.length ();
12907 unsigned num_waits
= 0;
12908 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
12910 /* ... push a placeholder. */
12911 args
.safe_push (integer_zero_node
);
12913 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
12914 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
12916 args
.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
12918 OMP_CLAUSE_WAIT_EXPR (c
)));
12922 if (!tagging
|| num_waits
)
12926 /* Now that we know the number, update the placeholder. */
12928 len
= oacc_launch_pack (GOMP_LAUNCH_WAIT
, NULL_TREE
, num_waits
);
12930 len
= build_int_cst (integer_type_node
, num_waits
);
12931 len
= fold_convert_loc (gimple_location (entry_stmt
),
12932 unsigned_type_node
, len
);
12933 args
[t_wait_idx
] = len
;
12938 gcc_unreachable ();
12941 /* Push terminal marker - zero. */
12942 args
.safe_push (oacc_launch_pack (0, NULL_TREE
, 0));
12944 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), args
);
12945 gimple_set_location (g
, gimple_location (entry_stmt
));
12946 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
12949 g
= gsi_stmt (gsi
);
12950 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
12951 gsi_remove (&gsi
, true);
12953 if (data_region
&& region
->exit
)
12955 gsi
= gsi_last_bb (region
->exit
);
12956 g
= gsi_stmt (gsi
);
12957 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
12958 gsi_remove (&gsi
, true);
12963 /* Expand the parallel region tree rooted at REGION. Expansion
12964 proceeds in depth-first order. Innermost regions are expanded
12965 first. This way, parallel regions that require a new function to
12966 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
12967 internal dependencies in their body. */
12970 expand_omp (struct omp_region
*region
)
12972 omp_any_child_fn_dumped
= false;
12975 location_t saved_location
;
12976 gimple
*inner_stmt
= NULL
;
12978 /* First, determine whether this is a combined parallel+workshare
12980 if (region
->type
== GIMPLE_OMP_PARALLEL
)
12981 determine_parallel_type (region
);
12983 if (region
->type
== GIMPLE_OMP_FOR
12984 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
12985 inner_stmt
= last_stmt (region
->inner
->entry
);
12988 expand_omp (region
->inner
);
12990 saved_location
= input_location
;
12991 if (gimple_has_location (last_stmt (region
->entry
)))
12992 input_location
= gimple_location (last_stmt (region
->entry
));
12994 switch (region
->type
)
12996 case GIMPLE_OMP_PARALLEL
:
12997 case GIMPLE_OMP_TASK
:
12998 expand_omp_taskreg (region
);
13001 case GIMPLE_OMP_FOR
:
13002 expand_omp_for (region
, inner_stmt
);
13005 case GIMPLE_OMP_SECTIONS
:
13006 expand_omp_sections (region
);
13009 case GIMPLE_OMP_SECTION
:
13010 /* Individual omp sections are handled together with their
13011 parent GIMPLE_OMP_SECTIONS region. */
13014 case GIMPLE_OMP_SINGLE
:
13015 expand_omp_single (region
);
13018 case GIMPLE_OMP_ORDERED
:
13020 gomp_ordered
*ord_stmt
13021 = as_a
<gomp_ordered
*> (last_stmt (region
->entry
));
13022 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
13023 OMP_CLAUSE_DEPEND
))
13025 /* We'll expand these when expanding corresponding
13026 worksharing region with ordered(n) clause. */
13027 gcc_assert (region
->outer
13028 && region
->outer
->type
== GIMPLE_OMP_FOR
);
13029 region
->ord_stmt
= ord_stmt
;
13034 case GIMPLE_OMP_MASTER
:
13035 case GIMPLE_OMP_TASKGROUP
:
13036 case GIMPLE_OMP_CRITICAL
:
13037 case GIMPLE_OMP_TEAMS
:
13038 expand_omp_synch (region
);
13041 case GIMPLE_OMP_ATOMIC_LOAD
:
13042 expand_omp_atomic (region
);
13045 case GIMPLE_OMP_TARGET
:
13046 expand_omp_target (region
);
13050 gcc_unreachable ();
13053 input_location
= saved_location
;
13054 region
= region
->next
;
13056 if (omp_any_child_fn_dumped
)
13059 dump_function_header (dump_file
, current_function_decl
, dump_flags
);
13060 omp_any_child_fn_dumped
= false;
13065 /* Helper for build_omp_regions. Scan the dominator tree starting at
13066 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
13067 true, the function ends once a single tree is built (otherwise, whole
13068 forest of OMP constructs may be built). */
13071 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
13074 gimple_stmt_iterator gsi
;
13078 gsi
= gsi_last_bb (bb
);
13079 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
13081 struct omp_region
*region
;
13082 enum gimple_code code
;
13084 stmt
= gsi_stmt (gsi
);
13085 code
= gimple_code (stmt
);
13086 if (code
== GIMPLE_OMP_RETURN
)
13088 /* STMT is the return point out of region PARENT. Mark it
13089 as the exit point and make PARENT the immediately
13090 enclosing region. */
13091 gcc_assert (parent
);
13094 parent
= parent
->outer
;
13096 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
13098 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
13099 GIMPLE_OMP_RETURN, but matches with
13100 GIMPLE_OMP_ATOMIC_LOAD. */
13101 gcc_assert (parent
);
13102 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
13105 parent
= parent
->outer
;
13107 else if (code
== GIMPLE_OMP_CONTINUE
)
13109 gcc_assert (parent
);
13112 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
13114 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
13115 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
13119 region
= new_omp_region (bb
, code
, parent
);
13121 if (code
== GIMPLE_OMP_TARGET
)
13123 switch (gimple_omp_target_kind (stmt
))
13125 case GF_OMP_TARGET_KIND_REGION
:
13126 case GF_OMP_TARGET_KIND_DATA
:
13127 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
13128 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
13129 case GF_OMP_TARGET_KIND_OACC_DATA
:
13131 case GF_OMP_TARGET_KIND_UPDATE
:
13132 case GF_OMP_TARGET_KIND_ENTER_DATA
:
13133 case GF_OMP_TARGET_KIND_EXIT_DATA
:
13134 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
13135 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
13136 /* ..., other than for those stand-alone directives... */
13140 gcc_unreachable ();
13143 else if (code
== GIMPLE_OMP_ORDERED
13144 && find_omp_clause (gimple_omp_ordered_clauses
13145 (as_a
<gomp_ordered
*> (stmt
)),
13146 OMP_CLAUSE_DEPEND
))
13147 /* #pragma omp ordered depend is also just a stand-alone
13150 /* ..., this directive becomes the parent for a new region. */
13156 if (single_tree
&& !parent
)
13159 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
13161 son
= next_dom_son (CDI_DOMINATORS
, son
))
13162 build_omp_regions_1 (son
, parent
, single_tree
);
13165 /* Builds the tree of OMP regions rooted at ROOT, storing it to
13166 root_omp_region. */
13169 build_omp_regions_root (basic_block root
)
13171 gcc_assert (root_omp_region
== NULL
);
13172 build_omp_regions_1 (root
, NULL
, true);
13173 gcc_assert (root_omp_region
!= NULL
);
13176 /* Expands omp construct (and its subconstructs) starting in HEAD. */
13179 omp_expand_local (basic_block head
)
13181 build_omp_regions_root (head
);
13182 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
13184 fprintf (dump_file
, "\nOMP region tree\n\n");
13185 dump_omp_region (dump_file
, root_omp_region
, 0);
13186 fprintf (dump_file
, "\n");
13189 remove_exit_barriers (root_omp_region
);
13190 expand_omp (root_omp_region
);
13192 free_omp_regions ();
13195 /* Scan the CFG and build a tree of OMP regions. Return the root of
13196 the OMP region tree. */
13199 build_omp_regions (void)
13201 gcc_assert (root_omp_region
== NULL
);
13202 calculate_dominance_info (CDI_DOMINATORS
);
13203 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
13206 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
13208 static unsigned int
13209 execute_expand_omp (void)
13211 build_omp_regions ();
13213 if (!root_omp_region
)
13218 fprintf (dump_file
, "\nOMP region tree\n\n");
13219 dump_omp_region (dump_file
, root_omp_region
, 0);
13220 fprintf (dump_file
, "\n");
13223 remove_exit_barriers (root_omp_region
);
13225 expand_omp (root_omp_region
);
13227 if (flag_checking
&& !loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
13228 verify_loop_structure ();
13229 cleanup_tree_cfg ();
13231 free_omp_regions ();
13236 /* OMP expansion -- the default pass, run before creation of SSA form. */
13240 const pass_data pass_data_expand_omp
=
13242 GIMPLE_PASS
, /* type */
13243 "ompexp", /* name */
13244 OPTGROUP_NONE
, /* optinfo_flags */
13245 TV_NONE
, /* tv_id */
13246 PROP_gimple_any
, /* properties_required */
13247 PROP_gimple_eomp
, /* properties_provided */
13248 0, /* properties_destroyed */
13249 0, /* todo_flags_start */
13250 0, /* todo_flags_finish */
13253 class pass_expand_omp
: public gimple_opt_pass
13256 pass_expand_omp (gcc::context
*ctxt
)
13257 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
13260 /* opt_pass methods: */
13261 virtual unsigned int execute (function
*)
13263 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
13264 || flag_openmp_simd
!= 0)
13265 && !seen_error ());
13267 /* This pass always runs, to provide PROP_gimple_eomp.
13268 But often, there is nothing to do. */
13272 return execute_expand_omp ();
13275 }; // class pass_expand_omp
13277 } // anon namespace
13280 make_pass_expand_omp (gcc::context
*ctxt
)
13282 return new pass_expand_omp (ctxt
);
13287 const pass_data pass_data_expand_omp_ssa
=
13289 GIMPLE_PASS
, /* type */
13290 "ompexpssa", /* name */
13291 OPTGROUP_NONE
, /* optinfo_flags */
13292 TV_NONE
, /* tv_id */
13293 PROP_cfg
| PROP_ssa
, /* properties_required */
13294 PROP_gimple_eomp
, /* properties_provided */
13295 0, /* properties_destroyed */
13296 0, /* todo_flags_start */
13297 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
13300 class pass_expand_omp_ssa
: public gimple_opt_pass
13303 pass_expand_omp_ssa (gcc::context
*ctxt
)
13304 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
13307 /* opt_pass methods: */
13308 virtual bool gate (function
*fun
)
13310 return !(fun
->curr_properties
& PROP_gimple_eomp
);
13312 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
13314 }; // class pass_expand_omp_ssa
13316 } // anon namespace
13319 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
13321 return new pass_expand_omp_ssa (ctxt
);
13324 /* Routines to lower OMP directives into OMP-GIMPLE. */
13326 /* If ctx is a worksharing context inside of a cancellable parallel
13327 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
13328 and conditional branch to parallel's cancel_label to handle
13329 cancellation in the implicit barrier. */
13332 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
13334 gimple
*omp_return
= gimple_seq_last_stmt (*body
);
13335 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
13336 if (gimple_omp_return_nowait_p (omp_return
))
13339 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
13340 && ctx
->outer
->cancellable
)
13342 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
13343 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
13344 tree lhs
= create_tmp_var (c_bool_type
);
13345 gimple_omp_return_set_lhs (omp_return
, lhs
);
13346 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
13347 gimple
*g
= gimple_build_cond (NE_EXPR
, lhs
,
13348 fold_convert (c_bool_type
,
13349 boolean_false_node
),
13350 ctx
->outer
->cancel_label
, fallthru_label
);
13351 gimple_seq_add_stmt (body
, g
);
13352 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
13356 /* Lower the OpenMP sections directive in the current statement in GSI_P.
13357 CTX is the enclosing OMP context for the current statement. */
13360 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13362 tree block
, control
;
13363 gimple_stmt_iterator tgsi
;
13364 gomp_sections
*stmt
;
13366 gbind
*new_stmt
, *bind
;
13367 gimple_seq ilist
, dlist
, olist
, new_body
;
13369 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
13371 push_gimplify_context ();
13375 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
13376 &ilist
, &dlist
, ctx
, NULL
);
13378 new_body
= gimple_omp_body (stmt
);
13379 gimple_omp_set_body (stmt
, NULL
);
13380 tgsi
= gsi_start (new_body
);
13381 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
13386 sec_start
= gsi_stmt (tgsi
);
13387 sctx
= maybe_lookup_ctx (sec_start
);
13390 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
13391 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
13392 GSI_CONTINUE_LINKING
);
13393 gimple_omp_set_body (sec_start
, NULL
);
13395 if (gsi_one_before_end_p (tgsi
))
13397 gimple_seq l
= NULL
;
13398 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
13400 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
13401 gimple_omp_section_set_last (sec_start
);
13404 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
13405 GSI_CONTINUE_LINKING
);
13408 block
= make_node (BLOCK
);
13409 bind
= gimple_build_bind (NULL
, new_body
, block
);
13412 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
13414 block
= make_node (BLOCK
);
13415 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
13416 gsi_replace (gsi_p
, new_stmt
, true);
13418 pop_gimplify_context (new_stmt
);
13419 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
13420 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
13421 if (BLOCK_VARS (block
))
13422 TREE_USED (block
) = 1;
13425 gimple_seq_add_seq (&new_body
, ilist
);
13426 gimple_seq_add_stmt (&new_body
, stmt
);
13427 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
13428 gimple_seq_add_stmt (&new_body
, bind
);
13430 control
= create_tmp_var (unsigned_type_node
, ".section");
13431 t
= gimple_build_omp_continue (control
, control
);
13432 gimple_omp_sections_set_control (stmt
, control
);
13433 gimple_seq_add_stmt (&new_body
, t
);
13435 gimple_seq_add_seq (&new_body
, olist
);
13436 if (ctx
->cancellable
)
13437 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
13438 gimple_seq_add_seq (&new_body
, dlist
);
13440 new_body
= maybe_catch_exception (new_body
);
13442 t
= gimple_build_omp_return
13443 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
13444 OMP_CLAUSE_NOWAIT
));
13445 gimple_seq_add_stmt (&new_body
, t
);
13446 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
13448 gimple_bind_set_body (new_stmt
, new_body
);
13452 /* A subroutine of lower_omp_single. Expand the simple form of
13453 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
13455 if (GOMP_single_start ())
13457 [ GOMP_barrier (); ] -> unless 'nowait' is present.
13459 FIXME. It may be better to delay expanding the logic of this until
13460 pass_expand_omp. The expanded logic may make the job more difficult
13461 to a synchronization analysis pass. */
13464 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
13466 location_t loc
= gimple_location (single_stmt
);
13467 tree tlabel
= create_artificial_label (loc
);
13468 tree flabel
= create_artificial_label (loc
);
13469 gimple
*call
, *cond
;
13472 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
13473 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
13474 call
= gimple_build_call (decl
, 0);
13475 gimple_call_set_lhs (call
, lhs
);
13476 gimple_seq_add_stmt (pre_p
, call
);
13478 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
13479 fold_convert_loc (loc
, TREE_TYPE (lhs
),
13480 boolean_true_node
),
13482 gimple_seq_add_stmt (pre_p
, cond
);
13483 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
13484 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
13485 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
13489 /* A subroutine of lower_omp_single. Expand the simple form of
13490 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
13492 #pragma omp single copyprivate (a, b, c)
13494 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
13497 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
13503 GOMP_single_copy_end (©out);
13514 FIXME. It may be better to delay expanding the logic of this until
13515 pass_expand_omp. The expanded logic may make the job more difficult
13516 to a synchronization analysis pass. */
13519 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
13522 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
13523 gimple_seq copyin_seq
;
13524 location_t loc
= gimple_location (single_stmt
);
13526 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
13528 ptr_type
= build_pointer_type (ctx
->record_type
);
13529 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
13531 l0
= create_artificial_label (loc
);
13532 l1
= create_artificial_label (loc
);
13533 l2
= create_artificial_label (loc
);
13535 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
13536 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
13537 t
= fold_convert_loc (loc
, ptr_type
, t
);
13538 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
13540 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
13541 build_int_cst (ptr_type
, 0));
13542 t
= build3 (COND_EXPR
, void_type_node
, t
,
13543 build_and_jump (&l0
), build_and_jump (&l1
));
13544 gimplify_and_add (t
, pre_p
);
13546 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
13548 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
13551 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
13554 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
13555 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
13556 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
13557 gimplify_and_add (t
, pre_p
);
13559 t
= build_and_jump (&l2
);
13560 gimplify_and_add (t
, pre_p
);
13562 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
13564 gimple_seq_add_seq (pre_p
, copyin_seq
);
13566 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
13570 /* Expand code for an OpenMP single directive. */
13573 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13577 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
13579 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
13581 push_gimplify_context ();
13583 block
= make_node (BLOCK
);
13584 bind
= gimple_build_bind (NULL
, NULL
, block
);
13585 gsi_replace (gsi_p
, bind
, true);
13588 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
13589 &bind_body
, &dlist
, ctx
, NULL
);
13590 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
13592 gimple_seq_add_stmt (&bind_body
, single_stmt
);
13594 if (ctx
->record_type
)
13595 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
13597 lower_omp_single_simple (single_stmt
, &bind_body
);
13599 gimple_omp_set_body (single_stmt
, NULL
);
13601 gimple_seq_add_seq (&bind_body
, dlist
);
13603 bind_body
= maybe_catch_exception (bind_body
);
13605 t
= gimple_build_omp_return
13606 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
13607 OMP_CLAUSE_NOWAIT
));
13608 gimple_seq_add_stmt (&bind_body_tail
, t
);
13609 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
13610 if (ctx
->record_type
)
13612 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
13613 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
13614 TREE_THIS_VOLATILE (clobber
) = 1;
13615 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
13616 clobber
), GSI_SAME_STMT
);
13618 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
13619 gimple_bind_set_body (bind
, bind_body
);
13621 pop_gimplify_context (bind
);
13623 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13624 BLOCK_VARS (block
) = ctx
->block_vars
;
13625 if (BLOCK_VARS (block
))
13626 TREE_USED (block
) = 1;
13630 /* Expand code for an OpenMP master directive. */
13633 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13635 tree block
, lab
= NULL
, x
, bfn_decl
;
13636 gimple
*stmt
= gsi_stmt (*gsi_p
);
13638 location_t loc
= gimple_location (stmt
);
13641 push_gimplify_context ();
13643 block
= make_node (BLOCK
);
13644 bind
= gimple_build_bind (NULL
, NULL
, block
);
13645 gsi_replace (gsi_p
, bind
, true);
13646 gimple_bind_add_stmt (bind
, stmt
);
13648 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
13649 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
13650 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
13651 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
13653 gimplify_and_add (x
, &tseq
);
13654 gimple_bind_add_seq (bind
, tseq
);
13656 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
13657 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
13658 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
13659 gimple_omp_set_body (stmt
, NULL
);
13661 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
13663 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
13665 pop_gimplify_context (bind
);
13667 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13668 BLOCK_VARS (block
) = ctx
->block_vars
;
13672 /* Expand code for an OpenMP taskgroup directive. */
13675 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13677 gimple
*stmt
= gsi_stmt (*gsi_p
);
13680 tree block
= make_node (BLOCK
);
13682 bind
= gimple_build_bind (NULL
, NULL
, block
);
13683 gsi_replace (gsi_p
, bind
, true);
13684 gimple_bind_add_stmt (bind
, stmt
);
13686 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
13688 gimple_bind_add_stmt (bind
, x
);
13690 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
13691 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
13692 gimple_omp_set_body (stmt
, NULL
);
13694 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
13696 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13697 BLOCK_VARS (block
) = ctx
->block_vars
;
13701 /* Fold the OMP_ORDERED_CLAUSES for the OMP_ORDERED in STMT if possible. */
13704 lower_omp_ordered_clauses (gimple_stmt_iterator
*gsi_p
, gomp_ordered
*ord_stmt
,
13707 struct omp_for_data fd
;
13708 if (!ctx
->outer
|| gimple_code (ctx
->outer
->stmt
) != GIMPLE_OMP_FOR
)
13711 unsigned int len
= gimple_omp_for_collapse (ctx
->outer
->stmt
);
13712 struct omp_for_data_loop
*loops
= XALLOCAVEC (struct omp_for_data_loop
, len
);
13713 extract_omp_for_data (as_a
<gomp_for
*> (ctx
->outer
->stmt
), &fd
, loops
);
13717 tree
*list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
13718 tree c
= gimple_omp_ordered_clauses (ord_stmt
);
13719 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
13720 && OMP_CLAUSE_DEPEND_KIND (c
) == OMP_CLAUSE_DEPEND_SINK
)
13722 /* Merge depend clauses from multiple adjacent
13723 #pragma omp ordered depend(sink:...) constructs
13724 into one #pragma omp ordered depend(sink:...), so that
13725 we can optimize them together. */
13726 gimple_stmt_iterator gsi
= *gsi_p
;
13728 while (!gsi_end_p (gsi
))
13730 gimple
*stmt
= gsi_stmt (gsi
);
13731 if (is_gimple_debug (stmt
)
13732 || gimple_code (stmt
) == GIMPLE_NOP
)
13737 if (gimple_code (stmt
) != GIMPLE_OMP_ORDERED
)
13739 gomp_ordered
*ord_stmt2
= as_a
<gomp_ordered
*> (stmt
);
13740 c
= gimple_omp_ordered_clauses (ord_stmt2
);
13742 || OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_DEPEND
13743 || OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
13746 list_p
= &OMP_CLAUSE_CHAIN (*list_p
);
13748 gsi_remove (&gsi
, true);
13752 /* Canonicalize sink dependence clauses into one folded clause if
13755 The basic algorithm is to create a sink vector whose first
13756 element is the GCD of all the first elements, and whose remaining
13757 elements are the minimum of the subsequent columns.
13759 We ignore dependence vectors whose first element is zero because
13760 such dependencies are known to be executed by the same thread.
13762 We take into account the direction of the loop, so a minimum
13763 becomes a maximum if the loop is iterating forwards. We also
13764 ignore sink clauses where the loop direction is unknown, or where
13765 the offsets are clearly invalid because they are not a multiple
13766 of the loop increment.
13770 #pragma omp for ordered(2)
13771 for (i=0; i < N; ++i)
13772 for (j=0; j < M; ++j)
13774 #pragma omp ordered \
13775 depend(sink:i-8,j-2) \
13776 depend(sink:i,j-1) \ // Completely ignored because i+0.
13777 depend(sink:i-4,j-3) \
13778 depend(sink:i-6,j-4)
13779 #pragma omp ordered depend(source)
13784 depend(sink:-gcd(8,4,6),-min(2,3,4))
13789 /* FIXME: Computing GCD's where the first element is zero is
13790 non-trivial in the presence of collapsed loops. Do this later. */
13791 if (fd
.collapse
> 1)
13794 wide_int
*folded_deps
= XALLOCAVEC (wide_int
, 2 * len
- 1);
13795 memset (folded_deps
, 0, sizeof (*folded_deps
) * (2 * len
- 1));
13796 tree folded_dep
= NULL_TREE
;
13797 /* TRUE if the first dimension's offset is negative. */
13798 bool neg_offset_p
= false;
13800 list_p
= gimple_omp_ordered_clauses_ptr (ord_stmt
);
13802 while ((c
= *list_p
) != NULL
)
13804 bool remove
= false;
13806 gcc_assert (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
);
13807 if (OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_SINK
)
13808 goto next_ordered_clause
;
13811 for (vec
= OMP_CLAUSE_DECL (c
), i
= 0;
13812 vec
&& TREE_CODE (vec
) == TREE_LIST
;
13813 vec
= TREE_CHAIN (vec
), ++i
)
13815 gcc_assert (i
< len
);
13817 /* extract_omp_for_data has canonicalized the condition. */
13818 gcc_assert (fd
.loops
[i
].cond_code
== LT_EXPR
13819 || fd
.loops
[i
].cond_code
== GT_EXPR
);
13820 bool forward
= fd
.loops
[i
].cond_code
== LT_EXPR
;
13821 bool maybe_lexically_later
= true;
13823 /* While the committee makes up its mind, bail if we have any
13824 non-constant steps. */
13825 if (TREE_CODE (fd
.loops
[i
].step
) != INTEGER_CST
)
13826 goto lower_omp_ordered_ret
;
13828 tree itype
= TREE_TYPE (TREE_VALUE (vec
));
13829 if (POINTER_TYPE_P (itype
))
13831 wide_int offset
= wide_int::from (TREE_PURPOSE (vec
),
13832 TYPE_PRECISION (itype
),
13833 TYPE_SIGN (itype
));
13835 /* Ignore invalid offsets that are not multiples of the step. */
13836 if (!wi::multiple_of_p
13837 (wi::abs (offset
), wi::abs ((wide_int
) fd
.loops
[i
].step
),
13840 warning_at (OMP_CLAUSE_LOCATION (c
), 0,
13841 "ignoring sink clause with offset that is not "
13842 "a multiple of the loop step");
13844 goto next_ordered_clause
;
13847 /* Calculate the first dimension. The first dimension of
13848 the folded dependency vector is the GCD of the first
13849 elements, while ignoring any first elements whose offset
13853 /* Ignore dependence vectors whose first dimension is 0. */
13857 goto next_ordered_clause
;
13861 if (!TYPE_UNSIGNED (itype
) && (forward
^ wi::neg_p (offset
)))
13863 error_at (OMP_CLAUSE_LOCATION (c
),
13864 "first offset must be in opposite direction "
13865 "of loop iterations");
13866 goto lower_omp_ordered_ret
;
13870 neg_offset_p
= forward
;
13871 /* Initialize the first time around. */
13872 if (folded_dep
== NULL_TREE
)
13875 folded_deps
[0] = offset
;
13878 folded_deps
[0] = wi::gcd (folded_deps
[0],
13882 /* Calculate minimum for the remaining dimensions. */
13885 folded_deps
[len
+ i
- 1] = offset
;
13886 if (folded_dep
== c
)
13887 folded_deps
[i
] = offset
;
13888 else if (maybe_lexically_later
13889 && !wi::eq_p (folded_deps
[i
], offset
))
13891 if (forward
^ wi::gts_p (folded_deps
[i
], offset
))
13895 for (j
= 1; j
<= i
; j
++)
13896 folded_deps
[j
] = folded_deps
[len
+ j
- 1];
13899 maybe_lexically_later
= false;
13903 gcc_assert (i
== len
);
13907 next_ordered_clause
:
13909 *list_p
= OMP_CLAUSE_CHAIN (c
);
13911 list_p
= &OMP_CLAUSE_CHAIN (c
);
13917 folded_deps
[0] = -folded_deps
[0];
13919 tree itype
= TREE_TYPE (TREE_VALUE (OMP_CLAUSE_DECL (folded_dep
)));
13920 if (POINTER_TYPE_P (itype
))
13923 TREE_PURPOSE (OMP_CLAUSE_DECL (folded_dep
))
13924 = wide_int_to_tree (itype
, folded_deps
[0]);
13925 OMP_CLAUSE_CHAIN (folded_dep
) = gimple_omp_ordered_clauses (ord_stmt
);
13926 *gimple_omp_ordered_clauses_ptr (ord_stmt
) = folded_dep
;
13929 lower_omp_ordered_ret
:
13931 /* Ordered without clauses is #pragma omp threads, while we want
13932 a nop instead if we remove all clauses. */
13933 if (gimple_omp_ordered_clauses (ord_stmt
) == NULL_TREE
)
13934 gsi_replace (gsi_p
, gimple_build_nop (), true);
13938 /* Expand code for an OpenMP ordered directive. */
13941 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
13944 gimple
*stmt
= gsi_stmt (*gsi_p
);
13945 gomp_ordered
*ord_stmt
= as_a
<gomp_ordered
*> (stmt
);
13949 = find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
), OMP_CLAUSE_SIMD
);
13951 if (find_omp_clause (gimple_omp_ordered_clauses (ord_stmt
),
13952 OMP_CLAUSE_DEPEND
))
13954 /* FIXME: This is needs to be moved to the expansion to verify various
13955 conditions only testable on cfg with dominators computed, and also
13956 all the depend clauses to be merged still might need to be available
13957 for the runtime checks. */
13959 lower_omp_ordered_clauses (gsi_p
, ord_stmt
, ctx
);
13963 push_gimplify_context ();
13965 block
= make_node (BLOCK
);
13966 bind
= gimple_build_bind (NULL
, NULL
, block
);
13967 gsi_replace (gsi_p
, bind
, true);
13968 gimple_bind_add_stmt (bind
, stmt
);
13972 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_START
, 0);
13973 cfun
->has_simduid_loops
= true;
13976 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
13978 gimple_bind_add_stmt (bind
, x
);
13980 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
13981 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
13982 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
13983 gimple_omp_set_body (stmt
, NULL
);
13986 x
= gimple_build_call_internal (IFN_GOMP_SIMD_ORDERED_END
, 0);
13988 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
),
13990 gimple_bind_add_stmt (bind
, x
);
13992 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
13994 pop_gimplify_context (bind
);
13996 gimple_bind_append_vars (bind
, ctx
->block_vars
);
13997 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14001 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
14002 substitution of a couple of function calls. But in the NAMED case,
14003 requires that languages coordinate a symbol name. It is therefore
14004 best put here in common code. */
14006 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
14009 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14012 tree name
, lock
, unlock
;
14013 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
14015 location_t loc
= gimple_location (stmt
);
14018 name
= gimple_omp_critical_name (stmt
);
14023 if (!critical_name_mutexes
)
14024 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
14026 tree
*n
= critical_name_mutexes
->get (name
);
14031 decl
= create_tmp_var_raw (ptr_type_node
);
14033 new_str
= ACONCAT ((".gomp_critical_user_",
14034 IDENTIFIER_POINTER (name
), NULL
));
14035 DECL_NAME (decl
) = get_identifier (new_str
);
14036 TREE_PUBLIC (decl
) = 1;
14037 TREE_STATIC (decl
) = 1;
14038 DECL_COMMON (decl
) = 1;
14039 DECL_ARTIFICIAL (decl
) = 1;
14040 DECL_IGNORED_P (decl
) = 1;
14042 varpool_node::finalize_decl (decl
);
14044 critical_name_mutexes
->put (name
, decl
);
14049 /* If '#pragma omp critical' is inside offloaded region or
14050 inside function marked as offloadable, the symbol must be
14051 marked as offloadable too. */
14053 if (cgraph_node::get (current_function_decl
)->offloadable
)
14054 varpool_node::get_create (decl
)->offloadable
= 1;
14056 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
14057 if (is_gimple_omp_offloaded (octx
->stmt
))
14059 varpool_node::get_create (decl
)->offloadable
= 1;
14063 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
14064 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
14066 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
14067 unlock
= build_call_expr_loc (loc
, unlock
, 1,
14068 build_fold_addr_expr_loc (loc
, decl
));
14072 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
14073 lock
= build_call_expr_loc (loc
, lock
, 0);
14075 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
14076 unlock
= build_call_expr_loc (loc
, unlock
, 0);
14079 push_gimplify_context ();
14081 block
= make_node (BLOCK
);
14082 bind
= gimple_build_bind (NULL
, NULL
, block
);
14083 gsi_replace (gsi_p
, bind
, true);
14084 gimple_bind_add_stmt (bind
, stmt
);
14086 tbody
= gimple_bind_body (bind
);
14087 gimplify_and_add (lock
, &tbody
);
14088 gimple_bind_set_body (bind
, tbody
);
14090 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14091 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
14092 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
14093 gimple_omp_set_body (stmt
, NULL
);
14095 tbody
= gimple_bind_body (bind
);
14096 gimplify_and_add (unlock
, &tbody
);
14097 gimple_bind_set_body (bind
, tbody
);
14099 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
14101 pop_gimplify_context (bind
);
14102 gimple_bind_append_vars (bind
, ctx
->block_vars
);
14103 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
14107 /* A subroutine of lower_omp_for. Generate code to emit the predicate
14108 for a lastprivate clause. Given a loop control predicate of (V
14109 cond N2), we gate the clause on (!(V cond N2)). The lowered form
14110 is appended to *DLIST, iterator initialization is appended to
14114 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
14115 gimple_seq
*dlist
, struct omp_context
*ctx
)
14117 tree clauses
, cond
, vinit
;
14118 enum tree_code cond_code
;
14121 cond_code
= fd
->loop
.cond_code
;
14122 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
14124 /* When possible, use a strict equality expression. This can let VRP
14125 type optimizations deduce the value and remove a copy. */
14126 if (tree_fits_shwi_p (fd
->loop
.step
))
14128 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
14129 if (step
== 1 || step
== -1)
14130 cond_code
= EQ_EXPR
;
14133 tree n2
= fd
->loop
.n2
;
14134 if (fd
->collapse
> 1
14135 && TREE_CODE (n2
) != INTEGER_CST
14136 && gimple_omp_for_combined_into_p (fd
->for_stmt
))
14138 struct omp_context
*taskreg_ctx
= NULL
;
14139 if (gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_FOR
)
14141 gomp_for
*gfor
= as_a
<gomp_for
*> (ctx
->outer
->stmt
);
14142 if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_FOR
14143 || gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
14145 if (gimple_omp_for_combined_into_p (gfor
))
14147 gcc_assert (ctx
->outer
->outer
14148 && is_parallel_ctx (ctx
->outer
->outer
));
14149 taskreg_ctx
= ctx
->outer
->outer
;
14153 struct omp_for_data outer_fd
;
14154 extract_omp_for_data (gfor
, &outer_fd
, NULL
);
14155 n2
= fold_convert (TREE_TYPE (n2
), outer_fd
.loop
.n2
);
14158 else if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_TASKLOOP
)
14159 taskreg_ctx
= ctx
->outer
->outer
;
14161 else if (is_taskreg_ctx (ctx
->outer
))
14162 taskreg_ctx
= ctx
->outer
;
14167 = find_omp_clause (gimple_omp_taskreg_clauses (taskreg_ctx
->stmt
),
14168 OMP_CLAUSE__LOOPTEMP_
);
14169 gcc_assert (innerc
);
14170 for (i
= 0; i
< fd
->collapse
; i
++)
14172 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14173 OMP_CLAUSE__LOOPTEMP_
);
14174 gcc_assert (innerc
);
14176 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
14177 OMP_CLAUSE__LOOPTEMP_
);
14179 n2
= fold_convert (TREE_TYPE (n2
),
14180 lookup_decl (OMP_CLAUSE_DECL (innerc
),
14184 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, n2
);
14186 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
14188 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
14189 if (!gimple_seq_empty_p (stmts
))
14191 gimple_seq_add_seq (&stmts
, *dlist
);
14194 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
14195 vinit
= fd
->loop
.n1
;
14196 if (cond_code
== EQ_EXPR
14197 && tree_fits_shwi_p (fd
->loop
.n2
)
14198 && ! integer_zerop (fd
->loop
.n2
))
14199 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
14201 vinit
= unshare_expr (vinit
);
14203 /* Initialize the iterator variable, so that threads that don't execute
14204 any iterations don't execute the lastprivate clauses by accident. */
14205 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
14210 /* Lower code for an OMP loop directive. */
14213 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14215 tree
*rhs_p
, block
;
14216 struct omp_for_data fd
, *fdp
= NULL
;
14217 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
14219 gimple_seq omp_for_body
, body
, dlist
;
14220 gimple_seq oacc_head
= NULL
, oacc_tail
= NULL
;
14223 push_gimplify_context ();
14225 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
14227 block
= make_node (BLOCK
);
14228 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
14229 /* Replace at gsi right away, so that 'stmt' is no member
14230 of a sequence anymore as we're going to add to a different
14232 gsi_replace (gsi_p
, new_stmt
, true);
14234 /* Move declaration of temporaries in the loop body before we make
14236 omp_for_body
= gimple_omp_body (stmt
);
14237 if (!gimple_seq_empty_p (omp_for_body
)
14238 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
14241 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
14242 tree vars
= gimple_bind_vars (inner_bind
);
14243 gimple_bind_append_vars (new_stmt
, vars
);
14244 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
14245 keep them on the inner_bind and it's block. */
14246 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
14247 if (gimple_bind_block (inner_bind
))
14248 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
14251 if (gimple_omp_for_combined_into_p (stmt
))
14253 extract_omp_for_data (stmt
, &fd
, NULL
);
14256 /* We need two temporaries with fd.loop.v type (istart/iend)
14257 and then (fd.collapse - 1) temporaries with the same
14258 type for count2 ... countN-1 vars if not constant. */
14260 tree type
= fd
.iter_type
;
14261 if (fd
.collapse
> 1
14262 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
14263 count
+= fd
.collapse
- 1;
14265 = (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
14266 || gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_TASKLOOP
);
14267 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
14268 tree clauses
= *pc
;
14271 = find_omp_clause (gimple_omp_taskreg_clauses (ctx
->outer
->stmt
),
14272 OMP_CLAUSE__LOOPTEMP_
);
14273 for (i
= 0; i
< count
; i
++)
14278 gcc_assert (outerc
);
14279 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
14280 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
14281 OMP_CLAUSE__LOOPTEMP_
);
14285 temp
= create_tmp_var (type
);
14286 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
14288 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
14289 OMP_CLAUSE_DECL (*pc
) = temp
;
14290 pc
= &OMP_CLAUSE_CHAIN (*pc
);
14295 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
14298 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
14300 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
14302 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
14304 /* Lower the header expressions. At this point, we can assume that
14305 the header is of the form:
14307 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
14309 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
14310 using the .omp_data_s mapping, if needed. */
14311 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
14313 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
14314 if (!is_gimple_min_invariant (*rhs_p
))
14315 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
14317 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
14318 if (!is_gimple_min_invariant (*rhs_p
))
14319 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
14321 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
14322 if (!is_gimple_min_invariant (*rhs_p
))
14323 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
14326 /* Once lowered, extract the bounds and clauses. */
14327 extract_omp_for_data (stmt
, &fd
, NULL
);
14329 if (is_gimple_omp_oacc (ctx
->stmt
)
14330 && !ctx_in_oacc_kernels_region (ctx
))
14331 lower_oacc_head_tail (gimple_location (stmt
),
14332 gimple_omp_for_clauses (stmt
),
14333 &oacc_head
, &oacc_tail
, ctx
);
14335 /* Add OpenACC partitioning and reduction markers just before the loop */
14337 gimple_seq_add_seq (&body
, oacc_head
);
14339 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
14341 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
14342 for (tree c
= gimple_omp_for_clauses (stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
14343 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
14344 && !OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
14346 OMP_CLAUSE_DECL (c
) = lookup_decl (OMP_CLAUSE_DECL (c
), ctx
);
14347 if (DECL_P (OMP_CLAUSE_LINEAR_STEP (c
)))
14348 OMP_CLAUSE_LINEAR_STEP (c
)
14349 = maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_LINEAR_STEP (c
),
14353 gimple_seq_add_stmt (&body
, stmt
);
14354 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
14356 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
14359 /* After the loop, add exit clauses. */
14360 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
14362 if (ctx
->cancellable
)
14363 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
14365 gimple_seq_add_seq (&body
, dlist
);
14367 body
= maybe_catch_exception (body
);
14369 /* Region exit marker goes at the end of the loop body. */
14370 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
14371 maybe_add_implicit_barrier_cancel (ctx
, &body
);
14373 /* Add OpenACC joining and reduction markers just after the loop. */
14375 gimple_seq_add_seq (&body
, oacc_tail
);
14377 pop_gimplify_context (new_stmt
);
14379 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
14380 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
14381 if (BLOCK_VARS (block
))
14382 TREE_USED (block
) = 1;
14384 gimple_bind_set_body (new_stmt
, body
);
14385 gimple_omp_set_body (stmt
, NULL
);
14386 gimple_omp_for_set_pre_body (stmt
, NULL
);
14389 /* Callback for walk_stmts. Check if the current statement only contains
14390 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
14393 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
14394 bool *handled_ops_p
,
14395 struct walk_stmt_info
*wi
)
14397 int *info
= (int *) wi
->info
;
14398 gimple
*stmt
= gsi_stmt (*gsi_p
);
14400 *handled_ops_p
= true;
14401 switch (gimple_code (stmt
))
14405 case GIMPLE_OMP_FOR
:
14406 case GIMPLE_OMP_SECTIONS
:
14407 *info
= *info
== 0 ? 1 : -1;
14416 struct omp_taskcopy_context
14418 /* This field must be at the beginning, as we do "inheritance": Some
14419 callback functions for tree-inline.c (e.g., omp_copy_decl)
14420 receive a copy_body_data pointer that is up-casted to an
14421 omp_context pointer. */
14427 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
14429 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
14431 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
14432 return create_tmp_var (TREE_TYPE (var
));
14438 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
14440 tree name
, new_fields
= NULL
, type
, f
;
14442 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
14443 name
= DECL_NAME (TYPE_NAME (orig_type
));
14444 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
14445 TYPE_DECL
, name
, type
);
14446 TYPE_NAME (type
) = name
;
14448 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
14450 tree new_f
= copy_node (f
);
14451 DECL_CONTEXT (new_f
) = type
;
14452 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
14453 TREE_CHAIN (new_f
) = new_fields
;
14454 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
14455 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
14456 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
14458 new_fields
= new_f
;
14459 tcctx
->cb
.decl_map
->put (f
, new_f
);
14461 TYPE_FIELDS (type
) = nreverse (new_fields
);
14462 layout_type (type
);
14466 /* Create task copyfn. */
14469 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
14471 struct function
*child_cfun
;
14472 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
14473 tree record_type
, srecord_type
, bind
, list
;
14474 bool record_needs_remap
= false, srecord_needs_remap
= false;
14476 struct omp_taskcopy_context tcctx
;
14477 location_t loc
= gimple_location (task_stmt
);
14479 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
14480 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
14481 gcc_assert (child_cfun
->cfg
== NULL
);
14482 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
14484 /* Reset DECL_CONTEXT on function arguments. */
14485 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
14486 DECL_CONTEXT (t
) = child_fn
;
14488 /* Populate the function. */
14489 push_gimplify_context ();
14490 push_cfun (child_cfun
);
14492 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
14493 TREE_SIDE_EFFECTS (bind
) = 1;
14495 DECL_SAVED_TREE (child_fn
) = bind
;
14496 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
14498 /* Remap src and dst argument types if needed. */
14499 record_type
= ctx
->record_type
;
14500 srecord_type
= ctx
->srecord_type
;
14501 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
14502 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
14504 record_needs_remap
= true;
14507 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
14508 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
14510 srecord_needs_remap
= true;
14514 if (record_needs_remap
|| srecord_needs_remap
)
14516 memset (&tcctx
, '\0', sizeof (tcctx
));
14517 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
14518 tcctx
.cb
.dst_fn
= child_fn
;
14519 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
14520 gcc_checking_assert (tcctx
.cb
.src_node
);
14521 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
14522 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
14523 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
14524 tcctx
.cb
.eh_lp_nr
= 0;
14525 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
14526 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
14529 if (record_needs_remap
)
14530 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
14531 if (srecord_needs_remap
)
14532 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
14535 tcctx
.cb
.decl_map
= NULL
;
14537 arg
= DECL_ARGUMENTS (child_fn
);
14538 TREE_TYPE (arg
) = build_pointer_type (record_type
);
14539 sarg
= DECL_CHAIN (arg
);
14540 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
14542 /* First pass: initialize temporaries used in record_type and srecord_type
14543 sizes and field offsets. */
14544 if (tcctx
.cb
.decl_map
)
14545 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
14546 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
14550 decl
= OMP_CLAUSE_DECL (c
);
14551 p
= tcctx
.cb
.decl_map
->get (decl
);
14554 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
14555 sf
= (tree
) n
->value
;
14556 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14557 src
= build_simple_mem_ref_loc (loc
, sarg
);
14558 src
= omp_build_component_ref (src
, sf
);
14559 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
14560 append_to_statement_list (t
, &list
);
14563 /* Second pass: copy shared var pointers and copy construct non-VLA
14564 firstprivate vars. */
14565 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
14566 switch (OMP_CLAUSE_CODE (c
))
14568 splay_tree_key key
;
14569 case OMP_CLAUSE_SHARED
:
14570 decl
= OMP_CLAUSE_DECL (c
);
14571 key
= (splay_tree_key
) decl
;
14572 if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c
))
14573 key
= (splay_tree_key
) &DECL_UID (decl
);
14574 n
= splay_tree_lookup (ctx
->field_map
, key
);
14577 f
= (tree
) n
->value
;
14578 if (tcctx
.cb
.decl_map
)
14579 f
= *tcctx
.cb
.decl_map
->get (f
);
14580 n
= splay_tree_lookup (ctx
->sfield_map
, key
);
14581 sf
= (tree
) n
->value
;
14582 if (tcctx
.cb
.decl_map
)
14583 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14584 src
= build_simple_mem_ref_loc (loc
, sarg
);
14585 src
= omp_build_component_ref (src
, sf
);
14586 dst
= build_simple_mem_ref_loc (loc
, arg
);
14587 dst
= omp_build_component_ref (dst
, f
);
14588 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
14589 append_to_statement_list (t
, &list
);
14591 case OMP_CLAUSE_FIRSTPRIVATE
:
14592 decl
= OMP_CLAUSE_DECL (c
);
14593 if (is_variable_sized (decl
))
14595 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
14598 f
= (tree
) n
->value
;
14599 if (tcctx
.cb
.decl_map
)
14600 f
= *tcctx
.cb
.decl_map
->get (f
);
14601 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
14604 sf
= (tree
) n
->value
;
14605 if (tcctx
.cb
.decl_map
)
14606 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14607 src
= build_simple_mem_ref_loc (loc
, sarg
);
14608 src
= omp_build_component_ref (src
, sf
);
14609 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
14610 src
= build_simple_mem_ref_loc (loc
, src
);
14614 dst
= build_simple_mem_ref_loc (loc
, arg
);
14615 dst
= omp_build_component_ref (dst
, f
);
14616 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
14617 append_to_statement_list (t
, &list
);
14619 case OMP_CLAUSE_PRIVATE
:
14620 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
14622 decl
= OMP_CLAUSE_DECL (c
);
14623 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
14624 f
= (tree
) n
->value
;
14625 if (tcctx
.cb
.decl_map
)
14626 f
= *tcctx
.cb
.decl_map
->get (f
);
14627 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
14630 sf
= (tree
) n
->value
;
14631 if (tcctx
.cb
.decl_map
)
14632 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14633 src
= build_simple_mem_ref_loc (loc
, sarg
);
14634 src
= omp_build_component_ref (src
, sf
);
14635 if (use_pointer_for_field (decl
, NULL
))
14636 src
= build_simple_mem_ref_loc (loc
, src
);
14640 dst
= build_simple_mem_ref_loc (loc
, arg
);
14641 dst
= omp_build_component_ref (dst
, f
);
14642 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
14643 append_to_statement_list (t
, &list
);
14649 /* Last pass: handle VLA firstprivates. */
14650 if (tcctx
.cb
.decl_map
)
14651 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
14652 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
14656 decl
= OMP_CLAUSE_DECL (c
);
14657 if (!is_variable_sized (decl
))
14659 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
14662 f
= (tree
) n
->value
;
14663 f
= *tcctx
.cb
.decl_map
->get (f
);
14664 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
14665 ind
= DECL_VALUE_EXPR (decl
);
14666 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
14667 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
14668 n
= splay_tree_lookup (ctx
->sfield_map
,
14669 (splay_tree_key
) TREE_OPERAND (ind
, 0));
14670 sf
= (tree
) n
->value
;
14671 sf
= *tcctx
.cb
.decl_map
->get (sf
);
14672 src
= build_simple_mem_ref_loc (loc
, sarg
);
14673 src
= omp_build_component_ref (src
, sf
);
14674 src
= build_simple_mem_ref_loc (loc
, src
);
14675 dst
= build_simple_mem_ref_loc (loc
, arg
);
14676 dst
= omp_build_component_ref (dst
, f
);
14677 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
14678 append_to_statement_list (t
, &list
);
14679 n
= splay_tree_lookup (ctx
->field_map
,
14680 (splay_tree_key
) TREE_OPERAND (ind
, 0));
14681 df
= (tree
) n
->value
;
14682 df
= *tcctx
.cb
.decl_map
->get (df
);
14683 ptr
= build_simple_mem_ref_loc (loc
, arg
);
14684 ptr
= omp_build_component_ref (ptr
, df
);
14685 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
14686 build_fold_addr_expr_loc (loc
, dst
));
14687 append_to_statement_list (t
, &list
);
14690 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
14691 append_to_statement_list (t
, &list
);
14693 if (tcctx
.cb
.decl_map
)
14694 delete tcctx
.cb
.decl_map
;
14695 pop_gimplify_context (NULL
);
14696 BIND_EXPR_BODY (bind
) = list
;
14701 lower_depend_clauses (tree
*pclauses
, gimple_seq
*iseq
, gimple_seq
*oseq
)
14705 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
14707 clauses
= find_omp_clause (*pclauses
, OMP_CLAUSE_DEPEND
);
14708 gcc_assert (clauses
);
14709 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
14710 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
14711 switch (OMP_CLAUSE_DEPEND_KIND (c
))
14713 case OMP_CLAUSE_DEPEND_IN
:
14716 case OMP_CLAUSE_DEPEND_OUT
:
14717 case OMP_CLAUSE_DEPEND_INOUT
:
14720 case OMP_CLAUSE_DEPEND_SOURCE
:
14721 case OMP_CLAUSE_DEPEND_SINK
:
14724 gcc_unreachable ();
14726 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
14727 tree array
= create_tmp_var (type
);
14728 TREE_ADDRESSABLE (array
) = 1;
14729 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
14731 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
14732 gimple_seq_add_stmt (iseq
, g
);
14733 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
14735 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
14736 gimple_seq_add_stmt (iseq
, g
);
14737 for (i
= 0; i
< 2; i
++)
14739 if ((i
? n_in
: n_out
) == 0)
14741 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
14742 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
14743 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
14745 tree t
= OMP_CLAUSE_DECL (c
);
14746 t
= fold_convert (ptr_type_node
, t
);
14747 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
14748 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
14749 NULL_TREE
, NULL_TREE
);
14750 g
= gimple_build_assign (r
, t
);
14751 gimple_seq_add_stmt (iseq
, g
);
14754 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
14755 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
14756 OMP_CLAUSE_CHAIN (c
) = *pclauses
;
14758 tree clobber
= build_constructor (type
, NULL
);
14759 TREE_THIS_VOLATILE (clobber
) = 1;
14760 g
= gimple_build_assign (array
, clobber
);
14761 gimple_seq_add_stmt (oseq
, g
);
14764 /* Lower the OpenMP parallel or task directive in the current statement
14765 in GSI_P. CTX holds context information for the directive. */
14768 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14772 gimple
*stmt
= gsi_stmt (*gsi_p
);
14773 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
14774 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
14775 location_t loc
= gimple_location (stmt
);
14777 clauses
= gimple_omp_taskreg_clauses (stmt
);
14779 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
14780 par_body
= gimple_bind_body (par_bind
);
14781 child_fn
= ctx
->cb
.dst_fn
;
14782 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
14783 && !gimple_omp_parallel_combined_p (stmt
))
14785 struct walk_stmt_info wi
;
14788 memset (&wi
, 0, sizeof (wi
));
14790 wi
.val_only
= true;
14791 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
14793 gimple_omp_parallel_set_combined_p (stmt
, true);
14795 gimple_seq dep_ilist
= NULL
;
14796 gimple_seq dep_olist
= NULL
;
14797 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
14798 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
14800 push_gimplify_context ();
14801 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
14802 lower_depend_clauses (gimple_omp_task_clauses_ptr (stmt
),
14803 &dep_ilist
, &dep_olist
);
14806 if (ctx
->srecord_type
)
14807 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
14809 push_gimplify_context ();
14814 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
14815 lower_omp (&par_body
, ctx
);
14816 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
14817 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
14819 /* Declare all the variables created by mapping and the variables
14820 declared in the scope of the parallel body. */
14821 record_vars_into (ctx
->block_vars
, child_fn
);
14822 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
14824 if (ctx
->record_type
)
14827 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
14828 : ctx
->record_type
, ".omp_data_o");
14829 DECL_NAMELESS (ctx
->sender_decl
) = 1;
14830 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
14831 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
14836 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
14837 lower_send_shared_vars (&ilist
, &olist
, ctx
);
14839 if (ctx
->record_type
)
14841 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
14842 TREE_THIS_VOLATILE (clobber
) = 1;
14843 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
14847 /* Once all the expansions are done, sequence all the different
14848 fragments inside gimple_omp_body. */
14852 if (ctx
->record_type
)
14854 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
14855 /* fixup_child_record_type might have changed receiver_decl's type. */
14856 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
14857 gimple_seq_add_stmt (&new_body
,
14858 gimple_build_assign (ctx
->receiver_decl
, t
));
14861 gimple_seq_add_seq (&new_body
, par_ilist
);
14862 gimple_seq_add_seq (&new_body
, par_body
);
14863 gimple_seq_add_seq (&new_body
, par_rlist
);
14864 if (ctx
->cancellable
)
14865 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
14866 gimple_seq_add_seq (&new_body
, par_olist
);
14867 new_body
= maybe_catch_exception (new_body
);
14868 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
)
14869 gimple_seq_add_stmt (&new_body
,
14870 gimple_build_omp_continue (integer_zero_node
,
14871 integer_zero_node
));
14872 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
14873 gimple_omp_set_body (stmt
, new_body
);
14875 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
14876 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
14877 gimple_bind_add_seq (bind
, ilist
);
14878 gimple_bind_add_stmt (bind
, stmt
);
14879 gimple_bind_add_seq (bind
, olist
);
14881 pop_gimplify_context (NULL
);
14885 gimple_bind_add_seq (dep_bind
, dep_ilist
);
14886 gimple_bind_add_stmt (dep_bind
, bind
);
14887 gimple_bind_add_seq (dep_bind
, dep_olist
);
14888 pop_gimplify_context (dep_bind
);
14892 /* Lower the GIMPLE_OMP_TARGET in the current statement
14893 in GSI_P. CTX holds context information for the directive. */
14896 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
14899 tree child_fn
, t
, c
;
14900 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
14901 gbind
*tgt_bind
, *bind
, *dep_bind
= NULL
;
14902 gimple_seq tgt_body
, olist
, ilist
, fplist
, new_body
;
14903 location_t loc
= gimple_location (stmt
);
14904 bool offloaded
, data_region
;
14905 unsigned int map_cnt
= 0;
14906 bool has_depend
= false;
14908 offloaded
= is_gimple_omp_offloaded (stmt
);
14909 switch (gimple_omp_target_kind (stmt
))
14911 case GF_OMP_TARGET_KIND_REGION
:
14912 case GF_OMP_TARGET_KIND_UPDATE
:
14913 case GF_OMP_TARGET_KIND_ENTER_DATA
:
14914 case GF_OMP_TARGET_KIND_EXIT_DATA
:
14915 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
14916 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
14917 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
14918 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
14919 data_region
= false;
14921 case GF_OMP_TARGET_KIND_DATA
:
14922 case GF_OMP_TARGET_KIND_OACC_DATA
:
14923 data_region
= true;
14926 gcc_unreachable ();
14929 clauses
= gimple_omp_target_clauses (stmt
);
14931 gimple_seq dep_ilist
= NULL
;
14932 gimple_seq dep_olist
= NULL
;
14933 if (find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
14935 push_gimplify_context ();
14936 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
14937 lower_depend_clauses (gimple_omp_target_clauses_ptr (stmt
),
14938 &dep_ilist
, &dep_olist
);
14946 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
14947 tgt_body
= gimple_bind_body (tgt_bind
);
14949 else if (data_region
)
14950 tgt_body
= gimple_omp_body (stmt
);
14951 child_fn
= ctx
->cb
.dst_fn
;
14953 push_gimplify_context ();
14956 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
14957 switch (OMP_CLAUSE_CODE (c
))
14963 case OMP_CLAUSE_MAP
:
14965 /* First check what we're prepared to handle in the following. */
14966 switch (OMP_CLAUSE_MAP_KIND (c
))
14968 case GOMP_MAP_ALLOC
:
14970 case GOMP_MAP_FROM
:
14971 case GOMP_MAP_TOFROM
:
14972 case GOMP_MAP_POINTER
:
14973 case GOMP_MAP_TO_PSET
:
14974 case GOMP_MAP_FORCE_DEALLOC
:
14975 case GOMP_MAP_RELEASE
:
14976 case GOMP_MAP_ALWAYS_TO
:
14977 case GOMP_MAP_ALWAYS_FROM
:
14978 case GOMP_MAP_ALWAYS_TOFROM
:
14979 case GOMP_MAP_FIRSTPRIVATE_POINTER
:
14980 case GOMP_MAP_FIRSTPRIVATE_REFERENCE
:
14981 case GOMP_MAP_STRUCT
:
14982 case GOMP_MAP_ALWAYS_POINTER
:
14984 case GOMP_MAP_FORCE_ALLOC
:
14985 case GOMP_MAP_FORCE_TO
:
14986 case GOMP_MAP_FORCE_FROM
:
14987 case GOMP_MAP_FORCE_TOFROM
:
14988 case GOMP_MAP_FORCE_PRESENT
:
14989 case GOMP_MAP_FORCE_DEVICEPTR
:
14990 gcc_assert (is_gimple_omp_oacc (stmt
));
14993 gcc_unreachable ();
14997 case OMP_CLAUSE_TO
:
14998 case OMP_CLAUSE_FROM
:
15000 var
= OMP_CLAUSE_DECL (c
);
15003 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
15004 || (!OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15005 && (OMP_CLAUSE_MAP_KIND (c
)
15006 != GOMP_MAP_FIRSTPRIVATE_POINTER
)))
15011 if (DECL_SIZE (var
)
15012 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
15014 tree var2
= DECL_VALUE_EXPR (var
);
15015 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
15016 var2
= TREE_OPERAND (var2
, 0);
15017 gcc_assert (DECL_P (var2
));
15022 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15023 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
15024 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
))
15026 if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15028 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
))
15029 && varpool_node::get_create (var
)->offloadable
)
15032 tree type
= build_pointer_type (TREE_TYPE (var
));
15033 tree new_var
= lookup_decl (var
, ctx
);
15034 x
= create_tmp_var_raw (type
, get_name (new_var
));
15035 gimple_add_tmp_var (x
);
15036 x
= build_simple_mem_ref (x
);
15037 SET_DECL_VALUE_EXPR (new_var
, x
);
15038 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15043 if (!maybe_lookup_field (var
, ctx
))
15048 x
= build_receiver_ref (var
, true, ctx
);
15049 tree new_var
= lookup_decl (var
, ctx
);
15051 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15052 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
15053 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15054 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15055 x
= build_simple_mem_ref (x
);
15056 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15058 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
15059 if (is_reference (new_var
))
15061 /* Create a local object to hold the instance
15063 tree type
= TREE_TYPE (TREE_TYPE (new_var
));
15064 const char *id
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
15065 tree inst
= create_tmp_var (type
, id
);
15066 gimplify_assign (inst
, fold_indirect_ref (x
), &fplist
);
15067 x
= build_fold_addr_expr (inst
);
15069 gimplify_assign (new_var
, x
, &fplist
);
15071 else if (DECL_P (new_var
))
15073 SET_DECL_VALUE_EXPR (new_var
, x
);
15074 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15077 gcc_unreachable ();
15082 case OMP_CLAUSE_FIRSTPRIVATE
:
15083 if (is_oacc_parallel (ctx
))
15084 goto oacc_firstprivate
;
15086 var
= OMP_CLAUSE_DECL (c
);
15087 if (!is_reference (var
)
15088 && !is_gimple_reg_type (TREE_TYPE (var
)))
15090 tree new_var
= lookup_decl (var
, ctx
);
15091 if (is_variable_sized (var
))
15093 tree pvar
= DECL_VALUE_EXPR (var
);
15094 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15095 pvar
= TREE_OPERAND (pvar
, 0);
15096 gcc_assert (DECL_P (pvar
));
15097 tree new_pvar
= lookup_decl (pvar
, ctx
);
15098 x
= build_fold_indirect_ref (new_pvar
);
15099 TREE_THIS_NOTRAP (x
) = 1;
15102 x
= build_receiver_ref (var
, true, ctx
);
15103 SET_DECL_VALUE_EXPR (new_var
, x
);
15104 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15108 case OMP_CLAUSE_PRIVATE
:
15109 if (is_gimple_omp_oacc (ctx
->stmt
))
15111 var
= OMP_CLAUSE_DECL (c
);
15112 if (is_variable_sized (var
))
15114 tree new_var
= lookup_decl (var
, ctx
);
15115 tree pvar
= DECL_VALUE_EXPR (var
);
15116 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15117 pvar
= TREE_OPERAND (pvar
, 0);
15118 gcc_assert (DECL_P (pvar
));
15119 tree new_pvar
= lookup_decl (pvar
, ctx
);
15120 x
= build_fold_indirect_ref (new_pvar
);
15121 TREE_THIS_NOTRAP (x
) = 1;
15122 SET_DECL_VALUE_EXPR (new_var
, x
);
15123 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15127 case OMP_CLAUSE_USE_DEVICE_PTR
:
15128 case OMP_CLAUSE_IS_DEVICE_PTR
:
15129 var
= OMP_CLAUSE_DECL (c
);
15131 if (is_variable_sized (var
))
15133 tree new_var
= lookup_decl (var
, ctx
);
15134 tree pvar
= DECL_VALUE_EXPR (var
);
15135 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15136 pvar
= TREE_OPERAND (pvar
, 0);
15137 gcc_assert (DECL_P (pvar
));
15138 tree new_pvar
= lookup_decl (pvar
, ctx
);
15139 x
= build_fold_indirect_ref (new_pvar
);
15140 TREE_THIS_NOTRAP (x
) = 1;
15141 SET_DECL_VALUE_EXPR (new_var
, x
);
15142 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15144 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15146 tree new_var
= lookup_decl (var
, ctx
);
15147 tree type
= build_pointer_type (TREE_TYPE (var
));
15148 x
= create_tmp_var_raw (type
, get_name (new_var
));
15149 gimple_add_tmp_var (x
);
15150 x
= build_simple_mem_ref (x
);
15151 SET_DECL_VALUE_EXPR (new_var
, x
);
15152 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
15159 target_nesting_level
++;
15160 lower_omp (&tgt_body
, ctx
);
15161 target_nesting_level
--;
15163 else if (data_region
)
15164 lower_omp (&tgt_body
, ctx
);
15168 /* Declare all the variables created by mapping and the variables
15169 declared in the scope of the target body. */
15170 record_vars_into (ctx
->block_vars
, child_fn
);
15171 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
15176 if (ctx
->record_type
)
15179 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
15180 DECL_NAMELESS (ctx
->sender_decl
) = 1;
15181 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
15182 t
= make_tree_vec (3);
15183 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
15184 TREE_VEC_ELT (t
, 1)
15185 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
15186 ".omp_data_sizes");
15187 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
15188 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
15189 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
15190 tree tkind_type
= short_unsigned_type_node
;
15191 int talign_shift
= 8;
15192 TREE_VEC_ELT (t
, 2)
15193 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
15194 ".omp_data_kinds");
15195 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
15196 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
15197 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
15198 gimple_omp_target_set_data_arg (stmt
, t
);
15200 vec
<constructor_elt
, va_gc
> *vsize
;
15201 vec
<constructor_elt
, va_gc
> *vkind
;
15202 vec_alloc (vsize
, map_cnt
);
15203 vec_alloc (vkind
, map_cnt
);
15204 unsigned int map_idx
= 0;
15206 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15207 switch (OMP_CLAUSE_CODE (c
))
15209 tree ovar
, nc
, s
, purpose
, var
, x
, type
;
15210 unsigned int talign
;
15215 case OMP_CLAUSE_MAP
:
15216 case OMP_CLAUSE_TO
:
15217 case OMP_CLAUSE_FROM
:
15218 oacc_firstprivate_map
:
15220 ovar
= OMP_CLAUSE_DECL (c
);
15221 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15222 && (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
15223 || (OMP_CLAUSE_MAP_KIND (c
)
15224 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
15226 if (!DECL_P (ovar
))
15228 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15229 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
15231 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
15232 == get_base_address (ovar
));
15233 nc
= OMP_CLAUSE_CHAIN (c
);
15234 ovar
= OMP_CLAUSE_DECL (nc
);
15238 tree x
= build_sender_ref (ovar
, ctx
);
15240 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
15241 gimplify_assign (x
, v
, &ilist
);
15247 if (DECL_SIZE (ovar
)
15248 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
15250 tree ovar2
= DECL_VALUE_EXPR (ovar
);
15251 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
15252 ovar2
= TREE_OPERAND (ovar2
, 0);
15253 gcc_assert (DECL_P (ovar2
));
15256 if (!maybe_lookup_field (ovar
, ctx
))
15260 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
15261 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
15262 talign
= DECL_ALIGN_UNIT (ovar
);
15265 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
15266 x
= build_sender_ref (ovar
, ctx
);
15268 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
15269 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
15270 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
15271 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
15273 gcc_assert (offloaded
);
15275 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
15276 mark_addressable (avar
);
15277 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
15278 talign
= DECL_ALIGN_UNIT (avar
);
15279 avar
= build_fold_addr_expr (avar
);
15280 gimplify_assign (x
, avar
, &ilist
);
15282 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15284 gcc_assert (is_gimple_omp_oacc (ctx
->stmt
));
15285 if (!is_reference (var
))
15286 var
= build_fold_addr_expr (var
);
15288 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
15289 gimplify_assign (x
, var
, &ilist
);
15291 else if (is_gimple_reg (var
))
15293 gcc_assert (offloaded
);
15294 tree avar
= create_tmp_var (TREE_TYPE (var
));
15295 mark_addressable (avar
);
15296 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
15297 if (GOMP_MAP_COPY_TO_P (map_kind
)
15298 || map_kind
== GOMP_MAP_POINTER
15299 || map_kind
== GOMP_MAP_TO_PSET
15300 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
15301 gimplify_assign (avar
, var
, &ilist
);
15302 avar
= build_fold_addr_expr (avar
);
15303 gimplify_assign (x
, avar
, &ilist
);
15304 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
15305 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
15306 && !TYPE_READONLY (TREE_TYPE (var
)))
15308 x
= unshare_expr (x
);
15309 x
= build_simple_mem_ref (x
);
15310 gimplify_assign (var
, x
, &olist
);
15315 var
= build_fold_addr_expr (var
);
15316 gimplify_assign (x
, var
, &ilist
);
15320 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
15322 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
15323 s
= TREE_TYPE (ovar
);
15324 if (TREE_CODE (s
) == REFERENCE_TYPE
)
15326 s
= TYPE_SIZE_UNIT (s
);
15329 s
= OMP_CLAUSE_SIZE (c
);
15330 if (s
== NULL_TREE
)
15331 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
15332 s
= fold_convert (size_type_node
, s
);
15333 purpose
= size_int (map_idx
++);
15334 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
15335 if (TREE_CODE (s
) != INTEGER_CST
)
15336 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
15338 unsigned HOST_WIDE_INT tkind
, tkind_zero
;
15339 switch (OMP_CLAUSE_CODE (c
))
15341 case OMP_CLAUSE_MAP
:
15342 tkind
= OMP_CLAUSE_MAP_KIND (c
);
15343 tkind_zero
= tkind
;
15344 if (OMP_CLAUSE_MAP_MAYBE_ZERO_LENGTH_ARRAY_SECTION (c
))
15347 case GOMP_MAP_ALLOC
:
15349 case GOMP_MAP_FROM
:
15350 case GOMP_MAP_TOFROM
:
15351 case GOMP_MAP_ALWAYS_TO
:
15352 case GOMP_MAP_ALWAYS_FROM
:
15353 case GOMP_MAP_ALWAYS_TOFROM
:
15354 case GOMP_MAP_RELEASE
:
15355 tkind_zero
= GOMP_MAP_ZERO_LEN_ARRAY_SECTION
;
15357 case GOMP_MAP_DELETE
:
15358 tkind_zero
= GOMP_MAP_DELETE_ZERO_LEN_ARRAY_SECTION
;
15362 if (tkind_zero
!= tkind
)
15364 if (integer_zerop (s
))
15365 tkind
= tkind_zero
;
15366 else if (integer_nonzerop (s
))
15367 tkind_zero
= tkind
;
15370 case OMP_CLAUSE_FIRSTPRIVATE
:
15371 gcc_checking_assert (is_gimple_omp_oacc (ctx
->stmt
));
15372 tkind
= GOMP_MAP_TO
;
15373 tkind_zero
= tkind
;
15375 case OMP_CLAUSE_TO
:
15376 tkind
= GOMP_MAP_TO
;
15377 tkind_zero
= tkind
;
15379 case OMP_CLAUSE_FROM
:
15380 tkind
= GOMP_MAP_FROM
;
15381 tkind_zero
= tkind
;
15384 gcc_unreachable ();
15386 gcc_checking_assert (tkind
15387 < (HOST_WIDE_INT_C (1U) << talign_shift
));
15388 gcc_checking_assert (tkind_zero
15389 < (HOST_WIDE_INT_C (1U) << talign_shift
));
15390 talign
= ceil_log2 (talign
);
15391 tkind
|= talign
<< talign_shift
;
15392 tkind_zero
|= talign
<< talign_shift
;
15393 gcc_checking_assert (tkind
15394 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
15395 gcc_checking_assert (tkind_zero
15396 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
15397 if (tkind
== tkind_zero
)
15398 x
= build_int_cstu (tkind_type
, tkind
);
15401 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 0;
15402 x
= build3 (COND_EXPR
, tkind_type
,
15403 fold_build2 (EQ_EXPR
, boolean_type_node
,
15404 unshare_expr (s
), size_zero_node
),
15405 build_int_cstu (tkind_type
, tkind_zero
),
15406 build_int_cstu (tkind_type
, tkind
));
15408 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
, x
);
15413 case OMP_CLAUSE_FIRSTPRIVATE
:
15414 if (is_oacc_parallel (ctx
))
15415 goto oacc_firstprivate_map
;
15416 ovar
= OMP_CLAUSE_DECL (c
);
15417 if (is_reference (ovar
))
15418 talign
= TYPE_ALIGN_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
15420 talign
= DECL_ALIGN_UNIT (ovar
);
15421 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
15422 x
= build_sender_ref (ovar
, ctx
);
15423 tkind
= GOMP_MAP_FIRSTPRIVATE
;
15424 type
= TREE_TYPE (ovar
);
15425 if (is_reference (ovar
))
15426 type
= TREE_TYPE (type
);
15427 bool use_firstprivate_int
, force_addr
;
15428 use_firstprivate_int
= false;
15429 force_addr
= false;
15430 if ((INTEGRAL_TYPE_P (type
)
15431 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
15432 || TREE_CODE (type
) == POINTER_TYPE
)
15433 use_firstprivate_int
= true;
15436 if (is_reference (var
))
15437 use_firstprivate_int
= false;
15438 else if (is_gimple_reg (var
))
15440 if (DECL_HAS_VALUE_EXPR_P (var
))
15442 tree v
= get_base_address (var
);
15443 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
15445 use_firstprivate_int
= false;
15449 switch (TREE_CODE (v
))
15453 use_firstprivate_int
= false;
15462 use_firstprivate_int
= false;
15464 if (use_firstprivate_int
)
15466 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
15468 if (is_reference (var
))
15469 t
= build_simple_mem_ref (var
);
15470 if (TREE_CODE (type
) != POINTER_TYPE
)
15471 t
= fold_convert (pointer_sized_int_node
, t
);
15472 t
= fold_convert (TREE_TYPE (x
), t
);
15473 gimplify_assign (x
, t
, &ilist
);
15475 else if (is_reference (var
))
15476 gimplify_assign (x
, var
, &ilist
);
15477 else if (!force_addr
&& is_gimple_reg (var
))
15479 tree avar
= create_tmp_var (TREE_TYPE (var
));
15480 mark_addressable (avar
);
15481 gimplify_assign (avar
, var
, &ilist
);
15482 avar
= build_fold_addr_expr (avar
);
15483 gimplify_assign (x
, avar
, &ilist
);
15487 var
= build_fold_addr_expr (var
);
15488 gimplify_assign (x
, var
, &ilist
);
15490 if (tkind
== GOMP_MAP_FIRSTPRIVATE_INT
)
15492 else if (is_reference (var
))
15493 s
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (ovar
)));
15495 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
15496 s
= fold_convert (size_type_node
, s
);
15497 purpose
= size_int (map_idx
++);
15498 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
15499 if (TREE_CODE (s
) != INTEGER_CST
)
15500 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
15502 gcc_checking_assert (tkind
15503 < (HOST_WIDE_INT_C (1U) << talign_shift
));
15504 talign
= ceil_log2 (talign
);
15505 tkind
|= talign
<< talign_shift
;
15506 gcc_checking_assert (tkind
15507 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
15508 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
15509 build_int_cstu (tkind_type
, tkind
));
15512 case OMP_CLAUSE_USE_DEVICE_PTR
:
15513 case OMP_CLAUSE_IS_DEVICE_PTR
:
15514 ovar
= OMP_CLAUSE_DECL (c
);
15515 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
15516 x
= build_sender_ref (ovar
, ctx
);
15517 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
15518 tkind
= GOMP_MAP_USE_DEVICE_PTR
;
15520 tkind
= GOMP_MAP_FIRSTPRIVATE_INT
;
15521 type
= TREE_TYPE (ovar
);
15522 if (TREE_CODE (type
) == ARRAY_TYPE
)
15523 var
= build_fold_addr_expr (var
);
15526 if (is_reference (ovar
))
15528 type
= TREE_TYPE (type
);
15529 if (TREE_CODE (type
) != ARRAY_TYPE
)
15530 var
= build_simple_mem_ref (var
);
15531 var
= fold_convert (TREE_TYPE (x
), var
);
15534 gimplify_assign (x
, var
, &ilist
);
15536 purpose
= size_int (map_idx
++);
15537 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
15538 gcc_checking_assert (tkind
15539 < (HOST_WIDE_INT_C (1U) << talign_shift
));
15540 gcc_checking_assert (tkind
15541 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
15542 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
15543 build_int_cstu (tkind_type
, tkind
));
15547 gcc_assert (map_idx
== map_cnt
);
15549 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
15550 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
15551 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
15552 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
15553 for (int i
= 1; i
<= 2; i
++)
15554 if (!TREE_STATIC (TREE_VEC_ELT (t
, i
)))
15556 gimple_seq initlist
= NULL
;
15557 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
15558 TREE_VEC_ELT (t
, i
)),
15559 &initlist
, true, NULL_TREE
);
15560 gimple_seq_add_seq (&ilist
, initlist
);
15562 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, i
)),
15564 TREE_THIS_VOLATILE (clobber
) = 1;
15565 gimple_seq_add_stmt (&olist
,
15566 gimple_build_assign (TREE_VEC_ELT (t
, i
),
15570 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
15571 TREE_THIS_VOLATILE (clobber
) = 1;
15572 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
15576 /* Once all the expansions are done, sequence all the different
15577 fragments inside gimple_omp_body. */
15582 && ctx
->record_type
)
15584 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
15585 /* fixup_child_record_type might have changed receiver_decl's type. */
15586 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
15587 gimple_seq_add_stmt (&new_body
,
15588 gimple_build_assign (ctx
->receiver_decl
, t
));
15590 gimple_seq_add_seq (&new_body
, fplist
);
15592 if (offloaded
|| data_region
)
15594 tree prev
= NULL_TREE
;
15595 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15596 switch (OMP_CLAUSE_CODE (c
))
15601 case OMP_CLAUSE_FIRSTPRIVATE
:
15602 if (is_gimple_omp_oacc (ctx
->stmt
))
15604 var
= OMP_CLAUSE_DECL (c
);
15605 if (is_reference (var
)
15606 || is_gimple_reg_type (TREE_TYPE (var
)))
15608 tree new_var
= lookup_decl (var
, ctx
);
15610 type
= TREE_TYPE (var
);
15611 if (is_reference (var
))
15612 type
= TREE_TYPE (type
);
15613 bool use_firstprivate_int
;
15614 use_firstprivate_int
= false;
15615 if ((INTEGRAL_TYPE_P (type
)
15616 && TYPE_PRECISION (type
) <= POINTER_SIZE
)
15617 || TREE_CODE (type
) == POINTER_TYPE
)
15618 use_firstprivate_int
= true;
15621 tree v
= lookup_decl_in_outer_ctx (var
, ctx
);
15622 if (is_reference (v
))
15623 use_firstprivate_int
= false;
15624 else if (is_gimple_reg (v
))
15626 if (DECL_HAS_VALUE_EXPR_P (v
))
15628 v
= get_base_address (v
);
15629 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
15630 use_firstprivate_int
= false;
15632 switch (TREE_CODE (v
))
15636 use_firstprivate_int
= false;
15644 use_firstprivate_int
= false;
15646 if (use_firstprivate_int
)
15648 x
= build_receiver_ref (var
, false, ctx
);
15649 if (TREE_CODE (type
) != POINTER_TYPE
)
15650 x
= fold_convert (pointer_sized_int_node
, x
);
15651 x
= fold_convert (type
, x
);
15652 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
15654 if (is_reference (var
))
15656 tree v
= create_tmp_var_raw (type
, get_name (var
));
15657 gimple_add_tmp_var (v
);
15658 TREE_ADDRESSABLE (v
) = 1;
15659 gimple_seq_add_stmt (&new_body
,
15660 gimple_build_assign (v
, x
));
15661 x
= build_fold_addr_expr (v
);
15663 gimple_seq_add_stmt (&new_body
,
15664 gimple_build_assign (new_var
, x
));
15668 x
= build_receiver_ref (var
, !is_reference (var
), ctx
);
15669 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
15671 gimple_seq_add_stmt (&new_body
,
15672 gimple_build_assign (new_var
, x
));
15675 else if (is_variable_sized (var
))
15677 tree pvar
= DECL_VALUE_EXPR (var
);
15678 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15679 pvar
= TREE_OPERAND (pvar
, 0);
15680 gcc_assert (DECL_P (pvar
));
15681 tree new_var
= lookup_decl (pvar
, ctx
);
15682 x
= build_receiver_ref (var
, false, ctx
);
15683 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15684 gimple_seq_add_stmt (&new_body
,
15685 gimple_build_assign (new_var
, x
));
15688 case OMP_CLAUSE_PRIVATE
:
15689 if (is_gimple_omp_oacc (ctx
->stmt
))
15691 var
= OMP_CLAUSE_DECL (c
);
15692 if (is_reference (var
))
15694 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
15695 tree new_var
= lookup_decl (var
, ctx
);
15696 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
15697 if (TREE_CONSTANT (x
))
15699 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
15701 gimple_add_tmp_var (x
);
15702 TREE_ADDRESSABLE (x
) = 1;
15703 x
= build_fold_addr_expr_loc (clause_loc
, x
);
15708 = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
15709 tree rtype
= TREE_TYPE (TREE_TYPE (new_var
));
15710 tree al
= size_int (TYPE_ALIGN (rtype
));
15711 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
15714 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
15715 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15716 gimple_seq_add_stmt (&new_body
,
15717 gimple_build_assign (new_var
, x
));
15720 case OMP_CLAUSE_USE_DEVICE_PTR
:
15721 case OMP_CLAUSE_IS_DEVICE_PTR
:
15722 var
= OMP_CLAUSE_DECL (c
);
15723 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_USE_DEVICE_PTR
)
15724 x
= build_sender_ref (var
, ctx
);
15726 x
= build_receiver_ref (var
, false, ctx
);
15727 if (is_variable_sized (var
))
15729 tree pvar
= DECL_VALUE_EXPR (var
);
15730 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15731 pvar
= TREE_OPERAND (pvar
, 0);
15732 gcc_assert (DECL_P (pvar
));
15733 tree new_var
= lookup_decl (pvar
, ctx
);
15734 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15735 gimple_seq_add_stmt (&new_body
,
15736 gimple_build_assign (new_var
, x
));
15738 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
15740 tree new_var
= lookup_decl (var
, ctx
);
15741 new_var
= DECL_VALUE_EXPR (new_var
);
15742 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
15743 new_var
= TREE_OPERAND (new_var
, 0);
15744 gcc_assert (DECL_P (new_var
));
15745 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15746 gimple_seq_add_stmt (&new_body
,
15747 gimple_build_assign (new_var
, x
));
15751 tree type
= TREE_TYPE (var
);
15752 tree new_var
= lookup_decl (var
, ctx
);
15753 if (is_reference (var
))
15755 type
= TREE_TYPE (type
);
15756 if (TREE_CODE (type
) != ARRAY_TYPE
)
15758 tree v
= create_tmp_var_raw (type
, get_name (var
));
15759 gimple_add_tmp_var (v
);
15760 TREE_ADDRESSABLE (v
) = 1;
15761 x
= fold_convert (type
, x
);
15762 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
,
15764 gimple_seq_add_stmt (&new_body
,
15765 gimple_build_assign (v
, x
));
15766 x
= build_fold_addr_expr (v
);
15769 x
= fold_convert (TREE_TYPE (new_var
), x
);
15770 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15771 gimple_seq_add_stmt (&new_body
,
15772 gimple_build_assign (new_var
, x
));
15776 /* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
15777 so that firstprivate vars holding OMP_CLAUSE_SIZE if needed
15778 are already handled. */
15779 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
15780 switch (OMP_CLAUSE_CODE (c
))
15785 case OMP_CLAUSE_MAP
:
15786 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_POINTER
15787 || OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
15789 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
15790 HOST_WIDE_INT offset
= 0;
15792 var
= OMP_CLAUSE_DECL (c
);
15794 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
15795 && is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
15797 && varpool_node::get_create (var
)->offloadable
)
15799 if (TREE_CODE (var
) == INDIRECT_REF
15800 && TREE_CODE (TREE_OPERAND (var
, 0)) == COMPONENT_REF
)
15801 var
= TREE_OPERAND (var
, 0);
15802 if (TREE_CODE (var
) == COMPONENT_REF
)
15804 var
= get_addr_base_and_unit_offset (var
, &offset
);
15805 gcc_assert (var
!= NULL_TREE
&& DECL_P (var
));
15807 else if (DECL_SIZE (var
)
15808 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
15810 tree var2
= DECL_VALUE_EXPR (var
);
15811 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
15812 var2
= TREE_OPERAND (var2
, 0);
15813 gcc_assert (DECL_P (var2
));
15816 tree new_var
= lookup_decl (var
, ctx
), x
;
15817 tree type
= TREE_TYPE (new_var
);
15819 if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == INDIRECT_REF
15820 && (TREE_CODE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0))
15823 type
= TREE_TYPE (TREE_OPERAND (OMP_CLAUSE_DECL (c
), 0));
15825 new_var
= build2 (MEM_REF
, type
,
15826 build_fold_addr_expr (new_var
),
15827 build_int_cst (build_pointer_type (type
),
15830 else if (TREE_CODE (OMP_CLAUSE_DECL (c
)) == COMPONENT_REF
)
15832 type
= TREE_TYPE (OMP_CLAUSE_DECL (c
));
15833 is_ref
= TREE_CODE (type
) == REFERENCE_TYPE
;
15834 new_var
= build2 (MEM_REF
, type
,
15835 build_fold_addr_expr (new_var
),
15836 build_int_cst (build_pointer_type (type
),
15840 is_ref
= is_reference (var
);
15841 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)
15843 bool ref_to_array
= false;
15846 type
= TREE_TYPE (type
);
15847 if (TREE_CODE (type
) == ARRAY_TYPE
)
15849 type
= build_pointer_type (type
);
15850 ref_to_array
= true;
15853 else if (TREE_CODE (type
) == ARRAY_TYPE
)
15855 tree decl2
= DECL_VALUE_EXPR (new_var
);
15856 gcc_assert (TREE_CODE (decl2
) == MEM_REF
);
15857 decl2
= TREE_OPERAND (decl2
, 0);
15858 gcc_assert (DECL_P (decl2
));
15860 type
= TREE_TYPE (new_var
);
15862 x
= build_receiver_ref (OMP_CLAUSE_DECL (prev
), false, ctx
);
15863 x
= fold_convert_loc (clause_loc
, type
, x
);
15864 if (!integer_zerop (OMP_CLAUSE_SIZE (c
)))
15866 tree bias
= OMP_CLAUSE_SIZE (c
);
15868 bias
= lookup_decl (bias
, ctx
);
15869 bias
= fold_convert_loc (clause_loc
, sizetype
, bias
);
15870 bias
= fold_build1_loc (clause_loc
, NEGATE_EXPR
, sizetype
,
15872 x
= fold_build2_loc (clause_loc
, POINTER_PLUS_EXPR
,
15873 TREE_TYPE (x
), x
, bias
);
15876 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
15877 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15878 if (is_ref
&& !ref_to_array
)
15880 tree t
= create_tmp_var_raw (type
, get_name (var
));
15881 gimple_add_tmp_var (t
);
15882 TREE_ADDRESSABLE (t
) = 1;
15883 gimple_seq_add_stmt (&new_body
,
15884 gimple_build_assign (t
, x
));
15885 x
= build_fold_addr_expr_loc (clause_loc
, t
);
15887 gimple_seq_add_stmt (&new_body
,
15888 gimple_build_assign (new_var
, x
));
15891 else if (OMP_CLAUSE_CHAIN (c
)
15892 && OMP_CLAUSE_CODE (OMP_CLAUSE_CHAIN (c
))
15894 && (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
15895 == GOMP_MAP_FIRSTPRIVATE_POINTER
15896 || (OMP_CLAUSE_MAP_KIND (OMP_CLAUSE_CHAIN (c
))
15897 == GOMP_MAP_FIRSTPRIVATE_REFERENCE
)))
15900 case OMP_CLAUSE_PRIVATE
:
15901 var
= OMP_CLAUSE_DECL (c
);
15902 if (is_variable_sized (var
))
15904 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
15905 tree new_var
= lookup_decl (var
, ctx
);
15906 tree pvar
= DECL_VALUE_EXPR (var
);
15907 gcc_assert (TREE_CODE (pvar
) == INDIRECT_REF
);
15908 pvar
= TREE_OPERAND (pvar
, 0);
15909 gcc_assert (DECL_P (pvar
));
15910 tree new_pvar
= lookup_decl (pvar
, ctx
);
15911 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN
);
15912 tree al
= size_int (DECL_ALIGN (var
));
15913 tree x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
15914 x
= build_call_expr_loc (clause_loc
, atmp
, 2, x
, al
);
15915 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_pvar
), x
);
15916 gimplify_expr (&x
, &new_body
, NULL
, is_gimple_val
, fb_rvalue
);
15917 gimple_seq_add_stmt (&new_body
,
15918 gimple_build_assign (new_pvar
, x
));
15923 gimple_seq fork_seq
= NULL
;
15924 gimple_seq join_seq
= NULL
;
15926 if (is_oacc_parallel (ctx
))
15928 /* If there are reductions on the offloaded region itself, treat
15929 them as a dummy GANG loop. */
15930 tree level
= build_int_cst (integer_type_node
, GOMP_DIM_GANG
);
15932 lower_oacc_reductions (gimple_location (ctx
->stmt
), clauses
, level
,
15933 false, NULL
, NULL
, &fork_seq
, &join_seq
, ctx
);
15936 gimple_seq_add_seq (&new_body
, fork_seq
);
15937 gimple_seq_add_seq (&new_body
, tgt_body
);
15938 gimple_seq_add_seq (&new_body
, join_seq
);
15941 new_body
= maybe_catch_exception (new_body
);
15943 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
15944 gimple_omp_set_body (stmt
, new_body
);
15947 bind
= gimple_build_bind (NULL
, NULL
,
15948 tgt_bind
? gimple_bind_block (tgt_bind
)
15950 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
15951 gimple_bind_add_seq (bind
, ilist
);
15952 gimple_bind_add_stmt (bind
, stmt
);
15953 gimple_bind_add_seq (bind
, olist
);
15955 pop_gimplify_context (NULL
);
15959 gimple_bind_add_seq (dep_bind
, dep_ilist
);
15960 gimple_bind_add_stmt (dep_bind
, bind
);
15961 gimple_bind_add_seq (dep_bind
, dep_olist
);
15962 pop_gimplify_context (dep_bind
);
15966 /* Expand code for an OpenMP teams directive. */
15969 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
15971 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
15972 push_gimplify_context ();
15974 tree block
= make_node (BLOCK
);
15975 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
15976 gsi_replace (gsi_p
, bind
, true);
15977 gimple_seq bind_body
= NULL
;
15978 gimple_seq dlist
= NULL
;
15979 gimple_seq olist
= NULL
;
15981 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
15982 OMP_CLAUSE_NUM_TEAMS
);
15983 if (num_teams
== NULL_TREE
)
15984 num_teams
= build_int_cst (unsigned_type_node
, 0);
15987 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
15988 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
15989 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
15991 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
15992 OMP_CLAUSE_THREAD_LIMIT
);
15993 if (thread_limit
== NULL_TREE
)
15994 thread_limit
= build_int_cst (unsigned_type_node
, 0);
15997 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
15998 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
15999 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
16003 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
16004 &bind_body
, &dlist
, ctx
, NULL
);
16005 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
16006 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
16007 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
16009 location_t loc
= gimple_location (teams_stmt
);
16010 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
16011 gimple
*call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
16012 gimple_set_location (call
, loc
);
16013 gimple_seq_add_stmt (&bind_body
, call
);
16015 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
16016 gimple_omp_set_body (teams_stmt
, NULL
);
16017 gimple_seq_add_seq (&bind_body
, olist
);
16018 gimple_seq_add_seq (&bind_body
, dlist
);
16019 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
16020 gimple_bind_set_body (bind
, bind_body
);
16022 pop_gimplify_context (bind
);
16024 gimple_bind_append_vars (bind
, ctx
->block_vars
);
16025 BLOCK_VARS (block
) = ctx
->block_vars
;
16026 if (BLOCK_VARS (block
))
16027 TREE_USED (block
) = 1;
16031 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
16032 regimplified. If DATA is non-NULL, lower_omp_1 is outside
16033 of OMP context, but with task_shared_vars set. */
16036 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
16041 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
16042 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
16045 if (task_shared_vars
16047 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
16050 /* If a global variable has been privatized, TREE_CONSTANT on
16051 ADDR_EXPR might be wrong. */
16052 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
16053 recompute_tree_invariant_for_addr_expr (t
);
16055 *walk_subtrees
= !IS_TYPE_OR_DECL_P (t
);
16059 /* Data to be communicated between lower_omp_regimplify_operands and
16060 lower_omp_regimplify_operands_p. */
16062 struct lower_omp_regimplify_operands_data
16068 /* Helper function for lower_omp_regimplify_operands. Find
16069 omp_member_access_dummy_var vars and adjust temporarily their
16070 DECL_VALUE_EXPRs if needed. */
16073 lower_omp_regimplify_operands_p (tree
*tp
, int *walk_subtrees
,
16076 tree t
= omp_member_access_dummy_var (*tp
);
16079 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
16080 lower_omp_regimplify_operands_data
*ldata
16081 = (lower_omp_regimplify_operands_data
*) wi
->info
;
16082 tree o
= maybe_lookup_decl (t
, ldata
->ctx
);
16085 ldata
->decls
->safe_push (DECL_VALUE_EXPR (*tp
));
16086 ldata
->decls
->safe_push (*tp
);
16087 tree v
= unshare_and_remap (DECL_VALUE_EXPR (*tp
), t
, o
);
16088 SET_DECL_VALUE_EXPR (*tp
, v
);
16091 *walk_subtrees
= !IS_TYPE_OR_DECL_P (*tp
);
16095 /* Wrapper around gimple_regimplify_operands that adjusts DECL_VALUE_EXPRs
16096 of omp_member_access_dummy_var vars during regimplification. */
16099 lower_omp_regimplify_operands (omp_context
*ctx
, gimple
*stmt
,
16100 gimple_stmt_iterator
*gsi_p
)
16102 auto_vec
<tree
, 10> decls
;
16105 struct walk_stmt_info wi
;
16106 memset (&wi
, '\0', sizeof (wi
));
16107 struct lower_omp_regimplify_operands_data data
;
16109 data
.decls
= &decls
;
16111 walk_gimple_op (stmt
, lower_omp_regimplify_operands_p
, &wi
);
16113 gimple_regimplify_operands (stmt
, gsi_p
);
16114 while (!decls
.is_empty ())
16116 tree t
= decls
.pop ();
16117 tree v
= decls
.pop ();
16118 SET_DECL_VALUE_EXPR (t
, v
);
16123 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
16125 gimple
*stmt
= gsi_stmt (*gsi_p
);
16126 struct walk_stmt_info wi
;
16129 if (gimple_has_location (stmt
))
16130 input_location
= gimple_location (stmt
);
16132 if (task_shared_vars
)
16133 memset (&wi
, '\0', sizeof (wi
));
16135 /* If we have issued syntax errors, avoid doing any heavy lifting.
16136 Just replace the OMP directives with a NOP to avoid
16137 confusing RTL expansion. */
16138 if (seen_error () && is_gimple_omp (stmt
))
16140 gsi_replace (gsi_p
, gimple_build_nop (), true);
16144 switch (gimple_code (stmt
))
16148 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
16149 if ((ctx
|| task_shared_vars
)
16150 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
16151 lower_omp_regimplify_p
,
16152 ctx
? NULL
: &wi
, NULL
)
16153 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
16154 lower_omp_regimplify_p
,
16155 ctx
? NULL
: &wi
, NULL
)))
16156 lower_omp_regimplify_operands (ctx
, cond_stmt
, gsi_p
);
16160 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
16162 case GIMPLE_EH_FILTER
:
16163 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
16166 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
16167 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
16169 case GIMPLE_TRANSACTION
:
16170 lower_omp (gimple_transaction_body_ptr (
16171 as_a
<gtransaction
*> (stmt
)),
16175 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
16177 case GIMPLE_OMP_PARALLEL
:
16178 case GIMPLE_OMP_TASK
:
16179 ctx
= maybe_lookup_ctx (stmt
);
16181 if (ctx
->cancellable
)
16182 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16183 lower_omp_taskreg (gsi_p
, ctx
);
16185 case GIMPLE_OMP_FOR
:
16186 ctx
= maybe_lookup_ctx (stmt
);
16188 if (ctx
->cancellable
)
16189 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16190 lower_omp_for (gsi_p
, ctx
);
16192 case GIMPLE_OMP_SECTIONS
:
16193 ctx
= maybe_lookup_ctx (stmt
);
16195 if (ctx
->cancellable
)
16196 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
16197 lower_omp_sections (gsi_p
, ctx
);
16199 case GIMPLE_OMP_SINGLE
:
16200 ctx
= maybe_lookup_ctx (stmt
);
16202 lower_omp_single (gsi_p
, ctx
);
16204 case GIMPLE_OMP_MASTER
:
16205 ctx
= maybe_lookup_ctx (stmt
);
16207 lower_omp_master (gsi_p
, ctx
);
16209 case GIMPLE_OMP_TASKGROUP
:
16210 ctx
= maybe_lookup_ctx (stmt
);
16212 lower_omp_taskgroup (gsi_p
, ctx
);
16214 case GIMPLE_OMP_ORDERED
:
16215 ctx
= maybe_lookup_ctx (stmt
);
16217 lower_omp_ordered (gsi_p
, ctx
);
16219 case GIMPLE_OMP_CRITICAL
:
16220 ctx
= maybe_lookup_ctx (stmt
);
16222 lower_omp_critical (gsi_p
, ctx
);
16224 case GIMPLE_OMP_ATOMIC_LOAD
:
16225 if ((ctx
|| task_shared_vars
)
16226 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
16227 as_a
<gomp_atomic_load
*> (stmt
)),
16228 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
16229 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
16231 case GIMPLE_OMP_TARGET
:
16232 ctx
= maybe_lookup_ctx (stmt
);
16234 lower_omp_target (gsi_p
, ctx
);
16236 case GIMPLE_OMP_TEAMS
:
16237 ctx
= maybe_lookup_ctx (stmt
);
16239 lower_omp_teams (gsi_p
, ctx
);
16243 call_stmt
= as_a
<gcall
*> (stmt
);
16244 fndecl
= gimple_call_fndecl (call_stmt
);
16246 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
16247 switch (DECL_FUNCTION_CODE (fndecl
))
16249 case BUILT_IN_GOMP_BARRIER
:
16253 case BUILT_IN_GOMP_CANCEL
:
16254 case BUILT_IN_GOMP_CANCELLATION_POINT
:
16257 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
16258 cctx
= cctx
->outer
;
16259 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
16260 if (!cctx
->cancellable
)
16262 if (DECL_FUNCTION_CODE (fndecl
)
16263 == BUILT_IN_GOMP_CANCELLATION_POINT
)
16265 stmt
= gimple_build_nop ();
16266 gsi_replace (gsi_p
, stmt
, false);
16270 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
16272 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
16273 gimple_call_set_fndecl (call_stmt
, fndecl
);
16274 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
16277 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
16278 gimple_call_set_lhs (call_stmt
, lhs
);
16279 tree fallthru_label
;
16280 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
16282 g
= gimple_build_label (fallthru_label
);
16283 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
16284 g
= gimple_build_cond (NE_EXPR
, lhs
,
16285 fold_convert (TREE_TYPE (lhs
),
16286 boolean_false_node
),
16287 cctx
->cancel_label
, fallthru_label
);
16288 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
16295 if ((ctx
|| task_shared_vars
)
16296 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
16299 /* Just remove clobbers, this should happen only if we have
16300 "privatized" local addressable variables in SIMD regions,
16301 the clobber isn't needed in that case and gimplifying address
16302 of the ARRAY_REF into a pointer and creating MEM_REF based
16303 clobber would create worse code than we get with the clobber
16305 if (gimple_clobber_p (stmt
))
16307 gsi_replace (gsi_p
, gimple_build_nop (), true);
16310 lower_omp_regimplify_operands (ctx
, stmt
, gsi_p
);
16317 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
16319 location_t saved_location
= input_location
;
16320 gimple_stmt_iterator gsi
;
16321 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
16322 lower_omp_1 (&gsi
, ctx
);
16323 /* During gimplification, we haven't folded statments inside offloading
16324 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */
16325 if (target_nesting_level
|| taskreg_nesting_level
)
16326 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
16328 input_location
= saved_location
;
16331 /* Main entry point. */
16333 static unsigned int
16334 execute_lower_omp (void)
16340 /* This pass always runs, to provide PROP_gimple_lomp.
16341 But often, there is nothing to do. */
16342 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
16343 && flag_openmp_simd
== 0)
16346 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
16347 delete_omp_context
);
16349 body
= gimple_body (current_function_decl
);
16350 scan_omp (&body
, NULL
);
16351 gcc_assert (taskreg_nesting_level
== 0);
16352 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
16353 finish_taskreg_scan (ctx
);
16354 taskreg_contexts
.release ();
16356 if (all_contexts
->root
)
16358 if (task_shared_vars
)
16359 push_gimplify_context ();
16360 lower_omp (&body
, NULL
);
16361 if (task_shared_vars
)
16362 pop_gimplify_context (NULL
);
16367 splay_tree_delete (all_contexts
);
16368 all_contexts
= NULL
;
16370 BITMAP_FREE (task_shared_vars
);
16376 const pass_data pass_data_lower_omp
=
16378 GIMPLE_PASS
, /* type */
16379 "omplower", /* name */
16380 OPTGROUP_NONE
, /* optinfo_flags */
16381 TV_NONE
, /* tv_id */
16382 PROP_gimple_any
, /* properties_required */
16383 PROP_gimple_lomp
, /* properties_provided */
16384 0, /* properties_destroyed */
16385 0, /* todo_flags_start */
16386 0, /* todo_flags_finish */
16389 class pass_lower_omp
: public gimple_opt_pass
16392 pass_lower_omp (gcc::context
*ctxt
)
16393 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
16396 /* opt_pass methods: */
16397 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
16399 }; // class pass_lower_omp
16401 } // anon namespace
16404 make_pass_lower_omp (gcc::context
*ctxt
)
16406 return new pass_lower_omp (ctxt
);
16409 /* The following is a utility to diagnose structured block violations.
16410 It is not part of the "omplower" pass, as that's invoked too late. It
16411 should be invoked by the respective front ends after gimplification. */
16413 static splay_tree all_labels
;
16415 /* Check for mismatched contexts and generate an error if needed. Return
16416 true if an error is detected. */
16419 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
16420 gimple
*branch_ctx
, gimple
*label_ctx
)
16422 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
16423 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
16425 if (label_ctx
== branch_ctx
)
16428 const char* kind
= NULL
;
16433 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
16434 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
16436 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
16437 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
16438 kind
= "Cilk Plus";
16442 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
16443 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
16445 gcc_checking_assert (kind
== NULL
);
16451 gcc_checking_assert (flag_openmp
);
16456 Previously we kept track of the label's entire context in diagnose_sb_[12]
16457 so we could traverse it and issue a correct "exit" or "enter" error
16458 message upon a structured block violation.
16460 We built the context by building a list with tree_cons'ing, but there is
16461 no easy counterpart in gimple tuples. It seems like far too much work
16462 for issuing exit/enter error messages. If someone really misses the
16463 distinct error message... patches welcome.
16467 /* Try to avoid confusing the user by producing and error message
16468 with correct "exit" or "enter" verbiage. We prefer "exit"
16469 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
16470 if (branch_ctx
== NULL
)
16476 if (TREE_VALUE (label_ctx
) == branch_ctx
)
16481 label_ctx
= TREE_CHAIN (label_ctx
);
16486 error ("invalid exit from %s structured block", kind
);
16488 error ("invalid entry to %s structured block", kind
);
16491 /* If it's obvious we have an invalid entry, be specific about the error. */
16492 if (branch_ctx
== NULL
)
16493 error ("invalid entry to %s structured block", kind
);
16496 /* Otherwise, be vague and lazy, but efficient. */
16497 error ("invalid branch to/from %s structured block", kind
);
16500 gsi_replace (gsi_p
, gimple_build_nop (), false);
16504 /* Pass 1: Create a minimal tree of structured blocks, and record
16505 where each label is found. */
16508 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
16509 struct walk_stmt_info
*wi
)
16511 gimple
*context
= (gimple
*) wi
->info
;
16512 gimple
*inner_context
;
16513 gimple
*stmt
= gsi_stmt (*gsi_p
);
16515 *handled_ops_p
= true;
16517 switch (gimple_code (stmt
))
16521 case GIMPLE_OMP_PARALLEL
:
16522 case GIMPLE_OMP_TASK
:
16523 case GIMPLE_OMP_SECTIONS
:
16524 case GIMPLE_OMP_SINGLE
:
16525 case GIMPLE_OMP_SECTION
:
16526 case GIMPLE_OMP_MASTER
:
16527 case GIMPLE_OMP_ORDERED
:
16528 case GIMPLE_OMP_CRITICAL
:
16529 case GIMPLE_OMP_TARGET
:
16530 case GIMPLE_OMP_TEAMS
:
16531 case GIMPLE_OMP_TASKGROUP
:
16532 /* The minimal context here is just the current OMP construct. */
16533 inner_context
= stmt
;
16534 wi
->info
= inner_context
;
16535 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
16536 wi
->info
= context
;
16539 case GIMPLE_OMP_FOR
:
16540 inner_context
= stmt
;
16541 wi
->info
= inner_context
;
16542 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
16544 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
16545 diagnose_sb_1
, NULL
, wi
);
16546 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
16547 wi
->info
= context
;
16551 splay_tree_insert (all_labels
,
16552 (splay_tree_key
) gimple_label_label (
16553 as_a
<glabel
*> (stmt
)),
16554 (splay_tree_value
) context
);
16564 /* Pass 2: Check each branch and see if its context differs from that of
16565 the destination label's context. */
16568 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
16569 struct walk_stmt_info
*wi
)
16571 gimple
*context
= (gimple
*) wi
->info
;
16573 gimple
*stmt
= gsi_stmt (*gsi_p
);
16575 *handled_ops_p
= true;
16577 switch (gimple_code (stmt
))
16581 case GIMPLE_OMP_PARALLEL
:
16582 case GIMPLE_OMP_TASK
:
16583 case GIMPLE_OMP_SECTIONS
:
16584 case GIMPLE_OMP_SINGLE
:
16585 case GIMPLE_OMP_SECTION
:
16586 case GIMPLE_OMP_MASTER
:
16587 case GIMPLE_OMP_ORDERED
:
16588 case GIMPLE_OMP_CRITICAL
:
16589 case GIMPLE_OMP_TARGET
:
16590 case GIMPLE_OMP_TEAMS
:
16591 case GIMPLE_OMP_TASKGROUP
:
16593 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
16594 wi
->info
= context
;
16597 case GIMPLE_OMP_FOR
:
16599 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
16601 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
16602 diagnose_sb_2
, NULL
, wi
);
16603 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
16604 wi
->info
= context
;
16609 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
16610 tree lab
= gimple_cond_true_label (cond_stmt
);
16613 n
= splay_tree_lookup (all_labels
,
16614 (splay_tree_key
) lab
);
16615 diagnose_sb_0 (gsi_p
, context
,
16616 n
? (gimple
*) n
->value
: NULL
);
16618 lab
= gimple_cond_false_label (cond_stmt
);
16621 n
= splay_tree_lookup (all_labels
,
16622 (splay_tree_key
) lab
);
16623 diagnose_sb_0 (gsi_p
, context
,
16624 n
? (gimple
*) n
->value
: NULL
);
16631 tree lab
= gimple_goto_dest (stmt
);
16632 if (TREE_CODE (lab
) != LABEL_DECL
)
16635 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
16636 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
*) n
->value
: NULL
);
16640 case GIMPLE_SWITCH
:
16642 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
16644 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
16646 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
16647 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
16648 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
*) n
->value
))
16654 case GIMPLE_RETURN
:
16655 diagnose_sb_0 (gsi_p
, context
, NULL
);
16665 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
16668 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
16671 gimple
*last
= last_stmt (bb
);
16672 enum gimple_code code
= gimple_code (last
);
16673 struct omp_region
*cur_region
= *region
;
16674 bool fallthru
= false;
16678 case GIMPLE_OMP_PARALLEL
:
16679 case GIMPLE_OMP_TASK
:
16680 case GIMPLE_OMP_FOR
:
16681 case GIMPLE_OMP_SINGLE
:
16682 case GIMPLE_OMP_TEAMS
:
16683 case GIMPLE_OMP_MASTER
:
16684 case GIMPLE_OMP_TASKGROUP
:
16685 case GIMPLE_OMP_CRITICAL
:
16686 case GIMPLE_OMP_SECTION
:
16687 cur_region
= new_omp_region (bb
, code
, cur_region
);
16691 case GIMPLE_OMP_ORDERED
:
16692 cur_region
= new_omp_region (bb
, code
, cur_region
);
16694 if (find_omp_clause (gimple_omp_ordered_clauses
16695 (as_a
<gomp_ordered
*> (last
)),
16696 OMP_CLAUSE_DEPEND
))
16697 cur_region
= cur_region
->outer
;
16700 case GIMPLE_OMP_TARGET
:
16701 cur_region
= new_omp_region (bb
, code
, cur_region
);
16703 switch (gimple_omp_target_kind (last
))
16705 case GF_OMP_TARGET_KIND_REGION
:
16706 case GF_OMP_TARGET_KIND_DATA
:
16707 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
16708 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
16709 case GF_OMP_TARGET_KIND_OACC_DATA
:
16711 case GF_OMP_TARGET_KIND_UPDATE
:
16712 case GF_OMP_TARGET_KIND_ENTER_DATA
:
16713 case GF_OMP_TARGET_KIND_EXIT_DATA
:
16714 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
16715 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
16716 cur_region
= cur_region
->outer
;
16719 gcc_unreachable ();
16723 case GIMPLE_OMP_SECTIONS
:
16724 cur_region
= new_omp_region (bb
, code
, cur_region
);
16728 case GIMPLE_OMP_SECTIONS_SWITCH
:
16732 case GIMPLE_OMP_ATOMIC_LOAD
:
16733 case GIMPLE_OMP_ATOMIC_STORE
:
16737 case GIMPLE_OMP_RETURN
:
16738 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
16739 somewhere other than the next block. This will be
16741 cur_region
->exit
= bb
;
16742 if (cur_region
->type
== GIMPLE_OMP_TASK
)
16743 /* Add an edge corresponding to not scheduling the task
16745 make_edge (cur_region
->entry
, bb
, EDGE_ABNORMAL
);
16746 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
16747 cur_region
= cur_region
->outer
;
16750 case GIMPLE_OMP_CONTINUE
:
16751 cur_region
->cont
= bb
;
16752 switch (cur_region
->type
)
16754 case GIMPLE_OMP_FOR
:
16755 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
16756 succs edges as abnormal to prevent splitting
16758 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
16759 /* Make the loopback edge. */
16760 make_edge (bb
, single_succ (cur_region
->entry
),
16763 /* Create an edge from GIMPLE_OMP_FOR to exit, which
16764 corresponds to the case that the body of the loop
16765 is not executed at all. */
16766 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
16767 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
16771 case GIMPLE_OMP_SECTIONS
:
16772 /* Wire up the edges into and out of the nested sections. */
16774 basic_block switch_bb
= single_succ (cur_region
->entry
);
16776 struct omp_region
*i
;
16777 for (i
= cur_region
->inner
; i
; i
= i
->next
)
16779 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
16780 make_edge (switch_bb
, i
->entry
, 0);
16781 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
16784 /* Make the loopback edge to the block with
16785 GIMPLE_OMP_SECTIONS_SWITCH. */
16786 make_edge (bb
, switch_bb
, 0);
16788 /* Make the edge from the switch to exit. */
16789 make_edge (switch_bb
, bb
->next_bb
, 0);
16794 case GIMPLE_OMP_TASK
:
16799 gcc_unreachable ();
16804 gcc_unreachable ();
16807 if (*region
!= cur_region
)
16809 *region
= cur_region
;
16811 *region_idx
= cur_region
->entry
->index
;
16819 static unsigned int
16820 diagnose_omp_structured_block_errors (void)
16822 struct walk_stmt_info wi
;
16823 gimple_seq body
= gimple_body (current_function_decl
);
16825 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
16827 memset (&wi
, 0, sizeof (wi
));
16828 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
16830 memset (&wi
, 0, sizeof (wi
));
16831 wi
.want_locations
= true;
16832 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
16834 gimple_set_body (current_function_decl
, body
);
16836 splay_tree_delete (all_labels
);
16844 const pass_data pass_data_diagnose_omp_blocks
=
16846 GIMPLE_PASS
, /* type */
16847 "*diagnose_omp_blocks", /* name */
16848 OPTGROUP_NONE
, /* optinfo_flags */
16849 TV_NONE
, /* tv_id */
16850 PROP_gimple_any
, /* properties_required */
16851 0, /* properties_provided */
16852 0, /* properties_destroyed */
16853 0, /* todo_flags_start */
16854 0, /* todo_flags_finish */
16857 class pass_diagnose_omp_blocks
: public gimple_opt_pass
16860 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
16861 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
16864 /* opt_pass methods: */
16865 virtual bool gate (function
*)
16867 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
16869 virtual unsigned int execute (function
*)
16871 return diagnose_omp_structured_block_errors ();
16874 }; // class pass_diagnose_omp_blocks
16876 } // anon namespace
16879 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
16881 return new pass_diagnose_omp_blocks (ctxt
);
16884 /* SIMD clone supporting code. */
16886 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
16887 of arguments to reserve space for. */
16889 static struct cgraph_simd_clone
*
16890 simd_clone_struct_alloc (int nargs
)
16892 struct cgraph_simd_clone
*clone_info
;
16893 size_t len
= (sizeof (struct cgraph_simd_clone
)
16894 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
16895 clone_info
= (struct cgraph_simd_clone
*)
16896 ggc_internal_cleared_alloc (len
);
16900 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
16903 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
16904 struct cgraph_simd_clone
*from
)
16906 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
16907 + ((from
->nargs
- from
->inbranch
)
16908 * sizeof (struct cgraph_simd_clone_arg
))));
16911 /* Return vector of parameter types of function FNDECL. This uses
16912 TYPE_ARG_TYPES if available, otherwise falls back to types of
16913 DECL_ARGUMENTS types. */
16916 simd_clone_vector_of_formal_parm_types (tree fndecl
)
16918 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
16919 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
16920 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
16923 FOR_EACH_VEC_ELT (args
, i
, arg
)
16924 args
[i
] = TREE_TYPE (args
[i
]);
16928 /* Given a simd function in NODE, extract the simd specific
16929 information from the OMP clauses passed in CLAUSES, and return
16930 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
16931 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
16932 otherwise set to FALSE. */
16934 static struct cgraph_simd_clone
*
16935 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
16936 bool *inbranch_specified
)
16938 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
16941 *inbranch_specified
= false;
16943 n
= args
.length ();
16944 if (n
> 0 && args
.last () == void_type_node
)
16947 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
16948 be cloned have a distinctive artificial label in addition to "omp
16952 && lookup_attribute ("cilk simd function",
16953 DECL_ATTRIBUTES (node
->decl
)));
16955 /* Allocate one more than needed just in case this is an in-branch
16956 clone which will require a mask argument. */
16957 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
16958 clone_info
->nargs
= n
;
16959 clone_info
->cilk_elemental
= cilk_clone
;
16966 clauses
= TREE_VALUE (clauses
);
16967 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
16970 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
16972 switch (OMP_CLAUSE_CODE (t
))
16974 case OMP_CLAUSE_INBRANCH
:
16975 clone_info
->inbranch
= 1;
16976 *inbranch_specified
= true;
16978 case OMP_CLAUSE_NOTINBRANCH
:
16979 clone_info
->inbranch
= 0;
16980 *inbranch_specified
= true;
16982 case OMP_CLAUSE_SIMDLEN
:
16983 clone_info
->simdlen
16984 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
16986 case OMP_CLAUSE_LINEAR
:
16988 tree decl
= OMP_CLAUSE_DECL (t
);
16989 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
16990 int argno
= TREE_INT_CST_LOW (decl
);
16991 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
16993 enum cgraph_simd_clone_arg_type arg_type
;
16994 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
16995 switch (OMP_CLAUSE_LINEAR_KIND (t
))
16997 case OMP_CLAUSE_LINEAR_REF
:
16999 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
;
17001 case OMP_CLAUSE_LINEAR_UVAL
:
17003 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
;
17005 case OMP_CLAUSE_LINEAR_VAL
:
17006 case OMP_CLAUSE_LINEAR_DEFAULT
:
17008 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
;
17011 gcc_unreachable ();
17014 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
17015 clone_info
->args
[argno
].arg_type
= arg_type
;
17016 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
17017 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
17018 && clone_info
->args
[argno
].linear_step
< n
);
17022 if (POINTER_TYPE_P (args
[argno
]))
17023 step
= fold_convert (ssizetype
, step
);
17024 if (!tree_fits_shwi_p (step
))
17026 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
17027 "ignoring large linear step");
17031 else if (integer_zerop (step
))
17033 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
17034 "ignoring zero linear step");
17040 enum cgraph_simd_clone_arg_type arg_type
;
17041 if (TREE_CODE (args
[argno
]) == REFERENCE_TYPE
)
17042 switch (OMP_CLAUSE_LINEAR_KIND (t
))
17044 case OMP_CLAUSE_LINEAR_REF
:
17046 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
;
17048 case OMP_CLAUSE_LINEAR_UVAL
:
17050 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
;
17052 case OMP_CLAUSE_LINEAR_VAL
:
17053 case OMP_CLAUSE_LINEAR_DEFAULT
:
17055 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
;
17058 gcc_unreachable ();
17061 arg_type
= SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
17062 clone_info
->args
[argno
].arg_type
= arg_type
;
17063 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
17068 case OMP_CLAUSE_UNIFORM
:
17070 tree decl
= OMP_CLAUSE_DECL (t
);
17071 int argno
= tree_to_uhwi (decl
);
17072 clone_info
->args
[argno
].arg_type
17073 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
17076 case OMP_CLAUSE_ALIGNED
:
17078 tree decl
= OMP_CLAUSE_DECL (t
);
17079 int argno
= tree_to_uhwi (decl
);
17080 clone_info
->args
[argno
].alignment
17081 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
17092 /* Given a SIMD clone in NODE, calculate the characteristic data
17093 type and return the coresponding type. The characteristic data
17094 type is computed as described in the Intel Vector ABI. */
17097 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
17098 struct cgraph_simd_clone
*clone_info
)
17100 tree type
= integer_type_node
;
17101 tree fndecl
= node
->decl
;
17103 /* a) For non-void function, the characteristic data type is the
17105 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
17106 type
= TREE_TYPE (TREE_TYPE (fndecl
));
17108 /* b) If the function has any non-uniform, non-linear parameters,
17109 then the characteristic data type is the type of the first
17113 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
17114 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
17115 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
17123 /* c) If the characteristic data type determined by a) or b) above
17124 is struct, union, or class type which is pass-by-value (except
17125 for the type that maps to the built-in complex data type), the
17126 characteristic data type is int. */
17127 if (RECORD_OR_UNION_TYPE_P (type
)
17128 && !aggregate_value_p (type
, NULL
)
17129 && TREE_CODE (type
) != COMPLEX_TYPE
)
17130 return integer_type_node
;
17132 /* d) If none of the above three classes is applicable, the
17133 characteristic data type is int. */
17137 /* e) For Intel Xeon Phi native and offload compilation, if the
17138 resulting characteristic data type is 8-bit or 16-bit integer
17139 data type, the characteristic data type is int. */
17140 /* Well, we don't handle Xeon Phi yet. */
17144 simd_clone_mangle (struct cgraph_node
*node
,
17145 struct cgraph_simd_clone
*clone_info
)
17147 char vecsize_mangle
= clone_info
->vecsize_mangle
;
17148 char mask
= clone_info
->inbranch
? 'M' : 'N';
17149 unsigned int simdlen
= clone_info
->simdlen
;
17153 gcc_assert (vecsize_mangle
&& simdlen
);
17155 pp_string (&pp
, "_ZGV");
17156 pp_character (&pp
, vecsize_mangle
);
17157 pp_character (&pp
, mask
);
17158 pp_decimal_int (&pp
, simdlen
);
17160 for (n
= 0; n
< clone_info
->nargs
; ++n
)
17162 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
17164 switch (arg
.arg_type
)
17166 case SIMD_CLONE_ARG_TYPE_UNIFORM
:
17167 pp_character (&pp
, 'u');
17169 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
17170 pp_character (&pp
, 'l');
17171 goto mangle_linear
;
17172 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
17173 pp_character (&pp
, 'R');
17174 goto mangle_linear
;
17175 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
17176 pp_character (&pp
, 'L');
17177 goto mangle_linear
;
17178 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
17179 pp_character (&pp
, 'U');
17180 goto mangle_linear
;
17182 gcc_assert (arg
.linear_step
!= 0);
17183 if (arg
.linear_step
> 1)
17184 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17185 else if (arg
.linear_step
< 0)
17187 pp_character (&pp
, 'n');
17188 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
17192 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
17193 pp_string (&pp
, "ls");
17194 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17196 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
17197 pp_string (&pp
, "Rs");
17198 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17200 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
17201 pp_string (&pp
, "Ls");
17202 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17204 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
17205 pp_string (&pp
, "Us");
17206 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
17209 pp_character (&pp
, 'v');
17213 pp_character (&pp
, 'a');
17214 pp_decimal_int (&pp
, arg
.alignment
);
17218 pp_underscore (&pp
);
17219 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
17222 pp_string (&pp
, str
);
17223 str
= pp_formatted_text (&pp
);
17225 /* If there already is a SIMD clone with the same mangled name, don't
17226 add another one. This can happen e.g. for
17227 #pragma omp declare simd
17228 #pragma omp declare simd simdlen(8)
17229 int foo (int, int);
17230 if the simdlen is assumed to be 8 for the first one, etc. */
17231 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
17232 clone
= clone
->simdclone
->next_clone
)
17233 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
17237 return get_identifier (str
);
17240 /* Create a simd clone of OLD_NODE and return it. */
17242 static struct cgraph_node
*
17243 simd_clone_create (struct cgraph_node
*old_node
)
17245 struct cgraph_node
*new_node
;
17246 if (old_node
->definition
)
17248 if (!old_node
->has_gimple_body_p ())
17250 old_node
->get_body ();
17251 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
17257 tree old_decl
= old_node
->decl
;
17258 tree new_decl
= copy_node (old_node
->decl
);
17259 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
17260 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
17261 SET_DECL_RTL (new_decl
, NULL
);
17262 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
17263 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
17264 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
17265 if (old_node
->in_other_partition
)
17266 new_node
->in_other_partition
= 1;
17267 symtab
->call_cgraph_insertion_hooks (new_node
);
17269 if (new_node
== NULL
)
17272 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
17274 /* The function cgraph_function_versioning () will force the new
17275 symbol local. Undo this, and inherit external visability from
17277 new_node
->local
.local
= old_node
->local
.local
;
17278 new_node
->externally_visible
= old_node
->externally_visible
;
17283 /* Adjust the return type of the given function to its appropriate
17284 vector counterpart. Returns a simd array to be used throughout the
17285 function as a return value. */
17288 simd_clone_adjust_return_type (struct cgraph_node
*node
)
17290 tree fndecl
= node
->decl
;
17291 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
17292 unsigned int veclen
;
17295 /* Adjust the function return type. */
17296 if (orig_rettype
== void_type_node
)
17298 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
17299 t
= TREE_TYPE (TREE_TYPE (fndecl
));
17300 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
17301 veclen
= node
->simdclone
->vecsize_int
;
17303 veclen
= node
->simdclone
->vecsize_float
;
17304 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
17305 if (veclen
> node
->simdclone
->simdlen
)
17306 veclen
= node
->simdclone
->simdlen
;
17307 if (POINTER_TYPE_P (t
))
17308 t
= pointer_sized_int_node
;
17309 if (veclen
== node
->simdclone
->simdlen
)
17310 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
17313 t
= build_vector_type (t
, veclen
);
17314 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
17316 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
17317 if (!node
->definition
)
17320 t
= DECL_RESULT (fndecl
);
17321 /* Adjust the DECL_RESULT. */
17322 gcc_assert (TREE_TYPE (t
) != void_type_node
);
17323 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
17326 tree atype
= build_array_type_nelts (orig_rettype
,
17327 node
->simdclone
->simdlen
);
17328 if (veclen
!= node
->simdclone
->simdlen
)
17329 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
17331 /* Set up a SIMD array to use as the return value. */
17332 tree retval
= create_tmp_var_raw (atype
, "retval");
17333 gimple_add_tmp_var (retval
);
17337 /* Each vector argument has a corresponding array to be used locally
17338 as part of the eventual loop. Create such temporary array and
17341 PREFIX is the prefix to be used for the temporary.
17343 TYPE is the inner element type.
17345 SIMDLEN is the number of elements. */
17348 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
17350 tree atype
= build_array_type_nelts (type
, simdlen
);
17351 tree avar
= create_tmp_var_raw (atype
, prefix
);
17352 gimple_add_tmp_var (avar
);
17356 /* Modify the function argument types to their corresponding vector
17357 counterparts if appropriate. Also, create one array for each simd
17358 argument to be used locally when using the function arguments as
17361 NODE is the function whose arguments are to be adjusted.
17363 Returns an adjustment vector that will be filled describing how the
17364 argument types will be adjusted. */
17366 static ipa_parm_adjustment_vec
17367 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
17370 ipa_parm_adjustment_vec adjustments
;
17372 if (node
->definition
)
17373 args
= ipa_get_vector_of_formal_parms (node
->decl
);
17375 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
17376 adjustments
.create (args
.length ());
17377 unsigned i
, j
, veclen
;
17378 struct ipa_parm_adjustment adj
;
17379 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
17381 memset (&adj
, 0, sizeof (adj
));
17382 tree parm
= args
[i
];
17383 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
17384 adj
.base_index
= i
;
17387 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
17388 node
->simdclone
->args
[i
].orig_type
= parm_type
;
17390 switch (node
->simdclone
->args
[i
].arg_type
)
17393 /* No adjustment necessary for scalar arguments. */
17394 adj
.op
= IPA_PARM_OP_COPY
;
17396 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
17397 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
17398 if (node
->definition
)
17399 node
->simdclone
->args
[i
].simd_array
17400 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
17401 TREE_TYPE (parm_type
),
17402 node
->simdclone
->simdlen
);
17403 adj
.op
= IPA_PARM_OP_COPY
;
17405 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
17406 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
17407 case SIMD_CLONE_ARG_TYPE_VECTOR
:
17408 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
17409 veclen
= node
->simdclone
->vecsize_int
;
17411 veclen
= node
->simdclone
->vecsize_float
;
17412 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
17413 if (veclen
> node
->simdclone
->simdlen
)
17414 veclen
= node
->simdclone
->simdlen
;
17415 adj
.arg_prefix
= "simd";
17416 if (POINTER_TYPE_P (parm_type
))
17417 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
17419 adj
.type
= build_vector_type (parm_type
, veclen
);
17420 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
17421 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
17423 adjustments
.safe_push (adj
);
17426 memset (&adj
, 0, sizeof (adj
));
17427 adj
.op
= IPA_PARM_OP_NEW
;
17428 adj
.arg_prefix
= "simd";
17429 adj
.base_index
= i
;
17430 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
17434 if (node
->definition
)
17435 node
->simdclone
->args
[i
].simd_array
17436 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
17437 parm_type
, node
->simdclone
->simdlen
);
17439 adjustments
.safe_push (adj
);
17442 if (node
->simdclone
->inbranch
)
17445 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
17448 memset (&adj
, 0, sizeof (adj
));
17449 adj
.op
= IPA_PARM_OP_NEW
;
17450 adj
.arg_prefix
= "mask";
17452 adj
.base_index
= i
;
17453 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
17454 veclen
= node
->simdclone
->vecsize_int
;
17456 veclen
= node
->simdclone
->vecsize_float
;
17457 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
17458 if (veclen
> node
->simdclone
->simdlen
)
17459 veclen
= node
->simdclone
->simdlen
;
17460 if (POINTER_TYPE_P (base_type
))
17461 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
17463 adj
.type
= build_vector_type (base_type
, veclen
);
17464 adjustments
.safe_push (adj
);
17466 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
17467 adjustments
.safe_push (adj
);
17469 /* We have previously allocated one extra entry for the mask. Use
17471 struct cgraph_simd_clone
*sc
= node
->simdclone
;
17473 if (node
->definition
)
17475 sc
->args
[i
].orig_arg
17476 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
17477 sc
->args
[i
].simd_array
17478 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
17480 sc
->args
[i
].orig_type
= base_type
;
17481 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
17484 if (node
->definition
)
17485 ipa_modify_formal_parameters (node
->decl
, adjustments
);
17488 tree new_arg_types
= NULL_TREE
, new_reversed
;
17489 bool last_parm_void
= false;
17490 if (args
.length () > 0 && args
.last () == void_type_node
)
17491 last_parm_void
= true;
17493 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
17494 j
= adjustments
.length ();
17495 for (i
= 0; i
< j
; i
++)
17497 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
17499 if (adj
->op
== IPA_PARM_OP_COPY
)
17500 ptype
= args
[adj
->base_index
];
17503 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
17505 new_reversed
= nreverse (new_arg_types
);
17506 if (last_parm_void
)
17509 TREE_CHAIN (new_arg_types
) = void_list_node
;
17511 new_reversed
= void_list_node
;
17514 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
17515 TYPE_ARG_TYPES (new_type
) = new_reversed
;
17516 TREE_TYPE (node
->decl
) = new_type
;
17518 adjustments
.release ();
17521 return adjustments
;
17524 /* Initialize and copy the function arguments in NODE to their
17525 corresponding local simd arrays. Returns a fresh gimple_seq with
17526 the instruction sequence generated. */
17529 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
17530 ipa_parm_adjustment_vec adjustments
)
17532 gimple_seq seq
= NULL
;
17533 unsigned i
= 0, j
= 0, k
;
17535 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
17537 arg
= DECL_CHAIN (arg
), i
++, j
++)
17539 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
17540 || POINTER_TYPE_P (TREE_TYPE (arg
)))
17543 node
->simdclone
->args
[i
].vector_arg
= arg
;
17545 tree array
= node
->simdclone
->args
[i
].simd_array
;
17546 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
17548 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
17549 tree ptr
= build_fold_addr_expr (array
);
17550 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
17551 build_int_cst (ptype
, 0));
17552 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
17553 gimplify_and_add (t
, &seq
);
17557 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
17558 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
17559 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
17561 tree ptr
= build_fold_addr_expr (array
);
17565 arg
= DECL_CHAIN (arg
);
17569 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
17570 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
17571 build_int_cst (ptype
, k
* elemsize
));
17572 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
17573 gimplify_and_add (t
, &seq
);
17580 /* Callback info for ipa_simd_modify_stmt_ops below. */
17582 struct modify_stmt_info
{
17583 ipa_parm_adjustment_vec adjustments
;
17585 /* True if the parent statement was modified by
17586 ipa_simd_modify_stmt_ops. */
17590 /* Callback for walk_gimple_op.
17592 Adjust operands from a given statement as specified in the
17593 adjustments vector in the callback data. */
17596 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
17598 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
17599 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
17600 tree
*orig_tp
= tp
;
17601 if (TREE_CODE (*tp
) == ADDR_EXPR
)
17602 tp
= &TREE_OPERAND (*tp
, 0);
17603 struct ipa_parm_adjustment
*cand
= NULL
;
17604 if (TREE_CODE (*tp
) == PARM_DECL
)
17605 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
17609 *walk_subtrees
= 0;
17612 tree repl
= NULL_TREE
;
17614 repl
= unshare_expr (cand
->new_decl
);
17619 *walk_subtrees
= 0;
17620 bool modified
= info
->modified
;
17621 info
->modified
= false;
17622 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
17623 if (!info
->modified
)
17625 info
->modified
= modified
;
17628 info
->modified
= modified
;
17637 repl
= build_fold_addr_expr (repl
);
17639 if (is_gimple_debug (info
->stmt
))
17641 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
17642 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
17643 DECL_ARTIFICIAL (vexpr
) = 1;
17644 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
17645 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
17650 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
17651 repl
= gimple_assign_lhs (stmt
);
17653 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
17654 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
17657 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
17659 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
17665 info
->modified
= true;
17669 /* Traverse the function body and perform all modifications as
17670 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
17671 modified such that the replacement/reduction value will now be an
17672 offset into the corresponding simd_array.
17674 This function will replace all function argument uses with their
17675 corresponding simd array elements, and ajust the return values
17679 ipa_simd_modify_function_body (struct cgraph_node
*node
,
17680 ipa_parm_adjustment_vec adjustments
,
17681 tree retval_array
, tree iter
)
17684 unsigned int i
, j
, l
;
17686 /* Re-use the adjustments array, but this time use it to replace
17687 every function argument use to an offset into the corresponding
17689 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
17691 if (!node
->simdclone
->args
[i
].vector_arg
)
17694 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
17695 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
17696 adjustments
[j
].new_decl
17697 = build4 (ARRAY_REF
,
17699 node
->simdclone
->args
[i
].simd_array
,
17701 NULL_TREE
, NULL_TREE
);
17702 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
17703 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
17704 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
17707 l
= adjustments
.length ();
17708 for (i
= 1; i
< num_ssa_names
; i
++)
17710 tree name
= ssa_name (i
);
17712 && SSA_NAME_VAR (name
)
17713 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
17715 for (j
= 0; j
< l
; j
++)
17716 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
17717 && adjustments
[j
].new_decl
)
17720 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
17723 = copy_var_decl (adjustments
[j
].base
,
17724 DECL_NAME (adjustments
[j
].base
),
17725 TREE_TYPE (adjustments
[j
].base
));
17726 adjustments
[j
].new_ssa_base
= base_var
;
17729 base_var
= adjustments
[j
].new_ssa_base
;
17730 if (SSA_NAME_IS_DEFAULT_DEF (name
))
17732 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
17733 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
17734 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
17735 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
17736 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
17737 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
17738 gimple
*stmt
= gimple_build_assign (name
, new_decl
);
17739 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
17742 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
17747 struct modify_stmt_info info
;
17748 info
.adjustments
= adjustments
;
17750 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
17752 gimple_stmt_iterator gsi
;
17754 gsi
= gsi_start_bb (bb
);
17755 while (!gsi_end_p (gsi
))
17757 gimple
*stmt
= gsi_stmt (gsi
);
17759 struct walk_stmt_info wi
;
17761 memset (&wi
, 0, sizeof (wi
));
17762 info
.modified
= false;
17764 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
17766 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
17768 tree retval
= gimple_return_retval (return_stmt
);
17771 gsi_remove (&gsi
, true);
17775 /* Replace `return foo' with `retval_array[iter] = foo'. */
17776 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
17777 retval_array
, iter
, NULL
, NULL
);
17778 stmt
= gimple_build_assign (ref
, retval
);
17779 gsi_replace (&gsi
, stmt
, true);
17780 info
.modified
= true;
17785 update_stmt (stmt
);
17786 if (maybe_clean_eh_stmt (stmt
))
17787 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
17794 /* Helper function of simd_clone_adjust, return linear step addend
17795 of Ith argument. */
17798 simd_clone_linear_addend (struct cgraph_node
*node
, unsigned int i
,
17799 tree addtype
, basic_block entry_bb
)
17801 tree ptype
= NULL_TREE
;
17802 switch (node
->simdclone
->args
[i
].arg_type
)
17804 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
:
17805 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
:
17806 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP
:
17807 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
:
17808 return build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
17809 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
:
17810 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
:
17811 ptype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
17813 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP
:
17814 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
:
17815 ptype
= TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
));
17818 gcc_unreachable ();
17821 unsigned int idx
= node
->simdclone
->args
[i
].linear_step
;
17822 tree arg
= node
->simdclone
->args
[idx
].orig_arg
;
17823 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg
)));
17824 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
17827 if (is_gimple_reg (arg
))
17828 ret
= get_or_create_ssa_default_def (cfun
, arg
);
17831 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (arg
)), arg
);
17832 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17833 ret
= gimple_assign_lhs (g
);
17835 if (TREE_CODE (TREE_TYPE (arg
)) == REFERENCE_TYPE
)
17837 g
= gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg
))),
17838 build_simple_mem_ref (ret
));
17839 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17840 ret
= gimple_assign_lhs (g
);
17842 if (!useless_type_conversion_p (addtype
, TREE_TYPE (ret
)))
17844 g
= gimple_build_assign (make_ssa_name (addtype
), NOP_EXPR
, ret
);
17845 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17846 ret
= gimple_assign_lhs (g
);
17848 if (POINTER_TYPE_P (ptype
))
17850 tree size
= TYPE_SIZE_UNIT (TREE_TYPE (ptype
));
17851 if (size
&& TREE_CODE (size
) == INTEGER_CST
)
17853 g
= gimple_build_assign (make_ssa_name (addtype
), MULT_EXPR
,
17854 ret
, fold_convert (addtype
, size
));
17855 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
17856 ret
= gimple_assign_lhs (g
);
17862 /* Adjust the argument types in NODE to their appropriate vector
17866 simd_clone_adjust (struct cgraph_node
*node
)
17868 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
17870 targetm
.simd_clone
.adjust (node
);
17872 tree retval
= simd_clone_adjust_return_type (node
);
17873 ipa_parm_adjustment_vec adjustments
17874 = simd_clone_adjust_argument_types (node
);
17876 push_gimplify_context ();
17878 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
17880 /* Adjust all uses of vector arguments accordingly. Adjust all
17881 return values accordingly. */
17882 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
17883 tree iter1
= make_ssa_name (iter
);
17884 tree iter2
= make_ssa_name (iter
);
17885 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
17887 /* Initialize the iteration variable. */
17888 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
17889 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
17890 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
17891 /* Insert the SIMD array and iv initialization at function
17893 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
17895 pop_gimplify_context (NULL
);
17897 /* Create a new BB right before the original exit BB, to hold the
17898 iteration increment and the condition/branch. */
17899 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
17900 basic_block incr_bb
= create_empty_bb (orig_exit
);
17901 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
17902 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
17903 flag. Set it now to be a FALLTHRU_EDGE. */
17904 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
17905 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
17906 for (unsigned i
= 0;
17907 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
17909 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
17910 redirect_edge_succ (e
, incr_bb
);
17912 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
17913 e
->probability
= REG_BR_PROB_BASE
;
17914 gsi
= gsi_last_bb (incr_bb
);
17915 gimple
*g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
17916 build_int_cst (unsigned_type_node
, 1));
17917 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17919 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
17920 struct loop
*loop
= alloc_loop ();
17921 cfun
->has_force_vectorize_loops
= true;
17922 loop
->safelen
= node
->simdclone
->simdlen
;
17923 loop
->force_vectorize
= true;
17924 loop
->header
= body_bb
;
17926 /* Branch around the body if the mask applies. */
17927 if (node
->simdclone
->inbranch
)
17929 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
17931 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
17932 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
17933 tree aref
= build4 (ARRAY_REF
,
17934 TREE_TYPE (TREE_TYPE (mask_array
)),
17937 g
= gimple_build_assign (mask
, aref
);
17938 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17939 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
17940 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
17942 aref
= build1 (VIEW_CONVERT_EXPR
,
17943 build_nonstandard_integer_type (bitsize
, 0), mask
);
17944 mask
= make_ssa_name (TREE_TYPE (aref
));
17945 g
= gimple_build_assign (mask
, aref
);
17946 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17949 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
17951 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17952 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
17953 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
17956 /* Generate the condition. */
17957 g
= gimple_build_cond (LT_EXPR
,
17959 build_int_cst (unsigned_type_node
,
17960 node
->simdclone
->simdlen
),
17962 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
17963 e
= split_block (incr_bb
, gsi_stmt (gsi
));
17964 basic_block latch_bb
= e
->dest
;
17965 basic_block new_exit_bb
;
17966 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
17967 loop
->latch
= latch_bb
;
17969 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
17971 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
17972 /* The successor of incr_bb is already pointing to latch_bb; just
17974 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
17975 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
17977 gphi
*phi
= create_phi_node (iter1
, body_bb
);
17978 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
17979 edge latch_edge
= single_succ_edge (latch_bb
);
17980 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
17982 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
17984 /* Generate the new return. */
17985 gsi
= gsi_last_bb (new_exit_bb
);
17987 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
17988 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
17989 retval
= TREE_OPERAND (retval
, 0);
17992 retval
= build1 (VIEW_CONVERT_EXPR
,
17993 TREE_TYPE (TREE_TYPE (node
->decl
)),
17995 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
17996 false, GSI_CONTINUE_LINKING
);
17998 g
= gimple_build_return (retval
);
17999 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
18001 /* Handle aligned clauses by replacing default defs of the aligned
18002 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
18003 lhs. Handle linear by adding PHIs. */
18004 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
18005 if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
18006 && (TREE_ADDRESSABLE (node
->simdclone
->args
[i
].orig_arg
)
18007 || !is_gimple_reg_type
18008 (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
18010 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
18011 if (is_gimple_reg_type (TREE_TYPE (orig_arg
)))
18012 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
18015 iter1
= create_tmp_var_raw (TREE_TYPE (orig_arg
));
18016 gimple_add_tmp_var (iter1
);
18018 gsi
= gsi_after_labels (entry_bb
);
18019 g
= gimple_build_assign (iter1
, orig_arg
);
18020 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
18021 gsi
= gsi_after_labels (body_bb
);
18022 g
= gimple_build_assign (orig_arg
, iter1
);
18023 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
18025 else if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
18026 && DECL_BY_REFERENCE (node
->simdclone
->args
[i
].orig_arg
)
18027 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
18029 && TREE_ADDRESSABLE
18030 (TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
18032 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
18033 tree def
= ssa_default_def (cfun
, orig_arg
);
18034 if (def
&& !has_zero_uses (def
))
18036 iter1
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg
)));
18037 gimple_add_tmp_var (iter1
);
18038 gsi
= gsi_after_labels (entry_bb
);
18039 g
= gimple_build_assign (iter1
, build_simple_mem_ref (def
));
18040 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
18041 gsi
= gsi_after_labels (body_bb
);
18042 g
= gimple_build_assign (build_simple_mem_ref (def
), iter1
);
18043 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
18046 else if (node
->simdclone
->args
[i
].alignment
18047 && node
->simdclone
->args
[i
].arg_type
18048 == SIMD_CLONE_ARG_TYPE_UNIFORM
18049 && (node
->simdclone
->args
[i
].alignment
18050 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
18051 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
18054 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
18055 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
18056 tree def
= ssa_default_def (cfun
, orig_arg
);
18057 if (def
&& !has_zero_uses (def
))
18059 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
18060 gimple_seq seq
= NULL
;
18061 bool need_cvt
= false;
18063 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
18065 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
18068 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
18069 gimple_call_set_lhs (g
, t
);
18070 gimple_seq_add_stmt_without_update (&seq
, g
);
18073 t
= make_ssa_name (orig_arg
);
18074 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
18075 gimple_seq_add_stmt_without_update (&seq
, g
);
18077 gsi_insert_seq_on_edge_immediate
18078 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
18080 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
18081 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
18083 node
->create_edge (cgraph_node::get_create (fn
),
18084 call
, entry_bb
->count
, freq
);
18086 imm_use_iterator iter
;
18087 use_operand_p use_p
;
18089 tree repl
= gimple_get_lhs (g
);
18090 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
18091 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
18094 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
18095 SET_USE (use_p
, repl
);
18098 else if ((node
->simdclone
->args
[i
].arg_type
18099 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
18100 || (node
->simdclone
->args
[i
].arg_type
18101 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP
)
18102 || (node
->simdclone
->args
[i
].arg_type
18103 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
18104 || (node
->simdclone
->args
[i
].arg_type
18105 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP
))
18107 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
18108 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
18109 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
18110 tree def
= NULL_TREE
;
18111 if (TREE_ADDRESSABLE (orig_arg
))
18113 def
= make_ssa_name (TREE_TYPE (orig_arg
));
18114 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
18115 iter2
= make_ssa_name (TREE_TYPE (orig_arg
));
18116 gsi
= gsi_after_labels (entry_bb
);
18117 g
= gimple_build_assign (def
, orig_arg
);
18118 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
18122 def
= ssa_default_def (cfun
, orig_arg
);
18123 if (!def
|| has_zero_uses (def
))
18127 iter1
= make_ssa_name (orig_arg
);
18128 iter2
= make_ssa_name (orig_arg
);
18133 phi
= create_phi_node (iter1
, body_bb
);
18134 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
18135 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
18136 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
18137 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
18138 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
18139 ? TREE_TYPE (orig_arg
) : sizetype
;
18140 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
18142 gsi
= gsi_last_bb (incr_bb
);
18143 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
18144 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18146 imm_use_iterator iter
;
18147 use_operand_p use_p
;
18149 if (TREE_ADDRESSABLE (orig_arg
))
18151 gsi
= gsi_after_labels (body_bb
);
18152 g
= gimple_build_assign (orig_arg
, iter1
);
18153 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
18156 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
18157 if (use_stmt
== phi
)
18160 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
18161 SET_USE (use_p
, iter1
);
18164 else if (node
->simdclone
->args
[i
].arg_type
18165 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP
18166 || (node
->simdclone
->args
[i
].arg_type
18167 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP
))
18169 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
18170 tree def
= ssa_default_def (cfun
, orig_arg
);
18171 gcc_assert (!TREE_ADDRESSABLE (orig_arg
)
18172 && TREE_CODE (TREE_TYPE (orig_arg
)) == REFERENCE_TYPE
);
18173 if (def
&& !has_zero_uses (def
))
18175 tree rtype
= TREE_TYPE (TREE_TYPE (orig_arg
));
18176 iter1
= make_ssa_name (orig_arg
);
18177 iter2
= make_ssa_name (orig_arg
);
18178 tree iter3
= make_ssa_name (rtype
);
18179 tree iter4
= make_ssa_name (rtype
);
18180 tree iter5
= make_ssa_name (rtype
);
18181 gsi
= gsi_after_labels (entry_bb
);
18183 = gimple_build_assign (iter3
, build_simple_mem_ref (def
));
18184 gsi_insert_before (&gsi
, load
, GSI_NEW_STMT
);
18186 tree array
= node
->simdclone
->args
[i
].simd_array
;
18187 TREE_ADDRESSABLE (array
) = 1;
18188 tree ptr
= build_fold_addr_expr (array
);
18189 phi
= create_phi_node (iter1
, body_bb
);
18190 add_phi_arg (phi
, ptr
, preheader_edge
, UNKNOWN_LOCATION
);
18191 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
18192 g
= gimple_build_assign (iter2
, POINTER_PLUS_EXPR
, iter1
,
18193 TYPE_SIZE_UNIT (TREE_TYPE (iter3
)));
18194 gsi
= gsi_last_bb (incr_bb
);
18195 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18197 phi
= create_phi_node (iter4
, body_bb
);
18198 add_phi_arg (phi
, iter3
, preheader_edge
, UNKNOWN_LOCATION
);
18199 add_phi_arg (phi
, iter5
, latch_edge
, UNKNOWN_LOCATION
);
18200 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
18201 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
18202 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (iter3
))
18203 ? TREE_TYPE (iter3
) : sizetype
;
18204 tree addcst
= simd_clone_linear_addend (node
, i
, addtype
,
18206 g
= gimple_build_assign (iter5
, code
, iter4
, addcst
);
18207 gsi
= gsi_last_bb (incr_bb
);
18208 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18210 g
= gimple_build_assign (build_simple_mem_ref (iter1
), iter4
);
18211 gsi
= gsi_after_labels (body_bb
);
18212 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18214 imm_use_iterator iter
;
18215 use_operand_p use_p
;
18217 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
18218 if (use_stmt
== load
)
18221 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
18222 SET_USE (use_p
, iter1
);
18224 if (!TYPE_READONLY (rtype
))
18226 tree v
= make_ssa_name (rtype
);
18227 tree aref
= build4 (ARRAY_REF
, rtype
, array
,
18228 size_zero_node
, NULL_TREE
,
18230 gsi
= gsi_after_labels (new_exit_bb
);
18231 g
= gimple_build_assign (v
, aref
);
18232 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18233 g
= gimple_build_assign (build_simple_mem_ref (def
), v
);
18234 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
18239 calculate_dominance_info (CDI_DOMINATORS
);
18240 add_loop (loop
, loop
->header
->loop_father
);
18241 update_ssa (TODO_update_ssa
);
18246 /* If the function in NODE is tagged as an elemental SIMD function,
18247 create the appropriate SIMD clones. */
18250 expand_simd_clones (struct cgraph_node
*node
)
18252 tree attr
= lookup_attribute ("omp declare simd",
18253 DECL_ATTRIBUTES (node
->decl
));
18254 if (attr
== NULL_TREE
18255 || node
->global
.inlined_to
18256 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
18260 #pragma omp declare simd
18262 in C, there we don't know the argument types at all. */
18263 if (!node
->definition
18264 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
18269 /* Start with parsing the "omp declare simd" attribute(s). */
18270 bool inbranch_clause_specified
;
18271 struct cgraph_simd_clone
*clone_info
18272 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
18273 &inbranch_clause_specified
);
18274 if (clone_info
== NULL
)
18277 int orig_simdlen
= clone_info
->simdlen
;
18278 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
18279 /* The target can return 0 (no simd clones should be created),
18280 1 (just one ISA of simd clones should be created) or higher
18281 count of ISA variants. In that case, clone_info is initialized
18282 for the first ISA variant. */
18284 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
18289 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
18290 also create one inbranch and one !inbranch clone of it. */
18291 for (int i
= 0; i
< count
* 2; i
++)
18293 struct cgraph_simd_clone
*clone
= clone_info
;
18294 if (inbranch_clause_specified
&& (i
& 1) != 0)
18299 clone
= simd_clone_struct_alloc (clone_info
->nargs
18301 simd_clone_struct_copy (clone
, clone_info
);
18302 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
18303 and simd_clone_adjust_argument_types did to the first
18305 clone
->nargs
-= clone_info
->inbranch
;
18306 clone
->simdlen
= orig_simdlen
;
18307 /* And call the target hook again to get the right ISA. */
18308 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
18312 clone
->inbranch
= 1;
18315 /* simd_clone_mangle might fail if such a clone has been created
18317 tree id
= simd_clone_mangle (node
, clone
);
18318 if (id
== NULL_TREE
)
18321 /* Only when we are sure we want to create the clone actually
18322 clone the function (or definitions) or create another
18323 extern FUNCTION_DECL (for prototypes without definitions). */
18324 struct cgraph_node
*n
= simd_clone_create (node
);
18328 n
->simdclone
= clone
;
18329 clone
->origin
= node
;
18330 clone
->next_clone
= NULL
;
18331 if (node
->simd_clones
== NULL
)
18333 clone
->prev_clone
= n
;
18334 node
->simd_clones
= n
;
18338 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
18339 clone
->prev_clone
->simdclone
->next_clone
= n
;
18340 node
->simd_clones
->simdclone
->prev_clone
= n
;
18342 symtab
->change_decl_assembler_name (n
->decl
, id
);
18343 /* And finally adjust the return type, parameters and for
18344 definitions also function body. */
18345 if (node
->definition
)
18346 simd_clone_adjust (n
);
18349 simd_clone_adjust_return_type (n
);
18350 simd_clone_adjust_argument_types (n
);
18354 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
18357 /* Entry point for IPA simd clone creation pass. */
18359 static unsigned int
18360 ipa_omp_simd_clone (void)
18362 struct cgraph_node
*node
;
18363 FOR_EACH_FUNCTION (node
)
18364 expand_simd_clones (node
);
18370 const pass_data pass_data_omp_simd_clone
=
18372 SIMPLE_IPA_PASS
, /* type */
18373 "simdclone", /* name */
18374 OPTGROUP_NONE
, /* optinfo_flags */
18375 TV_NONE
, /* tv_id */
18376 ( PROP_ssa
| PROP_cfg
), /* properties_required */
18377 0, /* properties_provided */
18378 0, /* properties_destroyed */
18379 0, /* todo_flags_start */
18380 0, /* todo_flags_finish */
18383 class pass_omp_simd_clone
: public simple_ipa_opt_pass
18386 pass_omp_simd_clone(gcc::context
*ctxt
)
18387 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
18390 /* opt_pass methods: */
18391 virtual bool gate (function
*);
18392 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
18396 pass_omp_simd_clone::gate (function
*)
18398 return ((flag_openmp
|| flag_openmp_simd
18400 || (in_lto_p
&& !flag_wpa
))
18401 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
18404 } // anon namespace
18406 simple_ipa_opt_pass
*
18407 make_pass_omp_simd_clone (gcc::context
*ctxt
)
18409 return new pass_omp_simd_clone (ctxt
);
18412 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
18413 adds their addresses and sizes to constructor-vector V_CTOR. */
18415 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
18416 vec
<constructor_elt
, va_gc
> *v_ctor
)
18418 unsigned len
= vec_safe_length (v_decls
);
18419 for (unsigned i
= 0; i
< len
; i
++)
18421 tree it
= (*v_decls
)[i
];
18422 bool is_function
= TREE_CODE (it
) != VAR_DECL
;
18424 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, build_fold_addr_expr (it
));
18426 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
,
18427 fold_convert (const_ptr_type_node
,
18428 DECL_SIZE_UNIT (it
)));
18432 /* Create new symbols containing (address, size) pairs for global variables,
18433 marked with "omp declare target" attribute, as well as addresses for the
18434 functions, which are outlined offloading regions. */
18436 omp_finish_file (void)
18438 unsigned num_funcs
= vec_safe_length (offload_funcs
);
18439 unsigned num_vars
= vec_safe_length (offload_vars
);
18441 if (num_funcs
== 0 && num_vars
== 0)
18444 if (targetm_common
.have_named_sections
)
18446 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
18447 vec_alloc (v_f
, num_funcs
);
18448 vec_alloc (v_v
, num_vars
* 2);
18450 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
18451 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
18453 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
18455 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
18457 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
18458 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
18459 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
18460 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
18461 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
18462 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
18463 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
18464 get_identifier (".offload_func_table"),
18466 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
18467 get_identifier (".offload_var_table"),
18469 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
18470 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
18471 otherwise a joint table in a binary will contain padding between
18472 tables from multiple object files. */
18473 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
18474 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
18475 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
18476 DECL_INITIAL (funcs_decl
) = ctor_f
;
18477 DECL_INITIAL (vars_decl
) = ctor_v
;
18478 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
18479 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
18481 varpool_node::finalize_decl (vars_decl
);
18482 varpool_node::finalize_decl (funcs_decl
);
18486 for (unsigned i
= 0; i
< num_funcs
; i
++)
18488 tree it
= (*offload_funcs
)[i
];
18489 targetm
.record_offload_symbol (it
);
18491 for (unsigned i
= 0; i
< num_vars
; i
++)
18493 tree it
= (*offload_vars
)[i
];
18494 targetm
.record_offload_symbol (it
);
18499 /* Find the number of threads (POS = false), or thread number (POS =
18500 true) for an OpenACC region partitioned as MASK. Setup code
18501 required for the calculation is added to SEQ. */
18504 oacc_thread_numbers (bool pos
, int mask
, gimple_seq
*seq
)
18506 tree res
= pos
? NULL_TREE
: build_int_cst (unsigned_type_node
, 1);
18509 /* Start at gang level, and examine relevant dimension indices. */
18510 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
18511 if (GOMP_DIM_MASK (ix
) & mask
)
18513 tree arg
= build_int_cst (unsigned_type_node
, ix
);
18517 /* We had an outer index, so scale that by the size of
18519 tree n
= create_tmp_var (integer_type_node
);
18521 = gimple_build_call_internal (IFN_GOACC_DIM_SIZE
, 1, arg
);
18523 gimple_call_set_lhs (call
, n
);
18524 gimple_seq_add_stmt (seq
, call
);
18525 res
= fold_build2 (MULT_EXPR
, integer_type_node
, res
, n
);
18529 /* Determine index in this dimension. */
18530 tree id
= create_tmp_var (integer_type_node
);
18531 gimple
*call
= gimple_build_call_internal
18532 (IFN_GOACC_DIM_POS
, 1, arg
);
18534 gimple_call_set_lhs (call
, id
);
18535 gimple_seq_add_stmt (seq
, call
);
18537 res
= fold_build2 (PLUS_EXPR
, integer_type_node
, res
, id
);
18543 if (res
== NULL_TREE
)
18544 res
= integer_zero_node
;
18549 /* Transform IFN_GOACC_LOOP calls to actual code. See
18550 expand_oacc_for for where these are generated. At the vector
18551 level, we stride loops, such that each member of a warp will
18552 operate on adjacent iterations. At the worker and gang level,
18553 each gang/warp executes a set of contiguous iterations. Chunking
18554 can override this such that each iteration engine executes a
18555 contiguous chunk, and then moves on to stride to the next chunk. */
18558 oacc_xform_loop (gcall
*call
)
18560 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
18561 enum ifn_goacc_loop_kind code
18562 = (enum ifn_goacc_loop_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
18563 tree dir
= gimple_call_arg (call
, 1);
18564 tree range
= gimple_call_arg (call
, 2);
18565 tree step
= gimple_call_arg (call
, 3);
18566 tree chunk_size
= NULL_TREE
;
18567 unsigned mask
= (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call
, 5));
18568 tree lhs
= gimple_call_lhs (call
);
18569 tree type
= TREE_TYPE (lhs
);
18570 tree diff_type
= TREE_TYPE (range
);
18571 tree r
= NULL_TREE
;
18572 gimple_seq seq
= NULL
;
18573 bool chunking
= false, striding
= true;
18574 unsigned outer_mask
= mask
& (~mask
+ 1); // Outermost partitioning
18575 unsigned inner_mask
= mask
& ~outer_mask
; // Inner partitioning (if any)
18577 #ifdef ACCEL_COMPILER
18578 chunk_size
= gimple_call_arg (call
, 4);
18579 if (integer_minus_onep (chunk_size
) /* Force static allocation. */
18580 || integer_zerop (chunk_size
)) /* Default (also static). */
18582 /* If we're at the gang level, we want each to execute a
18583 contiguous run of iterations. Otherwise we want each element
18585 striding
= !(outer_mask
& GOMP_DIM_MASK (GOMP_DIM_GANG
));
18590 /* Chunk of size 1 is striding. */
18591 striding
= integer_onep (chunk_size
);
18592 chunking
= !striding
;
18596 /* striding=true, chunking=true
18598 striding=true, chunking=false
18600 striding=false,chunking=true
18601 -> chunks=ceil (range/(chunksize*threads*step))
18602 striding=false,chunking=false
18603 -> chunk_size=ceil(range/(threads*step)),chunks=1 */
18604 push_gimplify_context (true);
18608 default: gcc_unreachable ();
18610 case IFN_GOACC_LOOP_CHUNKS
:
18612 r
= build_int_cst (type
, 1);
18616 = (range - dir) / (chunks * step * num_threads) + dir */
18617 tree per
= oacc_thread_numbers (false, mask
, &seq
);
18618 per
= fold_convert (type
, per
);
18619 chunk_size
= fold_convert (type
, chunk_size
);
18620 per
= fold_build2 (MULT_EXPR
, type
, per
, chunk_size
);
18621 per
= fold_build2 (MULT_EXPR
, type
, per
, step
);
18622 r
= build2 (MINUS_EXPR
, type
, range
, dir
);
18623 r
= build2 (PLUS_EXPR
, type
, r
, per
);
18624 r
= build2 (TRUNC_DIV_EXPR
, type
, r
, per
);
18628 case IFN_GOACC_LOOP_STEP
:
18630 /* If striding, step by the entire compute volume, otherwise
18631 step by the inner volume. */
18632 unsigned volume
= striding
? mask
: inner_mask
;
18634 r
= oacc_thread_numbers (false, volume
, &seq
);
18635 r
= build2 (MULT_EXPR
, type
, fold_convert (type
, r
), step
);
18639 case IFN_GOACC_LOOP_OFFSET
:
18642 r
= oacc_thread_numbers (true, mask
, &seq
);
18643 r
= fold_convert (diff_type
, r
);
18647 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
18648 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
18649 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
18650 inner_size
, outer_size
);
18652 volume
= fold_convert (diff_type
, volume
);
18654 chunk_size
= fold_convert (diff_type
, chunk_size
);
18657 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
18659 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
18660 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
18661 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
18664 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
18665 fold_convert (diff_type
, inner_size
));
18666 r
= oacc_thread_numbers (true, outer_mask
, &seq
);
18667 r
= fold_convert (diff_type
, r
);
18668 r
= build2 (MULT_EXPR
, diff_type
, r
, span
);
18670 tree inner
= oacc_thread_numbers (true, inner_mask
, &seq
);
18671 inner
= fold_convert (diff_type
, inner
);
18672 r
= fold_build2 (PLUS_EXPR
, diff_type
, r
, inner
);
18676 tree chunk
= fold_convert (diff_type
, gimple_call_arg (call
, 6));
18678 = fold_build2 (MULT_EXPR
, diff_type
, volume
, chunk_size
);
18679 per
= build2 (MULT_EXPR
, diff_type
, per
, chunk
);
18681 r
= build2 (PLUS_EXPR
, diff_type
, r
, per
);
18684 r
= fold_build2 (MULT_EXPR
, diff_type
, r
, step
);
18685 if (type
!= diff_type
)
18686 r
= fold_convert (type
, r
);
18689 case IFN_GOACC_LOOP_BOUND
:
18694 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
18695 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
18696 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
18697 inner_size
, outer_size
);
18699 volume
= fold_convert (diff_type
, volume
);
18701 chunk_size
= fold_convert (diff_type
, chunk_size
);
18704 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
18706 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
18707 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
18708 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
18711 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
18712 fold_convert (diff_type
, inner_size
));
18714 r
= fold_build2 (MULT_EXPR
, diff_type
, span
, step
);
18716 tree offset
= gimple_call_arg (call
, 6);
18717 r
= build2 (PLUS_EXPR
, diff_type
, r
,
18718 fold_convert (diff_type
, offset
));
18719 r
= build2 (integer_onep (dir
) ? MIN_EXPR
: MAX_EXPR
,
18720 diff_type
, r
, range
);
18722 if (diff_type
!= type
)
18723 r
= fold_convert (type
, r
);
18727 gimplify_assign (lhs
, r
, &seq
);
18729 pop_gimplify_context (NULL
);
18731 gsi_replace_with_seq (&gsi
, seq
, true);
18734 /* Validate and update the dimensions for offloaded FN. ATTRS is the
18735 raw attribute. DIMS is an array of dimensions, which is returned.
18736 Returns the function level dimensionality -- the level at which an
18737 offload routine wishes to partition a loop. */
18740 oacc_validate_dims (tree fn
, tree attrs
, int *dims
)
18742 tree purpose
[GOMP_DIM_MAX
];
18744 tree pos
= TREE_VALUE (attrs
);
18747 /* Make sure the attribute creator attached the dimension
18751 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
18753 purpose
[ix
] = TREE_PURPOSE (pos
);
18757 if (integer_zerop (purpose
[ix
]))
18759 else if (fn_level
< 0)
18763 tree val
= TREE_VALUE (pos
);
18764 dims
[ix
] = val
? TREE_INT_CST_LOW (val
) : -1;
18765 pos
= TREE_CHAIN (pos
);
18768 bool changed
= targetm
.goacc
.validate_dims (fn
, dims
, fn_level
);
18770 /* Default anything left to 1. */
18771 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
18780 /* Replace the attribute with new values. */
18782 for (ix
= GOMP_DIM_MAX
; ix
--;)
18783 pos
= tree_cons (purpose
[ix
],
18784 build_int_cst (integer_type_node
, dims
[ix
]),
18786 replace_oacc_fn_attrib (fn
, pos
);
18792 /* Create an empty OpenACC loop structure at LOC. */
18795 new_oacc_loop_raw (oacc_loop
*parent
, location_t loc
)
18797 oacc_loop
*loop
= XCNEW (oacc_loop
);
18799 loop
->parent
= parent
;
18800 loop
->child
= loop
->sibling
= NULL
;
18804 loop
->sibling
= parent
->child
;
18805 parent
->child
= loop
;
18809 loop
->marker
= NULL
;
18810 memset (loop
->heads
, 0, sizeof (loop
->heads
));
18811 memset (loop
->tails
, 0, sizeof (loop
->tails
));
18812 loop
->routine
= NULL_TREE
;
18814 loop
->mask
= loop
->flags
= 0;
18815 loop
->chunk_size
= 0;
18816 loop
->head_end
= NULL
;
18821 /* Create an outermost, dummy OpenACC loop for offloaded function
18825 new_oacc_loop_outer (tree decl
)
18827 return new_oacc_loop_raw (NULL
, DECL_SOURCE_LOCATION (decl
));
18830 /* Start a new OpenACC loop structure beginning at head marker HEAD.
18831 Link into PARENT loop. Return the new loop. */
18834 new_oacc_loop (oacc_loop
*parent
, gcall
*marker
)
18836 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (marker
));
18838 loop
->marker
= marker
;
18840 /* TODO: This is where device_type flattening would occur for the loop
18843 loop
->flags
= TREE_INT_CST_LOW (gimple_call_arg (marker
, 3));
18845 tree chunk_size
= integer_zero_node
;
18846 if (loop
->flags
& OLF_GANG_STATIC
)
18847 chunk_size
= gimple_call_arg (marker
, 4);
18848 loop
->chunk_size
= chunk_size
;
18853 /* Create a dummy loop encompassing a call to a openACC routine.
18854 Extract the routine's partitioning requirements. */
18857 new_oacc_loop_routine (oacc_loop
*parent
, gcall
*call
, tree decl
, tree attrs
)
18859 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (call
));
18860 int dims
[GOMP_DIM_MAX
];
18861 int level
= oacc_validate_dims (decl
, attrs
, dims
);
18863 gcc_assert (level
>= 0);
18865 loop
->marker
= call
;
18866 loop
->routine
= decl
;
18867 loop
->mask
= ((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1)
18868 ^ (GOMP_DIM_MASK (level
) - 1));
18871 /* Finish off the current OpenACC loop ending at tail marker TAIL.
18872 Return the parent loop. */
18875 finish_oacc_loop (oacc_loop
*loop
)
18877 return loop
->parent
;
18880 /* Free all OpenACC loop structures within LOOP (inclusive). */
18883 free_oacc_loop (oacc_loop
*loop
)
18886 free_oacc_loop (loop
->sibling
);
18888 free_oacc_loop (loop
->child
);
18893 /* Dump out the OpenACC loop head or tail beginning at FROM. */
18896 dump_oacc_loop_part (FILE *file
, gcall
*from
, int depth
,
18897 const char *title
, int level
)
18899 enum ifn_unique_kind kind
18900 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
18902 fprintf (file
, "%*s%s-%d:\n", depth
* 2, "", title
, level
);
18903 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
18905 gimple
*stmt
= gsi_stmt (gsi
);
18907 if (is_gimple_call (stmt
)
18908 && gimple_call_internal_p (stmt
)
18909 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
18911 enum ifn_unique_kind k
18912 = ((enum ifn_unique_kind
) TREE_INT_CST_LOW
18913 (gimple_call_arg (stmt
, 0)));
18915 if (k
== kind
&& stmt
!= from
)
18918 print_gimple_stmt (file
, stmt
, depth
* 2 + 2, 0);
18921 while (gsi_end_p (gsi
))
18922 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
18926 /* Dump OpenACC loops LOOP, its siblings and its children. */
18929 dump_oacc_loop (FILE *file
, oacc_loop
*loop
, int depth
)
18933 fprintf (file
, "%*sLoop %x(%x) %s:%u\n", depth
* 2, "",
18934 loop
->flags
, loop
->mask
,
18935 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
));
18938 print_gimple_stmt (file
, loop
->marker
, depth
* 2, 0);
18941 fprintf (file
, "%*sRoutine %s:%u:%s\n",
18942 depth
* 2, "", DECL_SOURCE_FILE (loop
->routine
),
18943 DECL_SOURCE_LINE (loop
->routine
),
18944 IDENTIFIER_POINTER (DECL_NAME (loop
->routine
)));
18946 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
18947 if (loop
->heads
[ix
])
18948 dump_oacc_loop_part (file
, loop
->heads
[ix
], depth
, "Head", ix
);
18949 for (ix
= GOMP_DIM_MAX
; ix
--;)
18950 if (loop
->tails
[ix
])
18951 dump_oacc_loop_part (file
, loop
->tails
[ix
], depth
, "Tail", ix
);
18954 dump_oacc_loop (file
, loop
->child
, depth
+ 1);
18956 dump_oacc_loop (file
, loop
->sibling
, depth
);
18959 void debug_oacc_loop (oacc_loop
*);
18961 /* Dump loops to stderr. */
18963 DEBUG_FUNCTION
void
18964 debug_oacc_loop (oacc_loop
*loop
)
18966 dump_oacc_loop (stderr
, loop
, 0);
18969 /* DFS walk of basic blocks BB onwards, creating OpenACC loop
18970 structures as we go. By construction these loops are properly
18974 oacc_loop_discover_walk (oacc_loop
*loop
, basic_block bb
)
18979 if (bb
->flags
& BB_VISITED
)
18983 bb
->flags
|= BB_VISITED
;
18985 /* Scan for loop markers. */
18986 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);
18989 gimple
*stmt
= gsi_stmt (gsi
);
18991 if (!is_gimple_call (stmt
))
18994 gcall
*call
= as_a
<gcall
*> (stmt
);
18996 /* If this is a routine, make a dummy loop for it. */
18997 if (tree decl
= gimple_call_fndecl (call
))
18998 if (tree attrs
= get_oacc_fn_attrib (decl
))
19000 gcc_assert (!marker
);
19001 new_oacc_loop_routine (loop
, call
, decl
, attrs
);
19004 if (!gimple_call_internal_p (call
))
19007 if (gimple_call_internal_fn (call
) != IFN_UNIQUE
)
19010 enum ifn_unique_kind kind
19011 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
19012 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
19013 || kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
19015 if (gimple_call_num_args (call
) == 2)
19017 gcc_assert (marker
&& !remaining
);
19019 if (kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
19020 loop
= finish_oacc_loop (loop
);
19022 loop
->head_end
= call
;
19026 int count
= TREE_INT_CST_LOW (gimple_call_arg (call
, 2));
19030 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
19031 loop
= new_oacc_loop (loop
, call
);
19034 gcc_assert (count
== remaining
);
19038 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
19039 loop
->heads
[marker
] = call
;
19041 loop
->tails
[remaining
] = call
;
19047 if (remaining
|| marker
)
19049 bb
= single_succ (bb
);
19050 gcc_assert (single_pred_p (bb
) && !(bb
->flags
& BB_VISITED
));
19054 /* Walk successor blocks. */
19058 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
19059 oacc_loop_discover_walk (loop
, e
->dest
);
19062 /* LOOP is the first sibling. Reverse the order in place and return
19063 the new first sibling. Recurse to child loops. */
19066 oacc_loop_sibling_nreverse (oacc_loop
*loop
)
19068 oacc_loop
*last
= NULL
;
19072 loop
->child
= oacc_loop_sibling_nreverse (loop
->child
);
19074 oacc_loop
*next
= loop
->sibling
;
19075 loop
->sibling
= last
;
19084 /* Discover the OpenACC loops marked up by HEAD and TAIL markers for
19085 the current function. */
19088 oacc_loop_discovery ()
19092 oacc_loop
*top
= new_oacc_loop_outer (current_function_decl
);
19093 oacc_loop_discover_walk (top
, ENTRY_BLOCK_PTR_FOR_FN (cfun
));
19095 /* The siblings were constructed in reverse order, reverse them so
19096 that diagnostics come out in an unsurprising order. */
19097 top
= oacc_loop_sibling_nreverse (top
);
19099 /* Reset the visited flags. */
19100 FOR_ALL_BB_FN (bb
, cfun
)
19101 bb
->flags
&= ~BB_VISITED
;
19106 /* Transform the abstract internal function markers starting at FROM
19107 to be for partitioning level LEVEL. Stop when we meet another HEAD
19111 oacc_loop_xform_head_tail (gcall
*from
, int level
)
19113 enum ifn_unique_kind kind
19114 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
19115 tree replacement
= build_int_cst (unsigned_type_node
, level
);
19117 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
19119 gimple
*stmt
= gsi_stmt (gsi
);
19121 if (is_gimple_call (stmt
)
19122 && gimple_call_internal_p (stmt
)
19123 && gimple_call_internal_fn (stmt
) == IFN_UNIQUE
)
19125 enum ifn_unique_kind k
19126 = ((enum ifn_unique_kind
)
19127 TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0)));
19129 if (k
== IFN_UNIQUE_OACC_FORK
|| k
== IFN_UNIQUE_OACC_JOIN
)
19130 *gimple_call_arg_ptr (stmt
, 2) = replacement
;
19131 else if (k
== kind
&& stmt
!= from
)
19134 else if (is_gimple_call (stmt
)
19135 && gimple_call_internal_p (stmt
)
19136 && gimple_call_internal_fn (stmt
) == IFN_GOACC_REDUCTION
)
19137 *gimple_call_arg_ptr (stmt
, 3) = replacement
;
19140 while (gsi_end_p (gsi
))
19141 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
19145 /* Transform the IFN_GOACC_LOOP internal functions by providing the
19146 determined partitioning mask and chunking argument. */
19149 oacc_loop_xform_loop (gcall
*end_marker
, tree mask_arg
, tree chunk_arg
)
19151 gimple_stmt_iterator gsi
= gsi_for_stmt (end_marker
);
19155 for (; !gsi_end_p (gsi
); gsi_next (&gsi
))
19157 gimple
*stmt
= gsi_stmt (gsi
);
19159 if (!is_gimple_call (stmt
))
19162 gcall
*call
= as_a
<gcall
*> (stmt
);
19164 if (!gimple_call_internal_p (call
))
19167 if (gimple_call_internal_fn (call
) != IFN_GOACC_LOOP
)
19170 *gimple_call_arg_ptr (call
, 5) = mask_arg
;
19171 *gimple_call_arg_ptr (call
, 4) = chunk_arg
;
19172 if (TREE_INT_CST_LOW (gimple_call_arg (call
, 0))
19173 == IFN_GOACC_LOOP_BOUND
)
19177 /* If we didn't see LOOP_BOUND, it should be in the single
19178 successor block. */
19179 basic_block bb
= single_succ (gsi_bb (gsi
));
19180 gsi
= gsi_start_bb (bb
);
19184 /* Process the discovered OpenACC loops, setting the correct
19185 partitioning level etc. */
19188 oacc_loop_process (oacc_loop
*loop
)
19191 oacc_loop_process (loop
->child
);
19193 if (loop
->mask
&& !loop
->routine
)
19196 unsigned mask
= loop
->mask
;
19197 unsigned dim
= GOMP_DIM_GANG
;
19198 tree mask_arg
= build_int_cst (unsigned_type_node
, mask
);
19199 tree chunk_arg
= loop
->chunk_size
;
19201 oacc_loop_xform_loop (loop
->head_end
, mask_arg
, chunk_arg
);
19203 for (ix
= 0; ix
!= GOMP_DIM_MAX
&& loop
->heads
[ix
]; ix
++)
19207 while (!(GOMP_DIM_MASK (dim
) & mask
))
19210 oacc_loop_xform_head_tail (loop
->heads
[ix
], dim
);
19211 oacc_loop_xform_head_tail (loop
->tails
[ix
], dim
);
19213 mask
^= GOMP_DIM_MASK (dim
);
19218 oacc_loop_process (loop
->sibling
);
19221 /* Walk the OpenACC loop heirarchy checking and assigning the
19222 programmer-specified partitionings. OUTER_MASK is the partitioning
19223 this loop is contained within. Return partitiong mask used within
19227 oacc_loop_fixed_partitions (oacc_loop
*loop
, unsigned outer_mask
)
19229 unsigned this_mask
= loop
->mask
;
19230 bool has_auto
= false;
19233 #ifdef ACCEL_COMPILER
19234 /* When device_type is supported, we want the device compiler to be
19235 noisy, if the loop parameters are device_type-specific. */
19239 if (!loop
->routine
)
19241 bool auto_par
= (loop
->flags
& OLF_AUTO
) != 0;
19242 bool seq_par
= (loop
->flags
& OLF_SEQ
) != 0;
19244 this_mask
= ((loop
->flags
>> OLF_DIM_BASE
)
19245 & (GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1));
19247 if ((this_mask
!= 0) + auto_par
+ seq_par
> 1)
19250 error_at (loop
->loc
,
19252 ? "%<seq%> overrides other OpenACC loop specifiers"
19253 : "%<auto%> conflicts with other OpenACC loop specifiers");
19255 loop
->flags
&= ~OLF_AUTO
;
19259 ~((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
);
19263 if (auto_par
&& (loop
->flags
& OLF_INDEPENDENT
))
19267 if (this_mask
& outer_mask
)
19269 const oacc_loop
*outer
;
19270 for (outer
= loop
->parent
; outer
; outer
= outer
->parent
)
19271 if (outer
->mask
& this_mask
)
19278 error_at (loop
->loc
,
19279 "%s uses same OpenACC parallelism as containing loop",
19280 loop
->routine
? "routine call" : "inner loop");
19281 inform (outer
->loc
, "containing loop here");
19284 error_at (loop
->loc
,
19285 "%s uses OpenACC parallelism disallowed by containing routine",
19286 loop
->routine
? "routine call" : "loop");
19289 inform (DECL_SOURCE_LOCATION (loop
->routine
),
19290 "routine %qD declared here", loop
->routine
);
19292 this_mask
&= ~outer_mask
;
19296 unsigned outermost
= this_mask
& -this_mask
;
19298 if (outermost
&& outermost
<= outer_mask
)
19302 error_at (loop
->loc
,
19303 "incorrectly nested OpenACC loop parallelism");
19305 const oacc_loop
*outer
;
19306 for (outer
= loop
->parent
;
19307 outer
->flags
&& outer
->flags
< outermost
;
19308 outer
= outer
->parent
)
19310 inform (outer
->loc
, "containing loop here");
19313 this_mask
&= ~outermost
;
19317 loop
->mask
= this_mask
;
19320 && oacc_loop_fixed_partitions (loop
->child
, outer_mask
| this_mask
))
19324 && oacc_loop_fixed_partitions (loop
->sibling
, outer_mask
))
19330 /* Walk the OpenACC loop heirarchy to check and assign partitioning
19334 oacc_loop_partition (oacc_loop
*loop
, int fn_level
)
19336 unsigned outer_mask
= 0;
19339 outer_mask
= GOMP_DIM_MASK (fn_level
) - 1;
19341 oacc_loop_fixed_partitions (loop
, outer_mask
);
19344 /* Default fork/join early expander. Delete the function calls if
19345 there is no RTL expander. */
19348 default_goacc_fork_join (gcall
*ARG_UNUSED (call
),
19349 const int *ARG_UNUSED (dims
), bool is_fork
)
19352 return targetm
.have_oacc_fork ();
19354 return targetm
.have_oacc_join ();
19357 /* Default goacc.reduction early expander.
19359 LHS-opt = IFN_REDUCTION (KIND, RES_PTR, VAR, LEVEL, OP, OFFSET)
19360 If RES_PTR is not integer-zerop:
19361 SETUP - emit 'LHS = *RES_PTR', LHS = NULL
19362 TEARDOWN - emit '*RES_PTR = VAR'
19364 emit 'LHS = VAR' */
19367 default_goacc_reduction (gcall
*call
)
19369 unsigned code
= (unsigned)TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
19370 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
19371 tree lhs
= gimple_call_lhs (call
);
19372 tree var
= gimple_call_arg (call
, 2);
19373 gimple_seq seq
= NULL
;
19375 if (code
== IFN_GOACC_REDUCTION_SETUP
19376 || code
== IFN_GOACC_REDUCTION_TEARDOWN
)
19378 /* Setup and Teardown need to copy from/to the receiver object,
19379 if there is one. */
19380 tree ref_to_res
= gimple_call_arg (call
, 1);
19382 if (!integer_zerop (ref_to_res
))
19384 tree dst
= build_simple_mem_ref (ref_to_res
);
19387 if (code
== IFN_GOACC_REDUCTION_SETUP
)
19393 gimple_seq_add_stmt (&seq
, gimple_build_assign (dst
, src
));
19397 /* Copy VAR to LHS, if there is an LHS. */
19399 gimple_seq_add_stmt (&seq
, gimple_build_assign (lhs
, var
));
19401 gsi_replace_with_seq (&gsi
, seq
, true);
19404 /* Main entry point for oacc transformations which run on the device
19405 compiler after LTO, so we know what the target device is at this
19406 point (including the host fallback). */
19408 static unsigned int
19409 execute_oacc_device_lower ()
19411 tree attrs
= get_oacc_fn_attrib (current_function_decl
);
19412 int dims
[GOMP_DIM_MAX
];
19415 /* Not an offloaded function. */
19418 int fn_level
= oacc_validate_dims (current_function_decl
, attrs
, dims
);
19420 /* Discover, partition and process the loops. */
19421 oacc_loop
*loops
= oacc_loop_discovery ();
19422 oacc_loop_partition (loops
, fn_level
);
19423 oacc_loop_process (loops
);
19426 fprintf (dump_file
, "OpenACC loops\n");
19427 dump_oacc_loop (dump_file
, loops
, 0);
19428 fprintf (dump_file
, "\n");
19431 /* Offloaded targets may introduce new basic blocks, which require
19432 dominance information to update SSA. */
19433 calculate_dominance_info (CDI_DOMINATORS
);
19435 /* Now lower internal loop functions to target-specific code
19438 FOR_ALL_BB_FN (bb
, cfun
)
19439 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);)
19441 gimple
*stmt
= gsi_stmt (gsi
);
19442 if (!is_gimple_call (stmt
))
19448 gcall
*call
= as_a
<gcall
*> (stmt
);
19449 if (!gimple_call_internal_p (call
))
19455 /* Rewind to allow rescan. */
19457 bool rescan
= false, remove
= false;
19458 enum internal_fn ifn_code
= gimple_call_internal_fn (call
);
19464 case IFN_GOACC_LOOP
:
19465 oacc_xform_loop (call
);
19469 case IFN_GOACC_REDUCTION
:
19470 /* Mark the function for SSA renaming. */
19471 mark_virtual_operands_for_renaming (cfun
);
19473 /* If the level is -1, this ended up being an unused
19474 axis. Handle as a default. */
19475 if (integer_minus_onep (gimple_call_arg (call
, 3)))
19476 default_goacc_reduction (call
);
19478 targetm
.goacc
.reduction (call
);
19484 enum ifn_unique_kind kind
19485 = ((enum ifn_unique_kind
)
19486 TREE_INT_CST_LOW (gimple_call_arg (call
, 0)));
19491 gcc_unreachable ();
19493 case IFN_UNIQUE_OACC_FORK
:
19494 case IFN_UNIQUE_OACC_JOIN
:
19495 if (integer_minus_onep (gimple_call_arg (call
, 2)))
19497 else if (!targetm
.goacc
.fork_join
19498 (call
, dims
, kind
== IFN_UNIQUE_OACC_FORK
))
19502 case IFN_UNIQUE_OACC_HEAD_MARK
:
19503 case IFN_UNIQUE_OACC_TAIL_MARK
:
19511 if (gsi_end_p (gsi
))
19512 /* We rewound past the beginning of the BB. */
19513 gsi
= gsi_start_bb (bb
);
19515 /* Undo the rewind. */
19520 if (gimple_vdef (call
))
19521 replace_uses_by (gimple_vdef (call
), gimple_vuse (call
));
19522 if (gimple_call_lhs (call
))
19524 /* Propagate the data dependency var. */
19525 gimple
*ass
= gimple_build_assign (gimple_call_lhs (call
),
19526 gimple_call_arg (call
, 1));
19527 gsi_replace (&gsi
, ass
, false);
19530 gsi_remove (&gsi
, true);
19533 /* If not rescanning, advance over the call. */
19537 free_oacc_loop (loops
);
19542 /* Default launch dimension validator. Force everything to 1. A
19543 backend that wants to provide larger dimensions must override this
19547 default_goacc_validate_dims (tree
ARG_UNUSED (decl
), int *dims
,
19548 int ARG_UNUSED (fn_level
))
19550 bool changed
= false;
19552 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
19564 /* Default dimension bound is unknown on accelerator and 1 on host. */
19567 default_goacc_dim_limit (int ARG_UNUSED (axis
))
19569 #ifdef ACCEL_COMPILER
19578 const pass_data pass_data_oacc_device_lower
=
19580 GIMPLE_PASS
, /* type */
19581 "oaccdevlow", /* name */
19582 OPTGROUP_NONE
, /* optinfo_flags */
19583 TV_NONE
, /* tv_id */
19584 PROP_cfg
, /* properties_required */
19585 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
19586 0, /* properties_destroyed */
19587 0, /* todo_flags_start */
19588 TODO_update_ssa
| TODO_cleanup_cfg
, /* todo_flags_finish */
19591 class pass_oacc_device_lower
: public gimple_opt_pass
19594 pass_oacc_device_lower (gcc::context
*ctxt
)
19595 : gimple_opt_pass (pass_data_oacc_device_lower
, ctxt
)
19598 /* opt_pass methods: */
19599 virtual unsigned int execute (function
*)
19601 bool gate
= flag_openacc
!= 0;
19606 return execute_oacc_device_lower ();
19609 }; // class pass_oacc_device_lower
19611 } // anon namespace
19614 make_pass_oacc_device_lower (gcc::context
*ctxt
)
19616 return new pass_oacc_device_lower (ctxt
);
19619 #include "gt-omp-low.h"