1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2015 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
32 #include "double-int.h"
39 #include "fold-const.h"
40 #include "stringpool.h"
41 #include "stor-layout.h"
44 #include "hard-reg-set.h"
46 #include "dominance.h"
49 #include "basic-block.h"
50 #include "tree-ssa-alias.h"
51 #include "internal-fn.h"
52 #include "gimple-fold.h"
53 #include "gimple-expr.h"
57 #include "gimple-iterator.h"
58 #include "gimplify-me.h"
59 #include "gimple-walk.h"
60 #include "tree-iterator.h"
61 #include "tree-inline.h"
62 #include "langhooks.h"
63 #include "diagnostic-core.h"
64 #include "gimple-ssa.h"
66 #include "plugin-api.h"
70 #include "tree-phinodes.h"
71 #include "ssa-iterators.h"
72 #include "tree-ssanames.h"
73 #include "tree-into-ssa.h"
76 #include "statistics.h"
78 #include "fixed-value.h"
79 #include "insn-config.h"
90 #include "tree-pass.h"
92 #include "splay-tree.h"
93 #include "insn-codes.h"
97 #include "common/common-target.h"
99 #include "gimple-low.h"
100 #include "tree-cfgcleanup.h"
101 #include "pretty-print.h"
102 #include "alloc-pool.h"
103 #include "symbol-summary.h"
104 #include "ipa-prop.h"
105 #include "tree-nested.h"
109 #include "lto-section-names.h"
110 #include "gomp-constants.h"
113 /* Lowering of OMP parallel and workshare constructs proceeds in two
114 phases. The first phase scans the function looking for OMP statements
115 and then for variables that must be replaced to satisfy data sharing
116 clauses. The second phase expands code for the constructs, as well as
117 re-gimplifying things when variables have been replaced with complex
120 Final code generation is done by pass_expand_omp. The flowgraph is
121 scanned for regions which are then moved to a new
122 function, to be invoked by the thread library, or offloaded. */
124 /* OMP region information. Every parallel and workshare
125 directive is enclosed between two markers, the OMP_* directive
126 and a corresponding OMP_RETURN statement. */
130 /* The enclosing region. */
131 struct omp_region
*outer
;
133 /* First child region. */
134 struct omp_region
*inner
;
136 /* Next peer region. */
137 struct omp_region
*next
;
139 /* Block containing the omp directive as its last stmt. */
142 /* Block containing the OMP_RETURN as its last stmt. */
145 /* Block containing the OMP_CONTINUE as its last stmt. */
148 /* If this is a combined parallel+workshare region, this is a list
149 of additional arguments needed by the combined parallel+workshare
151 vec
<tree
, va_gc
> *ws_args
;
153 /* The code for the omp directive of this region. */
154 enum gimple_code type
;
156 /* Schedule kind, only used for OMP_FOR type regions. */
157 enum omp_clause_schedule_kind sched_kind
;
159 /* True if this is a combined parallel+workshare region. */
160 bool is_combined_parallel
;
163 /* Levels of parallelism as defined by OpenACC. Increasing numbers
164 correspond to deeper loop nesting levels. */
166 #define MASK_WORKER 2
167 #define MASK_VECTOR 4
169 /* Context structure. Used to store information about each parallel
170 directive in the code. */
172 typedef struct omp_context
174 /* This field must be at the beginning, as we do "inheritance": Some
175 callback functions for tree-inline.c (e.g., omp_copy_decl)
176 receive a copy_body_data pointer that is up-casted to an
177 omp_context pointer. */
180 /* The tree of contexts corresponding to the encountered constructs. */
181 struct omp_context
*outer
;
184 /* Map variables to fields in a structure that allows communication
185 between sending and receiving threads. */
186 splay_tree field_map
;
191 /* These are used just by task contexts, if task firstprivate fn is
192 needed. srecord_type is used to communicate from the thread
193 that encountered the task construct to task firstprivate fn,
194 record_type is allocated by GOMP_task, initialized by task firstprivate
195 fn and passed to the task body fn. */
196 splay_tree sfield_map
;
199 /* A chain of variables to add to the top-level block surrounding the
200 construct. In the case of a parallel, this is in the child function. */
203 /* A map of reduction pointer variables. For accelerators, each
204 reduction variable is replaced with an array. Each thread, in turn,
205 is assigned to a slot on that array. */
206 splay_tree reduction_map
;
208 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
209 barriers should jump to during omplower pass. */
212 /* What to do with variables with implicitly determined sharing
214 enum omp_clause_default_kind default_kind
;
216 /* Nesting depth of this context. Used to beautify error messages re
217 invalid gotos. The outermost ctx is depth 1, with depth 0 being
218 reserved for the main body of the function. */
221 /* True if this parallel directive is nested within another. */
224 /* True if this construct can be cancelled. */
227 /* For OpenACC loops, a mask of gang, worker and vector used at
228 levels below this one. */
230 /* For OpenACC loops, a mask of gang, worker and vector used at
231 this level and above. For parallel and kernels clauses, a mask
232 indicating which of num_gangs/num_workers/num_vectors was used. */
236 /* A structure holding the elements of:
237 for (V = N1; V cond N2; V += STEP) [...] */
239 struct omp_for_data_loop
241 tree v
, n1
, n2
, step
;
242 enum tree_code cond_code
;
245 /* A structure describing the main elements of a parallel loop. */
249 struct omp_for_data_loop loop
;
254 bool have_nowait
, have_ordered
;
255 enum omp_clause_schedule_kind sched_kind
;
256 struct omp_for_data_loop
*loops
;
260 static splay_tree all_contexts
;
261 static int taskreg_nesting_level
;
262 static int target_nesting_level
;
263 static struct omp_region
*root_omp_region
;
264 static bitmap task_shared_vars
;
265 static vec
<omp_context
*> taskreg_contexts
;
267 static void scan_omp (gimple_seq
*, omp_context
*);
268 static tree
scan_omp_1_op (tree
*, int *, void *);
270 #define WALK_SUBSTMTS \
274 case GIMPLE_EH_FILTER: \
275 case GIMPLE_TRANSACTION: \
276 /* The sub-statements for these should be walked. */ \
277 *handled_ops_p = false; \
280 /* Helper function to get the name of the array containing the partial
281 reductions for OpenACC reductions. */
283 oacc_get_reduction_array_id (tree node
)
285 const char *id
= IDENTIFIER_POINTER (DECL_NAME (node
));
286 int len
= strlen ("OACC") + strlen (id
);
287 char *temp_name
= XALLOCAVEC (char, len
+ 1);
288 snprintf (temp_name
, len
+ 1, "OACC%s", id
);
289 return IDENTIFIER_POINTER (get_identifier (temp_name
));
292 /* Determine the number of threads OpenACC threads used to determine the
293 size of the array of partial reductions. Currently, this is num_gangs
294 * vector_length. This value may be different than GOACC_GET_NUM_THREADS,
295 because it is independed of the device used. */
298 oacc_max_threads (omp_context
*ctx
)
300 tree nthreads
, vector_length
, gangs
, clauses
;
302 gangs
= fold_convert (sizetype
, integer_one_node
);
303 vector_length
= gangs
;
305 /* The reduction clause may be nested inside a loop directive.
306 Scan for the innermost vector_length clause. */
307 for (omp_context
*oc
= ctx
; oc
; oc
= oc
->outer
)
309 if (gimple_code (oc
->stmt
) != GIMPLE_OMP_TARGET
310 || (gimple_omp_target_kind (oc
->stmt
)
311 != GF_OMP_TARGET_KIND_OACC_PARALLEL
))
314 clauses
= gimple_omp_target_clauses (oc
->stmt
);
316 vector_length
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
318 vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length
),
320 OMP_CLAUSE_VECTOR_LENGTH_EXPR
323 vector_length
= fold_convert (sizetype
, integer_one_node
);
325 gangs
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
327 gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (gangs
), sizetype
,
328 OMP_CLAUSE_NUM_GANGS_EXPR (gangs
));
330 gangs
= fold_convert (sizetype
, integer_one_node
);
335 nthreads
= fold_build2 (MULT_EXPR
, sizetype
, gangs
, vector_length
);
340 /* Holds offload tables with decls. */
341 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
343 /* Holds a decl for __OFFLOAD_TABLE__. */
344 static GTY(()) tree offload_symbol_decl
;
346 /* Get the __OFFLOAD_TABLE__ symbol. */
348 get_offload_symbol_decl (void)
350 if (!offload_symbol_decl
)
352 tree decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
353 get_identifier ("__OFFLOAD_TABLE__"),
355 TREE_ADDRESSABLE (decl
) = 1;
356 TREE_PUBLIC (decl
) = 1;
357 DECL_EXTERNAL (decl
) = 1;
358 DECL_WEAK (decl
) = 1;
359 DECL_ATTRIBUTES (decl
)
360 = tree_cons (get_identifier ("weak"),
361 NULL_TREE
, DECL_ATTRIBUTES (decl
));
362 offload_symbol_decl
= decl
;
364 return offload_symbol_decl
;
367 /* Convenience function for calling scan_omp_1_op on tree operands. */
370 scan_omp_op (tree
*tp
, omp_context
*ctx
)
372 struct walk_stmt_info wi
;
374 memset (&wi
, 0, sizeof (wi
));
376 wi
.want_locations
= true;
378 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
381 static void lower_omp (gimple_seq
*, omp_context
*);
382 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
383 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
385 /* Find an OMP clause of type KIND within CLAUSES. */
388 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
390 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
391 if (OMP_CLAUSE_CODE (clauses
) == kind
)
397 /* Return true if CTX is for an omp parallel. */
400 is_parallel_ctx (omp_context
*ctx
)
402 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
406 /* Return true if CTX is for an omp task. */
409 is_task_ctx (omp_context
*ctx
)
411 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
415 /* Return true if CTX is for an omp parallel or omp task. */
418 is_taskreg_ctx (omp_context
*ctx
)
420 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
421 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
425 /* Return true if REGION is a combined parallel+workshare region. */
428 is_combined_parallel (struct omp_region
*region
)
430 return region
->is_combined_parallel
;
434 /* Extract the header elements of parallel loop FOR_STMT and store
438 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
439 struct omp_for_data_loop
*loops
)
441 tree t
, var
, *collapse_iter
, *collapse_count
;
442 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
443 struct omp_for_data_loop
*loop
;
445 struct omp_for_data_loop dummy_loop
;
446 location_t loc
= gimple_location (for_stmt
);
447 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
448 bool distribute
= gimple_omp_for_kind (for_stmt
)
449 == GF_OMP_FOR_KIND_DISTRIBUTE
;
451 fd
->for_stmt
= for_stmt
;
453 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
454 if (fd
->collapse
> 1)
457 fd
->loops
= &fd
->loop
;
459 fd
->have_nowait
= distribute
|| simd
;
460 fd
->have_ordered
= false;
461 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
462 fd
->chunk_size
= NULL_TREE
;
463 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
464 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
465 collapse_iter
= NULL
;
466 collapse_count
= NULL
;
468 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
469 switch (OMP_CLAUSE_CODE (t
))
471 case OMP_CLAUSE_NOWAIT
:
472 fd
->have_nowait
= true;
474 case OMP_CLAUSE_ORDERED
:
475 fd
->have_ordered
= true;
477 case OMP_CLAUSE_SCHEDULE
:
478 gcc_assert (!distribute
);
479 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
480 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
482 case OMP_CLAUSE_DIST_SCHEDULE
:
483 gcc_assert (distribute
);
484 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
486 case OMP_CLAUSE_COLLAPSE
:
487 if (fd
->collapse
> 1)
489 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
490 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
497 /* FIXME: for now map schedule(auto) to schedule(static).
498 There should be analysis to determine whether all iterations
499 are approximately the same amount of work (then schedule(static)
500 is best) or if it varies (then schedule(dynamic,N) is better). */
501 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
503 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
504 gcc_assert (fd
->chunk_size
== NULL
);
506 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
507 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
508 gcc_assert (fd
->chunk_size
== NULL
);
509 else if (fd
->chunk_size
== NULL
)
511 /* We only need to compute a default chunk size for ordered
512 static loops and dynamic loops. */
513 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
515 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
516 ? integer_zero_node
: integer_one_node
;
519 for (i
= 0; i
< fd
->collapse
; i
++)
521 if (fd
->collapse
== 1)
523 else if (loops
!= NULL
)
528 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
529 gcc_assert (SSA_VAR_P (loop
->v
));
530 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
531 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
532 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
533 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
535 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
536 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
537 switch (loop
->cond_code
)
543 gcc_assert (gimple_omp_for_kind (for_stmt
)
544 == GF_OMP_FOR_KIND_CILKSIMD
545 || (gimple_omp_for_kind (for_stmt
)
546 == GF_OMP_FOR_KIND_CILKFOR
));
549 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
550 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
552 loop
->n2
= fold_build2_loc (loc
,
553 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
554 build_int_cst (TREE_TYPE (loop
->n2
), 1));
555 loop
->cond_code
= LT_EXPR
;
558 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
559 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
561 loop
->n2
= fold_build2_loc (loc
,
562 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
563 build_int_cst (TREE_TYPE (loop
->n2
), 1));
564 loop
->cond_code
= GT_EXPR
;
570 t
= gimple_omp_for_incr (for_stmt
, i
);
571 gcc_assert (TREE_OPERAND (t
, 0) == var
);
572 switch (TREE_CODE (t
))
575 loop
->step
= TREE_OPERAND (t
, 1);
577 case POINTER_PLUS_EXPR
:
578 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
581 loop
->step
= TREE_OPERAND (t
, 1);
582 loop
->step
= fold_build1_loc (loc
,
583 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
591 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
592 && !fd
->have_ordered
))
594 if (fd
->collapse
== 1)
595 iter_type
= TREE_TYPE (loop
->v
);
597 || TYPE_PRECISION (iter_type
)
598 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
600 = build_nonstandard_integer_type
601 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
603 else if (iter_type
!= long_long_unsigned_type_node
)
605 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
606 iter_type
= long_long_unsigned_type_node
;
607 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
608 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
609 >= TYPE_PRECISION (iter_type
))
613 if (loop
->cond_code
== LT_EXPR
)
614 n
= fold_build2_loc (loc
,
615 PLUS_EXPR
, TREE_TYPE (loop
->v
),
616 loop
->n2
, loop
->step
);
619 if (TREE_CODE (n
) != INTEGER_CST
620 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
621 iter_type
= long_long_unsigned_type_node
;
623 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
624 > TYPE_PRECISION (iter_type
))
628 if (loop
->cond_code
== LT_EXPR
)
631 n2
= fold_build2_loc (loc
,
632 PLUS_EXPR
, TREE_TYPE (loop
->v
),
633 loop
->n2
, loop
->step
);
637 n1
= fold_build2_loc (loc
,
638 MINUS_EXPR
, TREE_TYPE (loop
->v
),
639 loop
->n2
, loop
->step
);
642 if (TREE_CODE (n1
) != INTEGER_CST
643 || TREE_CODE (n2
) != INTEGER_CST
644 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
645 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
646 iter_type
= long_long_unsigned_type_node
;
650 if (collapse_count
&& *collapse_count
== NULL
)
652 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
653 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
654 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
655 if (t
&& integer_zerop (t
))
656 count
= build_zero_cst (long_long_unsigned_type_node
);
657 else if ((i
== 0 || count
!= NULL_TREE
)
658 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
659 && TREE_CONSTANT (loop
->n1
)
660 && TREE_CONSTANT (loop
->n2
)
661 && TREE_CODE (loop
->step
) == INTEGER_CST
)
663 tree itype
= TREE_TYPE (loop
->v
);
665 if (POINTER_TYPE_P (itype
))
666 itype
= signed_type_for (itype
);
667 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
668 t
= fold_build2_loc (loc
,
670 fold_convert_loc (loc
, itype
, loop
->step
), t
);
671 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
672 fold_convert_loc (loc
, itype
, loop
->n2
));
673 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
674 fold_convert_loc (loc
, itype
, loop
->n1
));
675 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
676 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
677 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
678 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
679 fold_convert_loc (loc
, itype
,
682 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
683 fold_convert_loc (loc
, itype
, loop
->step
));
684 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
685 if (count
!= NULL_TREE
)
686 count
= fold_build2_loc (loc
,
687 MULT_EXPR
, long_long_unsigned_type_node
,
691 if (TREE_CODE (count
) != INTEGER_CST
)
694 else if (count
&& !integer_zerop (count
))
701 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
702 || fd
->have_ordered
))
704 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
705 iter_type
= long_long_unsigned_type_node
;
707 iter_type
= long_integer_type_node
;
709 else if (collapse_iter
&& *collapse_iter
!= NULL
)
710 iter_type
= TREE_TYPE (*collapse_iter
);
711 fd
->iter_type
= iter_type
;
712 if (collapse_iter
&& *collapse_iter
== NULL
)
713 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
714 if (collapse_count
&& *collapse_count
== NULL
)
717 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
719 *collapse_count
= create_tmp_var (iter_type
, ".count");
722 if (fd
->collapse
> 1)
724 fd
->loop
.v
= *collapse_iter
;
725 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
726 fd
->loop
.n2
= *collapse_count
;
727 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
728 fd
->loop
.cond_code
= LT_EXPR
;
731 /* For OpenACC loops, force a chunk size of one, as this avoids the default
732 scheduling where several subsequent iterations are being executed by the
734 if (gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
736 gcc_assert (fd
->chunk_size
== NULL_TREE
);
737 fd
->chunk_size
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
742 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
743 is the immediate dominator of PAR_ENTRY_BB, return true if there
744 are no data dependencies that would prevent expanding the parallel
745 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
747 When expanding a combined parallel+workshare region, the call to
748 the child function may need additional arguments in the case of
749 GIMPLE_OMP_FOR regions. In some cases, these arguments are
750 computed out of variables passed in from the parent to the child
751 via 'struct .omp_data_s'. For instance:
753 #pragma omp parallel for schedule (guided, i * 4)
758 # BLOCK 2 (PAR_ENTRY_BB)
760 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
762 # BLOCK 3 (WS_ENTRY_BB)
763 .omp_data_i = &.omp_data_o;
764 D.1667 = .omp_data_i->i;
766 #pragma omp for schedule (guided, D.1598)
768 When we outline the parallel region, the call to the child function
769 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
770 that value is computed *after* the call site. So, in principle we
771 cannot do the transformation.
773 To see whether the code in WS_ENTRY_BB blocks the combined
774 parallel+workshare call, we collect all the variables used in the
775 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
776 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
779 FIXME. If we had the SSA form built at this point, we could merely
780 hoist the code in block 3 into block 2 and be done with it. But at
781 this point we don't have dataflow information and though we could
782 hack something up here, it is really not worth the aggravation. */
785 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
787 struct omp_for_data fd
;
788 gimple ws_stmt
= last_stmt (ws_entry_bb
);
790 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
793 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
795 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
797 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
799 if (fd
.iter_type
!= long_integer_type_node
)
802 /* FIXME. We give up too easily here. If any of these arguments
803 are not constants, they will likely involve variables that have
804 been mapped into fields of .omp_data_s for sharing with the child
805 function. With appropriate data flow, it would be possible to
807 if (!is_gimple_min_invariant (fd
.loop
.n1
)
808 || !is_gimple_min_invariant (fd
.loop
.n2
)
809 || !is_gimple_min_invariant (fd
.loop
.step
)
810 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
817 /* Collect additional arguments needed to emit a combined
818 parallel+workshare call. WS_STMT is the workshare directive being
821 static vec
<tree
, va_gc
> *
822 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
825 location_t loc
= gimple_location (ws_stmt
);
826 vec
<tree
, va_gc
> *ws_args
;
828 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
830 struct omp_for_data fd
;
833 extract_omp_for_data (for_stmt
, &fd
, NULL
);
837 if (gimple_omp_for_combined_into_p (for_stmt
))
840 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
841 OMP_CLAUSE__LOOPTEMP_
);
843 n1
= OMP_CLAUSE_DECL (innerc
);
844 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
845 OMP_CLAUSE__LOOPTEMP_
);
847 n2
= OMP_CLAUSE_DECL (innerc
);
850 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
852 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
853 ws_args
->quick_push (t
);
855 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
856 ws_args
->quick_push (t
);
858 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
859 ws_args
->quick_push (t
);
863 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
864 ws_args
->quick_push (t
);
869 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
871 /* Number of sections is equal to the number of edges from the
872 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
873 the exit of the sections region. */
874 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
875 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
876 vec_alloc (ws_args
, 1);
877 ws_args
->quick_push (t
);
885 /* Discover whether REGION is a combined parallel+workshare region. */
888 determine_parallel_type (struct omp_region
*region
)
890 basic_block par_entry_bb
, par_exit_bb
;
891 basic_block ws_entry_bb
, ws_exit_bb
;
893 if (region
== NULL
|| region
->inner
== NULL
894 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
895 || region
->inner
->cont
== NULL
)
898 /* We only support parallel+for and parallel+sections. */
899 if (region
->type
!= GIMPLE_OMP_PARALLEL
900 || (region
->inner
->type
!= GIMPLE_OMP_FOR
901 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
904 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
905 WS_EXIT_BB -> PAR_EXIT_BB. */
906 par_entry_bb
= region
->entry
;
907 par_exit_bb
= region
->exit
;
908 ws_entry_bb
= region
->inner
->entry
;
909 ws_exit_bb
= region
->inner
->exit
;
911 if (single_succ (par_entry_bb
) == ws_entry_bb
912 && single_succ (ws_exit_bb
) == par_exit_bb
913 && workshare_safe_to_combine_p (ws_entry_bb
)
914 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
915 || (last_and_only_stmt (ws_entry_bb
)
916 && last_and_only_stmt (par_exit_bb
))))
918 gimple par_stmt
= last_stmt (par_entry_bb
);
919 gimple ws_stmt
= last_stmt (ws_entry_bb
);
921 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
923 /* If this is a combined parallel loop, we need to determine
924 whether or not to use the combined library calls. There
925 are two cases where we do not apply the transformation:
926 static loops and any kind of ordered loop. In the first
927 case, we already open code the loop so there is no need
928 to do anything else. In the latter case, the combined
929 parallel loop call would still need extra synchronization
930 to implement ordered semantics, so there would not be any
931 gain in using the combined call. */
932 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
933 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
935 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
936 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
938 region
->is_combined_parallel
= false;
939 region
->inner
->is_combined_parallel
= false;
944 region
->is_combined_parallel
= true;
945 region
->inner
->is_combined_parallel
= true;
946 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
951 /* Return true if EXPR is variable sized. */
954 is_variable_sized (const_tree expr
)
956 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
959 /* Return true if DECL is a reference type. */
962 is_reference (tree decl
)
964 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
967 /* Return the type of a decl. If the decl is reference type,
968 return its base type. */
970 get_base_type (tree decl
)
972 tree type
= TREE_TYPE (decl
);
973 if (is_reference (decl
))
974 type
= TREE_TYPE (type
);
978 /* Lookup variables. The "maybe" form
979 allows for the variable form to not have been entered, otherwise we
980 assert that the variable must have been entered. */
983 lookup_decl (tree var
, omp_context
*ctx
)
985 tree
*n
= ctx
->cb
.decl_map
->get (var
);
990 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
992 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
993 return n
? *n
: NULL_TREE
;
997 lookup_field (tree var
, omp_context
*ctx
)
1000 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
1001 return (tree
) n
->value
;
1005 lookup_sfield (tree var
, omp_context
*ctx
)
1008 n
= splay_tree_lookup (ctx
->sfield_map
1009 ? ctx
->sfield_map
: ctx
->field_map
,
1010 (splay_tree_key
) var
);
1011 return (tree
) n
->value
;
1015 maybe_lookup_field (tree var
, omp_context
*ctx
)
1018 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
1019 return n
? (tree
) n
->value
: NULL_TREE
;
1023 lookup_oacc_reduction (const char *id
, omp_context
*ctx
)
1026 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) id
);
1027 return (tree
) n
->value
;
1031 maybe_lookup_oacc_reduction (tree var
, omp_context
*ctx
)
1033 splay_tree_node n
= NULL
;
1034 if (ctx
->reduction_map
)
1035 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) var
);
1036 return n
? (tree
) n
->value
: NULL_TREE
;
1039 /* Return true if DECL should be copied by pointer. SHARED_CTX is
1040 the parallel context if DECL is to be shared. */
1043 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
1045 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
1048 /* We can only use copy-in/copy-out semantics for shared variables
1049 when we know the value is not accessible from an outer scope. */
1052 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1054 /* ??? Trivially accessible from anywhere. But why would we even
1055 be passing an address in this case? Should we simply assert
1056 this to be false, or should we have a cleanup pass that removes
1057 these from the list of mappings? */
1058 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1061 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1062 without analyzing the expression whether or not its location
1063 is accessible to anyone else. In the case of nested parallel
1064 regions it certainly may be. */
1065 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1068 /* Do not use copy-in/copy-out for variables that have their
1070 if (TREE_ADDRESSABLE (decl
))
1073 /* lower_send_shared_vars only uses copy-in, but not copy-out
1075 if (TREE_READONLY (decl
)
1076 || ((TREE_CODE (decl
) == RESULT_DECL
1077 || TREE_CODE (decl
) == PARM_DECL
)
1078 && DECL_BY_REFERENCE (decl
)))
1081 /* Disallow copy-in/out in nested parallel if
1082 decl is shared in outer parallel, otherwise
1083 each thread could store the shared variable
1084 in its own copy-in location, making the
1085 variable no longer really shared. */
1086 if (shared_ctx
->is_nested
)
1090 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1091 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1098 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1099 c
; c
= OMP_CLAUSE_CHAIN (c
))
1100 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1101 && OMP_CLAUSE_DECL (c
) == decl
)
1105 goto maybe_mark_addressable_and_ret
;
1109 /* For tasks avoid using copy-in/out. As tasks can be
1110 deferred or executed in different thread, when GOMP_task
1111 returns, the task hasn't necessarily terminated. */
1112 if (is_task_ctx (shared_ctx
))
1115 maybe_mark_addressable_and_ret
:
1116 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1117 if (is_gimple_reg (outer
))
1119 /* Taking address of OUTER in lower_send_shared_vars
1120 might need regimplification of everything that uses the
1122 if (!task_shared_vars
)
1123 task_shared_vars
= BITMAP_ALLOC (NULL
);
1124 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1125 TREE_ADDRESSABLE (outer
) = 1;
1134 /* Construct a new automatic decl similar to VAR. */
1137 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1139 tree copy
= copy_var_decl (var
, name
, type
);
1141 DECL_CONTEXT (copy
) = current_function_decl
;
1142 DECL_CHAIN (copy
) = ctx
->block_vars
;
1143 ctx
->block_vars
= copy
;
1149 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1151 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1154 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1157 omp_build_component_ref (tree obj
, tree field
)
1159 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1160 if (TREE_THIS_VOLATILE (field
))
1161 TREE_THIS_VOLATILE (ret
) |= 1;
1162 if (TREE_READONLY (field
))
1163 TREE_READONLY (ret
) |= 1;
1167 /* Build tree nodes to access the field for VAR on the receiver side. */
1170 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1172 tree x
, field
= lookup_field (var
, ctx
);
1174 /* If the receiver record type was remapped in the child function,
1175 remap the field into the new record type. */
1176 x
= maybe_lookup_field (field
, ctx
);
1180 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1181 x
= omp_build_component_ref (x
, field
);
1183 x
= build_simple_mem_ref (x
);
1188 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1189 of a parallel, this is a component reference; for workshare constructs
1190 this is some variable. */
1193 build_outer_var_ref (tree var
, omp_context
*ctx
)
1197 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1199 else if (is_variable_sized (var
))
1201 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1202 x
= build_outer_var_ref (x
, ctx
);
1203 x
= build_simple_mem_ref (x
);
1205 else if (is_taskreg_ctx (ctx
))
1207 bool by_ref
= use_pointer_for_field (var
, NULL
);
1208 x
= build_receiver_ref (var
, by_ref
, ctx
);
1210 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1211 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1213 /* #pragma omp simd isn't a worksharing construct, and can reference even
1214 private vars in its linear etc. clauses. */
1216 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1217 x
= lookup_decl (var
, ctx
->outer
);
1218 else if (ctx
->outer
)
1219 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1223 else if (ctx
->outer
)
1224 x
= lookup_decl (var
, ctx
->outer
);
1225 else if (is_reference (var
))
1226 /* This can happen with orphaned constructs. If var is reference, it is
1227 possible it is shared and as such valid. */
1232 if (is_reference (var
))
1233 x
= build_simple_mem_ref (x
);
1238 /* Build tree nodes to access the field for VAR on the sender side. */
1241 build_sender_ref (tree var
, omp_context
*ctx
)
1243 tree field
= lookup_sfield (var
, ctx
);
1244 return omp_build_component_ref (ctx
->sender_decl
, field
);
1247 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1250 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1252 tree field
, type
, sfield
= NULL_TREE
;
1254 gcc_assert ((mask
& 1) == 0
1255 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1256 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1257 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1258 gcc_assert ((mask
& 3) == 3
1259 || !is_gimple_omp_oacc (ctx
->stmt
));
1261 type
= TREE_TYPE (var
);
1264 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1265 type
= build_pointer_type (build_pointer_type (type
));
1268 type
= build_pointer_type (type
);
1269 else if ((mask
& 3) == 1 && is_reference (var
))
1270 type
= TREE_TYPE (type
);
1272 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1273 FIELD_DECL
, DECL_NAME (var
), type
);
1275 /* Remember what variable this field was created for. This does have a
1276 side effect of making dwarf2out ignore this member, so for helpful
1277 debugging we clear it later in delete_omp_context. */
1278 DECL_ABSTRACT_ORIGIN (field
) = var
;
1279 if (type
== TREE_TYPE (var
))
1281 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1282 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1283 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1286 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1288 if ((mask
& 3) == 3)
1290 insert_field_into_struct (ctx
->record_type
, field
);
1291 if (ctx
->srecord_type
)
1293 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1294 FIELD_DECL
, DECL_NAME (var
), type
);
1295 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1296 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1297 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1298 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1299 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1304 if (ctx
->srecord_type
== NULL_TREE
)
1308 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1309 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1310 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1312 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1313 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1314 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1315 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1316 splay_tree_insert (ctx
->sfield_map
,
1317 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1318 (splay_tree_value
) sfield
);
1322 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1323 : ctx
->srecord_type
, field
);
1327 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1328 (splay_tree_value
) field
);
1329 if ((mask
& 2) && ctx
->sfield_map
)
1330 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1331 (splay_tree_value
) sfield
);
1335 install_var_local (tree var
, omp_context
*ctx
)
1337 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1338 insert_decl_map (&ctx
->cb
, var
, new_var
);
1342 /* Adjust the replacement for DECL in CTX for the new context. This means
1343 copying the DECL_VALUE_EXPR, and fixing up the type. */
1346 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1348 tree new_decl
, size
;
1350 new_decl
= lookup_decl (decl
, ctx
);
1352 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1354 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1355 && DECL_HAS_VALUE_EXPR_P (decl
))
1357 tree ve
= DECL_VALUE_EXPR (decl
);
1358 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1359 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1360 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1363 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1365 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1366 if (size
== error_mark_node
)
1367 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1368 DECL_SIZE (new_decl
) = size
;
1370 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1371 if (size
== error_mark_node
)
1372 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1373 DECL_SIZE_UNIT (new_decl
) = size
;
1377 /* The callback for remap_decl. Search all containing contexts for a
1378 mapping of the variable; this avoids having to duplicate the splay
1379 tree ahead of time. We know a mapping doesn't already exist in the
1380 given context. Create new mappings to implement default semantics. */
1383 omp_copy_decl (tree var
, copy_body_data
*cb
)
1385 omp_context
*ctx
= (omp_context
*) cb
;
1388 if (TREE_CODE (var
) == LABEL_DECL
)
1390 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1391 DECL_CONTEXT (new_var
) = current_function_decl
;
1392 insert_decl_map (&ctx
->cb
, var
, new_var
);
1396 while (!is_taskreg_ctx (ctx
))
1401 new_var
= maybe_lookup_decl (var
, ctx
);
1406 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1409 return error_mark_node
;
1413 /* Debugging dumps for parallel regions. */
1414 void dump_omp_region (FILE *, struct omp_region
*, int);
1415 void debug_omp_region (struct omp_region
*);
1416 void debug_all_omp_regions (void);
1418 /* Dump the parallel region tree rooted at REGION. */
1421 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1423 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1424 gimple_code_name
[region
->type
]);
1427 dump_omp_region (file
, region
->inner
, indent
+ 4);
1431 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1432 region
->cont
->index
);
1436 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1437 region
->exit
->index
);
1439 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1442 dump_omp_region (file
, region
->next
, indent
);
1446 debug_omp_region (struct omp_region
*region
)
1448 dump_omp_region (stderr
, region
, 0);
1452 debug_all_omp_regions (void)
1454 dump_omp_region (stderr
, root_omp_region
, 0);
1458 /* Create a new parallel region starting at STMT inside region PARENT. */
1460 static struct omp_region
*
1461 new_omp_region (basic_block bb
, enum gimple_code type
,
1462 struct omp_region
*parent
)
1464 struct omp_region
*region
= XCNEW (struct omp_region
);
1466 region
->outer
= parent
;
1468 region
->type
= type
;
1472 /* This is a nested region. Add it to the list of inner
1473 regions in PARENT. */
1474 region
->next
= parent
->inner
;
1475 parent
->inner
= region
;
1479 /* This is a toplevel region. Add it to the list of toplevel
1480 regions in ROOT_OMP_REGION. */
1481 region
->next
= root_omp_region
;
1482 root_omp_region
= region
;
1488 /* Release the memory associated with the region tree rooted at REGION. */
1491 free_omp_region_1 (struct omp_region
*region
)
1493 struct omp_region
*i
, *n
;
1495 for (i
= region
->inner
; i
; i
= n
)
1498 free_omp_region_1 (i
);
1504 /* Release the memory for the entire omp region tree. */
1507 free_omp_regions (void)
1509 struct omp_region
*r
, *n
;
1510 for (r
= root_omp_region
; r
; r
= n
)
1513 free_omp_region_1 (r
);
1515 root_omp_region
= NULL
;
1519 /* Create a new context, with OUTER_CTX being the surrounding context. */
1521 static omp_context
*
1522 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1524 omp_context
*ctx
= XCNEW (omp_context
);
1526 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1527 (splay_tree_value
) ctx
);
1532 ctx
->outer
= outer_ctx
;
1533 ctx
->cb
= outer_ctx
->cb
;
1534 ctx
->cb
.block
= NULL
;
1535 ctx
->depth
= outer_ctx
->depth
+ 1;
1536 ctx
->reduction_map
= outer_ctx
->reduction_map
;
1540 ctx
->cb
.src_fn
= current_function_decl
;
1541 ctx
->cb
.dst_fn
= current_function_decl
;
1542 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1543 gcc_checking_assert (ctx
->cb
.src_node
);
1544 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1545 ctx
->cb
.src_cfun
= cfun
;
1546 ctx
->cb
.copy_decl
= omp_copy_decl
;
1547 ctx
->cb
.eh_lp_nr
= 0;
1548 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1552 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1557 static gimple_seq
maybe_catch_exception (gimple_seq
);
1559 /* Finalize task copyfn. */
1562 finalize_task_copyfn (gomp_task
*task_stmt
)
1564 struct function
*child_cfun
;
1566 gimple_seq seq
= NULL
, new_seq
;
1569 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1570 if (child_fn
== NULL_TREE
)
1573 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1574 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1576 push_cfun (child_cfun
);
1577 bind
= gimplify_body (child_fn
, false);
1578 gimple_seq_add_stmt (&seq
, bind
);
1579 new_seq
= maybe_catch_exception (seq
);
1582 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1584 gimple_seq_add_stmt (&seq
, bind
);
1586 gimple_set_body (child_fn
, seq
);
1589 /* Inform the callgraph about the new function. */
1590 cgraph_node::add_new_function (child_fn
, false);
1593 /* Destroy a omp_context data structures. Called through the splay tree
1594 value delete callback. */
1597 delete_omp_context (splay_tree_value value
)
1599 omp_context
*ctx
= (omp_context
*) value
;
1601 delete ctx
->cb
.decl_map
;
1604 splay_tree_delete (ctx
->field_map
);
1605 if (ctx
->sfield_map
)
1606 splay_tree_delete (ctx
->sfield_map
);
1607 if (ctx
->reduction_map
1608 /* Shared over several omp_contexts. */
1609 && (ctx
->outer
== NULL
1610 || ctx
->reduction_map
!= ctx
->outer
->reduction_map
))
1611 splay_tree_delete (ctx
->reduction_map
);
1613 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1614 it produces corrupt debug information. */
1615 if (ctx
->record_type
)
1618 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1619 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1621 if (ctx
->srecord_type
)
1624 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1625 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1628 if (is_task_ctx (ctx
))
1629 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1634 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1638 fixup_child_record_type (omp_context
*ctx
)
1640 tree f
, type
= ctx
->record_type
;
1642 /* ??? It isn't sufficient to just call remap_type here, because
1643 variably_modified_type_p doesn't work the way we expect for
1644 record types. Testing each field for whether it needs remapping
1645 and creating a new record by hand works, however. */
1646 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1647 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1651 tree name
, new_fields
= NULL
;
1653 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1654 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1655 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1656 TYPE_DECL
, name
, type
);
1657 TYPE_NAME (type
) = name
;
1659 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1661 tree new_f
= copy_node (f
);
1662 DECL_CONTEXT (new_f
) = type
;
1663 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1664 DECL_CHAIN (new_f
) = new_fields
;
1665 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1666 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1668 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1672 /* Arrange to be able to look up the receiver field
1673 given the sender field. */
1674 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1675 (splay_tree_value
) new_f
);
1677 TYPE_FIELDS (type
) = nreverse (new_fields
);
1681 TREE_TYPE (ctx
->receiver_decl
)
1682 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1685 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1686 specified by CLAUSES. */
1689 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1692 bool scan_array_reductions
= false;
1694 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1698 switch (OMP_CLAUSE_CODE (c
))
1700 case OMP_CLAUSE_PRIVATE
:
1701 decl
= OMP_CLAUSE_DECL (c
);
1702 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1704 else if (!is_variable_sized (decl
))
1705 install_var_local (decl
, ctx
);
1708 case OMP_CLAUSE_SHARED
:
1709 decl
= OMP_CLAUSE_DECL (c
);
1710 /* Ignore shared directives in teams construct. */
1711 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1713 /* Global variables don't need to be copied,
1714 the receiver side will use them directly. */
1715 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1716 if (is_global_var (odecl
))
1718 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1721 gcc_assert (is_taskreg_ctx (ctx
));
1722 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1723 || !is_variable_sized (decl
));
1724 /* Global variables don't need to be copied,
1725 the receiver side will use them directly. */
1726 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1728 by_ref
= use_pointer_for_field (decl
, ctx
);
1729 if (! TREE_READONLY (decl
)
1730 || TREE_ADDRESSABLE (decl
)
1732 || is_reference (decl
))
1734 install_var_field (decl
, by_ref
, 3, ctx
);
1735 install_var_local (decl
, ctx
);
1738 /* We don't need to copy const scalar vars back. */
1739 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1742 case OMP_CLAUSE_LASTPRIVATE
:
1743 /* Let the corresponding firstprivate clause create
1745 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1749 case OMP_CLAUSE_FIRSTPRIVATE
:
1750 if (is_gimple_omp_oacc (ctx
->stmt
))
1752 sorry ("clause not supported yet");
1756 case OMP_CLAUSE_REDUCTION
:
1757 case OMP_CLAUSE_LINEAR
:
1758 decl
= OMP_CLAUSE_DECL (c
);
1760 if (is_variable_sized (decl
))
1762 if (is_task_ctx (ctx
))
1763 install_var_field (decl
, false, 1, ctx
);
1766 else if (is_taskreg_ctx (ctx
))
1769 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1770 by_ref
= use_pointer_for_field (decl
, NULL
);
1772 if (is_task_ctx (ctx
)
1773 && (global
|| by_ref
|| is_reference (decl
)))
1775 install_var_field (decl
, false, 1, ctx
);
1777 install_var_field (decl
, by_ref
, 2, ctx
);
1780 install_var_field (decl
, by_ref
, 3, ctx
);
1782 install_var_local (decl
, ctx
);
1783 if (is_gimple_omp_oacc (ctx
->stmt
)
1784 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
1786 /* Create a decl for the reduction array. */
1787 tree var
= OMP_CLAUSE_DECL (c
);
1788 tree type
= get_base_type (var
);
1789 tree ptype
= build_pointer_type (type
);
1790 tree array
= create_tmp_var (ptype
,
1791 oacc_get_reduction_array_id (var
));
1792 omp_context
*c
= (ctx
->field_map
? ctx
: ctx
->outer
);
1793 install_var_field (array
, true, 3, c
);
1794 install_var_local (array
, c
);
1796 /* Insert it into the current context. */
1797 splay_tree_insert (ctx
->reduction_map
, (splay_tree_key
)
1798 oacc_get_reduction_array_id (var
),
1799 (splay_tree_value
) array
);
1800 splay_tree_insert (ctx
->reduction_map
,
1801 (splay_tree_key
) array
,
1802 (splay_tree_value
) array
);
1806 case OMP_CLAUSE__LOOPTEMP_
:
1807 gcc_assert (is_parallel_ctx (ctx
));
1808 decl
= OMP_CLAUSE_DECL (c
);
1809 install_var_field (decl
, false, 3, ctx
);
1810 install_var_local (decl
, ctx
);
1813 case OMP_CLAUSE_COPYPRIVATE
:
1814 case OMP_CLAUSE_COPYIN
:
1815 decl
= OMP_CLAUSE_DECL (c
);
1816 by_ref
= use_pointer_for_field (decl
, NULL
);
1817 install_var_field (decl
, by_ref
, 3, ctx
);
1820 case OMP_CLAUSE_DEFAULT
:
1821 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1824 case OMP_CLAUSE_FINAL
:
1826 case OMP_CLAUSE_NUM_THREADS
:
1827 case OMP_CLAUSE_NUM_TEAMS
:
1828 case OMP_CLAUSE_THREAD_LIMIT
:
1829 case OMP_CLAUSE_DEVICE
:
1830 case OMP_CLAUSE_SCHEDULE
:
1831 case OMP_CLAUSE_DIST_SCHEDULE
:
1832 case OMP_CLAUSE_DEPEND
:
1833 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1834 case OMP_CLAUSE_NUM_GANGS
:
1835 case OMP_CLAUSE_NUM_WORKERS
:
1836 case OMP_CLAUSE_VECTOR_LENGTH
:
1838 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1842 case OMP_CLAUSE_FROM
:
1843 case OMP_CLAUSE_MAP
:
1845 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1846 decl
= OMP_CLAUSE_DECL (c
);
1847 /* Global variables with "omp declare target" attribute
1848 don't need to be copied, the receiver side will use them
1850 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1852 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1853 && varpool_node::get_create (decl
)->offloadable
)
1855 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1856 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
1858 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
1859 not offloaded; there is nothing to map for those. */
1860 if (!is_gimple_omp_offloaded (ctx
->stmt
)
1861 && !POINTER_TYPE_P (TREE_TYPE (decl
)))
1866 if (DECL_SIZE (decl
)
1867 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1869 tree decl2
= DECL_VALUE_EXPR (decl
);
1870 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1871 decl2
= TREE_OPERAND (decl2
, 0);
1872 gcc_assert (DECL_P (decl2
));
1873 install_var_field (decl2
, true, 3, ctx
);
1874 install_var_local (decl2
, ctx
);
1875 install_var_local (decl
, ctx
);
1879 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1880 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
1881 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1882 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1883 install_var_field (decl
, true, 7, ctx
);
1885 install_var_field (decl
, true, 3, ctx
);
1886 if (is_gimple_omp_offloaded (ctx
->stmt
))
1887 install_var_local (decl
, ctx
);
1892 tree base
= get_base_address (decl
);
1893 tree nc
= OMP_CLAUSE_CHAIN (c
);
1896 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1897 && OMP_CLAUSE_DECL (nc
) == base
1898 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
1899 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1901 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1902 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1908 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1909 decl
= OMP_CLAUSE_DECL (c
);
1911 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1912 (splay_tree_key
) decl
));
1914 = build_decl (OMP_CLAUSE_LOCATION (c
),
1915 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1916 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1917 insert_field_into_struct (ctx
->record_type
, field
);
1918 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1919 (splay_tree_value
) field
);
1924 case OMP_CLAUSE_NOWAIT
:
1925 case OMP_CLAUSE_ORDERED
:
1926 case OMP_CLAUSE_COLLAPSE
:
1927 case OMP_CLAUSE_UNTIED
:
1928 case OMP_CLAUSE_MERGEABLE
:
1929 case OMP_CLAUSE_PROC_BIND
:
1930 case OMP_CLAUSE_SAFELEN
:
1931 case OMP_CLAUSE_ASYNC
:
1932 case OMP_CLAUSE_WAIT
:
1933 case OMP_CLAUSE_GANG
:
1934 case OMP_CLAUSE_WORKER
:
1935 case OMP_CLAUSE_VECTOR
:
1938 case OMP_CLAUSE_ALIGNED
:
1939 decl
= OMP_CLAUSE_DECL (c
);
1940 if (is_global_var (decl
)
1941 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1942 install_var_local (decl
, ctx
);
1945 case OMP_CLAUSE_DEVICE_RESIDENT
:
1946 case OMP_CLAUSE_USE_DEVICE
:
1947 case OMP_CLAUSE__CACHE_
:
1948 case OMP_CLAUSE_INDEPENDENT
:
1949 case OMP_CLAUSE_AUTO
:
1950 case OMP_CLAUSE_SEQ
:
1951 sorry ("Clause not supported yet");
1959 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1961 switch (OMP_CLAUSE_CODE (c
))
1963 case OMP_CLAUSE_LASTPRIVATE
:
1964 /* Let the corresponding firstprivate clause create
1966 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1967 scan_array_reductions
= true;
1968 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1972 case OMP_CLAUSE_FIRSTPRIVATE
:
1973 if (is_gimple_omp_oacc (ctx
->stmt
))
1975 sorry ("clause not supported yet");
1979 case OMP_CLAUSE_PRIVATE
:
1980 case OMP_CLAUSE_REDUCTION
:
1981 case OMP_CLAUSE_LINEAR
:
1982 decl
= OMP_CLAUSE_DECL (c
);
1983 if (is_variable_sized (decl
))
1984 install_var_local (decl
, ctx
);
1985 fixup_remapped_decl (decl
, ctx
,
1986 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1987 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1988 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1989 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1990 scan_array_reductions
= true;
1991 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1992 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1993 scan_array_reductions
= true;
1996 case OMP_CLAUSE_SHARED
:
1997 /* Ignore shared directives in teams construct. */
1998 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2000 decl
= OMP_CLAUSE_DECL (c
);
2001 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2002 fixup_remapped_decl (decl
, ctx
, false);
2005 case OMP_CLAUSE_MAP
:
2006 if (!is_gimple_omp_offloaded (ctx
->stmt
))
2008 decl
= OMP_CLAUSE_DECL (c
);
2010 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
2011 && varpool_node::get_create (decl
)->offloadable
)
2015 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
2016 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
2017 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
2019 tree new_decl
= lookup_decl (decl
, ctx
);
2020 TREE_TYPE (new_decl
)
2021 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
2023 else if (DECL_SIZE (decl
)
2024 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
2026 tree decl2
= DECL_VALUE_EXPR (decl
);
2027 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
2028 decl2
= TREE_OPERAND (decl2
, 0);
2029 gcc_assert (DECL_P (decl2
));
2030 fixup_remapped_decl (decl2
, ctx
, false);
2031 fixup_remapped_decl (decl
, ctx
, true);
2034 fixup_remapped_decl (decl
, ctx
, false);
2038 case OMP_CLAUSE_COPYPRIVATE
:
2039 case OMP_CLAUSE_COPYIN
:
2040 case OMP_CLAUSE_DEFAULT
:
2042 case OMP_CLAUSE_NUM_THREADS
:
2043 case OMP_CLAUSE_NUM_TEAMS
:
2044 case OMP_CLAUSE_THREAD_LIMIT
:
2045 case OMP_CLAUSE_DEVICE
:
2046 case OMP_CLAUSE_SCHEDULE
:
2047 case OMP_CLAUSE_DIST_SCHEDULE
:
2048 case OMP_CLAUSE_NOWAIT
:
2049 case OMP_CLAUSE_ORDERED
:
2050 case OMP_CLAUSE_COLLAPSE
:
2051 case OMP_CLAUSE_UNTIED
:
2052 case OMP_CLAUSE_FINAL
:
2053 case OMP_CLAUSE_MERGEABLE
:
2054 case OMP_CLAUSE_PROC_BIND
:
2055 case OMP_CLAUSE_SAFELEN
:
2056 case OMP_CLAUSE_ALIGNED
:
2057 case OMP_CLAUSE_DEPEND
:
2058 case OMP_CLAUSE__LOOPTEMP_
:
2060 case OMP_CLAUSE_FROM
:
2061 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2062 case OMP_CLAUSE_ASYNC
:
2063 case OMP_CLAUSE_WAIT
:
2064 case OMP_CLAUSE_NUM_GANGS
:
2065 case OMP_CLAUSE_NUM_WORKERS
:
2066 case OMP_CLAUSE_VECTOR_LENGTH
:
2067 case OMP_CLAUSE_GANG
:
2068 case OMP_CLAUSE_WORKER
:
2069 case OMP_CLAUSE_VECTOR
:
2072 case OMP_CLAUSE_DEVICE_RESIDENT
:
2073 case OMP_CLAUSE_USE_DEVICE
:
2074 case OMP_CLAUSE__CACHE_
:
2075 case OMP_CLAUSE_INDEPENDENT
:
2076 case OMP_CLAUSE_AUTO
:
2077 case OMP_CLAUSE_SEQ
:
2078 sorry ("Clause not supported yet");
2086 gcc_checking_assert (!scan_array_reductions
2087 || !is_gimple_omp_oacc (ctx
->stmt
));
2088 if (scan_array_reductions
)
2089 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2090 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2091 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2093 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2094 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2096 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2097 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2098 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2099 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2100 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2101 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2104 /* Create a new name for omp child function. Returns an identifier. If
2105 IS_CILK_FOR is true then the suffix for the child function is
2109 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2112 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2113 return clone_function_name (current_function_decl
,
2114 task_copy
? "_omp_cpyfn" : "_omp_fn");
2117 /* Returns the type of the induction variable for the child function for
2118 _Cilk_for and the types for _high and _low variables based on TYPE. */
2121 cilk_for_check_loop_diff_type (tree type
)
2123 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2125 if (TYPE_UNSIGNED (type
))
2126 return uint32_type_node
;
2128 return integer_type_node
;
2132 if (TYPE_UNSIGNED (type
))
2133 return uint64_type_node
;
2135 return long_long_integer_type_node
;
2139 /* Build a decl for the omp child function. It'll not contain a body
2140 yet, just the bare decl. */
2143 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2145 tree decl
, type
, name
, t
;
2148 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2149 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2150 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2151 tree cilk_var_type
= NULL_TREE
;
2153 name
= create_omp_child_function_name (task_copy
,
2154 cilk_for_count
!= NULL_TREE
);
2156 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2157 ptr_type_node
, NULL_TREE
);
2158 else if (cilk_for_count
)
2160 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2161 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2162 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2163 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2166 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2168 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2170 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2173 ctx
->cb
.dst_fn
= decl
;
2175 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2177 TREE_STATIC (decl
) = 1;
2178 TREE_USED (decl
) = 1;
2179 DECL_ARTIFICIAL (decl
) = 1;
2180 DECL_IGNORED_P (decl
) = 0;
2181 TREE_PUBLIC (decl
) = 0;
2182 DECL_UNINLINABLE (decl
) = 1;
2183 DECL_EXTERNAL (decl
) = 0;
2184 DECL_CONTEXT (decl
) = NULL_TREE
;
2185 DECL_INITIAL (decl
) = make_node (BLOCK
);
2186 if (cgraph_node::get (current_function_decl
)->offloadable
)
2187 cgraph_node::get_create (decl
)->offloadable
= 1;
2191 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2192 if (is_gimple_omp_offloaded (octx
->stmt
))
2194 cgraph_node::get_create (decl
)->offloadable
= 1;
2195 #ifdef ENABLE_OFFLOADING
2196 g
->have_offload
= true;
2202 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2203 RESULT_DECL
, NULL_TREE
, void_type_node
);
2204 DECL_ARTIFICIAL (t
) = 1;
2205 DECL_IGNORED_P (t
) = 1;
2206 DECL_CONTEXT (t
) = decl
;
2207 DECL_RESULT (decl
) = t
;
2209 /* _Cilk_for's child function requires two extra parameters called
2210 __low and __high that are set the by Cilk runtime when it calls this
2214 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2215 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2216 DECL_ARTIFICIAL (t
) = 1;
2217 DECL_NAMELESS (t
) = 1;
2218 DECL_ARG_TYPE (t
) = ptr_type_node
;
2219 DECL_CONTEXT (t
) = current_function_decl
;
2221 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2222 DECL_ARGUMENTS (decl
) = t
;
2224 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2225 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2226 DECL_ARTIFICIAL (t
) = 1;
2227 DECL_NAMELESS (t
) = 1;
2228 DECL_ARG_TYPE (t
) = ptr_type_node
;
2229 DECL_CONTEXT (t
) = current_function_decl
;
2231 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2232 DECL_ARGUMENTS (decl
) = t
;
2235 tree data_name
= get_identifier (".omp_data_i");
2236 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2238 DECL_ARTIFICIAL (t
) = 1;
2239 DECL_NAMELESS (t
) = 1;
2240 DECL_ARG_TYPE (t
) = ptr_type_node
;
2241 DECL_CONTEXT (t
) = current_function_decl
;
2244 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2245 DECL_ARGUMENTS (decl
) = t
;
2247 ctx
->receiver_decl
= t
;
2250 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2251 PARM_DECL
, get_identifier (".omp_data_o"),
2253 DECL_ARTIFICIAL (t
) = 1;
2254 DECL_NAMELESS (t
) = 1;
2255 DECL_ARG_TYPE (t
) = ptr_type_node
;
2256 DECL_CONTEXT (t
) = current_function_decl
;
2258 TREE_ADDRESSABLE (t
) = 1;
2259 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2260 DECL_ARGUMENTS (decl
) = t
;
2263 /* Allocate memory for the function structure. The call to
2264 allocate_struct_function clobbers CFUN, so we need to restore
2266 push_struct_function (decl
);
2267 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2271 /* Callback for walk_gimple_seq. Check if combined parallel
2272 contains gimple_omp_for_combined_into_p OMP_FOR. */
2275 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2276 bool *handled_ops_p
,
2277 struct walk_stmt_info
*wi
)
2279 gimple stmt
= gsi_stmt (*gsi_p
);
2281 *handled_ops_p
= true;
2282 switch (gimple_code (stmt
))
2286 case GIMPLE_OMP_FOR
:
2287 if (gimple_omp_for_combined_into_p (stmt
)
2288 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
2291 return integer_zero_node
;
2300 /* Scan an OpenMP parallel directive. */
2303 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2307 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2309 /* Ignore parallel directives with empty bodies, unless there
2310 are copyin clauses. */
2312 && empty_body_p (gimple_omp_body (stmt
))
2313 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2314 OMP_CLAUSE_COPYIN
) == NULL
)
2316 gsi_replace (gsi
, gimple_build_nop (), false);
2320 if (gimple_omp_parallel_combined_p (stmt
))
2322 struct walk_stmt_info wi
;
2324 memset (&wi
, 0, sizeof (wi
));
2326 walk_gimple_seq (gimple_omp_body (stmt
),
2327 find_combined_for
, NULL
, &wi
);
2330 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
) wi
.info
);
2331 struct omp_for_data fd
;
2332 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2333 /* We need two temporaries with fd.loop.v type (istart/iend)
2334 and then (fd.collapse - 1) temporaries with the same
2335 type for count2 ... countN-1 vars if not constant. */
2336 size_t count
= 2, i
;
2337 tree type
= fd
.iter_type
;
2339 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2340 count
+= fd
.collapse
- 1;
2341 for (i
= 0; i
< count
; i
++)
2343 tree temp
= create_tmp_var (type
);
2344 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2345 OMP_CLAUSE__LOOPTEMP_
);
2346 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2347 OMP_CLAUSE_DECL (c
) = temp
;
2348 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2349 gimple_omp_parallel_set_clauses (stmt
, c
);
2354 ctx
= new_omp_context (stmt
, outer_ctx
);
2355 taskreg_contexts
.safe_push (ctx
);
2356 if (taskreg_nesting_level
> 1)
2357 ctx
->is_nested
= true;
2358 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2359 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2360 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2361 name
= create_tmp_var_name (".omp_data_s");
2362 name
= build_decl (gimple_location (stmt
),
2363 TYPE_DECL
, name
, ctx
->record_type
);
2364 DECL_ARTIFICIAL (name
) = 1;
2365 DECL_NAMELESS (name
) = 1;
2366 TYPE_NAME (ctx
->record_type
) = name
;
2367 create_omp_child_function (ctx
, false);
2368 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2370 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2371 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2373 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2374 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2377 /* Scan an OpenMP task directive. */
2380 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2384 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2386 /* Ignore task directives with empty bodies. */
2388 && empty_body_p (gimple_omp_body (stmt
)))
2390 gsi_replace (gsi
, gimple_build_nop (), false);
2394 ctx
= new_omp_context (stmt
, outer_ctx
);
2395 taskreg_contexts
.safe_push (ctx
);
2396 if (taskreg_nesting_level
> 1)
2397 ctx
->is_nested
= true;
2398 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2399 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2400 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2401 name
= create_tmp_var_name (".omp_data_s");
2402 name
= build_decl (gimple_location (stmt
),
2403 TYPE_DECL
, name
, ctx
->record_type
);
2404 DECL_ARTIFICIAL (name
) = 1;
2405 DECL_NAMELESS (name
) = 1;
2406 TYPE_NAME (ctx
->record_type
) = name
;
2407 create_omp_child_function (ctx
, false);
2408 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2410 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2412 if (ctx
->srecord_type
)
2414 name
= create_tmp_var_name (".omp_data_a");
2415 name
= build_decl (gimple_location (stmt
),
2416 TYPE_DECL
, name
, ctx
->srecord_type
);
2417 DECL_ARTIFICIAL (name
) = 1;
2418 DECL_NAMELESS (name
) = 1;
2419 TYPE_NAME (ctx
->srecord_type
) = name
;
2420 create_omp_child_function (ctx
, true);
2423 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2425 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2427 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2428 t
= build_int_cst (long_integer_type_node
, 0);
2429 gimple_omp_task_set_arg_size (stmt
, t
);
2430 t
= build_int_cst (long_integer_type_node
, 1);
2431 gimple_omp_task_set_arg_align (stmt
, t
);
2436 /* If any decls have been made addressable during scan_omp,
2437 adjust their fields if needed, and layout record types
2438 of parallel/task constructs. */
2441 finish_taskreg_scan (omp_context
*ctx
)
2443 if (ctx
->record_type
== NULL_TREE
)
2446 /* If any task_shared_vars were needed, verify all
2447 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2448 statements if use_pointer_for_field hasn't changed
2449 because of that. If it did, update field types now. */
2450 if (task_shared_vars
)
2454 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2455 c
; c
= OMP_CLAUSE_CHAIN (c
))
2456 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
2458 tree decl
= OMP_CLAUSE_DECL (c
);
2460 /* Global variables don't need to be copied,
2461 the receiver side will use them directly. */
2462 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2464 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2465 || !use_pointer_for_field (decl
, ctx
))
2467 tree field
= lookup_field (decl
, ctx
);
2468 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2469 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2471 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2472 TREE_THIS_VOLATILE (field
) = 0;
2473 DECL_USER_ALIGN (field
) = 0;
2474 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2475 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2476 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2477 if (ctx
->srecord_type
)
2479 tree sfield
= lookup_sfield (decl
, ctx
);
2480 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2481 TREE_THIS_VOLATILE (sfield
) = 0;
2482 DECL_USER_ALIGN (sfield
) = 0;
2483 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2484 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2485 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2490 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2492 layout_type (ctx
->record_type
);
2493 fixup_child_record_type (ctx
);
2497 location_t loc
= gimple_location (ctx
->stmt
);
2498 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2499 /* Move VLA fields to the end. */
2500 p
= &TYPE_FIELDS (ctx
->record_type
);
2502 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2503 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2506 *p
= TREE_CHAIN (*p
);
2507 TREE_CHAIN (*q
) = NULL_TREE
;
2508 q
= &TREE_CHAIN (*q
);
2511 p
= &DECL_CHAIN (*p
);
2513 layout_type (ctx
->record_type
);
2514 fixup_child_record_type (ctx
);
2515 if (ctx
->srecord_type
)
2516 layout_type (ctx
->srecord_type
);
2517 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2518 TYPE_SIZE_UNIT (ctx
->record_type
));
2519 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2520 t
= build_int_cst (long_integer_type_node
,
2521 TYPE_ALIGN_UNIT (ctx
->record_type
));
2522 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2527 static omp_context
*
2528 enclosing_target_ctx (omp_context
*ctx
)
2531 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
2533 gcc_assert (ctx
!= NULL
);
2538 oacc_loop_or_target_p (gimple stmt
)
2540 enum gimple_code outer_type
= gimple_code (stmt
);
2541 return ((outer_type
== GIMPLE_OMP_TARGET
2542 && ((gimple_omp_target_kind (stmt
)
2543 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
2544 || (gimple_omp_target_kind (stmt
)
2545 == GF_OMP_TARGET_KIND_OACC_KERNELS
)))
2546 || (outer_type
== GIMPLE_OMP_FOR
2547 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
));
2550 /* Scan a GIMPLE_OMP_FOR. */
2553 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2555 enum gimple_code outer_type
= GIMPLE_ERROR_MARK
;
2558 tree clauses
= gimple_omp_for_clauses (stmt
);
2561 outer_type
= gimple_code (outer_ctx
->stmt
);
2563 ctx
= new_omp_context (stmt
, outer_ctx
);
2565 if (is_gimple_omp_oacc (stmt
))
2567 if (outer_ctx
&& outer_type
== GIMPLE_OMP_FOR
)
2568 ctx
->gwv_this
= outer_ctx
->gwv_this
;
2569 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2572 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_GANG
)
2574 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WORKER
)
2576 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_VECTOR
)
2580 ctx
->gwv_this
|= val
;
2583 /* Skip; not nested inside a region. */
2586 if (!oacc_loop_or_target_p (outer_ctx
->stmt
))
2588 /* Skip; not nested inside an OpenACC region. */
2591 if (outer_type
== GIMPLE_OMP_FOR
)
2592 outer_ctx
->gwv_below
|= val
;
2593 if (OMP_CLAUSE_OPERAND (c
, 0) != NULL_TREE
)
2595 omp_context
*enclosing
= enclosing_target_ctx (outer_ctx
);
2596 if (gimple_omp_target_kind (enclosing
->stmt
)
2597 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
2598 error_at (gimple_location (stmt
),
2599 "no arguments allowed to gang, worker and vector clauses inside parallel");
2604 scan_sharing_clauses (clauses
, ctx
);
2606 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2607 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2609 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2610 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2611 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2612 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2614 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2616 if (is_gimple_omp_oacc (stmt
))
2618 if (ctx
->gwv_this
& ctx
->gwv_below
)
2619 error_at (gimple_location (stmt
),
2620 "gang, worker and vector may occur only once in a loop nest");
2621 else if (ctx
->gwv_below
!= 0
2622 && ctx
->gwv_this
> ctx
->gwv_below
)
2623 error_at (gimple_location (stmt
),
2624 "gang, worker and vector must occur in this order in a loop nest");
2625 if (outer_ctx
&& outer_type
== GIMPLE_OMP_FOR
)
2626 outer_ctx
->gwv_below
|= ctx
->gwv_below
;
2630 /* Scan an OpenMP sections directive. */
2633 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
2637 ctx
= new_omp_context (stmt
, outer_ctx
);
2638 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2639 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2642 /* Scan an OpenMP single directive. */
2645 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
2650 ctx
= new_omp_context (stmt
, outer_ctx
);
2651 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2652 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2653 name
= create_tmp_var_name (".omp_copy_s");
2654 name
= build_decl (gimple_location (stmt
),
2655 TYPE_DECL
, name
, ctx
->record_type
);
2656 TYPE_NAME (ctx
->record_type
) = name
;
2658 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2659 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2661 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2662 ctx
->record_type
= NULL
;
2664 layout_type (ctx
->record_type
);
2667 /* Scan a GIMPLE_OMP_TARGET. */
2670 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
2674 bool offloaded
= is_gimple_omp_offloaded (stmt
);
2675 tree clauses
= gimple_omp_target_clauses (stmt
);
2677 ctx
= new_omp_context (stmt
, outer_ctx
);
2678 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2679 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2680 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2681 name
= create_tmp_var_name (".omp_data_t");
2682 name
= build_decl (gimple_location (stmt
),
2683 TYPE_DECL
, name
, ctx
->record_type
);
2684 DECL_ARTIFICIAL (name
) = 1;
2685 DECL_NAMELESS (name
) = 1;
2686 TYPE_NAME (ctx
->record_type
) = name
;
2689 if (is_gimple_omp_oacc (stmt
))
2690 ctx
->reduction_map
= splay_tree_new (splay_tree_compare_pointers
,
2693 create_omp_child_function (ctx
, false);
2694 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2697 if (is_gimple_omp_oacc (stmt
))
2699 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2701 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_NUM_GANGS
)
2702 ctx
->gwv_this
|= MASK_GANG
;
2703 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_NUM_WORKERS
)
2704 ctx
->gwv_this
|= MASK_WORKER
;
2705 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_VECTOR_LENGTH
)
2706 ctx
->gwv_this
|= MASK_VECTOR
;
2710 scan_sharing_clauses (clauses
, ctx
);
2711 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2713 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2714 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2717 TYPE_FIELDS (ctx
->record_type
)
2718 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2719 #ifdef ENABLE_CHECKING
2721 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2722 for (field
= TYPE_FIELDS (ctx
->record_type
);
2724 field
= DECL_CHAIN (field
))
2725 gcc_assert (DECL_ALIGN (field
) == align
);
2727 layout_type (ctx
->record_type
);
2729 fixup_child_record_type (ctx
);
2733 /* Scan an OpenMP teams directive. */
2736 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
2738 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2739 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2740 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2743 /* Check nesting restrictions. */
2745 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2747 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
2748 inside an OpenACC CTX. */
2749 if (!(is_gimple_omp (stmt
)
2750 && is_gimple_omp_oacc (stmt
)))
2752 for (omp_context
*ctx_
= ctx
; ctx_
!= NULL
; ctx_
= ctx_
->outer
)
2753 if (is_gimple_omp (ctx_
->stmt
)
2754 && is_gimple_omp_oacc (ctx_
->stmt
))
2756 error_at (gimple_location (stmt
),
2757 "non-OpenACC construct inside of OpenACC region");
2764 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2765 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2767 error_at (gimple_location (stmt
),
2768 "OpenMP constructs may not be nested inside simd region");
2771 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2773 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2774 || (gimple_omp_for_kind (stmt
)
2775 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2776 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2778 error_at (gimple_location (stmt
),
2779 "only distribute or parallel constructs are allowed to "
2780 "be closely nested inside teams construct");
2785 switch (gimple_code (stmt
))
2787 case GIMPLE_OMP_FOR
:
2788 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
2790 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2792 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2794 error_at (gimple_location (stmt
),
2795 "distribute construct must be closely nested inside "
2803 if (is_gimple_call (stmt
)
2804 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2805 == BUILT_IN_GOMP_CANCEL
2806 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2807 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2809 const char *bad
= NULL
;
2810 const char *kind
= NULL
;
2813 error_at (gimple_location (stmt
), "orphaned %qs construct",
2814 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2815 == BUILT_IN_GOMP_CANCEL
2816 ? "#pragma omp cancel"
2817 : "#pragma omp cancellation point");
2820 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2821 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2825 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2826 bad
= "#pragma omp parallel";
2827 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2828 == BUILT_IN_GOMP_CANCEL
2829 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2830 ctx
->cancellable
= true;
2834 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2835 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2836 bad
= "#pragma omp for";
2837 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2838 == BUILT_IN_GOMP_CANCEL
2839 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2841 ctx
->cancellable
= true;
2842 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2844 warning_at (gimple_location (stmt
), 0,
2845 "%<#pragma omp cancel for%> inside "
2846 "%<nowait%> for construct");
2847 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2848 OMP_CLAUSE_ORDERED
))
2849 warning_at (gimple_location (stmt
), 0,
2850 "%<#pragma omp cancel for%> inside "
2851 "%<ordered%> for construct");
2856 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2857 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2858 bad
= "#pragma omp sections";
2859 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2860 == BUILT_IN_GOMP_CANCEL
2861 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2863 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2865 ctx
->cancellable
= true;
2866 if (find_omp_clause (gimple_omp_sections_clauses
2869 warning_at (gimple_location (stmt
), 0,
2870 "%<#pragma omp cancel sections%> inside "
2871 "%<nowait%> sections construct");
2875 gcc_assert (ctx
->outer
2876 && gimple_code (ctx
->outer
->stmt
)
2877 == GIMPLE_OMP_SECTIONS
);
2878 ctx
->outer
->cancellable
= true;
2879 if (find_omp_clause (gimple_omp_sections_clauses
2882 warning_at (gimple_location (stmt
), 0,
2883 "%<#pragma omp cancel sections%> inside "
2884 "%<nowait%> sections construct");
2890 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2891 bad
= "#pragma omp task";
2893 ctx
->cancellable
= true;
2897 error_at (gimple_location (stmt
), "invalid arguments");
2902 error_at (gimple_location (stmt
),
2903 "%<%s %s%> construct not closely nested inside of %qs",
2904 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2905 == BUILT_IN_GOMP_CANCEL
2906 ? "#pragma omp cancel"
2907 : "#pragma omp cancellation point", kind
, bad
);
2912 case GIMPLE_OMP_SECTIONS
:
2913 case GIMPLE_OMP_SINGLE
:
2914 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2915 switch (gimple_code (ctx
->stmt
))
2917 case GIMPLE_OMP_FOR
:
2918 case GIMPLE_OMP_SECTIONS
:
2919 case GIMPLE_OMP_SINGLE
:
2920 case GIMPLE_OMP_ORDERED
:
2921 case GIMPLE_OMP_MASTER
:
2922 case GIMPLE_OMP_TASK
:
2923 case GIMPLE_OMP_CRITICAL
:
2924 if (is_gimple_call (stmt
))
2926 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2927 != BUILT_IN_GOMP_BARRIER
)
2929 error_at (gimple_location (stmt
),
2930 "barrier region may not be closely nested inside "
2931 "of work-sharing, critical, ordered, master or "
2932 "explicit task region");
2935 error_at (gimple_location (stmt
),
2936 "work-sharing region may not be closely nested inside "
2937 "of work-sharing, critical, ordered, master or explicit "
2940 case GIMPLE_OMP_PARALLEL
:
2946 case GIMPLE_OMP_MASTER
:
2947 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2948 switch (gimple_code (ctx
->stmt
))
2950 case GIMPLE_OMP_FOR
:
2951 case GIMPLE_OMP_SECTIONS
:
2952 case GIMPLE_OMP_SINGLE
:
2953 case GIMPLE_OMP_TASK
:
2954 error_at (gimple_location (stmt
),
2955 "master region may not be closely nested inside "
2956 "of work-sharing or explicit task region");
2958 case GIMPLE_OMP_PARALLEL
:
2964 case GIMPLE_OMP_ORDERED
:
2965 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2966 switch (gimple_code (ctx
->stmt
))
2968 case GIMPLE_OMP_CRITICAL
:
2969 case GIMPLE_OMP_TASK
:
2970 error_at (gimple_location (stmt
),
2971 "ordered region may not be closely nested inside "
2972 "of critical or explicit task region");
2974 case GIMPLE_OMP_FOR
:
2975 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2976 OMP_CLAUSE_ORDERED
) == NULL
)
2978 error_at (gimple_location (stmt
),
2979 "ordered region must be closely nested inside "
2980 "a loop region with an ordered clause");
2984 case GIMPLE_OMP_PARALLEL
:
2985 error_at (gimple_location (stmt
),
2986 "ordered region must be closely nested inside "
2987 "a loop region with an ordered clause");
2993 case GIMPLE_OMP_CRITICAL
:
2996 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
2997 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2998 if (gomp_critical
*other_crit
2999 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
3000 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
3002 error_at (gimple_location (stmt
),
3003 "critical region may not be nested inside a critical "
3004 "region with the same name");
3009 case GIMPLE_OMP_TEAMS
:
3011 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
3012 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
3014 error_at (gimple_location (stmt
),
3015 "teams construct not closely nested inside of target "
3020 case GIMPLE_OMP_TARGET
:
3021 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
3023 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
3025 if (is_gimple_omp (stmt
)
3026 && is_gimple_omp_oacc (stmt
)
3027 && is_gimple_omp (ctx
->stmt
))
3029 error_at (gimple_location (stmt
),
3030 "OpenACC construct inside of non-OpenACC region");
3036 const char *stmt_name
, *ctx_stmt_name
;
3037 switch (gimple_omp_target_kind (stmt
))
3039 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3040 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3041 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3042 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3043 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3044 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3045 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3046 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
: stmt_name
= "enter/exit data"; break;
3047 default: gcc_unreachable ();
3049 switch (gimple_omp_target_kind (ctx
->stmt
))
3051 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3052 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3053 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: ctx_stmt_name
= "parallel"; break;
3054 case GF_OMP_TARGET_KIND_OACC_KERNELS
: ctx_stmt_name
= "kernels"; break;
3055 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3056 default: gcc_unreachable ();
3059 /* OpenACC/OpenMP mismatch? */
3060 if (is_gimple_omp_oacc (stmt
)
3061 != is_gimple_omp_oacc (ctx
->stmt
))
3063 error_at (gimple_location (stmt
),
3064 "%s %s construct inside of %s %s region",
3065 (is_gimple_omp_oacc (stmt
)
3066 ? "OpenACC" : "OpenMP"), stmt_name
,
3067 (is_gimple_omp_oacc (ctx
->stmt
)
3068 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3071 if (is_gimple_omp_offloaded (ctx
->stmt
))
3073 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3074 if (is_gimple_omp_oacc (ctx
->stmt
))
3076 error_at (gimple_location (stmt
),
3077 "%s construct inside of %s region",
3078 stmt_name
, ctx_stmt_name
);
3083 gcc_checking_assert (!is_gimple_omp_oacc (stmt
));
3084 warning_at (gimple_location (stmt
), 0,
3085 "%s construct inside of %s region",
3086 stmt_name
, ctx_stmt_name
);
3098 /* Helper function scan_omp.
3100 Callback for walk_tree or operators in walk_gimple_stmt used to
3101 scan for OMP directives in TP. */
3104 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3106 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3107 omp_context
*ctx
= (omp_context
*) wi
->info
;
3110 switch (TREE_CODE (t
))
3117 *tp
= remap_decl (t
, &ctx
->cb
);
3121 if (ctx
&& TYPE_P (t
))
3122 *tp
= remap_type (t
, &ctx
->cb
);
3123 else if (!DECL_P (t
))
3128 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3129 if (tem
!= TREE_TYPE (t
))
3131 if (TREE_CODE (t
) == INTEGER_CST
)
3132 *tp
= wide_int_to_tree (tem
, t
);
3134 TREE_TYPE (t
) = tem
;
3144 /* Return true if FNDECL is a setjmp or a longjmp. */
3147 setjmp_or_longjmp_p (const_tree fndecl
)
3149 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3150 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3151 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3154 tree declname
= DECL_NAME (fndecl
);
3157 const char *name
= IDENTIFIER_POINTER (declname
);
3158 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3162 /* Helper function for scan_omp.
3164 Callback for walk_gimple_stmt used to scan for OMP directives in
3165 the current statement in GSI. */
3168 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3169 struct walk_stmt_info
*wi
)
3171 gimple stmt
= gsi_stmt (*gsi
);
3172 omp_context
*ctx
= (omp_context
*) wi
->info
;
3174 if (gimple_has_location (stmt
))
3175 input_location
= gimple_location (stmt
);
3177 /* Check the nesting restrictions. */
3178 bool remove
= false;
3179 if (is_gimple_omp (stmt
))
3180 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3181 else if (is_gimple_call (stmt
))
3183 tree fndecl
= gimple_call_fndecl (stmt
);
3186 if (setjmp_or_longjmp_p (fndecl
)
3188 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3189 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3192 error_at (gimple_location (stmt
),
3193 "setjmp/longjmp inside simd construct");
3195 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3196 switch (DECL_FUNCTION_CODE (fndecl
))
3198 case BUILT_IN_GOMP_BARRIER
:
3199 case BUILT_IN_GOMP_CANCEL
:
3200 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3201 case BUILT_IN_GOMP_TASKYIELD
:
3202 case BUILT_IN_GOMP_TASKWAIT
:
3203 case BUILT_IN_GOMP_TASKGROUP_START
:
3204 case BUILT_IN_GOMP_TASKGROUP_END
:
3205 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3214 stmt
= gimple_build_nop ();
3215 gsi_replace (gsi
, stmt
, false);
3218 *handled_ops_p
= true;
3220 switch (gimple_code (stmt
))
3222 case GIMPLE_OMP_PARALLEL
:
3223 taskreg_nesting_level
++;
3224 scan_omp_parallel (gsi
, ctx
);
3225 taskreg_nesting_level
--;
3228 case GIMPLE_OMP_TASK
:
3229 taskreg_nesting_level
++;
3230 scan_omp_task (gsi
, ctx
);
3231 taskreg_nesting_level
--;
3234 case GIMPLE_OMP_FOR
:
3235 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3238 case GIMPLE_OMP_SECTIONS
:
3239 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3242 case GIMPLE_OMP_SINGLE
:
3243 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3246 case GIMPLE_OMP_SECTION
:
3247 case GIMPLE_OMP_MASTER
:
3248 case GIMPLE_OMP_TASKGROUP
:
3249 case GIMPLE_OMP_ORDERED
:
3250 case GIMPLE_OMP_CRITICAL
:
3251 ctx
= new_omp_context (stmt
, ctx
);
3252 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3255 case GIMPLE_OMP_TARGET
:
3256 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3259 case GIMPLE_OMP_TEAMS
:
3260 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3267 *handled_ops_p
= false;
3269 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3271 var
= DECL_CHAIN (var
))
3272 insert_decl_map (&ctx
->cb
, var
, var
);
3276 *handled_ops_p
= false;
3284 /* Scan all the statements starting at the current statement. CTX
3285 contains context information about the OMP directives and
3286 clauses found during the scan. */
3289 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
3291 location_t saved_location
;
3292 struct walk_stmt_info wi
;
3294 memset (&wi
, 0, sizeof (wi
));
3296 wi
.want_locations
= true;
3298 saved_location
= input_location
;
3299 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
3300 input_location
= saved_location
;
3303 /* Re-gimplification and code generation routines. */
3305 /* Build a call to GOMP_barrier. */
3308 build_omp_barrier (tree lhs
)
3310 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
3311 : BUILT_IN_GOMP_BARRIER
);
3312 gcall
*g
= gimple_build_call (fndecl
, 0);
3314 gimple_call_set_lhs (g
, lhs
);
3318 /* If a context was created for STMT when it was scanned, return it. */
3320 static omp_context
*
3321 maybe_lookup_ctx (gimple stmt
)
3324 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
3325 return n
? (omp_context
*) n
->value
: NULL
;
3329 /* Find the mapping for DECL in CTX or the immediately enclosing
3330 context that has a mapping for DECL.
3332 If CTX is a nested parallel directive, we may have to use the decl
3333 mappings created in CTX's parent context. Suppose that we have the
3334 following parallel nesting (variable UIDs showed for clarity):
3337 #omp parallel shared(iD.1562) -> outer parallel
3338 iD.1562 = iD.1562 + 1;
3340 #omp parallel shared (iD.1562) -> inner parallel
3341 iD.1562 = iD.1562 - 1;
3343 Each parallel structure will create a distinct .omp_data_s structure
3344 for copying iD.1562 in/out of the directive:
3346 outer parallel .omp_data_s.1.i -> iD.1562
3347 inner parallel .omp_data_s.2.i -> iD.1562
3349 A shared variable mapping will produce a copy-out operation before
3350 the parallel directive and a copy-in operation after it. So, in
3351 this case we would have:
3354 .omp_data_o.1.i = iD.1562;
3355 #omp parallel shared(iD.1562) -> outer parallel
3356 .omp_data_i.1 = &.omp_data_o.1
3357 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3359 .omp_data_o.2.i = iD.1562; -> **
3360 #omp parallel shared(iD.1562) -> inner parallel
3361 .omp_data_i.2 = &.omp_data_o.2
3362 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3365 ** This is a problem. The symbol iD.1562 cannot be referenced
3366 inside the body of the outer parallel region. But since we are
3367 emitting this copy operation while expanding the inner parallel
3368 directive, we need to access the CTX structure of the outer
3369 parallel directive to get the correct mapping:
3371 .omp_data_o.2.i = .omp_data_i.1->i
3373 Since there may be other workshare or parallel directives enclosing
3374 the parallel directive, it may be necessary to walk up the context
3375 parent chain. This is not a problem in general because nested
3376 parallelism happens only rarely. */
3379 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3384 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3385 t
= maybe_lookup_decl (decl
, up
);
3387 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
3389 return t
? t
: decl
;
3393 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3394 in outer contexts. */
3397 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3402 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3403 t
= maybe_lookup_decl (decl
, up
);
3405 return t
? t
: decl
;
3409 /* Construct the initialization value for reduction CLAUSE. */
3412 omp_reduction_init (tree clause
, tree type
)
3414 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
3415 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
3422 case TRUTH_ORIF_EXPR
:
3423 case TRUTH_XOR_EXPR
:
3425 return build_zero_cst (type
);
3428 case TRUTH_AND_EXPR
:
3429 case TRUTH_ANDIF_EXPR
:
3431 return fold_convert_loc (loc
, type
, integer_one_node
);
3434 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
3437 if (SCALAR_FLOAT_TYPE_P (type
))
3439 REAL_VALUE_TYPE max
, min
;
3440 if (HONOR_INFINITIES (type
))
3443 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
3446 real_maxval (&min
, 1, TYPE_MODE (type
));
3447 return build_real (type
, min
);
3451 gcc_assert (INTEGRAL_TYPE_P (type
));
3452 return TYPE_MIN_VALUE (type
);
3456 if (SCALAR_FLOAT_TYPE_P (type
))
3458 REAL_VALUE_TYPE max
;
3459 if (HONOR_INFINITIES (type
))
3462 real_maxval (&max
, 0, TYPE_MODE (type
));
3463 return build_real (type
, max
);
3467 gcc_assert (INTEGRAL_TYPE_P (type
));
3468 return TYPE_MAX_VALUE (type
);
3476 /* Return alignment to be assumed for var in CLAUSE, which should be
3477 OMP_CLAUSE_ALIGNED. */
3480 omp_clause_aligned_alignment (tree clause
)
3482 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
3483 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
3485 /* Otherwise return implementation defined alignment. */
3486 unsigned int al
= 1;
3487 machine_mode mode
, vmode
;
3488 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3490 vs
= 1 << floor_log2 (vs
);
3491 static enum mode_class classes
[]
3492 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
3493 for (int i
= 0; i
< 4; i
+= 2)
3494 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
3496 mode
= GET_MODE_WIDER_MODE (mode
))
3498 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
3499 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
3502 && GET_MODE_SIZE (vmode
) < vs
3503 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
3504 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
3506 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
3507 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
3509 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
3510 / GET_MODE_SIZE (mode
));
3511 if (TYPE_MODE (type
) != vmode
)
3513 if (TYPE_ALIGN_UNIT (type
) > al
)
3514 al
= TYPE_ALIGN_UNIT (type
);
3516 return build_int_cst (integer_type_node
, al
);
3519 /* Return maximum possible vectorization factor for the target. */
3526 || !flag_tree_loop_optimize
3527 || (!flag_tree_loop_vectorize
3528 && (global_options_set
.x_flag_tree_loop_vectorize
3529 || global_options_set
.x_flag_tree_vectorize
)))
3532 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3535 vs
= 1 << floor_log2 (vs
);
3538 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
3539 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
3540 return GET_MODE_NUNITS (vqimode
);
3544 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3548 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
3549 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
3553 max_vf
= omp_max_vf ();
3556 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3557 OMP_CLAUSE_SAFELEN
);
3558 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3560 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3562 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3566 idx
= create_tmp_var (unsigned_type_node
);
3567 lane
= create_tmp_var (unsigned_type_node
);
3573 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3574 tree avar
= create_tmp_var_raw (atype
);
3575 if (TREE_ADDRESSABLE (new_var
))
3576 TREE_ADDRESSABLE (avar
) = 1;
3577 DECL_ATTRIBUTES (avar
)
3578 = tree_cons (get_identifier ("omp simd array"), NULL
,
3579 DECL_ATTRIBUTES (avar
));
3580 gimple_add_tmp_var (avar
);
3581 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3582 NULL_TREE
, NULL_TREE
);
3583 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3584 NULL_TREE
, NULL_TREE
);
3585 if (DECL_P (new_var
))
3587 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3588 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3593 /* Helper function of lower_rec_input_clauses. For a reference
3594 in simd reduction, add an underlying variable it will reference. */
3597 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3599 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3600 if (TREE_CONSTANT (z
))
3602 const char *name
= NULL
;
3603 if (DECL_NAME (new_vard
))
3604 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3606 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3607 gimple_add_tmp_var (z
);
3608 TREE_ADDRESSABLE (z
) = 1;
3609 z
= build_fold_addr_expr_loc (loc
, z
);
3610 gimplify_assign (new_vard
, z
, ilist
);
3614 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3615 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3616 private variables. Initialization statements go in ILIST, while calls
3617 to destructors go in DLIST. */
3620 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3621 omp_context
*ctx
, struct omp_for_data
*fd
)
3623 tree c
, dtor
, copyin_seq
, x
, ptr
;
3624 bool copyin_by_ref
= false;
3625 bool lastprivate_firstprivate
= false;
3626 bool reduction_omp_orig_ref
= false;
3628 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3629 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
3631 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3632 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3633 gimple_seq llist
[2] = { NULL
, NULL
};
3637 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3638 with data sharing clauses referencing variable sized vars. That
3639 is unnecessarily hard to support and very unlikely to result in
3640 vectorized code anyway. */
3642 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3643 switch (OMP_CLAUSE_CODE (c
))
3645 case OMP_CLAUSE_LINEAR
:
3646 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3649 case OMP_CLAUSE_REDUCTION
:
3650 case OMP_CLAUSE_PRIVATE
:
3651 case OMP_CLAUSE_FIRSTPRIVATE
:
3652 case OMP_CLAUSE_LASTPRIVATE
:
3653 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3660 /* Do all the fixed sized types in the first pass, and the variable sized
3661 types in the second pass. This makes sure that the scalar arguments to
3662 the variable sized types are processed before we use them in the
3663 variable sized operations. */
3664 for (pass
= 0; pass
< 2; ++pass
)
3666 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3668 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3671 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3675 case OMP_CLAUSE_PRIVATE
:
3676 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3679 case OMP_CLAUSE_SHARED
:
3680 /* Ignore shared directives in teams construct. */
3681 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3683 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3685 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3688 case OMP_CLAUSE_FIRSTPRIVATE
:
3689 case OMP_CLAUSE_COPYIN
:
3690 case OMP_CLAUSE_LINEAR
:
3692 case OMP_CLAUSE_REDUCTION
:
3693 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3694 reduction_omp_orig_ref
= true;
3696 case OMP_CLAUSE__LOOPTEMP_
:
3697 /* Handle _looptemp_ clauses only on parallel. */
3701 case OMP_CLAUSE_LASTPRIVATE
:
3702 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3704 lastprivate_firstprivate
= true;
3708 /* Even without corresponding firstprivate, if
3709 decl is Fortran allocatable, it needs outer var
3712 && lang_hooks
.decls
.omp_private_outer_ref
3713 (OMP_CLAUSE_DECL (c
)))
3714 lastprivate_firstprivate
= true;
3716 case OMP_CLAUSE_ALIGNED
:
3719 var
= OMP_CLAUSE_DECL (c
);
3720 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3721 && !is_global_var (var
))
3723 new_var
= maybe_lookup_decl (var
, ctx
);
3724 if (new_var
== NULL_TREE
)
3725 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3726 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3727 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3728 omp_clause_aligned_alignment (c
));
3729 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3730 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3731 gimplify_and_add (x
, ilist
);
3733 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3734 && is_global_var (var
))
3736 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3737 new_var
= lookup_decl (var
, ctx
);
3738 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3739 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3740 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3741 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3742 omp_clause_aligned_alignment (c
));
3743 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3744 x
= create_tmp_var (ptype
);
3745 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3746 gimplify_and_add (t
, ilist
);
3747 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3748 SET_DECL_VALUE_EXPR (new_var
, t
);
3749 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3756 new_var
= var
= OMP_CLAUSE_DECL (c
);
3757 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3758 new_var
= lookup_decl (var
, ctx
);
3760 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3765 else if (is_variable_sized (var
))
3767 /* For variable sized types, we need to allocate the
3768 actual storage here. Call alloca and store the
3769 result in the pointer decl that we created elsewhere. */
3773 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3778 ptr
= DECL_VALUE_EXPR (new_var
);
3779 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3780 ptr
= TREE_OPERAND (ptr
, 0);
3781 gcc_assert (DECL_P (ptr
));
3782 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3784 /* void *tmp = __builtin_alloca */
3785 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3786 stmt
= gimple_build_call (atmp
, 1, x
);
3787 tmp
= create_tmp_var_raw (ptr_type_node
);
3788 gimple_add_tmp_var (tmp
);
3789 gimple_call_set_lhs (stmt
, tmp
);
3791 gimple_seq_add_stmt (ilist
, stmt
);
3793 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3794 gimplify_assign (ptr
, x
, ilist
);
3797 else if (is_reference (var
))
3799 /* For references that are being privatized for Fortran,
3800 allocate new backing storage for the new pointer
3801 variable. This allows us to avoid changing all the
3802 code that expects a pointer to something that expects
3803 a direct variable. */
3807 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3808 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3810 x
= build_receiver_ref (var
, false, ctx
);
3811 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3813 else if (TREE_CONSTANT (x
))
3815 /* For reduction in SIMD loop, defer adding the
3816 initialization of the reference, because if we decide
3817 to use SIMD array for it, the initilization could cause
3819 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3823 const char *name
= NULL
;
3824 if (DECL_NAME (var
))
3825 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3827 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3829 gimple_add_tmp_var (x
);
3830 TREE_ADDRESSABLE (x
) = 1;
3831 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3836 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3837 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3842 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3843 gimplify_assign (new_var
, x
, ilist
);
3846 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3848 else if (c_kind
== OMP_CLAUSE_REDUCTION
3849 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3857 switch (OMP_CLAUSE_CODE (c
))
3859 case OMP_CLAUSE_SHARED
:
3860 /* Ignore shared directives in teams construct. */
3861 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3863 /* Shared global vars are just accessed directly. */
3864 if (is_global_var (new_var
))
3866 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3867 needs to be delayed until after fixup_child_record_type so
3868 that we get the correct type during the dereference. */
3869 by_ref
= use_pointer_for_field (var
, ctx
);
3870 x
= build_receiver_ref (var
, by_ref
, ctx
);
3871 SET_DECL_VALUE_EXPR (new_var
, x
);
3872 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3874 /* ??? If VAR is not passed by reference, and the variable
3875 hasn't been initialized yet, then we'll get a warning for
3876 the store into the omp_data_s structure. Ideally, we'd be
3877 able to notice this and not store anything at all, but
3878 we're generating code too early. Suppress the warning. */
3880 TREE_NO_WARNING (var
) = 1;
3883 case OMP_CLAUSE_LASTPRIVATE
:
3884 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3888 case OMP_CLAUSE_PRIVATE
:
3889 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3890 x
= build_outer_var_ref (var
, ctx
);
3891 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3893 if (is_task_ctx (ctx
))
3894 x
= build_receiver_ref (var
, false, ctx
);
3896 x
= build_outer_var_ref (var
, ctx
);
3902 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3905 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3906 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3907 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3908 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3909 idx
, lane
, ivar
, lvar
))
3912 x
= lang_hooks
.decls
.omp_clause_default_ctor
3913 (c
, unshare_expr (ivar
), x
);
3915 gimplify_and_add (x
, &llist
[0]);
3918 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3921 gimple_seq tseq
= NULL
;
3924 gimplify_stmt (&dtor
, &tseq
);
3925 gimple_seq_add_seq (&llist
[1], tseq
);
3932 gimplify_and_add (nx
, ilist
);
3936 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3939 gimple_seq tseq
= NULL
;
3942 gimplify_stmt (&dtor
, &tseq
);
3943 gimple_seq_add_seq (dlist
, tseq
);
3947 case OMP_CLAUSE_LINEAR
:
3948 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3949 goto do_firstprivate
;
3950 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3953 x
= build_outer_var_ref (var
, ctx
);
3956 case OMP_CLAUSE_FIRSTPRIVATE
:
3957 if (is_task_ctx (ctx
))
3959 if (is_reference (var
) || is_variable_sized (var
))
3961 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3963 || use_pointer_for_field (var
, NULL
))
3965 x
= build_receiver_ref (var
, false, ctx
);
3966 SET_DECL_VALUE_EXPR (new_var
, x
);
3967 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3972 x
= build_outer_var_ref (var
, ctx
);
3975 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3976 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3978 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3979 tree stept
= TREE_TYPE (t
);
3980 tree ct
= find_omp_clause (clauses
,
3981 OMP_CLAUSE__LOOPTEMP_
);
3983 tree l
= OMP_CLAUSE_DECL (ct
);
3984 tree n1
= fd
->loop
.n1
;
3985 tree step
= fd
->loop
.step
;
3986 tree itype
= TREE_TYPE (l
);
3987 if (POINTER_TYPE_P (itype
))
3988 itype
= signed_type_for (itype
);
3989 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3990 if (TYPE_UNSIGNED (itype
)
3991 && fd
->loop
.cond_code
== GT_EXPR
)
3992 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3993 fold_build1 (NEGATE_EXPR
, itype
, l
),
3994 fold_build1 (NEGATE_EXPR
,
3997 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3998 t
= fold_build2 (MULT_EXPR
, stept
,
3999 fold_convert (stept
, l
), t
);
4001 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
4003 x
= lang_hooks
.decls
.omp_clause_linear_ctor
4005 gimplify_and_add (x
, ilist
);
4009 if (POINTER_TYPE_P (TREE_TYPE (x
)))
4010 x
= fold_build2 (POINTER_PLUS_EXPR
,
4011 TREE_TYPE (x
), x
, t
);
4013 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
4016 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
4017 || TREE_ADDRESSABLE (new_var
))
4018 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4019 idx
, lane
, ivar
, lvar
))
4021 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
4023 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
4024 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
4025 gimplify_and_add (x
, ilist
);
4026 gimple_stmt_iterator gsi
4027 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4029 = gimple_build_assign (unshare_expr (lvar
), iv
);
4030 gsi_insert_before_without_update (&gsi
, g
,
4032 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
4033 enum tree_code code
= PLUS_EXPR
;
4034 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
4035 code
= POINTER_PLUS_EXPR
;
4036 g
= gimple_build_assign (iv
, code
, iv
, t
);
4037 gsi_insert_before_without_update (&gsi
, g
,
4041 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4042 (c
, unshare_expr (ivar
), x
);
4043 gimplify_and_add (x
, &llist
[0]);
4044 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4047 gimple_seq tseq
= NULL
;
4050 gimplify_stmt (&dtor
, &tseq
);
4051 gimple_seq_add_seq (&llist
[1], tseq
);
4056 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
4057 gimplify_and_add (x
, ilist
);
4060 case OMP_CLAUSE__LOOPTEMP_
:
4061 gcc_assert (is_parallel_ctx (ctx
));
4062 x
= build_outer_var_ref (var
, ctx
);
4063 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4064 gimplify_and_add (x
, ilist
);
4067 case OMP_CLAUSE_COPYIN
:
4068 by_ref
= use_pointer_for_field (var
, NULL
);
4069 x
= build_receiver_ref (var
, by_ref
, ctx
);
4070 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
4071 append_to_statement_list (x
, ©in_seq
);
4072 copyin_by_ref
|= by_ref
;
4075 case OMP_CLAUSE_REDUCTION
:
4076 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4078 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4080 x
= build_outer_var_ref (var
, ctx
);
4082 if (is_reference (var
)
4083 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4085 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4086 SET_DECL_VALUE_EXPR (placeholder
, x
);
4087 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4088 tree new_vard
= new_var
;
4089 if (is_reference (var
))
4091 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4092 new_vard
= TREE_OPERAND (new_var
, 0);
4093 gcc_assert (DECL_P (new_vard
));
4096 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4097 idx
, lane
, ivar
, lvar
))
4099 if (new_vard
== new_var
)
4101 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
4102 SET_DECL_VALUE_EXPR (new_var
, ivar
);
4106 SET_DECL_VALUE_EXPR (new_vard
,
4107 build_fold_addr_expr (ivar
));
4108 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4110 x
= lang_hooks
.decls
.omp_clause_default_ctor
4111 (c
, unshare_expr (ivar
),
4112 build_outer_var_ref (var
, ctx
));
4114 gimplify_and_add (x
, &llist
[0]);
4115 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4117 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4118 lower_omp (&tseq
, ctx
);
4119 gimple_seq_add_seq (&llist
[0], tseq
);
4121 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4122 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4123 lower_omp (&tseq
, ctx
);
4124 gimple_seq_add_seq (&llist
[1], tseq
);
4125 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4126 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4127 if (new_vard
== new_var
)
4128 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4130 SET_DECL_VALUE_EXPR (new_vard
,
4131 build_fold_addr_expr (lvar
));
4132 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4137 gimplify_stmt (&dtor
, &tseq
);
4138 gimple_seq_add_seq (&llist
[1], tseq
);
4142 /* If this is a reference to constant size reduction var
4143 with placeholder, we haven't emitted the initializer
4144 for it because it is undesirable if SIMD arrays are used.
4145 But if they aren't used, we need to emit the deferred
4146 initialization now. */
4147 else if (is_reference (var
) && is_simd
)
4148 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4149 x
= lang_hooks
.decls
.omp_clause_default_ctor
4150 (c
, unshare_expr (new_var
),
4151 build_outer_var_ref (var
, ctx
));
4153 gimplify_and_add (x
, ilist
);
4154 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4156 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4157 lower_omp (&tseq
, ctx
);
4158 gimple_seq_add_seq (ilist
, tseq
);
4160 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4163 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4164 lower_omp (&tseq
, ctx
);
4165 gimple_seq_add_seq (dlist
, tseq
);
4166 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4168 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4173 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
4174 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
4175 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4177 /* reduction(-:var) sums up the partial results, so it
4178 acts identically to reduction(+:var). */
4179 if (code
== MINUS_EXPR
)
4182 tree new_vard
= new_var
;
4183 if (is_simd
&& is_reference (var
))
4185 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4186 new_vard
= TREE_OPERAND (new_var
, 0);
4187 gcc_assert (DECL_P (new_vard
));
4190 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4191 idx
, lane
, ivar
, lvar
))
4193 tree ref
= build_outer_var_ref (var
, ctx
);
4195 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
4197 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
4198 ref
= build_outer_var_ref (var
, ctx
);
4199 gimplify_assign (ref
, x
, &llist
[1]);
4201 if (new_vard
!= new_var
)
4203 SET_DECL_VALUE_EXPR (new_vard
,
4204 build_fold_addr_expr (lvar
));
4205 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4210 if (is_reference (var
) && is_simd
)
4211 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4212 gimplify_assign (new_var
, x
, ilist
);
4215 tree ref
= build_outer_var_ref (var
, ctx
);
4217 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4218 ref
= build_outer_var_ref (var
, ctx
);
4219 gimplify_assign (ref
, x
, dlist
);
4233 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
4234 /* Don't want uninit warnings on simduid, it is always uninitialized,
4235 but we use it not for the value, but for the DECL_UID only. */
4236 TREE_NO_WARNING (uid
) = 1;
4238 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
4239 gimple_call_set_lhs (g
, lane
);
4240 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4241 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
4242 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
4243 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
4244 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4245 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4246 g
= gimple_build_assign (lane
, INTEGER_CST
,
4247 build_int_cst (unsigned_type_node
, 0));
4248 gimple_seq_add_stmt (ilist
, g
);
4249 for (int i
= 0; i
< 2; i
++)
4252 tree vf
= create_tmp_var (unsigned_type_node
);
4253 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
4254 gimple_call_set_lhs (g
, vf
);
4255 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
4256 gimple_seq_add_stmt (seq
, g
);
4257 tree t
= build_int_cst (unsigned_type_node
, 0);
4258 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
4259 gimple_seq_add_stmt (seq
, g
);
4260 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4261 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
4262 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4263 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
4264 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
4265 gimple_seq_add_seq (seq
, llist
[i
]);
4266 t
= build_int_cst (unsigned_type_node
, 1);
4267 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
4268 gimple_seq_add_stmt (seq
, g
);
4269 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
4270 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
4271 gimple_seq_add_stmt (seq
, g
);
4272 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
4276 /* The copyin sequence is not to be executed by the main thread, since
4277 that would result in self-copies. Perhaps not visible to scalars,
4278 but it certainly is to C++ operator=. */
4281 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
4283 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
4284 build_int_cst (TREE_TYPE (x
), 0));
4285 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
4286 gimplify_and_add (x
, ilist
);
4289 /* If any copyin variable is passed by reference, we must ensure the
4290 master thread doesn't modify it before it is copied over in all
4291 threads. Similarly for variables in both firstprivate and
4292 lastprivate clauses we need to ensure the lastprivate copying
4293 happens after firstprivate copying in all threads. And similarly
4294 for UDRs if initializer expression refers to omp_orig. */
4295 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
4297 /* Don't add any barrier for #pragma omp simd or
4298 #pragma omp distribute. */
4299 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
4300 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
4301 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
4304 /* If max_vf is non-zero, then we can use only a vectorization factor
4305 up to the max_vf we chose. So stick it into the safelen clause. */
4308 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4309 OMP_CLAUSE_SAFELEN
);
4311 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
4312 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4315 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
4316 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
4318 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4319 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4325 /* Generate code to implement the LASTPRIVATE clauses. This is used for
4326 both parallel and workshare constructs. PREDICATE may be NULL if it's
4330 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
4333 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
4334 bool par_clauses
= false;
4335 tree simduid
= NULL
, lastlane
= NULL
;
4337 /* Early exit if there are no lastprivate or linear clauses. */
4338 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
4339 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
4340 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
4341 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
4343 if (clauses
== NULL
)
4345 /* If this was a workshare clause, see if it had been combined
4346 with its parallel. In that case, look for the clauses on the
4347 parallel statement itself. */
4348 if (is_parallel_ctx (ctx
))
4352 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4355 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4356 OMP_CLAUSE_LASTPRIVATE
);
4357 if (clauses
== NULL
)
4365 tree label_true
, arm1
, arm2
;
4367 label
= create_artificial_label (UNKNOWN_LOCATION
);
4368 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
4369 arm1
= TREE_OPERAND (predicate
, 0);
4370 arm2
= TREE_OPERAND (predicate
, 1);
4371 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4372 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4373 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
4375 gimple_seq_add_stmt (stmt_list
, stmt
);
4376 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
4379 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4380 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4382 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
4384 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
4387 for (c
= clauses
; c
;)
4390 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4392 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4393 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4394 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
4396 var
= OMP_CLAUSE_DECL (c
);
4397 new_var
= lookup_decl (var
, ctx
);
4399 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
4401 tree val
= DECL_VALUE_EXPR (new_var
);
4402 if (TREE_CODE (val
) == ARRAY_REF
4403 && VAR_P (TREE_OPERAND (val
, 0))
4404 && lookup_attribute ("omp simd array",
4405 DECL_ATTRIBUTES (TREE_OPERAND (val
,
4408 if (lastlane
== NULL
)
4410 lastlane
= create_tmp_var (unsigned_type_node
);
4412 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
4414 TREE_OPERAND (val
, 1));
4415 gimple_call_set_lhs (g
, lastlane
);
4416 gimple_seq_add_stmt (stmt_list
, g
);
4418 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
4419 TREE_OPERAND (val
, 0), lastlane
,
4420 NULL_TREE
, NULL_TREE
);
4424 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4425 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
4427 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
4428 gimple_seq_add_seq (stmt_list
,
4429 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
4430 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
4432 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4433 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
4435 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
4436 gimple_seq_add_seq (stmt_list
,
4437 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
4438 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
4441 x
= build_outer_var_ref (var
, ctx
);
4442 if (is_reference (var
))
4443 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4444 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
4445 gimplify_and_add (x
, stmt_list
);
4447 c
= OMP_CLAUSE_CHAIN (c
);
4448 if (c
== NULL
&& !par_clauses
)
4450 /* If this was a workshare clause, see if it had been combined
4451 with its parallel. In that case, continue looking for the
4452 clauses also on the parallel statement itself. */
4453 if (is_parallel_ctx (ctx
))
4457 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4460 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4461 OMP_CLAUSE_LASTPRIVATE
);
4467 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
4471 oacc_lower_reduction_var_helper (gimple_seq
*stmt_seqp
, omp_context
*ctx
,
4472 tree tid
, tree var
, tree new_var
)
4474 /* The atomic add at the end of the sum creates unnecessary
4475 write contention on accelerators. To work around this,
4476 create an array to store the partial reductions. Later, in
4477 lower_omp_for (for openacc), the values of array will be
4480 tree t
= NULL_TREE
, array
, x
;
4481 tree type
= get_base_type (var
);
4484 /* Now insert the partial reductions into the array. */
4486 /* Find the reduction array. */
4488 tree ptype
= build_pointer_type (type
);
4490 t
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
), ctx
);
4491 t
= build_receiver_ref (t
, false, ctx
->outer
);
4493 array
= create_tmp_var (ptype
);
4494 gimplify_assign (array
, t
, stmt_seqp
);
4496 tree ptr
= create_tmp_var (TREE_TYPE (array
));
4498 /* Find the reduction array. */
4500 /* testing a unary conversion. */
4501 tree offset
= create_tmp_var (sizetype
);
4502 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
),
4504 t
= create_tmp_var (sizetype
);
4505 gimplify_assign (t
, unshare_expr (fold_build1 (NOP_EXPR
, sizetype
, tid
)),
4507 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, t
);
4508 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4510 /* Offset expression. Does the POINTER_PLUS_EXPR take care
4511 of adding sizeof(var) to the array? */
4512 ptr
= create_tmp_var (ptype
);
4513 stmt
= gimple_build_assign (unshare_expr (ptr
), POINTER_PLUS_EXPR
, array
,
4515 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4517 /* Move the local sum to gfc$sum[i]. */
4518 x
= unshare_expr (build_simple_mem_ref (ptr
));
4519 stmt
= gimplify_assign (x
, new_var
, stmt_seqp
);
4522 /* Generate code to implement the REDUCTION clauses. */
4525 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
4527 gimple_seq sub_seq
= NULL
;
4529 tree x
, c
, tid
= NULL_TREE
;
4532 /* SIMD reductions are handled in lower_rec_input_clauses. */
4533 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4534 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4537 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
4538 update in that case, otherwise use a lock. */
4539 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
4540 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
4542 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4544 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4554 /* Initialize thread info for OpenACC. */
4555 if (is_gimple_omp_oacc (ctx
->stmt
))
4557 /* Get the current thread id. */
4558 tree call
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
4559 tid
= create_tmp_var (TREE_TYPE (TREE_TYPE (call
)));
4560 gimple stmt
= gimple_build_call (call
, 0);
4561 gimple_call_set_lhs (stmt
, tid
);
4562 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4565 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4567 tree var
, ref
, new_var
;
4568 enum tree_code code
;
4569 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4571 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4574 var
= OMP_CLAUSE_DECL (c
);
4575 new_var
= lookup_decl (var
, ctx
);
4576 if (is_reference (var
))
4577 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4578 ref
= build_outer_var_ref (var
, ctx
);
4579 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4581 /* reduction(-:var) sums up the partial results, so it acts
4582 identically to reduction(+:var). */
4583 if (code
== MINUS_EXPR
)
4586 if (is_gimple_omp_oacc (ctx
->stmt
))
4588 gcc_checking_assert (!OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
));
4590 oacc_lower_reduction_var_helper (stmt_seqp
, ctx
, tid
, var
, new_var
);
4592 else if (count
== 1)
4594 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
4596 addr
= save_expr (addr
);
4597 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
4598 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
4599 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
4600 gimplify_and_add (x
, stmt_seqp
);
4603 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4605 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4607 if (is_reference (var
)
4608 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4610 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
4611 SET_DECL_VALUE_EXPR (placeholder
, ref
);
4612 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4613 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
4614 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
4615 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4616 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
4620 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4621 ref
= build_outer_var_ref (var
, ctx
);
4622 gimplify_assign (ref
, x
, &sub_seq
);
4626 if (is_gimple_omp_oacc (ctx
->stmt
))
4629 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4631 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4633 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4635 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4637 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4641 /* Generate code to implement the COPYPRIVATE clauses. */
4644 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4649 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4651 tree var
, new_var
, ref
, x
;
4653 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4655 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4658 var
= OMP_CLAUSE_DECL (c
);
4659 by_ref
= use_pointer_for_field (var
, NULL
);
4661 ref
= build_sender_ref (var
, ctx
);
4662 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4665 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4666 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4668 gimplify_assign (ref
, x
, slist
);
4670 ref
= build_receiver_ref (var
, false, ctx
);
4673 ref
= fold_convert_loc (clause_loc
,
4674 build_pointer_type (TREE_TYPE (new_var
)),
4676 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4678 if (is_reference (var
))
4680 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4681 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4682 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4684 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4685 gimplify_and_add (x
, rlist
);
4690 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4691 and REDUCTION from the sender (aka parent) side. */
4694 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4699 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4701 tree val
, ref
, x
, var
;
4702 bool by_ref
, do_in
= false, do_out
= false;
4703 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4705 switch (OMP_CLAUSE_CODE (c
))
4707 case OMP_CLAUSE_PRIVATE
:
4708 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4711 case OMP_CLAUSE_FIRSTPRIVATE
:
4712 case OMP_CLAUSE_COPYIN
:
4713 case OMP_CLAUSE_LASTPRIVATE
:
4714 case OMP_CLAUSE_REDUCTION
:
4715 case OMP_CLAUSE__LOOPTEMP_
:
4721 val
= OMP_CLAUSE_DECL (c
);
4722 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4724 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4725 && is_global_var (var
))
4727 if (is_variable_sized (val
))
4729 by_ref
= use_pointer_for_field (val
, NULL
);
4731 switch (OMP_CLAUSE_CODE (c
))
4733 case OMP_CLAUSE_PRIVATE
:
4734 case OMP_CLAUSE_FIRSTPRIVATE
:
4735 case OMP_CLAUSE_COPYIN
:
4736 case OMP_CLAUSE__LOOPTEMP_
:
4740 case OMP_CLAUSE_LASTPRIVATE
:
4741 if (by_ref
|| is_reference (val
))
4743 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4750 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4755 case OMP_CLAUSE_REDUCTION
:
4757 do_out
= !(by_ref
|| is_reference (val
));
4766 ref
= build_sender_ref (val
, ctx
);
4767 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4768 gimplify_assign (ref
, x
, ilist
);
4769 if (is_task_ctx (ctx
))
4770 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4775 ref
= build_sender_ref (val
, ctx
);
4776 gimplify_assign (var
, ref
, olist
);
4781 /* Generate code to implement SHARED from the sender (aka parent)
4782 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4783 list things that got automatically shared. */
4786 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4788 tree var
, ovar
, nvar
, f
, x
, record_type
;
4790 if (ctx
->record_type
== NULL
)
4793 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4794 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4796 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4797 nvar
= maybe_lookup_decl (ovar
, ctx
);
4798 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4801 /* If CTX is a nested parallel directive. Find the immediately
4802 enclosing parallel or workshare construct that contains a
4803 mapping for OVAR. */
4804 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4806 if (use_pointer_for_field (ovar
, ctx
))
4808 x
= build_sender_ref (ovar
, ctx
);
4809 var
= build_fold_addr_expr (var
);
4810 gimplify_assign (x
, var
, ilist
);
4814 x
= build_sender_ref (ovar
, ctx
);
4815 gimplify_assign (x
, var
, ilist
);
4817 if (!TREE_READONLY (var
)
4818 /* We don't need to receive a new reference to a result
4819 or parm decl. In fact we may not store to it as we will
4820 invalidate any pending RSO and generate wrong gimple
4822 && !((TREE_CODE (var
) == RESULT_DECL
4823 || TREE_CODE (var
) == PARM_DECL
)
4824 && DECL_BY_REFERENCE (var
)))
4826 x
= build_sender_ref (ovar
, ctx
);
4827 gimplify_assign (var
, x
, olist
);
4834 /* A convenience function to build an empty GIMPLE_COND with just the
4838 gimple_build_cond_empty (tree cond
)
4840 enum tree_code pred_code
;
4843 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4844 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4848 /* Build the function calls to GOMP_parallel_start etc to actually
4849 generate the parallel operation. REGION is the parallel region
4850 being expanded. BB is the block where to insert the code. WS_ARGS
4851 will be set if this is a call to a combined parallel+workshare
4852 construct, it contains the list of additional arguments needed by
4853 the workshare construct. */
4856 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4857 gomp_parallel
*entry_stmt
,
4858 vec
<tree
, va_gc
> *ws_args
)
4860 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4861 gimple_stmt_iterator gsi
;
4863 enum built_in_function start_ix
;
4865 location_t clause_loc
;
4866 vec
<tree
, va_gc
> *args
;
4868 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4870 /* Determine what flavor of GOMP_parallel we will be
4872 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4873 if (is_combined_parallel (region
))
4875 switch (region
->inner
->type
)
4877 case GIMPLE_OMP_FOR
:
4878 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4879 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4880 + (region
->inner
->sched_kind
4881 == OMP_CLAUSE_SCHEDULE_RUNTIME
4882 ? 3 : region
->inner
->sched_kind
));
4883 start_ix
= (enum built_in_function
)start_ix2
;
4885 case GIMPLE_OMP_SECTIONS
:
4886 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4893 /* By default, the value of NUM_THREADS is zero (selected at run time)
4894 and there is no conditional. */
4896 val
= build_int_cst (unsigned_type_node
, 0);
4897 flags
= build_int_cst (unsigned_type_node
, 0);
4899 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4901 cond
= OMP_CLAUSE_IF_EXPR (c
);
4903 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4906 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4907 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4910 clause_loc
= gimple_location (entry_stmt
);
4912 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4914 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4916 /* Ensure 'val' is of the correct type. */
4917 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4919 /* If we found the clause 'if (cond)', build either
4920 (cond != 0) or (cond ? val : 1u). */
4923 cond
= gimple_boolify (cond
);
4925 if (integer_zerop (val
))
4926 val
= fold_build2_loc (clause_loc
,
4927 EQ_EXPR
, unsigned_type_node
, cond
,
4928 build_int_cst (TREE_TYPE (cond
), 0));
4931 basic_block cond_bb
, then_bb
, else_bb
;
4932 edge e
, e_then
, e_else
;
4933 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4935 tmp_var
= create_tmp_var (TREE_TYPE (val
));
4936 if (gimple_in_ssa_p (cfun
))
4938 tmp_then
= make_ssa_name (tmp_var
);
4939 tmp_else
= make_ssa_name (tmp_var
);
4940 tmp_join
= make_ssa_name (tmp_var
);
4949 e
= split_block (bb
, NULL
);
4954 then_bb
= create_empty_bb (cond_bb
);
4955 else_bb
= create_empty_bb (then_bb
);
4956 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4957 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4959 stmt
= gimple_build_cond_empty (cond
);
4960 gsi
= gsi_start_bb (cond_bb
);
4961 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4963 gsi
= gsi_start_bb (then_bb
);
4964 stmt
= gimple_build_assign (tmp_then
, val
);
4965 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4967 gsi
= gsi_start_bb (else_bb
);
4968 stmt
= gimple_build_assign
4969 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4970 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4972 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4973 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4974 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4975 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4976 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4977 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4979 if (gimple_in_ssa_p (cfun
))
4981 gphi
*phi
= create_phi_node (tmp_join
, bb
);
4982 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4983 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4989 gsi
= gsi_start_bb (bb
);
4990 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4991 false, GSI_CONTINUE_LINKING
);
4994 gsi
= gsi_last_bb (bb
);
4995 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4997 t1
= null_pointer_node
;
4999 t1
= build_fold_addr_expr (t
);
5000 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
5002 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
5003 args
->quick_push (t2
);
5004 args
->quick_push (t1
);
5005 args
->quick_push (val
);
5007 args
->splice (*ws_args
);
5008 args
->quick_push (flags
);
5010 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
5011 builtin_decl_explicit (start_ix
), args
);
5013 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5014 false, GSI_CONTINUE_LINKING
);
5017 /* Insert a function call whose name is FUNC_NAME with the information from
5018 ENTRY_STMT into the basic_block BB. */
5021 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
5022 vec
<tree
, va_gc
> *ws_args
)
5025 gimple_stmt_iterator gsi
;
5026 vec
<tree
, va_gc
> *args
;
5028 gcc_assert (vec_safe_length (ws_args
) == 2);
5029 tree func_name
= (*ws_args
)[0];
5030 tree grain
= (*ws_args
)[1];
5032 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
5033 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
5034 gcc_assert (count
!= NULL_TREE
);
5035 count
= OMP_CLAUSE_OPERAND (count
, 0);
5037 gsi
= gsi_last_bb (bb
);
5038 t
= gimple_omp_parallel_data_arg (entry_stmt
);
5040 t1
= null_pointer_node
;
5042 t1
= build_fold_addr_expr (t
);
5043 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
5045 vec_alloc (args
, 4);
5046 args
->quick_push (t2
);
5047 args
->quick_push (t1
);
5048 args
->quick_push (count
);
5049 args
->quick_push (grain
);
5050 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
5052 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
5053 GSI_CONTINUE_LINKING
);
5056 /* Build the function call to GOMP_task to actually
5057 generate the task operation. BB is the block where to insert the code. */
5060 expand_task_call (basic_block bb
, gomp_task
*entry_stmt
)
5062 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
5063 gimple_stmt_iterator gsi
;
5064 location_t loc
= gimple_location (entry_stmt
);
5066 clauses
= gimple_omp_task_clauses (entry_stmt
);
5068 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
5070 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
5072 cond
= boolean_true_node
;
5074 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
5075 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
5076 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
5077 flags
= build_int_cst (unsigned_type_node
,
5078 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
5080 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
5083 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
5084 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
5085 build_int_cst (unsigned_type_node
, 2),
5086 build_int_cst (unsigned_type_node
, 0));
5087 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
5090 depend
= OMP_CLAUSE_DECL (depend
);
5092 depend
= build_int_cst (ptr_type_node
, 0);
5094 gsi
= gsi_last_bb (bb
);
5095 t
= gimple_omp_task_data_arg (entry_stmt
);
5097 t2
= null_pointer_node
;
5099 t2
= build_fold_addr_expr_loc (loc
, t
);
5100 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
5101 t
= gimple_omp_task_copy_fn (entry_stmt
);
5103 t3
= null_pointer_node
;
5105 t3
= build_fold_addr_expr_loc (loc
, t
);
5107 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
5109 gimple_omp_task_arg_size (entry_stmt
),
5110 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
5113 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5114 false, GSI_CONTINUE_LINKING
);
5118 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
5119 catch handler and return it. This prevents programs from violating the
5120 structured block semantics with throws. */
5123 maybe_catch_exception (gimple_seq body
)
5128 if (!flag_exceptions
)
5131 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
5132 decl
= lang_hooks
.eh_protect_cleanup_actions ();
5134 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
5136 g
= gimple_build_eh_must_not_throw (decl
);
5137 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
5140 return gimple_seq_alloc_with_stmt (g
);
5143 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
5146 vec2chain (vec
<tree
, va_gc
> *v
)
5148 tree chain
= NULL_TREE
, t
;
5151 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
5153 DECL_CHAIN (t
) = chain
;
5161 /* Remove barriers in REGION->EXIT's block. Note that this is only
5162 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
5163 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
5164 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
5168 remove_exit_barrier (struct omp_region
*region
)
5170 gimple_stmt_iterator gsi
;
5171 basic_block exit_bb
;
5175 int any_addressable_vars
= -1;
5177 exit_bb
= region
->exit
;
5179 /* If the parallel region doesn't return, we don't have REGION->EXIT
5184 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
5185 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
5186 statements that can appear in between are extremely limited -- no
5187 memory operations at all. Here, we allow nothing at all, so the
5188 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
5189 gsi
= gsi_last_bb (exit_bb
);
5190 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5192 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
5195 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
5197 gsi
= gsi_last_bb (e
->src
);
5198 if (gsi_end_p (gsi
))
5200 stmt
= gsi_stmt (gsi
);
5201 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
5202 && !gimple_omp_return_nowait_p (stmt
))
5204 /* OpenMP 3.0 tasks unfortunately prevent this optimization
5205 in many cases. If there could be tasks queued, the barrier
5206 might be needed to let the tasks run before some local
5207 variable of the parallel that the task uses as shared
5208 runs out of scope. The task can be spawned either
5209 from within current function (this would be easy to check)
5210 or from some function it calls and gets passed an address
5211 of such a variable. */
5212 if (any_addressable_vars
< 0)
5214 gomp_parallel
*parallel_stmt
5215 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
5216 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
5217 tree local_decls
, block
, decl
;
5220 any_addressable_vars
= 0;
5221 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
5222 if (TREE_ADDRESSABLE (decl
))
5224 any_addressable_vars
= 1;
5227 for (block
= gimple_block (stmt
);
5228 !any_addressable_vars
5230 && TREE_CODE (block
) == BLOCK
;
5231 block
= BLOCK_SUPERCONTEXT (block
))
5233 for (local_decls
= BLOCK_VARS (block
);
5235 local_decls
= DECL_CHAIN (local_decls
))
5236 if (TREE_ADDRESSABLE (local_decls
))
5238 any_addressable_vars
= 1;
5241 if (block
== gimple_block (parallel_stmt
))
5245 if (!any_addressable_vars
)
5246 gimple_omp_return_set_nowait (stmt
);
5252 remove_exit_barriers (struct omp_region
*region
)
5254 if (region
->type
== GIMPLE_OMP_PARALLEL
)
5255 remove_exit_barrier (region
);
5259 region
= region
->inner
;
5260 remove_exit_barriers (region
);
5261 while (region
->next
)
5263 region
= region
->next
;
5264 remove_exit_barriers (region
);
5269 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
5270 calls. These can't be declared as const functions, but
5271 within one parallel body they are constant, so they can be
5272 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
5273 which are declared const. Similarly for task body, except
5274 that in untied task omp_get_thread_num () can change at any task
5275 scheduling point. */
5278 optimize_omp_library_calls (gimple entry_stmt
)
5281 gimple_stmt_iterator gsi
;
5282 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5283 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
5284 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5285 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
5286 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
5287 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
5288 OMP_CLAUSE_UNTIED
) != NULL
);
5290 FOR_EACH_BB_FN (bb
, cfun
)
5291 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5293 gimple call
= gsi_stmt (gsi
);
5296 if (is_gimple_call (call
)
5297 && (decl
= gimple_call_fndecl (call
))
5298 && DECL_EXTERNAL (decl
)
5299 && TREE_PUBLIC (decl
)
5300 && DECL_INITIAL (decl
) == NULL
)
5304 if (DECL_NAME (decl
) == thr_num_id
)
5306 /* In #pragma omp task untied omp_get_thread_num () can change
5307 during the execution of the task region. */
5310 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5312 else if (DECL_NAME (decl
) == num_thr_id
)
5313 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5317 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
5318 || gimple_call_num_args (call
) != 0)
5321 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
5324 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
5325 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
5326 TREE_TYPE (TREE_TYPE (built_in
))))
5329 gimple_call_set_fndecl (call
, built_in
);
5334 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
5338 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
5342 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
5343 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
5346 if (TREE_CODE (t
) == ADDR_EXPR
)
5347 recompute_tree_invariant_for_addr_expr (t
);
5349 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
5353 /* Prepend TO = FROM assignment before *GSI_P. */
5356 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
5358 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
5359 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
5360 true, GSI_SAME_STMT
);
5361 gimple stmt
= gimple_build_assign (to
, from
);
5362 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
5363 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
5364 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
5366 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
5367 gimple_regimplify_operands (stmt
, &gsi
);
5371 /* Expand the OpenMP parallel or task directive starting at REGION. */
5374 expand_omp_taskreg (struct omp_region
*region
)
5376 basic_block entry_bb
, exit_bb
, new_bb
;
5377 struct function
*child_cfun
;
5378 tree child_fn
, block
, t
;
5379 gimple_stmt_iterator gsi
;
5380 gimple entry_stmt
, stmt
;
5382 vec
<tree
, va_gc
> *ws_args
;
5384 entry_stmt
= last_stmt (region
->entry
);
5385 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
5386 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
5388 entry_bb
= region
->entry
;
5389 exit_bb
= region
->exit
;
5393 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
5394 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
5395 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
5398 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
5399 and the inner statement contains the name of the built-in function
5401 ws_args
= region
->inner
->ws_args
;
5402 else if (is_combined_parallel (region
))
5403 ws_args
= region
->ws_args
;
5407 if (child_cfun
->cfg
)
5409 /* Due to inlining, it may happen that we have already outlined
5410 the region, in which case all we need to do is make the
5411 sub-graph unreachable and emit the parallel call. */
5412 edge entry_succ_e
, exit_succ_e
;
5414 entry_succ_e
= single_succ_edge (entry_bb
);
5416 gsi
= gsi_last_bb (entry_bb
);
5417 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
5418 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
5419 gsi_remove (&gsi
, true);
5424 exit_succ_e
= single_succ_edge (exit_bb
);
5425 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
5427 remove_edge_and_dominated_blocks (entry_succ_e
);
5431 unsigned srcidx
, dstidx
, num
;
5433 /* If the parallel region needs data sent from the parent
5434 function, then the very first statement (except possible
5435 tree profile counter updates) of the parallel body
5436 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
5437 &.OMP_DATA_O is passed as an argument to the child function,
5438 we need to replace it with the argument as seen by the child
5441 In most cases, this will end up being the identity assignment
5442 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
5443 a function call that has been inlined, the original PARM_DECL
5444 .OMP_DATA_I may have been converted into a different local
5445 variable. In which case, we need to keep the assignment. */
5446 if (gimple_omp_taskreg_data_arg (entry_stmt
))
5448 basic_block entry_succ_bb
= single_succ (entry_bb
);
5450 gimple parcopy_stmt
= NULL
;
5452 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
5456 gcc_assert (!gsi_end_p (gsi
));
5457 stmt
= gsi_stmt (gsi
);
5458 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
5461 if (gimple_num_ops (stmt
) == 2)
5463 tree arg
= gimple_assign_rhs1 (stmt
);
5465 /* We're ignore the subcode because we're
5466 effectively doing a STRIP_NOPS. */
5468 if (TREE_CODE (arg
) == ADDR_EXPR
5469 && TREE_OPERAND (arg
, 0)
5470 == gimple_omp_taskreg_data_arg (entry_stmt
))
5472 parcopy_stmt
= stmt
;
5478 gcc_assert (parcopy_stmt
!= NULL
);
5479 arg
= DECL_ARGUMENTS (child_fn
);
5481 if (!gimple_in_ssa_p (cfun
))
5483 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
5484 gsi_remove (&gsi
, true);
5487 /* ?? Is setting the subcode really necessary ?? */
5488 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
5489 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
5494 /* If we are in ssa form, we must load the value from the default
5495 definition of the argument. That should not be defined now,
5496 since the argument is not used uninitialized. */
5497 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
5498 narg
= make_ssa_name (arg
, gimple_build_nop ());
5499 set_ssa_default_def (cfun
, arg
, narg
);
5500 /* ?? Is setting the subcode really necessary ?? */
5501 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
5502 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
5503 update_stmt (parcopy_stmt
);
5507 /* Declare local variables needed in CHILD_CFUN. */
5508 block
= DECL_INITIAL (child_fn
);
5509 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
5510 /* The gimplifier could record temporaries in parallel/task block
5511 rather than in containing function's local_decls chain,
5512 which would mean cgraph missed finalizing them. Do it now. */
5513 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
5514 if (TREE_CODE (t
) == VAR_DECL
5516 && !DECL_EXTERNAL (t
))
5517 varpool_node::finalize_decl (t
);
5518 DECL_SAVED_TREE (child_fn
) = NULL
;
5519 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5520 gimple_set_body (child_fn
, NULL
);
5521 TREE_USED (block
) = 1;
5523 /* Reset DECL_CONTEXT on function arguments. */
5524 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
5525 DECL_CONTEXT (t
) = child_fn
;
5527 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
5528 so that it can be moved to the child function. */
5529 gsi
= gsi_last_bb (entry_bb
);
5530 stmt
= gsi_stmt (gsi
);
5531 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
5532 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
5533 gsi_remove (&gsi
, true);
5534 e
= split_block (entry_bb
, stmt
);
5536 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
5538 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
5541 gsi
= gsi_last_bb (exit_bb
);
5542 gcc_assert (!gsi_end_p (gsi
)
5543 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5544 stmt
= gimple_build_return (NULL
);
5545 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5546 gsi_remove (&gsi
, true);
5549 /* Move the parallel region into CHILD_CFUN. */
5551 if (gimple_in_ssa_p (cfun
))
5553 init_tree_ssa (child_cfun
);
5554 init_ssa_operands (child_cfun
);
5555 child_cfun
->gimple_df
->in_ssa_p
= true;
5559 block
= gimple_block (entry_stmt
);
5561 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
5563 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
5564 /* When the OMP expansion process cannot guarantee an up-to-date
5565 loop tree arrange for the child function to fixup loops. */
5566 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
5567 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
5569 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5570 num
= vec_safe_length (child_cfun
->local_decls
);
5571 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
5573 t
= (*child_cfun
->local_decls
)[srcidx
];
5574 if (DECL_CONTEXT (t
) == cfun
->decl
)
5576 if (srcidx
!= dstidx
)
5577 (*child_cfun
->local_decls
)[dstidx
] = t
;
5581 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
5583 /* Inform the callgraph about the new function. */
5584 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
5585 cgraph_node::add_new_function (child_fn
, true);
5587 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5588 fixed in a following pass. */
5589 push_cfun (child_cfun
);
5591 optimize_omp_library_calls (entry_stmt
);
5592 cgraph_edge::rebuild_edges ();
5594 /* Some EH regions might become dead, see PR34608. If
5595 pass_cleanup_cfg isn't the first pass to happen with the
5596 new child, these dead EH edges might cause problems.
5597 Clean them up now. */
5598 if (flag_exceptions
)
5601 bool changed
= false;
5603 FOR_EACH_BB_FN (bb
, cfun
)
5604 changed
|= gimple_purge_dead_eh_edges (bb
);
5606 cleanup_tree_cfg ();
5608 if (gimple_in_ssa_p (cfun
))
5609 update_ssa (TODO_update_ssa
);
5613 /* Emit a library call to launch the children threads. */
5615 expand_cilk_for_call (new_bb
,
5616 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
5617 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5618 expand_parallel_call (region
, new_bb
,
5619 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
5621 expand_task_call (new_bb
, as_a
<gomp_task
*> (entry_stmt
));
5622 if (gimple_in_ssa_p (cfun
))
5623 update_ssa (TODO_update_ssa_only_virtuals
);
5627 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5628 of the combined collapse > 1 loop constructs, generate code like:
5629 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5634 count3 = (adj + N32 - N31) / STEP3;
5635 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5640 count2 = (adj + N22 - N21) / STEP2;
5641 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5646 count1 = (adj + N12 - N11) / STEP1;
5647 count = count1 * count2 * count3;
5648 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5650 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5651 of the combined loop constructs, just initialize COUNTS array
5652 from the _looptemp_ clauses. */
5654 /* NOTE: It *could* be better to moosh all of the BBs together,
5655 creating one larger BB with all the computation and the unexpected
5656 jump at the end. I.e.
5658 bool zero3, zero2, zero1, zero;
5661 count3 = (N32 - N31) /[cl] STEP3;
5663 count2 = (N22 - N21) /[cl] STEP2;
5665 count1 = (N12 - N11) /[cl] STEP1;
5666 zero = zero3 || zero2 || zero1;
5667 count = count1 * count2 * count3;
5668 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5670 After all, we expect the zero=false, and thus we expect to have to
5671 evaluate all of the comparison expressions, so short-circuiting
5672 oughtn't be a win. Since the condition isn't protecting a
5673 denominator, we're not concerned about divide-by-zero, so we can
5674 fully evaluate count even if a numerator turned out to be wrong.
5676 It seems like putting this all together would create much better
5677 scheduling opportunities, and less pressure on the chip's branch
5681 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5682 basic_block
&entry_bb
, tree
*counts
,
5683 basic_block
&zero_iter_bb
, int &first_zero_iter
,
5684 basic_block
&l2_dom_bb
)
5686 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5690 /* Collapsed loops need work for expansion into SSA form. */
5691 gcc_assert (!gimple_in_ssa_p (cfun
));
5693 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5694 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5696 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5697 isn't supposed to be handled, as the inner loop doesn't
5699 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5700 OMP_CLAUSE__LOOPTEMP_
);
5701 gcc_assert (innerc
);
5702 for (i
= 0; i
< fd
->collapse
; i
++)
5704 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5705 OMP_CLAUSE__LOOPTEMP_
);
5706 gcc_assert (innerc
);
5708 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5710 counts
[0] = NULL_TREE
;
5715 for (i
= 0; i
< fd
->collapse
; i
++)
5717 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5719 if (SSA_VAR_P (fd
->loop
.n2
)
5720 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5721 fold_convert (itype
, fd
->loops
[i
].n1
),
5722 fold_convert (itype
, fd
->loops
[i
].n2
)))
5723 == NULL_TREE
|| !integer_onep (t
)))
5727 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5728 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5729 true, GSI_SAME_STMT
);
5730 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5731 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5732 true, GSI_SAME_STMT
);
5733 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5734 NULL_TREE
, NULL_TREE
);
5735 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
5736 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
5737 expand_omp_regimplify_p
, NULL
, NULL
)
5738 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
5739 expand_omp_regimplify_p
, NULL
, NULL
))
5741 *gsi
= gsi_for_stmt (cond_stmt
);
5742 gimple_regimplify_operands (cond_stmt
, gsi
);
5744 e
= split_block (entry_bb
, cond_stmt
);
5745 if (zero_iter_bb
== NULL
)
5747 gassign
*assign_stmt
;
5748 first_zero_iter
= i
;
5749 zero_iter_bb
= create_empty_bb (entry_bb
);
5750 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5751 *gsi
= gsi_after_labels (zero_iter_bb
);
5752 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
5753 build_zero_cst (type
));
5754 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
5755 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5758 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5759 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5760 e
->flags
= EDGE_TRUE_VALUE
;
5761 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5762 if (l2_dom_bb
== NULL
)
5763 l2_dom_bb
= entry_bb
;
5765 *gsi
= gsi_last_bb (entry_bb
);
5768 if (POINTER_TYPE_P (itype
))
5769 itype
= signed_type_for (itype
);
5770 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5772 t
= fold_build2 (PLUS_EXPR
, itype
,
5773 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5774 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5775 fold_convert (itype
, fd
->loops
[i
].n2
));
5776 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5777 fold_convert (itype
, fd
->loops
[i
].n1
));
5778 /* ?? We could probably use CEIL_DIV_EXPR instead of
5779 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5780 generate the same code in the end because generically we
5781 don't know that the values involved must be negative for
5783 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5784 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5785 fold_build1 (NEGATE_EXPR
, itype
, t
),
5786 fold_build1 (NEGATE_EXPR
, itype
,
5787 fold_convert (itype
,
5788 fd
->loops
[i
].step
)));
5790 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5791 fold_convert (itype
, fd
->loops
[i
].step
));
5792 t
= fold_convert (type
, t
);
5793 if (TREE_CODE (t
) == INTEGER_CST
)
5797 counts
[i
] = create_tmp_reg (type
, ".count");
5798 expand_omp_build_assign (gsi
, counts
[i
], t
);
5800 if (SSA_VAR_P (fd
->loop
.n2
))
5805 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5806 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5812 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5814 V3 = N31 + (T % count3) * STEP3;
5816 V2 = N21 + (T % count2) * STEP2;
5818 V1 = N11 + T * STEP1;
5819 if this loop doesn't have an inner loop construct combined with it.
5820 If it does have an inner loop construct combined with it and the
5821 iteration count isn't known constant, store values from counts array
5822 into its _looptemp_ temporaries instead. */
5825 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5826 tree
*counts
, gimple inner_stmt
, tree startvar
)
5829 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5831 /* If fd->loop.n2 is constant, then no propagation of the counts
5832 is needed, they are constant. */
5833 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5836 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5837 ? gimple_omp_parallel_clauses (inner_stmt
)
5838 : gimple_omp_for_clauses (inner_stmt
);
5839 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5840 isn't supposed to be handled, as the inner loop doesn't
5842 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5843 gcc_assert (innerc
);
5844 for (i
= 0; i
< fd
->collapse
; i
++)
5846 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5847 OMP_CLAUSE__LOOPTEMP_
);
5848 gcc_assert (innerc
);
5851 tree tem
= OMP_CLAUSE_DECL (innerc
);
5852 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5853 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5854 false, GSI_CONTINUE_LINKING
);
5855 gassign
*stmt
= gimple_build_assign (tem
, t
);
5856 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5862 tree type
= TREE_TYPE (fd
->loop
.v
);
5863 tree tem
= create_tmp_reg (type
, ".tem");
5864 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
5865 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5867 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5869 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5871 if (POINTER_TYPE_P (vtype
))
5872 itype
= signed_type_for (vtype
);
5874 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5877 t
= fold_convert (itype
, t
);
5878 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5879 fold_convert (itype
, fd
->loops
[i
].step
));
5880 if (POINTER_TYPE_P (vtype
))
5881 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5883 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5884 t
= force_gimple_operand_gsi (gsi
, t
,
5885 DECL_P (fd
->loops
[i
].v
)
5886 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5888 GSI_CONTINUE_LINKING
);
5889 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5890 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5893 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5894 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5895 false, GSI_CONTINUE_LINKING
);
5896 stmt
= gimple_build_assign (tem
, t
);
5897 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5903 /* Helper function for expand_omp_for_*. Generate code like:
5906 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5910 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5917 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5918 basic_block body_bb
)
5920 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5922 gimple_stmt_iterator gsi
;
5928 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5930 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5932 bb
= create_empty_bb (last_bb
);
5933 add_bb_to_loop (bb
, last_bb
->loop_father
);
5934 gsi
= gsi_start_bb (bb
);
5936 if (i
< fd
->collapse
- 1)
5938 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5939 e
->probability
= REG_BR_PROB_BASE
/ 8;
5941 t
= fd
->loops
[i
+ 1].n1
;
5942 t
= force_gimple_operand_gsi (&gsi
, t
,
5943 DECL_P (fd
->loops
[i
+ 1].v
)
5944 && TREE_ADDRESSABLE (fd
->loops
[i
5947 GSI_CONTINUE_LINKING
);
5948 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5949 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5954 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5956 if (POINTER_TYPE_P (vtype
))
5957 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5959 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5960 t
= force_gimple_operand_gsi (&gsi
, t
,
5961 DECL_P (fd
->loops
[i
].v
)
5962 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5963 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5964 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5965 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5969 t
= fd
->loops
[i
].n2
;
5970 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5971 false, GSI_CONTINUE_LINKING
);
5972 tree v
= fd
->loops
[i
].v
;
5973 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5974 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5975 false, GSI_CONTINUE_LINKING
);
5976 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5977 stmt
= gimple_build_cond_empty (t
);
5978 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5979 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5980 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5983 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5991 /* A subroutine of expand_omp_for. Generate code for a parallel
5992 loop with any schedule. Given parameters:
5994 for (V = N1; V cond N2; V += STEP) BODY;
5996 where COND is "<" or ">", we generate pseudocode
5998 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5999 if (more) goto L0; else goto L3;
6006 if (V cond iend) goto L1; else goto L2;
6008 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6011 If this is a combined omp parallel loop, instead of the call to
6012 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
6013 If this is gimple_omp_for_combined_p loop, then instead of assigning
6014 V and iend in L0 we assign the first two _looptemp_ clause decls of the
6015 inner GIMPLE_OMP_FOR and V += STEP; and
6016 if (V cond iend) goto L1; else goto L2; are removed.
6018 For collapsed loops, given parameters:
6020 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6021 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6022 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6025 we generate pseudocode
6027 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
6032 count3 = (adj + N32 - N31) / STEP3;
6033 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
6038 count2 = (adj + N22 - N21) / STEP2;
6039 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
6044 count1 = (adj + N12 - N11) / STEP1;
6045 count = count1 * count2 * count3;
6050 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
6051 if (more) goto L0; else goto L3;
6055 V3 = N31 + (T % count3) * STEP3;
6057 V2 = N21 + (T % count2) * STEP2;
6059 V1 = N11 + T * STEP1;
6064 if (V < iend) goto L10; else goto L2;
6067 if (V3 cond3 N32) goto L1; else goto L11;
6071 if (V2 cond2 N22) goto L1; else goto L12;
6077 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6083 expand_omp_for_generic (struct omp_region
*region
,
6084 struct omp_for_data
*fd
,
6085 enum built_in_function start_fn
,
6086 enum built_in_function next_fn
,
6089 tree type
, istart0
, iend0
, iend
;
6090 tree t
, vmain
, vback
, bias
= NULL_TREE
;
6091 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
6092 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
6093 gimple_stmt_iterator gsi
;
6094 gassign
*assign_stmt
;
6095 bool in_combined_parallel
= is_combined_parallel (region
);
6096 bool broken_loop
= region
->cont
== NULL
;
6098 tree
*counts
= NULL
;
6101 gcc_assert (!broken_loop
|| !in_combined_parallel
);
6102 gcc_assert (fd
->iter_type
== long_integer_type_node
6103 || !in_combined_parallel
);
6105 type
= TREE_TYPE (fd
->loop
.v
);
6106 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
6107 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
6108 TREE_ADDRESSABLE (istart0
) = 1;
6109 TREE_ADDRESSABLE (iend0
) = 1;
6111 /* See if we need to bias by LLONG_MIN. */
6112 if (fd
->iter_type
== long_long_unsigned_type_node
6113 && TREE_CODE (type
) == INTEGER_TYPE
6114 && !TYPE_UNSIGNED (type
))
6118 if (fd
->loop
.cond_code
== LT_EXPR
)
6121 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6125 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6128 if (TREE_CODE (n1
) != INTEGER_CST
6129 || TREE_CODE (n2
) != INTEGER_CST
6130 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
6131 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
6134 entry_bb
= region
->entry
;
6135 cont_bb
= region
->cont
;
6137 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6138 gcc_assert (broken_loop
6139 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6140 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6141 l1_bb
= single_succ (l0_bb
);
6144 l2_bb
= create_empty_bb (cont_bb
);
6145 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
6146 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6150 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6151 exit_bb
= region
->exit
;
6153 gsi
= gsi_last_bb (entry_bb
);
6155 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6156 if (fd
->collapse
> 1)
6158 int first_zero_iter
= -1;
6159 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
6161 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6162 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6163 zero_iter_bb
, first_zero_iter
,
6168 /* Some counts[i] vars might be uninitialized if
6169 some loop has zero iterations. But the body shouldn't
6170 be executed in that case, so just avoid uninit warnings. */
6171 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
6172 if (SSA_VAR_P (counts
[i
]))
6173 TREE_NO_WARNING (counts
[i
]) = 1;
6175 e
= split_block (entry_bb
, gsi_stmt (gsi
));
6177 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
6178 gsi
= gsi_last_bb (entry_bb
);
6179 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
6180 get_immediate_dominator (CDI_DOMINATORS
,
6184 if (in_combined_parallel
)
6186 /* In a combined parallel loop, emit a call to
6187 GOMP_loop_foo_next. */
6188 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6189 build_fold_addr_expr (istart0
),
6190 build_fold_addr_expr (iend0
));
6194 tree t0
, t1
, t2
, t3
, t4
;
6195 /* If this is not a combined parallel loop, emit a call to
6196 GOMP_loop_foo_start in ENTRY_BB. */
6197 t4
= build_fold_addr_expr (iend0
);
6198 t3
= build_fold_addr_expr (istart0
);
6199 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
6202 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6204 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6205 OMP_CLAUSE__LOOPTEMP_
);
6206 gcc_assert (innerc
);
6207 t0
= OMP_CLAUSE_DECL (innerc
);
6208 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6209 OMP_CLAUSE__LOOPTEMP_
);
6210 gcc_assert (innerc
);
6211 t1
= OMP_CLAUSE_DECL (innerc
);
6213 if (POINTER_TYPE_P (TREE_TYPE (t0
))
6214 && TYPE_PRECISION (TREE_TYPE (t0
))
6215 != TYPE_PRECISION (fd
->iter_type
))
6217 /* Avoid casting pointers to integer of a different size. */
6218 tree itype
= signed_type_for (type
);
6219 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
6220 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
6224 t1
= fold_convert (fd
->iter_type
, t1
);
6225 t0
= fold_convert (fd
->iter_type
, t0
);
6229 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
6230 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
6232 if (fd
->iter_type
== long_integer_type_node
)
6236 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6237 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6238 6, t0
, t1
, t2
, t
, t3
, t4
);
6241 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6242 5, t0
, t1
, t2
, t3
, t4
);
6250 /* The GOMP_loop_ull_*start functions have additional boolean
6251 argument, true for < loops and false for > loops.
6252 In Fortran, the C bool type can be different from
6253 boolean_type_node. */
6254 bfn_decl
= builtin_decl_explicit (start_fn
);
6255 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
6256 t5
= build_int_cst (c_bool_type
,
6257 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
6260 tree bfn_decl
= builtin_decl_explicit (start_fn
);
6261 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6262 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
6265 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6266 6, t5
, t0
, t1
, t2
, t3
, t4
);
6269 if (TREE_TYPE (t
) != boolean_type_node
)
6270 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6271 t
, build_int_cst (TREE_TYPE (t
), 0));
6272 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6273 true, GSI_SAME_STMT
);
6274 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6276 /* Remove the GIMPLE_OMP_FOR statement. */
6277 gsi_remove (&gsi
, true);
6279 /* Iteration setup for sequential loop goes in L0_BB. */
6280 tree startvar
= fd
->loop
.v
;
6281 tree endvar
= NULL_TREE
;
6283 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6285 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
6286 && gimple_omp_for_kind (inner_stmt
)
6287 == GF_OMP_FOR_KIND_SIMD
);
6288 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
6289 OMP_CLAUSE__LOOPTEMP_
);
6290 gcc_assert (innerc
);
6291 startvar
= OMP_CLAUSE_DECL (innerc
);
6292 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6293 OMP_CLAUSE__LOOPTEMP_
);
6294 gcc_assert (innerc
);
6295 endvar
= OMP_CLAUSE_DECL (innerc
);
6298 gsi
= gsi_start_bb (l0_bb
);
6301 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6302 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6303 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6304 t
= fold_convert (TREE_TYPE (startvar
), t
);
6305 t
= force_gimple_operand_gsi (&gsi
, t
,
6307 && TREE_ADDRESSABLE (startvar
),
6308 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6309 assign_stmt
= gimple_build_assign (startvar
, t
);
6310 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6314 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6315 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6316 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6317 t
= fold_convert (TREE_TYPE (startvar
), t
);
6318 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6319 false, GSI_CONTINUE_LINKING
);
6322 assign_stmt
= gimple_build_assign (endvar
, iend
);
6323 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6324 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
6325 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
6327 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
6328 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6330 if (fd
->collapse
> 1)
6331 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6335 /* Code to control the increment and predicate for the sequential
6336 loop goes in the CONT_BB. */
6337 gsi
= gsi_last_bb (cont_bb
);
6338 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6339 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
6340 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6341 vback
= gimple_omp_continue_control_def (cont_stmt
);
6343 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6345 if (POINTER_TYPE_P (type
))
6346 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
6348 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
6349 t
= force_gimple_operand_gsi (&gsi
, t
,
6351 && TREE_ADDRESSABLE (vback
),
6352 NULL_TREE
, true, GSI_SAME_STMT
);
6353 assign_stmt
= gimple_build_assign (vback
, t
);
6354 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6356 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6357 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
6359 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6360 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6363 /* Remove GIMPLE_OMP_CONTINUE. */
6364 gsi_remove (&gsi
, true);
6366 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6367 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
6369 /* Emit code to get the next parallel iteration in L2_BB. */
6370 gsi
= gsi_start_bb (l2_bb
);
6372 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6373 build_fold_addr_expr (istart0
),
6374 build_fold_addr_expr (iend0
));
6375 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6376 false, GSI_CONTINUE_LINKING
);
6377 if (TREE_TYPE (t
) != boolean_type_node
)
6378 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6379 t
, build_int_cst (TREE_TYPE (t
), 0));
6380 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6381 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
6384 /* Add the loop cleanup function. */
6385 gsi
= gsi_last_bb (exit_bb
);
6386 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6387 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
6388 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6389 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
6391 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
6392 gcall
*call_stmt
= gimple_build_call (t
, 0);
6393 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6394 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
6395 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
6396 gsi_remove (&gsi
, true);
6398 /* Connect the new blocks. */
6399 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
6400 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
6406 e
= find_edge (cont_bb
, l3_bb
);
6407 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
6409 phis
= phi_nodes (l3_bb
);
6410 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
6412 gimple phi
= gsi_stmt (gsi
);
6413 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
6414 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
6418 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
6419 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
6420 e
= find_edge (cont_bb
, l1_bb
);
6421 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6426 else if (fd
->collapse
> 1)
6429 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6432 e
->flags
= EDGE_TRUE_VALUE
;
6435 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6436 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
6440 e
= find_edge (cont_bb
, l2_bb
);
6441 e
->flags
= EDGE_FALLTHRU
;
6443 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6445 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
6446 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
6447 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
6448 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
6449 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
6450 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
6451 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
6452 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
6454 struct loop
*outer_loop
= alloc_loop ();
6455 outer_loop
->header
= l0_bb
;
6456 outer_loop
->latch
= l2_bb
;
6457 add_loop (outer_loop
, l0_bb
->loop_father
);
6459 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6461 struct loop
*loop
= alloc_loop ();
6462 loop
->header
= l1_bb
;
6463 /* The loop may have multiple latches. */
6464 add_loop (loop
, outer_loop
);
6470 /* A subroutine of expand_omp_for. Generate code for a parallel
6471 loop with static schedule and no specified chunk size. Given
6474 for (V = N1; V cond N2; V += STEP) BODY;
6476 where COND is "<" or ">", we generate pseudocode
6478 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6483 if ((__typeof (V)) -1 > 0 && cond is >)
6484 n = -(adj + N2 - N1) / -STEP;
6486 n = (adj + N2 - N1) / STEP;
6489 if (threadid < tt) goto L3; else goto L4;
6494 s0 = q * threadid + tt;
6497 if (s0 >= e0) goto L2; else goto L0;
6503 if (V cond e) goto L1;
6508 expand_omp_for_static_nochunk (struct omp_region
*region
,
6509 struct omp_for_data
*fd
,
6512 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
6513 tree type
, itype
, vmain
, vback
;
6514 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
6515 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
6517 gimple_stmt_iterator gsi
;
6519 bool broken_loop
= region
->cont
== NULL
;
6520 tree
*counts
= NULL
;
6523 gcc_checking_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6524 != GF_OMP_FOR_KIND_OACC_LOOP
)
6527 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6528 if (POINTER_TYPE_P (type
))
6529 itype
= signed_type_for (type
);
6531 entry_bb
= region
->entry
;
6532 cont_bb
= region
->cont
;
6533 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6534 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6535 gcc_assert (broken_loop
6536 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
6537 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6538 body_bb
= single_succ (seq_start_bb
);
6541 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6542 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6544 exit_bb
= region
->exit
;
6546 /* Iteration space partitioning goes in ENTRY_BB. */
6547 gsi
= gsi_last_bb (entry_bb
);
6548 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6550 if (fd
->collapse
> 1)
6552 int first_zero_iter
= -1;
6553 basic_block l2_dom_bb
= NULL
;
6555 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6556 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6557 fin_bb
, first_zero_iter
,
6561 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6562 t
= integer_one_node
;
6564 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6565 fold_convert (type
, fd
->loop
.n1
),
6566 fold_convert (type
, fd
->loop
.n2
));
6567 if (fd
->collapse
== 1
6568 && TYPE_UNSIGNED (type
)
6569 && (t
== NULL_TREE
|| !integer_onep (t
)))
6571 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6572 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6573 true, GSI_SAME_STMT
);
6574 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6575 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6576 true, GSI_SAME_STMT
);
6577 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6578 NULL_TREE
, NULL_TREE
);
6579 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6580 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6581 expand_omp_regimplify_p
, NULL
, NULL
)
6582 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6583 expand_omp_regimplify_p
, NULL
, NULL
))
6585 gsi
= gsi_for_stmt (cond_stmt
);
6586 gimple_regimplify_operands (cond_stmt
, &gsi
);
6588 ep
= split_block (entry_bb
, cond_stmt
);
6589 ep
->flags
= EDGE_TRUE_VALUE
;
6590 entry_bb
= ep
->dest
;
6591 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6592 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6593 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6594 if (gimple_in_ssa_p (cfun
))
6596 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6597 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
6598 !gsi_end_p (gpi
); gsi_next (&gpi
))
6600 gphi
*phi
= gpi
.phi ();
6601 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6602 ep
, UNKNOWN_LOCATION
);
6605 gsi
= gsi_last_bb (entry_bb
);
6608 switch (gimple_omp_for_kind (fd
->for_stmt
))
6610 case GF_OMP_FOR_KIND_FOR
:
6611 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6612 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6614 case GF_OMP_FOR_KIND_DISTRIBUTE
:
6615 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
6616 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
6618 case GF_OMP_FOR_KIND_OACC_LOOP
:
6619 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
6620 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
6625 nthreads
= build_call_expr (nthreads
, 0);
6626 nthreads
= fold_convert (itype
, nthreads
);
6627 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
6628 true, GSI_SAME_STMT
);
6629 threadid
= build_call_expr (threadid
, 0);
6630 threadid
= fold_convert (itype
, threadid
);
6631 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
6632 true, GSI_SAME_STMT
);
6636 step
= fd
->loop
.step
;
6637 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6639 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6640 OMP_CLAUSE__LOOPTEMP_
);
6641 gcc_assert (innerc
);
6642 n1
= OMP_CLAUSE_DECL (innerc
);
6643 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6644 OMP_CLAUSE__LOOPTEMP_
);
6645 gcc_assert (innerc
);
6646 n2
= OMP_CLAUSE_DECL (innerc
);
6648 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6649 true, NULL_TREE
, true, GSI_SAME_STMT
);
6650 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6651 true, NULL_TREE
, true, GSI_SAME_STMT
);
6652 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6653 true, NULL_TREE
, true, GSI_SAME_STMT
);
6655 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6656 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6657 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6658 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6659 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6660 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6661 fold_build1 (NEGATE_EXPR
, itype
, t
),
6662 fold_build1 (NEGATE_EXPR
, itype
, step
));
6664 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6665 t
= fold_convert (itype
, t
);
6666 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6668 q
= create_tmp_reg (itype
, "q");
6669 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
6670 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6671 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
6673 tt
= create_tmp_reg (itype
, "tt");
6674 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
6675 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6676 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
6678 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
6679 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6680 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6682 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
6683 gsi
= gsi_last_bb (second_bb
);
6684 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6686 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
6688 gassign
*assign_stmt
6689 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
6690 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6692 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
6693 gsi
= gsi_last_bb (third_bb
);
6694 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6696 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6697 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6698 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6700 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6701 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6703 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6704 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6706 /* Remove the GIMPLE_OMP_FOR statement. */
6707 gsi_remove (&gsi
, true);
6709 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6710 gsi
= gsi_start_bb (seq_start_bb
);
6712 tree startvar
= fd
->loop
.v
;
6713 tree endvar
= NULL_TREE
;
6715 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6717 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6718 ? gimple_omp_parallel_clauses (inner_stmt
)
6719 : gimple_omp_for_clauses (inner_stmt
);
6720 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6721 gcc_assert (innerc
);
6722 startvar
= OMP_CLAUSE_DECL (innerc
);
6723 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6724 OMP_CLAUSE__LOOPTEMP_
);
6725 gcc_assert (innerc
);
6726 endvar
= OMP_CLAUSE_DECL (innerc
);
6728 t
= fold_convert (itype
, s0
);
6729 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6730 if (POINTER_TYPE_P (type
))
6731 t
= fold_build_pointer_plus (n1
, t
);
6733 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6734 t
= fold_convert (TREE_TYPE (startvar
), t
);
6735 t
= force_gimple_operand_gsi (&gsi
, t
,
6737 && TREE_ADDRESSABLE (startvar
),
6738 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6739 assign_stmt
= gimple_build_assign (startvar
, t
);
6740 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6742 t
= fold_convert (itype
, e0
);
6743 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6744 if (POINTER_TYPE_P (type
))
6745 t
= fold_build_pointer_plus (n1
, t
);
6747 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6748 t
= fold_convert (TREE_TYPE (startvar
), t
);
6749 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6750 false, GSI_CONTINUE_LINKING
);
6753 assign_stmt
= gimple_build_assign (endvar
, e
);
6754 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6755 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6756 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6758 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
6759 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6761 if (fd
->collapse
> 1)
6762 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6766 /* The code controlling the sequential loop replaces the
6767 GIMPLE_OMP_CONTINUE. */
6768 gsi
= gsi_last_bb (cont_bb
);
6769 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6770 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
6771 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6772 vback
= gimple_omp_continue_control_def (cont_stmt
);
6774 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6776 if (POINTER_TYPE_P (type
))
6777 t
= fold_build_pointer_plus (vmain
, step
);
6779 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6780 t
= force_gimple_operand_gsi (&gsi
, t
,
6782 && TREE_ADDRESSABLE (vback
),
6783 NULL_TREE
, true, GSI_SAME_STMT
);
6784 assign_stmt
= gimple_build_assign (vback
, t
);
6785 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6787 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6788 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6790 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6793 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6794 gsi_remove (&gsi
, true);
6796 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6797 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6800 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6801 gsi
= gsi_last_bb (exit_bb
);
6802 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6804 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6805 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
6806 gcc_checking_assert (t
== NULL_TREE
);
6808 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6810 gsi_remove (&gsi
, true);
6812 /* Connect all the blocks. */
6813 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6814 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6815 ep
= find_edge (entry_bb
, second_bb
);
6816 ep
->flags
= EDGE_TRUE_VALUE
;
6817 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6818 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6819 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6823 ep
= find_edge (cont_bb
, body_bb
);
6824 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6829 else if (fd
->collapse
> 1)
6832 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6835 ep
->flags
= EDGE_TRUE_VALUE
;
6836 find_edge (cont_bb
, fin_bb
)->flags
6837 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6840 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6841 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6842 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6844 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6845 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6846 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6847 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6849 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6851 struct loop
*loop
= alloc_loop ();
6852 loop
->header
= body_bb
;
6853 if (collapse_bb
== NULL
)
6854 loop
->latch
= cont_bb
;
6855 add_loop (loop
, body_bb
->loop_father
);
6860 /* A subroutine of expand_omp_for. Generate code for a parallel
6861 loop with static schedule and a specified chunk size. Given
6864 for (V = N1; V cond N2; V += STEP) BODY;
6866 where COND is "<" or ">", we generate pseudocode
6868 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6873 if ((__typeof (V)) -1 > 0 && cond is >)
6874 n = -(adj + N2 - N1) / -STEP;
6876 n = (adj + N2 - N1) / STEP;
6878 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6879 here so that V is defined
6880 if the loop is not entered
6882 s0 = (trip * nthreads + threadid) * CHUNK;
6883 e0 = min(s0 + CHUNK, n);
6884 if (s0 < n) goto L1; else goto L4;
6891 if (V cond e) goto L2; else goto L3;
6899 expand_omp_for_static_chunk (struct omp_region
*region
,
6900 struct omp_for_data
*fd
, gimple inner_stmt
)
6902 tree n
, s0
, e0
, e
, t
;
6903 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6904 tree type
, itype
, vmain
, vback
, vextra
;
6905 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6906 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6907 gimple_stmt_iterator gsi
;
6909 bool broken_loop
= region
->cont
== NULL
;
6910 tree
*counts
= NULL
;
6913 gcc_checking_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6914 != GF_OMP_FOR_KIND_OACC_LOOP
)
6917 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6918 if (POINTER_TYPE_P (type
))
6919 itype
= signed_type_for (type
);
6921 entry_bb
= region
->entry
;
6922 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6924 iter_part_bb
= se
->dest
;
6925 cont_bb
= region
->cont
;
6926 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6927 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6928 gcc_assert (broken_loop
6929 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6930 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6931 body_bb
= single_succ (seq_start_bb
);
6934 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6935 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6936 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6938 exit_bb
= region
->exit
;
6940 /* Trip and adjustment setup goes in ENTRY_BB. */
6941 gsi
= gsi_last_bb (entry_bb
);
6942 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6944 if (fd
->collapse
> 1)
6946 int first_zero_iter
= -1;
6947 basic_block l2_dom_bb
= NULL
;
6949 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6950 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6951 fin_bb
, first_zero_iter
,
6955 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6956 t
= integer_one_node
;
6958 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6959 fold_convert (type
, fd
->loop
.n1
),
6960 fold_convert (type
, fd
->loop
.n2
));
6961 if (fd
->collapse
== 1
6962 && TYPE_UNSIGNED (type
)
6963 && (t
== NULL_TREE
|| !integer_onep (t
)))
6965 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6966 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6967 true, GSI_SAME_STMT
);
6968 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6969 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6970 true, GSI_SAME_STMT
);
6971 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6972 NULL_TREE
, NULL_TREE
);
6973 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6974 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6975 expand_omp_regimplify_p
, NULL
, NULL
)
6976 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6977 expand_omp_regimplify_p
, NULL
, NULL
))
6979 gsi
= gsi_for_stmt (cond_stmt
);
6980 gimple_regimplify_operands (cond_stmt
, &gsi
);
6982 se
= split_block (entry_bb
, cond_stmt
);
6983 se
->flags
= EDGE_TRUE_VALUE
;
6984 entry_bb
= se
->dest
;
6985 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6986 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6987 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6988 if (gimple_in_ssa_p (cfun
))
6990 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6991 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
6992 !gsi_end_p (gpi
); gsi_next (&gpi
))
6994 gphi
*phi
= gpi
.phi ();
6995 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6996 se
, UNKNOWN_LOCATION
);
6999 gsi
= gsi_last_bb (entry_bb
);
7002 switch (gimple_omp_for_kind (fd
->for_stmt
))
7004 case GF_OMP_FOR_KIND_FOR
:
7005 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
7006 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
7008 case GF_OMP_FOR_KIND_DISTRIBUTE
:
7009 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
7010 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
7012 case GF_OMP_FOR_KIND_OACC_LOOP
:
7013 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
7014 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
7019 nthreads
= build_call_expr (nthreads
, 0);
7020 nthreads
= fold_convert (itype
, nthreads
);
7021 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
7022 true, GSI_SAME_STMT
);
7023 threadid
= build_call_expr (threadid
, 0);
7024 threadid
= fold_convert (itype
, threadid
);
7025 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
7026 true, GSI_SAME_STMT
);
7030 step
= fd
->loop
.step
;
7031 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7033 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7034 OMP_CLAUSE__LOOPTEMP_
);
7035 gcc_assert (innerc
);
7036 n1
= OMP_CLAUSE_DECL (innerc
);
7037 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7038 OMP_CLAUSE__LOOPTEMP_
);
7039 gcc_assert (innerc
);
7040 n2
= OMP_CLAUSE_DECL (innerc
);
7042 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
7043 true, NULL_TREE
, true, GSI_SAME_STMT
);
7044 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
7045 true, NULL_TREE
, true, GSI_SAME_STMT
);
7046 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
7047 true, NULL_TREE
, true, GSI_SAME_STMT
);
7049 = force_gimple_operand_gsi (&gsi
, fold_convert (itype
, fd
->chunk_size
),
7050 true, NULL_TREE
, true, GSI_SAME_STMT
);
7052 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
7053 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
7054 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
7055 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
7056 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
7057 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7058 fold_build1 (NEGATE_EXPR
, itype
, t
),
7059 fold_build1 (NEGATE_EXPR
, itype
, step
));
7061 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
7062 t
= fold_convert (itype
, t
);
7063 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7064 true, GSI_SAME_STMT
);
7066 trip_var
= create_tmp_reg (itype
, ".trip");
7067 if (gimple_in_ssa_p (cfun
))
7069 trip_init
= make_ssa_name (trip_var
);
7070 trip_main
= make_ssa_name (trip_var
);
7071 trip_back
= make_ssa_name (trip_var
);
7075 trip_init
= trip_var
;
7076 trip_main
= trip_var
;
7077 trip_back
= trip_var
;
7080 gassign
*assign_stmt
7081 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
7082 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
7084 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
7085 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7086 if (POINTER_TYPE_P (type
))
7087 t
= fold_build_pointer_plus (n1
, t
);
7089 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7090 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7091 true, GSI_SAME_STMT
);
7093 /* Remove the GIMPLE_OMP_FOR. */
7094 gsi_remove (&gsi
, true);
7096 /* Iteration space partitioning goes in ITER_PART_BB. */
7097 gsi
= gsi_last_bb (iter_part_bb
);
7099 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
7100 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
7101 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
7102 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7103 false, GSI_CONTINUE_LINKING
);
7105 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
7106 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
7107 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7108 false, GSI_CONTINUE_LINKING
);
7110 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
7111 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
7113 /* Setup code for sequential iteration goes in SEQ_START_BB. */
7114 gsi
= gsi_start_bb (seq_start_bb
);
7116 tree startvar
= fd
->loop
.v
;
7117 tree endvar
= NULL_TREE
;
7119 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7121 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
7122 ? gimple_omp_parallel_clauses (inner_stmt
)
7123 : gimple_omp_for_clauses (inner_stmt
);
7124 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7125 gcc_assert (innerc
);
7126 startvar
= OMP_CLAUSE_DECL (innerc
);
7127 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7128 OMP_CLAUSE__LOOPTEMP_
);
7129 gcc_assert (innerc
);
7130 endvar
= OMP_CLAUSE_DECL (innerc
);
7133 t
= fold_convert (itype
, s0
);
7134 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7135 if (POINTER_TYPE_P (type
))
7136 t
= fold_build_pointer_plus (n1
, t
);
7138 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7139 t
= fold_convert (TREE_TYPE (startvar
), t
);
7140 t
= force_gimple_operand_gsi (&gsi
, t
,
7142 && TREE_ADDRESSABLE (startvar
),
7143 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7144 assign_stmt
= gimple_build_assign (startvar
, t
);
7145 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7147 t
= fold_convert (itype
, e0
);
7148 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7149 if (POINTER_TYPE_P (type
))
7150 t
= fold_build_pointer_plus (n1
, t
);
7152 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7153 t
= fold_convert (TREE_TYPE (startvar
), t
);
7154 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7155 false, GSI_CONTINUE_LINKING
);
7158 assign_stmt
= gimple_build_assign (endvar
, e
);
7159 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7160 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
7161 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
7163 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
7164 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7166 if (fd
->collapse
> 1)
7167 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
7171 /* The code controlling the sequential loop goes in CONT_BB,
7172 replacing the GIMPLE_OMP_CONTINUE. */
7173 gsi
= gsi_last_bb (cont_bb
);
7174 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
7175 vmain
= gimple_omp_continue_control_use (cont_stmt
);
7176 vback
= gimple_omp_continue_control_def (cont_stmt
);
7178 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7180 if (POINTER_TYPE_P (type
))
7181 t
= fold_build_pointer_plus (vmain
, step
);
7183 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
7184 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
7185 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7186 true, GSI_SAME_STMT
);
7187 assign_stmt
= gimple_build_assign (vback
, t
);
7188 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
7190 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
7191 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
7193 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
7196 /* Remove GIMPLE_OMP_CONTINUE. */
7197 gsi_remove (&gsi
, true);
7199 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
7200 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
7202 /* Trip update code goes into TRIP_UPDATE_BB. */
7203 gsi
= gsi_start_bb (trip_update_bb
);
7205 t
= build_int_cst (itype
, 1);
7206 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
7207 assign_stmt
= gimple_build_assign (trip_back
, t
);
7208 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7211 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
7212 gsi
= gsi_last_bb (exit_bb
);
7213 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
7215 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
7216 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
7217 gcc_checking_assert (t
== NULL_TREE
);
7219 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
7221 gsi_remove (&gsi
, true);
7223 /* Connect the new blocks. */
7224 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
7225 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
7229 se
= find_edge (cont_bb
, body_bb
);
7230 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7235 else if (fd
->collapse
> 1)
7238 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
7241 se
->flags
= EDGE_TRUE_VALUE
;
7242 find_edge (cont_bb
, trip_update_bb
)->flags
7243 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
7245 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
7248 if (gimple_in_ssa_p (cfun
))
7256 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
7258 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
7259 remove arguments of the phi nodes in fin_bb. We need to create
7260 appropriate phi nodes in iter_part_bb instead. */
7261 se
= single_pred_edge (fin_bb
);
7262 re
= single_succ_edge (trip_update_bb
);
7263 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
7264 ene
= single_succ_edge (entry_bb
);
7266 psi
= gsi_start_phis (fin_bb
);
7267 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
7268 gsi_next (&psi
), ++i
)
7271 source_location locus
;
7274 t
= gimple_phi_result (phi
);
7275 gcc_assert (t
== redirect_edge_var_map_result (vm
));
7276 nphi
= create_phi_node (t
, iter_part_bb
);
7278 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
7279 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
7281 /* A special case -- fd->loop.v is not yet computed in
7282 iter_part_bb, we need to use vextra instead. */
7283 if (t
== fd
->loop
.v
)
7285 add_phi_arg (nphi
, t
, ene
, locus
);
7286 locus
= redirect_edge_var_map_location (vm
);
7287 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
7289 gcc_assert (gsi_end_p (psi
) && i
== head
->length ());
7290 redirect_edge_var_map_clear (re
);
7293 psi
= gsi_start_phis (fin_bb
);
7294 if (gsi_end_p (psi
))
7296 remove_phi_node (&psi
, false);
7299 /* Make phi node for trip. */
7300 phi
= create_phi_node (trip_main
, iter_part_bb
);
7301 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
7303 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
7308 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
7309 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
7310 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
7311 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
7312 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
7313 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
7314 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
7315 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
7316 recompute_dominator (CDI_DOMINATORS
, body_bb
));
7320 struct loop
*trip_loop
= alloc_loop ();
7321 trip_loop
->header
= iter_part_bb
;
7322 trip_loop
->latch
= trip_update_bb
;
7323 add_loop (trip_loop
, iter_part_bb
->loop_father
);
7325 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7327 struct loop
*loop
= alloc_loop ();
7328 loop
->header
= body_bb
;
7329 if (collapse_bb
== NULL
)
7330 loop
->latch
= cont_bb
;
7331 add_loop (loop
, trip_loop
);
7336 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
7338 for (V = N1; V cond N2; V += STEP) BODY;
7340 where COND is "<" or ">" or "!=", we generate pseudocode
7342 for (ind_var = low; ind_var < high; ind_var++)
7344 V = n1 + (ind_var * STEP)
7349 In the above pseudocode, low and high are function parameters of the
7350 child function. In the function below, we are inserting a temp.
7351 variable that will be making a call to two OMP functions that will not be
7352 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
7353 with _Cilk_for). These functions are replaced with low and high
7354 by the function that handles taskreg. */
7358 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
7360 bool broken_loop
= region
->cont
== NULL
;
7361 basic_block entry_bb
= region
->entry
;
7362 basic_block cont_bb
= region
->cont
;
7364 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7365 gcc_assert (broken_loop
7366 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7367 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7368 basic_block l1_bb
, l2_bb
;
7372 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7373 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7374 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7375 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7379 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7380 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7381 l2_bb
= single_succ (l1_bb
);
7383 basic_block exit_bb
= region
->exit
;
7384 basic_block l2_dom_bb
= NULL
;
7386 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
7388 /* Below statements until the "tree high_val = ..." are pseudo statements
7389 used to pass information to be used by expand_omp_taskreg.
7390 low_val and high_val will be replaced by the __low and __high
7391 parameter from the child function.
7393 The call_exprs part is a place-holder, it is mainly used
7394 to distinctly identify to the top-level part that this is
7395 where we should put low and high (reasoning given in header
7399 = gimple_omp_parallel_child_fn (
7400 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
7401 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
7402 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
7404 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
7406 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
7409 gcc_assert (low_val
&& high_val
);
7411 tree type
= TREE_TYPE (low_val
);
7412 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
7413 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7415 /* Not needed in SSA form right now. */
7416 gcc_assert (!gimple_in_ssa_p (cfun
));
7417 if (l2_dom_bb
== NULL
)
7423 gimple stmt
= gimple_build_assign (ind_var
, n1
);
7425 /* Replace the GIMPLE_OMP_FOR statement. */
7426 gsi_replace (&gsi
, stmt
, true);
7430 /* Code to control the increment goes in the CONT_BB. */
7431 gsi
= gsi_last_bb (cont_bb
);
7432 stmt
= gsi_stmt (gsi
);
7433 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7434 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
7435 build_one_cst (type
));
7437 /* Replace GIMPLE_OMP_CONTINUE. */
7438 gsi_replace (&gsi
, stmt
, true);
7441 /* Emit the condition in L1_BB. */
7442 gsi
= gsi_after_labels (l1_bb
);
7443 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
7444 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
7446 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
7447 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7448 fd
->loop
.n1
, fold_convert (sizetype
, t
));
7450 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7451 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
7452 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
7453 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7455 /* The condition is always '<' since the runtime will fill in the low
7457 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
7458 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
7460 /* Remove GIMPLE_OMP_RETURN. */
7461 gsi
= gsi_last_bb (exit_bb
);
7462 gsi_remove (&gsi
, true);
7464 /* Connect the new blocks. */
7465 remove_edge (FALLTHRU_EDGE (entry_bb
));
7470 remove_edge (BRANCH_EDGE (entry_bb
));
7471 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7473 e
= BRANCH_EDGE (l1_bb
);
7474 ne
= FALLTHRU_EDGE (l1_bb
);
7475 e
->flags
= EDGE_TRUE_VALUE
;
7479 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7481 ne
= single_succ_edge (l1_bb
);
7482 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7485 ne
->flags
= EDGE_FALSE_VALUE
;
7486 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7487 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7489 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7490 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7491 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7495 struct loop
*loop
= alloc_loop ();
7496 loop
->header
= l1_bb
;
7497 loop
->latch
= cont_bb
;
7498 add_loop (loop
, l1_bb
->loop_father
);
7499 loop
->safelen
= INT_MAX
;
7502 /* Pick the correct library function based on the precision of the
7503 induction variable type. */
7504 tree lib_fun
= NULL_TREE
;
7505 if (TYPE_PRECISION (type
) == 32)
7506 lib_fun
= cilk_for_32_fndecl
;
7507 else if (TYPE_PRECISION (type
) == 64)
7508 lib_fun
= cilk_for_64_fndecl
;
7512 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
7514 /* WS_ARGS contains the library function flavor to call:
7515 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
7516 user-defined grain value. If the user does not define one, then zero
7517 is passed in by the parser. */
7518 vec_alloc (region
->ws_args
, 2);
7519 region
->ws_args
->quick_push (lib_fun
);
7520 region
->ws_args
->quick_push (fd
->chunk_size
);
7523 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
7524 loop. Given parameters:
7526 for (V = N1; V cond N2; V += STEP) BODY;
7528 where COND is "<" or ">", we generate pseudocode
7536 if (V cond N2) goto L0; else goto L2;
7539 For collapsed loops, given parameters:
7541 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7542 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7543 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7546 we generate pseudocode
7552 count3 = (adj + N32 - N31) / STEP3;
7557 count2 = (adj + N22 - N21) / STEP2;
7562 count1 = (adj + N12 - N11) / STEP1;
7563 count = count1 * count2 * count3;
7573 V2 += (V3 cond3 N32) ? 0 : STEP2;
7574 V3 = (V3 cond3 N32) ? V3 : N31;
7575 V1 += (V2 cond2 N22) ? 0 : STEP1;
7576 V2 = (V2 cond2 N22) ? V2 : N21;
7578 if (V < count) goto L0; else goto L2;
7584 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
7587 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
7588 gimple_stmt_iterator gsi
;
7591 bool broken_loop
= region
->cont
== NULL
;
7593 tree
*counts
= NULL
;
7595 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7596 OMP_CLAUSE_SAFELEN
);
7597 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7598 OMP_CLAUSE__SIMDUID_
);
7601 type
= TREE_TYPE (fd
->loop
.v
);
7602 entry_bb
= region
->entry
;
7603 cont_bb
= region
->cont
;
7604 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7605 gcc_assert (broken_loop
7606 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7607 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7610 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7611 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7612 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7613 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7617 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7618 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7619 l2_bb
= single_succ (l1_bb
);
7621 exit_bb
= region
->exit
;
7624 gsi
= gsi_last_bb (entry_bb
);
7626 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7627 /* Not needed in SSA form right now. */
7628 gcc_assert (!gimple_in_ssa_p (cfun
));
7629 if (fd
->collapse
> 1)
7631 int first_zero_iter
= -1;
7632 basic_block zero_iter_bb
= l2_bb
;
7634 counts
= XALLOCAVEC (tree
, fd
->collapse
);
7635 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
7636 zero_iter_bb
, first_zero_iter
,
7639 if (l2_dom_bb
== NULL
)
7644 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7646 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7647 OMP_CLAUSE__LOOPTEMP_
);
7648 gcc_assert (innerc
);
7649 n1
= OMP_CLAUSE_DECL (innerc
);
7650 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7651 OMP_CLAUSE__LOOPTEMP_
);
7652 gcc_assert (innerc
);
7653 n2
= OMP_CLAUSE_DECL (innerc
);
7654 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7655 fold_convert (type
, n1
));
7656 if (fd
->collapse
> 1)
7659 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
7665 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7666 fold_convert (type
, fd
->loop
.n1
));
7667 if (fd
->collapse
> 1)
7668 for (i
= 0; i
< fd
->collapse
; i
++)
7670 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7671 if (POINTER_TYPE_P (itype
))
7672 itype
= signed_type_for (itype
);
7673 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
7674 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7678 /* Remove the GIMPLE_OMP_FOR statement. */
7679 gsi_remove (&gsi
, true);
7683 /* Code to control the increment goes in the CONT_BB. */
7684 gsi
= gsi_last_bb (cont_bb
);
7685 stmt
= gsi_stmt (gsi
);
7686 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7688 if (POINTER_TYPE_P (type
))
7689 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
7691 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
7692 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7694 if (fd
->collapse
> 1)
7696 i
= fd
->collapse
- 1;
7697 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7699 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
7700 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
7704 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
7706 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7709 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7711 for (i
= fd
->collapse
- 1; i
> 0; i
--)
7713 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7714 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
7715 if (POINTER_TYPE_P (itype2
))
7716 itype2
= signed_type_for (itype2
);
7717 t
= build3 (COND_EXPR
, itype2
,
7718 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7720 fold_convert (itype
, fd
->loops
[i
].n2
)),
7721 build_int_cst (itype2
, 0),
7722 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
7723 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
7724 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
7726 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
7727 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
7729 t
= build3 (COND_EXPR
, itype
,
7730 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7732 fold_convert (itype
, fd
->loops
[i
].n2
)),
7734 fold_convert (itype
, fd
->loops
[i
].n1
));
7735 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7739 /* Remove GIMPLE_OMP_CONTINUE. */
7740 gsi_remove (&gsi
, true);
7743 /* Emit the condition in L1_BB. */
7744 gsi
= gsi_start_bb (l1_bb
);
7746 t
= fold_convert (type
, n2
);
7747 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7748 false, GSI_CONTINUE_LINKING
);
7749 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
7750 cond_stmt
= gimple_build_cond_empty (t
);
7751 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
7752 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
7754 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
7757 gsi
= gsi_for_stmt (cond_stmt
);
7758 gimple_regimplify_operands (cond_stmt
, &gsi
);
7761 /* Remove GIMPLE_OMP_RETURN. */
7762 gsi
= gsi_last_bb (exit_bb
);
7763 gsi_remove (&gsi
, true);
7765 /* Connect the new blocks. */
7766 remove_edge (FALLTHRU_EDGE (entry_bb
));
7770 remove_edge (BRANCH_EDGE (entry_bb
));
7771 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7773 e
= BRANCH_EDGE (l1_bb
);
7774 ne
= FALLTHRU_EDGE (l1_bb
);
7775 e
->flags
= EDGE_TRUE_VALUE
;
7779 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7781 ne
= single_succ_edge (l1_bb
);
7782 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7785 ne
->flags
= EDGE_FALSE_VALUE
;
7786 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7787 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7789 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7790 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7791 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7795 struct loop
*loop
= alloc_loop ();
7796 loop
->header
= l1_bb
;
7797 loop
->latch
= cont_bb
;
7798 add_loop (loop
, l1_bb
->loop_father
);
7799 if (safelen
== NULL_TREE
)
7800 loop
->safelen
= INT_MAX
;
7803 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
7804 if (TREE_CODE (safelen
) != INTEGER_CST
)
7806 else if (!tree_fits_uhwi_p (safelen
)
7807 || tree_to_uhwi (safelen
) > INT_MAX
)
7808 loop
->safelen
= INT_MAX
;
7810 loop
->safelen
= tree_to_uhwi (safelen
);
7811 if (loop
->safelen
== 1)
7816 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
7817 cfun
->has_simduid_loops
= true;
7819 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7821 if ((flag_tree_loop_vectorize
7822 || (!global_options_set
.x_flag_tree_loop_vectorize
7823 && !global_options_set
.x_flag_tree_vectorize
))
7824 && flag_tree_loop_optimize
7825 && loop
->safelen
> 1)
7827 loop
->force_vectorize
= true;
7828 cfun
->has_force_vectorize_loops
= true;
7834 /* Expand the OMP loop defined by REGION. */
7837 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
7839 struct omp_for_data fd
;
7840 struct omp_for_data_loop
*loops
;
7843 = (struct omp_for_data_loop
*)
7844 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
7845 * sizeof (struct omp_for_data_loop
));
7846 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
7848 region
->sched_kind
= fd
.sched_kind
;
7850 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
7851 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7852 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7855 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
7856 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7857 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7860 /* If there isn't a continue then this is a degerate case where
7861 the introduction of abnormal edges during lowering will prevent
7862 original loops from being detected. Fix that up. */
7863 loops_state_set (LOOPS_NEED_FIXUP
);
7865 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
7866 expand_omp_simd (region
, &fd
);
7867 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
7868 expand_cilk_for (region
, &fd
);
7869 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
7870 && !fd
.have_ordered
)
7872 if (fd
.chunk_size
== NULL
)
7873 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
7875 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
7879 int fn_index
, start_ix
, next_ix
;
7881 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
7882 == GF_OMP_FOR_KIND_FOR
);
7883 if (fd
.chunk_size
== NULL
7884 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
7885 fd
.chunk_size
= integer_zero_node
;
7886 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
7887 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
7888 ? 3 : fd
.sched_kind
;
7889 fn_index
+= fd
.have_ordered
* 4;
7890 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
7891 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
7892 if (fd
.iter_type
== long_long_unsigned_type_node
)
7894 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7895 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
7896 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7897 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
7899 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
7900 (enum built_in_function
) next_ix
, inner_stmt
);
7903 if (gimple_in_ssa_p (cfun
))
7904 update_ssa (TODO_update_ssa_only_virtuals
);
7908 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7910 v = GOMP_sections_start (n);
7927 v = GOMP_sections_next ();
7932 If this is a combined parallel sections, replace the call to
7933 GOMP_sections_start with call to GOMP_sections_next. */
7936 expand_omp_sections (struct omp_region
*region
)
7938 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7940 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7941 gimple_stmt_iterator si
, switch_si
;
7942 gomp_sections
*sections_stmt
;
7944 gomp_continue
*cont
;
7947 struct omp_region
*inner
;
7949 bool exit_reachable
= region
->cont
!= NULL
;
7951 gcc_assert (region
->exit
!= NULL
);
7952 entry_bb
= region
->entry
;
7953 l0_bb
= single_succ (entry_bb
);
7954 l1_bb
= region
->cont
;
7955 l2_bb
= region
->exit
;
7956 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7957 l2
= gimple_block_label (l2_bb
);
7960 /* This can happen if there are reductions. */
7961 len
= EDGE_COUNT (l0_bb
->succs
);
7962 gcc_assert (len
> 0);
7963 e
= EDGE_SUCC (l0_bb
, len
- 1);
7964 si
= gsi_last_bb (e
->dest
);
7967 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7968 l2
= gimple_block_label (e
->dest
);
7970 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7972 si
= gsi_last_bb (e
->dest
);
7974 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7976 l2
= gimple_block_label (e
->dest
);
7982 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7984 default_bb
= create_empty_bb (l0_bb
);
7986 /* We will build a switch() with enough cases for all the
7987 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7988 and a default case to abort if something goes wrong. */
7989 len
= EDGE_COUNT (l0_bb
->succs
);
7991 /* Use vec::quick_push on label_vec throughout, since we know the size
7993 auto_vec
<tree
> label_vec (len
);
7995 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7996 GIMPLE_OMP_SECTIONS statement. */
7997 si
= gsi_last_bb (entry_bb
);
7998 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
7999 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
8000 vin
= gimple_omp_sections_control (sections_stmt
);
8001 if (!is_combined_parallel (region
))
8003 /* If we are not inside a combined parallel+sections region,
8004 call GOMP_sections_start. */
8005 t
= build_int_cst (unsigned_type_node
, len
- 1);
8006 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
8007 stmt
= gimple_build_call (u
, 1, t
);
8011 /* Otherwise, call GOMP_sections_next. */
8012 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
8013 stmt
= gimple_build_call (u
, 0);
8015 gimple_call_set_lhs (stmt
, vin
);
8016 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8017 gsi_remove (&si
, true);
8019 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
8021 switch_si
= gsi_last_bb (l0_bb
);
8022 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
8025 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
8026 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
8027 vmain
= gimple_omp_continue_control_use (cont
);
8028 vnext
= gimple_omp_continue_control_def (cont
);
8036 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
8037 label_vec
.quick_push (t
);
8040 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
8041 for (inner
= region
->inner
, casei
= 1;
8043 inner
= inner
->next
, i
++, casei
++)
8045 basic_block s_entry_bb
, s_exit_bb
;
8047 /* Skip optional reduction region. */
8048 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
8055 s_entry_bb
= inner
->entry
;
8056 s_exit_bb
= inner
->exit
;
8058 t
= gimple_block_label (s_entry_bb
);
8059 u
= build_int_cst (unsigned_type_node
, casei
);
8060 u
= build_case_label (u
, NULL
, t
);
8061 label_vec
.quick_push (u
);
8063 si
= gsi_last_bb (s_entry_bb
);
8064 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
8065 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
8066 gsi_remove (&si
, true);
8067 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
8069 if (s_exit_bb
== NULL
)
8072 si
= gsi_last_bb (s_exit_bb
);
8073 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8074 gsi_remove (&si
, true);
8076 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
8079 /* Error handling code goes in DEFAULT_BB. */
8080 t
= gimple_block_label (default_bb
);
8081 u
= build_case_label (NULL
, NULL
, t
);
8082 make_edge (l0_bb
, default_bb
, 0);
8083 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
8085 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
8086 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
8087 gsi_remove (&switch_si
, true);
8089 si
= gsi_start_bb (default_bb
);
8090 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
8091 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
8097 /* Code to get the next section goes in L1_BB. */
8098 si
= gsi_last_bb (l1_bb
);
8099 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
8101 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
8102 stmt
= gimple_build_call (bfn_decl
, 0);
8103 gimple_call_set_lhs (stmt
, vnext
);
8104 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8105 gsi_remove (&si
, true);
8107 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
8110 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
8111 si
= gsi_last_bb (l2_bb
);
8112 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
8113 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
8114 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
8115 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
8117 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
8118 stmt
= gimple_build_call (t
, 0);
8119 if (gimple_omp_return_lhs (gsi_stmt (si
)))
8120 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
8121 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8122 gsi_remove (&si
, true);
8124 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
8128 /* Expand code for an OpenMP single directive. We've already expanded
8129 much of the code, here we simply place the GOMP_barrier call. */
8132 expand_omp_single (struct omp_region
*region
)
8134 basic_block entry_bb
, exit_bb
;
8135 gimple_stmt_iterator si
;
8137 entry_bb
= region
->entry
;
8138 exit_bb
= region
->exit
;
8140 si
= gsi_last_bb (entry_bb
);
8141 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
8142 gsi_remove (&si
, true);
8143 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8145 si
= gsi_last_bb (exit_bb
);
8146 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
8148 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
8149 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
8151 gsi_remove (&si
, true);
8152 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8156 /* Generic expansion for OpenMP synchronization directives: master,
8157 ordered and critical. All we need to do here is remove the entry
8158 and exit markers for REGION. */
8161 expand_omp_synch (struct omp_region
*region
)
8163 basic_block entry_bb
, exit_bb
;
8164 gimple_stmt_iterator si
;
8166 entry_bb
= region
->entry
;
8167 exit_bb
= region
->exit
;
8169 si
= gsi_last_bb (entry_bb
);
8170 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
8171 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
8172 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
8173 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
8174 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
8175 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
8176 gsi_remove (&si
, true);
8177 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8181 si
= gsi_last_bb (exit_bb
);
8182 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8183 gsi_remove (&si
, true);
8184 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8188 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8189 operation as a normal volatile load. */
8192 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
8193 tree loaded_val
, int index
)
8195 enum built_in_function tmpbase
;
8196 gimple_stmt_iterator gsi
;
8197 basic_block store_bb
;
8200 tree decl
, call
, type
, itype
;
8202 gsi
= gsi_last_bb (load_bb
);
8203 stmt
= gsi_stmt (gsi
);
8204 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8205 loc
= gimple_location (stmt
);
8207 /* ??? If the target does not implement atomic_load_optab[mode], and mode
8208 is smaller than word size, then expand_atomic_load assumes that the load
8209 is atomic. We could avoid the builtin entirely in this case. */
8211 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8212 decl
= builtin_decl_explicit (tmpbase
);
8213 if (decl
== NULL_TREE
)
8216 type
= TREE_TYPE (loaded_val
);
8217 itype
= TREE_TYPE (TREE_TYPE (decl
));
8219 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
8220 build_int_cst (NULL
,
8221 gimple_omp_atomic_seq_cst_p (stmt
)
8223 : MEMMODEL_RELAXED
));
8224 if (!useless_type_conversion_p (type
, itype
))
8225 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8226 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8228 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8229 gsi_remove (&gsi
, true);
8231 store_bb
= single_succ (load_bb
);
8232 gsi
= gsi_last_bb (store_bb
);
8233 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8234 gsi_remove (&gsi
, true);
8236 if (gimple_in_ssa_p (cfun
))
8237 update_ssa (TODO_update_ssa_no_phi
);
8242 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8243 operation as a normal volatile store. */
8246 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
8247 tree loaded_val
, tree stored_val
, int index
)
8249 enum built_in_function tmpbase
;
8250 gimple_stmt_iterator gsi
;
8251 basic_block store_bb
= single_succ (load_bb
);
8254 tree decl
, call
, type
, itype
;
8258 gsi
= gsi_last_bb (load_bb
);
8259 stmt
= gsi_stmt (gsi
);
8260 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8262 /* If the load value is needed, then this isn't a store but an exchange. */
8263 exchange
= gimple_omp_atomic_need_value_p (stmt
);
8265 gsi
= gsi_last_bb (store_bb
);
8266 stmt
= gsi_stmt (gsi
);
8267 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
8268 loc
= gimple_location (stmt
);
8270 /* ??? If the target does not implement atomic_store_optab[mode], and mode
8271 is smaller than word size, then expand_atomic_store assumes that the store
8272 is atomic. We could avoid the builtin entirely in this case. */
8274 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
8275 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
8276 decl
= builtin_decl_explicit (tmpbase
);
8277 if (decl
== NULL_TREE
)
8280 type
= TREE_TYPE (stored_val
);
8282 /* Dig out the type of the function's second argument. */
8283 itype
= TREE_TYPE (decl
);
8284 itype
= TYPE_ARG_TYPES (itype
);
8285 itype
= TREE_CHAIN (itype
);
8286 itype
= TREE_VALUE (itype
);
8287 imode
= TYPE_MODE (itype
);
8289 if (exchange
&& !can_atomic_exchange_p (imode
, true))
8292 if (!useless_type_conversion_p (itype
, type
))
8293 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
8294 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
8295 build_int_cst (NULL
,
8296 gimple_omp_atomic_seq_cst_p (stmt
)
8298 : MEMMODEL_RELAXED
));
8301 if (!useless_type_conversion_p (type
, itype
))
8302 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8303 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8306 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8307 gsi_remove (&gsi
, true);
8309 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
8310 gsi
= gsi_last_bb (load_bb
);
8311 gsi_remove (&gsi
, true);
8313 if (gimple_in_ssa_p (cfun
))
8314 update_ssa (TODO_update_ssa_no_phi
);
8319 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8320 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
8321 size of the data type, and thus usable to find the index of the builtin
8322 decl. Returns false if the expression is not of the proper form. */
8325 expand_omp_atomic_fetch_op (basic_block load_bb
,
8326 tree addr
, tree loaded_val
,
8327 tree stored_val
, int index
)
8329 enum built_in_function oldbase
, newbase
, tmpbase
;
8330 tree decl
, itype
, call
;
8332 basic_block store_bb
= single_succ (load_bb
);
8333 gimple_stmt_iterator gsi
;
8336 enum tree_code code
;
8337 bool need_old
, need_new
;
8341 /* We expect to find the following sequences:
8344 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
8347 val = tmp OP something; (or: something OP tmp)
8348 GIMPLE_OMP_STORE (val)
8350 ???FIXME: Allow a more flexible sequence.
8351 Perhaps use data flow to pick the statements.
8355 gsi
= gsi_after_labels (store_bb
);
8356 stmt
= gsi_stmt (gsi
);
8357 loc
= gimple_location (stmt
);
8358 if (!is_gimple_assign (stmt
))
8361 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
8363 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
8364 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
8365 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
8366 gcc_checking_assert (!need_old
|| !need_new
);
8368 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
8371 /* Check for one of the supported fetch-op operations. */
8372 code
= gimple_assign_rhs_code (stmt
);
8376 case POINTER_PLUS_EXPR
:
8377 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
8378 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
8381 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
8382 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
8385 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
8386 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
8389 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
8390 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
8393 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
8394 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
8400 /* Make sure the expression is of the proper form. */
8401 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
8402 rhs
= gimple_assign_rhs2 (stmt
);
8403 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
8404 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
8405 rhs
= gimple_assign_rhs1 (stmt
);
8409 tmpbase
= ((enum built_in_function
)
8410 ((need_new
? newbase
: oldbase
) + index
+ 1));
8411 decl
= builtin_decl_explicit (tmpbase
);
8412 if (decl
== NULL_TREE
)
8414 itype
= TREE_TYPE (TREE_TYPE (decl
));
8415 imode
= TYPE_MODE (itype
);
8417 /* We could test all of the various optabs involved, but the fact of the
8418 matter is that (with the exception of i486 vs i586 and xadd) all targets
8419 that support any atomic operaton optab also implements compare-and-swap.
8420 Let optabs.c take care of expanding any compare-and-swap loop. */
8421 if (!can_compare_and_swap_p (imode
, true))
8424 gsi
= gsi_last_bb (load_bb
);
8425 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8427 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
8428 It only requires that the operation happen atomically. Thus we can
8429 use the RELAXED memory model. */
8430 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
8431 fold_convert_loc (loc
, itype
, rhs
),
8432 build_int_cst (NULL
,
8433 seq_cst
? MEMMODEL_SEQ_CST
8434 : MEMMODEL_RELAXED
));
8436 if (need_old
|| need_new
)
8438 lhs
= need_old
? loaded_val
: stored_val
;
8439 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
8440 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
8443 call
= fold_convert_loc (loc
, void_type_node
, call
);
8444 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8445 gsi_remove (&gsi
, true);
8447 gsi
= gsi_last_bb (store_bb
);
8448 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8449 gsi_remove (&gsi
, true);
8450 gsi
= gsi_last_bb (store_bb
);
8451 gsi_remove (&gsi
, true);
8453 if (gimple_in_ssa_p (cfun
))
8454 update_ssa (TODO_update_ssa_no_phi
);
8459 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8463 newval = rhs; // with oldval replacing *addr in rhs
8464 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
8465 if (oldval != newval)
8468 INDEX is log2 of the size of the data type, and thus usable to find the
8469 index of the builtin decl. */
8472 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
8473 tree addr
, tree loaded_val
, tree stored_val
,
8476 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
8477 tree type
, itype
, cmpxchg
, iaddr
;
8478 gimple_stmt_iterator si
;
8479 basic_block loop_header
= single_succ (load_bb
);
8482 enum built_in_function fncode
;
8484 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
8485 order to use the RELAXED memory model effectively. */
8486 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
8488 cmpxchg
= builtin_decl_explicit (fncode
);
8489 if (cmpxchg
== NULL_TREE
)
8491 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8492 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
8494 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
8497 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
8498 si
= gsi_last_bb (load_bb
);
8499 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8501 /* For floating-point values, we'll need to view-convert them to integers
8502 so that we can perform the atomic compare and swap. Simplify the
8503 following code by always setting up the "i"ntegral variables. */
8504 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
8508 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
8511 = force_gimple_operand_gsi (&si
,
8512 fold_convert (TREE_TYPE (iaddr
), addr
),
8513 false, NULL_TREE
, true, GSI_SAME_STMT
);
8514 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
8515 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8516 loadedi
= create_tmp_var (itype
);
8517 if (gimple_in_ssa_p (cfun
))
8518 loadedi
= make_ssa_name (loadedi
);
8523 loadedi
= loaded_val
;
8526 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8527 tree loaddecl
= builtin_decl_explicit (fncode
);
8530 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
8531 build_call_expr (loaddecl
, 2, iaddr
,
8532 build_int_cst (NULL_TREE
,
8533 MEMMODEL_RELAXED
)));
8535 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
8536 build_int_cst (TREE_TYPE (iaddr
), 0));
8539 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
8542 /* Move the value to the LOADEDI temporary. */
8543 if (gimple_in_ssa_p (cfun
))
8545 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
8546 phi
= create_phi_node (loadedi
, loop_header
);
8547 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
8551 gsi_insert_before (&si
,
8552 gimple_build_assign (loadedi
, initial
),
8554 if (loadedi
!= loaded_val
)
8556 gimple_stmt_iterator gsi2
;
8559 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
8560 gsi2
= gsi_start_bb (loop_header
);
8561 if (gimple_in_ssa_p (cfun
))
8564 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8565 true, GSI_SAME_STMT
);
8566 stmt
= gimple_build_assign (loaded_val
, x
);
8567 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
8571 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
8572 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8573 true, GSI_SAME_STMT
);
8576 gsi_remove (&si
, true);
8578 si
= gsi_last_bb (store_bb
);
8579 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8582 storedi
= stored_val
;
8585 force_gimple_operand_gsi (&si
,
8586 build1 (VIEW_CONVERT_EXPR
, itype
,
8587 stored_val
), true, NULL_TREE
, true,
8590 /* Build the compare&swap statement. */
8591 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
8592 new_storedi
= force_gimple_operand_gsi (&si
,
8593 fold_convert (TREE_TYPE (loadedi
),
8596 true, GSI_SAME_STMT
);
8598 if (gimple_in_ssa_p (cfun
))
8602 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
8603 stmt
= gimple_build_assign (old_vali
, loadedi
);
8604 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8606 stmt
= gimple_build_assign (loadedi
, new_storedi
);
8607 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8610 /* Note that we always perform the comparison as an integer, even for
8611 floating point. This allows the atomic operation to properly
8612 succeed even with NaNs and -0.0. */
8613 stmt
= gimple_build_cond_empty
8614 (build2 (NE_EXPR
, boolean_type_node
,
8615 new_storedi
, old_vali
));
8616 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8619 e
= single_succ_edge (store_bb
);
8620 e
->flags
&= ~EDGE_FALLTHRU
;
8621 e
->flags
|= EDGE_FALSE_VALUE
;
8623 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
8625 /* Copy the new value to loadedi (we already did that before the condition
8626 if we are not in SSA). */
8627 if (gimple_in_ssa_p (cfun
))
8629 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
8630 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
8633 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
8634 gsi_remove (&si
, true);
8636 struct loop
*loop
= alloc_loop ();
8637 loop
->header
= loop_header
;
8638 loop
->latch
= store_bb
;
8639 add_loop (loop
, loop_header
->loop_father
);
8641 if (gimple_in_ssa_p (cfun
))
8642 update_ssa (TODO_update_ssa_no_phi
);
8647 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8649 GOMP_atomic_start ();
8653 The result is not globally atomic, but works so long as all parallel
8654 references are within #pragma omp atomic directives. According to
8655 responses received from omp@openmp.org, appears to be within spec.
8656 Which makes sense, since that's how several other compilers handle
8657 this situation as well.
8658 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
8659 expanding. STORED_VAL is the operand of the matching
8660 GIMPLE_OMP_ATOMIC_STORE.
8663 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
8667 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
8672 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
8673 tree addr
, tree loaded_val
, tree stored_val
)
8675 gimple_stmt_iterator si
;
8679 si
= gsi_last_bb (load_bb
);
8680 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8682 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
8683 t
= build_call_expr (t
, 0);
8684 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8686 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
8687 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8688 gsi_remove (&si
, true);
8690 si
= gsi_last_bb (store_bb
);
8691 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8693 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
8695 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8697 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
8698 t
= build_call_expr (t
, 0);
8699 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8700 gsi_remove (&si
, true);
8702 if (gimple_in_ssa_p (cfun
))
8703 update_ssa (TODO_update_ssa_no_phi
);
8707 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
8708 using expand_omp_atomic_fetch_op. If it failed, we try to
8709 call expand_omp_atomic_pipeline, and if it fails too, the
8710 ultimate fallback is wrapping the operation in a mutex
8711 (expand_omp_atomic_mutex). REGION is the atomic region built
8712 by build_omp_regions_1(). */
8715 expand_omp_atomic (struct omp_region
*region
)
8717 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
8718 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
8719 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
8720 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
8721 tree addr
= gimple_omp_atomic_load_rhs (load
);
8722 tree stored_val
= gimple_omp_atomic_store_val (store
);
8723 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8724 HOST_WIDE_INT index
;
8726 /* Make sure the type is one of the supported sizes. */
8727 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
8728 index
= exact_log2 (index
);
8729 if (index
>= 0 && index
<= 4)
8731 unsigned int align
= TYPE_ALIGN_UNIT (type
);
8733 /* __sync builtins require strict data alignment. */
8734 if (exact_log2 (align
) >= index
)
8737 if (loaded_val
== stored_val
8738 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8739 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8740 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8741 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
8745 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8746 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8747 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8748 && store_bb
== single_succ (load_bb
)
8749 && first_stmt (store_bb
) == store
8750 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
8754 /* When possible, use specialized atomic update functions. */
8755 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
8756 && store_bb
== single_succ (load_bb
)
8757 && expand_omp_atomic_fetch_op (load_bb
, addr
,
8758 loaded_val
, stored_val
, index
))
8761 /* If we don't have specialized __sync builtins, try and implement
8762 as a compare and swap loop. */
8763 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
8764 loaded_val
, stored_val
, index
))
8769 /* The ultimate fallback is wrapping the operation in a mutex. */
8770 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
8774 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
8777 expand_omp_target (struct omp_region
*region
)
8779 basic_block entry_bb
, exit_bb
, new_bb
;
8780 struct function
*child_cfun
;
8781 tree child_fn
, block
, t
;
8782 gimple_stmt_iterator gsi
;
8783 gomp_target
*entry_stmt
;
8786 bool offloaded
, data_region
;
8788 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
8789 new_bb
= region
->entry
;
8791 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
8792 switch (gimple_omp_target_kind (entry_stmt
))
8794 case GF_OMP_TARGET_KIND_REGION
:
8795 case GF_OMP_TARGET_KIND_UPDATE
:
8796 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
8797 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
8798 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
8799 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
8800 data_region
= false;
8802 case GF_OMP_TARGET_KIND_DATA
:
8803 case GF_OMP_TARGET_KIND_OACC_DATA
:
8810 child_fn
= NULL_TREE
;
8814 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
8815 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
8818 /* Supported by expand_omp_taskreg, but not here. */
8819 if (child_cfun
!= NULL
)
8820 gcc_checking_assert (!child_cfun
->cfg
);
8821 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
8823 entry_bb
= region
->entry
;
8824 exit_bb
= region
->exit
;
8828 unsigned srcidx
, dstidx
, num
;
8830 /* If the offloading region needs data sent from the parent
8831 function, then the very first statement (except possible
8832 tree profile counter updates) of the offloading body
8833 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
8834 &.OMP_DATA_O is passed as an argument to the child function,
8835 we need to replace it with the argument as seen by the child
8838 In most cases, this will end up being the identity assignment
8839 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
8840 a function call that has been inlined, the original PARM_DECL
8841 .OMP_DATA_I may have been converted into a different local
8842 variable. In which case, we need to keep the assignment. */
8843 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
8846 basic_block entry_succ_bb
= single_succ (entry_bb
);
8847 gimple_stmt_iterator gsi
;
8849 gimple tgtcopy_stmt
= NULL
;
8850 tree sender
= TREE_VEC_ELT (data_arg
, 0);
8852 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
8854 gcc_assert (!gsi_end_p (gsi
));
8855 stmt
= gsi_stmt (gsi
);
8856 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
8859 if (gimple_num_ops (stmt
) == 2)
8861 tree arg
= gimple_assign_rhs1 (stmt
);
8863 /* We're ignoring the subcode because we're
8864 effectively doing a STRIP_NOPS. */
8866 if (TREE_CODE (arg
) == ADDR_EXPR
8867 && TREE_OPERAND (arg
, 0) == sender
)
8869 tgtcopy_stmt
= stmt
;
8875 gcc_assert (tgtcopy_stmt
!= NULL
);
8876 arg
= DECL_ARGUMENTS (child_fn
);
8878 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
8879 gsi_remove (&gsi
, true);
8882 /* Declare local variables needed in CHILD_CFUN. */
8883 block
= DECL_INITIAL (child_fn
);
8884 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
8885 /* The gimplifier could record temporaries in the offloading block
8886 rather than in containing function's local_decls chain,
8887 which would mean cgraph missed finalizing them. Do it now. */
8888 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
8889 if (TREE_CODE (t
) == VAR_DECL
8891 && !DECL_EXTERNAL (t
))
8892 varpool_node::finalize_decl (t
);
8893 DECL_SAVED_TREE (child_fn
) = NULL
;
8894 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8895 gimple_set_body (child_fn
, NULL
);
8896 TREE_USED (block
) = 1;
8898 /* Reset DECL_CONTEXT on function arguments. */
8899 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
8900 DECL_CONTEXT (t
) = child_fn
;
8902 /* Split ENTRY_BB at GIMPLE_*,
8903 so that it can be moved to the child function. */
8904 gsi
= gsi_last_bb (entry_bb
);
8905 stmt
= gsi_stmt (gsi
);
8907 && gimple_code (stmt
) == gimple_code (entry_stmt
));
8908 gsi_remove (&gsi
, true);
8909 e
= split_block (entry_bb
, stmt
);
8911 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8913 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8916 gsi
= gsi_last_bb (exit_bb
);
8917 gcc_assert (!gsi_end_p (gsi
)
8918 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8919 stmt
= gimple_build_return (NULL
);
8920 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8921 gsi_remove (&gsi
, true);
8924 /* Move the offloading region into CHILD_CFUN. */
8926 block
= gimple_block (entry_stmt
);
8928 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
8930 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8931 /* When the OMP expansion process cannot guarantee an up-to-date
8932 loop tree arrange for the child function to fixup loops. */
8933 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8934 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8936 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8937 num
= vec_safe_length (child_cfun
->local_decls
);
8938 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8940 t
= (*child_cfun
->local_decls
)[srcidx
];
8941 if (DECL_CONTEXT (t
) == cfun
->decl
)
8943 if (srcidx
!= dstidx
)
8944 (*child_cfun
->local_decls
)[dstidx
] = t
;
8948 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8950 /* Inform the callgraph about the new function. */
8951 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
8952 cgraph_node::add_new_function (child_fn
, true);
8954 #ifdef ENABLE_OFFLOADING
8955 /* Add the new function to the offload table. */
8956 vec_safe_push (offload_funcs
, child_fn
);
8959 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8960 fixed in a following pass. */
8961 push_cfun (child_cfun
);
8962 cgraph_edge::rebuild_edges ();
8964 #ifdef ENABLE_OFFLOADING
8965 /* Prevent IPA from removing child_fn as unreachable, since there are no
8966 refs from the parent function to child_fn in offload LTO mode. */
8967 struct cgraph_node
*node
= cgraph_node::get (child_fn
);
8968 node
->mark_force_output ();
8971 /* Some EH regions might become dead, see PR34608. If
8972 pass_cleanup_cfg isn't the first pass to happen with the
8973 new child, these dead EH edges might cause problems.
8974 Clean them up now. */
8975 if (flag_exceptions
)
8978 bool changed
= false;
8980 FOR_EACH_BB_FN (bb
, cfun
)
8981 changed
|= gimple_purge_dead_eh_edges (bb
);
8983 cleanup_tree_cfg ();
8988 /* Emit a library call to launch the offloading region, or do data
8990 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8991 enum built_in_function start_ix
;
8992 location_t clause_loc
;
8994 switch (gimple_omp_target_kind (entry_stmt
))
8996 case GF_OMP_TARGET_KIND_REGION
:
8997 start_ix
= BUILT_IN_GOMP_TARGET
;
8999 case GF_OMP_TARGET_KIND_DATA
:
9000 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
9002 case GF_OMP_TARGET_KIND_UPDATE
:
9003 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
9005 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
9006 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
9007 start_ix
= BUILT_IN_GOACC_PARALLEL
;
9009 case GF_OMP_TARGET_KIND_OACC_DATA
:
9010 start_ix
= BUILT_IN_GOACC_DATA_START
;
9012 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
9013 start_ix
= BUILT_IN_GOACC_UPDATE
;
9015 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9016 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
9022 clauses
= gimple_omp_target_clauses (entry_stmt
);
9024 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
9025 library choose) and there is no conditional. */
9027 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
9029 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
9031 cond
= OMP_CLAUSE_IF_EXPR (c
);
9033 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
9036 /* Even if we pass it to all library function calls, it is currently only
9037 defined/used for the OpenMP target ones. */
9038 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
9039 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
9040 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
);
9042 device
= OMP_CLAUSE_DEVICE_ID (c
);
9043 clause_loc
= OMP_CLAUSE_LOCATION (c
);
9046 clause_loc
= gimple_location (entry_stmt
);
9048 /* Ensure 'device' is of the correct type. */
9049 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
9051 /* If we found the clause 'if (cond)', build
9052 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
9055 cond
= gimple_boolify (cond
);
9057 basic_block cond_bb
, then_bb
, else_bb
;
9061 tmp_var
= create_tmp_var (TREE_TYPE (device
));
9063 e
= split_block (new_bb
, NULL
);
9066 gsi
= gsi_last_bb (new_bb
);
9068 e
= split_block (new_bb
, gsi_stmt (gsi
));
9074 then_bb
= create_empty_bb (cond_bb
);
9075 else_bb
= create_empty_bb (then_bb
);
9076 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
9077 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
9079 stmt
= gimple_build_cond_empty (cond
);
9080 gsi
= gsi_last_bb (cond_bb
);
9081 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9083 gsi
= gsi_start_bb (then_bb
);
9084 stmt
= gimple_build_assign (tmp_var
, device
);
9085 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9087 gsi
= gsi_start_bb (else_bb
);
9088 stmt
= gimple_build_assign (tmp_var
,
9089 build_int_cst (integer_type_node
,
9090 GOMP_DEVICE_HOST_FALLBACK
));
9091 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9093 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
9094 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
9095 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
9096 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
9097 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
9098 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
9103 gsi
= gsi_last_bb (new_bb
);
9104 t
= gimple_omp_target_data_arg (entry_stmt
);
9107 t1
= size_zero_node
;
9108 t2
= build_zero_cst (ptr_type_node
);
9114 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
9115 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
9116 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
9117 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
9118 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
9122 tree offload_table
= get_offload_symbol_decl ();
9124 /* The maximum number used by any start_ix, without varargs. */
9125 unsigned int argcnt
= 12;
9127 vec_alloc (args
, argcnt
);
9128 args
->quick_push (device
);
9130 args
->quick_push (build_fold_addr_expr (child_fn
));
9131 args
->quick_push (build_fold_addr_expr (offload_table
));
9132 args
->quick_push (t1
);
9133 args
->quick_push (t2
);
9134 args
->quick_push (t3
);
9135 args
->quick_push (t4
);
9138 case BUILT_IN_GOACC_DATA_START
:
9139 case BUILT_IN_GOMP_TARGET
:
9140 case BUILT_IN_GOMP_TARGET_DATA
:
9141 case BUILT_IN_GOMP_TARGET_UPDATE
:
9143 case BUILT_IN_GOACC_PARALLEL
:
9145 tree t_num_gangs
, t_num_workers
, t_vector_length
;
9147 /* Default values for num_gangs, num_workers, and vector_length. */
9148 t_num_gangs
= t_num_workers
= t_vector_length
9149 = fold_convert_loc (gimple_location (entry_stmt
),
9150 integer_type_node
, integer_one_node
);
9151 /* ..., but if present, use the value specified by the respective
9152 clause, making sure that are of the correct type. */
9153 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
9155 t_num_gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9157 OMP_CLAUSE_NUM_GANGS_EXPR (c
));
9158 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_WORKERS
);
9160 t_num_workers
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9162 OMP_CLAUSE_NUM_WORKERS_EXPR (c
));
9163 c
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
9165 t_vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9167 OMP_CLAUSE_VECTOR_LENGTH_EXPR (c
));
9168 args
->quick_push (t_num_gangs
);
9169 args
->quick_push (t_num_workers
);
9170 args
->quick_push (t_vector_length
);
9173 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
9174 case BUILT_IN_GOACC_UPDATE
:
9179 /* Default values for t_async. */
9180 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
9182 build_int_cst (integer_type_node
,
9184 /* ..., but if present, use the value specified by the respective
9185 clause, making sure that is of the correct type. */
9186 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
9188 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9190 OMP_CLAUSE_ASYNC_EXPR (c
));
9192 args
->quick_push (t_async
);
9193 /* Save the index, and... */
9194 t_wait_idx
= args
->length ();
9195 /* ... push a default value. */
9196 args
->quick_push (fold_convert_loc (gimple_location (entry_stmt
),
9198 integer_zero_node
));
9199 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
9204 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
9206 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
9208 args
->safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9210 OMP_CLAUSE_WAIT_EXPR (c
)));
9215 /* Now that we know the number, replace the default value. */
9216 args
->ordered_remove (t_wait_idx
);
9217 args
->quick_insert (t_wait_idx
,
9218 fold_convert_loc (gimple_location (entry_stmt
),
9220 build_int_cst (integer_type_node
, n
)));
9228 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), *args
);
9230 gimple_set_location (g
, gimple_location (entry_stmt
));
9231 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
9235 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
9236 gsi_remove (&gsi
, true);
9241 gsi
= gsi_last_bb (region
->exit
);
9243 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
9244 gsi_remove (&gsi
, true);
9249 /* Expand the parallel region tree rooted at REGION. Expansion
9250 proceeds in depth-first order. Innermost regions are expanded
9251 first. This way, parallel regions that require a new function to
9252 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
9253 internal dependencies in their body. */
9256 expand_omp (struct omp_region
*region
)
9260 location_t saved_location
;
9261 gimple inner_stmt
= NULL
;
9263 /* First, determine whether this is a combined parallel+workshare
9265 if (region
->type
== GIMPLE_OMP_PARALLEL
)
9266 determine_parallel_type (region
);
9268 if (region
->type
== GIMPLE_OMP_FOR
9269 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
9270 inner_stmt
= last_stmt (region
->inner
->entry
);
9273 expand_omp (region
->inner
);
9275 saved_location
= input_location
;
9276 if (gimple_has_location (last_stmt (region
->entry
)))
9277 input_location
= gimple_location (last_stmt (region
->entry
));
9279 switch (region
->type
)
9281 case GIMPLE_OMP_PARALLEL
:
9282 case GIMPLE_OMP_TASK
:
9283 expand_omp_taskreg (region
);
9286 case GIMPLE_OMP_FOR
:
9287 expand_omp_for (region
, inner_stmt
);
9290 case GIMPLE_OMP_SECTIONS
:
9291 expand_omp_sections (region
);
9294 case GIMPLE_OMP_SECTION
:
9295 /* Individual omp sections are handled together with their
9296 parent GIMPLE_OMP_SECTIONS region. */
9299 case GIMPLE_OMP_SINGLE
:
9300 expand_omp_single (region
);
9303 case GIMPLE_OMP_MASTER
:
9304 case GIMPLE_OMP_TASKGROUP
:
9305 case GIMPLE_OMP_ORDERED
:
9306 case GIMPLE_OMP_CRITICAL
:
9307 case GIMPLE_OMP_TEAMS
:
9308 expand_omp_synch (region
);
9311 case GIMPLE_OMP_ATOMIC_LOAD
:
9312 expand_omp_atomic (region
);
9315 case GIMPLE_OMP_TARGET
:
9316 expand_omp_target (region
);
9323 input_location
= saved_location
;
9324 region
= region
->next
;
9329 /* Helper for build_omp_regions. Scan the dominator tree starting at
9330 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
9331 true, the function ends once a single tree is built (otherwise, whole
9332 forest of OMP constructs may be built). */
9335 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
9338 gimple_stmt_iterator gsi
;
9342 gsi
= gsi_last_bb (bb
);
9343 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
9345 struct omp_region
*region
;
9346 enum gimple_code code
;
9348 stmt
= gsi_stmt (gsi
);
9349 code
= gimple_code (stmt
);
9350 if (code
== GIMPLE_OMP_RETURN
)
9352 /* STMT is the return point out of region PARENT. Mark it
9353 as the exit point and make PARENT the immediately
9354 enclosing region. */
9355 gcc_assert (parent
);
9358 parent
= parent
->outer
;
9360 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
9362 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
9363 GIMPLE_OMP_RETURN, but matches with
9364 GIMPLE_OMP_ATOMIC_LOAD. */
9365 gcc_assert (parent
);
9366 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
9369 parent
= parent
->outer
;
9371 else if (code
== GIMPLE_OMP_CONTINUE
)
9373 gcc_assert (parent
);
9376 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
9378 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
9379 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
9383 region
= new_omp_region (bb
, code
, parent
);
9385 if (code
== GIMPLE_OMP_TARGET
)
9387 switch (gimple_omp_target_kind (stmt
))
9389 case GF_OMP_TARGET_KIND_REGION
:
9390 case GF_OMP_TARGET_KIND_DATA
:
9391 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
9392 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
9393 case GF_OMP_TARGET_KIND_OACC_DATA
:
9395 case GF_OMP_TARGET_KIND_UPDATE
:
9396 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
9397 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9398 /* ..., other than for those stand-alone directives... */
9405 /* ..., this directive becomes the parent for a new region. */
9411 if (single_tree
&& !parent
)
9414 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
9416 son
= next_dom_son (CDI_DOMINATORS
, son
))
9417 build_omp_regions_1 (son
, parent
, single_tree
);
9420 /* Builds the tree of OMP regions rooted at ROOT, storing it to
9424 build_omp_regions_root (basic_block root
)
9426 gcc_assert (root_omp_region
== NULL
);
9427 build_omp_regions_1 (root
, NULL
, true);
9428 gcc_assert (root_omp_region
!= NULL
);
9431 /* Expands omp construct (and its subconstructs) starting in HEAD. */
9434 omp_expand_local (basic_block head
)
9436 build_omp_regions_root (head
);
9437 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
9439 fprintf (dump_file
, "\nOMP region tree\n\n");
9440 dump_omp_region (dump_file
, root_omp_region
, 0);
9441 fprintf (dump_file
, "\n");
9444 remove_exit_barriers (root_omp_region
);
9445 expand_omp (root_omp_region
);
9447 free_omp_regions ();
9450 /* Scan the CFG and build a tree of OMP regions. Return the root of
9451 the OMP region tree. */
9454 build_omp_regions (void)
9456 gcc_assert (root_omp_region
== NULL
);
9457 calculate_dominance_info (CDI_DOMINATORS
);
9458 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
9461 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
9464 execute_expand_omp (void)
9466 build_omp_regions ();
9468 if (!root_omp_region
)
9473 fprintf (dump_file
, "\nOMP region tree\n\n");
9474 dump_omp_region (dump_file
, root_omp_region
, 0);
9475 fprintf (dump_file
, "\n");
9478 remove_exit_barriers (root_omp_region
);
9480 expand_omp (root_omp_region
);
9482 cleanup_tree_cfg ();
9484 free_omp_regions ();
9489 /* OMP expansion -- the default pass, run before creation of SSA form. */
9493 const pass_data pass_data_expand_omp
=
9495 GIMPLE_PASS
, /* type */
9496 "ompexp", /* name */
9497 OPTGROUP_NONE
, /* optinfo_flags */
9498 TV_NONE
, /* tv_id */
9499 PROP_gimple_any
, /* properties_required */
9500 PROP_gimple_eomp
, /* properties_provided */
9501 0, /* properties_destroyed */
9502 0, /* todo_flags_start */
9503 0, /* todo_flags_finish */
9506 class pass_expand_omp
: public gimple_opt_pass
9509 pass_expand_omp (gcc::context
*ctxt
)
9510 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
9513 /* opt_pass methods: */
9514 virtual unsigned int execute (function
*)
9516 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
9517 || flag_openmp_simd
!= 0)
9520 /* This pass always runs, to provide PROP_gimple_eomp.
9521 But often, there is nothing to do. */
9525 return execute_expand_omp ();
9528 }; // class pass_expand_omp
9533 make_pass_expand_omp (gcc::context
*ctxt
)
9535 return new pass_expand_omp (ctxt
);
9540 const pass_data pass_data_expand_omp_ssa
=
9542 GIMPLE_PASS
, /* type */
9543 "ompexpssa", /* name */
9544 OPTGROUP_NONE
, /* optinfo_flags */
9545 TV_NONE
, /* tv_id */
9546 PROP_cfg
| PROP_ssa
, /* properties_required */
9547 PROP_gimple_eomp
, /* properties_provided */
9548 0, /* properties_destroyed */
9549 0, /* todo_flags_start */
9550 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
9553 class pass_expand_omp_ssa
: public gimple_opt_pass
9556 pass_expand_omp_ssa (gcc::context
*ctxt
)
9557 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
9560 /* opt_pass methods: */
9561 virtual bool gate (function
*fun
)
9563 return !(fun
->curr_properties
& PROP_gimple_eomp
);
9565 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
9567 }; // class pass_expand_omp_ssa
9572 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
9574 return new pass_expand_omp_ssa (ctxt
);
9577 /* Routines to lower OMP directives into OMP-GIMPLE. */
9579 /* Helper function to preform, potentially COMPLEX_TYPE, operation and
9580 convert it to gimple. */
9582 oacc_gimple_assign (tree dest
, tree_code op
, tree src
, gimple_seq
*seq
)
9586 if (TREE_CODE (TREE_TYPE (dest
)) != COMPLEX_TYPE
)
9588 stmt
= gimple_build_assign (dest
, op
, dest
, src
);
9589 gimple_seq_add_stmt (seq
, stmt
);
9593 tree t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9594 tree rdest
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9595 gimplify_assign (t
, rdest
, seq
);
9598 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9599 tree idest
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9600 gimplify_assign (t
, idest
, seq
);
9603 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
9604 tree rsrc
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9605 gimplify_assign (t
, rsrc
, seq
);
9608 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
9609 tree isrc
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9610 gimplify_assign (t
, isrc
, seq
);
9613 tree r
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9614 tree i
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9617 if (op
== PLUS_EXPR
)
9619 stmt
= gimple_build_assign (r
, op
, rdest
, rsrc
);
9620 gimple_seq_add_stmt (seq
, stmt
);
9622 stmt
= gimple_build_assign (i
, op
, idest
, isrc
);
9623 gimple_seq_add_stmt (seq
, stmt
);
9625 else if (op
== MULT_EXPR
)
9627 /* Let x = a + ib = dest, y = c + id = src.
9628 x * y = (ac - bd) + i(ad + bc) */
9629 tree ac
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9630 tree bd
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9631 tree ad
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9632 tree bc
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9634 stmt
= gimple_build_assign (ac
, MULT_EXPR
, rdest
, rsrc
);
9635 gimple_seq_add_stmt (seq
, stmt
);
9637 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, isrc
);
9638 gimple_seq_add_stmt (seq
, stmt
);
9640 stmt
= gimple_build_assign (r
, MINUS_EXPR
, ac
, bd
);
9641 gimple_seq_add_stmt (seq
, stmt
);
9643 stmt
= gimple_build_assign (ad
, MULT_EXPR
, rdest
, isrc
);
9644 gimple_seq_add_stmt (seq
, stmt
);
9646 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, rsrc
);
9647 gimple_seq_add_stmt (seq
, stmt
);
9649 stmt
= gimple_build_assign (i
, PLUS_EXPR
, ad
, bc
);
9650 gimple_seq_add_stmt (seq
, stmt
);
9655 result
= build2 (COMPLEX_EXPR
, TREE_TYPE (dest
), r
, i
);
9656 gimplify_assign (dest
, result
, seq
);
9659 /* Helper function to initialize local data for the reduction arrays.
9660 The reduction arrays need to be placed inside the calling function
9661 for accelerators, or else the host won't be able to preform the final
9665 oacc_initialize_reduction_data (tree clauses
, tree nthreads
,
9666 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
9672 /* Find the innermost OpenACC parallel context. */
9673 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
9674 && (gimple_omp_target_kind (ctx
->stmt
)
9675 == GF_OMP_TARGET_KIND_OACC_PARALLEL
))
9679 gcc_checking_assert (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
9680 && (gimple_omp_target_kind (octx
->stmt
)
9681 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
9683 /* Extract the clauses. */
9684 oc
= gimple_omp_target_clauses (octx
->stmt
);
9686 /* Find the last outer clause. */
9687 for (; oc
&& OMP_CLAUSE_CHAIN (oc
); oc
= OMP_CLAUSE_CHAIN (oc
))
9690 /* Allocate arrays for each reduction variable. */
9691 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9693 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9696 tree var
= OMP_CLAUSE_DECL (c
);
9697 tree type
= get_base_type (var
);
9698 tree array
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
),
9702 /* Calculate size of the reduction array. */
9703 t
= create_tmp_var (TREE_TYPE (nthreads
));
9704 stmt
= gimple_build_assign (t
, MULT_EXPR
, nthreads
,
9705 fold_convert (TREE_TYPE (nthreads
),
9706 TYPE_SIZE_UNIT (type
)));
9707 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9709 size
= create_tmp_var (sizetype
);
9710 gimplify_assign (size
, fold_build1 (NOP_EXPR
, sizetype
, t
), stmt_seqp
);
9712 /* Now allocate memory for it. */
9713 call
= unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA
));
9714 stmt
= gimple_build_call (call
, 1, size
);
9715 gimple_call_set_lhs (stmt
, array
);
9716 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9718 /* Map this array into the accelerator. */
9720 /* Add the reduction array to the list of clauses. */
9722 t
= build_omp_clause (gimple_location (ctx
->stmt
), OMP_CLAUSE_MAP
);
9723 OMP_CLAUSE_SET_MAP_KIND (t
, GOMP_MAP_FORCE_FROM
);
9724 OMP_CLAUSE_DECL (t
) = x
;
9725 OMP_CLAUSE_CHAIN (t
) = NULL
;
9727 OMP_CLAUSE_CHAIN (oc
) = t
;
9729 gimple_omp_target_set_clauses (as_a
<gomp_target
*> (octx
->stmt
), t
);
9730 OMP_CLAUSE_SIZE (t
) = size
;
9735 /* Helper function to process the array of partial reductions. Nthreads
9736 indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS
9737 cannot be used here, because nthreads on the host may be different than
9738 on the accelerator. */
9741 oacc_finalize_reduction_data (tree clauses
, tree nthreads
,
9742 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
9744 tree c
, x
, var
, array
, loop_header
, loop_body
, loop_exit
, type
;
9749 let var = the original reduction variable
9750 let array = reduction variable array
9752 for (i = 0; i < nthreads; i++)
9756 loop_header
= create_artificial_label (UNKNOWN_LOCATION
);
9757 loop_body
= create_artificial_label (UNKNOWN_LOCATION
);
9758 loop_exit
= create_artificial_label (UNKNOWN_LOCATION
);
9760 /* Create and initialize an index variable. */
9761 tree ix
= create_tmp_var (sizetype
);
9762 gimplify_assign (ix
, fold_build1 (NOP_EXPR
, sizetype
, integer_zero_node
),
9765 /* Insert the loop header label here. */
9766 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_header
));
9768 /* Exit loop if ix >= nthreads. */
9769 x
= create_tmp_var (sizetype
);
9770 gimplify_assign (x
, fold_build1 (NOP_EXPR
, sizetype
, nthreads
), stmt_seqp
);
9771 stmt
= gimple_build_cond (GE_EXPR
, ix
, x
, loop_exit
, loop_body
);
9772 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9774 /* Insert the loop body label here. */
9775 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_body
));
9777 /* Collapse each reduction array, one element at a time. */
9778 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9780 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9783 tree_code reduction_code
= OMP_CLAUSE_REDUCTION_CODE (c
);
9785 /* reduction(-:var) sums up the partial results, so it acts
9786 identically to reduction(+:var). */
9787 if (reduction_code
== MINUS_EXPR
)
9788 reduction_code
= PLUS_EXPR
;
9790 /* Set up reduction variable var. */
9791 var
= OMP_CLAUSE_DECL (c
);
9792 type
= get_base_type (var
);
9793 array
= lookup_oacc_reduction (oacc_get_reduction_array_id
9794 (OMP_CLAUSE_DECL (c
)), ctx
);
9796 /* Calculate the array offset. */
9797 tree offset
= create_tmp_var (sizetype
);
9798 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
), stmt_seqp
);
9799 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, ix
);
9800 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9802 tree ptr
= create_tmp_var (TREE_TYPE (array
));
9803 stmt
= gimple_build_assign (ptr
, POINTER_PLUS_EXPR
, array
, offset
);
9804 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9806 /* Extract array[ix] into mem. */
9807 tree mem
= create_tmp_var (type
);
9808 gimplify_assign (mem
, build_simple_mem_ref (ptr
), stmt_seqp
);
9810 /* Find the original reduction variable. */
9811 if (is_reference (var
))
9812 var
= build_simple_mem_ref (var
);
9814 tree t
= create_tmp_var (type
);
9816 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, t
, var
);
9817 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9819 /* var = var op mem */
9820 switch (OMP_CLAUSE_REDUCTION_CODE (c
))
9822 case TRUTH_ANDIF_EXPR
:
9823 case TRUTH_ORIF_EXPR
:
9824 t
= fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c
), integer_type_node
,
9826 gimplify_and_add (t
, stmt_seqp
);
9829 /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */
9830 oacc_gimple_assign (t
, OMP_CLAUSE_REDUCTION_CODE (c
), mem
,
9834 t
= fold_build1 (NOP_EXPR
, TREE_TYPE (var
), t
);
9835 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, var
, t
);
9836 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9839 /* Increment the induction variable. */
9840 tree one
= fold_build1 (NOP_EXPR
, sizetype
, integer_one_node
);
9841 stmt
= gimple_build_assign (ix
, PLUS_EXPR
, ix
, one
);
9842 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9844 /* Go back to the top of the loop. */
9845 gimple_seq_add_stmt (stmt_seqp
, gimple_build_goto (loop_header
));
9847 /* Place the loop exit label here. */
9848 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_exit
));
9851 /* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and
9852 scan that for reductions. */
9855 oacc_process_reduction_data (gimple_seq
*body
, gimple_seq
*in_stmt_seqp
,
9856 gimple_seq
*out_stmt_seqp
, omp_context
*ctx
)
9858 gimple_stmt_iterator gsi
;
9859 gimple_seq inner
= NULL
;
9861 /* A collapse clause may have inserted a new bind block. */
9862 gsi
= gsi_start (*body
);
9863 while (!gsi_end_p (gsi
))
9865 gimple stmt
= gsi_stmt (gsi
);
9866 if (gbind
*bind_stmt
= dyn_cast
<gbind
*> (stmt
))
9868 inner
= gimple_bind_body (bind_stmt
);
9870 gsi
= gsi_start (*body
);
9872 else if (dyn_cast
<gomp_for
*> (stmt
))
9878 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
9880 tree clauses
, nthreads
, t
, c
, acc_device
, acc_device_host
, call
,
9882 bool reduction_found
= false;
9884 gimple stmt
= gsi_stmt (gsi
);
9886 switch (gimple_code (stmt
))
9888 case GIMPLE_OMP_FOR
:
9889 clauses
= gimple_omp_for_clauses (stmt
);
9891 /* Search for a reduction clause. */
9892 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9893 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
9895 reduction_found
= true;
9899 if (!reduction_found
)
9902 ctx
= maybe_lookup_ctx (stmt
);
9905 /* Extract the number of threads. */
9906 nthreads
= create_tmp_var (sizetype
);
9907 t
= oacc_max_threads (ctx
);
9908 gimplify_assign (nthreads
, t
, in_stmt_seqp
);
9910 /* Determine if this is kernel will be executed on the host. */
9911 call
= builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE
);
9912 acc_device
= create_tmp_var (integer_type_node
, ".acc_device_type");
9913 stmt
= gimple_build_call (call
, 0);
9914 gimple_call_set_lhs (stmt
, acc_device
);
9915 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9917 /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */
9918 acc_device_host
= create_tmp_var (integer_type_node
,
9919 ".acc_device_host");
9920 gimplify_assign (acc_device_host
,
9921 build_int_cst (integer_type_node
,
9925 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9926 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9928 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9930 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9931 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9932 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9935 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9937 /* Also, set nthreads = 1 for ACC_DEVICE_TYPE=host_nonshm. */
9938 gimplify_assign (acc_device_host
,
9939 build_int_cst (integer_type_node
,
9940 GOMP_DEVICE_HOST_NONSHM
),
9943 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9944 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9946 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9948 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9949 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9950 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9953 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9955 oacc_initialize_reduction_data (clauses
, nthreads
, in_stmt_seqp
,
9957 oacc_finalize_reduction_data (clauses
, nthreads
, out_stmt_seqp
, ctx
);
9960 // Scan for other directives which support reduction here.
9966 /* If ctx is a worksharing context inside of a cancellable parallel
9967 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
9968 and conditional branch to parallel's cancel_label to handle
9969 cancellation in the implicit barrier. */
9972 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
9974 gimple omp_return
= gimple_seq_last_stmt (*body
);
9975 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
9976 if (gimple_omp_return_nowait_p (omp_return
))
9979 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
9980 && ctx
->outer
->cancellable
)
9982 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
9983 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
9984 tree lhs
= create_tmp_var (c_bool_type
);
9985 gimple_omp_return_set_lhs (omp_return
, lhs
);
9986 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
9987 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
9988 fold_convert (c_bool_type
,
9989 boolean_false_node
),
9990 ctx
->outer
->cancel_label
, fallthru_label
);
9991 gimple_seq_add_stmt (body
, g
);
9992 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
9996 /* Lower the OpenMP sections directive in the current statement in GSI_P.
9997 CTX is the enclosing OMP context for the current statement. */
10000 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10002 tree block
, control
;
10003 gimple_stmt_iterator tgsi
;
10004 gomp_sections
*stmt
;
10006 gbind
*new_stmt
, *bind
;
10007 gimple_seq ilist
, dlist
, olist
, new_body
;
10009 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
10011 push_gimplify_context ();
10015 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
10016 &ilist
, &dlist
, ctx
, NULL
);
10018 new_body
= gimple_omp_body (stmt
);
10019 gimple_omp_set_body (stmt
, NULL
);
10020 tgsi
= gsi_start (new_body
);
10021 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
10026 sec_start
= gsi_stmt (tgsi
);
10027 sctx
= maybe_lookup_ctx (sec_start
);
10030 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
10031 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
10032 GSI_CONTINUE_LINKING
);
10033 gimple_omp_set_body (sec_start
, NULL
);
10035 if (gsi_one_before_end_p (tgsi
))
10037 gimple_seq l
= NULL
;
10038 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
10040 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
10041 gimple_omp_section_set_last (sec_start
);
10044 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
10045 GSI_CONTINUE_LINKING
);
10048 block
= make_node (BLOCK
);
10049 bind
= gimple_build_bind (NULL
, new_body
, block
);
10052 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
10054 block
= make_node (BLOCK
);
10055 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10056 gsi_replace (gsi_p
, new_stmt
, true);
10058 pop_gimplify_context (new_stmt
);
10059 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10060 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10061 if (BLOCK_VARS (block
))
10062 TREE_USED (block
) = 1;
10065 gimple_seq_add_seq (&new_body
, ilist
);
10066 gimple_seq_add_stmt (&new_body
, stmt
);
10067 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
10068 gimple_seq_add_stmt (&new_body
, bind
);
10070 control
= create_tmp_var (unsigned_type_node
, ".section");
10071 t
= gimple_build_omp_continue (control
, control
);
10072 gimple_omp_sections_set_control (stmt
, control
);
10073 gimple_seq_add_stmt (&new_body
, t
);
10075 gimple_seq_add_seq (&new_body
, olist
);
10076 if (ctx
->cancellable
)
10077 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
10078 gimple_seq_add_seq (&new_body
, dlist
);
10080 new_body
= maybe_catch_exception (new_body
);
10082 t
= gimple_build_omp_return
10083 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
10084 OMP_CLAUSE_NOWAIT
));
10085 gimple_seq_add_stmt (&new_body
, t
);
10086 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
10088 gimple_bind_set_body (new_stmt
, new_body
);
10092 /* A subroutine of lower_omp_single. Expand the simple form of
10093 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
10095 if (GOMP_single_start ())
10097 [ GOMP_barrier (); ] -> unless 'nowait' is present.
10099 FIXME. It may be better to delay expanding the logic of this until
10100 pass_expand_omp. The expanded logic may make the job more difficult
10101 to a synchronization analysis pass. */
10104 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
10106 location_t loc
= gimple_location (single_stmt
);
10107 tree tlabel
= create_artificial_label (loc
);
10108 tree flabel
= create_artificial_label (loc
);
10112 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
10113 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
10114 call
= gimple_build_call (decl
, 0);
10115 gimple_call_set_lhs (call
, lhs
);
10116 gimple_seq_add_stmt (pre_p
, call
);
10118 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
10119 fold_convert_loc (loc
, TREE_TYPE (lhs
),
10120 boolean_true_node
),
10122 gimple_seq_add_stmt (pre_p
, cond
);
10123 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
10124 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10125 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
10129 /* A subroutine of lower_omp_single. Expand the simple form of
10130 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
10132 #pragma omp single copyprivate (a, b, c)
10134 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
10137 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
10143 GOMP_single_copy_end (©out);
10154 FIXME. It may be better to delay expanding the logic of this until
10155 pass_expand_omp. The expanded logic may make the job more difficult
10156 to a synchronization analysis pass. */
10159 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
10162 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
10163 gimple_seq copyin_seq
;
10164 location_t loc
= gimple_location (single_stmt
);
10166 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
10168 ptr_type
= build_pointer_type (ctx
->record_type
);
10169 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
10171 l0
= create_artificial_label (loc
);
10172 l1
= create_artificial_label (loc
);
10173 l2
= create_artificial_label (loc
);
10175 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
10176 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
10177 t
= fold_convert_loc (loc
, ptr_type
, t
);
10178 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
10180 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
10181 build_int_cst (ptr_type
, 0));
10182 t
= build3 (COND_EXPR
, void_type_node
, t
,
10183 build_and_jump (&l0
), build_and_jump (&l1
));
10184 gimplify_and_add (t
, pre_p
);
10186 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
10188 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10191 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
10194 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
10195 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
10196 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
10197 gimplify_and_add (t
, pre_p
);
10199 t
= build_and_jump (&l2
);
10200 gimplify_and_add (t
, pre_p
);
10202 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
10204 gimple_seq_add_seq (pre_p
, copyin_seq
);
10206 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
10210 /* Expand code for an OpenMP single directive. */
10213 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10217 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
10219 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
10221 push_gimplify_context ();
10223 block
= make_node (BLOCK
);
10224 bind
= gimple_build_bind (NULL
, NULL
, block
);
10225 gsi_replace (gsi_p
, bind
, true);
10228 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
10229 &bind_body
, &dlist
, ctx
, NULL
);
10230 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
10232 gimple_seq_add_stmt (&bind_body
, single_stmt
);
10234 if (ctx
->record_type
)
10235 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
10237 lower_omp_single_simple (single_stmt
, &bind_body
);
10239 gimple_omp_set_body (single_stmt
, NULL
);
10241 gimple_seq_add_seq (&bind_body
, dlist
);
10243 bind_body
= maybe_catch_exception (bind_body
);
10245 t
= gimple_build_omp_return
10246 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
10247 OMP_CLAUSE_NOWAIT
));
10248 gimple_seq_add_stmt (&bind_body_tail
, t
);
10249 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
10250 if (ctx
->record_type
)
10252 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
10253 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
10254 TREE_THIS_VOLATILE (clobber
) = 1;
10255 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
10256 clobber
), GSI_SAME_STMT
);
10258 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
10259 gimple_bind_set_body (bind
, bind_body
);
10261 pop_gimplify_context (bind
);
10263 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10264 BLOCK_VARS (block
) = ctx
->block_vars
;
10265 if (BLOCK_VARS (block
))
10266 TREE_USED (block
) = 1;
10270 /* Expand code for an OpenMP master directive. */
10273 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10275 tree block
, lab
= NULL
, x
, bfn_decl
;
10276 gimple stmt
= gsi_stmt (*gsi_p
);
10278 location_t loc
= gimple_location (stmt
);
10281 push_gimplify_context ();
10283 block
= make_node (BLOCK
);
10284 bind
= gimple_build_bind (NULL
, NULL
, block
);
10285 gsi_replace (gsi_p
, bind
, true);
10286 gimple_bind_add_stmt (bind
, stmt
);
10288 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
10289 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
10290 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
10291 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
10293 gimplify_and_add (x
, &tseq
);
10294 gimple_bind_add_seq (bind
, tseq
);
10296 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10297 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10298 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10299 gimple_omp_set_body (stmt
, NULL
);
10301 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
10303 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10305 pop_gimplify_context (bind
);
10307 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10308 BLOCK_VARS (block
) = ctx
->block_vars
;
10312 /* Expand code for an OpenMP taskgroup directive. */
10315 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10317 gimple stmt
= gsi_stmt (*gsi_p
);
10320 tree block
= make_node (BLOCK
);
10322 bind
= gimple_build_bind (NULL
, NULL
, block
);
10323 gsi_replace (gsi_p
, bind
, true);
10324 gimple_bind_add_stmt (bind
, stmt
);
10326 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
10328 gimple_bind_add_stmt (bind
, x
);
10330 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10331 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10332 gimple_omp_set_body (stmt
, NULL
);
10334 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10336 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10337 BLOCK_VARS (block
) = ctx
->block_vars
;
10341 /* Expand code for an OpenMP ordered directive. */
10344 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10347 gimple stmt
= gsi_stmt (*gsi_p
);
10351 push_gimplify_context ();
10353 block
= make_node (BLOCK
);
10354 bind
= gimple_build_bind (NULL
, NULL
, block
);
10355 gsi_replace (gsi_p
, bind
, true);
10356 gimple_bind_add_stmt (bind
, stmt
);
10358 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
10360 gimple_bind_add_stmt (bind
, x
);
10362 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10363 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10364 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10365 gimple_omp_set_body (stmt
, NULL
);
10367 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
10368 gimple_bind_add_stmt (bind
, x
);
10370 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10372 pop_gimplify_context (bind
);
10374 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10375 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10379 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
10380 substitution of a couple of function calls. But in the NAMED case,
10381 requires that languages coordinate a symbol name. It is therefore
10382 best put here in common code. */
10384 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
10387 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10390 tree name
, lock
, unlock
;
10391 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
10393 location_t loc
= gimple_location (stmt
);
10396 name
= gimple_omp_critical_name (stmt
);
10401 if (!critical_name_mutexes
)
10402 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
10404 tree
*n
= critical_name_mutexes
->get (name
);
10409 decl
= create_tmp_var_raw (ptr_type_node
);
10411 new_str
= ACONCAT ((".gomp_critical_user_",
10412 IDENTIFIER_POINTER (name
), NULL
));
10413 DECL_NAME (decl
) = get_identifier (new_str
);
10414 TREE_PUBLIC (decl
) = 1;
10415 TREE_STATIC (decl
) = 1;
10416 DECL_COMMON (decl
) = 1;
10417 DECL_ARTIFICIAL (decl
) = 1;
10418 DECL_IGNORED_P (decl
) = 1;
10420 varpool_node::finalize_decl (decl
);
10422 critical_name_mutexes
->put (name
, decl
);
10427 /* If '#pragma omp critical' is inside offloaded region or
10428 inside function marked as offloadable, the symbol must be
10429 marked as offloadable too. */
10431 if (cgraph_node::get (current_function_decl
)->offloadable
)
10432 varpool_node::get_create (decl
)->offloadable
= 1;
10434 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
10435 if (is_gimple_omp_offloaded (octx
->stmt
))
10437 varpool_node::get_create (decl
)->offloadable
= 1;
10441 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
10442 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
10444 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
10445 unlock
= build_call_expr_loc (loc
, unlock
, 1,
10446 build_fold_addr_expr_loc (loc
, decl
));
10450 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
10451 lock
= build_call_expr_loc (loc
, lock
, 0);
10453 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
10454 unlock
= build_call_expr_loc (loc
, unlock
, 0);
10457 push_gimplify_context ();
10459 block
= make_node (BLOCK
);
10460 bind
= gimple_build_bind (NULL
, NULL
, block
);
10461 gsi_replace (gsi_p
, bind
, true);
10462 gimple_bind_add_stmt (bind
, stmt
);
10464 tbody
= gimple_bind_body (bind
);
10465 gimplify_and_add (lock
, &tbody
);
10466 gimple_bind_set_body (bind
, tbody
);
10468 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10469 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10470 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10471 gimple_omp_set_body (stmt
, NULL
);
10473 tbody
= gimple_bind_body (bind
);
10474 gimplify_and_add (unlock
, &tbody
);
10475 gimple_bind_set_body (bind
, tbody
);
10477 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10479 pop_gimplify_context (bind
);
10480 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10481 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10485 /* A subroutine of lower_omp_for. Generate code to emit the predicate
10486 for a lastprivate clause. Given a loop control predicate of (V
10487 cond N2), we gate the clause on (!(V cond N2)). The lowered form
10488 is appended to *DLIST, iterator initialization is appended to
10492 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
10493 gimple_seq
*dlist
, struct omp_context
*ctx
)
10495 tree clauses
, cond
, vinit
;
10496 enum tree_code cond_code
;
10499 cond_code
= fd
->loop
.cond_code
;
10500 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
10502 /* When possible, use a strict equality expression. This can let VRP
10503 type optimizations deduce the value and remove a copy. */
10504 if (tree_fits_shwi_p (fd
->loop
.step
))
10506 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
10507 if (step
== 1 || step
== -1)
10508 cond_code
= EQ_EXPR
;
10511 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
10513 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
10515 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
10516 if (!gimple_seq_empty_p (stmts
))
10518 gimple_seq_add_seq (&stmts
, *dlist
);
10521 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
10522 vinit
= fd
->loop
.n1
;
10523 if (cond_code
== EQ_EXPR
10524 && tree_fits_shwi_p (fd
->loop
.n2
)
10525 && ! integer_zerop (fd
->loop
.n2
))
10526 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
10528 vinit
= unshare_expr (vinit
);
10530 /* Initialize the iterator variable, so that threads that don't execute
10531 any iterations don't execute the lastprivate clauses by accident. */
10532 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
10537 /* Lower code for an OMP loop directive. */
10540 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10542 tree
*rhs_p
, block
;
10543 struct omp_for_data fd
, *fdp
= NULL
;
10544 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
10546 gimple_seq omp_for_body
, body
, dlist
;
10549 push_gimplify_context ();
10551 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
10553 block
= make_node (BLOCK
);
10554 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10555 /* Replace at gsi right away, so that 'stmt' is no member
10556 of a sequence anymore as we're going to add to to a different
10558 gsi_replace (gsi_p
, new_stmt
, true);
10560 /* Move declaration of temporaries in the loop body before we make
10562 omp_for_body
= gimple_omp_body (stmt
);
10563 if (!gimple_seq_empty_p (omp_for_body
)
10564 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
10567 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
10568 tree vars
= gimple_bind_vars (inner_bind
);
10569 gimple_bind_append_vars (new_stmt
, vars
);
10570 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
10571 keep them on the inner_bind and it's block. */
10572 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
10573 if (gimple_bind_block (inner_bind
))
10574 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
10577 if (gimple_omp_for_combined_into_p (stmt
))
10579 extract_omp_for_data (stmt
, &fd
, NULL
);
10582 /* We need two temporaries with fd.loop.v type (istart/iend)
10583 and then (fd.collapse - 1) temporaries with the same
10584 type for count2 ... countN-1 vars if not constant. */
10586 tree type
= fd
.iter_type
;
10587 if (fd
.collapse
> 1
10588 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
10589 count
+= fd
.collapse
- 1;
10590 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
10591 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
10592 tree clauses
= *pc
;
10595 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
10596 OMP_CLAUSE__LOOPTEMP_
);
10597 for (i
= 0; i
< count
; i
++)
10602 gcc_assert (outerc
);
10603 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
10604 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
10605 OMP_CLAUSE__LOOPTEMP_
);
10609 temp
= create_tmp_var (type
);
10610 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
10612 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
10613 OMP_CLAUSE_DECL (*pc
) = temp
;
10614 pc
= &OMP_CLAUSE_CHAIN (*pc
);
10619 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
10622 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
10624 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
10626 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10628 /* Lower the header expressions. At this point, we can assume that
10629 the header is of the form:
10631 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
10633 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
10634 using the .omp_data_s mapping, if needed. */
10635 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
10637 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
10638 if (!is_gimple_min_invariant (*rhs_p
))
10639 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10641 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
10642 if (!is_gimple_min_invariant (*rhs_p
))
10643 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10645 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
10646 if (!is_gimple_min_invariant (*rhs_p
))
10647 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10650 /* Once lowered, extract the bounds and clauses. */
10651 extract_omp_for_data (stmt
, &fd
, NULL
);
10653 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
10655 gimple_seq_add_stmt (&body
, stmt
);
10656 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
10658 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
10661 /* After the loop, add exit clauses. */
10662 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
10664 if (ctx
->cancellable
)
10665 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
10667 gimple_seq_add_seq (&body
, dlist
);
10669 body
= maybe_catch_exception (body
);
10671 /* Region exit marker goes at the end of the loop body. */
10672 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
10673 maybe_add_implicit_barrier_cancel (ctx
, &body
);
10674 pop_gimplify_context (new_stmt
);
10676 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10677 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
10678 if (BLOCK_VARS (block
))
10679 TREE_USED (block
) = 1;
10681 gimple_bind_set_body (new_stmt
, body
);
10682 gimple_omp_set_body (stmt
, NULL
);
10683 gimple_omp_for_set_pre_body (stmt
, NULL
);
10686 /* Callback for walk_stmts. Check if the current statement only contains
10687 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
10690 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
10691 bool *handled_ops_p
,
10692 struct walk_stmt_info
*wi
)
10694 int *info
= (int *) wi
->info
;
10695 gimple stmt
= gsi_stmt (*gsi_p
);
10697 *handled_ops_p
= true;
10698 switch (gimple_code (stmt
))
10702 case GIMPLE_OMP_FOR
:
10703 case GIMPLE_OMP_SECTIONS
:
10704 *info
= *info
== 0 ? 1 : -1;
10713 struct omp_taskcopy_context
10715 /* This field must be at the beginning, as we do "inheritance": Some
10716 callback functions for tree-inline.c (e.g., omp_copy_decl)
10717 receive a copy_body_data pointer that is up-casted to an
10718 omp_context pointer. */
10724 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
10726 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
10728 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
10729 return create_tmp_var (TREE_TYPE (var
));
10735 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
10737 tree name
, new_fields
= NULL
, type
, f
;
10739 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
10740 name
= DECL_NAME (TYPE_NAME (orig_type
));
10741 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
10742 TYPE_DECL
, name
, type
);
10743 TYPE_NAME (type
) = name
;
10745 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
10747 tree new_f
= copy_node (f
);
10748 DECL_CONTEXT (new_f
) = type
;
10749 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
10750 TREE_CHAIN (new_f
) = new_fields
;
10751 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10752 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10753 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
10755 new_fields
= new_f
;
10756 tcctx
->cb
.decl_map
->put (f
, new_f
);
10758 TYPE_FIELDS (type
) = nreverse (new_fields
);
10759 layout_type (type
);
10763 /* Create task copyfn. */
10766 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
10768 struct function
*child_cfun
;
10769 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
10770 tree record_type
, srecord_type
, bind
, list
;
10771 bool record_needs_remap
= false, srecord_needs_remap
= false;
10773 struct omp_taskcopy_context tcctx
;
10774 location_t loc
= gimple_location (task_stmt
);
10776 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
10777 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
10778 gcc_assert (child_cfun
->cfg
== NULL
);
10779 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
10781 /* Reset DECL_CONTEXT on function arguments. */
10782 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
10783 DECL_CONTEXT (t
) = child_fn
;
10785 /* Populate the function. */
10786 push_gimplify_context ();
10787 push_cfun (child_cfun
);
10789 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
10790 TREE_SIDE_EFFECTS (bind
) = 1;
10792 DECL_SAVED_TREE (child_fn
) = bind
;
10793 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
10795 /* Remap src and dst argument types if needed. */
10796 record_type
= ctx
->record_type
;
10797 srecord_type
= ctx
->srecord_type
;
10798 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
10799 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10801 record_needs_remap
= true;
10804 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
10805 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10807 srecord_needs_remap
= true;
10811 if (record_needs_remap
|| srecord_needs_remap
)
10813 memset (&tcctx
, '\0', sizeof (tcctx
));
10814 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
10815 tcctx
.cb
.dst_fn
= child_fn
;
10816 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
10817 gcc_checking_assert (tcctx
.cb
.src_node
);
10818 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
10819 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
10820 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
10821 tcctx
.cb
.eh_lp_nr
= 0;
10822 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
10823 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
10826 if (record_needs_remap
)
10827 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
10828 if (srecord_needs_remap
)
10829 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
10832 tcctx
.cb
.decl_map
= NULL
;
10834 arg
= DECL_ARGUMENTS (child_fn
);
10835 TREE_TYPE (arg
) = build_pointer_type (record_type
);
10836 sarg
= DECL_CHAIN (arg
);
10837 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
10839 /* First pass: initialize temporaries used in record_type and srecord_type
10840 sizes and field offsets. */
10841 if (tcctx
.cb
.decl_map
)
10842 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10843 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10847 decl
= OMP_CLAUSE_DECL (c
);
10848 p
= tcctx
.cb
.decl_map
->get (decl
);
10851 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10852 sf
= (tree
) n
->value
;
10853 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10854 src
= build_simple_mem_ref_loc (loc
, sarg
);
10855 src
= omp_build_component_ref (src
, sf
);
10856 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
10857 append_to_statement_list (t
, &list
);
10860 /* Second pass: copy shared var pointers and copy construct non-VLA
10861 firstprivate vars. */
10862 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10863 switch (OMP_CLAUSE_CODE (c
))
10865 case OMP_CLAUSE_SHARED
:
10866 decl
= OMP_CLAUSE_DECL (c
);
10867 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10870 f
= (tree
) n
->value
;
10871 if (tcctx
.cb
.decl_map
)
10872 f
= *tcctx
.cb
.decl_map
->get (f
);
10873 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10874 sf
= (tree
) n
->value
;
10875 if (tcctx
.cb
.decl_map
)
10876 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10877 src
= build_simple_mem_ref_loc (loc
, sarg
);
10878 src
= omp_build_component_ref (src
, sf
);
10879 dst
= build_simple_mem_ref_loc (loc
, arg
);
10880 dst
= omp_build_component_ref (dst
, f
);
10881 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10882 append_to_statement_list (t
, &list
);
10884 case OMP_CLAUSE_FIRSTPRIVATE
:
10885 decl
= OMP_CLAUSE_DECL (c
);
10886 if (is_variable_sized (decl
))
10888 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10891 f
= (tree
) n
->value
;
10892 if (tcctx
.cb
.decl_map
)
10893 f
= *tcctx
.cb
.decl_map
->get (f
);
10894 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10897 sf
= (tree
) n
->value
;
10898 if (tcctx
.cb
.decl_map
)
10899 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10900 src
= build_simple_mem_ref_loc (loc
, sarg
);
10901 src
= omp_build_component_ref (src
, sf
);
10902 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
10903 src
= build_simple_mem_ref_loc (loc
, src
);
10907 dst
= build_simple_mem_ref_loc (loc
, arg
);
10908 dst
= omp_build_component_ref (dst
, f
);
10909 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10910 append_to_statement_list (t
, &list
);
10912 case OMP_CLAUSE_PRIVATE
:
10913 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
10915 decl
= OMP_CLAUSE_DECL (c
);
10916 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10917 f
= (tree
) n
->value
;
10918 if (tcctx
.cb
.decl_map
)
10919 f
= *tcctx
.cb
.decl_map
->get (f
);
10920 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10923 sf
= (tree
) n
->value
;
10924 if (tcctx
.cb
.decl_map
)
10925 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10926 src
= build_simple_mem_ref_loc (loc
, sarg
);
10927 src
= omp_build_component_ref (src
, sf
);
10928 if (use_pointer_for_field (decl
, NULL
))
10929 src
= build_simple_mem_ref_loc (loc
, src
);
10933 dst
= build_simple_mem_ref_loc (loc
, arg
);
10934 dst
= omp_build_component_ref (dst
, f
);
10935 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10936 append_to_statement_list (t
, &list
);
10942 /* Last pass: handle VLA firstprivates. */
10943 if (tcctx
.cb
.decl_map
)
10944 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10945 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10949 decl
= OMP_CLAUSE_DECL (c
);
10950 if (!is_variable_sized (decl
))
10952 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10955 f
= (tree
) n
->value
;
10956 f
= *tcctx
.cb
.decl_map
->get (f
);
10957 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
10958 ind
= DECL_VALUE_EXPR (decl
);
10959 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
10960 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
10961 n
= splay_tree_lookup (ctx
->sfield_map
,
10962 (splay_tree_key
) TREE_OPERAND (ind
, 0));
10963 sf
= (tree
) n
->value
;
10964 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10965 src
= build_simple_mem_ref_loc (loc
, sarg
);
10966 src
= omp_build_component_ref (src
, sf
);
10967 src
= build_simple_mem_ref_loc (loc
, src
);
10968 dst
= build_simple_mem_ref_loc (loc
, arg
);
10969 dst
= omp_build_component_ref (dst
, f
);
10970 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10971 append_to_statement_list (t
, &list
);
10972 n
= splay_tree_lookup (ctx
->field_map
,
10973 (splay_tree_key
) TREE_OPERAND (ind
, 0));
10974 df
= (tree
) n
->value
;
10975 df
= *tcctx
.cb
.decl_map
->get (df
);
10976 ptr
= build_simple_mem_ref_loc (loc
, arg
);
10977 ptr
= omp_build_component_ref (ptr
, df
);
10978 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
10979 build_fold_addr_expr_loc (loc
, dst
));
10980 append_to_statement_list (t
, &list
);
10983 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
10984 append_to_statement_list (t
, &list
);
10986 if (tcctx
.cb
.decl_map
)
10987 delete tcctx
.cb
.decl_map
;
10988 pop_gimplify_context (NULL
);
10989 BIND_EXPR_BODY (bind
) = list
;
10994 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
10998 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
11000 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
11001 OMP_CLAUSE_DEPEND
);
11002 gcc_assert (clauses
);
11003 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11004 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
11005 switch (OMP_CLAUSE_DEPEND_KIND (c
))
11007 case OMP_CLAUSE_DEPEND_IN
:
11010 case OMP_CLAUSE_DEPEND_OUT
:
11011 case OMP_CLAUSE_DEPEND_INOUT
:
11015 gcc_unreachable ();
11017 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
11018 tree array
= create_tmp_var (type
);
11019 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
11021 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
11022 gimple_seq_add_stmt (iseq
, g
);
11023 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
11025 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
11026 gimple_seq_add_stmt (iseq
, g
);
11027 for (i
= 0; i
< 2; i
++)
11029 if ((i
? n_in
: n_out
) == 0)
11031 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11032 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
11033 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
11035 tree t
= OMP_CLAUSE_DECL (c
);
11036 t
= fold_convert (ptr_type_node
, t
);
11037 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
11038 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
11039 NULL_TREE
, NULL_TREE
);
11040 g
= gimple_build_assign (r
, t
);
11041 gimple_seq_add_stmt (iseq
, g
);
11044 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
11045 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
11046 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
11047 OMP_CLAUSE_CHAIN (c
) = *p
;
11049 tree clobber
= build_constructor (type
, NULL
);
11050 TREE_THIS_VOLATILE (clobber
) = 1;
11051 g
= gimple_build_assign (array
, clobber
);
11052 gimple_seq_add_stmt (oseq
, g
);
11055 /* Lower the OpenMP parallel or task directive in the current statement
11056 in GSI_P. CTX holds context information for the directive. */
11059 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11063 gimple stmt
= gsi_stmt (*gsi_p
);
11064 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
11065 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
11066 location_t loc
= gimple_location (stmt
);
11068 clauses
= gimple_omp_taskreg_clauses (stmt
);
11070 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
11071 par_body
= gimple_bind_body (par_bind
);
11072 child_fn
= ctx
->cb
.dst_fn
;
11073 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
11074 && !gimple_omp_parallel_combined_p (stmt
))
11076 struct walk_stmt_info wi
;
11079 memset (&wi
, 0, sizeof (wi
));
11081 wi
.val_only
= true;
11082 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
11084 gimple_omp_parallel_set_combined_p (stmt
, true);
11086 gimple_seq dep_ilist
= NULL
;
11087 gimple_seq dep_olist
= NULL
;
11088 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
11089 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
11091 push_gimplify_context ();
11092 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
11093 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
11096 if (ctx
->srecord_type
)
11097 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
11099 push_gimplify_context ();
11104 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
11105 lower_omp (&par_body
, ctx
);
11106 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
11107 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
11109 /* Declare all the variables created by mapping and the variables
11110 declared in the scope of the parallel body. */
11111 record_vars_into (ctx
->block_vars
, child_fn
);
11112 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
11114 if (ctx
->record_type
)
11117 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
11118 : ctx
->record_type
, ".omp_data_o");
11119 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11120 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11121 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
11126 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
11127 lower_send_shared_vars (&ilist
, &olist
, ctx
);
11129 if (ctx
->record_type
)
11131 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
11132 TREE_THIS_VOLATILE (clobber
) = 1;
11133 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11137 /* Once all the expansions are done, sequence all the different
11138 fragments inside gimple_omp_body. */
11142 if (ctx
->record_type
)
11144 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11145 /* fixup_child_record_type might have changed receiver_decl's type. */
11146 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11147 gimple_seq_add_stmt (&new_body
,
11148 gimple_build_assign (ctx
->receiver_decl
, t
));
11151 gimple_seq_add_seq (&new_body
, par_ilist
);
11152 gimple_seq_add_seq (&new_body
, par_body
);
11153 gimple_seq_add_seq (&new_body
, par_rlist
);
11154 if (ctx
->cancellable
)
11155 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
11156 gimple_seq_add_seq (&new_body
, par_olist
);
11157 new_body
= maybe_catch_exception (new_body
);
11158 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11159 gimple_omp_set_body (stmt
, new_body
);
11161 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
11162 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
11163 gimple_bind_add_seq (bind
, ilist
);
11164 gimple_bind_add_stmt (bind
, stmt
);
11165 gimple_bind_add_seq (bind
, olist
);
11167 pop_gimplify_context (NULL
);
11171 gimple_bind_add_seq (dep_bind
, dep_ilist
);
11172 gimple_bind_add_stmt (dep_bind
, bind
);
11173 gimple_bind_add_seq (dep_bind
, dep_olist
);
11174 pop_gimplify_context (dep_bind
);
11178 /* Lower the GIMPLE_OMP_TARGET in the current statement
11179 in GSI_P. CTX holds context information for the directive. */
11182 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11185 tree child_fn
, t
, c
;
11186 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
11187 gbind
*tgt_bind
, *bind
;
11188 gimple_seq tgt_body
, olist
, ilist
, orlist
, irlist
, new_body
;
11189 location_t loc
= gimple_location (stmt
);
11190 bool offloaded
, data_region
;
11191 unsigned int map_cnt
= 0;
11193 offloaded
= is_gimple_omp_offloaded (stmt
);
11194 switch (gimple_omp_target_kind (stmt
))
11196 case GF_OMP_TARGET_KIND_REGION
:
11197 case GF_OMP_TARGET_KIND_UPDATE
:
11198 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
11199 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
11200 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
11201 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
11202 data_region
= false;
11204 case GF_OMP_TARGET_KIND_DATA
:
11205 case GF_OMP_TARGET_KIND_OACC_DATA
:
11206 data_region
= true;
11209 gcc_unreachable ();
11212 clauses
= gimple_omp_target_clauses (stmt
);
11218 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
11219 tgt_body
= gimple_bind_body (tgt_bind
);
11221 else if (data_region
)
11222 tgt_body
= gimple_omp_body (stmt
);
11223 child_fn
= ctx
->cb
.dst_fn
;
11225 push_gimplify_context ();
11230 && is_gimple_omp_oacc (stmt
))
11231 oacc_process_reduction_data (&tgt_body
, &irlist
, &orlist
, ctx
);
11233 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11234 switch (OMP_CLAUSE_CODE (c
))
11240 case OMP_CLAUSE_MAP
:
11241 #ifdef ENABLE_CHECKING
11242 /* First check what we're prepared to handle in the following. */
11243 switch (OMP_CLAUSE_MAP_KIND (c
))
11245 case GOMP_MAP_ALLOC
:
11247 case GOMP_MAP_FROM
:
11248 case GOMP_MAP_TOFROM
:
11249 case GOMP_MAP_POINTER
:
11250 case GOMP_MAP_TO_PSET
:
11252 case GOMP_MAP_FORCE_ALLOC
:
11253 case GOMP_MAP_FORCE_TO
:
11254 case GOMP_MAP_FORCE_FROM
:
11255 case GOMP_MAP_FORCE_TOFROM
:
11256 case GOMP_MAP_FORCE_PRESENT
:
11257 case GOMP_MAP_FORCE_DEALLOC
:
11258 case GOMP_MAP_FORCE_DEVICEPTR
:
11259 gcc_assert (is_gimple_omp_oacc (stmt
));
11262 gcc_unreachable ();
11266 case OMP_CLAUSE_TO
:
11267 case OMP_CLAUSE_FROM
:
11268 var
= OMP_CLAUSE_DECL (c
);
11271 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
11272 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11277 if (DECL_SIZE (var
)
11278 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
11280 tree var2
= DECL_VALUE_EXPR (var
);
11281 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
11282 var2
= TREE_OPERAND (var2
, 0);
11283 gcc_assert (DECL_P (var2
));
11287 if (!maybe_lookup_field (var
, ctx
))
11292 x
= build_receiver_ref (var
, true, ctx
);
11293 tree new_var
= lookup_decl (var
, ctx
);
11294 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
11295 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11296 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
11297 x
= build_simple_mem_ref (x
);
11298 SET_DECL_VALUE_EXPR (new_var
, x
);
11299 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
11306 target_nesting_level
++;
11307 lower_omp (&tgt_body
, ctx
);
11308 target_nesting_level
--;
11310 else if (data_region
)
11311 lower_omp (&tgt_body
, ctx
);
11315 /* Declare all the variables created by mapping and the variables
11316 declared in the scope of the target body. */
11317 record_vars_into (ctx
->block_vars
, child_fn
);
11318 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
11323 if (ctx
->record_type
)
11326 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
11327 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11328 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11329 t
= make_tree_vec (3);
11330 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
11331 TREE_VEC_ELT (t
, 1)
11332 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
11333 ".omp_data_sizes");
11334 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
11335 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
11336 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
11339 if (is_gimple_omp_oacc (stmt
))
11341 tkind_type
= short_unsigned_type_node
;
11346 tkind_type
= unsigned_char_type_node
;
11349 TREE_VEC_ELT (t
, 2)
11350 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
11351 ".omp_data_kinds");
11352 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
11353 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
11354 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
11355 gimple_omp_target_set_data_arg (stmt
, t
);
11357 vec
<constructor_elt
, va_gc
> *vsize
;
11358 vec
<constructor_elt
, va_gc
> *vkind
;
11359 vec_alloc (vsize
, map_cnt
);
11360 vec_alloc (vkind
, map_cnt
);
11361 unsigned int map_idx
= 0;
11363 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11364 switch (OMP_CLAUSE_CODE (c
))
11370 case OMP_CLAUSE_MAP
:
11371 case OMP_CLAUSE_TO
:
11372 case OMP_CLAUSE_FROM
:
11374 ovar
= OMP_CLAUSE_DECL (c
);
11375 if (!DECL_P (ovar
))
11377 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11378 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11380 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
11381 == get_base_address (ovar
));
11382 nc
= OMP_CLAUSE_CHAIN (c
);
11383 ovar
= OMP_CLAUSE_DECL (nc
);
11387 tree x
= build_sender_ref (ovar
, ctx
);
11389 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
11390 gimplify_assign (x
, v
, &ilist
);
11396 if (DECL_SIZE (ovar
)
11397 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
11399 tree ovar2
= DECL_VALUE_EXPR (ovar
);
11400 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
11401 ovar2
= TREE_OPERAND (ovar2
, 0);
11402 gcc_assert (DECL_P (ovar2
));
11405 if (!maybe_lookup_field (ovar
, ctx
))
11409 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
11410 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
11411 talign
= DECL_ALIGN_UNIT (ovar
);
11414 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
11415 tree x
= build_sender_ref (ovar
, ctx
);
11416 if (maybe_lookup_oacc_reduction (var
, ctx
))
11418 gcc_checking_assert (offloaded
11419 && is_gimple_omp_oacc (stmt
));
11420 gimplify_assign (x
, var
, &ilist
);
11422 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11423 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
11424 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11425 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
11427 gcc_assert (offloaded
);
11429 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
11430 mark_addressable (avar
);
11431 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
11432 talign
= DECL_ALIGN_UNIT (avar
);
11433 avar
= build_fold_addr_expr (avar
);
11434 gimplify_assign (x
, avar
, &ilist
);
11436 else if (is_gimple_reg (var
))
11438 gcc_assert (offloaded
);
11439 tree avar
= create_tmp_var (TREE_TYPE (var
));
11440 mark_addressable (avar
);
11441 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
11442 if (GOMP_MAP_COPY_TO_P (map_kind
)
11443 || map_kind
== GOMP_MAP_POINTER
11444 || map_kind
== GOMP_MAP_TO_PSET
11445 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
11446 gimplify_assign (avar
, var
, &ilist
);
11447 avar
= build_fold_addr_expr (avar
);
11448 gimplify_assign (x
, avar
, &ilist
);
11449 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
11450 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
11451 && !TYPE_READONLY (TREE_TYPE (var
)))
11453 x
= build_sender_ref (ovar
, ctx
);
11454 x
= build_simple_mem_ref (x
);
11455 gimplify_assign (var
, x
, &olist
);
11460 var
= build_fold_addr_expr (var
);
11461 gimplify_assign (x
, var
, &ilist
);
11464 tree s
= OMP_CLAUSE_SIZE (c
);
11465 if (s
== NULL_TREE
)
11466 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
11467 s
= fold_convert (size_type_node
, s
);
11468 tree purpose
= size_int (map_idx
++);
11469 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
11470 if (TREE_CODE (s
) != INTEGER_CST
)
11471 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
11473 unsigned HOST_WIDE_INT tkind
;
11474 switch (OMP_CLAUSE_CODE (c
))
11476 case OMP_CLAUSE_MAP
:
11477 tkind
= OMP_CLAUSE_MAP_KIND (c
);
11479 case OMP_CLAUSE_TO
:
11480 tkind
= GOMP_MAP_TO
;
11482 case OMP_CLAUSE_FROM
:
11483 tkind
= GOMP_MAP_FROM
;
11486 gcc_unreachable ();
11488 gcc_checking_assert (tkind
11489 < (HOST_WIDE_INT_C (1U) << talign_shift
));
11490 talign
= ceil_log2 (talign
);
11491 tkind
|= talign
<< talign_shift
;
11492 gcc_checking_assert (tkind
11493 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
11494 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
11495 build_int_cstu (tkind_type
, tkind
));
11500 gcc_assert (map_idx
== map_cnt
);
11502 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
11503 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
11504 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
11505 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
11506 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
11508 gimple_seq initlist
= NULL
;
11509 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
11510 TREE_VEC_ELT (t
, 1)),
11511 &initlist
, true, NULL_TREE
);
11512 gimple_seq_add_seq (&ilist
, initlist
);
11514 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
11516 TREE_THIS_VOLATILE (clobber
) = 1;
11517 gimple_seq_add_stmt (&olist
,
11518 gimple_build_assign (TREE_VEC_ELT (t
, 1),
11522 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
11523 TREE_THIS_VOLATILE (clobber
) = 1;
11524 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11528 /* Once all the expansions are done, sequence all the different
11529 fragments inside gimple_omp_body. */
11534 && ctx
->record_type
)
11536 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11537 /* fixup_child_record_type might have changed receiver_decl's type. */
11538 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11539 gimple_seq_add_stmt (&new_body
,
11540 gimple_build_assign (ctx
->receiver_decl
, t
));
11545 gimple_seq_add_seq (&new_body
, tgt_body
);
11546 new_body
= maybe_catch_exception (new_body
);
11548 else if (data_region
)
11549 new_body
= tgt_body
;
11550 if (offloaded
|| data_region
)
11552 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11553 gimple_omp_set_body (stmt
, new_body
);
11556 bind
= gimple_build_bind (NULL
, NULL
,
11557 tgt_bind
? gimple_bind_block (tgt_bind
)
11559 gsi_replace (gsi_p
, bind
, true);
11560 gimple_bind_add_seq (bind
, irlist
);
11561 gimple_bind_add_seq (bind
, ilist
);
11562 gimple_bind_add_stmt (bind
, stmt
);
11563 gimple_bind_add_seq (bind
, olist
);
11564 gimple_bind_add_seq (bind
, orlist
);
11566 pop_gimplify_context (NULL
);
11569 /* Expand code for an OpenMP teams directive. */
11572 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11574 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
11575 push_gimplify_context ();
11577 tree block
= make_node (BLOCK
);
11578 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
11579 gsi_replace (gsi_p
, bind
, true);
11580 gimple_seq bind_body
= NULL
;
11581 gimple_seq dlist
= NULL
;
11582 gimple_seq olist
= NULL
;
11584 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11585 OMP_CLAUSE_NUM_TEAMS
);
11586 if (num_teams
== NULL_TREE
)
11587 num_teams
= build_int_cst (unsigned_type_node
, 0);
11590 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
11591 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
11592 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
11594 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11595 OMP_CLAUSE_THREAD_LIMIT
);
11596 if (thread_limit
== NULL_TREE
)
11597 thread_limit
= build_int_cst (unsigned_type_node
, 0);
11600 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
11601 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
11602 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
11606 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
11607 &bind_body
, &dlist
, ctx
, NULL
);
11608 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
11609 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
11610 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
11612 location_t loc
= gimple_location (teams_stmt
);
11613 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
11614 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
11615 gimple_set_location (call
, loc
);
11616 gimple_seq_add_stmt (&bind_body
, call
);
11618 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
11619 gimple_omp_set_body (teams_stmt
, NULL
);
11620 gimple_seq_add_seq (&bind_body
, olist
);
11621 gimple_seq_add_seq (&bind_body
, dlist
);
11622 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
11623 gimple_bind_set_body (bind
, bind_body
);
11625 pop_gimplify_context (bind
);
11627 gimple_bind_append_vars (bind
, ctx
->block_vars
);
11628 BLOCK_VARS (block
) = ctx
->block_vars
;
11629 if (BLOCK_VARS (block
))
11630 TREE_USED (block
) = 1;
11634 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
11635 regimplified. If DATA is non-NULL, lower_omp_1 is outside
11636 of OMP context, but with task_shared_vars set. */
11639 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
11644 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
11645 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
11648 if (task_shared_vars
11650 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
11653 /* If a global variable has been privatized, TREE_CONSTANT on
11654 ADDR_EXPR might be wrong. */
11655 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
11656 recompute_tree_invariant_for_addr_expr (t
);
11658 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
11663 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11665 gimple stmt
= gsi_stmt (*gsi_p
);
11666 struct walk_stmt_info wi
;
11669 if (gimple_has_location (stmt
))
11670 input_location
= gimple_location (stmt
);
11672 if (task_shared_vars
)
11673 memset (&wi
, '\0', sizeof (wi
));
11675 /* If we have issued syntax errors, avoid doing any heavy lifting.
11676 Just replace the OMP directives with a NOP to avoid
11677 confusing RTL expansion. */
11678 if (seen_error () && is_gimple_omp (stmt
))
11680 gsi_replace (gsi_p
, gimple_build_nop (), true);
11684 switch (gimple_code (stmt
))
11688 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
11689 if ((ctx
|| task_shared_vars
)
11690 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
11691 lower_omp_regimplify_p
,
11692 ctx
? NULL
: &wi
, NULL
)
11693 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
11694 lower_omp_regimplify_p
,
11695 ctx
? NULL
: &wi
, NULL
)))
11696 gimple_regimplify_operands (cond_stmt
, gsi_p
);
11700 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
11702 case GIMPLE_EH_FILTER
:
11703 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
11706 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
11707 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
11709 case GIMPLE_TRANSACTION
:
11710 lower_omp (gimple_transaction_body_ptr (
11711 as_a
<gtransaction
*> (stmt
)),
11715 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
11717 case GIMPLE_OMP_PARALLEL
:
11718 case GIMPLE_OMP_TASK
:
11719 ctx
= maybe_lookup_ctx (stmt
);
11721 if (ctx
->cancellable
)
11722 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11723 lower_omp_taskreg (gsi_p
, ctx
);
11725 case GIMPLE_OMP_FOR
:
11726 ctx
= maybe_lookup_ctx (stmt
);
11728 if (ctx
->cancellable
)
11729 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11730 lower_omp_for (gsi_p
, ctx
);
11732 case GIMPLE_OMP_SECTIONS
:
11733 ctx
= maybe_lookup_ctx (stmt
);
11735 if (ctx
->cancellable
)
11736 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11737 lower_omp_sections (gsi_p
, ctx
);
11739 case GIMPLE_OMP_SINGLE
:
11740 ctx
= maybe_lookup_ctx (stmt
);
11742 lower_omp_single (gsi_p
, ctx
);
11744 case GIMPLE_OMP_MASTER
:
11745 ctx
= maybe_lookup_ctx (stmt
);
11747 lower_omp_master (gsi_p
, ctx
);
11749 case GIMPLE_OMP_TASKGROUP
:
11750 ctx
= maybe_lookup_ctx (stmt
);
11752 lower_omp_taskgroup (gsi_p
, ctx
);
11754 case GIMPLE_OMP_ORDERED
:
11755 ctx
= maybe_lookup_ctx (stmt
);
11757 lower_omp_ordered (gsi_p
, ctx
);
11759 case GIMPLE_OMP_CRITICAL
:
11760 ctx
= maybe_lookup_ctx (stmt
);
11762 lower_omp_critical (gsi_p
, ctx
);
11764 case GIMPLE_OMP_ATOMIC_LOAD
:
11765 if ((ctx
|| task_shared_vars
)
11766 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
11767 as_a
<gomp_atomic_load
*> (stmt
)),
11768 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
11769 gimple_regimplify_operands (stmt
, gsi_p
);
11771 case GIMPLE_OMP_TARGET
:
11772 ctx
= maybe_lookup_ctx (stmt
);
11774 lower_omp_target (gsi_p
, ctx
);
11776 case GIMPLE_OMP_TEAMS
:
11777 ctx
= maybe_lookup_ctx (stmt
);
11779 lower_omp_teams (gsi_p
, ctx
);
11783 call_stmt
= as_a
<gcall
*> (stmt
);
11784 fndecl
= gimple_call_fndecl (call_stmt
);
11786 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
11787 switch (DECL_FUNCTION_CODE (fndecl
))
11789 case BUILT_IN_GOMP_BARRIER
:
11793 case BUILT_IN_GOMP_CANCEL
:
11794 case BUILT_IN_GOMP_CANCELLATION_POINT
:
11797 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
11798 cctx
= cctx
->outer
;
11799 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
11800 if (!cctx
->cancellable
)
11802 if (DECL_FUNCTION_CODE (fndecl
)
11803 == BUILT_IN_GOMP_CANCELLATION_POINT
)
11805 stmt
= gimple_build_nop ();
11806 gsi_replace (gsi_p
, stmt
, false);
11810 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
11812 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
11813 gimple_call_set_fndecl (call_stmt
, fndecl
);
11814 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
11817 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
11818 gimple_call_set_lhs (call_stmt
, lhs
);
11819 tree fallthru_label
;
11820 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
11822 g
= gimple_build_label (fallthru_label
);
11823 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11824 g
= gimple_build_cond (NE_EXPR
, lhs
,
11825 fold_convert (TREE_TYPE (lhs
),
11826 boolean_false_node
),
11827 cctx
->cancel_label
, fallthru_label
);
11828 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11835 if ((ctx
|| task_shared_vars
)
11836 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
11839 /* Just remove clobbers, this should happen only if we have
11840 "privatized" local addressable variables in SIMD regions,
11841 the clobber isn't needed in that case and gimplifying address
11842 of the ARRAY_REF into a pointer and creating MEM_REF based
11843 clobber would create worse code than we get with the clobber
11845 if (gimple_clobber_p (stmt
))
11847 gsi_replace (gsi_p
, gimple_build_nop (), true);
11850 gimple_regimplify_operands (stmt
, gsi_p
);
11857 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
11859 location_t saved_location
= input_location
;
11860 gimple_stmt_iterator gsi
;
11861 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11862 lower_omp_1 (&gsi
, ctx
);
11863 /* During gimplification, we haven't folded statments inside offloading
11864 regions (gimplify.c:maybe_fold_stmt); do that now. */
11865 if (target_nesting_level
)
11866 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11868 input_location
= saved_location
;
11871 /* Main entry point. */
11873 static unsigned int
11874 execute_lower_omp (void)
11880 /* This pass always runs, to provide PROP_gimple_lomp.
11881 But often, there is nothing to do. */
11882 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
11883 && flag_openmp_simd
== 0)
11886 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
11887 delete_omp_context
);
11889 body
= gimple_body (current_function_decl
);
11890 scan_omp (&body
, NULL
);
11891 gcc_assert (taskreg_nesting_level
== 0);
11892 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
11893 finish_taskreg_scan (ctx
);
11894 taskreg_contexts
.release ();
11896 if (all_contexts
->root
)
11898 if (task_shared_vars
)
11899 push_gimplify_context ();
11900 lower_omp (&body
, NULL
);
11901 if (task_shared_vars
)
11902 pop_gimplify_context (NULL
);
11907 splay_tree_delete (all_contexts
);
11908 all_contexts
= NULL
;
11910 BITMAP_FREE (task_shared_vars
);
11916 const pass_data pass_data_lower_omp
=
11918 GIMPLE_PASS
, /* type */
11919 "omplower", /* name */
11920 OPTGROUP_NONE
, /* optinfo_flags */
11921 TV_NONE
, /* tv_id */
11922 PROP_gimple_any
, /* properties_required */
11923 PROP_gimple_lomp
, /* properties_provided */
11924 0, /* properties_destroyed */
11925 0, /* todo_flags_start */
11926 0, /* todo_flags_finish */
11929 class pass_lower_omp
: public gimple_opt_pass
11932 pass_lower_omp (gcc::context
*ctxt
)
11933 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
11936 /* opt_pass methods: */
11937 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
11939 }; // class pass_lower_omp
11941 } // anon namespace
11944 make_pass_lower_omp (gcc::context
*ctxt
)
11946 return new pass_lower_omp (ctxt
);
11949 /* The following is a utility to diagnose structured block violations.
11950 It is not part of the "omplower" pass, as that's invoked too late. It
11951 should be invoked by the respective front ends after gimplification. */
11953 static splay_tree all_labels
;
11955 /* Check for mismatched contexts and generate an error if needed. Return
11956 true if an error is detected. */
11959 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
11960 gimple branch_ctx
, gimple label_ctx
)
11962 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
11963 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
11965 if (label_ctx
== branch_ctx
)
11968 const char* kind
= NULL
;
11973 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
11974 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
11976 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
11977 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
11978 kind
= "Cilk Plus";
11982 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
11983 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
11985 gcc_checking_assert (kind
== NULL
);
11991 gcc_checking_assert (flag_openmp
);
11996 Previously we kept track of the label's entire context in diagnose_sb_[12]
11997 so we could traverse it and issue a correct "exit" or "enter" error
11998 message upon a structured block violation.
12000 We built the context by building a list with tree_cons'ing, but there is
12001 no easy counterpart in gimple tuples. It seems like far too much work
12002 for issuing exit/enter error messages. If someone really misses the
12003 distinct error message... patches welcome.
12007 /* Try to avoid confusing the user by producing and error message
12008 with correct "exit" or "enter" verbiage. We prefer "exit"
12009 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
12010 if (branch_ctx
== NULL
)
12016 if (TREE_VALUE (label_ctx
) == branch_ctx
)
12021 label_ctx
= TREE_CHAIN (label_ctx
);
12026 error ("invalid exit from %s structured block", kind
);
12028 error ("invalid entry to %s structured block", kind
);
12031 /* If it's obvious we have an invalid entry, be specific about the error. */
12032 if (branch_ctx
== NULL
)
12033 error ("invalid entry to %s structured block", kind
);
12036 /* Otherwise, be vague and lazy, but efficient. */
12037 error ("invalid branch to/from %s structured block", kind
);
12040 gsi_replace (gsi_p
, gimple_build_nop (), false);
12044 /* Pass 1: Create a minimal tree of structured blocks, and record
12045 where each label is found. */
12048 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12049 struct walk_stmt_info
*wi
)
12051 gimple context
= (gimple
) wi
->info
;
12052 gimple inner_context
;
12053 gimple stmt
= gsi_stmt (*gsi_p
);
12055 *handled_ops_p
= true;
12057 switch (gimple_code (stmt
))
12061 case GIMPLE_OMP_PARALLEL
:
12062 case GIMPLE_OMP_TASK
:
12063 case GIMPLE_OMP_SECTIONS
:
12064 case GIMPLE_OMP_SINGLE
:
12065 case GIMPLE_OMP_SECTION
:
12066 case GIMPLE_OMP_MASTER
:
12067 case GIMPLE_OMP_ORDERED
:
12068 case GIMPLE_OMP_CRITICAL
:
12069 case GIMPLE_OMP_TARGET
:
12070 case GIMPLE_OMP_TEAMS
:
12071 case GIMPLE_OMP_TASKGROUP
:
12072 /* The minimal context here is just the current OMP construct. */
12073 inner_context
= stmt
;
12074 wi
->info
= inner_context
;
12075 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12076 wi
->info
= context
;
12079 case GIMPLE_OMP_FOR
:
12080 inner_context
= stmt
;
12081 wi
->info
= inner_context
;
12082 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12084 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
12085 diagnose_sb_1
, NULL
, wi
);
12086 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12087 wi
->info
= context
;
12091 splay_tree_insert (all_labels
,
12092 (splay_tree_key
) gimple_label_label (
12093 as_a
<glabel
*> (stmt
)),
12094 (splay_tree_value
) context
);
12104 /* Pass 2: Check each branch and see if its context differs from that of
12105 the destination label's context. */
12108 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12109 struct walk_stmt_info
*wi
)
12111 gimple context
= (gimple
) wi
->info
;
12113 gimple stmt
= gsi_stmt (*gsi_p
);
12115 *handled_ops_p
= true;
12117 switch (gimple_code (stmt
))
12121 case GIMPLE_OMP_PARALLEL
:
12122 case GIMPLE_OMP_TASK
:
12123 case GIMPLE_OMP_SECTIONS
:
12124 case GIMPLE_OMP_SINGLE
:
12125 case GIMPLE_OMP_SECTION
:
12126 case GIMPLE_OMP_MASTER
:
12127 case GIMPLE_OMP_ORDERED
:
12128 case GIMPLE_OMP_CRITICAL
:
12129 case GIMPLE_OMP_TARGET
:
12130 case GIMPLE_OMP_TEAMS
:
12131 case GIMPLE_OMP_TASKGROUP
:
12133 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12134 wi
->info
= context
;
12137 case GIMPLE_OMP_FOR
:
12139 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12141 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
12142 diagnose_sb_2
, NULL
, wi
);
12143 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12144 wi
->info
= context
;
12149 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
12150 tree lab
= gimple_cond_true_label (cond_stmt
);
12153 n
= splay_tree_lookup (all_labels
,
12154 (splay_tree_key
) lab
);
12155 diagnose_sb_0 (gsi_p
, context
,
12156 n
? (gimple
) n
->value
: NULL
);
12158 lab
= gimple_cond_false_label (cond_stmt
);
12161 n
= splay_tree_lookup (all_labels
,
12162 (splay_tree_key
) lab
);
12163 diagnose_sb_0 (gsi_p
, context
,
12164 n
? (gimple
) n
->value
: NULL
);
12171 tree lab
= gimple_goto_dest (stmt
);
12172 if (TREE_CODE (lab
) != LABEL_DECL
)
12175 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12176 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
12180 case GIMPLE_SWITCH
:
12182 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
12184 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
12186 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
12187 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12188 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
12194 case GIMPLE_RETURN
:
12195 diagnose_sb_0 (gsi_p
, context
, NULL
);
12205 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
12208 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
12211 gimple last
= last_stmt (bb
);
12212 enum gimple_code code
= gimple_code (last
);
12213 struct omp_region
*cur_region
= *region
;
12214 bool fallthru
= false;
12218 case GIMPLE_OMP_PARALLEL
:
12219 case GIMPLE_OMP_TASK
:
12220 case GIMPLE_OMP_FOR
:
12221 case GIMPLE_OMP_SINGLE
:
12222 case GIMPLE_OMP_TEAMS
:
12223 case GIMPLE_OMP_MASTER
:
12224 case GIMPLE_OMP_TASKGROUP
:
12225 case GIMPLE_OMP_ORDERED
:
12226 case GIMPLE_OMP_CRITICAL
:
12227 case GIMPLE_OMP_SECTION
:
12228 cur_region
= new_omp_region (bb
, code
, cur_region
);
12232 case GIMPLE_OMP_TARGET
:
12233 cur_region
= new_omp_region (bb
, code
, cur_region
);
12235 switch (gimple_omp_target_kind (last
))
12237 case GF_OMP_TARGET_KIND_REGION
:
12238 case GF_OMP_TARGET_KIND_DATA
:
12239 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12240 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12241 case GF_OMP_TARGET_KIND_OACC_DATA
:
12243 case GF_OMP_TARGET_KIND_UPDATE
:
12244 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12245 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12246 cur_region
= cur_region
->outer
;
12249 gcc_unreachable ();
12253 case GIMPLE_OMP_SECTIONS
:
12254 cur_region
= new_omp_region (bb
, code
, cur_region
);
12258 case GIMPLE_OMP_SECTIONS_SWITCH
:
12262 case GIMPLE_OMP_ATOMIC_LOAD
:
12263 case GIMPLE_OMP_ATOMIC_STORE
:
12267 case GIMPLE_OMP_RETURN
:
12268 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
12269 somewhere other than the next block. This will be
12271 cur_region
->exit
= bb
;
12272 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
12273 cur_region
= cur_region
->outer
;
12276 case GIMPLE_OMP_CONTINUE
:
12277 cur_region
->cont
= bb
;
12278 switch (cur_region
->type
)
12280 case GIMPLE_OMP_FOR
:
12281 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
12282 succs edges as abnormal to prevent splitting
12284 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
12285 /* Make the loopback edge. */
12286 make_edge (bb
, single_succ (cur_region
->entry
),
12289 /* Create an edge from GIMPLE_OMP_FOR to exit, which
12290 corresponds to the case that the body of the loop
12291 is not executed at all. */
12292 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
12293 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
12297 case GIMPLE_OMP_SECTIONS
:
12298 /* Wire up the edges into and out of the nested sections. */
12300 basic_block switch_bb
= single_succ (cur_region
->entry
);
12302 struct omp_region
*i
;
12303 for (i
= cur_region
->inner
; i
; i
= i
->next
)
12305 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
12306 make_edge (switch_bb
, i
->entry
, 0);
12307 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
12310 /* Make the loopback edge to the block with
12311 GIMPLE_OMP_SECTIONS_SWITCH. */
12312 make_edge (bb
, switch_bb
, 0);
12314 /* Make the edge from the switch to exit. */
12315 make_edge (switch_bb
, bb
->next_bb
, 0);
12321 gcc_unreachable ();
12326 gcc_unreachable ();
12329 if (*region
!= cur_region
)
12331 *region
= cur_region
;
12333 *region_idx
= cur_region
->entry
->index
;
12341 static unsigned int
12342 diagnose_omp_structured_block_errors (void)
12344 struct walk_stmt_info wi
;
12345 gimple_seq body
= gimple_body (current_function_decl
);
12347 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
12349 memset (&wi
, 0, sizeof (wi
));
12350 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
12352 memset (&wi
, 0, sizeof (wi
));
12353 wi
.want_locations
= true;
12354 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
12356 gimple_set_body (current_function_decl
, body
);
12358 splay_tree_delete (all_labels
);
12366 const pass_data pass_data_diagnose_omp_blocks
=
12368 GIMPLE_PASS
, /* type */
12369 "*diagnose_omp_blocks", /* name */
12370 OPTGROUP_NONE
, /* optinfo_flags */
12371 TV_NONE
, /* tv_id */
12372 PROP_gimple_any
, /* properties_required */
12373 0, /* properties_provided */
12374 0, /* properties_destroyed */
12375 0, /* todo_flags_start */
12376 0, /* todo_flags_finish */
12379 class pass_diagnose_omp_blocks
: public gimple_opt_pass
12382 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12383 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
12386 /* opt_pass methods: */
12387 virtual bool gate (function
*)
12389 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
12391 virtual unsigned int execute (function
*)
12393 return diagnose_omp_structured_block_errors ();
12396 }; // class pass_diagnose_omp_blocks
12398 } // anon namespace
12401 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12403 return new pass_diagnose_omp_blocks (ctxt
);
12406 /* SIMD clone supporting code. */
12408 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
12409 of arguments to reserve space for. */
12411 static struct cgraph_simd_clone
*
12412 simd_clone_struct_alloc (int nargs
)
12414 struct cgraph_simd_clone
*clone_info
;
12415 size_t len
= (sizeof (struct cgraph_simd_clone
)
12416 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
12417 clone_info
= (struct cgraph_simd_clone
*)
12418 ggc_internal_cleared_alloc (len
);
12422 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
12425 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
12426 struct cgraph_simd_clone
*from
)
12428 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
12429 + ((from
->nargs
- from
->inbranch
)
12430 * sizeof (struct cgraph_simd_clone_arg
))));
12433 /* Return vector of parameter types of function FNDECL. This uses
12434 TYPE_ARG_TYPES if available, otherwise falls back to types of
12435 DECL_ARGUMENTS types. */
12438 simd_clone_vector_of_formal_parm_types (tree fndecl
)
12440 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
12441 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
12442 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
12445 FOR_EACH_VEC_ELT (args
, i
, arg
)
12446 args
[i
] = TREE_TYPE (args
[i
]);
12450 /* Given a simd function in NODE, extract the simd specific
12451 information from the OMP clauses passed in CLAUSES, and return
12452 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
12453 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
12454 otherwise set to FALSE. */
12456 static struct cgraph_simd_clone
*
12457 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
12458 bool *inbranch_specified
)
12460 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12463 *inbranch_specified
= false;
12465 n
= args
.length ();
12466 if (n
> 0 && args
.last () == void_type_node
)
12469 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
12470 be cloned have a distinctive artificial label in addition to "omp
12474 && lookup_attribute ("cilk simd function",
12475 DECL_ATTRIBUTES (node
->decl
)));
12477 /* Allocate one more than needed just in case this is an in-branch
12478 clone which will require a mask argument. */
12479 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
12480 clone_info
->nargs
= n
;
12481 clone_info
->cilk_elemental
= cilk_clone
;
12488 clauses
= TREE_VALUE (clauses
);
12489 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
12492 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
12494 switch (OMP_CLAUSE_CODE (t
))
12496 case OMP_CLAUSE_INBRANCH
:
12497 clone_info
->inbranch
= 1;
12498 *inbranch_specified
= true;
12500 case OMP_CLAUSE_NOTINBRANCH
:
12501 clone_info
->inbranch
= 0;
12502 *inbranch_specified
= true;
12504 case OMP_CLAUSE_SIMDLEN
:
12505 clone_info
->simdlen
12506 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
12508 case OMP_CLAUSE_LINEAR
:
12510 tree decl
= OMP_CLAUSE_DECL (t
);
12511 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
12512 int argno
= TREE_INT_CST_LOW (decl
);
12513 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
12515 clone_info
->args
[argno
].arg_type
12516 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
12517 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12518 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
12519 && clone_info
->args
[argno
].linear_step
< n
);
12523 if (POINTER_TYPE_P (args
[argno
]))
12524 step
= fold_convert (ssizetype
, step
);
12525 if (!tree_fits_shwi_p (step
))
12527 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12528 "ignoring large linear step");
12532 else if (integer_zerop (step
))
12534 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12535 "ignoring zero linear step");
12541 clone_info
->args
[argno
].arg_type
12542 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
12543 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12548 case OMP_CLAUSE_UNIFORM
:
12550 tree decl
= OMP_CLAUSE_DECL (t
);
12551 int argno
= tree_to_uhwi (decl
);
12552 clone_info
->args
[argno
].arg_type
12553 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
12556 case OMP_CLAUSE_ALIGNED
:
12558 tree decl
= OMP_CLAUSE_DECL (t
);
12559 int argno
= tree_to_uhwi (decl
);
12560 clone_info
->args
[argno
].alignment
12561 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
12572 /* Given a SIMD clone in NODE, calculate the characteristic data
12573 type and return the coresponding type. The characteristic data
12574 type is computed as described in the Intel Vector ABI. */
12577 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
12578 struct cgraph_simd_clone
*clone_info
)
12580 tree type
= integer_type_node
;
12581 tree fndecl
= node
->decl
;
12583 /* a) For non-void function, the characteristic data type is the
12585 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
12586 type
= TREE_TYPE (TREE_TYPE (fndecl
));
12588 /* b) If the function has any non-uniform, non-linear parameters,
12589 then the characteristic data type is the type of the first
12593 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
12594 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
12595 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
12603 /* c) If the characteristic data type determined by a) or b) above
12604 is struct, union, or class type which is pass-by-value (except
12605 for the type that maps to the built-in complex data type), the
12606 characteristic data type is int. */
12607 if (RECORD_OR_UNION_TYPE_P (type
)
12608 && !aggregate_value_p (type
, NULL
)
12609 && TREE_CODE (type
) != COMPLEX_TYPE
)
12610 return integer_type_node
;
12612 /* d) If none of the above three classes is applicable, the
12613 characteristic data type is int. */
12617 /* e) For Intel Xeon Phi native and offload compilation, if the
12618 resulting characteristic data type is 8-bit or 16-bit integer
12619 data type, the characteristic data type is int. */
12620 /* Well, we don't handle Xeon Phi yet. */
12624 simd_clone_mangle (struct cgraph_node
*node
,
12625 struct cgraph_simd_clone
*clone_info
)
12627 char vecsize_mangle
= clone_info
->vecsize_mangle
;
12628 char mask
= clone_info
->inbranch
? 'M' : 'N';
12629 unsigned int simdlen
= clone_info
->simdlen
;
12633 gcc_assert (vecsize_mangle
&& simdlen
);
12635 pp_string (&pp
, "_ZGV");
12636 pp_character (&pp
, vecsize_mangle
);
12637 pp_character (&pp
, mask
);
12638 pp_decimal_int (&pp
, simdlen
);
12640 for (n
= 0; n
< clone_info
->nargs
; ++n
)
12642 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
12644 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
12645 pp_character (&pp
, 'u');
12646 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
12648 gcc_assert (arg
.linear_step
!= 0);
12649 pp_character (&pp
, 'l');
12650 if (arg
.linear_step
> 1)
12651 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12652 else if (arg
.linear_step
< 0)
12654 pp_character (&pp
, 'n');
12655 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
12659 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
12661 pp_character (&pp
, 's');
12662 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12665 pp_character (&pp
, 'v');
12668 pp_character (&pp
, 'a');
12669 pp_decimal_int (&pp
, arg
.alignment
);
12673 pp_underscore (&pp
);
12675 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
)));
12676 const char *str
= pp_formatted_text (&pp
);
12678 /* If there already is a SIMD clone with the same mangled name, don't
12679 add another one. This can happen e.g. for
12680 #pragma omp declare simd
12681 #pragma omp declare simd simdlen(8)
12682 int foo (int, int);
12683 if the simdlen is assumed to be 8 for the first one, etc. */
12684 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
12685 clone
= clone
->simdclone
->next_clone
)
12686 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
12690 return get_identifier (str
);
12693 /* Create a simd clone of OLD_NODE and return it. */
12695 static struct cgraph_node
*
12696 simd_clone_create (struct cgraph_node
*old_node
)
12698 struct cgraph_node
*new_node
;
12699 if (old_node
->definition
)
12701 if (!old_node
->has_gimple_body_p ())
12703 old_node
->get_body ();
12704 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
12710 tree old_decl
= old_node
->decl
;
12711 tree new_decl
= copy_node (old_node
->decl
);
12712 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
12713 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
12714 SET_DECL_RTL (new_decl
, NULL
);
12715 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
12716 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
12717 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
12718 symtab
->call_cgraph_insertion_hooks (new_node
);
12720 if (new_node
== NULL
)
12723 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
12725 /* The function cgraph_function_versioning () will force the new
12726 symbol local. Undo this, and inherit external visability from
12728 new_node
->local
.local
= old_node
->local
.local
;
12729 new_node
->externally_visible
= old_node
->externally_visible
;
12734 /* Adjust the return type of the given function to its appropriate
12735 vector counterpart. Returns a simd array to be used throughout the
12736 function as a return value. */
12739 simd_clone_adjust_return_type (struct cgraph_node
*node
)
12741 tree fndecl
= node
->decl
;
12742 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
12743 unsigned int veclen
;
12746 /* Adjust the function return type. */
12747 if (orig_rettype
== void_type_node
)
12749 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
12750 t
= TREE_TYPE (TREE_TYPE (fndecl
));
12751 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
12752 veclen
= node
->simdclone
->vecsize_int
;
12754 veclen
= node
->simdclone
->vecsize_float
;
12755 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
12756 if (veclen
> node
->simdclone
->simdlen
)
12757 veclen
= node
->simdclone
->simdlen
;
12758 if (POINTER_TYPE_P (t
))
12759 t
= pointer_sized_int_node
;
12760 if (veclen
== node
->simdclone
->simdlen
)
12761 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
12764 t
= build_vector_type (t
, veclen
);
12765 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
12767 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
12768 if (!node
->definition
)
12771 t
= DECL_RESULT (fndecl
);
12772 /* Adjust the DECL_RESULT. */
12773 gcc_assert (TREE_TYPE (t
) != void_type_node
);
12774 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
12777 tree atype
= build_array_type_nelts (orig_rettype
,
12778 node
->simdclone
->simdlen
);
12779 if (veclen
!= node
->simdclone
->simdlen
)
12780 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
12782 /* Set up a SIMD array to use as the return value. */
12783 tree retval
= create_tmp_var_raw (atype
, "retval");
12784 gimple_add_tmp_var (retval
);
12788 /* Each vector argument has a corresponding array to be used locally
12789 as part of the eventual loop. Create such temporary array and
12792 PREFIX is the prefix to be used for the temporary.
12794 TYPE is the inner element type.
12796 SIMDLEN is the number of elements. */
12799 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
12801 tree atype
= build_array_type_nelts (type
, simdlen
);
12802 tree avar
= create_tmp_var_raw (atype
, prefix
);
12803 gimple_add_tmp_var (avar
);
12807 /* Modify the function argument types to their corresponding vector
12808 counterparts if appropriate. Also, create one array for each simd
12809 argument to be used locally when using the function arguments as
12812 NODE is the function whose arguments are to be adjusted.
12814 Returns an adjustment vector that will be filled describing how the
12815 argument types will be adjusted. */
12817 static ipa_parm_adjustment_vec
12818 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
12821 ipa_parm_adjustment_vec adjustments
;
12823 if (node
->definition
)
12824 args
= ipa_get_vector_of_formal_parms (node
->decl
);
12826 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12827 adjustments
.create (args
.length ());
12828 unsigned i
, j
, veclen
;
12829 struct ipa_parm_adjustment adj
;
12830 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
12832 memset (&adj
, 0, sizeof (adj
));
12833 tree parm
= args
[i
];
12834 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
12835 adj
.base_index
= i
;
12838 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
12839 node
->simdclone
->args
[i
].orig_type
= parm_type
;
12841 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
12843 /* No adjustment necessary for scalar arguments. */
12844 adj
.op
= IPA_PARM_OP_COPY
;
12848 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
12849 veclen
= node
->simdclone
->vecsize_int
;
12851 veclen
= node
->simdclone
->vecsize_float
;
12852 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
12853 if (veclen
> node
->simdclone
->simdlen
)
12854 veclen
= node
->simdclone
->simdlen
;
12855 adj
.arg_prefix
= "simd";
12856 if (POINTER_TYPE_P (parm_type
))
12857 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
12859 adj
.type
= build_vector_type (parm_type
, veclen
);
12860 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
12861 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12863 adjustments
.safe_push (adj
);
12866 memset (&adj
, 0, sizeof (adj
));
12867 adj
.op
= IPA_PARM_OP_NEW
;
12868 adj
.arg_prefix
= "simd";
12869 adj
.base_index
= i
;
12870 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
12874 if (node
->definition
)
12875 node
->simdclone
->args
[i
].simd_array
12876 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
12877 parm_type
, node
->simdclone
->simdlen
);
12879 adjustments
.safe_push (adj
);
12882 if (node
->simdclone
->inbranch
)
12885 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
12888 memset (&adj
, 0, sizeof (adj
));
12889 adj
.op
= IPA_PARM_OP_NEW
;
12890 adj
.arg_prefix
= "mask";
12892 adj
.base_index
= i
;
12893 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
12894 veclen
= node
->simdclone
->vecsize_int
;
12896 veclen
= node
->simdclone
->vecsize_float
;
12897 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
12898 if (veclen
> node
->simdclone
->simdlen
)
12899 veclen
= node
->simdclone
->simdlen
;
12900 if (POINTER_TYPE_P (base_type
))
12901 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
12903 adj
.type
= build_vector_type (base_type
, veclen
);
12904 adjustments
.safe_push (adj
);
12906 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12907 adjustments
.safe_push (adj
);
12909 /* We have previously allocated one extra entry for the mask. Use
12911 struct cgraph_simd_clone
*sc
= node
->simdclone
;
12913 if (node
->definition
)
12915 sc
->args
[i
].orig_arg
12916 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
12917 sc
->args
[i
].simd_array
12918 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
12920 sc
->args
[i
].orig_type
= base_type
;
12921 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
12924 if (node
->definition
)
12925 ipa_modify_formal_parameters (node
->decl
, adjustments
);
12928 tree new_arg_types
= NULL_TREE
, new_reversed
;
12929 bool last_parm_void
= false;
12930 if (args
.length () > 0 && args
.last () == void_type_node
)
12931 last_parm_void
= true;
12933 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
12934 j
= adjustments
.length ();
12935 for (i
= 0; i
< j
; i
++)
12937 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
12939 if (adj
->op
== IPA_PARM_OP_COPY
)
12940 ptype
= args
[adj
->base_index
];
12943 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
12945 new_reversed
= nreverse (new_arg_types
);
12946 if (last_parm_void
)
12949 TREE_CHAIN (new_arg_types
) = void_list_node
;
12951 new_reversed
= void_list_node
;
12954 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
12955 TYPE_ARG_TYPES (new_type
) = new_reversed
;
12956 TREE_TYPE (node
->decl
) = new_type
;
12958 adjustments
.release ();
12961 return adjustments
;
12964 /* Initialize and copy the function arguments in NODE to their
12965 corresponding local simd arrays. Returns a fresh gimple_seq with
12966 the instruction sequence generated. */
12969 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
12970 ipa_parm_adjustment_vec adjustments
)
12972 gimple_seq seq
= NULL
;
12973 unsigned i
= 0, j
= 0, k
;
12975 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
12977 arg
= DECL_CHAIN (arg
), i
++, j
++)
12979 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
12982 node
->simdclone
->args
[i
].vector_arg
= arg
;
12984 tree array
= node
->simdclone
->args
[i
].simd_array
;
12985 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
12987 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
12988 tree ptr
= build_fold_addr_expr (array
);
12989 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
12990 build_int_cst (ptype
, 0));
12991 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
12992 gimplify_and_add (t
, &seq
);
12996 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
12997 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
12998 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
13000 tree ptr
= build_fold_addr_expr (array
);
13004 arg
= DECL_CHAIN (arg
);
13008 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
13009 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
13010 build_int_cst (ptype
, k
* elemsize
));
13011 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
13012 gimplify_and_add (t
, &seq
);
13019 /* Callback info for ipa_simd_modify_stmt_ops below. */
13021 struct modify_stmt_info
{
13022 ipa_parm_adjustment_vec adjustments
;
13024 /* True if the parent statement was modified by
13025 ipa_simd_modify_stmt_ops. */
13029 /* Callback for walk_gimple_op.
13031 Adjust operands from a given statement as specified in the
13032 adjustments vector in the callback data. */
13035 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
13037 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
13038 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
13039 tree
*orig_tp
= tp
;
13040 if (TREE_CODE (*tp
) == ADDR_EXPR
)
13041 tp
= &TREE_OPERAND (*tp
, 0);
13042 struct ipa_parm_adjustment
*cand
= NULL
;
13043 if (TREE_CODE (*tp
) == PARM_DECL
)
13044 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
13048 *walk_subtrees
= 0;
13051 tree repl
= NULL_TREE
;
13053 repl
= unshare_expr (cand
->new_decl
);
13058 *walk_subtrees
= 0;
13059 bool modified
= info
->modified
;
13060 info
->modified
= false;
13061 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
13062 if (!info
->modified
)
13064 info
->modified
= modified
;
13067 info
->modified
= modified
;
13076 repl
= build_fold_addr_expr (repl
);
13078 if (is_gimple_debug (info
->stmt
))
13080 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
13081 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
13082 DECL_ARTIFICIAL (vexpr
) = 1;
13083 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
13084 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
13089 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
13090 repl
= gimple_assign_lhs (stmt
);
13092 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
13093 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13096 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
13098 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
13104 info
->modified
= true;
13108 /* Traverse the function body and perform all modifications as
13109 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
13110 modified such that the replacement/reduction value will now be an
13111 offset into the corresponding simd_array.
13113 This function will replace all function argument uses with their
13114 corresponding simd array elements, and ajust the return values
13118 ipa_simd_modify_function_body (struct cgraph_node
*node
,
13119 ipa_parm_adjustment_vec adjustments
,
13120 tree retval_array
, tree iter
)
13123 unsigned int i
, j
, l
;
13125 /* Re-use the adjustments array, but this time use it to replace
13126 every function argument use to an offset into the corresponding
13128 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
13130 if (!node
->simdclone
->args
[i
].vector_arg
)
13133 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
13134 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
13135 adjustments
[j
].new_decl
13136 = build4 (ARRAY_REF
,
13138 node
->simdclone
->args
[i
].simd_array
,
13140 NULL_TREE
, NULL_TREE
);
13141 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
13142 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
13143 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
13146 l
= adjustments
.length ();
13147 for (i
= 1; i
< num_ssa_names
; i
++)
13149 tree name
= ssa_name (i
);
13151 && SSA_NAME_VAR (name
)
13152 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
13154 for (j
= 0; j
< l
; j
++)
13155 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
13156 && adjustments
[j
].new_decl
)
13159 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
13162 = copy_var_decl (adjustments
[j
].base
,
13163 DECL_NAME (adjustments
[j
].base
),
13164 TREE_TYPE (adjustments
[j
].base
));
13165 adjustments
[j
].new_ssa_base
= base_var
;
13168 base_var
= adjustments
[j
].new_ssa_base
;
13169 if (SSA_NAME_IS_DEFAULT_DEF (name
))
13171 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13172 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
13173 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
13174 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
13175 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13176 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
13177 gimple stmt
= gimple_build_assign (name
, new_decl
);
13178 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13181 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13186 struct modify_stmt_info info
;
13187 info
.adjustments
= adjustments
;
13189 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
13191 gimple_stmt_iterator gsi
;
13193 gsi
= gsi_start_bb (bb
);
13194 while (!gsi_end_p (gsi
))
13196 gimple stmt
= gsi_stmt (gsi
);
13198 struct walk_stmt_info wi
;
13200 memset (&wi
, 0, sizeof (wi
));
13201 info
.modified
= false;
13203 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
13205 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
13207 tree retval
= gimple_return_retval (return_stmt
);
13210 gsi_remove (&gsi
, true);
13214 /* Replace `return foo' with `retval_array[iter] = foo'. */
13215 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
13216 retval_array
, iter
, NULL
, NULL
);
13217 stmt
= gimple_build_assign (ref
, retval
);
13218 gsi_replace (&gsi
, stmt
, true);
13219 info
.modified
= true;
13224 update_stmt (stmt
);
13225 if (maybe_clean_eh_stmt (stmt
))
13226 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
13233 /* Adjust the argument types in NODE to their appropriate vector
13237 simd_clone_adjust (struct cgraph_node
*node
)
13239 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
13241 targetm
.simd_clone
.adjust (node
);
13243 tree retval
= simd_clone_adjust_return_type (node
);
13244 ipa_parm_adjustment_vec adjustments
13245 = simd_clone_adjust_argument_types (node
);
13247 push_gimplify_context ();
13249 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
13251 /* Adjust all uses of vector arguments accordingly. Adjust all
13252 return values accordingly. */
13253 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
13254 tree iter1
= make_ssa_name (iter
);
13255 tree iter2
= make_ssa_name (iter
);
13256 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
13258 /* Initialize the iteration variable. */
13259 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13260 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
13261 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
13262 /* Insert the SIMD array and iv initialization at function
13264 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
13266 pop_gimplify_context (NULL
);
13268 /* Create a new BB right before the original exit BB, to hold the
13269 iteration increment and the condition/branch. */
13270 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
13271 basic_block incr_bb
= create_empty_bb (orig_exit
);
13272 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
13273 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
13274 flag. Set it now to be a FALLTHRU_EDGE. */
13275 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
13276 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
13277 for (unsigned i
= 0;
13278 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
13280 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
13281 redirect_edge_succ (e
, incr_bb
);
13283 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
13284 e
->probability
= REG_BR_PROB_BASE
;
13285 gsi
= gsi_last_bb (incr_bb
);
13286 gimple g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
13287 build_int_cst (unsigned_type_node
, 1));
13288 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13290 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
13291 struct loop
*loop
= alloc_loop ();
13292 cfun
->has_force_vectorize_loops
= true;
13293 loop
->safelen
= node
->simdclone
->simdlen
;
13294 loop
->force_vectorize
= true;
13295 loop
->header
= body_bb
;
13297 /* Branch around the body if the mask applies. */
13298 if (node
->simdclone
->inbranch
)
13300 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
13302 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
13303 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
13304 tree aref
= build4 (ARRAY_REF
,
13305 TREE_TYPE (TREE_TYPE (mask_array
)),
13308 g
= gimple_build_assign (mask
, aref
);
13309 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13310 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
13311 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
13313 aref
= build1 (VIEW_CONVERT_EXPR
,
13314 build_nonstandard_integer_type (bitsize
, 0), mask
);
13315 mask
= make_ssa_name (TREE_TYPE (aref
));
13316 g
= gimple_build_assign (mask
, aref
);
13317 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13320 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
13322 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13323 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
13324 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
13327 /* Generate the condition. */
13328 g
= gimple_build_cond (LT_EXPR
,
13330 build_int_cst (unsigned_type_node
,
13331 node
->simdclone
->simdlen
),
13333 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13334 e
= split_block (incr_bb
, gsi_stmt (gsi
));
13335 basic_block latch_bb
= e
->dest
;
13336 basic_block new_exit_bb
;
13337 new_exit_bb
= split_block (latch_bb
, NULL
)->dest
;
13338 loop
->latch
= latch_bb
;
13340 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
13342 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
13343 /* The successor of incr_bb is already pointing to latch_bb; just
13345 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
13346 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
13348 gphi
*phi
= create_phi_node (iter1
, body_bb
);
13349 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
13350 edge latch_edge
= single_succ_edge (latch_bb
);
13351 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
13353 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13355 /* Generate the new return. */
13356 gsi
= gsi_last_bb (new_exit_bb
);
13358 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
13359 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
13360 retval
= TREE_OPERAND (retval
, 0);
13363 retval
= build1 (VIEW_CONVERT_EXPR
,
13364 TREE_TYPE (TREE_TYPE (node
->decl
)),
13366 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
13367 false, GSI_CONTINUE_LINKING
);
13369 g
= gimple_build_return (retval
);
13370 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13372 /* Handle aligned clauses by replacing default defs of the aligned
13373 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
13374 lhs. Handle linear by adding PHIs. */
13375 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
13376 if (node
->simdclone
->args
[i
].alignment
13377 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
13378 && (node
->simdclone
->args
[i
].alignment
13379 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
13380 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
13383 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
13384 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13385 tree def
= ssa_default_def (cfun
, orig_arg
);
13386 if (def
&& !has_zero_uses (def
))
13388 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
13389 gimple_seq seq
= NULL
;
13390 bool need_cvt
= false;
13392 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
13394 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
13397 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
13398 gimple_call_set_lhs (g
, t
);
13399 gimple_seq_add_stmt_without_update (&seq
, g
);
13402 t
= make_ssa_name (orig_arg
);
13403 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
13404 gimple_seq_add_stmt_without_update (&seq
, g
);
13406 gsi_insert_seq_on_edge_immediate
13407 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
13409 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13410 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
13412 node
->create_edge (cgraph_node::get_create (fn
),
13413 call
, entry_bb
->count
, freq
);
13415 imm_use_iterator iter
;
13416 use_operand_p use_p
;
13418 tree repl
= gimple_get_lhs (g
);
13419 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13420 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
13423 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13424 SET_USE (use_p
, repl
);
13427 else if (node
->simdclone
->args
[i
].arg_type
13428 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
13430 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13431 tree def
= ssa_default_def (cfun
, orig_arg
);
13432 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13433 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
13434 if (def
&& !has_zero_uses (def
))
13436 iter1
= make_ssa_name (orig_arg
);
13437 iter2
= make_ssa_name (orig_arg
);
13438 phi
= create_phi_node (iter1
, body_bb
);
13439 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
13440 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13441 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13442 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
13443 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13444 ? TREE_TYPE (orig_arg
) : sizetype
;
13446 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
13447 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
13448 gsi
= gsi_last_bb (incr_bb
);
13449 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
13451 imm_use_iterator iter
;
13452 use_operand_p use_p
;
13454 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13455 if (use_stmt
== phi
)
13458 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13459 SET_USE (use_p
, iter1
);
13463 calculate_dominance_info (CDI_DOMINATORS
);
13464 add_loop (loop
, loop
->header
->loop_father
);
13465 update_ssa (TODO_update_ssa
);
13470 /* If the function in NODE is tagged as an elemental SIMD function,
13471 create the appropriate SIMD clones. */
13474 expand_simd_clones (struct cgraph_node
*node
)
13476 tree attr
= lookup_attribute ("omp declare simd",
13477 DECL_ATTRIBUTES (node
->decl
));
13478 if (attr
== NULL_TREE
13479 || node
->global
.inlined_to
13480 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
13484 #pragma omp declare simd
13486 in C, there we don't know the argument types at all. */
13487 if (!node
->definition
13488 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
13493 /* Start with parsing the "omp declare simd" attribute(s). */
13494 bool inbranch_clause_specified
;
13495 struct cgraph_simd_clone
*clone_info
13496 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
13497 &inbranch_clause_specified
);
13498 if (clone_info
== NULL
)
13501 int orig_simdlen
= clone_info
->simdlen
;
13502 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
13503 /* The target can return 0 (no simd clones should be created),
13504 1 (just one ISA of simd clones should be created) or higher
13505 count of ISA variants. In that case, clone_info is initialized
13506 for the first ISA variant. */
13508 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
13513 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
13514 also create one inbranch and one !inbranch clone of it. */
13515 for (int i
= 0; i
< count
* 2; i
++)
13517 struct cgraph_simd_clone
*clone
= clone_info
;
13518 if (inbranch_clause_specified
&& (i
& 1) != 0)
13523 clone
= simd_clone_struct_alloc (clone_info
->nargs
13525 simd_clone_struct_copy (clone
, clone_info
);
13526 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
13527 and simd_clone_adjust_argument_types did to the first
13529 clone
->nargs
-= clone_info
->inbranch
;
13530 clone
->simdlen
= orig_simdlen
;
13531 /* And call the target hook again to get the right ISA. */
13532 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
13536 clone
->inbranch
= 1;
13539 /* simd_clone_mangle might fail if such a clone has been created
13541 tree id
= simd_clone_mangle (node
, clone
);
13542 if (id
== NULL_TREE
)
13545 /* Only when we are sure we want to create the clone actually
13546 clone the function (or definitions) or create another
13547 extern FUNCTION_DECL (for prototypes without definitions). */
13548 struct cgraph_node
*n
= simd_clone_create (node
);
13552 n
->simdclone
= clone
;
13553 clone
->origin
= node
;
13554 clone
->next_clone
= NULL
;
13555 if (node
->simd_clones
== NULL
)
13557 clone
->prev_clone
= n
;
13558 node
->simd_clones
= n
;
13562 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
13563 clone
->prev_clone
->simdclone
->next_clone
= n
;
13564 node
->simd_clones
->simdclone
->prev_clone
= n
;
13566 symtab
->change_decl_assembler_name (n
->decl
, id
);
13567 /* And finally adjust the return type, parameters and for
13568 definitions also function body. */
13569 if (node
->definition
)
13570 simd_clone_adjust (n
);
13573 simd_clone_adjust_return_type (n
);
13574 simd_clone_adjust_argument_types (n
);
13578 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
13581 /* Entry point for IPA simd clone creation pass. */
13583 static unsigned int
13584 ipa_omp_simd_clone (void)
13586 struct cgraph_node
*node
;
13587 FOR_EACH_FUNCTION (node
)
13588 expand_simd_clones (node
);
13594 const pass_data pass_data_omp_simd_clone
=
13596 SIMPLE_IPA_PASS
, /* type */
13597 "simdclone", /* name */
13598 OPTGROUP_NONE
, /* optinfo_flags */
13599 TV_NONE
, /* tv_id */
13600 ( PROP_ssa
| PROP_cfg
), /* properties_required */
13601 0, /* properties_provided */
13602 0, /* properties_destroyed */
13603 0, /* todo_flags_start */
13604 0, /* todo_flags_finish */
13607 class pass_omp_simd_clone
: public simple_ipa_opt_pass
13610 pass_omp_simd_clone(gcc::context
*ctxt
)
13611 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
13614 /* opt_pass methods: */
13615 virtual bool gate (function
*);
13616 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
13620 pass_omp_simd_clone::gate (function
*)
13622 return ((flag_openmp
|| flag_openmp_simd
13624 || (in_lto_p
&& !flag_wpa
))
13625 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
13628 } // anon namespace
13630 simple_ipa_opt_pass
*
13631 make_pass_omp_simd_clone (gcc::context
*ctxt
)
13633 return new pass_omp_simd_clone (ctxt
);
13636 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
13637 adds their addresses and sizes to constructor-vector V_CTOR. */
13639 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
13640 vec
<constructor_elt
, va_gc
> *v_ctor
)
13642 unsigned len
= vec_safe_length (v_decls
);
13643 for (unsigned i
= 0; i
< len
; i
++)
13645 tree it
= (*v_decls
)[i
];
13646 bool is_function
= TREE_CODE (it
) != VAR_DECL
;
13648 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, build_fold_addr_expr (it
));
13650 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
,
13651 fold_convert (const_ptr_type_node
,
13652 DECL_SIZE_UNIT (it
)));
13656 /* Create new symbols containing (address, size) pairs for global variables,
13657 marked with "omp declare target" attribute, as well as addresses for the
13658 functions, which are outlined offloading regions. */
13660 omp_finish_file (void)
13662 unsigned num_funcs
= vec_safe_length (offload_funcs
);
13663 unsigned num_vars
= vec_safe_length (offload_vars
);
13665 if (num_funcs
== 0 && num_vars
== 0)
13668 if (targetm_common
.have_named_sections
)
13670 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
13671 vec_alloc (v_f
, num_funcs
);
13672 vec_alloc (v_v
, num_vars
* 2);
13674 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
13675 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
13677 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13679 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13681 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13682 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13683 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
13684 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
13685 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
13686 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
13687 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13688 get_identifier (".offload_func_table"),
13690 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13691 get_identifier (".offload_var_table"),
13693 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
13694 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
13695 otherwise a joint table in a binary will contain padding between
13696 tables from multiple object files. */
13697 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
13698 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
13699 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
13700 DECL_INITIAL (funcs_decl
) = ctor_f
;
13701 DECL_INITIAL (vars_decl
) = ctor_v
;
13702 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
13703 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
13705 varpool_node::finalize_decl (vars_decl
);
13706 varpool_node::finalize_decl (funcs_decl
);
13710 for (unsigned i
= 0; i
< num_funcs
; i
++)
13712 tree it
= (*offload_funcs
)[i
];
13713 targetm
.record_offload_symbol (it
);
13715 for (unsigned i
= 0; i
< num_vars
; i
++)
13717 tree it
= (*offload_vars
)[i
];
13718 targetm
.record_offload_symbol (it
);
13723 #include "gt-omp-low.h"