1 /* Lowering pass for OMP directives. Converts OMP directives into explicit
2 calls to the runtime library (libgomp), data marshalling to implement data
3 sharing and copying clauses, offloading to accelerators, and more.
5 Contributed by Diego Novillo <dnovillo@redhat.com>
7 Copyright (C) 2005-2015 Free Software Foundation, Inc.
9 This file is part of GCC.
11 GCC is free software; you can redistribute it and/or modify it under
12 the terms of the GNU General Public License as published by the Free
13 Software Foundation; either version 3, or (at your option) any later
16 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
17 WARRANTY; without even the implied warranty of MERCHANTABILITY or
18 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
21 You should have received a copy of the GNU General Public License
22 along with GCC; see the file COPYING3. If not see
23 <http://www.gnu.org/licenses/>. */
27 #include "coretypes.h"
35 #include "fold-const.h"
36 #include "stor-layout.h"
38 #include "internal-fn.h"
39 #include "gimple-fold.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
43 #include "gimple-walk.h"
44 #include "tree-iterator.h"
45 #include "tree-inline.h"
46 #include "langhooks.h"
47 #include "diagnostic-core.h"
50 #include "tree-into-ssa.h"
52 #include "insn-config.h"
63 #include "tree-pass.h"
65 #include "splay-tree.h"
66 #include "insn-codes.h"
70 #include "common/common-target.h"
72 #include "gimple-low.h"
73 #include "tree-cfgcleanup.h"
74 #include "pretty-print.h"
75 #include "alloc-pool.h"
76 #include "symbol-summary.h"
78 #include "tree-nested.h"
82 #include "lto-section-names.h"
83 #include "gomp-constants.h"
86 /* Lowering of OMP parallel and workshare constructs proceeds in two
87 phases. The first phase scans the function looking for OMP statements
88 and then for variables that must be replaced to satisfy data sharing
89 clauses. The second phase expands code for the constructs, as well as
90 re-gimplifying things when variables have been replaced with complex
93 Final code generation is done by pass_expand_omp. The flowgraph is
94 scanned for regions which are then moved to a new
95 function, to be invoked by the thread library, or offloaded. */
97 /* OMP region information. Every parallel and workshare
98 directive is enclosed between two markers, the OMP_* directive
99 and a corresponding OMP_RETURN statement. */
103 /* The enclosing region. */
104 struct omp_region
*outer
;
106 /* First child region. */
107 struct omp_region
*inner
;
109 /* Next peer region. */
110 struct omp_region
*next
;
112 /* Block containing the omp directive as its last stmt. */
115 /* Block containing the OMP_RETURN as its last stmt. */
118 /* Block containing the OMP_CONTINUE as its last stmt. */
121 /* If this is a combined parallel+workshare region, this is a list
122 of additional arguments needed by the combined parallel+workshare
124 vec
<tree
, va_gc
> *ws_args
;
126 /* The code for the omp directive of this region. */
127 enum gimple_code type
;
129 /* Schedule kind, only used for OMP_FOR type regions. */
130 enum omp_clause_schedule_kind sched_kind
;
132 /* True if this is a combined parallel+workshare region. */
133 bool is_combined_parallel
;
136 /* Levels of parallelism as defined by OpenACC. Increasing numbers
137 correspond to deeper loop nesting levels. */
139 #define MASK_WORKER 2
140 #define MASK_VECTOR 4
142 /* Context structure. Used to store information about each parallel
143 directive in the code. */
145 typedef struct omp_context
147 /* This field must be at the beginning, as we do "inheritance": Some
148 callback functions for tree-inline.c (e.g., omp_copy_decl)
149 receive a copy_body_data pointer that is up-casted to an
150 omp_context pointer. */
153 /* The tree of contexts corresponding to the encountered constructs. */
154 struct omp_context
*outer
;
157 /* Map variables to fields in a structure that allows communication
158 between sending and receiving threads. */
159 splay_tree field_map
;
164 /* These are used just by task contexts, if task firstprivate fn is
165 needed. srecord_type is used to communicate from the thread
166 that encountered the task construct to task firstprivate fn,
167 record_type is allocated by GOMP_task, initialized by task firstprivate
168 fn and passed to the task body fn. */
169 splay_tree sfield_map
;
172 /* A chain of variables to add to the top-level block surrounding the
173 construct. In the case of a parallel, this is in the child function. */
176 /* A map of reduction pointer variables. For accelerators, each
177 reduction variable is replaced with an array. Each thread, in turn,
178 is assigned to a slot on that array. */
179 splay_tree reduction_map
;
181 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
182 barriers should jump to during omplower pass. */
185 /* What to do with variables with implicitly determined sharing
187 enum omp_clause_default_kind default_kind
;
189 /* Nesting depth of this context. Used to beautify error messages re
190 invalid gotos. The outermost ctx is depth 1, with depth 0 being
191 reserved for the main body of the function. */
194 /* True if this parallel directive is nested within another. */
197 /* True if this construct can be cancelled. */
200 /* For OpenACC loops, a mask of gang, worker and vector used at
201 levels below this one. */
203 /* For OpenACC loops, a mask of gang, worker and vector used at
204 this level and above. For parallel and kernels clauses, a mask
205 indicating which of num_gangs/num_workers/num_vectors was used. */
209 /* A structure holding the elements of:
210 for (V = N1; V cond N2; V += STEP) [...] */
212 struct omp_for_data_loop
214 tree v
, n1
, n2
, step
;
215 enum tree_code cond_code
;
218 /* A structure describing the main elements of a parallel loop. */
222 struct omp_for_data_loop loop
;
227 bool have_nowait
, have_ordered
;
228 enum omp_clause_schedule_kind sched_kind
;
229 struct omp_for_data_loop
*loops
;
233 static splay_tree all_contexts
;
234 static int taskreg_nesting_level
;
235 static int target_nesting_level
;
236 static struct omp_region
*root_omp_region
;
237 static bitmap task_shared_vars
;
238 static vec
<omp_context
*> taskreg_contexts
;
240 static void scan_omp (gimple_seq
*, omp_context
*);
241 static tree
scan_omp_1_op (tree
*, int *, void *);
243 #define WALK_SUBSTMTS \
247 case GIMPLE_EH_FILTER: \
248 case GIMPLE_TRANSACTION: \
249 /* The sub-statements for these should be walked. */ \
250 *handled_ops_p = false; \
253 /* Helper function to get the name of the array containing the partial
254 reductions for OpenACC reductions. */
256 oacc_get_reduction_array_id (tree node
)
258 const char *id
= IDENTIFIER_POINTER (DECL_NAME (node
));
259 int len
= strlen ("OACC") + strlen (id
);
260 char *temp_name
= XALLOCAVEC (char, len
+ 1);
261 snprintf (temp_name
, len
+ 1, "OACC%s", id
);
262 return IDENTIFIER_POINTER (get_identifier (temp_name
));
265 /* Determine the number of threads OpenACC threads used to determine the
266 size of the array of partial reductions. Currently, this is num_gangs
267 * vector_length. This value may be different than GOACC_GET_NUM_THREADS,
268 because it is independed of the device used. */
271 oacc_max_threads (omp_context
*ctx
)
273 tree nthreads
, vector_length
, gangs
, clauses
;
275 gangs
= fold_convert (sizetype
, integer_one_node
);
276 vector_length
= gangs
;
278 /* The reduction clause may be nested inside a loop directive.
279 Scan for the innermost vector_length clause. */
280 for (omp_context
*oc
= ctx
; oc
; oc
= oc
->outer
)
282 if (gimple_code (oc
->stmt
) != GIMPLE_OMP_TARGET
283 || (gimple_omp_target_kind (oc
->stmt
)
284 != GF_OMP_TARGET_KIND_OACC_PARALLEL
))
287 clauses
= gimple_omp_target_clauses (oc
->stmt
);
289 vector_length
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
291 vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length
),
293 OMP_CLAUSE_VECTOR_LENGTH_EXPR
296 vector_length
= fold_convert (sizetype
, integer_one_node
);
298 gangs
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
300 gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (gangs
), sizetype
,
301 OMP_CLAUSE_NUM_GANGS_EXPR (gangs
));
303 gangs
= fold_convert (sizetype
, integer_one_node
);
308 nthreads
= fold_build2 (MULT_EXPR
, sizetype
, gangs
, vector_length
);
313 /* Holds offload tables with decls. */
314 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
316 /* Convenience function for calling scan_omp_1_op on tree operands. */
319 scan_omp_op (tree
*tp
, omp_context
*ctx
)
321 struct walk_stmt_info wi
;
323 memset (&wi
, 0, sizeof (wi
));
325 wi
.want_locations
= true;
327 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
330 static void lower_omp (gimple_seq
*, omp_context
*);
331 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
332 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
334 /* Find an OMP clause of type KIND within CLAUSES. */
337 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
339 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
340 if (OMP_CLAUSE_CODE (clauses
) == kind
)
346 /* Return true if CTX is for an omp parallel. */
349 is_parallel_ctx (omp_context
*ctx
)
351 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
355 /* Return true if CTX is for an omp task. */
358 is_task_ctx (omp_context
*ctx
)
360 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
364 /* Return true if CTX is for an omp parallel or omp task. */
367 is_taskreg_ctx (omp_context
*ctx
)
369 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
370 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
374 /* Return true if REGION is a combined parallel+workshare region. */
377 is_combined_parallel (struct omp_region
*region
)
379 return region
->is_combined_parallel
;
383 /* Extract the header elements of parallel loop FOR_STMT and store
387 extract_omp_for_data (gomp_for
*for_stmt
, struct omp_for_data
*fd
,
388 struct omp_for_data_loop
*loops
)
390 tree t
, var
, *collapse_iter
, *collapse_count
;
391 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
392 struct omp_for_data_loop
*loop
;
394 struct omp_for_data_loop dummy_loop
;
395 location_t loc
= gimple_location (for_stmt
);
396 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
397 bool distribute
= gimple_omp_for_kind (for_stmt
)
398 == GF_OMP_FOR_KIND_DISTRIBUTE
;
400 fd
->for_stmt
= for_stmt
;
402 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
403 if (fd
->collapse
> 1)
406 fd
->loops
= &fd
->loop
;
408 fd
->have_nowait
= distribute
|| simd
;
409 fd
->have_ordered
= false;
410 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
411 fd
->chunk_size
= NULL_TREE
;
412 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
413 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
414 collapse_iter
= NULL
;
415 collapse_count
= NULL
;
417 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
418 switch (OMP_CLAUSE_CODE (t
))
420 case OMP_CLAUSE_NOWAIT
:
421 fd
->have_nowait
= true;
423 case OMP_CLAUSE_ORDERED
:
424 fd
->have_ordered
= true;
426 case OMP_CLAUSE_SCHEDULE
:
427 gcc_assert (!distribute
);
428 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
429 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
431 case OMP_CLAUSE_DIST_SCHEDULE
:
432 gcc_assert (distribute
);
433 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
435 case OMP_CLAUSE_COLLAPSE
:
436 if (fd
->collapse
> 1)
438 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
439 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
446 /* FIXME: for now map schedule(auto) to schedule(static).
447 There should be analysis to determine whether all iterations
448 are approximately the same amount of work (then schedule(static)
449 is best) or if it varies (then schedule(dynamic,N) is better). */
450 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
452 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
453 gcc_assert (fd
->chunk_size
== NULL
);
455 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
456 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
457 gcc_assert (fd
->chunk_size
== NULL
);
458 else if (fd
->chunk_size
== NULL
)
460 /* We only need to compute a default chunk size for ordered
461 static loops and dynamic loops. */
462 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
464 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
465 ? integer_zero_node
: integer_one_node
;
468 for (i
= 0; i
< fd
->collapse
; i
++)
470 if (fd
->collapse
== 1)
472 else if (loops
!= NULL
)
477 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
478 gcc_assert (SSA_VAR_P (loop
->v
));
479 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
480 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
481 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
482 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
484 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
485 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
486 switch (loop
->cond_code
)
492 gcc_assert (gimple_omp_for_kind (for_stmt
)
493 == GF_OMP_FOR_KIND_CILKSIMD
494 || (gimple_omp_for_kind (for_stmt
)
495 == GF_OMP_FOR_KIND_CILKFOR
));
498 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
499 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
501 loop
->n2
= fold_build2_loc (loc
,
502 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
503 build_int_cst (TREE_TYPE (loop
->n2
), 1));
504 loop
->cond_code
= LT_EXPR
;
507 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
508 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
510 loop
->n2
= fold_build2_loc (loc
,
511 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
512 build_int_cst (TREE_TYPE (loop
->n2
), 1));
513 loop
->cond_code
= GT_EXPR
;
519 t
= gimple_omp_for_incr (for_stmt
, i
);
520 gcc_assert (TREE_OPERAND (t
, 0) == var
);
521 switch (TREE_CODE (t
))
524 loop
->step
= TREE_OPERAND (t
, 1);
526 case POINTER_PLUS_EXPR
:
527 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
530 loop
->step
= TREE_OPERAND (t
, 1);
531 loop
->step
= fold_build1_loc (loc
,
532 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
540 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
541 && !fd
->have_ordered
))
543 if (fd
->collapse
== 1)
544 iter_type
= TREE_TYPE (loop
->v
);
546 || TYPE_PRECISION (iter_type
)
547 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
549 = build_nonstandard_integer_type
550 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
552 else if (iter_type
!= long_long_unsigned_type_node
)
554 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
555 iter_type
= long_long_unsigned_type_node
;
556 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
557 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
558 >= TYPE_PRECISION (iter_type
))
562 if (loop
->cond_code
== LT_EXPR
)
563 n
= fold_build2_loc (loc
,
564 PLUS_EXPR
, TREE_TYPE (loop
->v
),
565 loop
->n2
, loop
->step
);
568 if (TREE_CODE (n
) != INTEGER_CST
569 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
570 iter_type
= long_long_unsigned_type_node
;
572 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
573 > TYPE_PRECISION (iter_type
))
577 if (loop
->cond_code
== LT_EXPR
)
580 n2
= fold_build2_loc (loc
,
581 PLUS_EXPR
, TREE_TYPE (loop
->v
),
582 loop
->n2
, loop
->step
);
586 n1
= fold_build2_loc (loc
,
587 MINUS_EXPR
, TREE_TYPE (loop
->v
),
588 loop
->n2
, loop
->step
);
591 if (TREE_CODE (n1
) != INTEGER_CST
592 || TREE_CODE (n2
) != INTEGER_CST
593 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
594 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
595 iter_type
= long_long_unsigned_type_node
;
599 if (collapse_count
&& *collapse_count
== NULL
)
601 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
602 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
603 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
604 if (t
&& integer_zerop (t
))
605 count
= build_zero_cst (long_long_unsigned_type_node
);
606 else if ((i
== 0 || count
!= NULL_TREE
)
607 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
608 && TREE_CONSTANT (loop
->n1
)
609 && TREE_CONSTANT (loop
->n2
)
610 && TREE_CODE (loop
->step
) == INTEGER_CST
)
612 tree itype
= TREE_TYPE (loop
->v
);
614 if (POINTER_TYPE_P (itype
))
615 itype
= signed_type_for (itype
);
616 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
617 t
= fold_build2_loc (loc
,
619 fold_convert_loc (loc
, itype
, loop
->step
), t
);
620 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
621 fold_convert_loc (loc
, itype
, loop
->n2
));
622 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
623 fold_convert_loc (loc
, itype
, loop
->n1
));
624 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
625 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
626 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
627 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
628 fold_convert_loc (loc
, itype
,
631 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
632 fold_convert_loc (loc
, itype
, loop
->step
));
633 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
634 if (count
!= NULL_TREE
)
635 count
= fold_build2_loc (loc
,
636 MULT_EXPR
, long_long_unsigned_type_node
,
640 if (TREE_CODE (count
) != INTEGER_CST
)
643 else if (count
&& !integer_zerop (count
))
650 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
651 || fd
->have_ordered
))
653 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
654 iter_type
= long_long_unsigned_type_node
;
656 iter_type
= long_integer_type_node
;
658 else if (collapse_iter
&& *collapse_iter
!= NULL
)
659 iter_type
= TREE_TYPE (*collapse_iter
);
660 fd
->iter_type
= iter_type
;
661 if (collapse_iter
&& *collapse_iter
== NULL
)
662 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
663 if (collapse_count
&& *collapse_count
== NULL
)
666 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
668 *collapse_count
= create_tmp_var (iter_type
, ".count");
671 if (fd
->collapse
> 1)
673 fd
->loop
.v
= *collapse_iter
;
674 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
675 fd
->loop
.n2
= *collapse_count
;
676 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
677 fd
->loop
.cond_code
= LT_EXPR
;
680 /* For OpenACC loops, force a chunk size of one, as this avoids the default
681 scheduling where several subsequent iterations are being executed by the
683 if (gimple_omp_for_kind (for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
685 gcc_assert (fd
->chunk_size
== NULL_TREE
);
686 fd
->chunk_size
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
691 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
692 is the immediate dominator of PAR_ENTRY_BB, return true if there
693 are no data dependencies that would prevent expanding the parallel
694 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
696 When expanding a combined parallel+workshare region, the call to
697 the child function may need additional arguments in the case of
698 GIMPLE_OMP_FOR regions. In some cases, these arguments are
699 computed out of variables passed in from the parent to the child
700 via 'struct .omp_data_s'. For instance:
702 #pragma omp parallel for schedule (guided, i * 4)
707 # BLOCK 2 (PAR_ENTRY_BB)
709 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
711 # BLOCK 3 (WS_ENTRY_BB)
712 .omp_data_i = &.omp_data_o;
713 D.1667 = .omp_data_i->i;
715 #pragma omp for schedule (guided, D.1598)
717 When we outline the parallel region, the call to the child function
718 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
719 that value is computed *after* the call site. So, in principle we
720 cannot do the transformation.
722 To see whether the code in WS_ENTRY_BB blocks the combined
723 parallel+workshare call, we collect all the variables used in the
724 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
725 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
728 FIXME. If we had the SSA form built at this point, we could merely
729 hoist the code in block 3 into block 2 and be done with it. But at
730 this point we don't have dataflow information and though we could
731 hack something up here, it is really not worth the aggravation. */
734 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
736 struct omp_for_data fd
;
737 gimple ws_stmt
= last_stmt (ws_entry_bb
);
739 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
742 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
744 extract_omp_for_data (as_a
<gomp_for
*> (ws_stmt
), &fd
, NULL
);
746 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
748 if (fd
.iter_type
!= long_integer_type_node
)
751 /* FIXME. We give up too easily here. If any of these arguments
752 are not constants, they will likely involve variables that have
753 been mapped into fields of .omp_data_s for sharing with the child
754 function. With appropriate data flow, it would be possible to
756 if (!is_gimple_min_invariant (fd
.loop
.n1
)
757 || !is_gimple_min_invariant (fd
.loop
.n2
)
758 || !is_gimple_min_invariant (fd
.loop
.step
)
759 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
766 /* Collect additional arguments needed to emit a combined
767 parallel+workshare call. WS_STMT is the workshare directive being
770 static vec
<tree
, va_gc
> *
771 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
774 location_t loc
= gimple_location (ws_stmt
);
775 vec
<tree
, va_gc
> *ws_args
;
777 if (gomp_for
*for_stmt
= dyn_cast
<gomp_for
*> (ws_stmt
))
779 struct omp_for_data fd
;
782 extract_omp_for_data (for_stmt
, &fd
, NULL
);
786 if (gimple_omp_for_combined_into_p (for_stmt
))
789 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
790 OMP_CLAUSE__LOOPTEMP_
);
792 n1
= OMP_CLAUSE_DECL (innerc
);
793 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
794 OMP_CLAUSE__LOOPTEMP_
);
796 n2
= OMP_CLAUSE_DECL (innerc
);
799 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
801 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
802 ws_args
->quick_push (t
);
804 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
805 ws_args
->quick_push (t
);
807 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
808 ws_args
->quick_push (t
);
812 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
813 ws_args
->quick_push (t
);
818 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
820 /* Number of sections is equal to the number of edges from the
821 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
822 the exit of the sections region. */
823 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
824 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
825 vec_alloc (ws_args
, 1);
826 ws_args
->quick_push (t
);
834 /* Discover whether REGION is a combined parallel+workshare region. */
837 determine_parallel_type (struct omp_region
*region
)
839 basic_block par_entry_bb
, par_exit_bb
;
840 basic_block ws_entry_bb
, ws_exit_bb
;
842 if (region
== NULL
|| region
->inner
== NULL
843 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
844 || region
->inner
->cont
== NULL
)
847 /* We only support parallel+for and parallel+sections. */
848 if (region
->type
!= GIMPLE_OMP_PARALLEL
849 || (region
->inner
->type
!= GIMPLE_OMP_FOR
850 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
853 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
854 WS_EXIT_BB -> PAR_EXIT_BB. */
855 par_entry_bb
= region
->entry
;
856 par_exit_bb
= region
->exit
;
857 ws_entry_bb
= region
->inner
->entry
;
858 ws_exit_bb
= region
->inner
->exit
;
860 if (single_succ (par_entry_bb
) == ws_entry_bb
861 && single_succ (ws_exit_bb
) == par_exit_bb
862 && workshare_safe_to_combine_p (ws_entry_bb
)
863 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
864 || (last_and_only_stmt (ws_entry_bb
)
865 && last_and_only_stmt (par_exit_bb
))))
867 gimple par_stmt
= last_stmt (par_entry_bb
);
868 gimple ws_stmt
= last_stmt (ws_entry_bb
);
870 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
872 /* If this is a combined parallel loop, we need to determine
873 whether or not to use the combined library calls. There
874 are two cases where we do not apply the transformation:
875 static loops and any kind of ordered loop. In the first
876 case, we already open code the loop so there is no need
877 to do anything else. In the latter case, the combined
878 parallel loop call would still need extra synchronization
879 to implement ordered semantics, so there would not be any
880 gain in using the combined call. */
881 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
882 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
884 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
885 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
887 region
->is_combined_parallel
= false;
888 region
->inner
->is_combined_parallel
= false;
893 region
->is_combined_parallel
= true;
894 region
->inner
->is_combined_parallel
= true;
895 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
900 /* Return true if EXPR is variable sized. */
903 is_variable_sized (const_tree expr
)
905 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
908 /* Return true if DECL is a reference type. */
911 is_reference (tree decl
)
913 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
916 /* Return the type of a decl. If the decl is reference type,
917 return its base type. */
919 get_base_type (tree decl
)
921 tree type
= TREE_TYPE (decl
);
922 if (is_reference (decl
))
923 type
= TREE_TYPE (type
);
927 /* Lookup variables. The "maybe" form
928 allows for the variable form to not have been entered, otherwise we
929 assert that the variable must have been entered. */
932 lookup_decl (tree var
, omp_context
*ctx
)
934 tree
*n
= ctx
->cb
.decl_map
->get (var
);
939 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
941 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
942 return n
? *n
: NULL_TREE
;
946 lookup_field (tree var
, omp_context
*ctx
)
949 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
950 return (tree
) n
->value
;
954 lookup_sfield (tree var
, omp_context
*ctx
)
957 n
= splay_tree_lookup (ctx
->sfield_map
958 ? ctx
->sfield_map
: ctx
->field_map
,
959 (splay_tree_key
) var
);
960 return (tree
) n
->value
;
964 maybe_lookup_field (tree var
, omp_context
*ctx
)
967 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
968 return n
? (tree
) n
->value
: NULL_TREE
;
972 lookup_oacc_reduction (const char *id
, omp_context
*ctx
)
975 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) id
);
976 return (tree
) n
->value
;
980 maybe_lookup_oacc_reduction (tree var
, omp_context
*ctx
)
982 splay_tree_node n
= NULL
;
983 if (ctx
->reduction_map
)
984 n
= splay_tree_lookup (ctx
->reduction_map
, (splay_tree_key
) var
);
985 return n
? (tree
) n
->value
: NULL_TREE
;
988 /* Return true if DECL should be copied by pointer. SHARED_CTX is
989 the parallel context if DECL is to be shared. */
992 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
994 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
997 /* We can only use copy-in/copy-out semantics for shared variables
998 when we know the value is not accessible from an outer scope. */
1001 gcc_assert (!is_gimple_omp_oacc (shared_ctx
->stmt
));
1003 /* ??? Trivially accessible from anywhere. But why would we even
1004 be passing an address in this case? Should we simply assert
1005 this to be false, or should we have a cleanup pass that removes
1006 these from the list of mappings? */
1007 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
1010 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
1011 without analyzing the expression whether or not its location
1012 is accessible to anyone else. In the case of nested parallel
1013 regions it certainly may be. */
1014 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
1017 /* Do not use copy-in/copy-out for variables that have their
1019 if (TREE_ADDRESSABLE (decl
))
1022 /* lower_send_shared_vars only uses copy-in, but not copy-out
1024 if (TREE_READONLY (decl
)
1025 || ((TREE_CODE (decl
) == RESULT_DECL
1026 || TREE_CODE (decl
) == PARM_DECL
)
1027 && DECL_BY_REFERENCE (decl
)))
1030 /* Disallow copy-in/out in nested parallel if
1031 decl is shared in outer parallel, otherwise
1032 each thread could store the shared variable
1033 in its own copy-in location, making the
1034 variable no longer really shared. */
1035 if (shared_ctx
->is_nested
)
1039 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
1040 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
1047 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
1048 c
; c
= OMP_CLAUSE_CHAIN (c
))
1049 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
1050 && OMP_CLAUSE_DECL (c
) == decl
)
1054 goto maybe_mark_addressable_and_ret
;
1058 /* For tasks avoid using copy-in/out. As tasks can be
1059 deferred or executed in different thread, when GOMP_task
1060 returns, the task hasn't necessarily terminated. */
1061 if (is_task_ctx (shared_ctx
))
1064 maybe_mark_addressable_and_ret
:
1065 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
1066 if (is_gimple_reg (outer
))
1068 /* Taking address of OUTER in lower_send_shared_vars
1069 might need regimplification of everything that uses the
1071 if (!task_shared_vars
)
1072 task_shared_vars
= BITMAP_ALLOC (NULL
);
1073 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
1074 TREE_ADDRESSABLE (outer
) = 1;
1083 /* Construct a new automatic decl similar to VAR. */
1086 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
1088 tree copy
= copy_var_decl (var
, name
, type
);
1090 DECL_CONTEXT (copy
) = current_function_decl
;
1091 DECL_CHAIN (copy
) = ctx
->block_vars
;
1092 ctx
->block_vars
= copy
;
1098 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
1100 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
1103 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
1106 omp_build_component_ref (tree obj
, tree field
)
1108 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
1109 if (TREE_THIS_VOLATILE (field
))
1110 TREE_THIS_VOLATILE (ret
) |= 1;
1111 if (TREE_READONLY (field
))
1112 TREE_READONLY (ret
) |= 1;
1116 /* Build tree nodes to access the field for VAR on the receiver side. */
1119 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
1121 tree x
, field
= lookup_field (var
, ctx
);
1123 /* If the receiver record type was remapped in the child function,
1124 remap the field into the new record type. */
1125 x
= maybe_lookup_field (field
, ctx
);
1129 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1130 x
= omp_build_component_ref (x
, field
);
1132 x
= build_simple_mem_ref (x
);
1137 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1138 of a parallel, this is a component reference; for workshare constructs
1139 this is some variable. */
1142 build_outer_var_ref (tree var
, omp_context
*ctx
)
1146 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1148 else if (is_variable_sized (var
))
1150 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1151 x
= build_outer_var_ref (x
, ctx
);
1152 x
= build_simple_mem_ref (x
);
1154 else if (is_taskreg_ctx (ctx
))
1156 bool by_ref
= use_pointer_for_field (var
, NULL
);
1157 x
= build_receiver_ref (var
, by_ref
, ctx
);
1159 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1160 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1162 /* #pragma omp simd isn't a worksharing construct, and can reference even
1163 private vars in its linear etc. clauses. */
1165 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1166 x
= lookup_decl (var
, ctx
->outer
);
1167 else if (ctx
->outer
)
1168 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1172 else if (ctx
->outer
)
1173 x
= lookup_decl (var
, ctx
->outer
);
1174 else if (is_reference (var
))
1175 /* This can happen with orphaned constructs. If var is reference, it is
1176 possible it is shared and as such valid. */
1181 if (is_reference (var
))
1182 x
= build_simple_mem_ref (x
);
1187 /* Build tree nodes to access the field for VAR on the sender side. */
1190 build_sender_ref (tree var
, omp_context
*ctx
)
1192 tree field
= lookup_sfield (var
, ctx
);
1193 return omp_build_component_ref (ctx
->sender_decl
, field
);
1196 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1199 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1201 tree field
, type
, sfield
= NULL_TREE
;
1203 gcc_assert ((mask
& 1) == 0
1204 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1205 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1206 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1207 gcc_assert ((mask
& 3) == 3
1208 || !is_gimple_omp_oacc (ctx
->stmt
));
1210 type
= TREE_TYPE (var
);
1213 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1214 type
= build_pointer_type (build_pointer_type (type
));
1217 type
= build_pointer_type (type
);
1218 else if ((mask
& 3) == 1 && is_reference (var
))
1219 type
= TREE_TYPE (type
);
1221 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1222 FIELD_DECL
, DECL_NAME (var
), type
);
1224 /* Remember what variable this field was created for. This does have a
1225 side effect of making dwarf2out ignore this member, so for helpful
1226 debugging we clear it later in delete_omp_context. */
1227 DECL_ABSTRACT_ORIGIN (field
) = var
;
1228 if (type
== TREE_TYPE (var
))
1230 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1231 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1232 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1235 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1237 if ((mask
& 3) == 3)
1239 insert_field_into_struct (ctx
->record_type
, field
);
1240 if (ctx
->srecord_type
)
1242 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1243 FIELD_DECL
, DECL_NAME (var
), type
);
1244 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1245 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1246 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1247 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1248 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1253 if (ctx
->srecord_type
== NULL_TREE
)
1257 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1258 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1259 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1261 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1262 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1263 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1264 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1265 splay_tree_insert (ctx
->sfield_map
,
1266 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1267 (splay_tree_value
) sfield
);
1271 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1272 : ctx
->srecord_type
, field
);
1276 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1277 (splay_tree_value
) field
);
1278 if ((mask
& 2) && ctx
->sfield_map
)
1279 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1280 (splay_tree_value
) sfield
);
1284 install_var_local (tree var
, omp_context
*ctx
)
1286 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1287 insert_decl_map (&ctx
->cb
, var
, new_var
);
1291 /* Adjust the replacement for DECL in CTX for the new context. This means
1292 copying the DECL_VALUE_EXPR, and fixing up the type. */
1295 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1297 tree new_decl
, size
;
1299 new_decl
= lookup_decl (decl
, ctx
);
1301 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1303 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1304 && DECL_HAS_VALUE_EXPR_P (decl
))
1306 tree ve
= DECL_VALUE_EXPR (decl
);
1307 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1308 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1309 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1312 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1314 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1315 if (size
== error_mark_node
)
1316 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1317 DECL_SIZE (new_decl
) = size
;
1319 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1320 if (size
== error_mark_node
)
1321 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1322 DECL_SIZE_UNIT (new_decl
) = size
;
1326 /* The callback for remap_decl. Search all containing contexts for a
1327 mapping of the variable; this avoids having to duplicate the splay
1328 tree ahead of time. We know a mapping doesn't already exist in the
1329 given context. Create new mappings to implement default semantics. */
1332 omp_copy_decl (tree var
, copy_body_data
*cb
)
1334 omp_context
*ctx
= (omp_context
*) cb
;
1337 if (TREE_CODE (var
) == LABEL_DECL
)
1339 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1340 DECL_CONTEXT (new_var
) = current_function_decl
;
1341 insert_decl_map (&ctx
->cb
, var
, new_var
);
1345 while (!is_taskreg_ctx (ctx
))
1350 new_var
= maybe_lookup_decl (var
, ctx
);
1355 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1358 return error_mark_node
;
1362 /* Debugging dumps for parallel regions. */
1363 void dump_omp_region (FILE *, struct omp_region
*, int);
1364 void debug_omp_region (struct omp_region
*);
1365 void debug_all_omp_regions (void);
1367 /* Dump the parallel region tree rooted at REGION. */
1370 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1372 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1373 gimple_code_name
[region
->type
]);
1376 dump_omp_region (file
, region
->inner
, indent
+ 4);
1380 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1381 region
->cont
->index
);
1385 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1386 region
->exit
->index
);
1388 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1391 dump_omp_region (file
, region
->next
, indent
);
1395 debug_omp_region (struct omp_region
*region
)
1397 dump_omp_region (stderr
, region
, 0);
1401 debug_all_omp_regions (void)
1403 dump_omp_region (stderr
, root_omp_region
, 0);
1407 /* Create a new parallel region starting at STMT inside region PARENT. */
1409 static struct omp_region
*
1410 new_omp_region (basic_block bb
, enum gimple_code type
,
1411 struct omp_region
*parent
)
1413 struct omp_region
*region
= XCNEW (struct omp_region
);
1415 region
->outer
= parent
;
1417 region
->type
= type
;
1421 /* This is a nested region. Add it to the list of inner
1422 regions in PARENT. */
1423 region
->next
= parent
->inner
;
1424 parent
->inner
= region
;
1428 /* This is a toplevel region. Add it to the list of toplevel
1429 regions in ROOT_OMP_REGION. */
1430 region
->next
= root_omp_region
;
1431 root_omp_region
= region
;
1437 /* Release the memory associated with the region tree rooted at REGION. */
1440 free_omp_region_1 (struct omp_region
*region
)
1442 struct omp_region
*i
, *n
;
1444 for (i
= region
->inner
; i
; i
= n
)
1447 free_omp_region_1 (i
);
1453 /* Release the memory for the entire omp region tree. */
1456 free_omp_regions (void)
1458 struct omp_region
*r
, *n
;
1459 for (r
= root_omp_region
; r
; r
= n
)
1462 free_omp_region_1 (r
);
1464 root_omp_region
= NULL
;
1468 /* Create a new context, with OUTER_CTX being the surrounding context. */
1470 static omp_context
*
1471 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1473 omp_context
*ctx
= XCNEW (omp_context
);
1475 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1476 (splay_tree_value
) ctx
);
1481 ctx
->outer
= outer_ctx
;
1482 ctx
->cb
= outer_ctx
->cb
;
1483 ctx
->cb
.block
= NULL
;
1484 ctx
->depth
= outer_ctx
->depth
+ 1;
1485 ctx
->reduction_map
= outer_ctx
->reduction_map
;
1489 ctx
->cb
.src_fn
= current_function_decl
;
1490 ctx
->cb
.dst_fn
= current_function_decl
;
1491 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1492 gcc_checking_assert (ctx
->cb
.src_node
);
1493 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1494 ctx
->cb
.src_cfun
= cfun
;
1495 ctx
->cb
.copy_decl
= omp_copy_decl
;
1496 ctx
->cb
.eh_lp_nr
= 0;
1497 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1501 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1506 static gimple_seq
maybe_catch_exception (gimple_seq
);
1508 /* Finalize task copyfn. */
1511 finalize_task_copyfn (gomp_task
*task_stmt
)
1513 struct function
*child_cfun
;
1515 gimple_seq seq
= NULL
, new_seq
;
1518 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1519 if (child_fn
== NULL_TREE
)
1522 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1523 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1525 push_cfun (child_cfun
);
1526 bind
= gimplify_body (child_fn
, false);
1527 gimple_seq_add_stmt (&seq
, bind
);
1528 new_seq
= maybe_catch_exception (seq
);
1531 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1533 gimple_seq_add_stmt (&seq
, bind
);
1535 gimple_set_body (child_fn
, seq
);
1538 /* Inform the callgraph about the new function. */
1539 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
1540 node
->parallelized_function
= 1;
1541 cgraph_node::add_new_function (child_fn
, false);
1544 /* Destroy a omp_context data structures. Called through the splay tree
1545 value delete callback. */
1548 delete_omp_context (splay_tree_value value
)
1550 omp_context
*ctx
= (omp_context
*) value
;
1552 delete ctx
->cb
.decl_map
;
1555 splay_tree_delete (ctx
->field_map
);
1556 if (ctx
->sfield_map
)
1557 splay_tree_delete (ctx
->sfield_map
);
1558 /* Reduction map is copied to nested contexts, so only delete it in the
1560 if (ctx
->reduction_map
1561 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
1562 && is_gimple_omp_offloaded (ctx
->stmt
)
1563 && is_gimple_omp_oacc (ctx
->stmt
))
1564 splay_tree_delete (ctx
->reduction_map
);
1566 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1567 it produces corrupt debug information. */
1568 if (ctx
->record_type
)
1571 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1572 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1574 if (ctx
->srecord_type
)
1577 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1578 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1581 if (is_task_ctx (ctx
))
1582 finalize_task_copyfn (as_a
<gomp_task
*> (ctx
->stmt
));
1587 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1591 fixup_child_record_type (omp_context
*ctx
)
1593 tree f
, type
= ctx
->record_type
;
1595 /* ??? It isn't sufficient to just call remap_type here, because
1596 variably_modified_type_p doesn't work the way we expect for
1597 record types. Testing each field for whether it needs remapping
1598 and creating a new record by hand works, however. */
1599 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1600 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1604 tree name
, new_fields
= NULL
;
1606 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1607 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1608 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1609 TYPE_DECL
, name
, type
);
1610 TYPE_NAME (type
) = name
;
1612 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1614 tree new_f
= copy_node (f
);
1615 DECL_CONTEXT (new_f
) = type
;
1616 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1617 DECL_CHAIN (new_f
) = new_fields
;
1618 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1619 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1621 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1625 /* Arrange to be able to look up the receiver field
1626 given the sender field. */
1627 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1628 (splay_tree_value
) new_f
);
1630 TYPE_FIELDS (type
) = nreverse (new_fields
);
1634 TREE_TYPE (ctx
->receiver_decl
)
1635 = build_qualified_type (build_reference_type (type
), TYPE_QUAL_RESTRICT
);
1638 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1639 specified by CLAUSES. */
1642 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1645 bool scan_array_reductions
= false;
1647 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1651 switch (OMP_CLAUSE_CODE (c
))
1653 case OMP_CLAUSE_PRIVATE
:
1654 decl
= OMP_CLAUSE_DECL (c
);
1655 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1657 else if (!is_variable_sized (decl
))
1658 install_var_local (decl
, ctx
);
1661 case OMP_CLAUSE_SHARED
:
1662 decl
= OMP_CLAUSE_DECL (c
);
1663 /* Ignore shared directives in teams construct. */
1664 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1666 /* Global variables don't need to be copied,
1667 the receiver side will use them directly. */
1668 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1669 if (is_global_var (odecl
))
1671 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1674 gcc_assert (is_taskreg_ctx (ctx
));
1675 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1676 || !is_variable_sized (decl
));
1677 /* Global variables don't need to be copied,
1678 the receiver side will use them directly. */
1679 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1681 by_ref
= use_pointer_for_field (decl
, ctx
);
1682 if (! TREE_READONLY (decl
)
1683 || TREE_ADDRESSABLE (decl
)
1685 || is_reference (decl
))
1687 install_var_field (decl
, by_ref
, 3, ctx
);
1688 install_var_local (decl
, ctx
);
1691 /* We don't need to copy const scalar vars back. */
1692 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1695 case OMP_CLAUSE_LASTPRIVATE
:
1696 /* Let the corresponding firstprivate clause create
1698 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1702 case OMP_CLAUSE_FIRSTPRIVATE
:
1703 if (is_gimple_omp_oacc (ctx
->stmt
))
1705 sorry ("clause not supported yet");
1709 case OMP_CLAUSE_REDUCTION
:
1710 case OMP_CLAUSE_LINEAR
:
1711 decl
= OMP_CLAUSE_DECL (c
);
1713 if (is_variable_sized (decl
))
1715 if (is_task_ctx (ctx
))
1716 install_var_field (decl
, false, 1, ctx
);
1719 else if (is_taskreg_ctx (ctx
))
1722 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1723 by_ref
= use_pointer_for_field (decl
, NULL
);
1725 if (is_task_ctx (ctx
)
1726 && (global
|| by_ref
|| is_reference (decl
)))
1728 install_var_field (decl
, false, 1, ctx
);
1730 install_var_field (decl
, by_ref
, 2, ctx
);
1733 install_var_field (decl
, by_ref
, 3, ctx
);
1735 install_var_local (decl
, ctx
);
1736 if (is_gimple_omp_oacc (ctx
->stmt
)
1737 && OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
1739 /* Create a decl for the reduction array. */
1740 tree var
= OMP_CLAUSE_DECL (c
);
1741 tree type
= get_base_type (var
);
1742 tree ptype
= build_pointer_type (type
);
1743 tree array
= create_tmp_var (ptype
,
1744 oacc_get_reduction_array_id (var
));
1745 omp_context
*c
= (ctx
->field_map
? ctx
: ctx
->outer
);
1746 install_var_field (array
, true, 3, c
);
1747 install_var_local (array
, c
);
1749 /* Insert it into the current context. */
1750 splay_tree_insert (ctx
->reduction_map
, (splay_tree_key
)
1751 oacc_get_reduction_array_id (var
),
1752 (splay_tree_value
) array
);
1753 splay_tree_insert (ctx
->reduction_map
,
1754 (splay_tree_key
) array
,
1755 (splay_tree_value
) array
);
1759 case OMP_CLAUSE__LOOPTEMP_
:
1760 gcc_assert (is_parallel_ctx (ctx
));
1761 decl
= OMP_CLAUSE_DECL (c
);
1762 install_var_field (decl
, false, 3, ctx
);
1763 install_var_local (decl
, ctx
);
1766 case OMP_CLAUSE_COPYPRIVATE
:
1767 case OMP_CLAUSE_COPYIN
:
1768 decl
= OMP_CLAUSE_DECL (c
);
1769 by_ref
= use_pointer_for_field (decl
, NULL
);
1770 install_var_field (decl
, by_ref
, 3, ctx
);
1773 case OMP_CLAUSE_DEFAULT
:
1774 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1777 case OMP_CLAUSE_FINAL
:
1779 case OMP_CLAUSE_NUM_THREADS
:
1780 case OMP_CLAUSE_NUM_TEAMS
:
1781 case OMP_CLAUSE_THREAD_LIMIT
:
1782 case OMP_CLAUSE_DEVICE
:
1783 case OMP_CLAUSE_SCHEDULE
:
1784 case OMP_CLAUSE_DIST_SCHEDULE
:
1785 case OMP_CLAUSE_DEPEND
:
1786 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1787 case OMP_CLAUSE_NUM_GANGS
:
1788 case OMP_CLAUSE_NUM_WORKERS
:
1789 case OMP_CLAUSE_VECTOR_LENGTH
:
1791 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1795 case OMP_CLAUSE_FROM
:
1796 case OMP_CLAUSE_MAP
:
1798 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1799 decl
= OMP_CLAUSE_DECL (c
);
1800 /* Global variables with "omp declare target" attribute
1801 don't need to be copied, the receiver side will use them
1803 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1805 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1806 && varpool_node::get_create (decl
)->offloadable
)
1808 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1809 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
)
1811 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are
1812 not offloaded; there is nothing to map for those. */
1813 if (!is_gimple_omp_offloaded (ctx
->stmt
)
1814 && !POINTER_TYPE_P (TREE_TYPE (decl
))
1815 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
1820 if (DECL_SIZE (decl
)
1821 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1823 tree decl2
= DECL_VALUE_EXPR (decl
);
1824 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1825 decl2
= TREE_OPERAND (decl2
, 0);
1826 gcc_assert (DECL_P (decl2
));
1827 install_var_field (decl2
, true, 3, ctx
);
1828 install_var_local (decl2
, ctx
);
1829 install_var_local (decl
, ctx
);
1833 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1834 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
1835 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1836 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1837 install_var_field (decl
, true, 7, ctx
);
1839 install_var_field (decl
, true, 3, ctx
);
1840 if (is_gimple_omp_offloaded (ctx
->stmt
))
1841 install_var_local (decl
, ctx
);
1846 tree base
= get_base_address (decl
);
1847 tree nc
= OMP_CLAUSE_CHAIN (c
);
1850 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1851 && OMP_CLAUSE_DECL (nc
) == base
1852 && OMP_CLAUSE_MAP_KIND (nc
) == GOMP_MAP_POINTER
1853 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1855 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1856 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1862 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1863 decl
= OMP_CLAUSE_DECL (c
);
1865 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1866 (splay_tree_key
) decl
));
1868 = build_decl (OMP_CLAUSE_LOCATION (c
),
1869 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1870 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1871 insert_field_into_struct (ctx
->record_type
, field
);
1872 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1873 (splay_tree_value
) field
);
1878 case OMP_CLAUSE_NOWAIT
:
1879 case OMP_CLAUSE_ORDERED
:
1880 case OMP_CLAUSE_COLLAPSE
:
1881 case OMP_CLAUSE_UNTIED
:
1882 case OMP_CLAUSE_MERGEABLE
:
1883 case OMP_CLAUSE_PROC_BIND
:
1884 case OMP_CLAUSE_SAFELEN
:
1885 case OMP_CLAUSE_ASYNC
:
1886 case OMP_CLAUSE_WAIT
:
1887 case OMP_CLAUSE_GANG
:
1888 case OMP_CLAUSE_WORKER
:
1889 case OMP_CLAUSE_VECTOR
:
1892 case OMP_CLAUSE_ALIGNED
:
1893 decl
= OMP_CLAUSE_DECL (c
);
1894 if (is_global_var (decl
)
1895 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1896 install_var_local (decl
, ctx
);
1899 case OMP_CLAUSE_DEVICE_RESIDENT
:
1900 case OMP_CLAUSE_USE_DEVICE
:
1901 case OMP_CLAUSE__CACHE_
:
1902 case OMP_CLAUSE_INDEPENDENT
:
1903 case OMP_CLAUSE_AUTO
:
1904 case OMP_CLAUSE_SEQ
:
1905 sorry ("Clause not supported yet");
1913 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1915 switch (OMP_CLAUSE_CODE (c
))
1917 case OMP_CLAUSE_LASTPRIVATE
:
1918 /* Let the corresponding firstprivate clause create
1920 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1921 scan_array_reductions
= true;
1922 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1926 case OMP_CLAUSE_FIRSTPRIVATE
:
1927 if (is_gimple_omp_oacc (ctx
->stmt
))
1929 sorry ("clause not supported yet");
1933 case OMP_CLAUSE_PRIVATE
:
1934 case OMP_CLAUSE_REDUCTION
:
1935 case OMP_CLAUSE_LINEAR
:
1936 decl
= OMP_CLAUSE_DECL (c
);
1937 if (is_variable_sized (decl
))
1938 install_var_local (decl
, ctx
);
1939 fixup_remapped_decl (decl
, ctx
,
1940 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1941 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1942 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1943 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1944 scan_array_reductions
= true;
1945 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1946 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1947 scan_array_reductions
= true;
1950 case OMP_CLAUSE_SHARED
:
1951 /* Ignore shared directives in teams construct. */
1952 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1954 decl
= OMP_CLAUSE_DECL (c
);
1955 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1956 fixup_remapped_decl (decl
, ctx
, false);
1959 case OMP_CLAUSE_MAP
:
1960 if (!is_gimple_omp_offloaded (ctx
->stmt
))
1962 decl
= OMP_CLAUSE_DECL (c
);
1964 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1965 && varpool_node::get_create (decl
)->offloadable
)
1969 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
1970 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1971 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1973 tree new_decl
= lookup_decl (decl
, ctx
);
1974 TREE_TYPE (new_decl
)
1975 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1977 else if (DECL_SIZE (decl
)
1978 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1980 tree decl2
= DECL_VALUE_EXPR (decl
);
1981 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1982 decl2
= TREE_OPERAND (decl2
, 0);
1983 gcc_assert (DECL_P (decl2
));
1984 fixup_remapped_decl (decl2
, ctx
, false);
1985 fixup_remapped_decl (decl
, ctx
, true);
1988 fixup_remapped_decl (decl
, ctx
, false);
1992 case OMP_CLAUSE_COPYPRIVATE
:
1993 case OMP_CLAUSE_COPYIN
:
1994 case OMP_CLAUSE_DEFAULT
:
1996 case OMP_CLAUSE_NUM_THREADS
:
1997 case OMP_CLAUSE_NUM_TEAMS
:
1998 case OMP_CLAUSE_THREAD_LIMIT
:
1999 case OMP_CLAUSE_DEVICE
:
2000 case OMP_CLAUSE_SCHEDULE
:
2001 case OMP_CLAUSE_DIST_SCHEDULE
:
2002 case OMP_CLAUSE_NOWAIT
:
2003 case OMP_CLAUSE_ORDERED
:
2004 case OMP_CLAUSE_COLLAPSE
:
2005 case OMP_CLAUSE_UNTIED
:
2006 case OMP_CLAUSE_FINAL
:
2007 case OMP_CLAUSE_MERGEABLE
:
2008 case OMP_CLAUSE_PROC_BIND
:
2009 case OMP_CLAUSE_SAFELEN
:
2010 case OMP_CLAUSE_ALIGNED
:
2011 case OMP_CLAUSE_DEPEND
:
2012 case OMP_CLAUSE__LOOPTEMP_
:
2014 case OMP_CLAUSE_FROM
:
2015 case OMP_CLAUSE__CILK_FOR_COUNT_
:
2016 case OMP_CLAUSE_ASYNC
:
2017 case OMP_CLAUSE_WAIT
:
2018 case OMP_CLAUSE_NUM_GANGS
:
2019 case OMP_CLAUSE_NUM_WORKERS
:
2020 case OMP_CLAUSE_VECTOR_LENGTH
:
2021 case OMP_CLAUSE_GANG
:
2022 case OMP_CLAUSE_WORKER
:
2023 case OMP_CLAUSE_VECTOR
:
2026 case OMP_CLAUSE_DEVICE_RESIDENT
:
2027 case OMP_CLAUSE_USE_DEVICE
:
2028 case OMP_CLAUSE__CACHE_
:
2029 case OMP_CLAUSE_INDEPENDENT
:
2030 case OMP_CLAUSE_AUTO
:
2031 case OMP_CLAUSE_SEQ
:
2032 sorry ("Clause not supported yet");
2040 gcc_checking_assert (!scan_array_reductions
2041 || !is_gimple_omp_oacc (ctx
->stmt
));
2042 if (scan_array_reductions
)
2043 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2044 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
2045 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
2047 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
2048 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
2050 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
2051 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
2052 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
2053 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
2054 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
2055 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
2058 /* Create a new name for omp child function. Returns an identifier. If
2059 IS_CILK_FOR is true then the suffix for the child function is
2063 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
2066 return clone_function_name (current_function_decl
, "_cilk_for_fn");
2067 return clone_function_name (current_function_decl
,
2068 task_copy
? "_omp_cpyfn" : "_omp_fn");
2071 /* Returns the type of the induction variable for the child function for
2072 _Cilk_for and the types for _high and _low variables based on TYPE. */
2075 cilk_for_check_loop_diff_type (tree type
)
2077 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
2079 if (TYPE_UNSIGNED (type
))
2080 return uint32_type_node
;
2082 return integer_type_node
;
2086 if (TYPE_UNSIGNED (type
))
2087 return uint64_type_node
;
2089 return long_long_integer_type_node
;
2093 /* Build a decl for the omp child function. It'll not contain a body
2094 yet, just the bare decl. */
2097 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
2099 tree decl
, type
, name
, t
;
2102 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2103 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
2104 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
2105 tree cilk_var_type
= NULL_TREE
;
2107 name
= create_omp_child_function_name (task_copy
,
2108 cilk_for_count
!= NULL_TREE
);
2110 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2111 ptr_type_node
, NULL_TREE
);
2112 else if (cilk_for_count
)
2114 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
2115 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
2116 type
= build_function_type_list (void_type_node
, ptr_type_node
,
2117 cilk_var_type
, cilk_var_type
, NULL_TREE
);
2120 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
2122 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
2124 gcc_checking_assert (!is_gimple_omp_oacc (ctx
->stmt
)
2127 ctx
->cb
.dst_fn
= decl
;
2129 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
2131 TREE_STATIC (decl
) = 1;
2132 TREE_USED (decl
) = 1;
2133 DECL_ARTIFICIAL (decl
) = 1;
2134 DECL_IGNORED_P (decl
) = 0;
2135 TREE_PUBLIC (decl
) = 0;
2136 DECL_UNINLINABLE (decl
) = 1;
2137 DECL_EXTERNAL (decl
) = 0;
2138 DECL_CONTEXT (decl
) = NULL_TREE
;
2139 DECL_INITIAL (decl
) = make_node (BLOCK
);
2140 if (cgraph_node::get (current_function_decl
)->offloadable
)
2141 cgraph_node::get_create (decl
)->offloadable
= 1;
2145 for (octx
= ctx
; octx
; octx
= octx
->outer
)
2146 if (is_gimple_omp_offloaded (octx
->stmt
))
2148 cgraph_node::get_create (decl
)->offloadable
= 1;
2149 #ifdef ENABLE_OFFLOADING
2150 g
->have_offload
= true;
2156 if (cgraph_node::get_create (decl
)->offloadable
2157 && !lookup_attribute ("omp declare target",
2158 DECL_ATTRIBUTES (current_function_decl
)))
2159 DECL_ATTRIBUTES (decl
)
2160 = tree_cons (get_identifier ("omp target entrypoint"),
2161 NULL_TREE
, DECL_ATTRIBUTES (decl
));
2163 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2164 RESULT_DECL
, NULL_TREE
, void_type_node
);
2165 DECL_ARTIFICIAL (t
) = 1;
2166 DECL_IGNORED_P (t
) = 1;
2167 DECL_CONTEXT (t
) = decl
;
2168 DECL_RESULT (decl
) = t
;
2170 /* _Cilk_for's child function requires two extra parameters called
2171 __low and __high that are set the by Cilk runtime when it calls this
2175 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2176 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
2177 DECL_ARTIFICIAL (t
) = 1;
2178 DECL_NAMELESS (t
) = 1;
2179 DECL_ARG_TYPE (t
) = ptr_type_node
;
2180 DECL_CONTEXT (t
) = current_function_decl
;
2182 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2183 DECL_ARGUMENTS (decl
) = t
;
2185 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2186 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
2187 DECL_ARTIFICIAL (t
) = 1;
2188 DECL_NAMELESS (t
) = 1;
2189 DECL_ARG_TYPE (t
) = ptr_type_node
;
2190 DECL_CONTEXT (t
) = current_function_decl
;
2192 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2193 DECL_ARGUMENTS (decl
) = t
;
2196 tree data_name
= get_identifier (".omp_data_i");
2197 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
2199 DECL_ARTIFICIAL (t
) = 1;
2200 DECL_NAMELESS (t
) = 1;
2201 DECL_ARG_TYPE (t
) = ptr_type_node
;
2202 DECL_CONTEXT (t
) = current_function_decl
;
2205 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2206 DECL_ARGUMENTS (decl
) = t
;
2208 ctx
->receiver_decl
= t
;
2211 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2212 PARM_DECL
, get_identifier (".omp_data_o"),
2214 DECL_ARTIFICIAL (t
) = 1;
2215 DECL_NAMELESS (t
) = 1;
2216 DECL_ARG_TYPE (t
) = ptr_type_node
;
2217 DECL_CONTEXT (t
) = current_function_decl
;
2219 TREE_ADDRESSABLE (t
) = 1;
2220 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2221 DECL_ARGUMENTS (decl
) = t
;
2224 /* Allocate memory for the function structure. The call to
2225 allocate_struct_function clobbers CFUN, so we need to restore
2227 push_struct_function (decl
);
2228 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2232 /* Callback for walk_gimple_seq. Check if combined parallel
2233 contains gimple_omp_for_combined_into_p OMP_FOR. */
2236 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2237 bool *handled_ops_p
,
2238 struct walk_stmt_info
*wi
)
2240 gimple stmt
= gsi_stmt (*gsi_p
);
2242 *handled_ops_p
= true;
2243 switch (gimple_code (stmt
))
2247 case GIMPLE_OMP_FOR
:
2248 if (gimple_omp_for_combined_into_p (stmt
)
2249 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
2252 return integer_zero_node
;
2261 /* Scan an OpenMP parallel directive. */
2264 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2268 gomp_parallel
*stmt
= as_a
<gomp_parallel
*> (gsi_stmt (*gsi
));
2270 /* Ignore parallel directives with empty bodies, unless there
2271 are copyin clauses. */
2273 && empty_body_p (gimple_omp_body (stmt
))
2274 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2275 OMP_CLAUSE_COPYIN
) == NULL
)
2277 gsi_replace (gsi
, gimple_build_nop (), false);
2281 if (gimple_omp_parallel_combined_p (stmt
))
2283 struct walk_stmt_info wi
;
2285 memset (&wi
, 0, sizeof (wi
));
2287 walk_gimple_seq (gimple_omp_body (stmt
),
2288 find_combined_for
, NULL
, &wi
);
2291 gomp_for
*for_stmt
= as_a
<gomp_for
*> ((gimple
) wi
.info
);
2292 struct omp_for_data fd
;
2293 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2294 /* We need two temporaries with fd.loop.v type (istart/iend)
2295 and then (fd.collapse - 1) temporaries with the same
2296 type for count2 ... countN-1 vars if not constant. */
2297 size_t count
= 2, i
;
2298 tree type
= fd
.iter_type
;
2300 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2301 count
+= fd
.collapse
- 1;
2302 for (i
= 0; i
< count
; i
++)
2304 tree temp
= create_tmp_var (type
);
2305 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2306 OMP_CLAUSE__LOOPTEMP_
);
2307 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2308 OMP_CLAUSE_DECL (c
) = temp
;
2309 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2310 gimple_omp_parallel_set_clauses (stmt
, c
);
2315 ctx
= new_omp_context (stmt
, outer_ctx
);
2316 taskreg_contexts
.safe_push (ctx
);
2317 if (taskreg_nesting_level
> 1)
2318 ctx
->is_nested
= true;
2319 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2320 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2321 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2322 name
= create_tmp_var_name (".omp_data_s");
2323 name
= build_decl (gimple_location (stmt
),
2324 TYPE_DECL
, name
, ctx
->record_type
);
2325 DECL_ARTIFICIAL (name
) = 1;
2326 DECL_NAMELESS (name
) = 1;
2327 TYPE_NAME (ctx
->record_type
) = name
;
2328 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2329 create_omp_child_function (ctx
, false);
2330 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2332 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2333 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2335 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2336 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2339 /* Scan an OpenMP task directive. */
2342 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2346 gomp_task
*stmt
= as_a
<gomp_task
*> (gsi_stmt (*gsi
));
2348 /* Ignore task directives with empty bodies. */
2350 && empty_body_p (gimple_omp_body (stmt
)))
2352 gsi_replace (gsi
, gimple_build_nop (), false);
2356 ctx
= new_omp_context (stmt
, outer_ctx
);
2357 taskreg_contexts
.safe_push (ctx
);
2358 if (taskreg_nesting_level
> 1)
2359 ctx
->is_nested
= true;
2360 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2361 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2362 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2363 name
= create_tmp_var_name (".omp_data_s");
2364 name
= build_decl (gimple_location (stmt
),
2365 TYPE_DECL
, name
, ctx
->record_type
);
2366 DECL_ARTIFICIAL (name
) = 1;
2367 DECL_NAMELESS (name
) = 1;
2368 TYPE_NAME (ctx
->record_type
) = name
;
2369 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2370 create_omp_child_function (ctx
, false);
2371 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2373 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2375 if (ctx
->srecord_type
)
2377 name
= create_tmp_var_name (".omp_data_a");
2378 name
= build_decl (gimple_location (stmt
),
2379 TYPE_DECL
, name
, ctx
->srecord_type
);
2380 DECL_ARTIFICIAL (name
) = 1;
2381 DECL_NAMELESS (name
) = 1;
2382 TYPE_NAME (ctx
->srecord_type
) = name
;
2383 TYPE_ARTIFICIAL (ctx
->srecord_type
) = 1;
2384 create_omp_child_function (ctx
, true);
2387 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2389 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2391 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2392 t
= build_int_cst (long_integer_type_node
, 0);
2393 gimple_omp_task_set_arg_size (stmt
, t
);
2394 t
= build_int_cst (long_integer_type_node
, 1);
2395 gimple_omp_task_set_arg_align (stmt
, t
);
2400 /* If any decls have been made addressable during scan_omp,
2401 adjust their fields if needed, and layout record types
2402 of parallel/task constructs. */
2405 finish_taskreg_scan (omp_context
*ctx
)
2407 if (ctx
->record_type
== NULL_TREE
)
2410 /* If any task_shared_vars were needed, verify all
2411 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2412 statements if use_pointer_for_field hasn't changed
2413 because of that. If it did, update field types now. */
2414 if (task_shared_vars
)
2418 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2419 c
; c
= OMP_CLAUSE_CHAIN (c
))
2420 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
2422 tree decl
= OMP_CLAUSE_DECL (c
);
2424 /* Global variables don't need to be copied,
2425 the receiver side will use them directly. */
2426 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2428 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2429 || !use_pointer_for_field (decl
, ctx
))
2431 tree field
= lookup_field (decl
, ctx
);
2432 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2433 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2435 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2436 TREE_THIS_VOLATILE (field
) = 0;
2437 DECL_USER_ALIGN (field
) = 0;
2438 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2439 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2440 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2441 if (ctx
->srecord_type
)
2443 tree sfield
= lookup_sfield (decl
, ctx
);
2444 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2445 TREE_THIS_VOLATILE (sfield
) = 0;
2446 DECL_USER_ALIGN (sfield
) = 0;
2447 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2448 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2449 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2454 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2456 layout_type (ctx
->record_type
);
2457 fixup_child_record_type (ctx
);
2461 location_t loc
= gimple_location (ctx
->stmt
);
2462 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2463 /* Move VLA fields to the end. */
2464 p
= &TYPE_FIELDS (ctx
->record_type
);
2466 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2467 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2470 *p
= TREE_CHAIN (*p
);
2471 TREE_CHAIN (*q
) = NULL_TREE
;
2472 q
= &TREE_CHAIN (*q
);
2475 p
= &DECL_CHAIN (*p
);
2477 layout_type (ctx
->record_type
);
2478 fixup_child_record_type (ctx
);
2479 if (ctx
->srecord_type
)
2480 layout_type (ctx
->srecord_type
);
2481 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2482 TYPE_SIZE_UNIT (ctx
->record_type
));
2483 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2484 t
= build_int_cst (long_integer_type_node
,
2485 TYPE_ALIGN_UNIT (ctx
->record_type
));
2486 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2491 static omp_context
*
2492 enclosing_target_ctx (omp_context
*ctx
)
2495 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
2497 gcc_assert (ctx
!= NULL
);
2502 oacc_loop_or_target_p (gimple stmt
)
2504 enum gimple_code outer_type
= gimple_code (stmt
);
2505 return ((outer_type
== GIMPLE_OMP_TARGET
2506 && ((gimple_omp_target_kind (stmt
)
2507 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
2508 || (gimple_omp_target_kind (stmt
)
2509 == GF_OMP_TARGET_KIND_OACC_KERNELS
)))
2510 || (outer_type
== GIMPLE_OMP_FOR
2511 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
));
2514 /* Scan a GIMPLE_OMP_FOR. */
2517 scan_omp_for (gomp_for
*stmt
, omp_context
*outer_ctx
)
2519 enum gimple_code outer_type
= GIMPLE_ERROR_MARK
;
2522 tree clauses
= gimple_omp_for_clauses (stmt
);
2525 outer_type
= gimple_code (outer_ctx
->stmt
);
2527 ctx
= new_omp_context (stmt
, outer_ctx
);
2529 if (is_gimple_omp_oacc (stmt
))
2531 if (outer_ctx
&& outer_type
== GIMPLE_OMP_FOR
)
2532 ctx
->gwv_this
= outer_ctx
->gwv_this
;
2533 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2536 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_GANG
)
2538 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WORKER
)
2540 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_VECTOR
)
2544 ctx
->gwv_this
|= val
;
2547 /* Skip; not nested inside a region. */
2550 if (!oacc_loop_or_target_p (outer_ctx
->stmt
))
2552 /* Skip; not nested inside an OpenACC region. */
2555 if (outer_type
== GIMPLE_OMP_FOR
)
2556 outer_ctx
->gwv_below
|= val
;
2557 if (OMP_CLAUSE_OPERAND (c
, 0) != NULL_TREE
)
2559 omp_context
*enclosing
= enclosing_target_ctx (outer_ctx
);
2560 if (gimple_omp_target_kind (enclosing
->stmt
)
2561 == GF_OMP_TARGET_KIND_OACC_PARALLEL
)
2562 error_at (gimple_location (stmt
),
2563 "no arguments allowed to gang, worker and vector clauses inside parallel");
2568 scan_sharing_clauses (clauses
, ctx
);
2570 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2571 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2573 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2574 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2575 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2576 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2578 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2580 if (is_gimple_omp_oacc (stmt
))
2582 if (ctx
->gwv_this
& ctx
->gwv_below
)
2583 error_at (gimple_location (stmt
),
2584 "gang, worker and vector may occur only once in a loop nest");
2585 else if (ctx
->gwv_below
!= 0
2586 && ctx
->gwv_this
> ctx
->gwv_below
)
2587 error_at (gimple_location (stmt
),
2588 "gang, worker and vector must occur in this order in a loop nest");
2589 if (outer_ctx
&& outer_type
== GIMPLE_OMP_FOR
)
2590 outer_ctx
->gwv_below
|= ctx
->gwv_below
;
2594 /* Scan an OpenMP sections directive. */
2597 scan_omp_sections (gomp_sections
*stmt
, omp_context
*outer_ctx
)
2601 ctx
= new_omp_context (stmt
, outer_ctx
);
2602 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2603 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2606 /* Scan an OpenMP single directive. */
2609 scan_omp_single (gomp_single
*stmt
, omp_context
*outer_ctx
)
2614 ctx
= new_omp_context (stmt
, outer_ctx
);
2615 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2616 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2617 name
= create_tmp_var_name (".omp_copy_s");
2618 name
= build_decl (gimple_location (stmt
),
2619 TYPE_DECL
, name
, ctx
->record_type
);
2620 TYPE_NAME (ctx
->record_type
) = name
;
2622 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2623 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2625 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2626 ctx
->record_type
= NULL
;
2628 layout_type (ctx
->record_type
);
2631 /* Scan a GIMPLE_OMP_TARGET. */
2634 scan_omp_target (gomp_target
*stmt
, omp_context
*outer_ctx
)
2638 bool offloaded
= is_gimple_omp_offloaded (stmt
);
2639 tree clauses
= gimple_omp_target_clauses (stmt
);
2641 ctx
= new_omp_context (stmt
, outer_ctx
);
2642 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2643 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2644 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2645 name
= create_tmp_var_name (".omp_data_t");
2646 name
= build_decl (gimple_location (stmt
),
2647 TYPE_DECL
, name
, ctx
->record_type
);
2648 DECL_ARTIFICIAL (name
) = 1;
2649 DECL_NAMELESS (name
) = 1;
2650 TYPE_NAME (ctx
->record_type
) = name
;
2651 TYPE_ARTIFICIAL (ctx
->record_type
) = 1;
2654 if (is_gimple_omp_oacc (stmt
))
2655 ctx
->reduction_map
= splay_tree_new (splay_tree_compare_pointers
,
2658 create_omp_child_function (ctx
, false);
2659 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2662 if (is_gimple_omp_oacc (stmt
))
2664 for (tree c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
2666 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_NUM_GANGS
)
2667 ctx
->gwv_this
|= MASK_GANG
;
2668 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_NUM_WORKERS
)
2669 ctx
->gwv_this
|= MASK_WORKER
;
2670 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_VECTOR_LENGTH
)
2671 ctx
->gwv_this
|= MASK_VECTOR
;
2675 scan_sharing_clauses (clauses
, ctx
);
2676 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2678 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2679 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2682 TYPE_FIELDS (ctx
->record_type
)
2683 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2684 #ifdef ENABLE_CHECKING
2686 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2687 for (field
= TYPE_FIELDS (ctx
->record_type
);
2689 field
= DECL_CHAIN (field
))
2690 gcc_assert (DECL_ALIGN (field
) == align
);
2692 layout_type (ctx
->record_type
);
2694 fixup_child_record_type (ctx
);
2698 /* Scan an OpenMP teams directive. */
2701 scan_omp_teams (gomp_teams
*stmt
, omp_context
*outer_ctx
)
2703 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2704 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2705 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2708 /* Check nesting restrictions. */
2710 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2712 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
2713 inside an OpenACC CTX. */
2714 if (!(is_gimple_omp (stmt
)
2715 && is_gimple_omp_oacc (stmt
)))
2717 for (omp_context
*ctx_
= ctx
; ctx_
!= NULL
; ctx_
= ctx_
->outer
)
2718 if (is_gimple_omp (ctx_
->stmt
)
2719 && is_gimple_omp_oacc (ctx_
->stmt
))
2721 error_at (gimple_location (stmt
),
2722 "non-OpenACC construct inside of OpenACC region");
2729 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2730 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2732 error_at (gimple_location (stmt
),
2733 "OpenMP constructs may not be nested inside simd region");
2736 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2738 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2739 || (gimple_omp_for_kind (stmt
)
2740 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2741 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2743 error_at (gimple_location (stmt
),
2744 "only distribute or parallel constructs are allowed to "
2745 "be closely nested inside teams construct");
2750 switch (gimple_code (stmt
))
2752 case GIMPLE_OMP_FOR
:
2753 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
2755 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2757 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2759 error_at (gimple_location (stmt
),
2760 "distribute construct must be closely nested inside "
2768 if (is_gimple_call (stmt
)
2769 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2770 == BUILT_IN_GOMP_CANCEL
2771 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2772 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2774 const char *bad
= NULL
;
2775 const char *kind
= NULL
;
2778 error_at (gimple_location (stmt
), "orphaned %qs construct",
2779 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2780 == BUILT_IN_GOMP_CANCEL
2781 ? "#pragma omp cancel"
2782 : "#pragma omp cancellation point");
2785 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2786 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2790 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2791 bad
= "#pragma omp parallel";
2792 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2793 == BUILT_IN_GOMP_CANCEL
2794 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2795 ctx
->cancellable
= true;
2799 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2800 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2801 bad
= "#pragma omp for";
2802 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2803 == BUILT_IN_GOMP_CANCEL
2804 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2806 ctx
->cancellable
= true;
2807 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2809 warning_at (gimple_location (stmt
), 0,
2810 "%<#pragma omp cancel for%> inside "
2811 "%<nowait%> for construct");
2812 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2813 OMP_CLAUSE_ORDERED
))
2814 warning_at (gimple_location (stmt
), 0,
2815 "%<#pragma omp cancel for%> inside "
2816 "%<ordered%> for construct");
2821 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2822 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2823 bad
= "#pragma omp sections";
2824 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2825 == BUILT_IN_GOMP_CANCEL
2826 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2828 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2830 ctx
->cancellable
= true;
2831 if (find_omp_clause (gimple_omp_sections_clauses
2834 warning_at (gimple_location (stmt
), 0,
2835 "%<#pragma omp cancel sections%> inside "
2836 "%<nowait%> sections construct");
2840 gcc_assert (ctx
->outer
2841 && gimple_code (ctx
->outer
->stmt
)
2842 == GIMPLE_OMP_SECTIONS
);
2843 ctx
->outer
->cancellable
= true;
2844 if (find_omp_clause (gimple_omp_sections_clauses
2847 warning_at (gimple_location (stmt
), 0,
2848 "%<#pragma omp cancel sections%> inside "
2849 "%<nowait%> sections construct");
2855 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2856 bad
= "#pragma omp task";
2858 ctx
->cancellable
= true;
2862 error_at (gimple_location (stmt
), "invalid arguments");
2867 error_at (gimple_location (stmt
),
2868 "%<%s %s%> construct not closely nested inside of %qs",
2869 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2870 == BUILT_IN_GOMP_CANCEL
2871 ? "#pragma omp cancel"
2872 : "#pragma omp cancellation point", kind
, bad
);
2877 case GIMPLE_OMP_SECTIONS
:
2878 case GIMPLE_OMP_SINGLE
:
2879 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2880 switch (gimple_code (ctx
->stmt
))
2882 case GIMPLE_OMP_FOR
:
2883 case GIMPLE_OMP_SECTIONS
:
2884 case GIMPLE_OMP_SINGLE
:
2885 case GIMPLE_OMP_ORDERED
:
2886 case GIMPLE_OMP_MASTER
:
2887 case GIMPLE_OMP_TASK
:
2888 case GIMPLE_OMP_CRITICAL
:
2889 if (is_gimple_call (stmt
))
2891 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2892 != BUILT_IN_GOMP_BARRIER
)
2894 error_at (gimple_location (stmt
),
2895 "barrier region may not be closely nested inside "
2896 "of work-sharing, critical, ordered, master or "
2897 "explicit task region");
2900 error_at (gimple_location (stmt
),
2901 "work-sharing region may not be closely nested inside "
2902 "of work-sharing, critical, ordered, master or explicit "
2905 case GIMPLE_OMP_PARALLEL
:
2911 case GIMPLE_OMP_MASTER
:
2912 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2913 switch (gimple_code (ctx
->stmt
))
2915 case GIMPLE_OMP_FOR
:
2916 case GIMPLE_OMP_SECTIONS
:
2917 case GIMPLE_OMP_SINGLE
:
2918 case GIMPLE_OMP_TASK
:
2919 error_at (gimple_location (stmt
),
2920 "master region may not be closely nested inside "
2921 "of work-sharing or explicit task region");
2923 case GIMPLE_OMP_PARALLEL
:
2929 case GIMPLE_OMP_ORDERED
:
2930 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2931 switch (gimple_code (ctx
->stmt
))
2933 case GIMPLE_OMP_CRITICAL
:
2934 case GIMPLE_OMP_TASK
:
2935 error_at (gimple_location (stmt
),
2936 "ordered region may not be closely nested inside "
2937 "of critical or explicit task region");
2939 case GIMPLE_OMP_FOR
:
2940 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2941 OMP_CLAUSE_ORDERED
) == NULL
)
2943 error_at (gimple_location (stmt
),
2944 "ordered region must be closely nested inside "
2945 "a loop region with an ordered clause");
2949 case GIMPLE_OMP_PARALLEL
:
2950 error_at (gimple_location (stmt
),
2951 "ordered region must be closely nested inside "
2952 "a loop region with an ordered clause");
2958 case GIMPLE_OMP_CRITICAL
:
2961 = gimple_omp_critical_name (as_a
<gomp_critical
*> (stmt
));
2962 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2963 if (gomp_critical
*other_crit
2964 = dyn_cast
<gomp_critical
*> (ctx
->stmt
))
2965 if (this_stmt_name
== gimple_omp_critical_name (other_crit
))
2967 error_at (gimple_location (stmt
),
2968 "critical region may not be nested inside a critical "
2969 "region with the same name");
2974 case GIMPLE_OMP_TEAMS
:
2976 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2977 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2979 error_at (gimple_location (stmt
),
2980 "teams construct not closely nested inside of target "
2985 case GIMPLE_OMP_TARGET
:
2986 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2988 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
)
2990 if (is_gimple_omp (stmt
)
2991 && is_gimple_omp_oacc (stmt
)
2992 && is_gimple_omp (ctx
->stmt
))
2994 error_at (gimple_location (stmt
),
2995 "OpenACC construct inside of non-OpenACC region");
3001 const char *stmt_name
, *ctx_stmt_name
;
3002 switch (gimple_omp_target_kind (stmt
))
3004 case GF_OMP_TARGET_KIND_REGION
: stmt_name
= "target"; break;
3005 case GF_OMP_TARGET_KIND_DATA
: stmt_name
= "target data"; break;
3006 case GF_OMP_TARGET_KIND_UPDATE
: stmt_name
= "target update"; break;
3007 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: stmt_name
= "parallel"; break;
3008 case GF_OMP_TARGET_KIND_OACC_KERNELS
: stmt_name
= "kernels"; break;
3009 case GF_OMP_TARGET_KIND_OACC_DATA
: stmt_name
= "data"; break;
3010 case GF_OMP_TARGET_KIND_OACC_UPDATE
: stmt_name
= "update"; break;
3011 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
: stmt_name
= "enter/exit data"; break;
3012 default: gcc_unreachable ();
3014 switch (gimple_omp_target_kind (ctx
->stmt
))
3016 case GF_OMP_TARGET_KIND_REGION
: ctx_stmt_name
= "target"; break;
3017 case GF_OMP_TARGET_KIND_DATA
: ctx_stmt_name
= "target data"; break;
3018 case GF_OMP_TARGET_KIND_OACC_PARALLEL
: ctx_stmt_name
= "parallel"; break;
3019 case GF_OMP_TARGET_KIND_OACC_KERNELS
: ctx_stmt_name
= "kernels"; break;
3020 case GF_OMP_TARGET_KIND_OACC_DATA
: ctx_stmt_name
= "data"; break;
3021 default: gcc_unreachable ();
3024 /* OpenACC/OpenMP mismatch? */
3025 if (is_gimple_omp_oacc (stmt
)
3026 != is_gimple_omp_oacc (ctx
->stmt
))
3028 error_at (gimple_location (stmt
),
3029 "%s %s construct inside of %s %s region",
3030 (is_gimple_omp_oacc (stmt
)
3031 ? "OpenACC" : "OpenMP"), stmt_name
,
3032 (is_gimple_omp_oacc (ctx
->stmt
)
3033 ? "OpenACC" : "OpenMP"), ctx_stmt_name
);
3036 if (is_gimple_omp_offloaded (ctx
->stmt
))
3038 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */
3039 if (is_gimple_omp_oacc (ctx
->stmt
))
3041 error_at (gimple_location (stmt
),
3042 "%s construct inside of %s region",
3043 stmt_name
, ctx_stmt_name
);
3048 gcc_checking_assert (!is_gimple_omp_oacc (stmt
));
3049 warning_at (gimple_location (stmt
), 0,
3050 "%s construct inside of %s region",
3051 stmt_name
, ctx_stmt_name
);
3063 /* Helper function scan_omp.
3065 Callback for walk_tree or operators in walk_gimple_stmt used to
3066 scan for OMP directives in TP. */
3069 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
3071 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
3072 omp_context
*ctx
= (omp_context
*) wi
->info
;
3075 switch (TREE_CODE (t
))
3082 *tp
= remap_decl (t
, &ctx
->cb
);
3086 if (ctx
&& TYPE_P (t
))
3087 *tp
= remap_type (t
, &ctx
->cb
);
3088 else if (!DECL_P (t
))
3093 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
3094 if (tem
!= TREE_TYPE (t
))
3096 if (TREE_CODE (t
) == INTEGER_CST
)
3097 *tp
= wide_int_to_tree (tem
, t
);
3099 TREE_TYPE (t
) = tem
;
3109 /* Return true if FNDECL is a setjmp or a longjmp. */
3112 setjmp_or_longjmp_p (const_tree fndecl
)
3114 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
3115 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
3116 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
3119 tree declname
= DECL_NAME (fndecl
);
3122 const char *name
= IDENTIFIER_POINTER (declname
);
3123 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
3127 /* Helper function for scan_omp.
3129 Callback for walk_gimple_stmt used to scan for OMP directives in
3130 the current statement in GSI. */
3133 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
3134 struct walk_stmt_info
*wi
)
3136 gimple stmt
= gsi_stmt (*gsi
);
3137 omp_context
*ctx
= (omp_context
*) wi
->info
;
3139 if (gimple_has_location (stmt
))
3140 input_location
= gimple_location (stmt
);
3142 /* Check the nesting restrictions. */
3143 bool remove
= false;
3144 if (is_gimple_omp (stmt
))
3145 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3146 else if (is_gimple_call (stmt
))
3148 tree fndecl
= gimple_call_fndecl (stmt
);
3151 if (setjmp_or_longjmp_p (fndecl
)
3153 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3154 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3157 error_at (gimple_location (stmt
),
3158 "setjmp/longjmp inside simd construct");
3160 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
3161 switch (DECL_FUNCTION_CODE (fndecl
))
3163 case BUILT_IN_GOMP_BARRIER
:
3164 case BUILT_IN_GOMP_CANCEL
:
3165 case BUILT_IN_GOMP_CANCELLATION_POINT
:
3166 case BUILT_IN_GOMP_TASKYIELD
:
3167 case BUILT_IN_GOMP_TASKWAIT
:
3168 case BUILT_IN_GOMP_TASKGROUP_START
:
3169 case BUILT_IN_GOMP_TASKGROUP_END
:
3170 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
3179 stmt
= gimple_build_nop ();
3180 gsi_replace (gsi
, stmt
, false);
3183 *handled_ops_p
= true;
3185 switch (gimple_code (stmt
))
3187 case GIMPLE_OMP_PARALLEL
:
3188 taskreg_nesting_level
++;
3189 scan_omp_parallel (gsi
, ctx
);
3190 taskreg_nesting_level
--;
3193 case GIMPLE_OMP_TASK
:
3194 taskreg_nesting_level
++;
3195 scan_omp_task (gsi
, ctx
);
3196 taskreg_nesting_level
--;
3199 case GIMPLE_OMP_FOR
:
3200 scan_omp_for (as_a
<gomp_for
*> (stmt
), ctx
);
3203 case GIMPLE_OMP_SECTIONS
:
3204 scan_omp_sections (as_a
<gomp_sections
*> (stmt
), ctx
);
3207 case GIMPLE_OMP_SINGLE
:
3208 scan_omp_single (as_a
<gomp_single
*> (stmt
), ctx
);
3211 case GIMPLE_OMP_SECTION
:
3212 case GIMPLE_OMP_MASTER
:
3213 case GIMPLE_OMP_TASKGROUP
:
3214 case GIMPLE_OMP_ORDERED
:
3215 case GIMPLE_OMP_CRITICAL
:
3216 ctx
= new_omp_context (stmt
, ctx
);
3217 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
3220 case GIMPLE_OMP_TARGET
:
3221 scan_omp_target (as_a
<gomp_target
*> (stmt
), ctx
);
3224 case GIMPLE_OMP_TEAMS
:
3225 scan_omp_teams (as_a
<gomp_teams
*> (stmt
), ctx
);
3232 *handled_ops_p
= false;
3234 for (var
= gimple_bind_vars (as_a
<gbind
*> (stmt
));
3236 var
= DECL_CHAIN (var
))
3237 insert_decl_map (&ctx
->cb
, var
, var
);
3241 *handled_ops_p
= false;
3249 /* Scan all the statements starting at the current statement. CTX
3250 contains context information about the OMP directives and
3251 clauses found during the scan. */
3254 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
3256 location_t saved_location
;
3257 struct walk_stmt_info wi
;
3259 memset (&wi
, 0, sizeof (wi
));
3261 wi
.want_locations
= true;
3263 saved_location
= input_location
;
3264 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
3265 input_location
= saved_location
;
3268 /* Re-gimplification and code generation routines. */
3270 /* Build a call to GOMP_barrier. */
3273 build_omp_barrier (tree lhs
)
3275 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
3276 : BUILT_IN_GOMP_BARRIER
);
3277 gcall
*g
= gimple_build_call (fndecl
, 0);
3279 gimple_call_set_lhs (g
, lhs
);
3283 /* If a context was created for STMT when it was scanned, return it. */
3285 static omp_context
*
3286 maybe_lookup_ctx (gimple stmt
)
3289 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
3290 return n
? (omp_context
*) n
->value
: NULL
;
3294 /* Find the mapping for DECL in CTX or the immediately enclosing
3295 context that has a mapping for DECL.
3297 If CTX is a nested parallel directive, we may have to use the decl
3298 mappings created in CTX's parent context. Suppose that we have the
3299 following parallel nesting (variable UIDs showed for clarity):
3302 #omp parallel shared(iD.1562) -> outer parallel
3303 iD.1562 = iD.1562 + 1;
3305 #omp parallel shared (iD.1562) -> inner parallel
3306 iD.1562 = iD.1562 - 1;
3308 Each parallel structure will create a distinct .omp_data_s structure
3309 for copying iD.1562 in/out of the directive:
3311 outer parallel .omp_data_s.1.i -> iD.1562
3312 inner parallel .omp_data_s.2.i -> iD.1562
3314 A shared variable mapping will produce a copy-out operation before
3315 the parallel directive and a copy-in operation after it. So, in
3316 this case we would have:
3319 .omp_data_o.1.i = iD.1562;
3320 #omp parallel shared(iD.1562) -> outer parallel
3321 .omp_data_i.1 = &.omp_data_o.1
3322 .omp_data_i.1->i = .omp_data_i.1->i + 1;
3324 .omp_data_o.2.i = iD.1562; -> **
3325 #omp parallel shared(iD.1562) -> inner parallel
3326 .omp_data_i.2 = &.omp_data_o.2
3327 .omp_data_i.2->i = .omp_data_i.2->i - 1;
3330 ** This is a problem. The symbol iD.1562 cannot be referenced
3331 inside the body of the outer parallel region. But since we are
3332 emitting this copy operation while expanding the inner parallel
3333 directive, we need to access the CTX structure of the outer
3334 parallel directive to get the correct mapping:
3336 .omp_data_o.2.i = .omp_data_i.1->i
3338 Since there may be other workshare or parallel directives enclosing
3339 the parallel directive, it may be necessary to walk up the context
3340 parent chain. This is not a problem in general because nested
3341 parallelism happens only rarely. */
3344 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3349 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3350 t
= maybe_lookup_decl (decl
, up
);
3352 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
3354 return t
? t
: decl
;
3358 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
3359 in outer contexts. */
3362 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
3367 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
3368 t
= maybe_lookup_decl (decl
, up
);
3370 return t
? t
: decl
;
3374 /* Construct the initialization value for reduction CLAUSE. */
3377 omp_reduction_init (tree clause
, tree type
)
3379 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
3380 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
3387 case TRUTH_ORIF_EXPR
:
3388 case TRUTH_XOR_EXPR
:
3390 return build_zero_cst (type
);
3393 case TRUTH_AND_EXPR
:
3394 case TRUTH_ANDIF_EXPR
:
3396 return fold_convert_loc (loc
, type
, integer_one_node
);
3399 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
3402 if (SCALAR_FLOAT_TYPE_P (type
))
3404 REAL_VALUE_TYPE max
, min
;
3405 if (HONOR_INFINITIES (type
))
3408 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
3411 real_maxval (&min
, 1, TYPE_MODE (type
));
3412 return build_real (type
, min
);
3416 gcc_assert (INTEGRAL_TYPE_P (type
));
3417 return TYPE_MIN_VALUE (type
);
3421 if (SCALAR_FLOAT_TYPE_P (type
))
3423 REAL_VALUE_TYPE max
;
3424 if (HONOR_INFINITIES (type
))
3427 real_maxval (&max
, 0, TYPE_MODE (type
));
3428 return build_real (type
, max
);
3432 gcc_assert (INTEGRAL_TYPE_P (type
));
3433 return TYPE_MAX_VALUE (type
);
3441 /* Return alignment to be assumed for var in CLAUSE, which should be
3442 OMP_CLAUSE_ALIGNED. */
3445 omp_clause_aligned_alignment (tree clause
)
3447 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
3448 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
3450 /* Otherwise return implementation defined alignment. */
3451 unsigned int al
= 1;
3452 machine_mode mode
, vmode
;
3453 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3455 vs
= 1 << floor_log2 (vs
);
3456 static enum mode_class classes
[]
3457 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
3458 for (int i
= 0; i
< 4; i
+= 2)
3459 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
3461 mode
= GET_MODE_WIDER_MODE (mode
))
3463 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
3464 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
3467 && GET_MODE_SIZE (vmode
) < vs
3468 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
3469 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
3471 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
3472 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
3474 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
3475 / GET_MODE_SIZE (mode
));
3476 if (TYPE_MODE (type
) != vmode
)
3478 if (TYPE_ALIGN_UNIT (type
) > al
)
3479 al
= TYPE_ALIGN_UNIT (type
);
3481 return build_int_cst (integer_type_node
, al
);
3484 /* Return maximum possible vectorization factor for the target. */
3491 || !flag_tree_loop_optimize
3492 || (!flag_tree_loop_vectorize
3493 && (global_options_set
.x_flag_tree_loop_vectorize
3494 || global_options_set
.x_flag_tree_vectorize
)))
3497 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3500 vs
= 1 << floor_log2 (vs
);
3503 machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
3504 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
3505 return GET_MODE_NUNITS (vqimode
);
3509 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3513 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
3514 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
3518 max_vf
= omp_max_vf ();
3521 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3522 OMP_CLAUSE_SAFELEN
);
3523 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3525 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3527 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3531 idx
= create_tmp_var (unsigned_type_node
);
3532 lane
= create_tmp_var (unsigned_type_node
);
3538 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3539 tree avar
= create_tmp_var_raw (atype
);
3540 if (TREE_ADDRESSABLE (new_var
))
3541 TREE_ADDRESSABLE (avar
) = 1;
3542 DECL_ATTRIBUTES (avar
)
3543 = tree_cons (get_identifier ("omp simd array"), NULL
,
3544 DECL_ATTRIBUTES (avar
));
3545 gimple_add_tmp_var (avar
);
3546 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3547 NULL_TREE
, NULL_TREE
);
3548 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3549 NULL_TREE
, NULL_TREE
);
3550 if (DECL_P (new_var
))
3552 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3553 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3558 /* Helper function of lower_rec_input_clauses. For a reference
3559 in simd reduction, add an underlying variable it will reference. */
3562 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3564 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3565 if (TREE_CONSTANT (z
))
3567 const char *name
= NULL
;
3568 if (DECL_NAME (new_vard
))
3569 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3571 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3572 gimple_add_tmp_var (z
);
3573 TREE_ADDRESSABLE (z
) = 1;
3574 z
= build_fold_addr_expr_loc (loc
, z
);
3575 gimplify_assign (new_vard
, z
, ilist
);
3579 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3580 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3581 private variables. Initialization statements go in ILIST, while calls
3582 to destructors go in DLIST. */
3585 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3586 omp_context
*ctx
, struct omp_for_data
*fd
)
3588 tree c
, dtor
, copyin_seq
, x
, ptr
;
3589 bool copyin_by_ref
= false;
3590 bool lastprivate_firstprivate
= false;
3591 bool reduction_omp_orig_ref
= false;
3593 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3594 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
3596 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3597 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3598 gimple_seq llist
[2] = { NULL
, NULL
};
3602 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3603 with data sharing clauses referencing variable sized vars. That
3604 is unnecessarily hard to support and very unlikely to result in
3605 vectorized code anyway. */
3607 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3608 switch (OMP_CLAUSE_CODE (c
))
3610 case OMP_CLAUSE_LINEAR
:
3611 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3614 case OMP_CLAUSE_REDUCTION
:
3615 case OMP_CLAUSE_PRIVATE
:
3616 case OMP_CLAUSE_FIRSTPRIVATE
:
3617 case OMP_CLAUSE_LASTPRIVATE
:
3618 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3625 /* Do all the fixed sized types in the first pass, and the variable sized
3626 types in the second pass. This makes sure that the scalar arguments to
3627 the variable sized types are processed before we use them in the
3628 variable sized operations. */
3629 for (pass
= 0; pass
< 2; ++pass
)
3631 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3633 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3636 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3640 case OMP_CLAUSE_PRIVATE
:
3641 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3644 case OMP_CLAUSE_SHARED
:
3645 /* Ignore shared directives in teams construct. */
3646 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3648 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3650 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3653 case OMP_CLAUSE_FIRSTPRIVATE
:
3654 case OMP_CLAUSE_COPYIN
:
3655 case OMP_CLAUSE_LINEAR
:
3657 case OMP_CLAUSE_REDUCTION
:
3658 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3659 reduction_omp_orig_ref
= true;
3661 case OMP_CLAUSE__LOOPTEMP_
:
3662 /* Handle _looptemp_ clauses only on parallel. */
3666 case OMP_CLAUSE_LASTPRIVATE
:
3667 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3669 lastprivate_firstprivate
= true;
3673 /* Even without corresponding firstprivate, if
3674 decl is Fortran allocatable, it needs outer var
3677 && lang_hooks
.decls
.omp_private_outer_ref
3678 (OMP_CLAUSE_DECL (c
)))
3679 lastprivate_firstprivate
= true;
3681 case OMP_CLAUSE_ALIGNED
:
3684 var
= OMP_CLAUSE_DECL (c
);
3685 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3686 && !is_global_var (var
))
3688 new_var
= maybe_lookup_decl (var
, ctx
);
3689 if (new_var
== NULL_TREE
)
3690 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3691 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3692 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3693 omp_clause_aligned_alignment (c
));
3694 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3695 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3696 gimplify_and_add (x
, ilist
);
3698 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3699 && is_global_var (var
))
3701 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3702 new_var
= lookup_decl (var
, ctx
);
3703 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3704 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3705 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3706 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3707 omp_clause_aligned_alignment (c
));
3708 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3709 x
= create_tmp_var (ptype
);
3710 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3711 gimplify_and_add (t
, ilist
);
3712 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3713 SET_DECL_VALUE_EXPR (new_var
, t
);
3714 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3721 new_var
= var
= OMP_CLAUSE_DECL (c
);
3722 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3723 new_var
= lookup_decl (var
, ctx
);
3725 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3730 else if (is_variable_sized (var
))
3732 /* For variable sized types, we need to allocate the
3733 actual storage here. Call alloca and store the
3734 result in the pointer decl that we created elsewhere. */
3738 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3743 ptr
= DECL_VALUE_EXPR (new_var
);
3744 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3745 ptr
= TREE_OPERAND (ptr
, 0);
3746 gcc_assert (DECL_P (ptr
));
3747 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3749 /* void *tmp = __builtin_alloca */
3750 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3751 stmt
= gimple_build_call (atmp
, 1, x
);
3752 tmp
= create_tmp_var_raw (ptr_type_node
);
3753 gimple_add_tmp_var (tmp
);
3754 gimple_call_set_lhs (stmt
, tmp
);
3756 gimple_seq_add_stmt (ilist
, stmt
);
3758 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3759 gimplify_assign (ptr
, x
, ilist
);
3762 else if (is_reference (var
))
3764 /* For references that are being privatized for Fortran,
3765 allocate new backing storage for the new pointer
3766 variable. This allows us to avoid changing all the
3767 code that expects a pointer to something that expects
3768 a direct variable. */
3772 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3773 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3775 x
= build_receiver_ref (var
, false, ctx
);
3776 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3778 else if (TREE_CONSTANT (x
))
3780 /* For reduction in SIMD loop, defer adding the
3781 initialization of the reference, because if we decide
3782 to use SIMD array for it, the initilization could cause
3784 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3788 const char *name
= NULL
;
3789 if (DECL_NAME (var
))
3790 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3792 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3794 gimple_add_tmp_var (x
);
3795 TREE_ADDRESSABLE (x
) = 1;
3796 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3801 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3802 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3807 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3808 gimplify_assign (new_var
, x
, ilist
);
3811 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3813 else if (c_kind
== OMP_CLAUSE_REDUCTION
3814 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3822 switch (OMP_CLAUSE_CODE (c
))
3824 case OMP_CLAUSE_SHARED
:
3825 /* Ignore shared directives in teams construct. */
3826 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3828 /* Shared global vars are just accessed directly. */
3829 if (is_global_var (new_var
))
3831 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3832 needs to be delayed until after fixup_child_record_type so
3833 that we get the correct type during the dereference. */
3834 by_ref
= use_pointer_for_field (var
, ctx
);
3835 x
= build_receiver_ref (var
, by_ref
, ctx
);
3836 SET_DECL_VALUE_EXPR (new_var
, x
);
3837 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3839 /* ??? If VAR is not passed by reference, and the variable
3840 hasn't been initialized yet, then we'll get a warning for
3841 the store into the omp_data_s structure. Ideally, we'd be
3842 able to notice this and not store anything at all, but
3843 we're generating code too early. Suppress the warning. */
3845 TREE_NO_WARNING (var
) = 1;
3848 case OMP_CLAUSE_LASTPRIVATE
:
3849 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3853 case OMP_CLAUSE_PRIVATE
:
3854 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3855 x
= build_outer_var_ref (var
, ctx
);
3856 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3858 if (is_task_ctx (ctx
))
3859 x
= build_receiver_ref (var
, false, ctx
);
3861 x
= build_outer_var_ref (var
, ctx
);
3867 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3870 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3871 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3872 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3873 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3874 idx
, lane
, ivar
, lvar
))
3877 x
= lang_hooks
.decls
.omp_clause_default_ctor
3878 (c
, unshare_expr (ivar
), x
);
3880 gimplify_and_add (x
, &llist
[0]);
3883 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3886 gimple_seq tseq
= NULL
;
3889 gimplify_stmt (&dtor
, &tseq
);
3890 gimple_seq_add_seq (&llist
[1], tseq
);
3897 gimplify_and_add (nx
, ilist
);
3901 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3904 gimple_seq tseq
= NULL
;
3907 gimplify_stmt (&dtor
, &tseq
);
3908 gimple_seq_add_seq (dlist
, tseq
);
3912 case OMP_CLAUSE_LINEAR
:
3913 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3914 goto do_firstprivate
;
3915 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3918 x
= build_outer_var_ref (var
, ctx
);
3921 case OMP_CLAUSE_FIRSTPRIVATE
:
3922 if (is_task_ctx (ctx
))
3924 if (is_reference (var
) || is_variable_sized (var
))
3926 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3928 || use_pointer_for_field (var
, NULL
))
3930 x
= build_receiver_ref (var
, false, ctx
);
3931 SET_DECL_VALUE_EXPR (new_var
, x
);
3932 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3937 x
= build_outer_var_ref (var
, ctx
);
3940 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3941 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3943 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3944 tree stept
= TREE_TYPE (t
);
3945 tree ct
= find_omp_clause (clauses
,
3946 OMP_CLAUSE__LOOPTEMP_
);
3948 tree l
= OMP_CLAUSE_DECL (ct
);
3949 tree n1
= fd
->loop
.n1
;
3950 tree step
= fd
->loop
.step
;
3951 tree itype
= TREE_TYPE (l
);
3952 if (POINTER_TYPE_P (itype
))
3953 itype
= signed_type_for (itype
);
3954 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3955 if (TYPE_UNSIGNED (itype
)
3956 && fd
->loop
.cond_code
== GT_EXPR
)
3957 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3958 fold_build1 (NEGATE_EXPR
, itype
, l
),
3959 fold_build1 (NEGATE_EXPR
,
3962 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3963 t
= fold_build2 (MULT_EXPR
, stept
,
3964 fold_convert (stept
, l
), t
);
3966 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3968 x
= lang_hooks
.decls
.omp_clause_linear_ctor
3970 gimplify_and_add (x
, ilist
);
3974 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3975 x
= fold_build2 (POINTER_PLUS_EXPR
,
3976 TREE_TYPE (x
), x
, t
);
3978 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3981 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3982 || TREE_ADDRESSABLE (new_var
))
3983 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3984 idx
, lane
, ivar
, lvar
))
3986 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3988 tree iv
= create_tmp_var (TREE_TYPE (new_var
));
3989 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3990 gimplify_and_add (x
, ilist
);
3991 gimple_stmt_iterator gsi
3992 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3994 = gimple_build_assign (unshare_expr (lvar
), iv
);
3995 gsi_insert_before_without_update (&gsi
, g
,
3997 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3998 enum tree_code code
= PLUS_EXPR
;
3999 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
4000 code
= POINTER_PLUS_EXPR
;
4001 g
= gimple_build_assign (iv
, code
, iv
, t
);
4002 gsi_insert_before_without_update (&gsi
, g
,
4006 x
= lang_hooks
.decls
.omp_clause_copy_ctor
4007 (c
, unshare_expr (ivar
), x
);
4008 gimplify_and_add (x
, &llist
[0]);
4009 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4012 gimple_seq tseq
= NULL
;
4015 gimplify_stmt (&dtor
, &tseq
);
4016 gimple_seq_add_seq (&llist
[1], tseq
);
4021 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
4022 gimplify_and_add (x
, ilist
);
4025 case OMP_CLAUSE__LOOPTEMP_
:
4026 gcc_assert (is_parallel_ctx (ctx
));
4027 x
= build_outer_var_ref (var
, ctx
);
4028 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
4029 gimplify_and_add (x
, ilist
);
4032 case OMP_CLAUSE_COPYIN
:
4033 by_ref
= use_pointer_for_field (var
, NULL
);
4034 x
= build_receiver_ref (var
, by_ref
, ctx
);
4035 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
4036 append_to_statement_list (x
, ©in_seq
);
4037 copyin_by_ref
|= by_ref
;
4040 case OMP_CLAUSE_REDUCTION
:
4041 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4043 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4045 x
= build_outer_var_ref (var
, ctx
);
4047 if (is_reference (var
)
4048 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4050 x
= build_fold_addr_expr_loc (clause_loc
, x
);
4051 SET_DECL_VALUE_EXPR (placeholder
, x
);
4052 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4053 tree new_vard
= new_var
;
4054 if (is_reference (var
))
4056 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4057 new_vard
= TREE_OPERAND (new_var
, 0);
4058 gcc_assert (DECL_P (new_vard
));
4061 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4062 idx
, lane
, ivar
, lvar
))
4064 if (new_vard
== new_var
)
4066 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
4067 SET_DECL_VALUE_EXPR (new_var
, ivar
);
4071 SET_DECL_VALUE_EXPR (new_vard
,
4072 build_fold_addr_expr (ivar
));
4073 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4075 x
= lang_hooks
.decls
.omp_clause_default_ctor
4076 (c
, unshare_expr (ivar
),
4077 build_outer_var_ref (var
, ctx
));
4079 gimplify_and_add (x
, &llist
[0]);
4080 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4082 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4083 lower_omp (&tseq
, ctx
);
4084 gimple_seq_add_seq (&llist
[0], tseq
);
4086 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4087 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4088 lower_omp (&tseq
, ctx
);
4089 gimple_seq_add_seq (&llist
[1], tseq
);
4090 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4091 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4092 if (new_vard
== new_var
)
4093 SET_DECL_VALUE_EXPR (new_var
, lvar
);
4095 SET_DECL_VALUE_EXPR (new_vard
,
4096 build_fold_addr_expr (lvar
));
4097 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
4102 gimplify_stmt (&dtor
, &tseq
);
4103 gimple_seq_add_seq (&llist
[1], tseq
);
4107 /* If this is a reference to constant size reduction var
4108 with placeholder, we haven't emitted the initializer
4109 for it because it is undesirable if SIMD arrays are used.
4110 But if they aren't used, we need to emit the deferred
4111 initialization now. */
4112 else if (is_reference (var
) && is_simd
)
4113 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4114 x
= lang_hooks
.decls
.omp_clause_default_ctor
4115 (c
, unshare_expr (new_var
),
4116 build_outer_var_ref (var
, ctx
));
4118 gimplify_and_add (x
, ilist
);
4119 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
4121 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
4122 lower_omp (&tseq
, ctx
);
4123 gimple_seq_add_seq (ilist
, tseq
);
4125 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
4128 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
4129 lower_omp (&tseq
, ctx
);
4130 gimple_seq_add_seq (dlist
, tseq
);
4131 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4133 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
4138 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
4139 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
4140 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4142 /* reduction(-:var) sums up the partial results, so it
4143 acts identically to reduction(+:var). */
4144 if (code
== MINUS_EXPR
)
4147 tree new_vard
= new_var
;
4148 if (is_simd
&& is_reference (var
))
4150 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
4151 new_vard
= TREE_OPERAND (new_var
, 0);
4152 gcc_assert (DECL_P (new_vard
));
4155 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
4156 idx
, lane
, ivar
, lvar
))
4158 tree ref
= build_outer_var_ref (var
, ctx
);
4160 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
4162 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
4163 ref
= build_outer_var_ref (var
, ctx
);
4164 gimplify_assign (ref
, x
, &llist
[1]);
4166 if (new_vard
!= new_var
)
4168 SET_DECL_VALUE_EXPR (new_vard
,
4169 build_fold_addr_expr (lvar
));
4170 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
4175 if (is_reference (var
) && is_simd
)
4176 handle_simd_reference (clause_loc
, new_vard
, ilist
);
4177 gimplify_assign (new_var
, x
, ilist
);
4180 tree ref
= build_outer_var_ref (var
, ctx
);
4182 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4183 ref
= build_outer_var_ref (var
, ctx
);
4184 gimplify_assign (ref
, x
, dlist
);
4198 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
4199 /* Don't want uninit warnings on simduid, it is always uninitialized,
4200 but we use it not for the value, but for the DECL_UID only. */
4201 TREE_NO_WARNING (uid
) = 1;
4203 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
4204 gimple_call_set_lhs (g
, lane
);
4205 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
4206 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
4207 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
4208 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
4209 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4210 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4211 g
= gimple_build_assign (lane
, INTEGER_CST
,
4212 build_int_cst (unsigned_type_node
, 0));
4213 gimple_seq_add_stmt (ilist
, g
);
4214 for (int i
= 0; i
< 2; i
++)
4217 tree vf
= create_tmp_var (unsigned_type_node
);
4218 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
4219 gimple_call_set_lhs (g
, vf
);
4220 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
4221 gimple_seq_add_stmt (seq
, g
);
4222 tree t
= build_int_cst (unsigned_type_node
, 0);
4223 g
= gimple_build_assign (idx
, INTEGER_CST
, t
);
4224 gimple_seq_add_stmt (seq
, g
);
4225 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
4226 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
4227 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
4228 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
4229 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
4230 gimple_seq_add_seq (seq
, llist
[i
]);
4231 t
= build_int_cst (unsigned_type_node
, 1);
4232 g
= gimple_build_assign (idx
, PLUS_EXPR
, idx
, t
);
4233 gimple_seq_add_stmt (seq
, g
);
4234 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
4235 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
4236 gimple_seq_add_stmt (seq
, g
);
4237 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
4241 /* The copyin sequence is not to be executed by the main thread, since
4242 that would result in self-copies. Perhaps not visible to scalars,
4243 but it certainly is to C++ operator=. */
4246 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
4248 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
4249 build_int_cst (TREE_TYPE (x
), 0));
4250 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
4251 gimplify_and_add (x
, ilist
);
4254 /* If any copyin variable is passed by reference, we must ensure the
4255 master thread doesn't modify it before it is copied over in all
4256 threads. Similarly for variables in both firstprivate and
4257 lastprivate clauses we need to ensure the lastprivate copying
4258 happens after firstprivate copying in all threads. And similarly
4259 for UDRs if initializer expression refers to omp_orig. */
4260 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
4262 /* Don't add any barrier for #pragma omp simd or
4263 #pragma omp distribute. */
4264 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
4265 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
4266 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
4269 /* If max_vf is non-zero, then we can use only a vectorization factor
4270 up to the max_vf we chose. So stick it into the safelen clause. */
4273 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
4274 OMP_CLAUSE_SAFELEN
);
4276 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
4277 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
4280 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
4281 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
4283 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
4284 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
4290 /* Generate code to implement the LASTPRIVATE clauses. This is used for
4291 both parallel and workshare constructs. PREDICATE may be NULL if it's
4295 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
4298 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
4299 bool par_clauses
= false;
4300 tree simduid
= NULL
, lastlane
= NULL
;
4302 /* Early exit if there are no lastprivate or linear clauses. */
4303 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
4304 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
4305 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
4306 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
4308 if (clauses
== NULL
)
4310 /* If this was a workshare clause, see if it had been combined
4311 with its parallel. In that case, look for the clauses on the
4312 parallel statement itself. */
4313 if (is_parallel_ctx (ctx
))
4317 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4320 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4321 OMP_CLAUSE_LASTPRIVATE
);
4322 if (clauses
== NULL
)
4330 tree label_true
, arm1
, arm2
;
4332 label
= create_artificial_label (UNKNOWN_LOCATION
);
4333 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
4334 arm1
= TREE_OPERAND (predicate
, 0);
4335 arm2
= TREE_OPERAND (predicate
, 1);
4336 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4337 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
4338 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
4340 gimple_seq_add_stmt (stmt_list
, stmt
);
4341 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
4344 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4345 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4347 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
4349 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
4352 for (c
= clauses
; c
;)
4355 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4357 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4358 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4359 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
4361 var
= OMP_CLAUSE_DECL (c
);
4362 new_var
= lookup_decl (var
, ctx
);
4364 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
4366 tree val
= DECL_VALUE_EXPR (new_var
);
4367 if (TREE_CODE (val
) == ARRAY_REF
4368 && VAR_P (TREE_OPERAND (val
, 0))
4369 && lookup_attribute ("omp simd array",
4370 DECL_ATTRIBUTES (TREE_OPERAND (val
,
4373 if (lastlane
== NULL
)
4375 lastlane
= create_tmp_var (unsigned_type_node
);
4377 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
4379 TREE_OPERAND (val
, 1));
4380 gimple_call_set_lhs (g
, lastlane
);
4381 gimple_seq_add_stmt (stmt_list
, g
);
4383 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
4384 TREE_OPERAND (val
, 0), lastlane
,
4385 NULL_TREE
, NULL_TREE
);
4389 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4390 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
4392 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
4393 gimple_seq_add_seq (stmt_list
,
4394 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
4395 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
4397 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4398 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
4400 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
4401 gimple_seq_add_seq (stmt_list
,
4402 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
4403 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
4406 x
= build_outer_var_ref (var
, ctx
);
4407 if (is_reference (var
))
4408 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4409 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
4410 gimplify_and_add (x
, stmt_list
);
4412 c
= OMP_CLAUSE_CHAIN (c
);
4413 if (c
== NULL
&& !par_clauses
)
4415 /* If this was a workshare clause, see if it had been combined
4416 with its parallel. In that case, continue looking for the
4417 clauses also on the parallel statement itself. */
4418 if (is_parallel_ctx (ctx
))
4422 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4425 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4426 OMP_CLAUSE_LASTPRIVATE
);
4432 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
4436 oacc_lower_reduction_var_helper (gimple_seq
*stmt_seqp
, omp_context
*ctx
,
4437 tree tid
, tree var
, tree new_var
)
4439 /* The atomic add at the end of the sum creates unnecessary
4440 write contention on accelerators. To work around this,
4441 create an array to store the partial reductions. Later, in
4442 lower_omp_for (for openacc), the values of array will be
4445 tree t
= NULL_TREE
, array
, x
;
4446 tree type
= get_base_type (var
);
4449 /* Now insert the partial reductions into the array. */
4451 /* Find the reduction array. */
4453 tree ptype
= build_pointer_type (type
);
4455 t
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
), ctx
);
4456 t
= build_receiver_ref (t
, false, ctx
->outer
);
4458 array
= create_tmp_var (ptype
);
4459 gimplify_assign (array
, t
, stmt_seqp
);
4461 tree ptr
= create_tmp_var (TREE_TYPE (array
));
4463 /* Find the reduction array. */
4465 /* testing a unary conversion. */
4466 tree offset
= create_tmp_var (sizetype
);
4467 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
),
4469 t
= create_tmp_var (sizetype
);
4470 gimplify_assign (t
, unshare_expr (fold_build1 (NOP_EXPR
, sizetype
, tid
)),
4472 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, t
);
4473 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4475 /* Offset expression. Does the POINTER_PLUS_EXPR take care
4476 of adding sizeof(var) to the array? */
4477 ptr
= create_tmp_var (ptype
);
4478 stmt
= gimple_build_assign (unshare_expr (ptr
), POINTER_PLUS_EXPR
, array
,
4480 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4482 /* Move the local sum to gfc$sum[i]. */
4483 x
= unshare_expr (build_simple_mem_ref (ptr
));
4484 stmt
= gimplify_assign (x
, new_var
, stmt_seqp
);
4487 /* Generate code to implement the REDUCTION clauses. */
4490 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
4492 gimple_seq sub_seq
= NULL
;
4494 tree x
, c
, tid
= NULL_TREE
;
4497 /* SIMD reductions are handled in lower_rec_input_clauses. */
4498 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4499 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4502 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
4503 update in that case, otherwise use a lock. */
4504 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
4505 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
4507 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4509 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4519 /* Initialize thread info for OpenACC. */
4520 if (is_gimple_omp_oacc (ctx
->stmt
))
4522 /* Get the current thread id. */
4523 tree call
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
4524 tid
= create_tmp_var (TREE_TYPE (TREE_TYPE (call
)));
4525 gimple stmt
= gimple_build_call (call
, 0);
4526 gimple_call_set_lhs (stmt
, tid
);
4527 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4530 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4532 tree var
, ref
, new_var
;
4533 enum tree_code code
;
4534 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4536 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4539 var
= OMP_CLAUSE_DECL (c
);
4540 new_var
= lookup_decl (var
, ctx
);
4541 if (is_reference (var
))
4542 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4543 ref
= build_outer_var_ref (var
, ctx
);
4544 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4546 /* reduction(-:var) sums up the partial results, so it acts
4547 identically to reduction(+:var). */
4548 if (code
== MINUS_EXPR
)
4551 if (is_gimple_omp_oacc (ctx
->stmt
))
4553 gcc_checking_assert (!OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
));
4555 oacc_lower_reduction_var_helper (stmt_seqp
, ctx
, tid
, var
, new_var
);
4557 else if (count
== 1)
4559 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
4561 addr
= save_expr (addr
);
4562 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
4563 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
4564 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
4565 gimplify_and_add (x
, stmt_seqp
);
4568 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4570 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4572 if (is_reference (var
)
4573 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4575 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
4576 SET_DECL_VALUE_EXPR (placeholder
, ref
);
4577 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4578 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
4579 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
4580 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4581 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
4585 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4586 ref
= build_outer_var_ref (var
, ctx
);
4587 gimplify_assign (ref
, x
, &sub_seq
);
4591 if (is_gimple_omp_oacc (ctx
->stmt
))
4594 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4596 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4598 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4600 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4602 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4606 /* Generate code to implement the COPYPRIVATE clauses. */
4609 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4614 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4616 tree var
, new_var
, ref
, x
;
4618 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4620 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4623 var
= OMP_CLAUSE_DECL (c
);
4624 by_ref
= use_pointer_for_field (var
, NULL
);
4626 ref
= build_sender_ref (var
, ctx
);
4627 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4630 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4631 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4633 gimplify_assign (ref
, x
, slist
);
4635 ref
= build_receiver_ref (var
, false, ctx
);
4638 ref
= fold_convert_loc (clause_loc
,
4639 build_pointer_type (TREE_TYPE (new_var
)),
4641 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4643 if (is_reference (var
))
4645 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4646 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4647 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4649 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4650 gimplify_and_add (x
, rlist
);
4655 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4656 and REDUCTION from the sender (aka parent) side. */
4659 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4664 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4666 tree val
, ref
, x
, var
;
4667 bool by_ref
, do_in
= false, do_out
= false;
4668 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4670 switch (OMP_CLAUSE_CODE (c
))
4672 case OMP_CLAUSE_PRIVATE
:
4673 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4676 case OMP_CLAUSE_FIRSTPRIVATE
:
4677 case OMP_CLAUSE_COPYIN
:
4678 case OMP_CLAUSE_LASTPRIVATE
:
4679 case OMP_CLAUSE_REDUCTION
:
4680 case OMP_CLAUSE__LOOPTEMP_
:
4686 val
= OMP_CLAUSE_DECL (c
);
4687 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4689 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4690 && is_global_var (var
))
4692 if (is_variable_sized (val
))
4694 by_ref
= use_pointer_for_field (val
, NULL
);
4696 switch (OMP_CLAUSE_CODE (c
))
4698 case OMP_CLAUSE_PRIVATE
:
4699 case OMP_CLAUSE_FIRSTPRIVATE
:
4700 case OMP_CLAUSE_COPYIN
:
4701 case OMP_CLAUSE__LOOPTEMP_
:
4705 case OMP_CLAUSE_LASTPRIVATE
:
4706 if (by_ref
|| is_reference (val
))
4708 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4715 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4720 case OMP_CLAUSE_REDUCTION
:
4722 do_out
= !(by_ref
|| is_reference (val
));
4731 ref
= build_sender_ref (val
, ctx
);
4732 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4733 gimplify_assign (ref
, x
, ilist
);
4734 if (is_task_ctx (ctx
))
4735 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4740 ref
= build_sender_ref (val
, ctx
);
4741 gimplify_assign (var
, ref
, olist
);
4746 /* Generate code to implement SHARED from the sender (aka parent)
4747 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4748 list things that got automatically shared. */
4751 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4753 tree var
, ovar
, nvar
, f
, x
, record_type
;
4755 if (ctx
->record_type
== NULL
)
4758 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4759 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4761 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4762 nvar
= maybe_lookup_decl (ovar
, ctx
);
4763 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4766 /* If CTX is a nested parallel directive. Find the immediately
4767 enclosing parallel or workshare construct that contains a
4768 mapping for OVAR. */
4769 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4771 if (use_pointer_for_field (ovar
, ctx
))
4773 x
= build_sender_ref (ovar
, ctx
);
4774 var
= build_fold_addr_expr (var
);
4775 gimplify_assign (x
, var
, ilist
);
4779 x
= build_sender_ref (ovar
, ctx
);
4780 gimplify_assign (x
, var
, ilist
);
4782 if (!TREE_READONLY (var
)
4783 /* We don't need to receive a new reference to a result
4784 or parm decl. In fact we may not store to it as we will
4785 invalidate any pending RSO and generate wrong gimple
4787 && !((TREE_CODE (var
) == RESULT_DECL
4788 || TREE_CODE (var
) == PARM_DECL
)
4789 && DECL_BY_REFERENCE (var
)))
4791 x
= build_sender_ref (ovar
, ctx
);
4792 gimplify_assign (var
, x
, olist
);
4799 /* A convenience function to build an empty GIMPLE_COND with just the
4803 gimple_build_cond_empty (tree cond
)
4805 enum tree_code pred_code
;
4808 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4809 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4813 /* Build the function calls to GOMP_parallel_start etc to actually
4814 generate the parallel operation. REGION is the parallel region
4815 being expanded. BB is the block where to insert the code. WS_ARGS
4816 will be set if this is a call to a combined parallel+workshare
4817 construct, it contains the list of additional arguments needed by
4818 the workshare construct. */
4821 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4822 gomp_parallel
*entry_stmt
,
4823 vec
<tree
, va_gc
> *ws_args
)
4825 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4826 gimple_stmt_iterator gsi
;
4828 enum built_in_function start_ix
;
4830 location_t clause_loc
;
4831 vec
<tree
, va_gc
> *args
;
4833 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4835 /* Determine what flavor of GOMP_parallel we will be
4837 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4838 if (is_combined_parallel (region
))
4840 switch (region
->inner
->type
)
4842 case GIMPLE_OMP_FOR
:
4843 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4844 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4845 + (region
->inner
->sched_kind
4846 == OMP_CLAUSE_SCHEDULE_RUNTIME
4847 ? 3 : region
->inner
->sched_kind
));
4848 start_ix
= (enum built_in_function
)start_ix2
;
4850 case GIMPLE_OMP_SECTIONS
:
4851 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4858 /* By default, the value of NUM_THREADS is zero (selected at run time)
4859 and there is no conditional. */
4861 val
= build_int_cst (unsigned_type_node
, 0);
4862 flags
= build_int_cst (unsigned_type_node
, 0);
4864 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4866 cond
= OMP_CLAUSE_IF_EXPR (c
);
4868 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4871 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4872 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4875 clause_loc
= gimple_location (entry_stmt
);
4877 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4879 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4881 /* Ensure 'val' is of the correct type. */
4882 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4884 /* If we found the clause 'if (cond)', build either
4885 (cond != 0) or (cond ? val : 1u). */
4888 cond
= gimple_boolify (cond
);
4890 if (integer_zerop (val
))
4891 val
= fold_build2_loc (clause_loc
,
4892 EQ_EXPR
, unsigned_type_node
, cond
,
4893 build_int_cst (TREE_TYPE (cond
), 0));
4896 basic_block cond_bb
, then_bb
, else_bb
;
4897 edge e
, e_then
, e_else
;
4898 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4900 tmp_var
= create_tmp_var (TREE_TYPE (val
));
4901 if (gimple_in_ssa_p (cfun
))
4903 tmp_then
= make_ssa_name (tmp_var
);
4904 tmp_else
= make_ssa_name (tmp_var
);
4905 tmp_join
= make_ssa_name (tmp_var
);
4914 e
= split_block_after_labels (bb
);
4919 then_bb
= create_empty_bb (cond_bb
);
4920 else_bb
= create_empty_bb (then_bb
);
4921 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4922 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4924 stmt
= gimple_build_cond_empty (cond
);
4925 gsi
= gsi_start_bb (cond_bb
);
4926 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4928 gsi
= gsi_start_bb (then_bb
);
4929 stmt
= gimple_build_assign (tmp_then
, val
);
4930 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4932 gsi
= gsi_start_bb (else_bb
);
4933 stmt
= gimple_build_assign
4934 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4935 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4937 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4938 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4939 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4940 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4941 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4942 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4944 if (gimple_in_ssa_p (cfun
))
4946 gphi
*phi
= create_phi_node (tmp_join
, bb
);
4947 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4948 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4954 gsi
= gsi_start_bb (bb
);
4955 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4956 false, GSI_CONTINUE_LINKING
);
4959 gsi
= gsi_last_bb (bb
);
4960 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4962 t1
= null_pointer_node
;
4964 t1
= build_fold_addr_expr (t
);
4965 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4967 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4968 args
->quick_push (t2
);
4969 args
->quick_push (t1
);
4970 args
->quick_push (val
);
4972 args
->splice (*ws_args
);
4973 args
->quick_push (flags
);
4975 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4976 builtin_decl_explicit (start_ix
), args
);
4978 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4979 false, GSI_CONTINUE_LINKING
);
4982 /* Insert a function call whose name is FUNC_NAME with the information from
4983 ENTRY_STMT into the basic_block BB. */
4986 expand_cilk_for_call (basic_block bb
, gomp_parallel
*entry_stmt
,
4987 vec
<tree
, va_gc
> *ws_args
)
4990 gimple_stmt_iterator gsi
;
4991 vec
<tree
, va_gc
> *args
;
4993 gcc_assert (vec_safe_length (ws_args
) == 2);
4994 tree func_name
= (*ws_args
)[0];
4995 tree grain
= (*ws_args
)[1];
4997 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4998 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
4999 gcc_assert (count
!= NULL_TREE
);
5000 count
= OMP_CLAUSE_OPERAND (count
, 0);
5002 gsi
= gsi_last_bb (bb
);
5003 t
= gimple_omp_parallel_data_arg (entry_stmt
);
5005 t1
= null_pointer_node
;
5007 t1
= build_fold_addr_expr (t
);
5008 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
5010 vec_alloc (args
, 4);
5011 args
->quick_push (t2
);
5012 args
->quick_push (t1
);
5013 args
->quick_push (count
);
5014 args
->quick_push (grain
);
5015 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
5017 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
5018 GSI_CONTINUE_LINKING
);
5021 /* Build the function call to GOMP_task to actually
5022 generate the task operation. BB is the block where to insert the code. */
5025 expand_task_call (basic_block bb
, gomp_task
*entry_stmt
)
5027 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
5028 gimple_stmt_iterator gsi
;
5029 location_t loc
= gimple_location (entry_stmt
);
5031 clauses
= gimple_omp_task_clauses (entry_stmt
);
5033 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
5035 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
5037 cond
= boolean_true_node
;
5039 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
5040 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
5041 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
5042 flags
= build_int_cst (unsigned_type_node
,
5043 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
5045 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
5048 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
5049 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
5050 build_int_cst (unsigned_type_node
, 2),
5051 build_int_cst (unsigned_type_node
, 0));
5052 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
5055 depend
= OMP_CLAUSE_DECL (depend
);
5057 depend
= build_int_cst (ptr_type_node
, 0);
5059 gsi
= gsi_last_bb (bb
);
5060 t
= gimple_omp_task_data_arg (entry_stmt
);
5062 t2
= null_pointer_node
;
5064 t2
= build_fold_addr_expr_loc (loc
, t
);
5065 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
5066 t
= gimple_omp_task_copy_fn (entry_stmt
);
5068 t3
= null_pointer_node
;
5070 t3
= build_fold_addr_expr_loc (loc
, t
);
5072 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
5074 gimple_omp_task_arg_size (entry_stmt
),
5075 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
5078 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5079 false, GSI_CONTINUE_LINKING
);
5083 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
5084 catch handler and return it. This prevents programs from violating the
5085 structured block semantics with throws. */
5088 maybe_catch_exception (gimple_seq body
)
5093 if (!flag_exceptions
)
5096 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
5097 decl
= lang_hooks
.eh_protect_cleanup_actions ();
5099 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
5101 g
= gimple_build_eh_must_not_throw (decl
);
5102 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
5105 return gimple_seq_alloc_with_stmt (g
);
5108 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
5111 vec2chain (vec
<tree
, va_gc
> *v
)
5113 tree chain
= NULL_TREE
, t
;
5116 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
5118 DECL_CHAIN (t
) = chain
;
5126 /* Remove barriers in REGION->EXIT's block. Note that this is only
5127 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
5128 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
5129 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
5133 remove_exit_barrier (struct omp_region
*region
)
5135 gimple_stmt_iterator gsi
;
5136 basic_block exit_bb
;
5140 int any_addressable_vars
= -1;
5142 exit_bb
= region
->exit
;
5144 /* If the parallel region doesn't return, we don't have REGION->EXIT
5149 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
5150 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
5151 statements that can appear in between are extremely limited -- no
5152 memory operations at all. Here, we allow nothing at all, so the
5153 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
5154 gsi
= gsi_last_bb (exit_bb
);
5155 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5157 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
5160 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
5162 gsi
= gsi_last_bb (e
->src
);
5163 if (gsi_end_p (gsi
))
5165 stmt
= gsi_stmt (gsi
);
5166 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
5167 && !gimple_omp_return_nowait_p (stmt
))
5169 /* OpenMP 3.0 tasks unfortunately prevent this optimization
5170 in many cases. If there could be tasks queued, the barrier
5171 might be needed to let the tasks run before some local
5172 variable of the parallel that the task uses as shared
5173 runs out of scope. The task can be spawned either
5174 from within current function (this would be easy to check)
5175 or from some function it calls and gets passed an address
5176 of such a variable. */
5177 if (any_addressable_vars
< 0)
5179 gomp_parallel
*parallel_stmt
5180 = as_a
<gomp_parallel
*> (last_stmt (region
->entry
));
5181 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
5182 tree local_decls
, block
, decl
;
5185 any_addressable_vars
= 0;
5186 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
5187 if (TREE_ADDRESSABLE (decl
))
5189 any_addressable_vars
= 1;
5192 for (block
= gimple_block (stmt
);
5193 !any_addressable_vars
5195 && TREE_CODE (block
) == BLOCK
;
5196 block
= BLOCK_SUPERCONTEXT (block
))
5198 for (local_decls
= BLOCK_VARS (block
);
5200 local_decls
= DECL_CHAIN (local_decls
))
5201 if (TREE_ADDRESSABLE (local_decls
))
5203 any_addressable_vars
= 1;
5206 if (block
== gimple_block (parallel_stmt
))
5210 if (!any_addressable_vars
)
5211 gimple_omp_return_set_nowait (stmt
);
5217 remove_exit_barriers (struct omp_region
*region
)
5219 if (region
->type
== GIMPLE_OMP_PARALLEL
)
5220 remove_exit_barrier (region
);
5224 region
= region
->inner
;
5225 remove_exit_barriers (region
);
5226 while (region
->next
)
5228 region
= region
->next
;
5229 remove_exit_barriers (region
);
5234 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
5235 calls. These can't be declared as const functions, but
5236 within one parallel body they are constant, so they can be
5237 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
5238 which are declared const. Similarly for task body, except
5239 that in untied task omp_get_thread_num () can change at any task
5240 scheduling point. */
5243 optimize_omp_library_calls (gimple entry_stmt
)
5246 gimple_stmt_iterator gsi
;
5247 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5248 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
5249 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5250 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
5251 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
5252 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
5253 OMP_CLAUSE_UNTIED
) != NULL
);
5255 FOR_EACH_BB_FN (bb
, cfun
)
5256 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5258 gimple call
= gsi_stmt (gsi
);
5261 if (is_gimple_call (call
)
5262 && (decl
= gimple_call_fndecl (call
))
5263 && DECL_EXTERNAL (decl
)
5264 && TREE_PUBLIC (decl
)
5265 && DECL_INITIAL (decl
) == NULL
)
5269 if (DECL_NAME (decl
) == thr_num_id
)
5271 /* In #pragma omp task untied omp_get_thread_num () can change
5272 during the execution of the task region. */
5275 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
5277 else if (DECL_NAME (decl
) == num_thr_id
)
5278 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
5282 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
5283 || gimple_call_num_args (call
) != 0)
5286 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
5289 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
5290 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
5291 TREE_TYPE (TREE_TYPE (built_in
))))
5294 gimple_call_set_fndecl (call
, built_in
);
5299 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
5303 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
5307 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
5308 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
5311 if (TREE_CODE (t
) == ADDR_EXPR
)
5312 recompute_tree_invariant_for_addr_expr (t
);
5314 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
5318 /* Prepend TO = FROM assignment before *GSI_P. */
5321 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
5323 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
5324 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
5325 true, GSI_SAME_STMT
);
5326 gimple stmt
= gimple_build_assign (to
, from
);
5327 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
5328 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
5329 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
5331 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
5332 gimple_regimplify_operands (stmt
, &gsi
);
5336 /* Expand the OpenMP parallel or task directive starting at REGION. */
5339 expand_omp_taskreg (struct omp_region
*region
)
5341 basic_block entry_bb
, exit_bb
, new_bb
;
5342 struct function
*child_cfun
;
5343 tree child_fn
, block
, t
;
5344 gimple_stmt_iterator gsi
;
5345 gimple entry_stmt
, stmt
;
5347 vec
<tree
, va_gc
> *ws_args
;
5349 entry_stmt
= last_stmt (region
->entry
);
5350 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
5351 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
5353 entry_bb
= region
->entry
;
5354 if (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
)
5355 exit_bb
= region
->cont
;
5357 exit_bb
= region
->exit
;
5361 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
5362 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
5363 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
5366 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
5367 and the inner statement contains the name of the built-in function
5369 ws_args
= region
->inner
->ws_args
;
5370 else if (is_combined_parallel (region
))
5371 ws_args
= region
->ws_args
;
5375 if (child_cfun
->cfg
)
5377 /* Due to inlining, it may happen that we have already outlined
5378 the region, in which case all we need to do is make the
5379 sub-graph unreachable and emit the parallel call. */
5380 edge entry_succ_e
, exit_succ_e
;
5382 entry_succ_e
= single_succ_edge (entry_bb
);
5384 gsi
= gsi_last_bb (entry_bb
);
5385 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
5386 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
5387 gsi_remove (&gsi
, true);
5392 exit_succ_e
= single_succ_edge (exit_bb
);
5393 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
5395 remove_edge_and_dominated_blocks (entry_succ_e
);
5399 unsigned srcidx
, dstidx
, num
;
5401 /* If the parallel region needs data sent from the parent
5402 function, then the very first statement (except possible
5403 tree profile counter updates) of the parallel body
5404 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
5405 &.OMP_DATA_O is passed as an argument to the child function,
5406 we need to replace it with the argument as seen by the child
5409 In most cases, this will end up being the identity assignment
5410 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
5411 a function call that has been inlined, the original PARM_DECL
5412 .OMP_DATA_I may have been converted into a different local
5413 variable. In which case, we need to keep the assignment. */
5414 if (gimple_omp_taskreg_data_arg (entry_stmt
))
5416 basic_block entry_succ_bb
5417 = single_succ_p (entry_bb
) ? single_succ (entry_bb
)
5418 : FALLTHRU_EDGE (entry_bb
)->dest
;
5420 gimple parcopy_stmt
= NULL
;
5422 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
5426 gcc_assert (!gsi_end_p (gsi
));
5427 stmt
= gsi_stmt (gsi
);
5428 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
5431 if (gimple_num_ops (stmt
) == 2)
5433 tree arg
= gimple_assign_rhs1 (stmt
);
5435 /* We're ignore the subcode because we're
5436 effectively doing a STRIP_NOPS. */
5438 if (TREE_CODE (arg
) == ADDR_EXPR
5439 && TREE_OPERAND (arg
, 0)
5440 == gimple_omp_taskreg_data_arg (entry_stmt
))
5442 parcopy_stmt
= stmt
;
5448 gcc_assert (parcopy_stmt
!= NULL
);
5449 arg
= DECL_ARGUMENTS (child_fn
);
5451 if (!gimple_in_ssa_p (cfun
))
5453 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
5454 gsi_remove (&gsi
, true);
5457 /* ?? Is setting the subcode really necessary ?? */
5458 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
5459 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
5464 /* If we are in ssa form, we must load the value from the default
5465 definition of the argument. That should not be defined now,
5466 since the argument is not used uninitialized. */
5467 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
5468 narg
= make_ssa_name (arg
, gimple_build_nop ());
5469 set_ssa_default_def (cfun
, arg
, narg
);
5470 /* ?? Is setting the subcode really necessary ?? */
5471 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
5472 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
5473 update_stmt (parcopy_stmt
);
5477 /* Declare local variables needed in CHILD_CFUN. */
5478 block
= DECL_INITIAL (child_fn
);
5479 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
5480 /* The gimplifier could record temporaries in parallel/task block
5481 rather than in containing function's local_decls chain,
5482 which would mean cgraph missed finalizing them. Do it now. */
5483 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
5484 if (TREE_CODE (t
) == VAR_DECL
5486 && !DECL_EXTERNAL (t
))
5487 varpool_node::finalize_decl (t
);
5488 DECL_SAVED_TREE (child_fn
) = NULL
;
5489 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5490 gimple_set_body (child_fn
, NULL
);
5491 TREE_USED (block
) = 1;
5493 /* Reset DECL_CONTEXT on function arguments. */
5494 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
5495 DECL_CONTEXT (t
) = child_fn
;
5497 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
5498 so that it can be moved to the child function. */
5499 gsi
= gsi_last_bb (entry_bb
);
5500 stmt
= gsi_stmt (gsi
);
5501 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
5502 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
5503 e
= split_block (entry_bb
, stmt
);
5504 gsi_remove (&gsi
, true);
5507 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5508 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
5511 e2
= make_edge (e
->src
, BRANCH_EDGE (entry_bb
)->dest
, EDGE_ABNORMAL
);
5512 gcc_assert (e2
->dest
== region
->exit
);
5513 remove_edge (BRANCH_EDGE (entry_bb
));
5514 set_immediate_dominator (CDI_DOMINATORS
, e2
->dest
, e
->src
);
5515 gsi
= gsi_last_bb (region
->exit
);
5516 gcc_assert (!gsi_end_p (gsi
)
5517 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5518 gsi_remove (&gsi
, true);
5521 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */
5524 gsi
= gsi_last_bb (exit_bb
);
5525 gcc_assert (!gsi_end_p (gsi
)
5526 && (gimple_code (gsi_stmt (gsi
))
5527 == (e2
? GIMPLE_OMP_CONTINUE
: GIMPLE_OMP_RETURN
)));
5528 stmt
= gimple_build_return (NULL
);
5529 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5530 gsi_remove (&gsi
, true);
5533 /* Move the parallel region into CHILD_CFUN. */
5535 if (gimple_in_ssa_p (cfun
))
5537 init_tree_ssa (child_cfun
);
5538 init_ssa_operands (child_cfun
);
5539 child_cfun
->gimple_df
->in_ssa_p
= true;
5543 block
= gimple_block (entry_stmt
);
5545 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
5547 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
5550 basic_block dest_bb
= e2
->dest
;
5552 make_edge (new_bb
, dest_bb
, EDGE_FALLTHRU
);
5554 set_immediate_dominator (CDI_DOMINATORS
, dest_bb
, new_bb
);
5556 /* When the OMP expansion process cannot guarantee an up-to-date
5557 loop tree arrange for the child function to fixup loops. */
5558 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
5559 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
5561 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5562 num
= vec_safe_length (child_cfun
->local_decls
);
5563 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
5565 t
= (*child_cfun
->local_decls
)[srcidx
];
5566 if (DECL_CONTEXT (t
) == cfun
->decl
)
5568 if (srcidx
!= dstidx
)
5569 (*child_cfun
->local_decls
)[dstidx
] = t
;
5573 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
5575 /* Inform the callgraph about the new function. */
5576 child_cfun
->curr_properties
= cfun
->curr_properties
;
5577 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
5578 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
5579 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
5580 node
->parallelized_function
= 1;
5581 cgraph_node::add_new_function (child_fn
, true);
5583 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5584 fixed in a following pass. */
5585 push_cfun (child_cfun
);
5587 optimize_omp_library_calls (entry_stmt
);
5588 cgraph_edge::rebuild_edges ();
5590 /* Some EH regions might become dead, see PR34608. If
5591 pass_cleanup_cfg isn't the first pass to happen with the
5592 new child, these dead EH edges might cause problems.
5593 Clean them up now. */
5594 if (flag_exceptions
)
5597 bool changed
= false;
5599 FOR_EACH_BB_FN (bb
, cfun
)
5600 changed
|= gimple_purge_dead_eh_edges (bb
);
5602 cleanup_tree_cfg ();
5604 if (gimple_in_ssa_p (cfun
))
5605 update_ssa (TODO_update_ssa
);
5609 /* Emit a library call to launch the children threads. */
5611 expand_cilk_for_call (new_bb
,
5612 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
5613 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5614 expand_parallel_call (region
, new_bb
,
5615 as_a
<gomp_parallel
*> (entry_stmt
), ws_args
);
5617 expand_task_call (new_bb
, as_a
<gomp_task
*> (entry_stmt
));
5618 if (gimple_in_ssa_p (cfun
))
5619 update_ssa (TODO_update_ssa_only_virtuals
);
5623 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5624 of the combined collapse > 1 loop constructs, generate code like:
5625 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5630 count3 = (adj + N32 - N31) / STEP3;
5631 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5636 count2 = (adj + N22 - N21) / STEP2;
5637 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5642 count1 = (adj + N12 - N11) / STEP1;
5643 count = count1 * count2 * count3;
5644 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5646 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5647 of the combined loop constructs, just initialize COUNTS array
5648 from the _looptemp_ clauses. */
5650 /* NOTE: It *could* be better to moosh all of the BBs together,
5651 creating one larger BB with all the computation and the unexpected
5652 jump at the end. I.e.
5654 bool zero3, zero2, zero1, zero;
5657 count3 = (N32 - N31) /[cl] STEP3;
5659 count2 = (N22 - N21) /[cl] STEP2;
5661 count1 = (N12 - N11) /[cl] STEP1;
5662 zero = zero3 || zero2 || zero1;
5663 count = count1 * count2 * count3;
5664 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5666 After all, we expect the zero=false, and thus we expect to have to
5667 evaluate all of the comparison expressions, so short-circuiting
5668 oughtn't be a win. Since the condition isn't protecting a
5669 denominator, we're not concerned about divide-by-zero, so we can
5670 fully evaluate count even if a numerator turned out to be wrong.
5672 It seems like putting this all together would create much better
5673 scheduling opportunities, and less pressure on the chip's branch
5677 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5678 basic_block
&entry_bb
, tree
*counts
,
5679 basic_block
&zero_iter_bb
, int &first_zero_iter
,
5680 basic_block
&l2_dom_bb
)
5682 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5686 /* Collapsed loops need work for expansion into SSA form. */
5687 gcc_assert (!gimple_in_ssa_p (cfun
));
5689 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5690 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5692 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5693 isn't supposed to be handled, as the inner loop doesn't
5695 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5696 OMP_CLAUSE__LOOPTEMP_
);
5697 gcc_assert (innerc
);
5698 for (i
= 0; i
< fd
->collapse
; i
++)
5700 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5701 OMP_CLAUSE__LOOPTEMP_
);
5702 gcc_assert (innerc
);
5704 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5706 counts
[0] = NULL_TREE
;
5711 for (i
= 0; i
< fd
->collapse
; i
++)
5713 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5715 if (SSA_VAR_P (fd
->loop
.n2
)
5716 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5717 fold_convert (itype
, fd
->loops
[i
].n1
),
5718 fold_convert (itype
, fd
->loops
[i
].n2
)))
5719 == NULL_TREE
|| !integer_onep (t
)))
5723 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5724 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5725 true, GSI_SAME_STMT
);
5726 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5727 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5728 true, GSI_SAME_STMT
);
5729 cond_stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5730 NULL_TREE
, NULL_TREE
);
5731 gsi_insert_before (gsi
, cond_stmt
, GSI_SAME_STMT
);
5732 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
5733 expand_omp_regimplify_p
, NULL
, NULL
)
5734 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
5735 expand_omp_regimplify_p
, NULL
, NULL
))
5737 *gsi
= gsi_for_stmt (cond_stmt
);
5738 gimple_regimplify_operands (cond_stmt
, gsi
);
5740 e
= split_block (entry_bb
, cond_stmt
);
5741 if (zero_iter_bb
== NULL
)
5743 gassign
*assign_stmt
;
5744 first_zero_iter
= i
;
5745 zero_iter_bb
= create_empty_bb (entry_bb
);
5746 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5747 *gsi
= gsi_after_labels (zero_iter_bb
);
5748 assign_stmt
= gimple_build_assign (fd
->loop
.n2
,
5749 build_zero_cst (type
));
5750 gsi_insert_before (gsi
, assign_stmt
, GSI_SAME_STMT
);
5751 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5754 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5755 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5756 e
->flags
= EDGE_TRUE_VALUE
;
5757 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5758 if (l2_dom_bb
== NULL
)
5759 l2_dom_bb
= entry_bb
;
5761 *gsi
= gsi_last_bb (entry_bb
);
5764 if (POINTER_TYPE_P (itype
))
5765 itype
= signed_type_for (itype
);
5766 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5768 t
= fold_build2 (PLUS_EXPR
, itype
,
5769 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5770 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5771 fold_convert (itype
, fd
->loops
[i
].n2
));
5772 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5773 fold_convert (itype
, fd
->loops
[i
].n1
));
5774 /* ?? We could probably use CEIL_DIV_EXPR instead of
5775 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5776 generate the same code in the end because generically we
5777 don't know that the values involved must be negative for
5779 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5780 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5781 fold_build1 (NEGATE_EXPR
, itype
, t
),
5782 fold_build1 (NEGATE_EXPR
, itype
,
5783 fold_convert (itype
,
5784 fd
->loops
[i
].step
)));
5786 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5787 fold_convert (itype
, fd
->loops
[i
].step
));
5788 t
= fold_convert (type
, t
);
5789 if (TREE_CODE (t
) == INTEGER_CST
)
5793 counts
[i
] = create_tmp_reg (type
, ".count");
5794 expand_omp_build_assign (gsi
, counts
[i
], t
);
5796 if (SSA_VAR_P (fd
->loop
.n2
))
5801 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5802 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5808 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5810 V3 = N31 + (T % count3) * STEP3;
5812 V2 = N21 + (T % count2) * STEP2;
5814 V1 = N11 + T * STEP1;
5815 if this loop doesn't have an inner loop construct combined with it.
5816 If it does have an inner loop construct combined with it and the
5817 iteration count isn't known constant, store values from counts array
5818 into its _looptemp_ temporaries instead. */
5821 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5822 tree
*counts
, gimple inner_stmt
, tree startvar
)
5825 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5827 /* If fd->loop.n2 is constant, then no propagation of the counts
5828 is needed, they are constant. */
5829 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5832 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5833 ? gimple_omp_parallel_clauses (inner_stmt
)
5834 : gimple_omp_for_clauses (inner_stmt
);
5835 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5836 isn't supposed to be handled, as the inner loop doesn't
5838 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5839 gcc_assert (innerc
);
5840 for (i
= 0; i
< fd
->collapse
; i
++)
5842 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5843 OMP_CLAUSE__LOOPTEMP_
);
5844 gcc_assert (innerc
);
5847 tree tem
= OMP_CLAUSE_DECL (innerc
);
5848 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5849 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5850 false, GSI_CONTINUE_LINKING
);
5851 gassign
*stmt
= gimple_build_assign (tem
, t
);
5852 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5858 tree type
= TREE_TYPE (fd
->loop
.v
);
5859 tree tem
= create_tmp_reg (type
, ".tem");
5860 gassign
*stmt
= gimple_build_assign (tem
, startvar
);
5861 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5863 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5865 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5867 if (POINTER_TYPE_P (vtype
))
5868 itype
= signed_type_for (vtype
);
5870 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5873 t
= fold_convert (itype
, t
);
5874 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5875 fold_convert (itype
, fd
->loops
[i
].step
));
5876 if (POINTER_TYPE_P (vtype
))
5877 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5879 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5880 t
= force_gimple_operand_gsi (gsi
, t
,
5881 DECL_P (fd
->loops
[i
].v
)
5882 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5884 GSI_CONTINUE_LINKING
);
5885 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5886 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5889 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5890 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5891 false, GSI_CONTINUE_LINKING
);
5892 stmt
= gimple_build_assign (tem
, t
);
5893 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5899 /* Helper function for expand_omp_for_*. Generate code like:
5902 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5906 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5913 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5914 basic_block body_bb
)
5916 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5918 gimple_stmt_iterator gsi
;
5924 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5926 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5928 bb
= create_empty_bb (last_bb
);
5929 add_bb_to_loop (bb
, last_bb
->loop_father
);
5930 gsi
= gsi_start_bb (bb
);
5932 if (i
< fd
->collapse
- 1)
5934 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5935 e
->probability
= REG_BR_PROB_BASE
/ 8;
5937 t
= fd
->loops
[i
+ 1].n1
;
5938 t
= force_gimple_operand_gsi (&gsi
, t
,
5939 DECL_P (fd
->loops
[i
+ 1].v
)
5940 && TREE_ADDRESSABLE (fd
->loops
[i
5943 GSI_CONTINUE_LINKING
);
5944 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5945 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5950 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5952 if (POINTER_TYPE_P (vtype
))
5953 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5955 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5956 t
= force_gimple_operand_gsi (&gsi
, t
,
5957 DECL_P (fd
->loops
[i
].v
)
5958 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5959 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5960 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5961 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5965 t
= fd
->loops
[i
].n2
;
5966 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5967 false, GSI_CONTINUE_LINKING
);
5968 tree v
= fd
->loops
[i
].v
;
5969 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5970 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5971 false, GSI_CONTINUE_LINKING
);
5972 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5973 stmt
= gimple_build_cond_empty (t
);
5974 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5975 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5976 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5979 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5987 /* A subroutine of expand_omp_for. Generate code for a parallel
5988 loop with any schedule. Given parameters:
5990 for (V = N1; V cond N2; V += STEP) BODY;
5992 where COND is "<" or ">", we generate pseudocode
5994 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5995 if (more) goto L0; else goto L3;
6002 if (V cond iend) goto L1; else goto L2;
6004 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6007 If this is a combined omp parallel loop, instead of the call to
6008 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
6009 If this is gimple_omp_for_combined_p loop, then instead of assigning
6010 V and iend in L0 we assign the first two _looptemp_ clause decls of the
6011 inner GIMPLE_OMP_FOR and V += STEP; and
6012 if (V cond iend) goto L1; else goto L2; are removed.
6014 For collapsed loops, given parameters:
6016 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6017 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6018 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6021 we generate pseudocode
6023 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
6028 count3 = (adj + N32 - N31) / STEP3;
6029 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
6034 count2 = (adj + N22 - N21) / STEP2;
6035 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
6040 count1 = (adj + N12 - N11) / STEP1;
6041 count = count1 * count2 * count3;
6046 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
6047 if (more) goto L0; else goto L3;
6051 V3 = N31 + (T % count3) * STEP3;
6053 V2 = N21 + (T % count2) * STEP2;
6055 V1 = N11 + T * STEP1;
6060 if (V < iend) goto L10; else goto L2;
6063 if (V3 cond3 N32) goto L1; else goto L11;
6067 if (V2 cond2 N22) goto L1; else goto L12;
6073 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
6079 expand_omp_for_generic (struct omp_region
*region
,
6080 struct omp_for_data
*fd
,
6081 enum built_in_function start_fn
,
6082 enum built_in_function next_fn
,
6085 tree type
, istart0
, iend0
, iend
;
6086 tree t
, vmain
, vback
, bias
= NULL_TREE
;
6087 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
6088 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
6089 gimple_stmt_iterator gsi
;
6090 gassign
*assign_stmt
;
6091 bool in_combined_parallel
= is_combined_parallel (region
);
6092 bool broken_loop
= region
->cont
== NULL
;
6094 tree
*counts
= NULL
;
6097 gcc_assert (!broken_loop
|| !in_combined_parallel
);
6098 gcc_assert (fd
->iter_type
== long_integer_type_node
6099 || !in_combined_parallel
);
6101 type
= TREE_TYPE (fd
->loop
.v
);
6102 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
6103 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
6104 TREE_ADDRESSABLE (istart0
) = 1;
6105 TREE_ADDRESSABLE (iend0
) = 1;
6107 /* See if we need to bias by LLONG_MIN. */
6108 if (fd
->iter_type
== long_long_unsigned_type_node
6109 && TREE_CODE (type
) == INTEGER_TYPE
6110 && !TYPE_UNSIGNED (type
))
6114 if (fd
->loop
.cond_code
== LT_EXPR
)
6117 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6121 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
6124 if (TREE_CODE (n1
) != INTEGER_CST
6125 || TREE_CODE (n2
) != INTEGER_CST
6126 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
6127 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
6130 entry_bb
= region
->entry
;
6131 cont_bb
= region
->cont
;
6133 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6134 gcc_assert (broken_loop
6135 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6136 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6137 l1_bb
= single_succ (l0_bb
);
6140 l2_bb
= create_empty_bb (cont_bb
);
6141 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
6142 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6146 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6147 exit_bb
= region
->exit
;
6149 gsi
= gsi_last_bb (entry_bb
);
6151 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6152 if (fd
->collapse
> 1)
6154 int first_zero_iter
= -1;
6155 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
6157 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6158 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6159 zero_iter_bb
, first_zero_iter
,
6164 /* Some counts[i] vars might be uninitialized if
6165 some loop has zero iterations. But the body shouldn't
6166 be executed in that case, so just avoid uninit warnings. */
6167 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
6168 if (SSA_VAR_P (counts
[i
]))
6169 TREE_NO_WARNING (counts
[i
]) = 1;
6171 e
= split_block (entry_bb
, gsi_stmt (gsi
));
6173 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
6174 gsi
= gsi_last_bb (entry_bb
);
6175 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
6176 get_immediate_dominator (CDI_DOMINATORS
,
6180 if (in_combined_parallel
)
6182 /* In a combined parallel loop, emit a call to
6183 GOMP_loop_foo_next. */
6184 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6185 build_fold_addr_expr (istart0
),
6186 build_fold_addr_expr (iend0
));
6190 tree t0
, t1
, t2
, t3
, t4
;
6191 /* If this is not a combined parallel loop, emit a call to
6192 GOMP_loop_foo_start in ENTRY_BB. */
6193 t4
= build_fold_addr_expr (iend0
);
6194 t3
= build_fold_addr_expr (istart0
);
6195 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
6198 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6200 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6201 OMP_CLAUSE__LOOPTEMP_
);
6202 gcc_assert (innerc
);
6203 t0
= OMP_CLAUSE_DECL (innerc
);
6204 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6205 OMP_CLAUSE__LOOPTEMP_
);
6206 gcc_assert (innerc
);
6207 t1
= OMP_CLAUSE_DECL (innerc
);
6209 if (POINTER_TYPE_P (TREE_TYPE (t0
))
6210 && TYPE_PRECISION (TREE_TYPE (t0
))
6211 != TYPE_PRECISION (fd
->iter_type
))
6213 /* Avoid casting pointers to integer of a different size. */
6214 tree itype
= signed_type_for (type
);
6215 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
6216 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
6220 t1
= fold_convert (fd
->iter_type
, t1
);
6221 t0
= fold_convert (fd
->iter_type
, t0
);
6225 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
6226 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
6228 if (fd
->iter_type
== long_integer_type_node
)
6232 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6233 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6234 6, t0
, t1
, t2
, t
, t3
, t4
);
6237 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6238 5, t0
, t1
, t2
, t3
, t4
);
6246 /* The GOMP_loop_ull_*start functions have additional boolean
6247 argument, true for < loops and false for > loops.
6248 In Fortran, the C bool type can be different from
6249 boolean_type_node. */
6250 bfn_decl
= builtin_decl_explicit (start_fn
);
6251 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
6252 t5
= build_int_cst (c_bool_type
,
6253 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
6256 tree bfn_decl
= builtin_decl_explicit (start_fn
);
6257 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
6258 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
6261 t
= build_call_expr (builtin_decl_explicit (start_fn
),
6262 6, t5
, t0
, t1
, t2
, t3
, t4
);
6265 if (TREE_TYPE (t
) != boolean_type_node
)
6266 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6267 t
, build_int_cst (TREE_TYPE (t
), 0));
6268 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6269 true, GSI_SAME_STMT
);
6270 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6272 /* Remove the GIMPLE_OMP_FOR statement. */
6273 gsi_remove (&gsi
, true);
6275 /* Iteration setup for sequential loop goes in L0_BB. */
6276 tree startvar
= fd
->loop
.v
;
6277 tree endvar
= NULL_TREE
;
6279 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6281 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
6282 && gimple_omp_for_kind (inner_stmt
)
6283 == GF_OMP_FOR_KIND_SIMD
);
6284 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
6285 OMP_CLAUSE__LOOPTEMP_
);
6286 gcc_assert (innerc
);
6287 startvar
= OMP_CLAUSE_DECL (innerc
);
6288 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6289 OMP_CLAUSE__LOOPTEMP_
);
6290 gcc_assert (innerc
);
6291 endvar
= OMP_CLAUSE_DECL (innerc
);
6294 gsi
= gsi_start_bb (l0_bb
);
6297 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6298 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6299 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6300 t
= fold_convert (TREE_TYPE (startvar
), t
);
6301 t
= force_gimple_operand_gsi (&gsi
, t
,
6303 && TREE_ADDRESSABLE (startvar
),
6304 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6305 assign_stmt
= gimple_build_assign (startvar
, t
);
6306 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6310 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
6311 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
6312 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
6313 t
= fold_convert (TREE_TYPE (startvar
), t
);
6314 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6315 false, GSI_CONTINUE_LINKING
);
6318 assign_stmt
= gimple_build_assign (endvar
, iend
);
6319 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6320 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
6321 assign_stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
6323 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, iend
);
6324 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6326 if (fd
->collapse
> 1)
6327 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6331 /* Code to control the increment and predicate for the sequential
6332 loop goes in the CONT_BB. */
6333 gsi
= gsi_last_bb (cont_bb
);
6334 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6335 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
6336 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6337 vback
= gimple_omp_continue_control_def (cont_stmt
);
6339 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6341 if (POINTER_TYPE_P (type
))
6342 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
6344 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
6345 t
= force_gimple_operand_gsi (&gsi
, t
,
6347 && TREE_ADDRESSABLE (vback
),
6348 NULL_TREE
, true, GSI_SAME_STMT
);
6349 assign_stmt
= gimple_build_assign (vback
, t
);
6350 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6352 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6353 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
6355 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6356 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6359 /* Remove GIMPLE_OMP_CONTINUE. */
6360 gsi_remove (&gsi
, true);
6362 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6363 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
6365 /* Emit code to get the next parallel iteration in L2_BB. */
6366 gsi
= gsi_start_bb (l2_bb
);
6368 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
6369 build_fold_addr_expr (istart0
),
6370 build_fold_addr_expr (iend0
));
6371 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6372 false, GSI_CONTINUE_LINKING
);
6373 if (TREE_TYPE (t
) != boolean_type_node
)
6374 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
6375 t
, build_int_cst (TREE_TYPE (t
), 0));
6376 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6377 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
6380 /* Add the loop cleanup function. */
6381 gsi
= gsi_last_bb (exit_bb
);
6382 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6383 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
6384 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6385 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
6387 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
6388 gcall
*call_stmt
= gimple_build_call (t
, 0);
6389 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
6390 gimple_call_set_lhs (call_stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
6391 gsi_insert_after (&gsi
, call_stmt
, GSI_SAME_STMT
);
6392 gsi_remove (&gsi
, true);
6394 /* Connect the new blocks. */
6395 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
6396 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
6402 e
= find_edge (cont_bb
, l3_bb
);
6403 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
6405 phis
= phi_nodes (l3_bb
);
6406 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
6408 gimple phi
= gsi_stmt (gsi
);
6409 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
6410 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
6414 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
6415 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
6416 e
= find_edge (cont_bb
, l1_bb
);
6417 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6422 else if (fd
->collapse
> 1)
6425 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6428 e
->flags
= EDGE_TRUE_VALUE
;
6431 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6432 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
6436 e
= find_edge (cont_bb
, l2_bb
);
6437 e
->flags
= EDGE_FALLTHRU
;
6439 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6441 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
6442 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
6443 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
6444 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
6445 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
6446 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
6447 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
6448 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
6450 struct loop
*outer_loop
= alloc_loop ();
6451 outer_loop
->header
= l0_bb
;
6452 outer_loop
->latch
= l2_bb
;
6453 add_loop (outer_loop
, l0_bb
->loop_father
);
6455 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6457 struct loop
*loop
= alloc_loop ();
6458 loop
->header
= l1_bb
;
6459 /* The loop may have multiple latches. */
6460 add_loop (loop
, outer_loop
);
6466 /* A subroutine of expand_omp_for. Generate code for a parallel
6467 loop with static schedule and no specified chunk size. Given
6470 for (V = N1; V cond N2; V += STEP) BODY;
6472 where COND is "<" or ">", we generate pseudocode
6474 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6479 if ((__typeof (V)) -1 > 0 && cond is >)
6480 n = -(adj + N2 - N1) / -STEP;
6482 n = (adj + N2 - N1) / STEP;
6485 if (threadid < tt) goto L3; else goto L4;
6490 s0 = q * threadid + tt;
6493 if (s0 >= e0) goto L2; else goto L0;
6499 if (V cond e) goto L1;
6504 expand_omp_for_static_nochunk (struct omp_region
*region
,
6505 struct omp_for_data
*fd
,
6508 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
6509 tree type
, itype
, vmain
, vback
;
6510 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
6511 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
6513 gimple_stmt_iterator gsi
;
6515 bool broken_loop
= region
->cont
== NULL
;
6516 tree
*counts
= NULL
;
6519 gcc_checking_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6520 != GF_OMP_FOR_KIND_OACC_LOOP
)
6523 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6524 if (POINTER_TYPE_P (type
))
6525 itype
= signed_type_for (type
);
6527 entry_bb
= region
->entry
;
6528 cont_bb
= region
->cont
;
6529 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6530 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6531 gcc_assert (broken_loop
6532 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
6533 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6534 body_bb
= single_succ (seq_start_bb
);
6537 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6538 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6540 exit_bb
= region
->exit
;
6542 /* Iteration space partitioning goes in ENTRY_BB. */
6543 gsi
= gsi_last_bb (entry_bb
);
6544 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6546 if (fd
->collapse
> 1)
6548 int first_zero_iter
= -1;
6549 basic_block l2_dom_bb
= NULL
;
6551 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6552 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6553 fin_bb
, first_zero_iter
,
6557 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6558 t
= integer_one_node
;
6560 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6561 fold_convert (type
, fd
->loop
.n1
),
6562 fold_convert (type
, fd
->loop
.n2
));
6563 if (fd
->collapse
== 1
6564 && TYPE_UNSIGNED (type
)
6565 && (t
== NULL_TREE
|| !integer_onep (t
)))
6567 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6568 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6569 true, GSI_SAME_STMT
);
6570 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6571 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6572 true, GSI_SAME_STMT
);
6573 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6574 NULL_TREE
, NULL_TREE
);
6575 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6576 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6577 expand_omp_regimplify_p
, NULL
, NULL
)
6578 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6579 expand_omp_regimplify_p
, NULL
, NULL
))
6581 gsi
= gsi_for_stmt (cond_stmt
);
6582 gimple_regimplify_operands (cond_stmt
, &gsi
);
6584 ep
= split_block (entry_bb
, cond_stmt
);
6585 ep
->flags
= EDGE_TRUE_VALUE
;
6586 entry_bb
= ep
->dest
;
6587 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6588 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6589 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6590 if (gimple_in_ssa_p (cfun
))
6592 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6593 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
6594 !gsi_end_p (gpi
); gsi_next (&gpi
))
6596 gphi
*phi
= gpi
.phi ();
6597 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6598 ep
, UNKNOWN_LOCATION
);
6601 gsi
= gsi_last_bb (entry_bb
);
6604 switch (gimple_omp_for_kind (fd
->for_stmt
))
6606 case GF_OMP_FOR_KIND_FOR
:
6607 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
6608 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
6610 case GF_OMP_FOR_KIND_DISTRIBUTE
:
6611 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
6612 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
6614 case GF_OMP_FOR_KIND_OACC_LOOP
:
6615 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
6616 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
6621 nthreads
= build_call_expr (nthreads
, 0);
6622 nthreads
= fold_convert (itype
, nthreads
);
6623 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
6624 true, GSI_SAME_STMT
);
6625 threadid
= build_call_expr (threadid
, 0);
6626 threadid
= fold_convert (itype
, threadid
);
6627 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
6628 true, GSI_SAME_STMT
);
6632 step
= fd
->loop
.step
;
6633 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6635 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6636 OMP_CLAUSE__LOOPTEMP_
);
6637 gcc_assert (innerc
);
6638 n1
= OMP_CLAUSE_DECL (innerc
);
6639 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6640 OMP_CLAUSE__LOOPTEMP_
);
6641 gcc_assert (innerc
);
6642 n2
= OMP_CLAUSE_DECL (innerc
);
6644 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6645 true, NULL_TREE
, true, GSI_SAME_STMT
);
6646 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6647 true, NULL_TREE
, true, GSI_SAME_STMT
);
6648 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6649 true, NULL_TREE
, true, GSI_SAME_STMT
);
6651 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6652 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6653 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6654 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6655 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6656 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6657 fold_build1 (NEGATE_EXPR
, itype
, t
),
6658 fold_build1 (NEGATE_EXPR
, itype
, step
));
6660 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6661 t
= fold_convert (itype
, t
);
6662 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6664 q
= create_tmp_reg (itype
, "q");
6665 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
6666 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6667 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
6669 tt
= create_tmp_reg (itype
, "tt");
6670 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
6671 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6672 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
6674 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
6675 gcond
*cond_stmt
= gimple_build_cond_empty (t
);
6676 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6678 second_bb
= split_block (entry_bb
, cond_stmt
)->dest
;
6679 gsi
= gsi_last_bb (second_bb
);
6680 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6682 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
6684 gassign
*assign_stmt
6685 = gimple_build_assign (q
, PLUS_EXPR
, q
, build_int_cst (itype
, 1));
6686 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6688 third_bb
= split_block (second_bb
, assign_stmt
)->dest
;
6689 gsi
= gsi_last_bb (third_bb
);
6690 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6692 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6693 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6694 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6696 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6697 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6699 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6700 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6702 /* Remove the GIMPLE_OMP_FOR statement. */
6703 gsi_remove (&gsi
, true);
6705 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6706 gsi
= gsi_start_bb (seq_start_bb
);
6708 tree startvar
= fd
->loop
.v
;
6709 tree endvar
= NULL_TREE
;
6711 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6713 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6714 ? gimple_omp_parallel_clauses (inner_stmt
)
6715 : gimple_omp_for_clauses (inner_stmt
);
6716 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6717 gcc_assert (innerc
);
6718 startvar
= OMP_CLAUSE_DECL (innerc
);
6719 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6720 OMP_CLAUSE__LOOPTEMP_
);
6721 gcc_assert (innerc
);
6722 endvar
= OMP_CLAUSE_DECL (innerc
);
6724 t
= fold_convert (itype
, s0
);
6725 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6726 if (POINTER_TYPE_P (type
))
6727 t
= fold_build_pointer_plus (n1
, t
);
6729 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6730 t
= fold_convert (TREE_TYPE (startvar
), t
);
6731 t
= force_gimple_operand_gsi (&gsi
, t
,
6733 && TREE_ADDRESSABLE (startvar
),
6734 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6735 assign_stmt
= gimple_build_assign (startvar
, t
);
6736 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6738 t
= fold_convert (itype
, e0
);
6739 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6740 if (POINTER_TYPE_P (type
))
6741 t
= fold_build_pointer_plus (n1
, t
);
6743 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6744 t
= fold_convert (TREE_TYPE (startvar
), t
);
6745 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6746 false, GSI_CONTINUE_LINKING
);
6749 assign_stmt
= gimple_build_assign (endvar
, e
);
6750 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6751 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6752 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6754 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
6755 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
6757 if (fd
->collapse
> 1)
6758 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6762 /* The code controlling the sequential loop replaces the
6763 GIMPLE_OMP_CONTINUE. */
6764 gsi
= gsi_last_bb (cont_bb
);
6765 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
6766 gcc_assert (gimple_code (cont_stmt
) == GIMPLE_OMP_CONTINUE
);
6767 vmain
= gimple_omp_continue_control_use (cont_stmt
);
6768 vback
= gimple_omp_continue_control_def (cont_stmt
);
6770 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6772 if (POINTER_TYPE_P (type
))
6773 t
= fold_build_pointer_plus (vmain
, step
);
6775 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6776 t
= force_gimple_operand_gsi (&gsi
, t
,
6778 && TREE_ADDRESSABLE (vback
),
6779 NULL_TREE
, true, GSI_SAME_STMT
);
6780 assign_stmt
= gimple_build_assign (vback
, t
);
6781 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
6783 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6784 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6786 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6789 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6790 gsi_remove (&gsi
, true);
6792 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6793 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6796 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6797 gsi
= gsi_last_bb (exit_bb
);
6798 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6800 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6801 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
6802 gcc_checking_assert (t
== NULL_TREE
);
6804 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6806 gsi_remove (&gsi
, true);
6808 /* Connect all the blocks. */
6809 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6810 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6811 ep
= find_edge (entry_bb
, second_bb
);
6812 ep
->flags
= EDGE_TRUE_VALUE
;
6813 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6814 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6815 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6819 ep
= find_edge (cont_bb
, body_bb
);
6820 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6825 else if (fd
->collapse
> 1)
6828 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6831 ep
->flags
= EDGE_TRUE_VALUE
;
6832 find_edge (cont_bb
, fin_bb
)->flags
6833 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6836 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6837 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6838 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6840 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6841 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6842 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6843 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6845 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6847 struct loop
*loop
= alloc_loop ();
6848 loop
->header
= body_bb
;
6849 if (collapse_bb
== NULL
)
6850 loop
->latch
= cont_bb
;
6851 add_loop (loop
, body_bb
->loop_father
);
6856 /* A subroutine of expand_omp_for. Generate code for a parallel
6857 loop with static schedule and a specified chunk size. Given
6860 for (V = N1; V cond N2; V += STEP) BODY;
6862 where COND is "<" or ">", we generate pseudocode
6864 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6869 if ((__typeof (V)) -1 > 0 && cond is >)
6870 n = -(adj + N2 - N1) / -STEP;
6872 n = (adj + N2 - N1) / STEP;
6874 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6875 here so that V is defined
6876 if the loop is not entered
6878 s0 = (trip * nthreads + threadid) * CHUNK;
6879 e0 = min(s0 + CHUNK, n);
6880 if (s0 < n) goto L1; else goto L4;
6887 if (V cond e) goto L2; else goto L3;
6895 expand_omp_for_static_chunk (struct omp_region
*region
,
6896 struct omp_for_data
*fd
, gimple inner_stmt
)
6898 tree n
, s0
, e0
, e
, t
;
6899 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6900 tree type
, itype
, vmain
, vback
, vextra
;
6901 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6902 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6903 gimple_stmt_iterator gsi
;
6905 bool broken_loop
= region
->cont
== NULL
;
6906 tree
*counts
= NULL
;
6909 gcc_checking_assert ((gimple_omp_for_kind (fd
->for_stmt
)
6910 != GF_OMP_FOR_KIND_OACC_LOOP
)
6913 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6914 if (POINTER_TYPE_P (type
))
6915 itype
= signed_type_for (type
);
6917 entry_bb
= region
->entry
;
6918 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6920 iter_part_bb
= se
->dest
;
6921 cont_bb
= region
->cont
;
6922 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6923 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6924 gcc_assert (broken_loop
6925 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6926 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6927 body_bb
= single_succ (seq_start_bb
);
6930 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6931 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6932 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6934 exit_bb
= region
->exit
;
6936 /* Trip and adjustment setup goes in ENTRY_BB. */
6937 gsi
= gsi_last_bb (entry_bb
);
6938 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6940 if (fd
->collapse
> 1)
6942 int first_zero_iter
= -1;
6943 basic_block l2_dom_bb
= NULL
;
6945 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6946 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6947 fin_bb
, first_zero_iter
,
6951 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6952 t
= integer_one_node
;
6954 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6955 fold_convert (type
, fd
->loop
.n1
),
6956 fold_convert (type
, fd
->loop
.n2
));
6957 if (fd
->collapse
== 1
6958 && TYPE_UNSIGNED (type
)
6959 && (t
== NULL_TREE
|| !integer_onep (t
)))
6961 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6962 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6963 true, GSI_SAME_STMT
);
6964 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6965 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6966 true, GSI_SAME_STMT
);
6967 gcond
*cond_stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6968 NULL_TREE
, NULL_TREE
);
6969 gsi_insert_before (&gsi
, cond_stmt
, GSI_SAME_STMT
);
6970 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
6971 expand_omp_regimplify_p
, NULL
, NULL
)
6972 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
6973 expand_omp_regimplify_p
, NULL
, NULL
))
6975 gsi
= gsi_for_stmt (cond_stmt
);
6976 gimple_regimplify_operands (cond_stmt
, &gsi
);
6978 se
= split_block (entry_bb
, cond_stmt
);
6979 se
->flags
= EDGE_TRUE_VALUE
;
6980 entry_bb
= se
->dest
;
6981 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6982 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6983 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6984 if (gimple_in_ssa_p (cfun
))
6986 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6987 for (gphi_iterator gpi
= gsi_start_phis (fin_bb
);
6988 !gsi_end_p (gpi
); gsi_next (&gpi
))
6990 gphi
*phi
= gpi
.phi ();
6991 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6992 se
, UNKNOWN_LOCATION
);
6995 gsi
= gsi_last_bb (entry_bb
);
6998 switch (gimple_omp_for_kind (fd
->for_stmt
))
7000 case GF_OMP_FOR_KIND_FOR
:
7001 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
7002 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
7004 case GF_OMP_FOR_KIND_DISTRIBUTE
:
7005 nthreads
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS
);
7006 threadid
= builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM
);
7008 case GF_OMP_FOR_KIND_OACC_LOOP
:
7009 nthreads
= builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS
);
7010 threadid
= builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM
);
7015 nthreads
= build_call_expr (nthreads
, 0);
7016 nthreads
= fold_convert (itype
, nthreads
);
7017 nthreads
= force_gimple_operand_gsi (&gsi
, nthreads
, true, NULL_TREE
,
7018 true, GSI_SAME_STMT
);
7019 threadid
= build_call_expr (threadid
, 0);
7020 threadid
= fold_convert (itype
, threadid
);
7021 threadid
= force_gimple_operand_gsi (&gsi
, threadid
, true, NULL_TREE
,
7022 true, GSI_SAME_STMT
);
7026 step
= fd
->loop
.step
;
7027 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7029 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7030 OMP_CLAUSE__LOOPTEMP_
);
7031 gcc_assert (innerc
);
7032 n1
= OMP_CLAUSE_DECL (innerc
);
7033 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7034 OMP_CLAUSE__LOOPTEMP_
);
7035 gcc_assert (innerc
);
7036 n2
= OMP_CLAUSE_DECL (innerc
);
7038 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
7039 true, NULL_TREE
, true, GSI_SAME_STMT
);
7040 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
7041 true, NULL_TREE
, true, GSI_SAME_STMT
);
7042 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
7043 true, NULL_TREE
, true, GSI_SAME_STMT
);
7045 = force_gimple_operand_gsi (&gsi
, fold_convert (itype
, fd
->chunk_size
),
7046 true, NULL_TREE
, true, GSI_SAME_STMT
);
7048 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
7049 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
7050 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
7051 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
7052 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
7053 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
7054 fold_build1 (NEGATE_EXPR
, itype
, t
),
7055 fold_build1 (NEGATE_EXPR
, itype
, step
));
7057 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
7058 t
= fold_convert (itype
, t
);
7059 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7060 true, GSI_SAME_STMT
);
7062 trip_var
= create_tmp_reg (itype
, ".trip");
7063 if (gimple_in_ssa_p (cfun
))
7065 trip_init
= make_ssa_name (trip_var
);
7066 trip_main
= make_ssa_name (trip_var
);
7067 trip_back
= make_ssa_name (trip_var
);
7071 trip_init
= trip_var
;
7072 trip_main
= trip_var
;
7073 trip_back
= trip_var
;
7076 gassign
*assign_stmt
7077 = gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
7078 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
7080 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
7081 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7082 if (POINTER_TYPE_P (type
))
7083 t
= fold_build_pointer_plus (n1
, t
);
7085 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7086 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7087 true, GSI_SAME_STMT
);
7089 /* Remove the GIMPLE_OMP_FOR. */
7090 gsi_remove (&gsi
, true);
7092 /* Iteration space partitioning goes in ITER_PART_BB. */
7093 gsi
= gsi_last_bb (iter_part_bb
);
7095 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
7096 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
7097 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
7098 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7099 false, GSI_CONTINUE_LINKING
);
7101 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
7102 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
7103 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7104 false, GSI_CONTINUE_LINKING
);
7106 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
7107 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
7109 /* Setup code for sequential iteration goes in SEQ_START_BB. */
7110 gsi
= gsi_start_bb (seq_start_bb
);
7112 tree startvar
= fd
->loop
.v
;
7113 tree endvar
= NULL_TREE
;
7115 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7117 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
7118 ? gimple_omp_parallel_clauses (inner_stmt
)
7119 : gimple_omp_for_clauses (inner_stmt
);
7120 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
7121 gcc_assert (innerc
);
7122 startvar
= OMP_CLAUSE_DECL (innerc
);
7123 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7124 OMP_CLAUSE__LOOPTEMP_
);
7125 gcc_assert (innerc
);
7126 endvar
= OMP_CLAUSE_DECL (innerc
);
7129 t
= fold_convert (itype
, s0
);
7130 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7131 if (POINTER_TYPE_P (type
))
7132 t
= fold_build_pointer_plus (n1
, t
);
7134 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7135 t
= fold_convert (TREE_TYPE (startvar
), t
);
7136 t
= force_gimple_operand_gsi (&gsi
, t
,
7138 && TREE_ADDRESSABLE (startvar
),
7139 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
7140 assign_stmt
= gimple_build_assign (startvar
, t
);
7141 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7143 t
= fold_convert (itype
, e0
);
7144 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
7145 if (POINTER_TYPE_P (type
))
7146 t
= fold_build_pointer_plus (n1
, t
);
7148 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
7149 t
= fold_convert (TREE_TYPE (startvar
), t
);
7150 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7151 false, GSI_CONTINUE_LINKING
);
7154 assign_stmt
= gimple_build_assign (endvar
, e
);
7155 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7156 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
7157 assign_stmt
= gimple_build_assign (fd
->loop
.v
, e
);
7159 assign_stmt
= gimple_build_assign (fd
->loop
.v
, NOP_EXPR
, e
);
7160 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7162 if (fd
->collapse
> 1)
7163 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
7167 /* The code controlling the sequential loop goes in CONT_BB,
7168 replacing the GIMPLE_OMP_CONTINUE. */
7169 gsi
= gsi_last_bb (cont_bb
);
7170 gomp_continue
*cont_stmt
= as_a
<gomp_continue
*> (gsi_stmt (gsi
));
7171 vmain
= gimple_omp_continue_control_use (cont_stmt
);
7172 vback
= gimple_omp_continue_control_def (cont_stmt
);
7174 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7176 if (POINTER_TYPE_P (type
))
7177 t
= fold_build_pointer_plus (vmain
, step
);
7179 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
7180 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
7181 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7182 true, GSI_SAME_STMT
);
7183 assign_stmt
= gimple_build_assign (vback
, t
);
7184 gsi_insert_before (&gsi
, assign_stmt
, GSI_SAME_STMT
);
7186 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
7187 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
7189 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
7192 /* Remove GIMPLE_OMP_CONTINUE. */
7193 gsi_remove (&gsi
, true);
7195 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
7196 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
7198 /* Trip update code goes into TRIP_UPDATE_BB. */
7199 gsi
= gsi_start_bb (trip_update_bb
);
7201 t
= build_int_cst (itype
, 1);
7202 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
7203 assign_stmt
= gimple_build_assign (trip_back
, t
);
7204 gsi_insert_after (&gsi
, assign_stmt
, GSI_CONTINUE_LINKING
);
7207 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
7208 gsi
= gsi_last_bb (exit_bb
);
7209 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
7211 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
7212 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_OACC_LOOP
)
7213 gcc_checking_assert (t
== NULL_TREE
);
7215 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
7217 gsi_remove (&gsi
, true);
7219 /* Connect the new blocks. */
7220 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
7221 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
7225 se
= find_edge (cont_bb
, body_bb
);
7226 if (gimple_omp_for_combined_p (fd
->for_stmt
))
7231 else if (fd
->collapse
> 1)
7234 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
7237 se
->flags
= EDGE_TRUE_VALUE
;
7238 find_edge (cont_bb
, trip_update_bb
)->flags
7239 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
7241 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
7244 if (gimple_in_ssa_p (cfun
))
7252 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
7254 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
7255 remove arguments of the phi nodes in fin_bb. We need to create
7256 appropriate phi nodes in iter_part_bb instead. */
7257 se
= single_pred_edge (fin_bb
);
7258 re
= single_succ_edge (trip_update_bb
);
7259 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
7260 ene
= single_succ_edge (entry_bb
);
7262 psi
= gsi_start_phis (fin_bb
);
7263 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
7264 gsi_next (&psi
), ++i
)
7267 source_location locus
;
7270 t
= gimple_phi_result (phi
);
7271 gcc_assert (t
== redirect_edge_var_map_result (vm
));
7272 nphi
= create_phi_node (t
, iter_part_bb
);
7274 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
7275 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
7277 /* A special case -- fd->loop.v is not yet computed in
7278 iter_part_bb, we need to use vextra instead. */
7279 if (t
== fd
->loop
.v
)
7281 add_phi_arg (nphi
, t
, ene
, locus
);
7282 locus
= redirect_edge_var_map_location (vm
);
7283 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
7285 gcc_assert (gsi_end_p (psi
) && i
== head
->length ());
7286 redirect_edge_var_map_clear (re
);
7289 psi
= gsi_start_phis (fin_bb
);
7290 if (gsi_end_p (psi
))
7292 remove_phi_node (&psi
, false);
7295 /* Make phi node for trip. */
7296 phi
= create_phi_node (trip_main
, iter_part_bb
);
7297 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
7299 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
7304 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
7305 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
7306 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
7307 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
7308 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
7309 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
7310 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
7311 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
7312 recompute_dominator (CDI_DOMINATORS
, body_bb
));
7316 struct loop
*trip_loop
= alloc_loop ();
7317 trip_loop
->header
= iter_part_bb
;
7318 trip_loop
->latch
= trip_update_bb
;
7319 add_loop (trip_loop
, iter_part_bb
->loop_father
);
7321 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
7323 struct loop
*loop
= alloc_loop ();
7324 loop
->header
= body_bb
;
7325 if (collapse_bb
== NULL
)
7326 loop
->latch
= cont_bb
;
7327 add_loop (loop
, trip_loop
);
7332 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
7334 for (V = N1; V cond N2; V += STEP) BODY;
7336 where COND is "<" or ">" or "!=", we generate pseudocode
7338 for (ind_var = low; ind_var < high; ind_var++)
7340 V = n1 + (ind_var * STEP)
7345 In the above pseudocode, low and high are function parameters of the
7346 child function. In the function below, we are inserting a temp.
7347 variable that will be making a call to two OMP functions that will not be
7348 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
7349 with _Cilk_for). These functions are replaced with low and high
7350 by the function that handles taskreg. */
7354 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
7356 bool broken_loop
= region
->cont
== NULL
;
7357 basic_block entry_bb
= region
->entry
;
7358 basic_block cont_bb
= region
->cont
;
7360 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7361 gcc_assert (broken_loop
7362 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7363 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7364 basic_block l1_bb
, l2_bb
;
7368 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7369 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7370 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7371 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7375 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7376 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7377 l2_bb
= single_succ (l1_bb
);
7379 basic_block exit_bb
= region
->exit
;
7380 basic_block l2_dom_bb
= NULL
;
7382 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
7384 /* Below statements until the "tree high_val = ..." are pseudo statements
7385 used to pass information to be used by expand_omp_taskreg.
7386 low_val and high_val will be replaced by the __low and __high
7387 parameter from the child function.
7389 The call_exprs part is a place-holder, it is mainly used
7390 to distinctly identify to the top-level part that this is
7391 where we should put low and high (reasoning given in header
7395 = gimple_omp_parallel_child_fn (
7396 as_a
<gomp_parallel
*> (last_stmt (region
->outer
->entry
)));
7397 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
7398 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
7400 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
7402 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
7405 gcc_assert (low_val
&& high_val
);
7407 tree type
= TREE_TYPE (low_val
);
7408 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
7409 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7411 /* Not needed in SSA form right now. */
7412 gcc_assert (!gimple_in_ssa_p (cfun
));
7413 if (l2_dom_bb
== NULL
)
7419 gimple stmt
= gimple_build_assign (ind_var
, n1
);
7421 /* Replace the GIMPLE_OMP_FOR statement. */
7422 gsi_replace (&gsi
, stmt
, true);
7426 /* Code to control the increment goes in the CONT_BB. */
7427 gsi
= gsi_last_bb (cont_bb
);
7428 stmt
= gsi_stmt (gsi
);
7429 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7430 stmt
= gimple_build_assign (ind_var
, PLUS_EXPR
, ind_var
,
7431 build_one_cst (type
));
7433 /* Replace GIMPLE_OMP_CONTINUE. */
7434 gsi_replace (&gsi
, stmt
, true);
7437 /* Emit the condition in L1_BB. */
7438 gsi
= gsi_after_labels (l1_bb
);
7439 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
7440 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
7442 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
7443 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7444 fd
->loop
.n1
, fold_convert (sizetype
, t
));
7446 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
7447 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
7448 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
7449 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7451 /* The condition is always '<' since the runtime will fill in the low
7453 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
7454 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
7456 /* Remove GIMPLE_OMP_RETURN. */
7457 gsi
= gsi_last_bb (exit_bb
);
7458 gsi_remove (&gsi
, true);
7460 /* Connect the new blocks. */
7461 remove_edge (FALLTHRU_EDGE (entry_bb
));
7466 remove_edge (BRANCH_EDGE (entry_bb
));
7467 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7469 e
= BRANCH_EDGE (l1_bb
);
7470 ne
= FALLTHRU_EDGE (l1_bb
);
7471 e
->flags
= EDGE_TRUE_VALUE
;
7475 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7477 ne
= single_succ_edge (l1_bb
);
7478 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7481 ne
->flags
= EDGE_FALSE_VALUE
;
7482 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7483 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7485 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7486 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7487 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7491 struct loop
*loop
= alloc_loop ();
7492 loop
->header
= l1_bb
;
7493 loop
->latch
= cont_bb
;
7494 add_loop (loop
, l1_bb
->loop_father
);
7495 loop
->safelen
= INT_MAX
;
7498 /* Pick the correct library function based on the precision of the
7499 induction variable type. */
7500 tree lib_fun
= NULL_TREE
;
7501 if (TYPE_PRECISION (type
) == 32)
7502 lib_fun
= cilk_for_32_fndecl
;
7503 else if (TYPE_PRECISION (type
) == 64)
7504 lib_fun
= cilk_for_64_fndecl
;
7508 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
7510 /* WS_ARGS contains the library function flavor to call:
7511 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
7512 user-defined grain value. If the user does not define one, then zero
7513 is passed in by the parser. */
7514 vec_alloc (region
->ws_args
, 2);
7515 region
->ws_args
->quick_push (lib_fun
);
7516 region
->ws_args
->quick_push (fd
->chunk_size
);
7519 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
7520 loop. Given parameters:
7522 for (V = N1; V cond N2; V += STEP) BODY;
7524 where COND is "<" or ">", we generate pseudocode
7532 if (V cond N2) goto L0; else goto L2;
7535 For collapsed loops, given parameters:
7537 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7538 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7539 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7542 we generate pseudocode
7548 count3 = (adj + N32 - N31) / STEP3;
7553 count2 = (adj + N22 - N21) / STEP2;
7558 count1 = (adj + N12 - N11) / STEP1;
7559 count = count1 * count2 * count3;
7569 V2 += (V3 cond3 N32) ? 0 : STEP2;
7570 V3 = (V3 cond3 N32) ? V3 : N31;
7571 V1 += (V2 cond2 N22) ? 0 : STEP1;
7572 V2 = (V2 cond2 N22) ? V2 : N21;
7574 if (V < count) goto L0; else goto L2;
7580 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
7583 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
7584 gimple_stmt_iterator gsi
;
7587 bool broken_loop
= region
->cont
== NULL
;
7589 tree
*counts
= NULL
;
7591 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7592 OMP_CLAUSE_SAFELEN
);
7593 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7594 OMP_CLAUSE__SIMDUID_
);
7597 type
= TREE_TYPE (fd
->loop
.v
);
7598 entry_bb
= region
->entry
;
7599 cont_bb
= region
->cont
;
7600 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7601 gcc_assert (broken_loop
7602 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7603 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7606 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7607 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7608 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7609 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7613 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7614 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7615 l2_bb
= single_succ (l1_bb
);
7617 exit_bb
= region
->exit
;
7620 gsi
= gsi_last_bb (entry_bb
);
7622 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7623 /* Not needed in SSA form right now. */
7624 gcc_assert (!gimple_in_ssa_p (cfun
));
7625 if (fd
->collapse
> 1)
7627 int first_zero_iter
= -1;
7628 basic_block zero_iter_bb
= l2_bb
;
7630 counts
= XALLOCAVEC (tree
, fd
->collapse
);
7631 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
7632 zero_iter_bb
, first_zero_iter
,
7635 if (l2_dom_bb
== NULL
)
7640 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7642 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7643 OMP_CLAUSE__LOOPTEMP_
);
7644 gcc_assert (innerc
);
7645 n1
= OMP_CLAUSE_DECL (innerc
);
7646 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7647 OMP_CLAUSE__LOOPTEMP_
);
7648 gcc_assert (innerc
);
7649 n2
= OMP_CLAUSE_DECL (innerc
);
7650 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7651 fold_convert (type
, n1
));
7652 if (fd
->collapse
> 1)
7655 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
7661 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7662 fold_convert (type
, fd
->loop
.n1
));
7663 if (fd
->collapse
> 1)
7664 for (i
= 0; i
< fd
->collapse
; i
++)
7666 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7667 if (POINTER_TYPE_P (itype
))
7668 itype
= signed_type_for (itype
);
7669 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
7670 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7674 /* Remove the GIMPLE_OMP_FOR statement. */
7675 gsi_remove (&gsi
, true);
7679 /* Code to control the increment goes in the CONT_BB. */
7680 gsi
= gsi_last_bb (cont_bb
);
7681 stmt
= gsi_stmt (gsi
);
7682 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7684 if (POINTER_TYPE_P (type
))
7685 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
7687 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
7688 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7690 if (fd
->collapse
> 1)
7692 i
= fd
->collapse
- 1;
7693 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7695 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
7696 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
7700 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
7702 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7705 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7707 for (i
= fd
->collapse
- 1; i
> 0; i
--)
7709 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7710 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
7711 if (POINTER_TYPE_P (itype2
))
7712 itype2
= signed_type_for (itype2
);
7713 t
= build3 (COND_EXPR
, itype2
,
7714 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7716 fold_convert (itype
, fd
->loops
[i
].n2
)),
7717 build_int_cst (itype2
, 0),
7718 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
7719 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
7720 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
7722 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
7723 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
7725 t
= build3 (COND_EXPR
, itype
,
7726 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7728 fold_convert (itype
, fd
->loops
[i
].n2
)),
7730 fold_convert (itype
, fd
->loops
[i
].n1
));
7731 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7735 /* Remove GIMPLE_OMP_CONTINUE. */
7736 gsi_remove (&gsi
, true);
7739 /* Emit the condition in L1_BB. */
7740 gsi
= gsi_start_bb (l1_bb
);
7742 t
= fold_convert (type
, n2
);
7743 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7744 false, GSI_CONTINUE_LINKING
);
7745 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
7746 cond_stmt
= gimple_build_cond_empty (t
);
7747 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
7748 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
7750 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
), expand_omp_regimplify_p
,
7753 gsi
= gsi_for_stmt (cond_stmt
);
7754 gimple_regimplify_operands (cond_stmt
, &gsi
);
7757 /* Remove GIMPLE_OMP_RETURN. */
7758 gsi
= gsi_last_bb (exit_bb
);
7759 gsi_remove (&gsi
, true);
7761 /* Connect the new blocks. */
7762 remove_edge (FALLTHRU_EDGE (entry_bb
));
7766 remove_edge (BRANCH_EDGE (entry_bb
));
7767 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7769 e
= BRANCH_EDGE (l1_bb
);
7770 ne
= FALLTHRU_EDGE (l1_bb
);
7771 e
->flags
= EDGE_TRUE_VALUE
;
7775 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7777 ne
= single_succ_edge (l1_bb
);
7778 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7781 ne
->flags
= EDGE_FALSE_VALUE
;
7782 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7783 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7785 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7786 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7787 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7791 struct loop
*loop
= alloc_loop ();
7792 loop
->header
= l1_bb
;
7793 loop
->latch
= cont_bb
;
7794 add_loop (loop
, l1_bb
->loop_father
);
7795 if (safelen
== NULL_TREE
)
7796 loop
->safelen
= INT_MAX
;
7799 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
7800 if (TREE_CODE (safelen
) != INTEGER_CST
)
7802 else if (!tree_fits_uhwi_p (safelen
)
7803 || tree_to_uhwi (safelen
) > INT_MAX
)
7804 loop
->safelen
= INT_MAX
;
7806 loop
->safelen
= tree_to_uhwi (safelen
);
7807 if (loop
->safelen
== 1)
7812 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
7813 cfun
->has_simduid_loops
= true;
7815 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7817 if ((flag_tree_loop_vectorize
7818 || (!global_options_set
.x_flag_tree_loop_vectorize
7819 && !global_options_set
.x_flag_tree_vectorize
))
7820 && flag_tree_loop_optimize
7821 && loop
->safelen
> 1)
7823 loop
->force_vectorize
= true;
7824 cfun
->has_force_vectorize_loops
= true;
7828 cfun
->has_simduid_loops
= true;
7832 /* Expand the OMP loop defined by REGION. */
7835 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
7837 struct omp_for_data fd
;
7838 struct omp_for_data_loop
*loops
;
7841 = (struct omp_for_data_loop
*)
7842 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
7843 * sizeof (struct omp_for_data_loop
));
7844 extract_omp_for_data (as_a
<gomp_for
*> (last_stmt (region
->entry
)),
7846 region
->sched_kind
= fd
.sched_kind
;
7848 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
7849 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7850 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7853 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
7854 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7855 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7858 /* If there isn't a continue then this is a degerate case where
7859 the introduction of abnormal edges during lowering will prevent
7860 original loops from being detected. Fix that up. */
7861 loops_state_set (LOOPS_NEED_FIXUP
);
7863 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
7864 expand_omp_simd (region
, &fd
);
7865 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
7866 expand_cilk_for (region
, &fd
);
7867 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
7868 && !fd
.have_ordered
)
7870 if (fd
.chunk_size
== NULL
)
7871 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
7873 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
7877 int fn_index
, start_ix
, next_ix
;
7879 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
7880 == GF_OMP_FOR_KIND_FOR
);
7881 if (fd
.chunk_size
== NULL
7882 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
7883 fd
.chunk_size
= integer_zero_node
;
7884 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
7885 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
7886 ? 3 : fd
.sched_kind
;
7887 fn_index
+= fd
.have_ordered
* 4;
7888 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
7889 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
7890 if (fd
.iter_type
== long_long_unsigned_type_node
)
7892 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7893 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
7894 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7895 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
7897 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
7898 (enum built_in_function
) next_ix
, inner_stmt
);
7901 if (gimple_in_ssa_p (cfun
))
7902 update_ssa (TODO_update_ssa_only_virtuals
);
7906 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7908 v = GOMP_sections_start (n);
7925 v = GOMP_sections_next ();
7930 If this is a combined parallel sections, replace the call to
7931 GOMP_sections_start with call to GOMP_sections_next. */
7934 expand_omp_sections (struct omp_region
*region
)
7936 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7938 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7939 gimple_stmt_iterator si
, switch_si
;
7940 gomp_sections
*sections_stmt
;
7942 gomp_continue
*cont
;
7945 struct omp_region
*inner
;
7947 bool exit_reachable
= region
->cont
!= NULL
;
7949 gcc_assert (region
->exit
!= NULL
);
7950 entry_bb
= region
->entry
;
7951 l0_bb
= single_succ (entry_bb
);
7952 l1_bb
= region
->cont
;
7953 l2_bb
= region
->exit
;
7954 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7955 l2
= gimple_block_label (l2_bb
);
7958 /* This can happen if there are reductions. */
7959 len
= EDGE_COUNT (l0_bb
->succs
);
7960 gcc_assert (len
> 0);
7961 e
= EDGE_SUCC (l0_bb
, len
- 1);
7962 si
= gsi_last_bb (e
->dest
);
7965 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7966 l2
= gimple_block_label (e
->dest
);
7968 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7970 si
= gsi_last_bb (e
->dest
);
7972 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7974 l2
= gimple_block_label (e
->dest
);
7980 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7982 default_bb
= create_empty_bb (l0_bb
);
7984 /* We will build a switch() with enough cases for all the
7985 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7986 and a default case to abort if something goes wrong. */
7987 len
= EDGE_COUNT (l0_bb
->succs
);
7989 /* Use vec::quick_push on label_vec throughout, since we know the size
7991 auto_vec
<tree
> label_vec (len
);
7993 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7994 GIMPLE_OMP_SECTIONS statement. */
7995 si
= gsi_last_bb (entry_bb
);
7996 sections_stmt
= as_a
<gomp_sections
*> (gsi_stmt (si
));
7997 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
7998 vin
= gimple_omp_sections_control (sections_stmt
);
7999 if (!is_combined_parallel (region
))
8001 /* If we are not inside a combined parallel+sections region,
8002 call GOMP_sections_start. */
8003 t
= build_int_cst (unsigned_type_node
, len
- 1);
8004 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
8005 stmt
= gimple_build_call (u
, 1, t
);
8009 /* Otherwise, call GOMP_sections_next. */
8010 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
8011 stmt
= gimple_build_call (u
, 0);
8013 gimple_call_set_lhs (stmt
, vin
);
8014 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8015 gsi_remove (&si
, true);
8017 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
8019 switch_si
= gsi_last_bb (l0_bb
);
8020 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
8023 cont
= as_a
<gomp_continue
*> (last_stmt (l1_bb
));
8024 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
8025 vmain
= gimple_omp_continue_control_use (cont
);
8026 vnext
= gimple_omp_continue_control_def (cont
);
8034 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
8035 label_vec
.quick_push (t
);
8038 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
8039 for (inner
= region
->inner
, casei
= 1;
8041 inner
= inner
->next
, i
++, casei
++)
8043 basic_block s_entry_bb
, s_exit_bb
;
8045 /* Skip optional reduction region. */
8046 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
8053 s_entry_bb
= inner
->entry
;
8054 s_exit_bb
= inner
->exit
;
8056 t
= gimple_block_label (s_entry_bb
);
8057 u
= build_int_cst (unsigned_type_node
, casei
);
8058 u
= build_case_label (u
, NULL
, t
);
8059 label_vec
.quick_push (u
);
8061 si
= gsi_last_bb (s_entry_bb
);
8062 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
8063 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
8064 gsi_remove (&si
, true);
8065 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
8067 if (s_exit_bb
== NULL
)
8070 si
= gsi_last_bb (s_exit_bb
);
8071 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8072 gsi_remove (&si
, true);
8074 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
8077 /* Error handling code goes in DEFAULT_BB. */
8078 t
= gimple_block_label (default_bb
);
8079 u
= build_case_label (NULL
, NULL
, t
);
8080 make_edge (l0_bb
, default_bb
, 0);
8081 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
8083 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
8084 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
8085 gsi_remove (&switch_si
, true);
8087 si
= gsi_start_bb (default_bb
);
8088 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
8089 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
8095 /* Code to get the next section goes in L1_BB. */
8096 si
= gsi_last_bb (l1_bb
);
8097 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
8099 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
8100 stmt
= gimple_build_call (bfn_decl
, 0);
8101 gimple_call_set_lhs (stmt
, vnext
);
8102 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8103 gsi_remove (&si
, true);
8105 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
8108 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
8109 si
= gsi_last_bb (l2_bb
);
8110 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
8111 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
8112 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
8113 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
8115 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
8116 stmt
= gimple_build_call (t
, 0);
8117 if (gimple_omp_return_lhs (gsi_stmt (si
)))
8118 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
8119 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
8120 gsi_remove (&si
, true);
8122 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
8126 /* Expand code for an OpenMP single directive. We've already expanded
8127 much of the code, here we simply place the GOMP_barrier call. */
8130 expand_omp_single (struct omp_region
*region
)
8132 basic_block entry_bb
, exit_bb
;
8133 gimple_stmt_iterator si
;
8135 entry_bb
= region
->entry
;
8136 exit_bb
= region
->exit
;
8138 si
= gsi_last_bb (entry_bb
);
8139 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
8140 gsi_remove (&si
, true);
8141 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8143 si
= gsi_last_bb (exit_bb
);
8144 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
8146 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
8147 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
8149 gsi_remove (&si
, true);
8150 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8154 /* Generic expansion for OpenMP synchronization directives: master,
8155 ordered and critical. All we need to do here is remove the entry
8156 and exit markers for REGION. */
8159 expand_omp_synch (struct omp_region
*region
)
8161 basic_block entry_bb
, exit_bb
;
8162 gimple_stmt_iterator si
;
8164 entry_bb
= region
->entry
;
8165 exit_bb
= region
->exit
;
8167 si
= gsi_last_bb (entry_bb
);
8168 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
8169 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
8170 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
8171 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
8172 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
8173 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
8174 gsi_remove (&si
, true);
8175 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8179 si
= gsi_last_bb (exit_bb
);
8180 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
8181 gsi_remove (&si
, true);
8182 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
8186 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8187 operation as a normal volatile load. */
8190 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
8191 tree loaded_val
, int index
)
8193 enum built_in_function tmpbase
;
8194 gimple_stmt_iterator gsi
;
8195 basic_block store_bb
;
8198 tree decl
, call
, type
, itype
;
8200 gsi
= gsi_last_bb (load_bb
);
8201 stmt
= gsi_stmt (gsi
);
8202 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8203 loc
= gimple_location (stmt
);
8205 /* ??? If the target does not implement atomic_load_optab[mode], and mode
8206 is smaller than word size, then expand_atomic_load assumes that the load
8207 is atomic. We could avoid the builtin entirely in this case. */
8209 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8210 decl
= builtin_decl_explicit (tmpbase
);
8211 if (decl
== NULL_TREE
)
8214 type
= TREE_TYPE (loaded_val
);
8215 itype
= TREE_TYPE (TREE_TYPE (decl
));
8217 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
8218 build_int_cst (NULL
,
8219 gimple_omp_atomic_seq_cst_p (stmt
)
8221 : MEMMODEL_RELAXED
));
8222 if (!useless_type_conversion_p (type
, itype
))
8223 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8224 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8226 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8227 gsi_remove (&gsi
, true);
8229 store_bb
= single_succ (load_bb
);
8230 gsi
= gsi_last_bb (store_bb
);
8231 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8232 gsi_remove (&gsi
, true);
8234 if (gimple_in_ssa_p (cfun
))
8235 update_ssa (TODO_update_ssa_no_phi
);
8240 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8241 operation as a normal volatile store. */
8244 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
8245 tree loaded_val
, tree stored_val
, int index
)
8247 enum built_in_function tmpbase
;
8248 gimple_stmt_iterator gsi
;
8249 basic_block store_bb
= single_succ (load_bb
);
8252 tree decl
, call
, type
, itype
;
8256 gsi
= gsi_last_bb (load_bb
);
8257 stmt
= gsi_stmt (gsi
);
8258 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
8260 /* If the load value is needed, then this isn't a store but an exchange. */
8261 exchange
= gimple_omp_atomic_need_value_p (stmt
);
8263 gsi
= gsi_last_bb (store_bb
);
8264 stmt
= gsi_stmt (gsi
);
8265 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
8266 loc
= gimple_location (stmt
);
8268 /* ??? If the target does not implement atomic_store_optab[mode], and mode
8269 is smaller than word size, then expand_atomic_store assumes that the store
8270 is atomic. We could avoid the builtin entirely in this case. */
8272 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
8273 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
8274 decl
= builtin_decl_explicit (tmpbase
);
8275 if (decl
== NULL_TREE
)
8278 type
= TREE_TYPE (stored_val
);
8280 /* Dig out the type of the function's second argument. */
8281 itype
= TREE_TYPE (decl
);
8282 itype
= TYPE_ARG_TYPES (itype
);
8283 itype
= TREE_CHAIN (itype
);
8284 itype
= TREE_VALUE (itype
);
8285 imode
= TYPE_MODE (itype
);
8287 if (exchange
&& !can_atomic_exchange_p (imode
, true))
8290 if (!useless_type_conversion_p (itype
, type
))
8291 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
8292 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
8293 build_int_cst (NULL
,
8294 gimple_omp_atomic_seq_cst_p (stmt
)
8296 : MEMMODEL_RELAXED
));
8299 if (!useless_type_conversion_p (type
, itype
))
8300 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
8301 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
8304 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8305 gsi_remove (&gsi
, true);
8307 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
8308 gsi
= gsi_last_bb (load_bb
);
8309 gsi_remove (&gsi
, true);
8311 if (gimple_in_ssa_p (cfun
))
8312 update_ssa (TODO_update_ssa_no_phi
);
8317 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
8318 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
8319 size of the data type, and thus usable to find the index of the builtin
8320 decl. Returns false if the expression is not of the proper form. */
8323 expand_omp_atomic_fetch_op (basic_block load_bb
,
8324 tree addr
, tree loaded_val
,
8325 tree stored_val
, int index
)
8327 enum built_in_function oldbase
, newbase
, tmpbase
;
8328 tree decl
, itype
, call
;
8330 basic_block store_bb
= single_succ (load_bb
);
8331 gimple_stmt_iterator gsi
;
8334 enum tree_code code
;
8335 bool need_old
, need_new
;
8339 /* We expect to find the following sequences:
8342 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
8345 val = tmp OP something; (or: something OP tmp)
8346 GIMPLE_OMP_STORE (val)
8348 ???FIXME: Allow a more flexible sequence.
8349 Perhaps use data flow to pick the statements.
8353 gsi
= gsi_after_labels (store_bb
);
8354 stmt
= gsi_stmt (gsi
);
8355 loc
= gimple_location (stmt
);
8356 if (!is_gimple_assign (stmt
))
8359 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
8361 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
8362 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
8363 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
8364 gcc_checking_assert (!need_old
|| !need_new
);
8366 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
8369 /* Check for one of the supported fetch-op operations. */
8370 code
= gimple_assign_rhs_code (stmt
);
8374 case POINTER_PLUS_EXPR
:
8375 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
8376 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
8379 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
8380 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
8383 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
8384 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
8387 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
8388 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
8391 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
8392 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
8398 /* Make sure the expression is of the proper form. */
8399 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
8400 rhs
= gimple_assign_rhs2 (stmt
);
8401 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
8402 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
8403 rhs
= gimple_assign_rhs1 (stmt
);
8407 tmpbase
= ((enum built_in_function
)
8408 ((need_new
? newbase
: oldbase
) + index
+ 1));
8409 decl
= builtin_decl_explicit (tmpbase
);
8410 if (decl
== NULL_TREE
)
8412 itype
= TREE_TYPE (TREE_TYPE (decl
));
8413 imode
= TYPE_MODE (itype
);
8415 /* We could test all of the various optabs involved, but the fact of the
8416 matter is that (with the exception of i486 vs i586 and xadd) all targets
8417 that support any atomic operaton optab also implements compare-and-swap.
8418 Let optabs.c take care of expanding any compare-and-swap loop. */
8419 if (!can_compare_and_swap_p (imode
, true))
8422 gsi
= gsi_last_bb (load_bb
);
8423 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8425 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
8426 It only requires that the operation happen atomically. Thus we can
8427 use the RELAXED memory model. */
8428 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
8429 fold_convert_loc (loc
, itype
, rhs
),
8430 build_int_cst (NULL
,
8431 seq_cst
? MEMMODEL_SEQ_CST
8432 : MEMMODEL_RELAXED
));
8434 if (need_old
|| need_new
)
8436 lhs
= need_old
? loaded_val
: stored_val
;
8437 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
8438 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
8441 call
= fold_convert_loc (loc
, void_type_node
, call
);
8442 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8443 gsi_remove (&gsi
, true);
8445 gsi
= gsi_last_bb (store_bb
);
8446 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
8447 gsi_remove (&gsi
, true);
8448 gsi
= gsi_last_bb (store_bb
);
8449 gsi_remove (&gsi
, true);
8451 if (gimple_in_ssa_p (cfun
))
8452 update_ssa (TODO_update_ssa_no_phi
);
8457 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8461 newval = rhs; // with oldval replacing *addr in rhs
8462 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
8463 if (oldval != newval)
8466 INDEX is log2 of the size of the data type, and thus usable to find the
8467 index of the builtin decl. */
8470 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
8471 tree addr
, tree loaded_val
, tree stored_val
,
8474 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
8475 tree type
, itype
, cmpxchg
, iaddr
;
8476 gimple_stmt_iterator si
;
8477 basic_block loop_header
= single_succ (load_bb
);
8480 enum built_in_function fncode
;
8482 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
8483 order to use the RELAXED memory model effectively. */
8484 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
8486 cmpxchg
= builtin_decl_explicit (fncode
);
8487 if (cmpxchg
== NULL_TREE
)
8489 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8490 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
8492 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
8495 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
8496 si
= gsi_last_bb (load_bb
);
8497 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8499 /* For floating-point values, we'll need to view-convert them to integers
8500 so that we can perform the atomic compare and swap. Simplify the
8501 following code by always setting up the "i"ntegral variables. */
8502 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
8506 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
8509 = force_gimple_operand_gsi (&si
,
8510 fold_convert (TREE_TYPE (iaddr
), addr
),
8511 false, NULL_TREE
, true, GSI_SAME_STMT
);
8512 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
8513 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8514 loadedi
= create_tmp_var (itype
);
8515 if (gimple_in_ssa_p (cfun
))
8516 loadedi
= make_ssa_name (loadedi
);
8521 loadedi
= loaded_val
;
8524 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8525 tree loaddecl
= builtin_decl_explicit (fncode
);
8528 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
8529 build_call_expr (loaddecl
, 2, iaddr
,
8530 build_int_cst (NULL_TREE
,
8531 MEMMODEL_RELAXED
)));
8533 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
8534 build_int_cst (TREE_TYPE (iaddr
), 0));
8537 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
8540 /* Move the value to the LOADEDI temporary. */
8541 if (gimple_in_ssa_p (cfun
))
8543 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
8544 phi
= create_phi_node (loadedi
, loop_header
);
8545 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
8549 gsi_insert_before (&si
,
8550 gimple_build_assign (loadedi
, initial
),
8552 if (loadedi
!= loaded_val
)
8554 gimple_stmt_iterator gsi2
;
8557 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
8558 gsi2
= gsi_start_bb (loop_header
);
8559 if (gimple_in_ssa_p (cfun
))
8562 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8563 true, GSI_SAME_STMT
);
8564 stmt
= gimple_build_assign (loaded_val
, x
);
8565 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
8569 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
8570 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8571 true, GSI_SAME_STMT
);
8574 gsi_remove (&si
, true);
8576 si
= gsi_last_bb (store_bb
);
8577 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8580 storedi
= stored_val
;
8583 force_gimple_operand_gsi (&si
,
8584 build1 (VIEW_CONVERT_EXPR
, itype
,
8585 stored_val
), true, NULL_TREE
, true,
8588 /* Build the compare&swap statement. */
8589 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
8590 new_storedi
= force_gimple_operand_gsi (&si
,
8591 fold_convert (TREE_TYPE (loadedi
),
8594 true, GSI_SAME_STMT
);
8596 if (gimple_in_ssa_p (cfun
))
8600 old_vali
= create_tmp_var (TREE_TYPE (loadedi
));
8601 stmt
= gimple_build_assign (old_vali
, loadedi
);
8602 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8604 stmt
= gimple_build_assign (loadedi
, new_storedi
);
8605 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8608 /* Note that we always perform the comparison as an integer, even for
8609 floating point. This allows the atomic operation to properly
8610 succeed even with NaNs and -0.0. */
8611 stmt
= gimple_build_cond_empty
8612 (build2 (NE_EXPR
, boolean_type_node
,
8613 new_storedi
, old_vali
));
8614 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8617 e
= single_succ_edge (store_bb
);
8618 e
->flags
&= ~EDGE_FALLTHRU
;
8619 e
->flags
|= EDGE_FALSE_VALUE
;
8621 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
8623 /* Copy the new value to loadedi (we already did that before the condition
8624 if we are not in SSA). */
8625 if (gimple_in_ssa_p (cfun
))
8627 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
8628 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
8631 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
8632 gsi_remove (&si
, true);
8634 struct loop
*loop
= alloc_loop ();
8635 loop
->header
= loop_header
;
8636 loop
->latch
= store_bb
;
8637 add_loop (loop
, loop_header
->loop_father
);
8639 if (gimple_in_ssa_p (cfun
))
8640 update_ssa (TODO_update_ssa_no_phi
);
8645 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8647 GOMP_atomic_start ();
8651 The result is not globally atomic, but works so long as all parallel
8652 references are within #pragma omp atomic directives. According to
8653 responses received from omp@openmp.org, appears to be within spec.
8654 Which makes sense, since that's how several other compilers handle
8655 this situation as well.
8656 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
8657 expanding. STORED_VAL is the operand of the matching
8658 GIMPLE_OMP_ATOMIC_STORE.
8661 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
8665 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
8670 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
8671 tree addr
, tree loaded_val
, tree stored_val
)
8673 gimple_stmt_iterator si
;
8677 si
= gsi_last_bb (load_bb
);
8678 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8680 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
8681 t
= build_call_expr (t
, 0);
8682 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8684 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
8685 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8686 gsi_remove (&si
, true);
8688 si
= gsi_last_bb (store_bb
);
8689 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8691 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
8693 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8695 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
8696 t
= build_call_expr (t
, 0);
8697 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8698 gsi_remove (&si
, true);
8700 if (gimple_in_ssa_p (cfun
))
8701 update_ssa (TODO_update_ssa_no_phi
);
8705 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
8706 using expand_omp_atomic_fetch_op. If it failed, we try to
8707 call expand_omp_atomic_pipeline, and if it fails too, the
8708 ultimate fallback is wrapping the operation in a mutex
8709 (expand_omp_atomic_mutex). REGION is the atomic region built
8710 by build_omp_regions_1(). */
8713 expand_omp_atomic (struct omp_region
*region
)
8715 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
8716 gomp_atomic_load
*load
= as_a
<gomp_atomic_load
*> (last_stmt (load_bb
));
8717 gomp_atomic_store
*store
= as_a
<gomp_atomic_store
*> (last_stmt (store_bb
));
8718 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
8719 tree addr
= gimple_omp_atomic_load_rhs (load
);
8720 tree stored_val
= gimple_omp_atomic_store_val (store
);
8721 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8722 HOST_WIDE_INT index
;
8724 /* Make sure the type is one of the supported sizes. */
8725 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
8726 index
= exact_log2 (index
);
8727 if (index
>= 0 && index
<= 4)
8729 unsigned int align
= TYPE_ALIGN_UNIT (type
);
8731 /* __sync builtins require strict data alignment. */
8732 if (exact_log2 (align
) >= index
)
8735 if (loaded_val
== stored_val
8736 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8737 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8738 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8739 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
8743 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8744 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8745 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8746 && store_bb
== single_succ (load_bb
)
8747 && first_stmt (store_bb
) == store
8748 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
8752 /* When possible, use specialized atomic update functions. */
8753 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
8754 && store_bb
== single_succ (load_bb
)
8755 && expand_omp_atomic_fetch_op (load_bb
, addr
,
8756 loaded_val
, stored_val
, index
))
8759 /* If we don't have specialized __sync builtins, try and implement
8760 as a compare and swap loop. */
8761 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
8762 loaded_val
, stored_val
, index
))
8767 /* The ultimate fallback is wrapping the operation in a mutex. */
8768 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
8772 /* Expand the GIMPLE_OMP_TARGET starting at REGION. */
8775 expand_omp_target (struct omp_region
*region
)
8777 basic_block entry_bb
, exit_bb
, new_bb
;
8778 struct function
*child_cfun
;
8779 tree child_fn
, block
, t
;
8780 gimple_stmt_iterator gsi
;
8781 gomp_target
*entry_stmt
;
8784 bool offloaded
, data_region
;
8786 entry_stmt
= as_a
<gomp_target
*> (last_stmt (region
->entry
));
8787 new_bb
= region
->entry
;
8789 offloaded
= is_gimple_omp_offloaded (entry_stmt
);
8790 switch (gimple_omp_target_kind (entry_stmt
))
8792 case GF_OMP_TARGET_KIND_REGION
:
8793 case GF_OMP_TARGET_KIND_UPDATE
:
8794 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
8795 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
8796 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
8797 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
8798 data_region
= false;
8800 case GF_OMP_TARGET_KIND_DATA
:
8801 case GF_OMP_TARGET_KIND_OACC_DATA
:
8808 child_fn
= NULL_TREE
;
8812 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
8813 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
8816 /* Supported by expand_omp_taskreg, but not here. */
8817 if (child_cfun
!= NULL
)
8818 gcc_checking_assert (!child_cfun
->cfg
);
8819 gcc_checking_assert (!gimple_in_ssa_p (cfun
));
8821 entry_bb
= region
->entry
;
8822 exit_bb
= region
->exit
;
8826 unsigned srcidx
, dstidx
, num
;
8828 /* If the offloading region needs data sent from the parent
8829 function, then the very first statement (except possible
8830 tree profile counter updates) of the offloading body
8831 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
8832 &.OMP_DATA_O is passed as an argument to the child function,
8833 we need to replace it with the argument as seen by the child
8836 In most cases, this will end up being the identity assignment
8837 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had
8838 a function call that has been inlined, the original PARM_DECL
8839 .OMP_DATA_I may have been converted into a different local
8840 variable. In which case, we need to keep the assignment. */
8841 tree data_arg
= gimple_omp_target_data_arg (entry_stmt
);
8844 basic_block entry_succ_bb
= single_succ (entry_bb
);
8845 gimple_stmt_iterator gsi
;
8847 gimple tgtcopy_stmt
= NULL
;
8848 tree sender
= TREE_VEC_ELT (data_arg
, 0);
8850 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
8852 gcc_assert (!gsi_end_p (gsi
));
8853 stmt
= gsi_stmt (gsi
);
8854 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
8857 if (gimple_num_ops (stmt
) == 2)
8859 tree arg
= gimple_assign_rhs1 (stmt
);
8861 /* We're ignoring the subcode because we're
8862 effectively doing a STRIP_NOPS. */
8864 if (TREE_CODE (arg
) == ADDR_EXPR
8865 && TREE_OPERAND (arg
, 0) == sender
)
8867 tgtcopy_stmt
= stmt
;
8873 gcc_assert (tgtcopy_stmt
!= NULL
);
8874 arg
= DECL_ARGUMENTS (child_fn
);
8876 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
8877 gsi_remove (&gsi
, true);
8880 /* Declare local variables needed in CHILD_CFUN. */
8881 block
= DECL_INITIAL (child_fn
);
8882 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
8883 /* The gimplifier could record temporaries in the offloading block
8884 rather than in containing function's local_decls chain,
8885 which would mean cgraph missed finalizing them. Do it now. */
8886 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
8887 if (TREE_CODE (t
) == VAR_DECL
8889 && !DECL_EXTERNAL (t
))
8890 varpool_node::finalize_decl (t
);
8891 DECL_SAVED_TREE (child_fn
) = NULL
;
8892 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8893 gimple_set_body (child_fn
, NULL
);
8894 TREE_USED (block
) = 1;
8896 /* Reset DECL_CONTEXT on function arguments. */
8897 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
8898 DECL_CONTEXT (t
) = child_fn
;
8900 /* Split ENTRY_BB at GIMPLE_*,
8901 so that it can be moved to the child function. */
8902 gsi
= gsi_last_bb (entry_bb
);
8903 stmt
= gsi_stmt (gsi
);
8905 && gimple_code (stmt
) == gimple_code (entry_stmt
));
8906 e
= split_block (entry_bb
, stmt
);
8907 gsi_remove (&gsi
, true);
8909 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8911 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8914 gsi
= gsi_last_bb (exit_bb
);
8915 gcc_assert (!gsi_end_p (gsi
)
8916 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8917 stmt
= gimple_build_return (NULL
);
8918 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8919 gsi_remove (&gsi
, true);
8922 /* Move the offloading region into CHILD_CFUN. */
8924 block
= gimple_block (entry_stmt
);
8926 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
8928 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8929 /* When the OMP expansion process cannot guarantee an up-to-date
8930 loop tree arrange for the child function to fixup loops. */
8931 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8932 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8934 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8935 num
= vec_safe_length (child_cfun
->local_decls
);
8936 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8938 t
= (*child_cfun
->local_decls
)[srcidx
];
8939 if (DECL_CONTEXT (t
) == cfun
->decl
)
8941 if (srcidx
!= dstidx
)
8942 (*child_cfun
->local_decls
)[dstidx
] = t
;
8946 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8948 /* Inform the callgraph about the new function. */
8949 child_cfun
->curr_properties
= cfun
->curr_properties
;
8950 child_cfun
->has_simduid_loops
|= cfun
->has_simduid_loops
;
8951 child_cfun
->has_force_vectorize_loops
|= cfun
->has_force_vectorize_loops
;
8952 cgraph_node
*node
= cgraph_node::get_create (child_fn
);
8953 node
->parallelized_function
= 1;
8954 cgraph_node::add_new_function (child_fn
, true);
8956 #ifdef ENABLE_OFFLOADING
8957 /* Add the new function to the offload table. */
8958 vec_safe_push (offload_funcs
, child_fn
);
8961 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8962 fixed in a following pass. */
8963 push_cfun (child_cfun
);
8964 cgraph_edge::rebuild_edges ();
8966 #ifdef ENABLE_OFFLOADING
8967 /* Prevent IPA from removing child_fn as unreachable, since there are no
8968 refs from the parent function to child_fn in offload LTO mode. */
8969 cgraph_node::get (child_fn
)->mark_force_output ();
8972 /* Some EH regions might become dead, see PR34608. If
8973 pass_cleanup_cfg isn't the first pass to happen with the
8974 new child, these dead EH edges might cause problems.
8975 Clean them up now. */
8976 if (flag_exceptions
)
8979 bool changed
= false;
8981 FOR_EACH_BB_FN (bb
, cfun
)
8982 changed
|= gimple_purge_dead_eh_edges (bb
);
8984 cleanup_tree_cfg ();
8989 /* Emit a library call to launch the offloading region, or do data
8991 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8992 enum built_in_function start_ix
;
8993 location_t clause_loc
;
8995 switch (gimple_omp_target_kind (entry_stmt
))
8997 case GF_OMP_TARGET_KIND_REGION
:
8998 start_ix
= BUILT_IN_GOMP_TARGET
;
9000 case GF_OMP_TARGET_KIND_DATA
:
9001 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
9003 case GF_OMP_TARGET_KIND_UPDATE
:
9004 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
9006 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
9007 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
9008 start_ix
= BUILT_IN_GOACC_PARALLEL
;
9010 case GF_OMP_TARGET_KIND_OACC_DATA
:
9011 start_ix
= BUILT_IN_GOACC_DATA_START
;
9013 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
9014 start_ix
= BUILT_IN_GOACC_UPDATE
;
9016 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9017 start_ix
= BUILT_IN_GOACC_ENTER_EXIT_DATA
;
9023 clauses
= gimple_omp_target_clauses (entry_stmt
);
9025 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime
9026 library choose) and there is no conditional. */
9028 device
= build_int_cst (integer_type_node
, GOMP_DEVICE_ICV
);
9030 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
9032 cond
= OMP_CLAUSE_IF_EXPR (c
);
9034 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
9037 /* Even if we pass it to all library function calls, it is currently only
9038 defined/used for the OpenMP target ones. */
9039 gcc_checking_assert (start_ix
== BUILT_IN_GOMP_TARGET
9040 || start_ix
== BUILT_IN_GOMP_TARGET_DATA
9041 || start_ix
== BUILT_IN_GOMP_TARGET_UPDATE
);
9043 device
= OMP_CLAUSE_DEVICE_ID (c
);
9044 clause_loc
= OMP_CLAUSE_LOCATION (c
);
9047 clause_loc
= gimple_location (entry_stmt
);
9049 /* Ensure 'device' is of the correct type. */
9050 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
9052 /* If we found the clause 'if (cond)', build
9053 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */
9056 cond
= gimple_boolify (cond
);
9058 basic_block cond_bb
, then_bb
, else_bb
;
9062 tmp_var
= create_tmp_var (TREE_TYPE (device
));
9064 e
= split_block_after_labels (new_bb
);
9067 gsi
= gsi_last_bb (new_bb
);
9069 e
= split_block (new_bb
, gsi_stmt (gsi
));
9075 then_bb
= create_empty_bb (cond_bb
);
9076 else_bb
= create_empty_bb (then_bb
);
9077 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
9078 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
9080 stmt
= gimple_build_cond_empty (cond
);
9081 gsi
= gsi_last_bb (cond_bb
);
9082 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9084 gsi
= gsi_start_bb (then_bb
);
9085 stmt
= gimple_build_assign (tmp_var
, device
);
9086 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9088 gsi
= gsi_start_bb (else_bb
);
9089 stmt
= gimple_build_assign (tmp_var
,
9090 build_int_cst (integer_type_node
,
9091 GOMP_DEVICE_HOST_FALLBACK
));
9092 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
9094 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
9095 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
9096 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
9097 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
9098 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
9099 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
9104 gsi
= gsi_last_bb (new_bb
);
9105 t
= gimple_omp_target_data_arg (entry_stmt
);
9108 t1
= size_zero_node
;
9109 t2
= build_zero_cst (ptr_type_node
);
9115 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
9116 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
9117 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
9118 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
9119 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
9123 /* The maximum number used by any start_ix, without varargs. */
9124 auto_vec
<tree
, 11> args
;
9125 args
.quick_push (device
);
9127 args
.quick_push (build_fold_addr_expr (child_fn
));
9130 case BUILT_IN_GOMP_TARGET
:
9131 case BUILT_IN_GOMP_TARGET_DATA
:
9132 case BUILT_IN_GOMP_TARGET_UPDATE
:
9133 /* This const void * is part of the current ABI, but we're not actually
9135 args
.quick_push (build_zero_cst (ptr_type_node
));
9137 case BUILT_IN_GOACC_DATA_START
:
9138 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
9139 case BUILT_IN_GOACC_PARALLEL
:
9140 case BUILT_IN_GOACC_UPDATE
:
9145 args
.quick_push (t1
);
9146 args
.quick_push (t2
);
9147 args
.quick_push (t3
);
9148 args
.quick_push (t4
);
9151 case BUILT_IN_GOACC_DATA_START
:
9152 case BUILT_IN_GOMP_TARGET
:
9153 case BUILT_IN_GOMP_TARGET_DATA
:
9154 case BUILT_IN_GOMP_TARGET_UPDATE
:
9156 case BUILT_IN_GOACC_PARALLEL
:
9158 tree t_num_gangs
, t_num_workers
, t_vector_length
;
9160 /* Default values for num_gangs, num_workers, and vector_length. */
9161 t_num_gangs
= t_num_workers
= t_vector_length
9162 = fold_convert_loc (gimple_location (entry_stmt
),
9163 integer_type_node
, integer_one_node
);
9164 /* ..., but if present, use the value specified by the respective
9165 clause, making sure that are of the correct type. */
9166 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_GANGS
);
9168 t_num_gangs
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9170 OMP_CLAUSE_NUM_GANGS_EXPR (c
));
9171 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_WORKERS
);
9173 t_num_workers
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9175 OMP_CLAUSE_NUM_WORKERS_EXPR (c
));
9176 c
= find_omp_clause (clauses
, OMP_CLAUSE_VECTOR_LENGTH
);
9178 t_vector_length
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9180 OMP_CLAUSE_VECTOR_LENGTH_EXPR (c
));
9181 args
.quick_push (t_num_gangs
);
9182 args
.quick_push (t_num_workers
);
9183 args
.quick_push (t_vector_length
);
9186 case BUILT_IN_GOACC_ENTER_EXIT_DATA
:
9187 case BUILT_IN_GOACC_UPDATE
:
9192 /* Default values for t_async. */
9193 t_async
= fold_convert_loc (gimple_location (entry_stmt
),
9195 build_int_cst (integer_type_node
,
9197 /* ..., but if present, use the value specified by the respective
9198 clause, making sure that is of the correct type. */
9199 c
= find_omp_clause (clauses
, OMP_CLAUSE_ASYNC
);
9201 t_async
= fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9203 OMP_CLAUSE_ASYNC_EXPR (c
));
9205 args
.quick_push (t_async
);
9206 /* Save the index, and... */
9207 t_wait_idx
= args
.length ();
9208 /* ... push a default value. */
9209 args
.quick_push (fold_convert_loc (gimple_location (entry_stmt
),
9211 integer_zero_node
));
9212 c
= find_omp_clause (clauses
, OMP_CLAUSE_WAIT
);
9217 for (; c
; c
= OMP_CLAUSE_CHAIN (c
))
9219 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_WAIT
)
9221 args
.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c
),
9223 OMP_CLAUSE_WAIT_EXPR (c
)));
9228 /* Now that we know the number, replace the default value. */
9229 args
.ordered_remove (t_wait_idx
);
9230 args
.quick_insert (t_wait_idx
,
9231 fold_convert_loc (gimple_location (entry_stmt
),
9233 build_int_cst (integer_type_node
, n
)));
9241 g
= gimple_build_call_vec (builtin_decl_explicit (start_ix
), args
);
9242 gimple_set_location (g
, gimple_location (entry_stmt
));
9243 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
9247 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
9248 gsi_remove (&gsi
, true);
9253 gsi
= gsi_last_bb (region
->exit
);
9255 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
9256 gsi_remove (&gsi
, true);
9261 /* Expand the parallel region tree rooted at REGION. Expansion
9262 proceeds in depth-first order. Innermost regions are expanded
9263 first. This way, parallel regions that require a new function to
9264 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
9265 internal dependencies in their body. */
9268 expand_omp (struct omp_region
*region
)
9272 location_t saved_location
;
9273 gimple inner_stmt
= NULL
;
9275 /* First, determine whether this is a combined parallel+workshare
9277 if (region
->type
== GIMPLE_OMP_PARALLEL
)
9278 determine_parallel_type (region
);
9280 if (region
->type
== GIMPLE_OMP_FOR
9281 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
9282 inner_stmt
= last_stmt (region
->inner
->entry
);
9285 expand_omp (region
->inner
);
9287 saved_location
= input_location
;
9288 if (gimple_has_location (last_stmt (region
->entry
)))
9289 input_location
= gimple_location (last_stmt (region
->entry
));
9291 switch (region
->type
)
9293 case GIMPLE_OMP_PARALLEL
:
9294 case GIMPLE_OMP_TASK
:
9295 expand_omp_taskreg (region
);
9298 case GIMPLE_OMP_FOR
:
9299 expand_omp_for (region
, inner_stmt
);
9302 case GIMPLE_OMP_SECTIONS
:
9303 expand_omp_sections (region
);
9306 case GIMPLE_OMP_SECTION
:
9307 /* Individual omp sections are handled together with their
9308 parent GIMPLE_OMP_SECTIONS region. */
9311 case GIMPLE_OMP_SINGLE
:
9312 expand_omp_single (region
);
9315 case GIMPLE_OMP_MASTER
:
9316 case GIMPLE_OMP_TASKGROUP
:
9317 case GIMPLE_OMP_ORDERED
:
9318 case GIMPLE_OMP_CRITICAL
:
9319 case GIMPLE_OMP_TEAMS
:
9320 expand_omp_synch (region
);
9323 case GIMPLE_OMP_ATOMIC_LOAD
:
9324 expand_omp_atomic (region
);
9327 case GIMPLE_OMP_TARGET
:
9328 expand_omp_target (region
);
9335 input_location
= saved_location
;
9336 region
= region
->next
;
9341 /* Helper for build_omp_regions. Scan the dominator tree starting at
9342 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
9343 true, the function ends once a single tree is built (otherwise, whole
9344 forest of OMP constructs may be built). */
9347 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
9350 gimple_stmt_iterator gsi
;
9354 gsi
= gsi_last_bb (bb
);
9355 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
9357 struct omp_region
*region
;
9358 enum gimple_code code
;
9360 stmt
= gsi_stmt (gsi
);
9361 code
= gimple_code (stmt
);
9362 if (code
== GIMPLE_OMP_RETURN
)
9364 /* STMT is the return point out of region PARENT. Mark it
9365 as the exit point and make PARENT the immediately
9366 enclosing region. */
9367 gcc_assert (parent
);
9370 parent
= parent
->outer
;
9372 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
9374 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
9375 GIMPLE_OMP_RETURN, but matches with
9376 GIMPLE_OMP_ATOMIC_LOAD. */
9377 gcc_assert (parent
);
9378 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
9381 parent
= parent
->outer
;
9383 else if (code
== GIMPLE_OMP_CONTINUE
)
9385 gcc_assert (parent
);
9388 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
9390 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
9391 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
9395 region
= new_omp_region (bb
, code
, parent
);
9397 if (code
== GIMPLE_OMP_TARGET
)
9399 switch (gimple_omp_target_kind (stmt
))
9401 case GF_OMP_TARGET_KIND_REGION
:
9402 case GF_OMP_TARGET_KIND_DATA
:
9403 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
9404 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
9405 case GF_OMP_TARGET_KIND_OACC_DATA
:
9407 case GF_OMP_TARGET_KIND_UPDATE
:
9408 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
9409 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
9410 /* ..., other than for those stand-alone directives... */
9417 /* ..., this directive becomes the parent for a new region. */
9423 if (single_tree
&& !parent
)
9426 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
9428 son
= next_dom_son (CDI_DOMINATORS
, son
))
9429 build_omp_regions_1 (son
, parent
, single_tree
);
9432 /* Builds the tree of OMP regions rooted at ROOT, storing it to
9436 build_omp_regions_root (basic_block root
)
9438 gcc_assert (root_omp_region
== NULL
);
9439 build_omp_regions_1 (root
, NULL
, true);
9440 gcc_assert (root_omp_region
!= NULL
);
9443 /* Expands omp construct (and its subconstructs) starting in HEAD. */
9446 omp_expand_local (basic_block head
)
9448 build_omp_regions_root (head
);
9449 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
9451 fprintf (dump_file
, "\nOMP region tree\n\n");
9452 dump_omp_region (dump_file
, root_omp_region
, 0);
9453 fprintf (dump_file
, "\n");
9456 remove_exit_barriers (root_omp_region
);
9457 expand_omp (root_omp_region
);
9459 free_omp_regions ();
9462 /* Scan the CFG and build a tree of OMP regions. Return the root of
9463 the OMP region tree. */
9466 build_omp_regions (void)
9468 gcc_assert (root_omp_region
== NULL
);
9469 calculate_dominance_info (CDI_DOMINATORS
);
9470 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
9473 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
9476 execute_expand_omp (void)
9478 build_omp_regions ();
9480 if (!root_omp_region
)
9485 fprintf (dump_file
, "\nOMP region tree\n\n");
9486 dump_omp_region (dump_file
, root_omp_region
, 0);
9487 fprintf (dump_file
, "\n");
9490 remove_exit_barriers (root_omp_region
);
9492 expand_omp (root_omp_region
);
9494 cleanup_tree_cfg ();
9496 free_omp_regions ();
9501 /* OMP expansion -- the default pass, run before creation of SSA form. */
9505 const pass_data pass_data_expand_omp
=
9507 GIMPLE_PASS
, /* type */
9508 "ompexp", /* name */
9509 OPTGROUP_NONE
, /* optinfo_flags */
9510 TV_NONE
, /* tv_id */
9511 PROP_gimple_any
, /* properties_required */
9512 PROP_gimple_eomp
, /* properties_provided */
9513 0, /* properties_destroyed */
9514 0, /* todo_flags_start */
9515 0, /* todo_flags_finish */
9518 class pass_expand_omp
: public gimple_opt_pass
9521 pass_expand_omp (gcc::context
*ctxt
)
9522 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
9525 /* opt_pass methods: */
9526 virtual unsigned int execute (function
*)
9528 bool gate
= ((flag_cilkplus
!= 0 || flag_openacc
!= 0 || flag_openmp
!= 0
9529 || flag_openmp_simd
!= 0)
9532 /* This pass always runs, to provide PROP_gimple_eomp.
9533 But often, there is nothing to do. */
9537 return execute_expand_omp ();
9540 }; // class pass_expand_omp
9545 make_pass_expand_omp (gcc::context
*ctxt
)
9547 return new pass_expand_omp (ctxt
);
9552 const pass_data pass_data_expand_omp_ssa
=
9554 GIMPLE_PASS
, /* type */
9555 "ompexpssa", /* name */
9556 OPTGROUP_NONE
, /* optinfo_flags */
9557 TV_NONE
, /* tv_id */
9558 PROP_cfg
| PROP_ssa
, /* properties_required */
9559 PROP_gimple_eomp
, /* properties_provided */
9560 0, /* properties_destroyed */
9561 0, /* todo_flags_start */
9562 TODO_cleanup_cfg
| TODO_rebuild_alias
, /* todo_flags_finish */
9565 class pass_expand_omp_ssa
: public gimple_opt_pass
9568 pass_expand_omp_ssa (gcc::context
*ctxt
)
9569 : gimple_opt_pass (pass_data_expand_omp_ssa
, ctxt
)
9572 /* opt_pass methods: */
9573 virtual bool gate (function
*fun
)
9575 return !(fun
->curr_properties
& PROP_gimple_eomp
);
9577 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
9579 }; // class pass_expand_omp_ssa
9584 make_pass_expand_omp_ssa (gcc::context
*ctxt
)
9586 return new pass_expand_omp_ssa (ctxt
);
9589 /* Routines to lower OMP directives into OMP-GIMPLE. */
9591 /* Helper function to preform, potentially COMPLEX_TYPE, operation and
9592 convert it to gimple. */
9594 oacc_gimple_assign (tree dest
, tree_code op
, tree src
, gimple_seq
*seq
)
9598 if (TREE_CODE (TREE_TYPE (dest
)) != COMPLEX_TYPE
)
9600 stmt
= gimple_build_assign (dest
, op
, dest
, src
);
9601 gimple_seq_add_stmt (seq
, stmt
);
9605 tree t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9606 tree rdest
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9607 gimplify_assign (t
, rdest
, seq
);
9610 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9611 tree idest
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (dest
)), dest
);
9612 gimplify_assign (t
, idest
, seq
);
9615 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
9616 tree rsrc
= fold_build1 (REALPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9617 gimplify_assign (t
, rsrc
, seq
);
9620 t
= create_tmp_var (TREE_TYPE (TREE_TYPE (src
)));
9621 tree isrc
= fold_build1 (IMAGPART_EXPR
, TREE_TYPE (TREE_TYPE (src
)), src
);
9622 gimplify_assign (t
, isrc
, seq
);
9625 tree r
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9626 tree i
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9629 if (op
== PLUS_EXPR
)
9631 stmt
= gimple_build_assign (r
, op
, rdest
, rsrc
);
9632 gimple_seq_add_stmt (seq
, stmt
);
9634 stmt
= gimple_build_assign (i
, op
, idest
, isrc
);
9635 gimple_seq_add_stmt (seq
, stmt
);
9637 else if (op
== MULT_EXPR
)
9639 /* Let x = a + ib = dest, y = c + id = src.
9640 x * y = (ac - bd) + i(ad + bc) */
9641 tree ac
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9642 tree bd
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9643 tree ad
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9644 tree bc
= create_tmp_var (TREE_TYPE (TREE_TYPE (dest
)));
9646 stmt
= gimple_build_assign (ac
, MULT_EXPR
, rdest
, rsrc
);
9647 gimple_seq_add_stmt (seq
, stmt
);
9649 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, isrc
);
9650 gimple_seq_add_stmt (seq
, stmt
);
9652 stmt
= gimple_build_assign (r
, MINUS_EXPR
, ac
, bd
);
9653 gimple_seq_add_stmt (seq
, stmt
);
9655 stmt
= gimple_build_assign (ad
, MULT_EXPR
, rdest
, isrc
);
9656 gimple_seq_add_stmt (seq
, stmt
);
9658 stmt
= gimple_build_assign (bd
, MULT_EXPR
, idest
, rsrc
);
9659 gimple_seq_add_stmt (seq
, stmt
);
9661 stmt
= gimple_build_assign (i
, PLUS_EXPR
, ad
, bc
);
9662 gimple_seq_add_stmt (seq
, stmt
);
9667 result
= build2 (COMPLEX_EXPR
, TREE_TYPE (dest
), r
, i
);
9668 gimplify_assign (dest
, result
, seq
);
9671 /* Helper function to initialize local data for the reduction arrays.
9672 The reduction arrays need to be placed inside the calling function
9673 for accelerators, or else the host won't be able to preform the final
9677 oacc_initialize_reduction_data (tree clauses
, tree nthreads
,
9678 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
9684 /* Find the innermost OpenACC parallel context. */
9685 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
9686 && (gimple_omp_target_kind (ctx
->stmt
)
9687 == GF_OMP_TARGET_KIND_OACC_PARALLEL
))
9691 gcc_checking_assert (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
9692 && (gimple_omp_target_kind (octx
->stmt
)
9693 == GF_OMP_TARGET_KIND_OACC_PARALLEL
));
9695 /* Extract the clauses. */
9696 oc
= gimple_omp_target_clauses (octx
->stmt
);
9698 /* Find the last outer clause. */
9699 for (; oc
&& OMP_CLAUSE_CHAIN (oc
); oc
= OMP_CLAUSE_CHAIN (oc
))
9702 /* Allocate arrays for each reduction variable. */
9703 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9705 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9708 tree var
= OMP_CLAUSE_DECL (c
);
9709 tree type
= get_base_type (var
);
9710 tree array
= lookup_oacc_reduction (oacc_get_reduction_array_id (var
),
9714 /* Calculate size of the reduction array. */
9715 t
= create_tmp_var (TREE_TYPE (nthreads
));
9716 stmt
= gimple_build_assign (t
, MULT_EXPR
, nthreads
,
9717 fold_convert (TREE_TYPE (nthreads
),
9718 TYPE_SIZE_UNIT (type
)));
9719 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9721 size
= create_tmp_var (sizetype
);
9722 gimplify_assign (size
, fold_build1 (NOP_EXPR
, sizetype
, t
), stmt_seqp
);
9724 /* Now allocate memory for it. */
9725 call
= unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA
));
9726 stmt
= gimple_build_call (call
, 1, size
);
9727 gimple_call_set_lhs (stmt
, array
);
9728 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9730 /* Map this array into the accelerator. */
9732 /* Add the reduction array to the list of clauses. */
9734 t
= build_omp_clause (gimple_location (ctx
->stmt
), OMP_CLAUSE_MAP
);
9735 OMP_CLAUSE_SET_MAP_KIND (t
, GOMP_MAP_FORCE_FROM
);
9736 OMP_CLAUSE_DECL (t
) = x
;
9737 OMP_CLAUSE_CHAIN (t
) = NULL
;
9739 OMP_CLAUSE_CHAIN (oc
) = t
;
9741 gimple_omp_target_set_clauses (as_a
<gomp_target
*> (octx
->stmt
), t
);
9742 OMP_CLAUSE_SIZE (t
) = size
;
9747 /* Helper function to process the array of partial reductions. Nthreads
9748 indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS
9749 cannot be used here, because nthreads on the host may be different than
9750 on the accelerator. */
9753 oacc_finalize_reduction_data (tree clauses
, tree nthreads
,
9754 gimple_seq
*stmt_seqp
, omp_context
*ctx
)
9756 tree c
, x
, var
, array
, loop_header
, loop_body
, loop_exit
, type
;
9761 let var = the original reduction variable
9762 let array = reduction variable array
9764 for (i = 0; i < nthreads; i++)
9768 loop_header
= create_artificial_label (UNKNOWN_LOCATION
);
9769 loop_body
= create_artificial_label (UNKNOWN_LOCATION
);
9770 loop_exit
= create_artificial_label (UNKNOWN_LOCATION
);
9772 /* Create and initialize an index variable. */
9773 tree ix
= create_tmp_var (sizetype
);
9774 gimplify_assign (ix
, fold_build1 (NOP_EXPR
, sizetype
, integer_zero_node
),
9777 /* Insert the loop header label here. */
9778 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_header
));
9780 /* Exit loop if ix >= nthreads. */
9781 x
= create_tmp_var (sizetype
);
9782 gimplify_assign (x
, fold_build1 (NOP_EXPR
, sizetype
, nthreads
), stmt_seqp
);
9783 stmt
= gimple_build_cond (GE_EXPR
, ix
, x
, loop_exit
, loop_body
);
9784 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9786 /* Insert the loop body label here. */
9787 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_body
));
9789 /* Collapse each reduction array, one element at a time. */
9790 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9792 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
9795 tree_code reduction_code
= OMP_CLAUSE_REDUCTION_CODE (c
);
9797 /* reduction(-:var) sums up the partial results, so it acts
9798 identically to reduction(+:var). */
9799 if (reduction_code
== MINUS_EXPR
)
9800 reduction_code
= PLUS_EXPR
;
9802 /* Set up reduction variable var. */
9803 var
= OMP_CLAUSE_DECL (c
);
9804 type
= get_base_type (var
);
9805 array
= lookup_oacc_reduction (oacc_get_reduction_array_id
9806 (OMP_CLAUSE_DECL (c
)), ctx
);
9808 /* Calculate the array offset. */
9809 tree offset
= create_tmp_var (sizetype
);
9810 gimplify_assign (offset
, TYPE_SIZE_UNIT (type
), stmt_seqp
);
9811 stmt
= gimple_build_assign (offset
, MULT_EXPR
, offset
, ix
);
9812 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9814 tree ptr
= create_tmp_var (TREE_TYPE (array
));
9815 stmt
= gimple_build_assign (ptr
, POINTER_PLUS_EXPR
, array
, offset
);
9816 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9818 /* Extract array[ix] into mem. */
9819 tree mem
= create_tmp_var (type
);
9820 gimplify_assign (mem
, build_simple_mem_ref (ptr
), stmt_seqp
);
9822 /* Find the original reduction variable. */
9823 if (is_reference (var
))
9824 var
= build_simple_mem_ref (var
);
9826 tree t
= create_tmp_var (type
);
9828 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, t
, var
);
9829 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9831 /* var = var op mem */
9832 switch (OMP_CLAUSE_REDUCTION_CODE (c
))
9834 case TRUTH_ANDIF_EXPR
:
9835 case TRUTH_ORIF_EXPR
:
9836 t
= fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c
), integer_type_node
,
9838 gimplify_and_add (t
, stmt_seqp
);
9841 /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */
9842 oacc_gimple_assign (t
, OMP_CLAUSE_REDUCTION_CODE (c
), mem
,
9846 t
= fold_build1 (NOP_EXPR
, TREE_TYPE (var
), t
);
9847 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, var
, t
);
9848 gimplify_and_add (unshare_expr(x
), stmt_seqp
);
9851 /* Increment the induction variable. */
9852 tree one
= fold_build1 (NOP_EXPR
, sizetype
, integer_one_node
);
9853 stmt
= gimple_build_assign (ix
, PLUS_EXPR
, ix
, one
);
9854 gimple_seq_add_stmt (stmt_seqp
, stmt
);
9856 /* Go back to the top of the loop. */
9857 gimple_seq_add_stmt (stmt_seqp
, gimple_build_goto (loop_header
));
9859 /* Place the loop exit label here. */
9860 gimple_seq_add_stmt (stmt_seqp
, gimple_build_label (loop_exit
));
9863 /* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and
9864 scan that for reductions. */
9867 oacc_process_reduction_data (gimple_seq
*body
, gimple_seq
*in_stmt_seqp
,
9868 gimple_seq
*out_stmt_seqp
, omp_context
*ctx
)
9870 gimple_stmt_iterator gsi
;
9871 gimple_seq inner
= NULL
;
9873 /* A collapse clause may have inserted a new bind block. */
9874 gsi
= gsi_start (*body
);
9875 while (!gsi_end_p (gsi
))
9877 gimple stmt
= gsi_stmt (gsi
);
9878 if (gbind
*bind_stmt
= dyn_cast
<gbind
*> (stmt
))
9880 inner
= gimple_bind_body (bind_stmt
);
9882 gsi
= gsi_start (*body
);
9884 else if (dyn_cast
<gomp_for
*> (stmt
))
9890 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
9892 tree clauses
, nthreads
, t
, c
, acc_device
, acc_device_host
, call
,
9894 bool reduction_found
= false;
9896 gimple stmt
= gsi_stmt (gsi
);
9898 switch (gimple_code (stmt
))
9900 case GIMPLE_OMP_FOR
:
9901 clauses
= gimple_omp_for_clauses (stmt
);
9903 /* Search for a reduction clause. */
9904 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9905 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
9907 reduction_found
= true;
9911 if (!reduction_found
)
9914 ctx
= maybe_lookup_ctx (stmt
);
9917 /* Extract the number of threads. */
9918 nthreads
= create_tmp_var (sizetype
);
9919 t
= oacc_max_threads (ctx
);
9920 gimplify_assign (nthreads
, t
, in_stmt_seqp
);
9922 /* Determine if this is kernel will be executed on the host. */
9923 call
= builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE
);
9924 acc_device
= create_tmp_var (integer_type_node
, ".acc_device_type");
9925 stmt
= gimple_build_call (call
, 0);
9926 gimple_call_set_lhs (stmt
, acc_device
);
9927 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9929 /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */
9930 acc_device_host
= create_tmp_var (integer_type_node
,
9931 ".acc_device_host");
9932 gimplify_assign (acc_device_host
,
9933 build_int_cst (integer_type_node
,
9937 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9938 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9940 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9942 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9943 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9944 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9947 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9949 /* Also, set nthreads = 1 for ACC_DEVICE_TYPE=host_nonshm. */
9950 gimplify_assign (acc_device_host
,
9951 build_int_cst (integer_type_node
,
9952 GOMP_DEVICE_HOST_NONSHM
),
9955 enter
= create_artificial_label (UNKNOWN_LOCATION
);
9956 exit
= create_artificial_label (UNKNOWN_LOCATION
);
9958 stmt
= gimple_build_cond (EQ_EXPR
, acc_device
, acc_device_host
,
9960 gimple_seq_add_stmt (in_stmt_seqp
, stmt
);
9961 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (enter
));
9962 gimplify_assign (nthreads
, fold_build1 (NOP_EXPR
, sizetype
,
9965 gimple_seq_add_stmt (in_stmt_seqp
, gimple_build_label (exit
));
9967 oacc_initialize_reduction_data (clauses
, nthreads
, in_stmt_seqp
,
9969 oacc_finalize_reduction_data (clauses
, nthreads
, out_stmt_seqp
, ctx
);
9972 // Scan for other directives which support reduction here.
9978 /* If ctx is a worksharing context inside of a cancellable parallel
9979 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
9980 and conditional branch to parallel's cancel_label to handle
9981 cancellation in the implicit barrier. */
9984 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
9986 gimple omp_return
= gimple_seq_last_stmt (*body
);
9987 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
9988 if (gimple_omp_return_nowait_p (omp_return
))
9991 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
9992 && ctx
->outer
->cancellable
)
9994 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
9995 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
9996 tree lhs
= create_tmp_var (c_bool_type
);
9997 gimple_omp_return_set_lhs (omp_return
, lhs
);
9998 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
9999 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
10000 fold_convert (c_bool_type
,
10001 boolean_false_node
),
10002 ctx
->outer
->cancel_label
, fallthru_label
);
10003 gimple_seq_add_stmt (body
, g
);
10004 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
10008 /* Lower the OpenMP sections directive in the current statement in GSI_P.
10009 CTX is the enclosing OMP context for the current statement. */
10012 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10014 tree block
, control
;
10015 gimple_stmt_iterator tgsi
;
10016 gomp_sections
*stmt
;
10018 gbind
*new_stmt
, *bind
;
10019 gimple_seq ilist
, dlist
, olist
, new_body
;
10021 stmt
= as_a
<gomp_sections
*> (gsi_stmt (*gsi_p
));
10023 push_gimplify_context ();
10027 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
10028 &ilist
, &dlist
, ctx
, NULL
);
10030 new_body
= gimple_omp_body (stmt
);
10031 gimple_omp_set_body (stmt
, NULL
);
10032 tgsi
= gsi_start (new_body
);
10033 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
10038 sec_start
= gsi_stmt (tgsi
);
10039 sctx
= maybe_lookup_ctx (sec_start
);
10042 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
10043 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
10044 GSI_CONTINUE_LINKING
);
10045 gimple_omp_set_body (sec_start
, NULL
);
10047 if (gsi_one_before_end_p (tgsi
))
10049 gimple_seq l
= NULL
;
10050 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
10052 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
10053 gimple_omp_section_set_last (sec_start
);
10056 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
10057 GSI_CONTINUE_LINKING
);
10060 block
= make_node (BLOCK
);
10061 bind
= gimple_build_bind (NULL
, new_body
, block
);
10064 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
10066 block
= make_node (BLOCK
);
10067 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10068 gsi_replace (gsi_p
, new_stmt
, true);
10070 pop_gimplify_context (new_stmt
);
10071 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10072 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10073 if (BLOCK_VARS (block
))
10074 TREE_USED (block
) = 1;
10077 gimple_seq_add_seq (&new_body
, ilist
);
10078 gimple_seq_add_stmt (&new_body
, stmt
);
10079 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
10080 gimple_seq_add_stmt (&new_body
, bind
);
10082 control
= create_tmp_var (unsigned_type_node
, ".section");
10083 t
= gimple_build_omp_continue (control
, control
);
10084 gimple_omp_sections_set_control (stmt
, control
);
10085 gimple_seq_add_stmt (&new_body
, t
);
10087 gimple_seq_add_seq (&new_body
, olist
);
10088 if (ctx
->cancellable
)
10089 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
10090 gimple_seq_add_seq (&new_body
, dlist
);
10092 new_body
= maybe_catch_exception (new_body
);
10094 t
= gimple_build_omp_return
10095 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
10096 OMP_CLAUSE_NOWAIT
));
10097 gimple_seq_add_stmt (&new_body
, t
);
10098 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
10100 gimple_bind_set_body (new_stmt
, new_body
);
10104 /* A subroutine of lower_omp_single. Expand the simple form of
10105 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
10107 if (GOMP_single_start ())
10109 [ GOMP_barrier (); ] -> unless 'nowait' is present.
10111 FIXME. It may be better to delay expanding the logic of this until
10112 pass_expand_omp. The expanded logic may make the job more difficult
10113 to a synchronization analysis pass. */
10116 lower_omp_single_simple (gomp_single
*single_stmt
, gimple_seq
*pre_p
)
10118 location_t loc
= gimple_location (single_stmt
);
10119 tree tlabel
= create_artificial_label (loc
);
10120 tree flabel
= create_artificial_label (loc
);
10124 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
10125 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)));
10126 call
= gimple_build_call (decl
, 0);
10127 gimple_call_set_lhs (call
, lhs
);
10128 gimple_seq_add_stmt (pre_p
, call
);
10130 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
10131 fold_convert_loc (loc
, TREE_TYPE (lhs
),
10132 boolean_true_node
),
10134 gimple_seq_add_stmt (pre_p
, cond
);
10135 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
10136 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10137 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
10141 /* A subroutine of lower_omp_single. Expand the simple form of
10142 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
10144 #pragma omp single copyprivate (a, b, c)
10146 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
10149 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
10155 GOMP_single_copy_end (©out);
10166 FIXME. It may be better to delay expanding the logic of this until
10167 pass_expand_omp. The expanded logic may make the job more difficult
10168 to a synchronization analysis pass. */
10171 lower_omp_single_copy (gomp_single
*single_stmt
, gimple_seq
*pre_p
,
10174 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
10175 gimple_seq copyin_seq
;
10176 location_t loc
= gimple_location (single_stmt
);
10178 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
10180 ptr_type
= build_pointer_type (ctx
->record_type
);
10181 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
10183 l0
= create_artificial_label (loc
);
10184 l1
= create_artificial_label (loc
);
10185 l2
= create_artificial_label (loc
);
10187 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
10188 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
10189 t
= fold_convert_loc (loc
, ptr_type
, t
);
10190 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
10192 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
10193 build_int_cst (ptr_type
, 0));
10194 t
= build3 (COND_EXPR
, void_type_node
, t
,
10195 build_and_jump (&l0
), build_and_jump (&l1
));
10196 gimplify_and_add (t
, pre_p
);
10198 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
10200 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
10203 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
10206 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
10207 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
10208 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
10209 gimplify_and_add (t
, pre_p
);
10211 t
= build_and_jump (&l2
);
10212 gimplify_and_add (t
, pre_p
);
10214 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
10216 gimple_seq_add_seq (pre_p
, copyin_seq
);
10218 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
10222 /* Expand code for an OpenMP single directive. */
10225 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10229 gomp_single
*single_stmt
= as_a
<gomp_single
*> (gsi_stmt (*gsi_p
));
10231 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
10233 push_gimplify_context ();
10235 block
= make_node (BLOCK
);
10236 bind
= gimple_build_bind (NULL
, NULL
, block
);
10237 gsi_replace (gsi_p
, bind
, true);
10240 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
10241 &bind_body
, &dlist
, ctx
, NULL
);
10242 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
10244 gimple_seq_add_stmt (&bind_body
, single_stmt
);
10246 if (ctx
->record_type
)
10247 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
10249 lower_omp_single_simple (single_stmt
, &bind_body
);
10251 gimple_omp_set_body (single_stmt
, NULL
);
10253 gimple_seq_add_seq (&bind_body
, dlist
);
10255 bind_body
= maybe_catch_exception (bind_body
);
10257 t
= gimple_build_omp_return
10258 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
10259 OMP_CLAUSE_NOWAIT
));
10260 gimple_seq_add_stmt (&bind_body_tail
, t
);
10261 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
10262 if (ctx
->record_type
)
10264 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
10265 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
10266 TREE_THIS_VOLATILE (clobber
) = 1;
10267 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
10268 clobber
), GSI_SAME_STMT
);
10270 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
10271 gimple_bind_set_body (bind
, bind_body
);
10273 pop_gimplify_context (bind
);
10275 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10276 BLOCK_VARS (block
) = ctx
->block_vars
;
10277 if (BLOCK_VARS (block
))
10278 TREE_USED (block
) = 1;
10282 /* Expand code for an OpenMP master directive. */
10285 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10287 tree block
, lab
= NULL
, x
, bfn_decl
;
10288 gimple stmt
= gsi_stmt (*gsi_p
);
10290 location_t loc
= gimple_location (stmt
);
10293 push_gimplify_context ();
10295 block
= make_node (BLOCK
);
10296 bind
= gimple_build_bind (NULL
, NULL
, block
);
10297 gsi_replace (gsi_p
, bind
, true);
10298 gimple_bind_add_stmt (bind
, stmt
);
10300 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
10301 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
10302 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
10303 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
10305 gimplify_and_add (x
, &tseq
);
10306 gimple_bind_add_seq (bind
, tseq
);
10308 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10309 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10310 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10311 gimple_omp_set_body (stmt
, NULL
);
10313 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
10315 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10317 pop_gimplify_context (bind
);
10319 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10320 BLOCK_VARS (block
) = ctx
->block_vars
;
10324 /* Expand code for an OpenMP taskgroup directive. */
10327 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10329 gimple stmt
= gsi_stmt (*gsi_p
);
10332 tree block
= make_node (BLOCK
);
10334 bind
= gimple_build_bind (NULL
, NULL
, block
);
10335 gsi_replace (gsi_p
, bind
, true);
10336 gimple_bind_add_stmt (bind
, stmt
);
10338 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
10340 gimple_bind_add_stmt (bind
, x
);
10342 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10343 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10344 gimple_omp_set_body (stmt
, NULL
);
10346 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10348 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10349 BLOCK_VARS (block
) = ctx
->block_vars
;
10353 /* Expand code for an OpenMP ordered directive. */
10356 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10359 gimple stmt
= gsi_stmt (*gsi_p
);
10363 push_gimplify_context ();
10365 block
= make_node (BLOCK
);
10366 bind
= gimple_build_bind (NULL
, NULL
, block
);
10367 gsi_replace (gsi_p
, bind
, true);
10368 gimple_bind_add_stmt (bind
, stmt
);
10370 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
10372 gimple_bind_add_stmt (bind
, x
);
10374 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10375 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10376 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10377 gimple_omp_set_body (stmt
, NULL
);
10379 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
10380 gimple_bind_add_stmt (bind
, x
);
10382 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10384 pop_gimplify_context (bind
);
10386 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10387 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10391 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
10392 substitution of a couple of function calls. But in the NAMED case,
10393 requires that languages coordinate a symbol name. It is therefore
10394 best put here in common code. */
10396 static GTY(()) hash_map
<tree
, tree
> *critical_name_mutexes
;
10399 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10402 tree name
, lock
, unlock
;
10403 gomp_critical
*stmt
= as_a
<gomp_critical
*> (gsi_stmt (*gsi_p
));
10405 location_t loc
= gimple_location (stmt
);
10408 name
= gimple_omp_critical_name (stmt
);
10413 if (!critical_name_mutexes
)
10414 critical_name_mutexes
= hash_map
<tree
, tree
>::create_ggc (10);
10416 tree
*n
= critical_name_mutexes
->get (name
);
10421 decl
= create_tmp_var_raw (ptr_type_node
);
10423 new_str
= ACONCAT ((".gomp_critical_user_",
10424 IDENTIFIER_POINTER (name
), NULL
));
10425 DECL_NAME (decl
) = get_identifier (new_str
);
10426 TREE_PUBLIC (decl
) = 1;
10427 TREE_STATIC (decl
) = 1;
10428 DECL_COMMON (decl
) = 1;
10429 DECL_ARTIFICIAL (decl
) = 1;
10430 DECL_IGNORED_P (decl
) = 1;
10432 varpool_node::finalize_decl (decl
);
10434 critical_name_mutexes
->put (name
, decl
);
10439 /* If '#pragma omp critical' is inside offloaded region or
10440 inside function marked as offloadable, the symbol must be
10441 marked as offloadable too. */
10443 if (cgraph_node::get (current_function_decl
)->offloadable
)
10444 varpool_node::get_create (decl
)->offloadable
= 1;
10446 for (octx
= ctx
->outer
; octx
; octx
= octx
->outer
)
10447 if (is_gimple_omp_offloaded (octx
->stmt
))
10449 varpool_node::get_create (decl
)->offloadable
= 1;
10453 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
10454 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
10456 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
10457 unlock
= build_call_expr_loc (loc
, unlock
, 1,
10458 build_fold_addr_expr_loc (loc
, decl
));
10462 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
10463 lock
= build_call_expr_loc (loc
, lock
, 0);
10465 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
10466 unlock
= build_call_expr_loc (loc
, unlock
, 0);
10469 push_gimplify_context ();
10471 block
= make_node (BLOCK
);
10472 bind
= gimple_build_bind (NULL
, NULL
, block
);
10473 gsi_replace (gsi_p
, bind
, true);
10474 gimple_bind_add_stmt (bind
, stmt
);
10476 tbody
= gimple_bind_body (bind
);
10477 gimplify_and_add (lock
, &tbody
);
10478 gimple_bind_set_body (bind
, tbody
);
10480 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10481 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
10482 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
10483 gimple_omp_set_body (stmt
, NULL
);
10485 tbody
= gimple_bind_body (bind
);
10486 gimplify_and_add (unlock
, &tbody
);
10487 gimple_bind_set_body (bind
, tbody
);
10489 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
10491 pop_gimplify_context (bind
);
10492 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10493 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
10497 /* A subroutine of lower_omp_for. Generate code to emit the predicate
10498 for a lastprivate clause. Given a loop control predicate of (V
10499 cond N2), we gate the clause on (!(V cond N2)). The lowered form
10500 is appended to *DLIST, iterator initialization is appended to
10504 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
10505 gimple_seq
*dlist
, struct omp_context
*ctx
)
10507 tree clauses
, cond
, vinit
;
10508 enum tree_code cond_code
;
10511 cond_code
= fd
->loop
.cond_code
;
10512 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
10514 /* When possible, use a strict equality expression. This can let VRP
10515 type optimizations deduce the value and remove a copy. */
10516 if (tree_fits_shwi_p (fd
->loop
.step
))
10518 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
10519 if (step
== 1 || step
== -1)
10520 cond_code
= EQ_EXPR
;
10523 tree n2
= fd
->loop
.n2
;
10524 if (fd
->collapse
> 1
10525 && TREE_CODE (n2
) != INTEGER_CST
10526 && gimple_omp_for_combined_into_p (fd
->for_stmt
)
10527 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_FOR
)
10529 gomp_for
*gfor
= as_a
<gomp_for
*> (ctx
->outer
->stmt
);
10530 if (gimple_omp_for_kind (gfor
) == GF_OMP_FOR_KIND_FOR
)
10532 struct omp_for_data outer_fd
;
10533 extract_omp_for_data (gfor
, &outer_fd
, NULL
);
10534 n2
= fold_convert (TREE_TYPE (n2
), outer_fd
.loop
.n2
);
10537 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, n2
);
10539 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
10541 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
10542 if (!gimple_seq_empty_p (stmts
))
10544 gimple_seq_add_seq (&stmts
, *dlist
);
10547 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
10548 vinit
= fd
->loop
.n1
;
10549 if (cond_code
== EQ_EXPR
10550 && tree_fits_shwi_p (fd
->loop
.n2
)
10551 && ! integer_zerop (fd
->loop
.n2
))
10552 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
10554 vinit
= unshare_expr (vinit
);
10556 /* Initialize the iterator variable, so that threads that don't execute
10557 any iterations don't execute the lastprivate clauses by accident. */
10558 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
10563 /* Lower code for an OMP loop directive. */
10566 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10568 tree
*rhs_p
, block
;
10569 struct omp_for_data fd
, *fdp
= NULL
;
10570 gomp_for
*stmt
= as_a
<gomp_for
*> (gsi_stmt (*gsi_p
));
10572 gimple_seq omp_for_body
, body
, dlist
;
10575 push_gimplify_context ();
10577 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
10579 block
= make_node (BLOCK
);
10580 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
10581 /* Replace at gsi right away, so that 'stmt' is no member
10582 of a sequence anymore as we're going to add to to a different
10584 gsi_replace (gsi_p
, new_stmt
, true);
10586 /* Move declaration of temporaries in the loop body before we make
10588 omp_for_body
= gimple_omp_body (stmt
);
10589 if (!gimple_seq_empty_p (omp_for_body
)
10590 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
10593 = as_a
<gbind
*> (gimple_seq_first_stmt (omp_for_body
));
10594 tree vars
= gimple_bind_vars (inner_bind
);
10595 gimple_bind_append_vars (new_stmt
, vars
);
10596 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
10597 keep them on the inner_bind and it's block. */
10598 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
10599 if (gimple_bind_block (inner_bind
))
10600 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
10603 if (gimple_omp_for_combined_into_p (stmt
))
10605 extract_omp_for_data (stmt
, &fd
, NULL
);
10608 /* We need two temporaries with fd.loop.v type (istart/iend)
10609 and then (fd.collapse - 1) temporaries with the same
10610 type for count2 ... countN-1 vars if not constant. */
10612 tree type
= fd
.iter_type
;
10613 if (fd
.collapse
> 1
10614 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
10615 count
+= fd
.collapse
- 1;
10616 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
10617 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
10618 tree clauses
= *pc
;
10621 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
10622 OMP_CLAUSE__LOOPTEMP_
);
10623 for (i
= 0; i
< count
; i
++)
10628 gcc_assert (outerc
);
10629 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
10630 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
10631 OMP_CLAUSE__LOOPTEMP_
);
10635 temp
= create_tmp_var (type
);
10636 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
10638 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
10639 OMP_CLAUSE_DECL (*pc
) = temp
;
10640 pc
= &OMP_CLAUSE_CHAIN (*pc
);
10645 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
10648 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
10650 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
10652 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
10654 /* Lower the header expressions. At this point, we can assume that
10655 the header is of the form:
10657 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
10659 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
10660 using the .omp_data_s mapping, if needed. */
10661 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
10663 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
10664 if (!is_gimple_min_invariant (*rhs_p
))
10665 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10667 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
10668 if (!is_gimple_min_invariant (*rhs_p
))
10669 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10671 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
10672 if (!is_gimple_min_invariant (*rhs_p
))
10673 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
10676 /* Once lowered, extract the bounds and clauses. */
10677 extract_omp_for_data (stmt
, &fd
, NULL
);
10679 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
10681 gimple_seq_add_stmt (&body
, stmt
);
10682 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
10684 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
10687 /* After the loop, add exit clauses. */
10688 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
10690 if (ctx
->cancellable
)
10691 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
10693 gimple_seq_add_seq (&body
, dlist
);
10695 body
= maybe_catch_exception (body
);
10697 /* Region exit marker goes at the end of the loop body. */
10698 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
10699 maybe_add_implicit_barrier_cancel (ctx
, &body
);
10700 pop_gimplify_context (new_stmt
);
10702 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
10703 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
10704 if (BLOCK_VARS (block
))
10705 TREE_USED (block
) = 1;
10707 gimple_bind_set_body (new_stmt
, body
);
10708 gimple_omp_set_body (stmt
, NULL
);
10709 gimple_omp_for_set_pre_body (stmt
, NULL
);
10712 /* Callback for walk_stmts. Check if the current statement only contains
10713 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
10716 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
10717 bool *handled_ops_p
,
10718 struct walk_stmt_info
*wi
)
10720 int *info
= (int *) wi
->info
;
10721 gimple stmt
= gsi_stmt (*gsi_p
);
10723 *handled_ops_p
= true;
10724 switch (gimple_code (stmt
))
10728 case GIMPLE_OMP_FOR
:
10729 case GIMPLE_OMP_SECTIONS
:
10730 *info
= *info
== 0 ? 1 : -1;
10739 struct omp_taskcopy_context
10741 /* This field must be at the beginning, as we do "inheritance": Some
10742 callback functions for tree-inline.c (e.g., omp_copy_decl)
10743 receive a copy_body_data pointer that is up-casted to an
10744 omp_context pointer. */
10750 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
10752 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
10754 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
10755 return create_tmp_var (TREE_TYPE (var
));
10761 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
10763 tree name
, new_fields
= NULL
, type
, f
;
10765 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
10766 name
= DECL_NAME (TYPE_NAME (orig_type
));
10767 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
10768 TYPE_DECL
, name
, type
);
10769 TYPE_NAME (type
) = name
;
10771 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
10773 tree new_f
= copy_node (f
);
10774 DECL_CONTEXT (new_f
) = type
;
10775 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
10776 TREE_CHAIN (new_f
) = new_fields
;
10777 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10778 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
10779 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
10781 new_fields
= new_f
;
10782 tcctx
->cb
.decl_map
->put (f
, new_f
);
10784 TYPE_FIELDS (type
) = nreverse (new_fields
);
10785 layout_type (type
);
10789 /* Create task copyfn. */
10792 create_task_copyfn (gomp_task
*task_stmt
, omp_context
*ctx
)
10794 struct function
*child_cfun
;
10795 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
10796 tree record_type
, srecord_type
, bind
, list
;
10797 bool record_needs_remap
= false, srecord_needs_remap
= false;
10799 struct omp_taskcopy_context tcctx
;
10800 location_t loc
= gimple_location (task_stmt
);
10802 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
10803 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
10804 gcc_assert (child_cfun
->cfg
== NULL
);
10805 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
10807 /* Reset DECL_CONTEXT on function arguments. */
10808 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
10809 DECL_CONTEXT (t
) = child_fn
;
10811 /* Populate the function. */
10812 push_gimplify_context ();
10813 push_cfun (child_cfun
);
10815 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
10816 TREE_SIDE_EFFECTS (bind
) = 1;
10818 DECL_SAVED_TREE (child_fn
) = bind
;
10819 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
10821 /* Remap src and dst argument types if needed. */
10822 record_type
= ctx
->record_type
;
10823 srecord_type
= ctx
->srecord_type
;
10824 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
10825 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10827 record_needs_remap
= true;
10830 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
10831 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
10833 srecord_needs_remap
= true;
10837 if (record_needs_remap
|| srecord_needs_remap
)
10839 memset (&tcctx
, '\0', sizeof (tcctx
));
10840 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
10841 tcctx
.cb
.dst_fn
= child_fn
;
10842 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
10843 gcc_checking_assert (tcctx
.cb
.src_node
);
10844 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
10845 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
10846 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
10847 tcctx
.cb
.eh_lp_nr
= 0;
10848 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
10849 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
10852 if (record_needs_remap
)
10853 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
10854 if (srecord_needs_remap
)
10855 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
10858 tcctx
.cb
.decl_map
= NULL
;
10860 arg
= DECL_ARGUMENTS (child_fn
);
10861 TREE_TYPE (arg
) = build_pointer_type (record_type
);
10862 sarg
= DECL_CHAIN (arg
);
10863 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
10865 /* First pass: initialize temporaries used in record_type and srecord_type
10866 sizes and field offsets. */
10867 if (tcctx
.cb
.decl_map
)
10868 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10869 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10873 decl
= OMP_CLAUSE_DECL (c
);
10874 p
= tcctx
.cb
.decl_map
->get (decl
);
10877 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10878 sf
= (tree
) n
->value
;
10879 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10880 src
= build_simple_mem_ref_loc (loc
, sarg
);
10881 src
= omp_build_component_ref (src
, sf
);
10882 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
10883 append_to_statement_list (t
, &list
);
10886 /* Second pass: copy shared var pointers and copy construct non-VLA
10887 firstprivate vars. */
10888 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10889 switch (OMP_CLAUSE_CODE (c
))
10891 case OMP_CLAUSE_SHARED
:
10892 decl
= OMP_CLAUSE_DECL (c
);
10893 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10896 f
= (tree
) n
->value
;
10897 if (tcctx
.cb
.decl_map
)
10898 f
= *tcctx
.cb
.decl_map
->get (f
);
10899 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10900 sf
= (tree
) n
->value
;
10901 if (tcctx
.cb
.decl_map
)
10902 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10903 src
= build_simple_mem_ref_loc (loc
, sarg
);
10904 src
= omp_build_component_ref (src
, sf
);
10905 dst
= build_simple_mem_ref_loc (loc
, arg
);
10906 dst
= omp_build_component_ref (dst
, f
);
10907 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10908 append_to_statement_list (t
, &list
);
10910 case OMP_CLAUSE_FIRSTPRIVATE
:
10911 decl
= OMP_CLAUSE_DECL (c
);
10912 if (is_variable_sized (decl
))
10914 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10917 f
= (tree
) n
->value
;
10918 if (tcctx
.cb
.decl_map
)
10919 f
= *tcctx
.cb
.decl_map
->get (f
);
10920 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10923 sf
= (tree
) n
->value
;
10924 if (tcctx
.cb
.decl_map
)
10925 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10926 src
= build_simple_mem_ref_loc (loc
, sarg
);
10927 src
= omp_build_component_ref (src
, sf
);
10928 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
10929 src
= build_simple_mem_ref_loc (loc
, src
);
10933 dst
= build_simple_mem_ref_loc (loc
, arg
);
10934 dst
= omp_build_component_ref (dst
, f
);
10935 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10936 append_to_statement_list (t
, &list
);
10938 case OMP_CLAUSE_PRIVATE
:
10939 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
10941 decl
= OMP_CLAUSE_DECL (c
);
10942 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10943 f
= (tree
) n
->value
;
10944 if (tcctx
.cb
.decl_map
)
10945 f
= *tcctx
.cb
.decl_map
->get (f
);
10946 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
10949 sf
= (tree
) n
->value
;
10950 if (tcctx
.cb
.decl_map
)
10951 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10952 src
= build_simple_mem_ref_loc (loc
, sarg
);
10953 src
= omp_build_component_ref (src
, sf
);
10954 if (use_pointer_for_field (decl
, NULL
))
10955 src
= build_simple_mem_ref_loc (loc
, src
);
10959 dst
= build_simple_mem_ref_loc (loc
, arg
);
10960 dst
= omp_build_component_ref (dst
, f
);
10961 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
10962 append_to_statement_list (t
, &list
);
10968 /* Last pass: handle VLA firstprivates. */
10969 if (tcctx
.cb
.decl_map
)
10970 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
10971 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
10975 decl
= OMP_CLAUSE_DECL (c
);
10976 if (!is_variable_sized (decl
))
10978 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
10981 f
= (tree
) n
->value
;
10982 f
= *tcctx
.cb
.decl_map
->get (f
);
10983 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
10984 ind
= DECL_VALUE_EXPR (decl
);
10985 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
10986 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
10987 n
= splay_tree_lookup (ctx
->sfield_map
,
10988 (splay_tree_key
) TREE_OPERAND (ind
, 0));
10989 sf
= (tree
) n
->value
;
10990 sf
= *tcctx
.cb
.decl_map
->get (sf
);
10991 src
= build_simple_mem_ref_loc (loc
, sarg
);
10992 src
= omp_build_component_ref (src
, sf
);
10993 src
= build_simple_mem_ref_loc (loc
, src
);
10994 dst
= build_simple_mem_ref_loc (loc
, arg
);
10995 dst
= omp_build_component_ref (dst
, f
);
10996 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
10997 append_to_statement_list (t
, &list
);
10998 n
= splay_tree_lookup (ctx
->field_map
,
10999 (splay_tree_key
) TREE_OPERAND (ind
, 0));
11000 df
= (tree
) n
->value
;
11001 df
= *tcctx
.cb
.decl_map
->get (df
);
11002 ptr
= build_simple_mem_ref_loc (loc
, arg
);
11003 ptr
= omp_build_component_ref (ptr
, df
);
11004 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
11005 build_fold_addr_expr_loc (loc
, dst
));
11006 append_to_statement_list (t
, &list
);
11009 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
11010 append_to_statement_list (t
, &list
);
11012 if (tcctx
.cb
.decl_map
)
11013 delete tcctx
.cb
.decl_map
;
11014 pop_gimplify_context (NULL
);
11015 BIND_EXPR_BODY (bind
) = list
;
11020 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
11024 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
11026 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
11027 OMP_CLAUSE_DEPEND
);
11028 gcc_assert (clauses
);
11029 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11030 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
11031 switch (OMP_CLAUSE_DEPEND_KIND (c
))
11033 case OMP_CLAUSE_DEPEND_IN
:
11036 case OMP_CLAUSE_DEPEND_OUT
:
11037 case OMP_CLAUSE_DEPEND_INOUT
:
11041 gcc_unreachable ();
11043 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
11044 tree array
= create_tmp_var (type
);
11045 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
11047 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
11048 gimple_seq_add_stmt (iseq
, g
);
11049 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
11051 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
11052 gimple_seq_add_stmt (iseq
, g
);
11053 for (i
= 0; i
< 2; i
++)
11055 if ((i
? n_in
: n_out
) == 0)
11057 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11058 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
11059 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
11061 tree t
= OMP_CLAUSE_DECL (c
);
11062 t
= fold_convert (ptr_type_node
, t
);
11063 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
11064 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
11065 NULL_TREE
, NULL_TREE
);
11066 g
= gimple_build_assign (r
, t
);
11067 gimple_seq_add_stmt (iseq
, g
);
11070 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
11071 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
11072 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
11073 OMP_CLAUSE_CHAIN (c
) = *p
;
11075 tree clobber
= build_constructor (type
, NULL
);
11076 TREE_THIS_VOLATILE (clobber
) = 1;
11077 g
= gimple_build_assign (array
, clobber
);
11078 gimple_seq_add_stmt (oseq
, g
);
11081 /* Lower the OpenMP parallel or task directive in the current statement
11082 in GSI_P. CTX holds context information for the directive. */
11085 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11089 gimple stmt
= gsi_stmt (*gsi_p
);
11090 gbind
*par_bind
, *bind
, *dep_bind
= NULL
;
11091 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
11092 location_t loc
= gimple_location (stmt
);
11094 clauses
= gimple_omp_taskreg_clauses (stmt
);
11096 = as_a
<gbind
*> (gimple_seq_first_stmt (gimple_omp_body (stmt
)));
11097 par_body
= gimple_bind_body (par_bind
);
11098 child_fn
= ctx
->cb
.dst_fn
;
11099 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
11100 && !gimple_omp_parallel_combined_p (stmt
))
11102 struct walk_stmt_info wi
;
11105 memset (&wi
, 0, sizeof (wi
));
11107 wi
.val_only
= true;
11108 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
11110 gimple_omp_parallel_set_combined_p (stmt
, true);
11112 gimple_seq dep_ilist
= NULL
;
11113 gimple_seq dep_olist
= NULL
;
11114 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
11115 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
11117 push_gimplify_context ();
11118 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
11119 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
11122 if (ctx
->srecord_type
)
11123 create_task_copyfn (as_a
<gomp_task
*> (stmt
), ctx
);
11125 push_gimplify_context ();
11130 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
11131 lower_omp (&par_body
, ctx
);
11132 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
11133 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
11135 /* Declare all the variables created by mapping and the variables
11136 declared in the scope of the parallel body. */
11137 record_vars_into (ctx
->block_vars
, child_fn
);
11138 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
11140 if (ctx
->record_type
)
11143 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
11144 : ctx
->record_type
, ".omp_data_o");
11145 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11146 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11147 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
11152 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
11153 lower_send_shared_vars (&ilist
, &olist
, ctx
);
11155 if (ctx
->record_type
)
11157 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
11158 TREE_THIS_VOLATILE (clobber
) = 1;
11159 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11163 /* Once all the expansions are done, sequence all the different
11164 fragments inside gimple_omp_body. */
11168 if (ctx
->record_type
)
11170 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11171 /* fixup_child_record_type might have changed receiver_decl's type. */
11172 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11173 gimple_seq_add_stmt (&new_body
,
11174 gimple_build_assign (ctx
->receiver_decl
, t
));
11177 gimple_seq_add_seq (&new_body
, par_ilist
);
11178 gimple_seq_add_seq (&new_body
, par_body
);
11179 gimple_seq_add_seq (&new_body
, par_rlist
);
11180 if (ctx
->cancellable
)
11181 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
11182 gimple_seq_add_seq (&new_body
, par_olist
);
11183 new_body
= maybe_catch_exception (new_body
);
11184 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
)
11185 gimple_seq_add_stmt (&new_body
,
11186 gimple_build_omp_continue (integer_zero_node
,
11187 integer_zero_node
));
11188 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11189 gimple_omp_set_body (stmt
, new_body
);
11191 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
11192 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
11193 gimple_bind_add_seq (bind
, ilist
);
11194 gimple_bind_add_stmt (bind
, stmt
);
11195 gimple_bind_add_seq (bind
, olist
);
11197 pop_gimplify_context (NULL
);
11201 gimple_bind_add_seq (dep_bind
, dep_ilist
);
11202 gimple_bind_add_stmt (dep_bind
, bind
);
11203 gimple_bind_add_seq (dep_bind
, dep_olist
);
11204 pop_gimplify_context (dep_bind
);
11208 /* Lower the GIMPLE_OMP_TARGET in the current statement
11209 in GSI_P. CTX holds context information for the directive. */
11212 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11215 tree child_fn
, t
, c
;
11216 gomp_target
*stmt
= as_a
<gomp_target
*> (gsi_stmt (*gsi_p
));
11217 gbind
*tgt_bind
, *bind
;
11218 gimple_seq tgt_body
, olist
, ilist
, orlist
, irlist
, new_body
;
11219 location_t loc
= gimple_location (stmt
);
11220 bool offloaded
, data_region
;
11221 unsigned int map_cnt
= 0;
11223 offloaded
= is_gimple_omp_offloaded (stmt
);
11224 switch (gimple_omp_target_kind (stmt
))
11226 case GF_OMP_TARGET_KIND_REGION
:
11227 case GF_OMP_TARGET_KIND_UPDATE
:
11228 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
11229 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
11230 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
11231 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
11232 data_region
= false;
11234 case GF_OMP_TARGET_KIND_DATA
:
11235 case GF_OMP_TARGET_KIND_OACC_DATA
:
11236 data_region
= true;
11239 gcc_unreachable ();
11242 clauses
= gimple_omp_target_clauses (stmt
);
11248 tgt_bind
= gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt
));
11249 tgt_body
= gimple_bind_body (tgt_bind
);
11251 else if (data_region
)
11252 tgt_body
= gimple_omp_body (stmt
);
11253 child_fn
= ctx
->cb
.dst_fn
;
11255 push_gimplify_context ();
11260 && is_gimple_omp_oacc (stmt
))
11261 oacc_process_reduction_data (&tgt_body
, &irlist
, &orlist
, ctx
);
11263 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11264 switch (OMP_CLAUSE_CODE (c
))
11270 case OMP_CLAUSE_MAP
:
11271 #ifdef ENABLE_CHECKING
11272 /* First check what we're prepared to handle in the following. */
11273 switch (OMP_CLAUSE_MAP_KIND (c
))
11275 case GOMP_MAP_ALLOC
:
11277 case GOMP_MAP_FROM
:
11278 case GOMP_MAP_TOFROM
:
11279 case GOMP_MAP_POINTER
:
11280 case GOMP_MAP_TO_PSET
:
11282 case GOMP_MAP_FORCE_ALLOC
:
11283 case GOMP_MAP_FORCE_TO
:
11284 case GOMP_MAP_FORCE_FROM
:
11285 case GOMP_MAP_FORCE_TOFROM
:
11286 case GOMP_MAP_FORCE_PRESENT
:
11287 case GOMP_MAP_FORCE_DEALLOC
:
11288 case GOMP_MAP_FORCE_DEVICEPTR
:
11289 gcc_assert (is_gimple_omp_oacc (stmt
));
11292 gcc_unreachable ();
11296 case OMP_CLAUSE_TO
:
11297 case OMP_CLAUSE_FROM
:
11298 var
= OMP_CLAUSE_DECL (c
);
11301 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
11302 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11307 if (DECL_SIZE (var
)
11308 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
11310 tree var2
= DECL_VALUE_EXPR (var
);
11311 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
11312 var2
= TREE_OPERAND (var2
, 0);
11313 gcc_assert (DECL_P (var2
));
11317 if (!maybe_lookup_field (var
, ctx
))
11322 x
= build_receiver_ref (var
, true, ctx
);
11323 tree new_var
= lookup_decl (var
, ctx
);
11324 if (OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
11325 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11326 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
11327 x
= build_simple_mem_ref (x
);
11328 SET_DECL_VALUE_EXPR (new_var
, x
);
11329 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
11336 target_nesting_level
++;
11337 lower_omp (&tgt_body
, ctx
);
11338 target_nesting_level
--;
11340 else if (data_region
)
11341 lower_omp (&tgt_body
, ctx
);
11345 /* Declare all the variables created by mapping and the variables
11346 declared in the scope of the target body. */
11347 record_vars_into (ctx
->block_vars
, child_fn
);
11348 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
11353 if (ctx
->record_type
)
11356 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
11357 DECL_NAMELESS (ctx
->sender_decl
) = 1;
11358 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
11359 t
= make_tree_vec (3);
11360 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
11361 TREE_VEC_ELT (t
, 1)
11362 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
11363 ".omp_data_sizes");
11364 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
11365 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
11366 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
11369 if (is_gimple_omp_oacc (stmt
))
11371 tkind_type
= short_unsigned_type_node
;
11376 tkind_type
= unsigned_char_type_node
;
11379 TREE_VEC_ELT (t
, 2)
11380 = create_tmp_var (build_array_type_nelts (tkind_type
, map_cnt
),
11381 ".omp_data_kinds");
11382 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
11383 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
11384 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
11385 gimple_omp_target_set_data_arg (stmt
, t
);
11387 vec
<constructor_elt
, va_gc
> *vsize
;
11388 vec
<constructor_elt
, va_gc
> *vkind
;
11389 vec_alloc (vsize
, map_cnt
);
11390 vec_alloc (vkind
, map_cnt
);
11391 unsigned int map_idx
= 0;
11393 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
11394 switch (OMP_CLAUSE_CODE (c
))
11400 case OMP_CLAUSE_MAP
:
11401 case OMP_CLAUSE_TO
:
11402 case OMP_CLAUSE_FROM
:
11404 ovar
= OMP_CLAUSE_DECL (c
);
11405 if (!DECL_P (ovar
))
11407 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11408 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
11410 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
11411 == get_base_address (ovar
));
11412 nc
= OMP_CLAUSE_CHAIN (c
);
11413 ovar
= OMP_CLAUSE_DECL (nc
);
11417 tree x
= build_sender_ref (ovar
, ctx
);
11419 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
11420 gimplify_assign (x
, v
, &ilist
);
11426 if (DECL_SIZE (ovar
)
11427 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
11429 tree ovar2
= DECL_VALUE_EXPR (ovar
);
11430 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
11431 ovar2
= TREE_OPERAND (ovar2
, 0);
11432 gcc_assert (DECL_P (ovar2
));
11435 if (!maybe_lookup_field (ovar
, ctx
))
11439 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
11440 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
11441 talign
= DECL_ALIGN_UNIT (ovar
);
11444 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
11445 tree x
= build_sender_ref (ovar
, ctx
);
11446 if (maybe_lookup_oacc_reduction (var
, ctx
))
11448 gcc_checking_assert (offloaded
11449 && is_gimple_omp_oacc (stmt
));
11450 gimplify_assign (x
, var
, &ilist
);
11452 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
11453 && OMP_CLAUSE_MAP_KIND (c
) == GOMP_MAP_POINTER
11454 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
11455 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
11457 gcc_assert (offloaded
);
11459 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)));
11460 mark_addressable (avar
);
11461 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
11462 talign
= DECL_ALIGN_UNIT (avar
);
11463 avar
= build_fold_addr_expr (avar
);
11464 gimplify_assign (x
, avar
, &ilist
);
11466 else if (is_gimple_reg (var
))
11468 gcc_assert (offloaded
);
11469 tree avar
= create_tmp_var (TREE_TYPE (var
));
11470 mark_addressable (avar
);
11471 enum gomp_map_kind map_kind
= OMP_CLAUSE_MAP_KIND (c
);
11472 if (GOMP_MAP_COPY_TO_P (map_kind
)
11473 || map_kind
== GOMP_MAP_POINTER
11474 || map_kind
== GOMP_MAP_TO_PSET
11475 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
11476 gimplify_assign (avar
, var
, &ilist
);
11477 avar
= build_fold_addr_expr (avar
);
11478 gimplify_assign (x
, avar
, &ilist
);
11479 if ((GOMP_MAP_COPY_FROM_P (map_kind
)
11480 || map_kind
== GOMP_MAP_FORCE_DEVICEPTR
)
11481 && !TYPE_READONLY (TREE_TYPE (var
)))
11483 x
= build_sender_ref (ovar
, ctx
);
11484 x
= build_simple_mem_ref (x
);
11485 gimplify_assign (var
, x
, &olist
);
11490 var
= build_fold_addr_expr (var
);
11491 gimplify_assign (x
, var
, &ilist
);
11494 tree s
= OMP_CLAUSE_SIZE (c
);
11495 if (s
== NULL_TREE
)
11496 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
11497 s
= fold_convert (size_type_node
, s
);
11498 tree purpose
= size_int (map_idx
++);
11499 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
11500 if (TREE_CODE (s
) != INTEGER_CST
)
11501 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
11503 unsigned HOST_WIDE_INT tkind
;
11504 switch (OMP_CLAUSE_CODE (c
))
11506 case OMP_CLAUSE_MAP
:
11507 tkind
= OMP_CLAUSE_MAP_KIND (c
);
11509 case OMP_CLAUSE_TO
:
11510 tkind
= GOMP_MAP_TO
;
11512 case OMP_CLAUSE_FROM
:
11513 tkind
= GOMP_MAP_FROM
;
11516 gcc_unreachable ();
11518 gcc_checking_assert (tkind
11519 < (HOST_WIDE_INT_C (1U) << talign_shift
));
11520 talign
= ceil_log2 (talign
);
11521 tkind
|= talign
<< talign_shift
;
11522 gcc_checking_assert (tkind
11523 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type
)));
11524 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
11525 build_int_cstu (tkind_type
, tkind
));
11530 gcc_assert (map_idx
== map_cnt
);
11532 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
11533 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
11534 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
11535 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
11536 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
11538 gimple_seq initlist
= NULL
;
11539 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
11540 TREE_VEC_ELT (t
, 1)),
11541 &initlist
, true, NULL_TREE
);
11542 gimple_seq_add_seq (&ilist
, initlist
);
11544 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
11546 TREE_THIS_VOLATILE (clobber
) = 1;
11547 gimple_seq_add_stmt (&olist
,
11548 gimple_build_assign (TREE_VEC_ELT (t
, 1),
11552 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
11553 TREE_THIS_VOLATILE (clobber
) = 1;
11554 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
11558 /* Once all the expansions are done, sequence all the different
11559 fragments inside gimple_omp_body. */
11564 && ctx
->record_type
)
11566 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
11567 /* fixup_child_record_type might have changed receiver_decl's type. */
11568 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
11569 gimple_seq_add_stmt (&new_body
,
11570 gimple_build_assign (ctx
->receiver_decl
, t
));
11575 gimple_seq_add_seq (&new_body
, tgt_body
);
11576 new_body
= maybe_catch_exception (new_body
);
11578 else if (data_region
)
11579 new_body
= tgt_body
;
11580 if (offloaded
|| data_region
)
11582 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
11583 gimple_omp_set_body (stmt
, new_body
);
11586 bind
= gimple_build_bind (NULL
, NULL
,
11587 tgt_bind
? gimple_bind_block (tgt_bind
)
11589 gsi_replace (gsi_p
, bind
, true);
11590 gimple_bind_add_seq (bind
, irlist
);
11591 gimple_bind_add_seq (bind
, ilist
);
11592 gimple_bind_add_stmt (bind
, stmt
);
11593 gimple_bind_add_seq (bind
, olist
);
11594 gimple_bind_add_seq (bind
, orlist
);
11596 pop_gimplify_context (NULL
);
11599 /* Expand code for an OpenMP teams directive. */
11602 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11604 gomp_teams
*teams_stmt
= as_a
<gomp_teams
*> (gsi_stmt (*gsi_p
));
11605 push_gimplify_context ();
11607 tree block
= make_node (BLOCK
);
11608 gbind
*bind
= gimple_build_bind (NULL
, NULL
, block
);
11609 gsi_replace (gsi_p
, bind
, true);
11610 gimple_seq bind_body
= NULL
;
11611 gimple_seq dlist
= NULL
;
11612 gimple_seq olist
= NULL
;
11614 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11615 OMP_CLAUSE_NUM_TEAMS
);
11616 if (num_teams
== NULL_TREE
)
11617 num_teams
= build_int_cst (unsigned_type_node
, 0);
11620 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
11621 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
11622 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
11624 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
11625 OMP_CLAUSE_THREAD_LIMIT
);
11626 if (thread_limit
== NULL_TREE
)
11627 thread_limit
= build_int_cst (unsigned_type_node
, 0);
11630 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
11631 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
11632 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
11636 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
11637 &bind_body
, &dlist
, ctx
, NULL
);
11638 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
11639 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
11640 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
11642 location_t loc
= gimple_location (teams_stmt
);
11643 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
11644 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
11645 gimple_set_location (call
, loc
);
11646 gimple_seq_add_stmt (&bind_body
, call
);
11648 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
11649 gimple_omp_set_body (teams_stmt
, NULL
);
11650 gimple_seq_add_seq (&bind_body
, olist
);
11651 gimple_seq_add_seq (&bind_body
, dlist
);
11652 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
11653 gimple_bind_set_body (bind
, bind_body
);
11655 pop_gimplify_context (bind
);
11657 gimple_bind_append_vars (bind
, ctx
->block_vars
);
11658 BLOCK_VARS (block
) = ctx
->block_vars
;
11659 if (BLOCK_VARS (block
))
11660 TREE_USED (block
) = 1;
11664 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
11665 regimplified. If DATA is non-NULL, lower_omp_1 is outside
11666 of OMP context, but with task_shared_vars set. */
11669 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
11674 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
11675 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
11678 if (task_shared_vars
11680 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
11683 /* If a global variable has been privatized, TREE_CONSTANT on
11684 ADDR_EXPR might be wrong. */
11685 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
11686 recompute_tree_invariant_for_addr_expr (t
);
11688 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
11693 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
11695 gimple stmt
= gsi_stmt (*gsi_p
);
11696 struct walk_stmt_info wi
;
11699 if (gimple_has_location (stmt
))
11700 input_location
= gimple_location (stmt
);
11702 if (task_shared_vars
)
11703 memset (&wi
, '\0', sizeof (wi
));
11705 /* If we have issued syntax errors, avoid doing any heavy lifting.
11706 Just replace the OMP directives with a NOP to avoid
11707 confusing RTL expansion. */
11708 if (seen_error () && is_gimple_omp (stmt
))
11710 gsi_replace (gsi_p
, gimple_build_nop (), true);
11714 switch (gimple_code (stmt
))
11718 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
11719 if ((ctx
|| task_shared_vars
)
11720 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt
),
11721 lower_omp_regimplify_p
,
11722 ctx
? NULL
: &wi
, NULL
)
11723 || walk_tree (gimple_cond_rhs_ptr (cond_stmt
),
11724 lower_omp_regimplify_p
,
11725 ctx
? NULL
: &wi
, NULL
)))
11726 gimple_regimplify_operands (cond_stmt
, gsi_p
);
11730 lower_omp (gimple_catch_handler_ptr (as_a
<gcatch
*> (stmt
)), ctx
);
11732 case GIMPLE_EH_FILTER
:
11733 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
11736 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
11737 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
11739 case GIMPLE_TRANSACTION
:
11740 lower_omp (gimple_transaction_body_ptr (
11741 as_a
<gtransaction
*> (stmt
)),
11745 lower_omp (gimple_bind_body_ptr (as_a
<gbind
*> (stmt
)), ctx
);
11747 case GIMPLE_OMP_PARALLEL
:
11748 case GIMPLE_OMP_TASK
:
11749 ctx
= maybe_lookup_ctx (stmt
);
11751 if (ctx
->cancellable
)
11752 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11753 lower_omp_taskreg (gsi_p
, ctx
);
11755 case GIMPLE_OMP_FOR
:
11756 ctx
= maybe_lookup_ctx (stmt
);
11758 if (ctx
->cancellable
)
11759 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11760 lower_omp_for (gsi_p
, ctx
);
11762 case GIMPLE_OMP_SECTIONS
:
11763 ctx
= maybe_lookup_ctx (stmt
);
11765 if (ctx
->cancellable
)
11766 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
11767 lower_omp_sections (gsi_p
, ctx
);
11769 case GIMPLE_OMP_SINGLE
:
11770 ctx
= maybe_lookup_ctx (stmt
);
11772 lower_omp_single (gsi_p
, ctx
);
11774 case GIMPLE_OMP_MASTER
:
11775 ctx
= maybe_lookup_ctx (stmt
);
11777 lower_omp_master (gsi_p
, ctx
);
11779 case GIMPLE_OMP_TASKGROUP
:
11780 ctx
= maybe_lookup_ctx (stmt
);
11782 lower_omp_taskgroup (gsi_p
, ctx
);
11784 case GIMPLE_OMP_ORDERED
:
11785 ctx
= maybe_lookup_ctx (stmt
);
11787 lower_omp_ordered (gsi_p
, ctx
);
11789 case GIMPLE_OMP_CRITICAL
:
11790 ctx
= maybe_lookup_ctx (stmt
);
11792 lower_omp_critical (gsi_p
, ctx
);
11794 case GIMPLE_OMP_ATOMIC_LOAD
:
11795 if ((ctx
|| task_shared_vars
)
11796 && walk_tree (gimple_omp_atomic_load_rhs_ptr (
11797 as_a
<gomp_atomic_load
*> (stmt
)),
11798 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
11799 gimple_regimplify_operands (stmt
, gsi_p
);
11801 case GIMPLE_OMP_TARGET
:
11802 ctx
= maybe_lookup_ctx (stmt
);
11804 lower_omp_target (gsi_p
, ctx
);
11806 case GIMPLE_OMP_TEAMS
:
11807 ctx
= maybe_lookup_ctx (stmt
);
11809 lower_omp_teams (gsi_p
, ctx
);
11813 call_stmt
= as_a
<gcall
*> (stmt
);
11814 fndecl
= gimple_call_fndecl (call_stmt
);
11816 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
11817 switch (DECL_FUNCTION_CODE (fndecl
))
11819 case BUILT_IN_GOMP_BARRIER
:
11823 case BUILT_IN_GOMP_CANCEL
:
11824 case BUILT_IN_GOMP_CANCELLATION_POINT
:
11827 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
11828 cctx
= cctx
->outer
;
11829 gcc_assert (gimple_call_lhs (call_stmt
) == NULL_TREE
);
11830 if (!cctx
->cancellable
)
11832 if (DECL_FUNCTION_CODE (fndecl
)
11833 == BUILT_IN_GOMP_CANCELLATION_POINT
)
11835 stmt
= gimple_build_nop ();
11836 gsi_replace (gsi_p
, stmt
, false);
11840 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
11842 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
11843 gimple_call_set_fndecl (call_stmt
, fndecl
);
11844 gimple_call_set_fntype (call_stmt
, TREE_TYPE (fndecl
));
11847 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)));
11848 gimple_call_set_lhs (call_stmt
, lhs
);
11849 tree fallthru_label
;
11850 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
11852 g
= gimple_build_label (fallthru_label
);
11853 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11854 g
= gimple_build_cond (NE_EXPR
, lhs
,
11855 fold_convert (TREE_TYPE (lhs
),
11856 boolean_false_node
),
11857 cctx
->cancel_label
, fallthru_label
);
11858 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
11865 if ((ctx
|| task_shared_vars
)
11866 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
11869 /* Just remove clobbers, this should happen only if we have
11870 "privatized" local addressable variables in SIMD regions,
11871 the clobber isn't needed in that case and gimplifying address
11872 of the ARRAY_REF into a pointer and creating MEM_REF based
11873 clobber would create worse code than we get with the clobber
11875 if (gimple_clobber_p (stmt
))
11877 gsi_replace (gsi_p
, gimple_build_nop (), true);
11880 gimple_regimplify_operands (stmt
, gsi_p
);
11887 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
11889 location_t saved_location
= input_location
;
11890 gimple_stmt_iterator gsi
;
11891 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11892 lower_omp_1 (&gsi
, ctx
);
11893 /* During gimplification, we haven't folded statments inside offloading
11894 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */
11895 if (target_nesting_level
|| taskreg_nesting_level
)
11896 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
11898 input_location
= saved_location
;
11901 /* Main entry point. */
11903 static unsigned int
11904 execute_lower_omp (void)
11910 /* This pass always runs, to provide PROP_gimple_lomp.
11911 But often, there is nothing to do. */
11912 if (flag_cilkplus
== 0 && flag_openacc
== 0 && flag_openmp
== 0
11913 && flag_openmp_simd
== 0)
11916 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
11917 delete_omp_context
);
11919 body
= gimple_body (current_function_decl
);
11920 scan_omp (&body
, NULL
);
11921 gcc_assert (taskreg_nesting_level
== 0);
11922 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
11923 finish_taskreg_scan (ctx
);
11924 taskreg_contexts
.release ();
11926 if (all_contexts
->root
)
11928 if (task_shared_vars
)
11929 push_gimplify_context ();
11930 lower_omp (&body
, NULL
);
11931 if (task_shared_vars
)
11932 pop_gimplify_context (NULL
);
11937 splay_tree_delete (all_contexts
);
11938 all_contexts
= NULL
;
11940 BITMAP_FREE (task_shared_vars
);
11946 const pass_data pass_data_lower_omp
=
11948 GIMPLE_PASS
, /* type */
11949 "omplower", /* name */
11950 OPTGROUP_NONE
, /* optinfo_flags */
11951 TV_NONE
, /* tv_id */
11952 PROP_gimple_any
, /* properties_required */
11953 PROP_gimple_lomp
, /* properties_provided */
11954 0, /* properties_destroyed */
11955 0, /* todo_flags_start */
11956 0, /* todo_flags_finish */
11959 class pass_lower_omp
: public gimple_opt_pass
11962 pass_lower_omp (gcc::context
*ctxt
)
11963 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
11966 /* opt_pass methods: */
11967 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
11969 }; // class pass_lower_omp
11971 } // anon namespace
11974 make_pass_lower_omp (gcc::context
*ctxt
)
11976 return new pass_lower_omp (ctxt
);
11979 /* The following is a utility to diagnose structured block violations.
11980 It is not part of the "omplower" pass, as that's invoked too late. It
11981 should be invoked by the respective front ends after gimplification. */
11983 static splay_tree all_labels
;
11985 /* Check for mismatched contexts and generate an error if needed. Return
11986 true if an error is detected. */
11989 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
11990 gimple branch_ctx
, gimple label_ctx
)
11992 gcc_checking_assert (!branch_ctx
|| is_gimple_omp (branch_ctx
));
11993 gcc_checking_assert (!label_ctx
|| is_gimple_omp (label_ctx
));
11995 if (label_ctx
== branch_ctx
)
11998 const char* kind
= NULL
;
12003 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
12004 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
12006 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
12007 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
12008 kind
= "Cilk Plus";
12012 if ((branch_ctx
&& is_gimple_omp_oacc (branch_ctx
))
12013 || (label_ctx
&& is_gimple_omp_oacc (label_ctx
)))
12015 gcc_checking_assert (kind
== NULL
);
12021 gcc_checking_assert (flag_openmp
);
12026 Previously we kept track of the label's entire context in diagnose_sb_[12]
12027 so we could traverse it and issue a correct "exit" or "enter" error
12028 message upon a structured block violation.
12030 We built the context by building a list with tree_cons'ing, but there is
12031 no easy counterpart in gimple tuples. It seems like far too much work
12032 for issuing exit/enter error messages. If someone really misses the
12033 distinct error message... patches welcome.
12037 /* Try to avoid confusing the user by producing and error message
12038 with correct "exit" or "enter" verbiage. We prefer "exit"
12039 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
12040 if (branch_ctx
== NULL
)
12046 if (TREE_VALUE (label_ctx
) == branch_ctx
)
12051 label_ctx
= TREE_CHAIN (label_ctx
);
12056 error ("invalid exit from %s structured block", kind
);
12058 error ("invalid entry to %s structured block", kind
);
12061 /* If it's obvious we have an invalid entry, be specific about the error. */
12062 if (branch_ctx
== NULL
)
12063 error ("invalid entry to %s structured block", kind
);
12066 /* Otherwise, be vague and lazy, but efficient. */
12067 error ("invalid branch to/from %s structured block", kind
);
12070 gsi_replace (gsi_p
, gimple_build_nop (), false);
12074 /* Pass 1: Create a minimal tree of structured blocks, and record
12075 where each label is found. */
12078 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12079 struct walk_stmt_info
*wi
)
12081 gimple context
= (gimple
) wi
->info
;
12082 gimple inner_context
;
12083 gimple stmt
= gsi_stmt (*gsi_p
);
12085 *handled_ops_p
= true;
12087 switch (gimple_code (stmt
))
12091 case GIMPLE_OMP_PARALLEL
:
12092 case GIMPLE_OMP_TASK
:
12093 case GIMPLE_OMP_SECTIONS
:
12094 case GIMPLE_OMP_SINGLE
:
12095 case GIMPLE_OMP_SECTION
:
12096 case GIMPLE_OMP_MASTER
:
12097 case GIMPLE_OMP_ORDERED
:
12098 case GIMPLE_OMP_CRITICAL
:
12099 case GIMPLE_OMP_TARGET
:
12100 case GIMPLE_OMP_TEAMS
:
12101 case GIMPLE_OMP_TASKGROUP
:
12102 /* The minimal context here is just the current OMP construct. */
12103 inner_context
= stmt
;
12104 wi
->info
= inner_context
;
12105 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12106 wi
->info
= context
;
12109 case GIMPLE_OMP_FOR
:
12110 inner_context
= stmt
;
12111 wi
->info
= inner_context
;
12112 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12114 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
12115 diagnose_sb_1
, NULL
, wi
);
12116 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
12117 wi
->info
= context
;
12121 splay_tree_insert (all_labels
,
12122 (splay_tree_key
) gimple_label_label (
12123 as_a
<glabel
*> (stmt
)),
12124 (splay_tree_value
) context
);
12134 /* Pass 2: Check each branch and see if its context differs from that of
12135 the destination label's context. */
12138 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
12139 struct walk_stmt_info
*wi
)
12141 gimple context
= (gimple
) wi
->info
;
12143 gimple stmt
= gsi_stmt (*gsi_p
);
12145 *handled_ops_p
= true;
12147 switch (gimple_code (stmt
))
12151 case GIMPLE_OMP_PARALLEL
:
12152 case GIMPLE_OMP_TASK
:
12153 case GIMPLE_OMP_SECTIONS
:
12154 case GIMPLE_OMP_SINGLE
:
12155 case GIMPLE_OMP_SECTION
:
12156 case GIMPLE_OMP_MASTER
:
12157 case GIMPLE_OMP_ORDERED
:
12158 case GIMPLE_OMP_CRITICAL
:
12159 case GIMPLE_OMP_TARGET
:
12160 case GIMPLE_OMP_TEAMS
:
12161 case GIMPLE_OMP_TASKGROUP
:
12163 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12164 wi
->info
= context
;
12167 case GIMPLE_OMP_FOR
:
12169 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
12171 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
12172 diagnose_sb_2
, NULL
, wi
);
12173 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
12174 wi
->info
= context
;
12179 gcond
*cond_stmt
= as_a
<gcond
*> (stmt
);
12180 tree lab
= gimple_cond_true_label (cond_stmt
);
12183 n
= splay_tree_lookup (all_labels
,
12184 (splay_tree_key
) lab
);
12185 diagnose_sb_0 (gsi_p
, context
,
12186 n
? (gimple
) n
->value
: NULL
);
12188 lab
= gimple_cond_false_label (cond_stmt
);
12191 n
= splay_tree_lookup (all_labels
,
12192 (splay_tree_key
) lab
);
12193 diagnose_sb_0 (gsi_p
, context
,
12194 n
? (gimple
) n
->value
: NULL
);
12201 tree lab
= gimple_goto_dest (stmt
);
12202 if (TREE_CODE (lab
) != LABEL_DECL
)
12205 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12206 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
12210 case GIMPLE_SWITCH
:
12212 gswitch
*switch_stmt
= as_a
<gswitch
*> (stmt
);
12214 for (i
= 0; i
< gimple_switch_num_labels (switch_stmt
); ++i
)
12216 tree lab
= CASE_LABEL (gimple_switch_label (switch_stmt
, i
));
12217 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
12218 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
12224 case GIMPLE_RETURN
:
12225 diagnose_sb_0 (gsi_p
, context
, NULL
);
12235 /* Called from tree-cfg.c::make_edges to create cfg edges for all relevant
12238 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
12241 gimple last
= last_stmt (bb
);
12242 enum gimple_code code
= gimple_code (last
);
12243 struct omp_region
*cur_region
= *region
;
12244 bool fallthru
= false;
12248 case GIMPLE_OMP_PARALLEL
:
12249 case GIMPLE_OMP_TASK
:
12250 case GIMPLE_OMP_FOR
:
12251 case GIMPLE_OMP_SINGLE
:
12252 case GIMPLE_OMP_TEAMS
:
12253 case GIMPLE_OMP_MASTER
:
12254 case GIMPLE_OMP_TASKGROUP
:
12255 case GIMPLE_OMP_ORDERED
:
12256 case GIMPLE_OMP_CRITICAL
:
12257 case GIMPLE_OMP_SECTION
:
12258 cur_region
= new_omp_region (bb
, code
, cur_region
);
12262 case GIMPLE_OMP_TARGET
:
12263 cur_region
= new_omp_region (bb
, code
, cur_region
);
12265 switch (gimple_omp_target_kind (last
))
12267 case GF_OMP_TARGET_KIND_REGION
:
12268 case GF_OMP_TARGET_KIND_DATA
:
12269 case GF_OMP_TARGET_KIND_OACC_PARALLEL
:
12270 case GF_OMP_TARGET_KIND_OACC_KERNELS
:
12271 case GF_OMP_TARGET_KIND_OACC_DATA
:
12273 case GF_OMP_TARGET_KIND_UPDATE
:
12274 case GF_OMP_TARGET_KIND_OACC_UPDATE
:
12275 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA
:
12276 cur_region
= cur_region
->outer
;
12279 gcc_unreachable ();
12283 case GIMPLE_OMP_SECTIONS
:
12284 cur_region
= new_omp_region (bb
, code
, cur_region
);
12288 case GIMPLE_OMP_SECTIONS_SWITCH
:
12292 case GIMPLE_OMP_ATOMIC_LOAD
:
12293 case GIMPLE_OMP_ATOMIC_STORE
:
12297 case GIMPLE_OMP_RETURN
:
12298 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
12299 somewhere other than the next block. This will be
12301 cur_region
->exit
= bb
;
12302 if (cur_region
->type
== GIMPLE_OMP_TASK
)
12303 /* Add an edge corresponding to not scheduling the task
12305 make_edge (cur_region
->entry
, bb
, EDGE_ABNORMAL
);
12306 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
12307 cur_region
= cur_region
->outer
;
12310 case GIMPLE_OMP_CONTINUE
:
12311 cur_region
->cont
= bb
;
12312 switch (cur_region
->type
)
12314 case GIMPLE_OMP_FOR
:
12315 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
12316 succs edges as abnormal to prevent splitting
12318 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
12319 /* Make the loopback edge. */
12320 make_edge (bb
, single_succ (cur_region
->entry
),
12323 /* Create an edge from GIMPLE_OMP_FOR to exit, which
12324 corresponds to the case that the body of the loop
12325 is not executed at all. */
12326 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
12327 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
12331 case GIMPLE_OMP_SECTIONS
:
12332 /* Wire up the edges into and out of the nested sections. */
12334 basic_block switch_bb
= single_succ (cur_region
->entry
);
12336 struct omp_region
*i
;
12337 for (i
= cur_region
->inner
; i
; i
= i
->next
)
12339 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
12340 make_edge (switch_bb
, i
->entry
, 0);
12341 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
12344 /* Make the loopback edge to the block with
12345 GIMPLE_OMP_SECTIONS_SWITCH. */
12346 make_edge (bb
, switch_bb
, 0);
12348 /* Make the edge from the switch to exit. */
12349 make_edge (switch_bb
, bb
->next_bb
, 0);
12354 case GIMPLE_OMP_TASK
:
12359 gcc_unreachable ();
12364 gcc_unreachable ();
12367 if (*region
!= cur_region
)
12369 *region
= cur_region
;
12371 *region_idx
= cur_region
->entry
->index
;
12379 static unsigned int
12380 diagnose_omp_structured_block_errors (void)
12382 struct walk_stmt_info wi
;
12383 gimple_seq body
= gimple_body (current_function_decl
);
12385 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
12387 memset (&wi
, 0, sizeof (wi
));
12388 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
12390 memset (&wi
, 0, sizeof (wi
));
12391 wi
.want_locations
= true;
12392 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
12394 gimple_set_body (current_function_decl
, body
);
12396 splay_tree_delete (all_labels
);
12404 const pass_data pass_data_diagnose_omp_blocks
=
12406 GIMPLE_PASS
, /* type */
12407 "*diagnose_omp_blocks", /* name */
12408 OPTGROUP_NONE
, /* optinfo_flags */
12409 TV_NONE
, /* tv_id */
12410 PROP_gimple_any
, /* properties_required */
12411 0, /* properties_provided */
12412 0, /* properties_destroyed */
12413 0, /* todo_flags_start */
12414 0, /* todo_flags_finish */
12417 class pass_diagnose_omp_blocks
: public gimple_opt_pass
12420 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12421 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
12424 /* opt_pass methods: */
12425 virtual bool gate (function
*)
12427 return flag_cilkplus
|| flag_openacc
|| flag_openmp
;
12429 virtual unsigned int execute (function
*)
12431 return diagnose_omp_structured_block_errors ();
12434 }; // class pass_diagnose_omp_blocks
12436 } // anon namespace
12439 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
12441 return new pass_diagnose_omp_blocks (ctxt
);
12444 /* SIMD clone supporting code. */
12446 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
12447 of arguments to reserve space for. */
12449 static struct cgraph_simd_clone
*
12450 simd_clone_struct_alloc (int nargs
)
12452 struct cgraph_simd_clone
*clone_info
;
12453 size_t len
= (sizeof (struct cgraph_simd_clone
)
12454 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
12455 clone_info
= (struct cgraph_simd_clone
*)
12456 ggc_internal_cleared_alloc (len
);
12460 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
12463 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
12464 struct cgraph_simd_clone
*from
)
12466 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
12467 + ((from
->nargs
- from
->inbranch
)
12468 * sizeof (struct cgraph_simd_clone_arg
))));
12471 /* Return vector of parameter types of function FNDECL. This uses
12472 TYPE_ARG_TYPES if available, otherwise falls back to types of
12473 DECL_ARGUMENTS types. */
12476 simd_clone_vector_of_formal_parm_types (tree fndecl
)
12478 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
12479 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
12480 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
12483 FOR_EACH_VEC_ELT (args
, i
, arg
)
12484 args
[i
] = TREE_TYPE (args
[i
]);
12488 /* Given a simd function in NODE, extract the simd specific
12489 information from the OMP clauses passed in CLAUSES, and return
12490 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
12491 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
12492 otherwise set to FALSE. */
12494 static struct cgraph_simd_clone
*
12495 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
12496 bool *inbranch_specified
)
12498 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12501 *inbranch_specified
= false;
12503 n
= args
.length ();
12504 if (n
> 0 && args
.last () == void_type_node
)
12507 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
12508 be cloned have a distinctive artificial label in addition to "omp
12512 && lookup_attribute ("cilk simd function",
12513 DECL_ATTRIBUTES (node
->decl
)));
12515 /* Allocate one more than needed just in case this is an in-branch
12516 clone which will require a mask argument. */
12517 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
12518 clone_info
->nargs
= n
;
12519 clone_info
->cilk_elemental
= cilk_clone
;
12526 clauses
= TREE_VALUE (clauses
);
12527 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
12530 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
12532 switch (OMP_CLAUSE_CODE (t
))
12534 case OMP_CLAUSE_INBRANCH
:
12535 clone_info
->inbranch
= 1;
12536 *inbranch_specified
= true;
12538 case OMP_CLAUSE_NOTINBRANCH
:
12539 clone_info
->inbranch
= 0;
12540 *inbranch_specified
= true;
12542 case OMP_CLAUSE_SIMDLEN
:
12543 clone_info
->simdlen
12544 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
12546 case OMP_CLAUSE_LINEAR
:
12548 tree decl
= OMP_CLAUSE_DECL (t
);
12549 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
12550 int argno
= TREE_INT_CST_LOW (decl
);
12551 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
12553 clone_info
->args
[argno
].arg_type
12554 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
12555 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12556 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
12557 && clone_info
->args
[argno
].linear_step
< n
);
12561 if (POINTER_TYPE_P (args
[argno
]))
12562 step
= fold_convert (ssizetype
, step
);
12563 if (!tree_fits_shwi_p (step
))
12565 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12566 "ignoring large linear step");
12570 else if (integer_zerop (step
))
12572 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
12573 "ignoring zero linear step");
12579 clone_info
->args
[argno
].arg_type
12580 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
12581 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
12586 case OMP_CLAUSE_UNIFORM
:
12588 tree decl
= OMP_CLAUSE_DECL (t
);
12589 int argno
= tree_to_uhwi (decl
);
12590 clone_info
->args
[argno
].arg_type
12591 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
12594 case OMP_CLAUSE_ALIGNED
:
12596 tree decl
= OMP_CLAUSE_DECL (t
);
12597 int argno
= tree_to_uhwi (decl
);
12598 clone_info
->args
[argno
].alignment
12599 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
12610 /* Given a SIMD clone in NODE, calculate the characteristic data
12611 type and return the coresponding type. The characteristic data
12612 type is computed as described in the Intel Vector ABI. */
12615 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
12616 struct cgraph_simd_clone
*clone_info
)
12618 tree type
= integer_type_node
;
12619 tree fndecl
= node
->decl
;
12621 /* a) For non-void function, the characteristic data type is the
12623 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
12624 type
= TREE_TYPE (TREE_TYPE (fndecl
));
12626 /* b) If the function has any non-uniform, non-linear parameters,
12627 then the characteristic data type is the type of the first
12631 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
12632 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
12633 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
12641 /* c) If the characteristic data type determined by a) or b) above
12642 is struct, union, or class type which is pass-by-value (except
12643 for the type that maps to the built-in complex data type), the
12644 characteristic data type is int. */
12645 if (RECORD_OR_UNION_TYPE_P (type
)
12646 && !aggregate_value_p (type
, NULL
)
12647 && TREE_CODE (type
) != COMPLEX_TYPE
)
12648 return integer_type_node
;
12650 /* d) If none of the above three classes is applicable, the
12651 characteristic data type is int. */
12655 /* e) For Intel Xeon Phi native and offload compilation, if the
12656 resulting characteristic data type is 8-bit or 16-bit integer
12657 data type, the characteristic data type is int. */
12658 /* Well, we don't handle Xeon Phi yet. */
12662 simd_clone_mangle (struct cgraph_node
*node
,
12663 struct cgraph_simd_clone
*clone_info
)
12665 char vecsize_mangle
= clone_info
->vecsize_mangle
;
12666 char mask
= clone_info
->inbranch
? 'M' : 'N';
12667 unsigned int simdlen
= clone_info
->simdlen
;
12671 gcc_assert (vecsize_mangle
&& simdlen
);
12673 pp_string (&pp
, "_ZGV");
12674 pp_character (&pp
, vecsize_mangle
);
12675 pp_character (&pp
, mask
);
12676 pp_decimal_int (&pp
, simdlen
);
12678 for (n
= 0; n
< clone_info
->nargs
; ++n
)
12680 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
12682 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
12683 pp_character (&pp
, 'u');
12684 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
12686 gcc_assert (arg
.linear_step
!= 0);
12687 pp_character (&pp
, 'l');
12688 if (arg
.linear_step
> 1)
12689 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12690 else if (arg
.linear_step
< 0)
12692 pp_character (&pp
, 'n');
12693 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
12697 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
12699 pp_character (&pp
, 's');
12700 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
12703 pp_character (&pp
, 'v');
12706 pp_character (&pp
, 'a');
12707 pp_decimal_int (&pp
, arg
.alignment
);
12711 pp_underscore (&pp
);
12712 const char *str
= IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
));
12715 pp_string (&pp
, str
);
12716 str
= pp_formatted_text (&pp
);
12718 /* If there already is a SIMD clone with the same mangled name, don't
12719 add another one. This can happen e.g. for
12720 #pragma omp declare simd
12721 #pragma omp declare simd simdlen(8)
12722 int foo (int, int);
12723 if the simdlen is assumed to be 8 for the first one, etc. */
12724 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
12725 clone
= clone
->simdclone
->next_clone
)
12726 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
12730 return get_identifier (str
);
12733 /* Create a simd clone of OLD_NODE and return it. */
12735 static struct cgraph_node
*
12736 simd_clone_create (struct cgraph_node
*old_node
)
12738 struct cgraph_node
*new_node
;
12739 if (old_node
->definition
)
12741 if (!old_node
->has_gimple_body_p ())
12743 old_node
->get_body ();
12744 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
12750 tree old_decl
= old_node
->decl
;
12751 tree new_decl
= copy_node (old_node
->decl
);
12752 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
12753 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
12754 SET_DECL_RTL (new_decl
, NULL
);
12755 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
12756 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
12757 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
12758 symtab
->call_cgraph_insertion_hooks (new_node
);
12760 if (new_node
== NULL
)
12763 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
12765 /* The function cgraph_function_versioning () will force the new
12766 symbol local. Undo this, and inherit external visability from
12768 new_node
->local
.local
= old_node
->local
.local
;
12769 new_node
->externally_visible
= old_node
->externally_visible
;
12774 /* Adjust the return type of the given function to its appropriate
12775 vector counterpart. Returns a simd array to be used throughout the
12776 function as a return value. */
12779 simd_clone_adjust_return_type (struct cgraph_node
*node
)
12781 tree fndecl
= node
->decl
;
12782 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
12783 unsigned int veclen
;
12786 /* Adjust the function return type. */
12787 if (orig_rettype
== void_type_node
)
12789 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
12790 t
= TREE_TYPE (TREE_TYPE (fndecl
));
12791 if (INTEGRAL_TYPE_P (t
) || POINTER_TYPE_P (t
))
12792 veclen
= node
->simdclone
->vecsize_int
;
12794 veclen
= node
->simdclone
->vecsize_float
;
12795 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (t
));
12796 if (veclen
> node
->simdclone
->simdlen
)
12797 veclen
= node
->simdclone
->simdlen
;
12798 if (POINTER_TYPE_P (t
))
12799 t
= pointer_sized_int_node
;
12800 if (veclen
== node
->simdclone
->simdlen
)
12801 t
= build_vector_type (t
, node
->simdclone
->simdlen
);
12804 t
= build_vector_type (t
, veclen
);
12805 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
12807 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
12808 if (!node
->definition
)
12811 t
= DECL_RESULT (fndecl
);
12812 /* Adjust the DECL_RESULT. */
12813 gcc_assert (TREE_TYPE (t
) != void_type_node
);
12814 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
12817 tree atype
= build_array_type_nelts (orig_rettype
,
12818 node
->simdclone
->simdlen
);
12819 if (veclen
!= node
->simdclone
->simdlen
)
12820 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
12822 /* Set up a SIMD array to use as the return value. */
12823 tree retval
= create_tmp_var_raw (atype
, "retval");
12824 gimple_add_tmp_var (retval
);
12828 /* Each vector argument has a corresponding array to be used locally
12829 as part of the eventual loop. Create such temporary array and
12832 PREFIX is the prefix to be used for the temporary.
12834 TYPE is the inner element type.
12836 SIMDLEN is the number of elements. */
12839 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
12841 tree atype
= build_array_type_nelts (type
, simdlen
);
12842 tree avar
= create_tmp_var_raw (atype
, prefix
);
12843 gimple_add_tmp_var (avar
);
12847 /* Modify the function argument types to their corresponding vector
12848 counterparts if appropriate. Also, create one array for each simd
12849 argument to be used locally when using the function arguments as
12852 NODE is the function whose arguments are to be adjusted.
12854 Returns an adjustment vector that will be filled describing how the
12855 argument types will be adjusted. */
12857 static ipa_parm_adjustment_vec
12858 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
12861 ipa_parm_adjustment_vec adjustments
;
12863 if (node
->definition
)
12864 args
= ipa_get_vector_of_formal_parms (node
->decl
);
12866 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
12867 adjustments
.create (args
.length ());
12868 unsigned i
, j
, veclen
;
12869 struct ipa_parm_adjustment adj
;
12870 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
12872 memset (&adj
, 0, sizeof (adj
));
12873 tree parm
= args
[i
];
12874 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
12875 adj
.base_index
= i
;
12878 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
12879 node
->simdclone
->args
[i
].orig_type
= parm_type
;
12881 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
12883 /* No adjustment necessary for scalar arguments. */
12884 adj
.op
= IPA_PARM_OP_COPY
;
12888 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
12889 veclen
= node
->simdclone
->vecsize_int
;
12891 veclen
= node
->simdclone
->vecsize_float
;
12892 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
12893 if (veclen
> node
->simdclone
->simdlen
)
12894 veclen
= node
->simdclone
->simdlen
;
12895 adj
.arg_prefix
= "simd";
12896 if (POINTER_TYPE_P (parm_type
))
12897 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
12899 adj
.type
= build_vector_type (parm_type
, veclen
);
12900 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
12901 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12903 adjustments
.safe_push (adj
);
12906 memset (&adj
, 0, sizeof (adj
));
12907 adj
.op
= IPA_PARM_OP_NEW
;
12908 adj
.arg_prefix
= "simd";
12909 adj
.base_index
= i
;
12910 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
12914 if (node
->definition
)
12915 node
->simdclone
->args
[i
].simd_array
12916 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
12917 parm_type
, node
->simdclone
->simdlen
);
12919 adjustments
.safe_push (adj
);
12922 if (node
->simdclone
->inbranch
)
12925 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
12928 memset (&adj
, 0, sizeof (adj
));
12929 adj
.op
= IPA_PARM_OP_NEW
;
12930 adj
.arg_prefix
= "mask";
12932 adj
.base_index
= i
;
12933 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
12934 veclen
= node
->simdclone
->vecsize_int
;
12936 veclen
= node
->simdclone
->vecsize_float
;
12937 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
12938 if (veclen
> node
->simdclone
->simdlen
)
12939 veclen
= node
->simdclone
->simdlen
;
12940 if (POINTER_TYPE_P (base_type
))
12941 adj
.type
= build_vector_type (pointer_sized_int_node
, veclen
);
12943 adj
.type
= build_vector_type (base_type
, veclen
);
12944 adjustments
.safe_push (adj
);
12946 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
12947 adjustments
.safe_push (adj
);
12949 /* We have previously allocated one extra entry for the mask. Use
12951 struct cgraph_simd_clone
*sc
= node
->simdclone
;
12953 if (node
->definition
)
12955 sc
->args
[i
].orig_arg
12956 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
12957 sc
->args
[i
].simd_array
12958 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
12960 sc
->args
[i
].orig_type
= base_type
;
12961 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
12964 if (node
->definition
)
12965 ipa_modify_formal_parameters (node
->decl
, adjustments
);
12968 tree new_arg_types
= NULL_TREE
, new_reversed
;
12969 bool last_parm_void
= false;
12970 if (args
.length () > 0 && args
.last () == void_type_node
)
12971 last_parm_void
= true;
12973 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
12974 j
= adjustments
.length ();
12975 for (i
= 0; i
< j
; i
++)
12977 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
12979 if (adj
->op
== IPA_PARM_OP_COPY
)
12980 ptype
= args
[adj
->base_index
];
12983 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
12985 new_reversed
= nreverse (new_arg_types
);
12986 if (last_parm_void
)
12989 TREE_CHAIN (new_arg_types
) = void_list_node
;
12991 new_reversed
= void_list_node
;
12994 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
12995 TYPE_ARG_TYPES (new_type
) = new_reversed
;
12996 TREE_TYPE (node
->decl
) = new_type
;
12998 adjustments
.release ();
13001 return adjustments
;
13004 /* Initialize and copy the function arguments in NODE to their
13005 corresponding local simd arrays. Returns a fresh gimple_seq with
13006 the instruction sequence generated. */
13009 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
13010 ipa_parm_adjustment_vec adjustments
)
13012 gimple_seq seq
= NULL
;
13013 unsigned i
= 0, j
= 0, k
;
13015 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
13017 arg
= DECL_CHAIN (arg
), i
++, j
++)
13019 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
13022 node
->simdclone
->args
[i
].vector_arg
= arg
;
13024 tree array
= node
->simdclone
->args
[i
].simd_array
;
13025 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
13027 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
13028 tree ptr
= build_fold_addr_expr (array
);
13029 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
13030 build_int_cst (ptype
, 0));
13031 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
13032 gimplify_and_add (t
, &seq
);
13036 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
13037 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
13038 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
13040 tree ptr
= build_fold_addr_expr (array
);
13044 arg
= DECL_CHAIN (arg
);
13048 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
13049 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
13050 build_int_cst (ptype
, k
* elemsize
));
13051 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
13052 gimplify_and_add (t
, &seq
);
13059 /* Callback info for ipa_simd_modify_stmt_ops below. */
13061 struct modify_stmt_info
{
13062 ipa_parm_adjustment_vec adjustments
;
13064 /* True if the parent statement was modified by
13065 ipa_simd_modify_stmt_ops. */
13069 /* Callback for walk_gimple_op.
13071 Adjust operands from a given statement as specified in the
13072 adjustments vector in the callback data. */
13075 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
13077 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
13078 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
13079 tree
*orig_tp
= tp
;
13080 if (TREE_CODE (*tp
) == ADDR_EXPR
)
13081 tp
= &TREE_OPERAND (*tp
, 0);
13082 struct ipa_parm_adjustment
*cand
= NULL
;
13083 if (TREE_CODE (*tp
) == PARM_DECL
)
13084 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
13088 *walk_subtrees
= 0;
13091 tree repl
= NULL_TREE
;
13093 repl
= unshare_expr (cand
->new_decl
);
13098 *walk_subtrees
= 0;
13099 bool modified
= info
->modified
;
13100 info
->modified
= false;
13101 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
13102 if (!info
->modified
)
13104 info
->modified
= modified
;
13107 info
->modified
= modified
;
13116 repl
= build_fold_addr_expr (repl
);
13118 if (is_gimple_debug (info
->stmt
))
13120 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
13121 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
13122 DECL_ARTIFICIAL (vexpr
) = 1;
13123 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
13124 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
13129 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
)), repl
);
13130 repl
= gimple_assign_lhs (stmt
);
13132 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
13133 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13136 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
13138 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
13144 info
->modified
= true;
13148 /* Traverse the function body and perform all modifications as
13149 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
13150 modified such that the replacement/reduction value will now be an
13151 offset into the corresponding simd_array.
13153 This function will replace all function argument uses with their
13154 corresponding simd array elements, and ajust the return values
13158 ipa_simd_modify_function_body (struct cgraph_node
*node
,
13159 ipa_parm_adjustment_vec adjustments
,
13160 tree retval_array
, tree iter
)
13163 unsigned int i
, j
, l
;
13165 /* Re-use the adjustments array, but this time use it to replace
13166 every function argument use to an offset into the corresponding
13168 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
13170 if (!node
->simdclone
->args
[i
].vector_arg
)
13173 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
13174 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
13175 adjustments
[j
].new_decl
13176 = build4 (ARRAY_REF
,
13178 node
->simdclone
->args
[i
].simd_array
,
13180 NULL_TREE
, NULL_TREE
);
13181 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
13182 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
13183 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
13186 l
= adjustments
.length ();
13187 for (i
= 1; i
< num_ssa_names
; i
++)
13189 tree name
= ssa_name (i
);
13191 && SSA_NAME_VAR (name
)
13192 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
13194 for (j
= 0; j
< l
; j
++)
13195 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
13196 && adjustments
[j
].new_decl
)
13199 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
13202 = copy_var_decl (adjustments
[j
].base
,
13203 DECL_NAME (adjustments
[j
].base
),
13204 TREE_TYPE (adjustments
[j
].base
));
13205 adjustments
[j
].new_ssa_base
= base_var
;
13208 base_var
= adjustments
[j
].new_ssa_base
;
13209 if (SSA_NAME_IS_DEFAULT_DEF (name
))
13211 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13212 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
13213 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
13214 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
13215 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13216 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
13217 gimple stmt
= gimple_build_assign (name
, new_decl
);
13218 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
13221 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
13226 struct modify_stmt_info info
;
13227 info
.adjustments
= adjustments
;
13229 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
13231 gimple_stmt_iterator gsi
;
13233 gsi
= gsi_start_bb (bb
);
13234 while (!gsi_end_p (gsi
))
13236 gimple stmt
= gsi_stmt (gsi
);
13238 struct walk_stmt_info wi
;
13240 memset (&wi
, 0, sizeof (wi
));
13241 info
.modified
= false;
13243 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
13245 if (greturn
*return_stmt
= dyn_cast
<greturn
*> (stmt
))
13247 tree retval
= gimple_return_retval (return_stmt
);
13250 gsi_remove (&gsi
, true);
13254 /* Replace `return foo' with `retval_array[iter] = foo'. */
13255 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
13256 retval_array
, iter
, NULL
, NULL
);
13257 stmt
= gimple_build_assign (ref
, retval
);
13258 gsi_replace (&gsi
, stmt
, true);
13259 info
.modified
= true;
13264 update_stmt (stmt
);
13265 if (maybe_clean_eh_stmt (stmt
))
13266 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
13273 /* Adjust the argument types in NODE to their appropriate vector
13277 simd_clone_adjust (struct cgraph_node
*node
)
13279 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
13281 targetm
.simd_clone
.adjust (node
);
13283 tree retval
= simd_clone_adjust_return_type (node
);
13284 ipa_parm_adjustment_vec adjustments
13285 = simd_clone_adjust_argument_types (node
);
13287 push_gimplify_context ();
13289 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
13291 /* Adjust all uses of vector arguments accordingly. Adjust all
13292 return values accordingly. */
13293 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
13294 tree iter1
= make_ssa_name (iter
);
13295 tree iter2
= make_ssa_name (iter
);
13296 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
13298 /* Initialize the iteration variable. */
13299 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13300 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
13301 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
13302 /* Insert the SIMD array and iv initialization at function
13304 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
13306 pop_gimplify_context (NULL
);
13308 /* Create a new BB right before the original exit BB, to hold the
13309 iteration increment and the condition/branch. */
13310 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
13311 basic_block incr_bb
= create_empty_bb (orig_exit
);
13312 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
13313 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
13314 flag. Set it now to be a FALLTHRU_EDGE. */
13315 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
13316 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
13317 for (unsigned i
= 0;
13318 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
13320 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
13321 redirect_edge_succ (e
, incr_bb
);
13323 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
13324 e
->probability
= REG_BR_PROB_BASE
;
13325 gsi
= gsi_last_bb (incr_bb
);
13326 gimple g
= gimple_build_assign (iter2
, PLUS_EXPR
, iter1
,
13327 build_int_cst (unsigned_type_node
, 1));
13328 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13330 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
13331 struct loop
*loop
= alloc_loop ();
13332 cfun
->has_force_vectorize_loops
= true;
13333 loop
->safelen
= node
->simdclone
->simdlen
;
13334 loop
->force_vectorize
= true;
13335 loop
->header
= body_bb
;
13337 /* Branch around the body if the mask applies. */
13338 if (node
->simdclone
->inbranch
)
13340 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
13342 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
13343 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)));
13344 tree aref
= build4 (ARRAY_REF
,
13345 TREE_TYPE (TREE_TYPE (mask_array
)),
13348 g
= gimple_build_assign (mask
, aref
);
13349 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13350 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
13351 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
13353 aref
= build1 (VIEW_CONVERT_EXPR
,
13354 build_nonstandard_integer_type (bitsize
, 0), mask
);
13355 mask
= make_ssa_name (TREE_TYPE (aref
));
13356 g
= gimple_build_assign (mask
, aref
);
13357 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13360 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
13362 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13363 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
13364 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
13367 /* Generate the condition. */
13368 g
= gimple_build_cond (LT_EXPR
,
13370 build_int_cst (unsigned_type_node
,
13371 node
->simdclone
->simdlen
),
13373 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13374 e
= split_block (incr_bb
, gsi_stmt (gsi
));
13375 basic_block latch_bb
= e
->dest
;
13376 basic_block new_exit_bb
;
13377 new_exit_bb
= split_block_after_labels (latch_bb
)->dest
;
13378 loop
->latch
= latch_bb
;
13380 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
13382 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
13383 /* The successor of incr_bb is already pointing to latch_bb; just
13385 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
13386 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
13388 gphi
*phi
= create_phi_node (iter1
, body_bb
);
13389 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
13390 edge latch_edge
= single_succ_edge (latch_bb
);
13391 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
13393 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13395 /* Generate the new return. */
13396 gsi
= gsi_last_bb (new_exit_bb
);
13398 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
13399 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
13400 retval
= TREE_OPERAND (retval
, 0);
13403 retval
= build1 (VIEW_CONVERT_EXPR
,
13404 TREE_TYPE (TREE_TYPE (node
->decl
)),
13406 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
13407 false, GSI_CONTINUE_LINKING
);
13409 g
= gimple_build_return (retval
);
13410 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
13412 /* Handle aligned clauses by replacing default defs of the aligned
13413 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
13414 lhs. Handle linear by adding PHIs. */
13415 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
13416 if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
13417 && (TREE_ADDRESSABLE (node
->simdclone
->args
[i
].orig_arg
)
13418 || !is_gimple_reg_type
13419 (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
13421 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13422 if (is_gimple_reg_type (TREE_TYPE (orig_arg
)))
13423 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
13426 iter1
= create_tmp_var_raw (TREE_TYPE (orig_arg
));
13427 gimple_add_tmp_var (iter1
);
13429 gsi
= gsi_after_labels (entry_bb
);
13430 g
= gimple_build_assign (iter1
, orig_arg
);
13431 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
13432 gsi
= gsi_after_labels (body_bb
);
13433 g
= gimple_build_assign (orig_arg
, iter1
);
13434 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
13436 else if (node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
13437 && DECL_BY_REFERENCE (node
->simdclone
->args
[i
].orig_arg
)
13438 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
13440 && TREE_ADDRESSABLE
13441 (TREE_TYPE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))))
13443 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13444 tree def
= ssa_default_def (cfun
, orig_arg
);
13445 if (def
&& !has_zero_uses (def
))
13447 iter1
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg
)));
13448 gimple_add_tmp_var (iter1
);
13449 gsi
= gsi_after_labels (entry_bb
);
13450 g
= gimple_build_assign (iter1
, build_simple_mem_ref (def
));
13451 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
13452 gsi
= gsi_after_labels (body_bb
);
13453 g
= gimple_build_assign (build_simple_mem_ref (def
), iter1
);
13454 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
13457 else if (node
->simdclone
->args
[i
].alignment
13458 && node
->simdclone
->args
[i
].arg_type
13459 == SIMD_CLONE_ARG_TYPE_UNIFORM
13460 && (node
->simdclone
->args
[i
].alignment
13461 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
13462 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
13465 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
13466 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13467 tree def
= ssa_default_def (cfun
, orig_arg
);
13468 if (def
&& !has_zero_uses (def
))
13470 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
13471 gimple_seq seq
= NULL
;
13472 bool need_cvt
= false;
13474 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
13476 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
13479 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
);
13480 gimple_call_set_lhs (g
, t
);
13481 gimple_seq_add_stmt_without_update (&seq
, g
);
13484 t
= make_ssa_name (orig_arg
);
13485 g
= gimple_build_assign (t
, NOP_EXPR
, gimple_call_lhs (g
));
13486 gimple_seq_add_stmt_without_update (&seq
, g
);
13488 gsi_insert_seq_on_edge_immediate
13489 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
13491 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
13492 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
13494 node
->create_edge (cgraph_node::get_create (fn
),
13495 call
, entry_bb
->count
, freq
);
13497 imm_use_iterator iter
;
13498 use_operand_p use_p
;
13500 tree repl
= gimple_get_lhs (g
);
13501 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13502 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
13505 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13506 SET_USE (use_p
, repl
);
13509 else if (node
->simdclone
->args
[i
].arg_type
13510 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
13512 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
13513 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13514 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
13515 tree def
= NULL_TREE
;
13516 if (TREE_ADDRESSABLE (orig_arg
))
13518 def
= make_ssa_name (TREE_TYPE (orig_arg
));
13519 iter1
= make_ssa_name (TREE_TYPE (orig_arg
));
13520 iter2
= make_ssa_name (TREE_TYPE (orig_arg
));
13521 gsi
= gsi_after_labels (entry_bb
);
13522 g
= gimple_build_assign (def
, orig_arg
);
13523 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
13527 def
= ssa_default_def (cfun
, orig_arg
);
13528 if (!def
|| has_zero_uses (def
))
13532 iter1
= make_ssa_name (orig_arg
);
13533 iter2
= make_ssa_name (orig_arg
);
13538 phi
= create_phi_node (iter1
, body_bb
);
13539 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
13540 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
13541 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13542 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
13543 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
13544 ? TREE_TYPE (orig_arg
) : sizetype
;
13546 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
13547 g
= gimple_build_assign (iter2
, code
, iter1
, addcst
);
13548 gsi
= gsi_last_bb (incr_bb
);
13549 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
13551 imm_use_iterator iter
;
13552 use_operand_p use_p
;
13554 if (TREE_ADDRESSABLE (orig_arg
))
13556 gsi
= gsi_after_labels (body_bb
);
13557 g
= gimple_build_assign (orig_arg
, iter1
);
13558 gsi_insert_before (&gsi
, g
, GSI_NEW_STMT
);
13561 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
13562 if (use_stmt
== phi
)
13565 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
13566 SET_USE (use_p
, iter1
);
13570 calculate_dominance_info (CDI_DOMINATORS
);
13571 add_loop (loop
, loop
->header
->loop_father
);
13572 update_ssa (TODO_update_ssa
);
13577 /* If the function in NODE is tagged as an elemental SIMD function,
13578 create the appropriate SIMD clones. */
13581 expand_simd_clones (struct cgraph_node
*node
)
13583 tree attr
= lookup_attribute ("omp declare simd",
13584 DECL_ATTRIBUTES (node
->decl
));
13585 if (attr
== NULL_TREE
13586 || node
->global
.inlined_to
13587 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
13591 #pragma omp declare simd
13593 in C, there we don't know the argument types at all. */
13594 if (!node
->definition
13595 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
13600 /* Start with parsing the "omp declare simd" attribute(s). */
13601 bool inbranch_clause_specified
;
13602 struct cgraph_simd_clone
*clone_info
13603 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
13604 &inbranch_clause_specified
);
13605 if (clone_info
== NULL
)
13608 int orig_simdlen
= clone_info
->simdlen
;
13609 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
13610 /* The target can return 0 (no simd clones should be created),
13611 1 (just one ISA of simd clones should be created) or higher
13612 count of ISA variants. In that case, clone_info is initialized
13613 for the first ISA variant. */
13615 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
13620 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
13621 also create one inbranch and one !inbranch clone of it. */
13622 for (int i
= 0; i
< count
* 2; i
++)
13624 struct cgraph_simd_clone
*clone
= clone_info
;
13625 if (inbranch_clause_specified
&& (i
& 1) != 0)
13630 clone
= simd_clone_struct_alloc (clone_info
->nargs
13632 simd_clone_struct_copy (clone
, clone_info
);
13633 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
13634 and simd_clone_adjust_argument_types did to the first
13636 clone
->nargs
-= clone_info
->inbranch
;
13637 clone
->simdlen
= orig_simdlen
;
13638 /* And call the target hook again to get the right ISA. */
13639 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
13643 clone
->inbranch
= 1;
13646 /* simd_clone_mangle might fail if such a clone has been created
13648 tree id
= simd_clone_mangle (node
, clone
);
13649 if (id
== NULL_TREE
)
13652 /* Only when we are sure we want to create the clone actually
13653 clone the function (or definitions) or create another
13654 extern FUNCTION_DECL (for prototypes without definitions). */
13655 struct cgraph_node
*n
= simd_clone_create (node
);
13659 n
->simdclone
= clone
;
13660 clone
->origin
= node
;
13661 clone
->next_clone
= NULL
;
13662 if (node
->simd_clones
== NULL
)
13664 clone
->prev_clone
= n
;
13665 node
->simd_clones
= n
;
13669 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
13670 clone
->prev_clone
->simdclone
->next_clone
= n
;
13671 node
->simd_clones
->simdclone
->prev_clone
= n
;
13673 symtab
->change_decl_assembler_name (n
->decl
, id
);
13674 /* And finally adjust the return type, parameters and for
13675 definitions also function body. */
13676 if (node
->definition
)
13677 simd_clone_adjust (n
);
13680 simd_clone_adjust_return_type (n
);
13681 simd_clone_adjust_argument_types (n
);
13685 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
13688 /* Entry point for IPA simd clone creation pass. */
13690 static unsigned int
13691 ipa_omp_simd_clone (void)
13693 struct cgraph_node
*node
;
13694 FOR_EACH_FUNCTION (node
)
13695 expand_simd_clones (node
);
13701 const pass_data pass_data_omp_simd_clone
=
13703 SIMPLE_IPA_PASS
, /* type */
13704 "simdclone", /* name */
13705 OPTGROUP_NONE
, /* optinfo_flags */
13706 TV_NONE
, /* tv_id */
13707 ( PROP_ssa
| PROP_cfg
), /* properties_required */
13708 0, /* properties_provided */
13709 0, /* properties_destroyed */
13710 0, /* todo_flags_start */
13711 0, /* todo_flags_finish */
13714 class pass_omp_simd_clone
: public simple_ipa_opt_pass
13717 pass_omp_simd_clone(gcc::context
*ctxt
)
13718 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
13721 /* opt_pass methods: */
13722 virtual bool gate (function
*);
13723 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
13727 pass_omp_simd_clone::gate (function
*)
13729 return ((flag_openmp
|| flag_openmp_simd
13731 || (in_lto_p
&& !flag_wpa
))
13732 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
13735 } // anon namespace
13737 simple_ipa_opt_pass
*
13738 make_pass_omp_simd_clone (gcc::context
*ctxt
)
13740 return new pass_omp_simd_clone (ctxt
);
13743 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
13744 adds their addresses and sizes to constructor-vector V_CTOR. */
13746 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
13747 vec
<constructor_elt
, va_gc
> *v_ctor
)
13749 unsigned len
= vec_safe_length (v_decls
);
13750 for (unsigned i
= 0; i
< len
; i
++)
13752 tree it
= (*v_decls
)[i
];
13753 bool is_function
= TREE_CODE (it
) != VAR_DECL
;
13755 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, build_fold_addr_expr (it
));
13757 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
,
13758 fold_convert (const_ptr_type_node
,
13759 DECL_SIZE_UNIT (it
)));
13763 /* Create new symbols containing (address, size) pairs for global variables,
13764 marked with "omp declare target" attribute, as well as addresses for the
13765 functions, which are outlined offloading regions. */
13767 omp_finish_file (void)
13769 unsigned num_funcs
= vec_safe_length (offload_funcs
);
13770 unsigned num_vars
= vec_safe_length (offload_vars
);
13772 if (num_funcs
== 0 && num_vars
== 0)
13775 if (targetm_common
.have_named_sections
)
13777 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
13778 vec_alloc (v_f
, num_funcs
);
13779 vec_alloc (v_v
, num_vars
* 2);
13781 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
13782 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
13784 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13786 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
13788 TYPE_ALIGN (vars_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13789 TYPE_ALIGN (funcs_decl_type
) = TYPE_ALIGN (pointer_sized_int_node
);
13790 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
13791 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
13792 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
13793 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
13794 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13795 get_identifier (".offload_func_table"),
13797 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
13798 get_identifier (".offload_var_table"),
13800 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
13801 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
13802 otherwise a joint table in a binary will contain padding between
13803 tables from multiple object files. */
13804 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
13805 DECL_ALIGN (funcs_decl
) = TYPE_ALIGN (funcs_decl_type
);
13806 DECL_ALIGN (vars_decl
) = TYPE_ALIGN (vars_decl_type
);
13807 DECL_INITIAL (funcs_decl
) = ctor_f
;
13808 DECL_INITIAL (vars_decl
) = ctor_v
;
13809 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
13810 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
13812 varpool_node::finalize_decl (vars_decl
);
13813 varpool_node::finalize_decl (funcs_decl
);
13817 for (unsigned i
= 0; i
< num_funcs
; i
++)
13819 tree it
= (*offload_funcs
)[i
];
13820 targetm
.record_offload_symbol (it
);
13822 for (unsigned i
= 0; i
< num_vars
; i
++)
13824 tree it
= (*offload_vars
)[i
];
13825 targetm
.record_offload_symbol (it
);
13830 #include "gt-omp-low.h"