1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2014 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
32 #include "pointer-set.h"
33 #include "basic-block.h"
34 #include "tree-ssa-alias.h"
35 #include "internal-fn.h"
36 #include "gimple-fold.h"
37 #include "gimple-expr.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
43 #include "gimple-walk.h"
44 #include "tree-iterator.h"
45 #include "tree-inline.h"
46 #include "langhooks.h"
47 #include "diagnostic-core.h"
48 #include "gimple-ssa.h"
51 #include "tree-phinodes.h"
52 #include "ssa-iterators.h"
53 #include "tree-ssanames.h"
54 #include "tree-into-ssa.h"
61 #include "tree-pass.h"
63 #include "splay-tree.h"
68 #include "gimple-low.h"
69 #include "tree-cfgcleanup.h"
70 #include "pretty-print.h"
72 #include "tree-nested.h"
76 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
77 phases. The first phase scans the function looking for OMP statements
78 and then for variables that must be replaced to satisfy data sharing
79 clauses. The second phase expands code for the constructs, as well as
80 re-gimplifying things when variables have been replaced with complex
83 Final code generation is done by pass_expand_omp. The flowgraph is
84 scanned for parallel regions which are then moved to a new
85 function, to be invoked by the thread library. */
87 /* Parallel region information. Every parallel and workshare
88 directive is enclosed between two markers, the OMP_* directive
89 and a corresponding OMP_RETURN statement. */
93 /* The enclosing region. */
94 struct omp_region
*outer
;
96 /* First child region. */
97 struct omp_region
*inner
;
99 /* Next peer region. */
100 struct omp_region
*next
;
102 /* Block containing the omp directive as its last stmt. */
105 /* Block containing the OMP_RETURN as its last stmt. */
108 /* Block containing the OMP_CONTINUE as its last stmt. */
111 /* If this is a combined parallel+workshare region, this is a list
112 of additional arguments needed by the combined parallel+workshare
114 vec
<tree
, va_gc
> *ws_args
;
116 /* The code for the omp directive of this region. */
117 enum gimple_code type
;
119 /* Schedule kind, only used for OMP_FOR type regions. */
120 enum omp_clause_schedule_kind sched_kind
;
122 /* True if this is a combined parallel+workshare region. */
123 bool is_combined_parallel
;
126 /* Context structure. Used to store information about each parallel
127 directive in the code. */
129 typedef struct omp_context
131 /* This field must be at the beginning, as we do "inheritance": Some
132 callback functions for tree-inline.c (e.g., omp_copy_decl)
133 receive a copy_body_data pointer that is up-casted to an
134 omp_context pointer. */
137 /* The tree of contexts corresponding to the encountered constructs. */
138 struct omp_context
*outer
;
141 /* Map variables to fields in a structure that allows communication
142 between sending and receiving threads. */
143 splay_tree field_map
;
148 /* These are used just by task contexts, if task firstprivate fn is
149 needed. srecord_type is used to communicate from the thread
150 that encountered the task construct to task firstprivate fn,
151 record_type is allocated by GOMP_task, initialized by task firstprivate
152 fn and passed to the task body fn. */
153 splay_tree sfield_map
;
156 /* A chain of variables to add to the top-level block surrounding the
157 construct. In the case of a parallel, this is in the child function. */
160 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
161 barriers should jump to during omplower pass. */
164 /* What to do with variables with implicitly determined sharing
166 enum omp_clause_default_kind default_kind
;
168 /* Nesting depth of this context. Used to beautify error messages re
169 invalid gotos. The outermost ctx is depth 1, with depth 0 being
170 reserved for the main body of the function. */
173 /* True if this parallel directive is nested within another. */
176 /* True if this construct can be cancelled. */
181 struct omp_for_data_loop
183 tree v
, n1
, n2
, step
;
184 enum tree_code cond_code
;
187 /* A structure describing the main elements of a parallel loop. */
191 struct omp_for_data_loop loop
;
196 bool have_nowait
, have_ordered
;
197 enum omp_clause_schedule_kind sched_kind
;
198 struct omp_for_data_loop
*loops
;
202 static splay_tree all_contexts
;
203 static int taskreg_nesting_level
;
204 static int target_nesting_level
;
205 static struct omp_region
*root_omp_region
;
206 static bitmap task_shared_vars
;
208 static void scan_omp (gimple_seq
*, omp_context
*);
209 static tree
scan_omp_1_op (tree
*, int *, void *);
211 #define WALK_SUBSTMTS \
215 case GIMPLE_EH_FILTER: \
216 case GIMPLE_TRANSACTION: \
217 /* The sub-statements for these should be walked. */ \
218 *handled_ops_p = false; \
221 /* Convenience function for calling scan_omp_1_op on tree operands. */
224 scan_omp_op (tree
*tp
, omp_context
*ctx
)
226 struct walk_stmt_info wi
;
228 memset (&wi
, 0, sizeof (wi
));
230 wi
.want_locations
= true;
232 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
235 static void lower_omp (gimple_seq
*, omp_context
*);
236 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
237 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
239 /* Find an OpenMP clause of type KIND within CLAUSES. */
242 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
244 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
245 if (OMP_CLAUSE_CODE (clauses
) == kind
)
251 /* Return true if CTX is for an omp parallel. */
254 is_parallel_ctx (omp_context
*ctx
)
256 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
260 /* Return true if CTX is for an omp task. */
263 is_task_ctx (omp_context
*ctx
)
265 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
269 /* Return true if CTX is for an omp parallel or omp task. */
272 is_taskreg_ctx (omp_context
*ctx
)
274 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
275 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
279 /* Return true if REGION is a combined parallel+workshare region. */
282 is_combined_parallel (struct omp_region
*region
)
284 return region
->is_combined_parallel
;
288 /* Extract the header elements of parallel loop FOR_STMT and store
292 extract_omp_for_data (gimple for_stmt
, struct omp_for_data
*fd
,
293 struct omp_for_data_loop
*loops
)
295 tree t
, var
, *collapse_iter
, *collapse_count
;
296 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
297 struct omp_for_data_loop
*loop
;
299 struct omp_for_data_loop dummy_loop
;
300 location_t loc
= gimple_location (for_stmt
);
301 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
302 bool distribute
= gimple_omp_for_kind (for_stmt
)
303 == GF_OMP_FOR_KIND_DISTRIBUTE
;
305 fd
->for_stmt
= for_stmt
;
307 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
308 if (fd
->collapse
> 1)
311 fd
->loops
= &fd
->loop
;
313 fd
->have_nowait
= distribute
|| simd
;
314 fd
->have_ordered
= false;
315 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
316 fd
->chunk_size
= NULL_TREE
;
317 collapse_iter
= NULL
;
318 collapse_count
= NULL
;
320 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
321 switch (OMP_CLAUSE_CODE (t
))
323 case OMP_CLAUSE_NOWAIT
:
324 fd
->have_nowait
= true;
326 case OMP_CLAUSE_ORDERED
:
327 fd
->have_ordered
= true;
329 case OMP_CLAUSE_SCHEDULE
:
330 gcc_assert (!distribute
);
331 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
332 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
334 case OMP_CLAUSE_DIST_SCHEDULE
:
335 gcc_assert (distribute
);
336 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
338 case OMP_CLAUSE_COLLAPSE
:
339 if (fd
->collapse
> 1)
341 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
342 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
348 /* FIXME: for now map schedule(auto) to schedule(static).
349 There should be analysis to determine whether all iterations
350 are approximately the same amount of work (then schedule(static)
351 is best) or if it varies (then schedule(dynamic,N) is better). */
352 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
354 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
355 gcc_assert (fd
->chunk_size
== NULL
);
357 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
358 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
359 gcc_assert (fd
->chunk_size
== NULL
);
360 else if (fd
->chunk_size
== NULL
)
362 /* We only need to compute a default chunk size for ordered
363 static loops and dynamic loops. */
364 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
366 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
367 ? integer_zero_node
: integer_one_node
;
370 for (i
= 0; i
< fd
->collapse
; i
++)
372 if (fd
->collapse
== 1)
374 else if (loops
!= NULL
)
379 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
380 gcc_assert (SSA_VAR_P (loop
->v
));
381 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
382 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
383 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
384 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
386 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
387 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
388 switch (loop
->cond_code
)
394 gcc_assert (gimple_omp_for_kind (for_stmt
)
395 == GF_OMP_FOR_KIND_CILKSIMD
);
398 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
399 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
401 loop
->n2
= fold_build2_loc (loc
,
402 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
403 build_int_cst (TREE_TYPE (loop
->n2
), 1));
404 loop
->cond_code
= LT_EXPR
;
407 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
408 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
410 loop
->n2
= fold_build2_loc (loc
,
411 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
412 build_int_cst (TREE_TYPE (loop
->n2
), 1));
413 loop
->cond_code
= GT_EXPR
;
419 t
= gimple_omp_for_incr (for_stmt
, i
);
420 gcc_assert (TREE_OPERAND (t
, 0) == var
);
421 switch (TREE_CODE (t
))
424 loop
->step
= TREE_OPERAND (t
, 1);
426 case POINTER_PLUS_EXPR
:
427 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
430 loop
->step
= TREE_OPERAND (t
, 1);
431 loop
->step
= fold_build1_loc (loc
,
432 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
440 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
441 && !fd
->have_ordered
))
443 if (fd
->collapse
== 1)
444 iter_type
= TREE_TYPE (loop
->v
);
446 || TYPE_PRECISION (iter_type
)
447 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
449 = build_nonstandard_integer_type
450 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
452 else if (iter_type
!= long_long_unsigned_type_node
)
454 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
455 iter_type
= long_long_unsigned_type_node
;
456 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
457 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
458 >= TYPE_PRECISION (iter_type
))
462 if (loop
->cond_code
== LT_EXPR
)
463 n
= fold_build2_loc (loc
,
464 PLUS_EXPR
, TREE_TYPE (loop
->v
),
465 loop
->n2
, loop
->step
);
468 if (TREE_CODE (n
) != INTEGER_CST
469 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
470 iter_type
= long_long_unsigned_type_node
;
472 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
473 > TYPE_PRECISION (iter_type
))
477 if (loop
->cond_code
== LT_EXPR
)
480 n2
= fold_build2_loc (loc
,
481 PLUS_EXPR
, TREE_TYPE (loop
->v
),
482 loop
->n2
, loop
->step
);
486 n1
= fold_build2_loc (loc
,
487 MINUS_EXPR
, TREE_TYPE (loop
->v
),
488 loop
->n2
, loop
->step
);
491 if (TREE_CODE (n1
) != INTEGER_CST
492 || TREE_CODE (n2
) != INTEGER_CST
493 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
494 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
495 iter_type
= long_long_unsigned_type_node
;
499 if (collapse_count
&& *collapse_count
== NULL
)
501 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
502 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
503 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
504 if (t
&& integer_zerop (t
))
505 count
= build_zero_cst (long_long_unsigned_type_node
);
506 else if ((i
== 0 || count
!= NULL_TREE
)
507 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
508 && TREE_CONSTANT (loop
->n1
)
509 && TREE_CONSTANT (loop
->n2
)
510 && TREE_CODE (loop
->step
) == INTEGER_CST
)
512 tree itype
= TREE_TYPE (loop
->v
);
514 if (POINTER_TYPE_P (itype
))
515 itype
= signed_type_for (itype
);
516 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
517 t
= fold_build2_loc (loc
,
519 fold_convert_loc (loc
, itype
, loop
->step
), t
);
520 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
521 fold_convert_loc (loc
, itype
, loop
->n2
));
522 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
523 fold_convert_loc (loc
, itype
, loop
->n1
));
524 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
525 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
526 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
527 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
528 fold_convert_loc (loc
, itype
,
531 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
532 fold_convert_loc (loc
, itype
, loop
->step
));
533 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
534 if (count
!= NULL_TREE
)
535 count
= fold_build2_loc (loc
,
536 MULT_EXPR
, long_long_unsigned_type_node
,
540 if (TREE_CODE (count
) != INTEGER_CST
)
543 else if (count
&& !integer_zerop (count
))
550 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
551 || fd
->have_ordered
))
553 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
554 iter_type
= long_long_unsigned_type_node
;
556 iter_type
= long_integer_type_node
;
558 else if (collapse_iter
&& *collapse_iter
!= NULL
)
559 iter_type
= TREE_TYPE (*collapse_iter
);
560 fd
->iter_type
= iter_type
;
561 if (collapse_iter
&& *collapse_iter
== NULL
)
562 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
563 if (collapse_count
&& *collapse_count
== NULL
)
566 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
568 *collapse_count
= create_tmp_var (iter_type
, ".count");
571 if (fd
->collapse
> 1)
573 fd
->loop
.v
= *collapse_iter
;
574 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
575 fd
->loop
.n2
= *collapse_count
;
576 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
577 fd
->loop
.cond_code
= LT_EXPR
;
582 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
583 is the immediate dominator of PAR_ENTRY_BB, return true if there
584 are no data dependencies that would prevent expanding the parallel
585 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
587 When expanding a combined parallel+workshare region, the call to
588 the child function may need additional arguments in the case of
589 GIMPLE_OMP_FOR regions. In some cases, these arguments are
590 computed out of variables passed in from the parent to the child
591 via 'struct .omp_data_s'. For instance:
593 #pragma omp parallel for schedule (guided, i * 4)
598 # BLOCK 2 (PAR_ENTRY_BB)
600 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
602 # BLOCK 3 (WS_ENTRY_BB)
603 .omp_data_i = &.omp_data_o;
604 D.1667 = .omp_data_i->i;
606 #pragma omp for schedule (guided, D.1598)
608 When we outline the parallel region, the call to the child function
609 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
610 that value is computed *after* the call site. So, in principle we
611 cannot do the transformation.
613 To see whether the code in WS_ENTRY_BB blocks the combined
614 parallel+workshare call, we collect all the variables used in the
615 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
616 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
619 FIXME. If we had the SSA form built at this point, we could merely
620 hoist the code in block 3 into block 2 and be done with it. But at
621 this point we don't have dataflow information and though we could
622 hack something up here, it is really not worth the aggravation. */
625 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
627 struct omp_for_data fd
;
628 gimple ws_stmt
= last_stmt (ws_entry_bb
);
630 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
633 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
635 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
637 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
639 if (fd
.iter_type
!= long_integer_type_node
)
642 /* FIXME. We give up too easily here. If any of these arguments
643 are not constants, they will likely involve variables that have
644 been mapped into fields of .omp_data_s for sharing with the child
645 function. With appropriate data flow, it would be possible to
647 if (!is_gimple_min_invariant (fd
.loop
.n1
)
648 || !is_gimple_min_invariant (fd
.loop
.n2
)
649 || !is_gimple_min_invariant (fd
.loop
.step
)
650 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
657 /* Collect additional arguments needed to emit a combined
658 parallel+workshare call. WS_STMT is the workshare directive being
661 static vec
<tree
, va_gc
> *
662 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
665 location_t loc
= gimple_location (ws_stmt
);
666 vec
<tree
, va_gc
> *ws_args
;
668 if (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
)
670 struct omp_for_data fd
;
673 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
677 if (gimple_omp_for_combined_into_p (ws_stmt
))
680 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
681 OMP_CLAUSE__LOOPTEMP_
);
683 n1
= OMP_CLAUSE_DECL (innerc
);
684 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
685 OMP_CLAUSE__LOOPTEMP_
);
687 n2
= OMP_CLAUSE_DECL (innerc
);
690 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
692 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
693 ws_args
->quick_push (t
);
695 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
696 ws_args
->quick_push (t
);
698 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
699 ws_args
->quick_push (t
);
703 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
704 ws_args
->quick_push (t
);
709 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
711 /* Number of sections is equal to the number of edges from the
712 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
713 the exit of the sections region. */
714 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
715 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
716 vec_alloc (ws_args
, 1);
717 ws_args
->quick_push (t
);
725 /* Discover whether REGION is a combined parallel+workshare region. */
728 determine_parallel_type (struct omp_region
*region
)
730 basic_block par_entry_bb
, par_exit_bb
;
731 basic_block ws_entry_bb
, ws_exit_bb
;
733 if (region
== NULL
|| region
->inner
== NULL
734 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
735 || region
->inner
->cont
== NULL
)
738 /* We only support parallel+for and parallel+sections. */
739 if (region
->type
!= GIMPLE_OMP_PARALLEL
740 || (region
->inner
->type
!= GIMPLE_OMP_FOR
741 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
744 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
745 WS_EXIT_BB -> PAR_EXIT_BB. */
746 par_entry_bb
= region
->entry
;
747 par_exit_bb
= region
->exit
;
748 ws_entry_bb
= region
->inner
->entry
;
749 ws_exit_bb
= region
->inner
->exit
;
751 if (single_succ (par_entry_bb
) == ws_entry_bb
752 && single_succ (ws_exit_bb
) == par_exit_bb
753 && workshare_safe_to_combine_p (ws_entry_bb
)
754 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
755 || (last_and_only_stmt (ws_entry_bb
)
756 && last_and_only_stmt (par_exit_bb
))))
758 gimple par_stmt
= last_stmt (par_entry_bb
);
759 gimple ws_stmt
= last_stmt (ws_entry_bb
);
761 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
763 /* If this is a combined parallel loop, we need to determine
764 whether or not to use the combined library calls. There
765 are two cases where we do not apply the transformation:
766 static loops and any kind of ordered loop. In the first
767 case, we already open code the loop so there is no need
768 to do anything else. In the latter case, the combined
769 parallel loop call would still need extra synchronization
770 to implement ordered semantics, so there would not be any
771 gain in using the combined call. */
772 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
773 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
775 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
776 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
778 region
->is_combined_parallel
= false;
779 region
->inner
->is_combined_parallel
= false;
784 region
->is_combined_parallel
= true;
785 region
->inner
->is_combined_parallel
= true;
786 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
791 /* Return true if EXPR is variable sized. */
794 is_variable_sized (const_tree expr
)
796 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
799 /* Return true if DECL is a reference type. */
802 is_reference (tree decl
)
804 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
807 /* Lookup variables in the decl or field splay trees. The "maybe" form
808 allows for the variable form to not have been entered, otherwise we
809 assert that the variable must have been entered. */
812 lookup_decl (tree var
, omp_context
*ctx
)
815 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
820 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
823 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
824 return n
? *n
: NULL_TREE
;
828 lookup_field (tree var
, omp_context
*ctx
)
831 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
832 return (tree
) n
->value
;
836 lookup_sfield (tree var
, omp_context
*ctx
)
839 n
= splay_tree_lookup (ctx
->sfield_map
840 ? ctx
->sfield_map
: ctx
->field_map
,
841 (splay_tree_key
) var
);
842 return (tree
) n
->value
;
846 maybe_lookup_field (tree var
, omp_context
*ctx
)
849 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
850 return n
? (tree
) n
->value
: NULL_TREE
;
853 /* Return true if DECL should be copied by pointer. SHARED_CTX is
854 the parallel context if DECL is to be shared. */
857 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
859 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
862 /* We can only use copy-in/copy-out semantics for shared variables
863 when we know the value is not accessible from an outer scope. */
866 /* ??? Trivially accessible from anywhere. But why would we even
867 be passing an address in this case? Should we simply assert
868 this to be false, or should we have a cleanup pass that removes
869 these from the list of mappings? */
870 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
873 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
874 without analyzing the expression whether or not its location
875 is accessible to anyone else. In the case of nested parallel
876 regions it certainly may be. */
877 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
880 /* Do not use copy-in/copy-out for variables that have their
882 if (TREE_ADDRESSABLE (decl
))
885 /* lower_send_shared_vars only uses copy-in, but not copy-out
887 if (TREE_READONLY (decl
)
888 || ((TREE_CODE (decl
) == RESULT_DECL
889 || TREE_CODE (decl
) == PARM_DECL
)
890 && DECL_BY_REFERENCE (decl
)))
893 /* Disallow copy-in/out in nested parallel if
894 decl is shared in outer parallel, otherwise
895 each thread could store the shared variable
896 in its own copy-in location, making the
897 variable no longer really shared. */
898 if (shared_ctx
->is_nested
)
902 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
903 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
910 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
911 c
; c
= OMP_CLAUSE_CHAIN (c
))
912 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
913 && OMP_CLAUSE_DECL (c
) == decl
)
917 goto maybe_mark_addressable_and_ret
;
921 /* For tasks avoid using copy-in/out. As tasks can be
922 deferred or executed in different thread, when GOMP_task
923 returns, the task hasn't necessarily terminated. */
924 if (is_task_ctx (shared_ctx
))
927 maybe_mark_addressable_and_ret
:
928 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
929 if (is_gimple_reg (outer
))
931 /* Taking address of OUTER in lower_send_shared_vars
932 might need regimplification of everything that uses the
934 if (!task_shared_vars
)
935 task_shared_vars
= BITMAP_ALLOC (NULL
);
936 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
937 TREE_ADDRESSABLE (outer
) = 1;
946 /* Construct a new automatic decl similar to VAR. */
949 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
951 tree copy
= copy_var_decl (var
, name
, type
);
953 DECL_CONTEXT (copy
) = current_function_decl
;
954 DECL_CHAIN (copy
) = ctx
->block_vars
;
955 ctx
->block_vars
= copy
;
961 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
963 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
966 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
969 omp_build_component_ref (tree obj
, tree field
)
971 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
972 if (TREE_THIS_VOLATILE (field
))
973 TREE_THIS_VOLATILE (ret
) |= 1;
974 if (TREE_READONLY (field
))
975 TREE_READONLY (ret
) |= 1;
979 /* Build tree nodes to access the field for VAR on the receiver side. */
982 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
984 tree x
, field
= lookup_field (var
, ctx
);
986 /* If the receiver record type was remapped in the child function,
987 remap the field into the new record type. */
988 x
= maybe_lookup_field (field
, ctx
);
992 x
= build_simple_mem_ref (ctx
->receiver_decl
);
993 x
= omp_build_component_ref (x
, field
);
995 x
= build_simple_mem_ref (x
);
1000 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1001 of a parallel, this is a component reference; for workshare constructs
1002 this is some variable. */
1005 build_outer_var_ref (tree var
, omp_context
*ctx
)
1009 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1011 else if (is_variable_sized (var
))
1013 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1014 x
= build_outer_var_ref (x
, ctx
);
1015 x
= build_simple_mem_ref (x
);
1017 else if (is_taskreg_ctx (ctx
))
1019 bool by_ref
= use_pointer_for_field (var
, NULL
);
1020 x
= build_receiver_ref (var
, by_ref
, ctx
);
1022 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1023 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1025 /* #pragma omp simd isn't a worksharing construct, and can reference even
1026 private vars in its linear etc. clauses. */
1028 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1029 x
= lookup_decl (var
, ctx
->outer
);
1030 else if (ctx
->outer
)
1031 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1035 else if (ctx
->outer
)
1036 x
= lookup_decl (var
, ctx
->outer
);
1037 else if (is_reference (var
))
1038 /* This can happen with orphaned constructs. If var is reference, it is
1039 possible it is shared and as such valid. */
1044 if (is_reference (var
))
1045 x
= build_simple_mem_ref (x
);
1050 /* Build tree nodes to access the field for VAR on the sender side. */
1053 build_sender_ref (tree var
, omp_context
*ctx
)
1055 tree field
= lookup_sfield (var
, ctx
);
1056 return omp_build_component_ref (ctx
->sender_decl
, field
);
1059 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1062 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1064 tree field
, type
, sfield
= NULL_TREE
;
1066 gcc_assert ((mask
& 1) == 0
1067 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1068 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1069 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1071 type
= TREE_TYPE (var
);
1074 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1075 type
= build_pointer_type (build_pointer_type (type
));
1078 type
= build_pointer_type (type
);
1079 else if ((mask
& 3) == 1 && is_reference (var
))
1080 type
= TREE_TYPE (type
);
1082 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1083 FIELD_DECL
, DECL_NAME (var
), type
);
1085 /* Remember what variable this field was created for. This does have a
1086 side effect of making dwarf2out ignore this member, so for helpful
1087 debugging we clear it later in delete_omp_context. */
1088 DECL_ABSTRACT_ORIGIN (field
) = var
;
1089 if (type
== TREE_TYPE (var
))
1091 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1092 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1093 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1096 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1098 if ((mask
& 3) == 3)
1100 insert_field_into_struct (ctx
->record_type
, field
);
1101 if (ctx
->srecord_type
)
1103 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1104 FIELD_DECL
, DECL_NAME (var
), type
);
1105 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1106 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1107 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1108 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1109 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1114 if (ctx
->srecord_type
== NULL_TREE
)
1118 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1119 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1120 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1122 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1123 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1124 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1125 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1126 splay_tree_insert (ctx
->sfield_map
,
1127 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1128 (splay_tree_value
) sfield
);
1132 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1133 : ctx
->srecord_type
, field
);
1137 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1138 (splay_tree_value
) field
);
1139 if ((mask
& 2) && ctx
->sfield_map
)
1140 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1141 (splay_tree_value
) sfield
);
1145 install_var_local (tree var
, omp_context
*ctx
)
1147 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1148 insert_decl_map (&ctx
->cb
, var
, new_var
);
1152 /* Adjust the replacement for DECL in CTX for the new context. This means
1153 copying the DECL_VALUE_EXPR, and fixing up the type. */
1156 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1158 tree new_decl
, size
;
1160 new_decl
= lookup_decl (decl
, ctx
);
1162 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1164 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1165 && DECL_HAS_VALUE_EXPR_P (decl
))
1167 tree ve
= DECL_VALUE_EXPR (decl
);
1168 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1169 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1170 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1173 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1175 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1176 if (size
== error_mark_node
)
1177 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1178 DECL_SIZE (new_decl
) = size
;
1180 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1181 if (size
== error_mark_node
)
1182 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1183 DECL_SIZE_UNIT (new_decl
) = size
;
1187 /* The callback for remap_decl. Search all containing contexts for a
1188 mapping of the variable; this avoids having to duplicate the splay
1189 tree ahead of time. We know a mapping doesn't already exist in the
1190 given context. Create new mappings to implement default semantics. */
1193 omp_copy_decl (tree var
, copy_body_data
*cb
)
1195 omp_context
*ctx
= (omp_context
*) cb
;
1198 if (TREE_CODE (var
) == LABEL_DECL
)
1200 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1201 DECL_CONTEXT (new_var
) = current_function_decl
;
1202 insert_decl_map (&ctx
->cb
, var
, new_var
);
1206 while (!is_taskreg_ctx (ctx
))
1211 new_var
= maybe_lookup_decl (var
, ctx
);
1216 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1219 return error_mark_node
;
1223 /* Debugging dumps for parallel regions. */
1224 void dump_omp_region (FILE *, struct omp_region
*, int);
1225 void debug_omp_region (struct omp_region
*);
1226 void debug_all_omp_regions (void);
1228 /* Dump the parallel region tree rooted at REGION. */
1231 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1233 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1234 gimple_code_name
[region
->type
]);
1237 dump_omp_region (file
, region
->inner
, indent
+ 4);
1241 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1242 region
->cont
->index
);
1246 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1247 region
->exit
->index
);
1249 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1252 dump_omp_region (file
, region
->next
, indent
);
1256 debug_omp_region (struct omp_region
*region
)
1258 dump_omp_region (stderr
, region
, 0);
1262 debug_all_omp_regions (void)
1264 dump_omp_region (stderr
, root_omp_region
, 0);
1268 /* Create a new parallel region starting at STMT inside region PARENT. */
1270 static struct omp_region
*
1271 new_omp_region (basic_block bb
, enum gimple_code type
,
1272 struct omp_region
*parent
)
1274 struct omp_region
*region
= XCNEW (struct omp_region
);
1276 region
->outer
= parent
;
1278 region
->type
= type
;
1282 /* This is a nested region. Add it to the list of inner
1283 regions in PARENT. */
1284 region
->next
= parent
->inner
;
1285 parent
->inner
= region
;
1289 /* This is a toplevel region. Add it to the list of toplevel
1290 regions in ROOT_OMP_REGION. */
1291 region
->next
= root_omp_region
;
1292 root_omp_region
= region
;
1298 /* Release the memory associated with the region tree rooted at REGION. */
1301 free_omp_region_1 (struct omp_region
*region
)
1303 struct omp_region
*i
, *n
;
1305 for (i
= region
->inner
; i
; i
= n
)
1308 free_omp_region_1 (i
);
1314 /* Release the memory for the entire omp region tree. */
1317 free_omp_regions (void)
1319 struct omp_region
*r
, *n
;
1320 for (r
= root_omp_region
; r
; r
= n
)
1323 free_omp_region_1 (r
);
1325 root_omp_region
= NULL
;
1329 /* Create a new context, with OUTER_CTX being the surrounding context. */
1331 static omp_context
*
1332 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1334 omp_context
*ctx
= XCNEW (omp_context
);
1336 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1337 (splay_tree_value
) ctx
);
1342 ctx
->outer
= outer_ctx
;
1343 ctx
->cb
= outer_ctx
->cb
;
1344 ctx
->cb
.block
= NULL
;
1345 ctx
->depth
= outer_ctx
->depth
+ 1;
1349 ctx
->cb
.src_fn
= current_function_decl
;
1350 ctx
->cb
.dst_fn
= current_function_decl
;
1351 ctx
->cb
.src_node
= cgraph_get_node (current_function_decl
);
1352 gcc_checking_assert (ctx
->cb
.src_node
);
1353 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1354 ctx
->cb
.src_cfun
= cfun
;
1355 ctx
->cb
.copy_decl
= omp_copy_decl
;
1356 ctx
->cb
.eh_lp_nr
= 0;
1357 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1361 ctx
->cb
.decl_map
= pointer_map_create ();
1366 static gimple_seq
maybe_catch_exception (gimple_seq
);
1368 /* Finalize task copyfn. */
1371 finalize_task_copyfn (gimple task_stmt
)
1373 struct function
*child_cfun
;
1375 gimple_seq seq
= NULL
, new_seq
;
1378 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1379 if (child_fn
== NULL_TREE
)
1382 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1383 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1385 push_cfun (child_cfun
);
1386 bind
= gimplify_body (child_fn
, false);
1387 gimple_seq_add_stmt (&seq
, bind
);
1388 new_seq
= maybe_catch_exception (seq
);
1391 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1393 gimple_seq_add_stmt (&seq
, bind
);
1395 gimple_set_body (child_fn
, seq
);
1398 /* Inform the callgraph about the new function. */
1399 cgraph_add_new_function (child_fn
, false);
1402 /* Destroy a omp_context data structures. Called through the splay tree
1403 value delete callback. */
1406 delete_omp_context (splay_tree_value value
)
1408 omp_context
*ctx
= (omp_context
*) value
;
1410 pointer_map_destroy (ctx
->cb
.decl_map
);
1413 splay_tree_delete (ctx
->field_map
);
1414 if (ctx
->sfield_map
)
1415 splay_tree_delete (ctx
->sfield_map
);
1417 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1418 it produces corrupt debug information. */
1419 if (ctx
->record_type
)
1422 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1423 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1425 if (ctx
->srecord_type
)
1428 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1429 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1432 if (is_task_ctx (ctx
))
1433 finalize_task_copyfn (ctx
->stmt
);
1438 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1442 fixup_child_record_type (omp_context
*ctx
)
1444 tree f
, type
= ctx
->record_type
;
1446 /* ??? It isn't sufficient to just call remap_type here, because
1447 variably_modified_type_p doesn't work the way we expect for
1448 record types. Testing each field for whether it needs remapping
1449 and creating a new record by hand works, however. */
1450 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1451 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1455 tree name
, new_fields
= NULL
;
1457 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1458 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1459 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1460 TYPE_DECL
, name
, type
);
1461 TYPE_NAME (type
) = name
;
1463 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1465 tree new_f
= copy_node (f
);
1466 DECL_CONTEXT (new_f
) = type
;
1467 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1468 DECL_CHAIN (new_f
) = new_fields
;
1469 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1470 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1472 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1476 /* Arrange to be able to look up the receiver field
1477 given the sender field. */
1478 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1479 (splay_tree_value
) new_f
);
1481 TYPE_FIELDS (type
) = nreverse (new_fields
);
1485 TREE_TYPE (ctx
->receiver_decl
) = build_pointer_type (type
);
1488 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1489 specified by CLAUSES. */
1492 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1495 bool scan_array_reductions
= false;
1497 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1501 switch (OMP_CLAUSE_CODE (c
))
1503 case OMP_CLAUSE_PRIVATE
:
1504 decl
= OMP_CLAUSE_DECL (c
);
1505 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1507 else if (!is_variable_sized (decl
))
1508 install_var_local (decl
, ctx
);
1511 case OMP_CLAUSE_SHARED
:
1512 decl
= OMP_CLAUSE_DECL (c
);
1513 /* Ignore shared directives in teams construct. */
1514 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1516 /* Global variables don't need to be copied,
1517 the receiver side will use them directly. */
1518 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1519 if (is_global_var (odecl
))
1521 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1524 gcc_assert (is_taskreg_ctx (ctx
));
1525 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1526 || !is_variable_sized (decl
));
1527 /* Global variables don't need to be copied,
1528 the receiver side will use them directly. */
1529 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1531 by_ref
= use_pointer_for_field (decl
, ctx
);
1532 if (! TREE_READONLY (decl
)
1533 || TREE_ADDRESSABLE (decl
)
1535 || is_reference (decl
))
1537 install_var_field (decl
, by_ref
, 3, ctx
);
1538 install_var_local (decl
, ctx
);
1541 /* We don't need to copy const scalar vars back. */
1542 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1545 case OMP_CLAUSE_LASTPRIVATE
:
1546 /* Let the corresponding firstprivate clause create
1548 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1552 case OMP_CLAUSE_FIRSTPRIVATE
:
1553 case OMP_CLAUSE_REDUCTION
:
1554 case OMP_CLAUSE_LINEAR
:
1555 decl
= OMP_CLAUSE_DECL (c
);
1557 if (is_variable_sized (decl
))
1559 if (is_task_ctx (ctx
))
1560 install_var_field (decl
, false, 1, ctx
);
1563 else if (is_taskreg_ctx (ctx
))
1566 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1567 by_ref
= use_pointer_for_field (decl
, NULL
);
1569 if (is_task_ctx (ctx
)
1570 && (global
|| by_ref
|| is_reference (decl
)))
1572 install_var_field (decl
, false, 1, ctx
);
1574 install_var_field (decl
, by_ref
, 2, ctx
);
1577 install_var_field (decl
, by_ref
, 3, ctx
);
1579 install_var_local (decl
, ctx
);
1582 case OMP_CLAUSE__LOOPTEMP_
:
1583 gcc_assert (is_parallel_ctx (ctx
));
1584 decl
= OMP_CLAUSE_DECL (c
);
1585 install_var_field (decl
, false, 3, ctx
);
1586 install_var_local (decl
, ctx
);
1589 case OMP_CLAUSE_COPYPRIVATE
:
1590 case OMP_CLAUSE_COPYIN
:
1591 decl
= OMP_CLAUSE_DECL (c
);
1592 by_ref
= use_pointer_for_field (decl
, NULL
);
1593 install_var_field (decl
, by_ref
, 3, ctx
);
1596 case OMP_CLAUSE_DEFAULT
:
1597 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1600 case OMP_CLAUSE_FINAL
:
1602 case OMP_CLAUSE_NUM_THREADS
:
1603 case OMP_CLAUSE_NUM_TEAMS
:
1604 case OMP_CLAUSE_THREAD_LIMIT
:
1605 case OMP_CLAUSE_DEVICE
:
1606 case OMP_CLAUSE_SCHEDULE
:
1607 case OMP_CLAUSE_DIST_SCHEDULE
:
1608 case OMP_CLAUSE_DEPEND
:
1610 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1614 case OMP_CLAUSE_FROM
:
1615 case OMP_CLAUSE_MAP
:
1617 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1618 decl
= OMP_CLAUSE_DECL (c
);
1619 /* Global variables with "omp declare target" attribute
1620 don't need to be copied, the receiver side will use them
1622 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1624 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1625 && lookup_attribute ("omp declare target",
1626 DECL_ATTRIBUTES (decl
)))
1628 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1629 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1631 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1632 #pragma omp target data, there is nothing to map for
1634 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
1635 && !POINTER_TYPE_P (TREE_TYPE (decl
)))
1640 if (DECL_SIZE (decl
)
1641 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1643 tree decl2
= DECL_VALUE_EXPR (decl
);
1644 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1645 decl2
= TREE_OPERAND (decl2
, 0);
1646 gcc_assert (DECL_P (decl2
));
1647 install_var_field (decl2
, true, 3, ctx
);
1648 install_var_local (decl2
, ctx
);
1649 install_var_local (decl
, ctx
);
1653 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1654 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1655 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1656 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1657 install_var_field (decl
, true, 7, ctx
);
1659 install_var_field (decl
, true, 3, ctx
);
1660 if (gimple_omp_target_kind (ctx
->stmt
)
1661 == GF_OMP_TARGET_KIND_REGION
)
1662 install_var_local (decl
, ctx
);
1667 tree base
= get_base_address (decl
);
1668 tree nc
= OMP_CLAUSE_CHAIN (c
);
1671 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1672 && OMP_CLAUSE_DECL (nc
) == base
1673 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1674 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1676 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1677 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1683 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1684 decl
= OMP_CLAUSE_DECL (c
);
1686 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1687 (splay_tree_key
) decl
));
1689 = build_decl (OMP_CLAUSE_LOCATION (c
),
1690 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1691 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1692 insert_field_into_struct (ctx
->record_type
, field
);
1693 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1694 (splay_tree_value
) field
);
1699 case OMP_CLAUSE_NOWAIT
:
1700 case OMP_CLAUSE_ORDERED
:
1701 case OMP_CLAUSE_COLLAPSE
:
1702 case OMP_CLAUSE_UNTIED
:
1703 case OMP_CLAUSE_MERGEABLE
:
1704 case OMP_CLAUSE_PROC_BIND
:
1705 case OMP_CLAUSE_SAFELEN
:
1708 case OMP_CLAUSE_ALIGNED
:
1709 decl
= OMP_CLAUSE_DECL (c
);
1710 if (is_global_var (decl
)
1711 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1712 install_var_local (decl
, ctx
);
1720 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1722 switch (OMP_CLAUSE_CODE (c
))
1724 case OMP_CLAUSE_LASTPRIVATE
:
1725 /* Let the corresponding firstprivate clause create
1727 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1728 scan_array_reductions
= true;
1729 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1733 case OMP_CLAUSE_PRIVATE
:
1734 case OMP_CLAUSE_FIRSTPRIVATE
:
1735 case OMP_CLAUSE_REDUCTION
:
1736 case OMP_CLAUSE_LINEAR
:
1737 decl
= OMP_CLAUSE_DECL (c
);
1738 if (is_variable_sized (decl
))
1739 install_var_local (decl
, ctx
);
1740 fixup_remapped_decl (decl
, ctx
,
1741 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1742 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1743 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1744 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1745 scan_array_reductions
= true;
1746 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1747 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1748 scan_array_reductions
= true;
1751 case OMP_CLAUSE_SHARED
:
1752 /* Ignore shared directives in teams construct. */
1753 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1755 decl
= OMP_CLAUSE_DECL (c
);
1756 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1757 fixup_remapped_decl (decl
, ctx
, false);
1760 case OMP_CLAUSE_MAP
:
1761 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
)
1763 decl
= OMP_CLAUSE_DECL (c
);
1765 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1766 && lookup_attribute ("omp declare target",
1767 DECL_ATTRIBUTES (decl
)))
1771 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1772 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1773 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1775 tree new_decl
= lookup_decl (decl
, ctx
);
1776 TREE_TYPE (new_decl
)
1777 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1779 else if (DECL_SIZE (decl
)
1780 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1782 tree decl2
= DECL_VALUE_EXPR (decl
);
1783 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1784 decl2
= TREE_OPERAND (decl2
, 0);
1785 gcc_assert (DECL_P (decl2
));
1786 fixup_remapped_decl (decl2
, ctx
, false);
1787 fixup_remapped_decl (decl
, ctx
, true);
1790 fixup_remapped_decl (decl
, ctx
, false);
1794 case OMP_CLAUSE_COPYPRIVATE
:
1795 case OMP_CLAUSE_COPYIN
:
1796 case OMP_CLAUSE_DEFAULT
:
1798 case OMP_CLAUSE_NUM_THREADS
:
1799 case OMP_CLAUSE_NUM_TEAMS
:
1800 case OMP_CLAUSE_THREAD_LIMIT
:
1801 case OMP_CLAUSE_DEVICE
:
1802 case OMP_CLAUSE_SCHEDULE
:
1803 case OMP_CLAUSE_DIST_SCHEDULE
:
1804 case OMP_CLAUSE_NOWAIT
:
1805 case OMP_CLAUSE_ORDERED
:
1806 case OMP_CLAUSE_COLLAPSE
:
1807 case OMP_CLAUSE_UNTIED
:
1808 case OMP_CLAUSE_FINAL
:
1809 case OMP_CLAUSE_MERGEABLE
:
1810 case OMP_CLAUSE_PROC_BIND
:
1811 case OMP_CLAUSE_SAFELEN
:
1812 case OMP_CLAUSE_ALIGNED
:
1813 case OMP_CLAUSE_DEPEND
:
1814 case OMP_CLAUSE__LOOPTEMP_
:
1816 case OMP_CLAUSE_FROM
:
1824 if (scan_array_reductions
)
1825 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1826 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1827 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1829 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
1830 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
1832 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
1833 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1834 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
1835 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1836 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1837 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
1840 /* Create a new name for omp child function. Returns an identifier. */
1843 create_omp_child_function_name (bool task_copy
)
1845 return (clone_function_name (current_function_decl
,
1846 task_copy
? "_omp_cpyfn" : "_omp_fn"));
1849 /* Build a decl for the omp child function. It'll not contain a body
1850 yet, just the bare decl. */
1853 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
1855 tree decl
, type
, name
, t
;
1857 name
= create_omp_child_function_name (task_copy
);
1859 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1860 ptr_type_node
, NULL_TREE
);
1862 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
1864 decl
= build_decl (gimple_location (ctx
->stmt
),
1865 FUNCTION_DECL
, name
, type
);
1868 ctx
->cb
.dst_fn
= decl
;
1870 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
1872 TREE_STATIC (decl
) = 1;
1873 TREE_USED (decl
) = 1;
1874 DECL_ARTIFICIAL (decl
) = 1;
1875 DECL_NAMELESS (decl
) = 1;
1876 DECL_IGNORED_P (decl
) = 0;
1877 TREE_PUBLIC (decl
) = 0;
1878 DECL_UNINLINABLE (decl
) = 1;
1879 DECL_EXTERNAL (decl
) = 0;
1880 DECL_CONTEXT (decl
) = NULL_TREE
;
1881 DECL_INITIAL (decl
) = make_node (BLOCK
);
1882 bool target_p
= false;
1883 if (lookup_attribute ("omp declare target",
1884 DECL_ATTRIBUTES (current_function_decl
)))
1889 for (octx
= ctx
; octx
; octx
= octx
->outer
)
1890 if (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
1891 && gimple_omp_target_kind (octx
->stmt
)
1892 == GF_OMP_TARGET_KIND_REGION
)
1899 DECL_ATTRIBUTES (decl
)
1900 = tree_cons (get_identifier ("omp declare target"),
1901 NULL_TREE
, DECL_ATTRIBUTES (decl
));
1903 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1904 RESULT_DECL
, NULL_TREE
, void_type_node
);
1905 DECL_ARTIFICIAL (t
) = 1;
1906 DECL_IGNORED_P (t
) = 1;
1907 DECL_CONTEXT (t
) = decl
;
1908 DECL_RESULT (decl
) = t
;
1910 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1911 PARM_DECL
, get_identifier (".omp_data_i"), ptr_type_node
);
1912 DECL_ARTIFICIAL (t
) = 1;
1913 DECL_NAMELESS (t
) = 1;
1914 DECL_ARG_TYPE (t
) = ptr_type_node
;
1915 DECL_CONTEXT (t
) = current_function_decl
;
1917 DECL_ARGUMENTS (decl
) = t
;
1919 ctx
->receiver_decl
= t
;
1922 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1923 PARM_DECL
, get_identifier (".omp_data_o"),
1925 DECL_ARTIFICIAL (t
) = 1;
1926 DECL_NAMELESS (t
) = 1;
1927 DECL_ARG_TYPE (t
) = ptr_type_node
;
1928 DECL_CONTEXT (t
) = current_function_decl
;
1930 TREE_ADDRESSABLE (t
) = 1;
1931 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1932 DECL_ARGUMENTS (decl
) = t
;
1935 /* Allocate memory for the function structure. The call to
1936 allocate_struct_function clobbers CFUN, so we need to restore
1938 push_struct_function (decl
);
1939 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
1943 /* Callback for walk_gimple_seq. Check if combined parallel
1944 contains gimple_omp_for_combined_into_p OMP_FOR. */
1947 find_combined_for (gimple_stmt_iterator
*gsi_p
,
1948 bool *handled_ops_p
,
1949 struct walk_stmt_info
*wi
)
1951 gimple stmt
= gsi_stmt (*gsi_p
);
1953 *handled_ops_p
= true;
1954 switch (gimple_code (stmt
))
1958 case GIMPLE_OMP_FOR
:
1959 if (gimple_omp_for_combined_into_p (stmt
)
1960 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
1963 return integer_zero_node
;
1972 /* Scan an OpenMP parallel directive. */
1975 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
1979 gimple stmt
= gsi_stmt (*gsi
);
1981 /* Ignore parallel directives with empty bodies, unless there
1982 are copyin clauses. */
1984 && empty_body_p (gimple_omp_body (stmt
))
1985 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
1986 OMP_CLAUSE_COPYIN
) == NULL
)
1988 gsi_replace (gsi
, gimple_build_nop (), false);
1992 if (gimple_omp_parallel_combined_p (stmt
))
1995 struct walk_stmt_info wi
;
1997 memset (&wi
, 0, sizeof (wi
));
1999 walk_gimple_seq (gimple_omp_body (stmt
),
2000 find_combined_for
, NULL
, &wi
);
2001 for_stmt
= (gimple
) wi
.info
;
2004 struct omp_for_data fd
;
2005 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2006 /* We need two temporaries with fd.loop.v type (istart/iend)
2007 and then (fd.collapse - 1) temporaries with the same
2008 type for count2 ... countN-1 vars if not constant. */
2009 size_t count
= 2, i
;
2010 tree type
= fd
.iter_type
;
2012 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2013 count
+= fd
.collapse
- 1;
2014 for (i
= 0; i
< count
; i
++)
2016 tree temp
= create_tmp_var (type
, NULL
);
2017 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2018 OMP_CLAUSE__LOOPTEMP_
);
2019 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2020 OMP_CLAUSE_DECL (c
) = temp
;
2021 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2022 gimple_omp_parallel_set_clauses (stmt
, c
);
2027 ctx
= new_omp_context (stmt
, outer_ctx
);
2028 if (taskreg_nesting_level
> 1)
2029 ctx
->is_nested
= true;
2030 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2031 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2032 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2033 name
= create_tmp_var_name (".omp_data_s");
2034 name
= build_decl (gimple_location (stmt
),
2035 TYPE_DECL
, name
, ctx
->record_type
);
2036 DECL_ARTIFICIAL (name
) = 1;
2037 DECL_NAMELESS (name
) = 1;
2038 TYPE_NAME (ctx
->record_type
) = name
;
2039 create_omp_child_function (ctx
, false);
2040 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2042 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2043 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2045 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2046 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2049 layout_type (ctx
->record_type
);
2050 fixup_child_record_type (ctx
);
2054 /* Scan an OpenMP task directive. */
2057 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2061 gimple stmt
= gsi_stmt (*gsi
);
2062 location_t loc
= gimple_location (stmt
);
2064 /* Ignore task directives with empty bodies. */
2066 && empty_body_p (gimple_omp_body (stmt
)))
2068 gsi_replace (gsi
, gimple_build_nop (), false);
2072 ctx
= new_omp_context (stmt
, outer_ctx
);
2073 if (taskreg_nesting_level
> 1)
2074 ctx
->is_nested
= true;
2075 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2076 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2077 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2078 name
= create_tmp_var_name (".omp_data_s");
2079 name
= build_decl (gimple_location (stmt
),
2080 TYPE_DECL
, name
, ctx
->record_type
);
2081 DECL_ARTIFICIAL (name
) = 1;
2082 DECL_NAMELESS (name
) = 1;
2083 TYPE_NAME (ctx
->record_type
) = name
;
2084 create_omp_child_function (ctx
, false);
2085 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2087 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2089 if (ctx
->srecord_type
)
2091 name
= create_tmp_var_name (".omp_data_a");
2092 name
= build_decl (gimple_location (stmt
),
2093 TYPE_DECL
, name
, ctx
->srecord_type
);
2094 DECL_ARTIFICIAL (name
) = 1;
2095 DECL_NAMELESS (name
) = 1;
2096 TYPE_NAME (ctx
->srecord_type
) = name
;
2097 create_omp_child_function (ctx
, true);
2100 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2102 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2104 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2105 t
= build_int_cst (long_integer_type_node
, 0);
2106 gimple_omp_task_set_arg_size (stmt
, t
);
2107 t
= build_int_cst (long_integer_type_node
, 1);
2108 gimple_omp_task_set_arg_align (stmt
, t
);
2112 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2113 /* Move VLA fields to the end. */
2114 p
= &TYPE_FIELDS (ctx
->record_type
);
2116 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2117 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2120 *p
= TREE_CHAIN (*p
);
2121 TREE_CHAIN (*q
) = NULL_TREE
;
2122 q
= &TREE_CHAIN (*q
);
2125 p
= &DECL_CHAIN (*p
);
2127 layout_type (ctx
->record_type
);
2128 fixup_child_record_type (ctx
);
2129 if (ctx
->srecord_type
)
2130 layout_type (ctx
->srecord_type
);
2131 t
= fold_convert_loc (loc
, long_integer_type_node
,
2132 TYPE_SIZE_UNIT (ctx
->record_type
));
2133 gimple_omp_task_set_arg_size (stmt
, t
);
2134 t
= build_int_cst (long_integer_type_node
,
2135 TYPE_ALIGN_UNIT (ctx
->record_type
));
2136 gimple_omp_task_set_arg_align (stmt
, t
);
2141 /* Scan an OpenMP loop directive. */
2144 scan_omp_for (gimple stmt
, omp_context
*outer_ctx
)
2149 ctx
= new_omp_context (stmt
, outer_ctx
);
2151 scan_sharing_clauses (gimple_omp_for_clauses (stmt
), ctx
);
2153 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2154 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2156 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2157 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2158 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2159 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2161 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2164 /* Scan an OpenMP sections directive. */
2167 scan_omp_sections (gimple stmt
, omp_context
*outer_ctx
)
2171 ctx
= new_omp_context (stmt
, outer_ctx
);
2172 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2173 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2176 /* Scan an OpenMP single directive. */
2179 scan_omp_single (gimple stmt
, omp_context
*outer_ctx
)
2184 ctx
= new_omp_context (stmt
, outer_ctx
);
2185 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2186 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2187 name
= create_tmp_var_name (".omp_copy_s");
2188 name
= build_decl (gimple_location (stmt
),
2189 TYPE_DECL
, name
, ctx
->record_type
);
2190 TYPE_NAME (ctx
->record_type
) = name
;
2192 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2193 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2195 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2196 ctx
->record_type
= NULL
;
2198 layout_type (ctx
->record_type
);
2201 /* Scan an OpenMP target{, data, update} directive. */
2204 scan_omp_target (gimple stmt
, omp_context
*outer_ctx
)
2208 int kind
= gimple_omp_target_kind (stmt
);
2210 ctx
= new_omp_context (stmt
, outer_ctx
);
2211 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2212 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2213 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2214 name
= create_tmp_var_name (".omp_data_t");
2215 name
= build_decl (gimple_location (stmt
),
2216 TYPE_DECL
, name
, ctx
->record_type
);
2217 DECL_ARTIFICIAL (name
) = 1;
2218 DECL_NAMELESS (name
) = 1;
2219 TYPE_NAME (ctx
->record_type
) = name
;
2220 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2222 create_omp_child_function (ctx
, false);
2223 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2226 scan_sharing_clauses (gimple_omp_target_clauses (stmt
), ctx
);
2227 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2229 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2230 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2233 TYPE_FIELDS (ctx
->record_type
)
2234 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2235 #ifdef ENABLE_CHECKING
2237 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2238 for (field
= TYPE_FIELDS (ctx
->record_type
);
2240 field
= DECL_CHAIN (field
))
2241 gcc_assert (DECL_ALIGN (field
) == align
);
2243 layout_type (ctx
->record_type
);
2244 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2245 fixup_child_record_type (ctx
);
2249 /* Scan an OpenMP teams directive. */
2252 scan_omp_teams (gimple stmt
, omp_context
*outer_ctx
)
2254 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2255 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2256 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2259 /* Check OpenMP nesting restrictions. */
2261 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2265 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2266 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2268 error_at (gimple_location (stmt
),
2269 "OpenMP constructs may not be nested inside simd region");
2272 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2274 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2275 || (gimple_omp_for_kind (stmt
)
2276 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2277 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2279 error_at (gimple_location (stmt
),
2280 "only distribute or parallel constructs are allowed to "
2281 "be closely nested inside teams construct");
2286 switch (gimple_code (stmt
))
2288 case GIMPLE_OMP_FOR
:
2289 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
2291 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2293 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2295 error_at (gimple_location (stmt
),
2296 "distribute construct must be closely nested inside "
2304 if (is_gimple_call (stmt
)
2305 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2306 == BUILT_IN_GOMP_CANCEL
2307 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2308 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2310 const char *bad
= NULL
;
2311 const char *kind
= NULL
;
2314 error_at (gimple_location (stmt
), "orphaned %qs construct",
2315 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2316 == BUILT_IN_GOMP_CANCEL
2317 ? "#pragma omp cancel"
2318 : "#pragma omp cancellation point");
2321 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2322 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2326 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2327 bad
= "#pragma omp parallel";
2328 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2329 == BUILT_IN_GOMP_CANCEL
2330 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2331 ctx
->cancellable
= true;
2335 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2336 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2337 bad
= "#pragma omp for";
2338 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2339 == BUILT_IN_GOMP_CANCEL
2340 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2342 ctx
->cancellable
= true;
2343 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2345 warning_at (gimple_location (stmt
), 0,
2346 "%<#pragma omp cancel for%> inside "
2347 "%<nowait%> for construct");
2348 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2349 OMP_CLAUSE_ORDERED
))
2350 warning_at (gimple_location (stmt
), 0,
2351 "%<#pragma omp cancel for%> inside "
2352 "%<ordered%> for construct");
2357 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2358 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2359 bad
= "#pragma omp sections";
2360 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2361 == BUILT_IN_GOMP_CANCEL
2362 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2364 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2366 ctx
->cancellable
= true;
2367 if (find_omp_clause (gimple_omp_sections_clauses
2370 warning_at (gimple_location (stmt
), 0,
2371 "%<#pragma omp cancel sections%> inside "
2372 "%<nowait%> sections construct");
2376 gcc_assert (ctx
->outer
2377 && gimple_code (ctx
->outer
->stmt
)
2378 == GIMPLE_OMP_SECTIONS
);
2379 ctx
->outer
->cancellable
= true;
2380 if (find_omp_clause (gimple_omp_sections_clauses
2383 warning_at (gimple_location (stmt
), 0,
2384 "%<#pragma omp cancel sections%> inside "
2385 "%<nowait%> sections construct");
2391 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2392 bad
= "#pragma omp task";
2394 ctx
->cancellable
= true;
2398 error_at (gimple_location (stmt
), "invalid arguments");
2403 error_at (gimple_location (stmt
),
2404 "%<%s %s%> construct not closely nested inside of %qs",
2405 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2406 == BUILT_IN_GOMP_CANCEL
2407 ? "#pragma omp cancel"
2408 : "#pragma omp cancellation point", kind
, bad
);
2413 case GIMPLE_OMP_SECTIONS
:
2414 case GIMPLE_OMP_SINGLE
:
2415 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2416 switch (gimple_code (ctx
->stmt
))
2418 case GIMPLE_OMP_FOR
:
2419 case GIMPLE_OMP_SECTIONS
:
2420 case GIMPLE_OMP_SINGLE
:
2421 case GIMPLE_OMP_ORDERED
:
2422 case GIMPLE_OMP_MASTER
:
2423 case GIMPLE_OMP_TASK
:
2424 case GIMPLE_OMP_CRITICAL
:
2425 if (is_gimple_call (stmt
))
2427 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2428 != BUILT_IN_GOMP_BARRIER
)
2430 error_at (gimple_location (stmt
),
2431 "barrier region may not be closely nested inside "
2432 "of work-sharing, critical, ordered, master or "
2433 "explicit task region");
2436 error_at (gimple_location (stmt
),
2437 "work-sharing region may not be closely nested inside "
2438 "of work-sharing, critical, ordered, master or explicit "
2441 case GIMPLE_OMP_PARALLEL
:
2447 case GIMPLE_OMP_MASTER
:
2448 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2449 switch (gimple_code (ctx
->stmt
))
2451 case GIMPLE_OMP_FOR
:
2452 case GIMPLE_OMP_SECTIONS
:
2453 case GIMPLE_OMP_SINGLE
:
2454 case GIMPLE_OMP_TASK
:
2455 error_at (gimple_location (stmt
),
2456 "master region may not be closely nested inside "
2457 "of work-sharing or explicit task region");
2459 case GIMPLE_OMP_PARALLEL
:
2465 case GIMPLE_OMP_ORDERED
:
2466 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2467 switch (gimple_code (ctx
->stmt
))
2469 case GIMPLE_OMP_CRITICAL
:
2470 case GIMPLE_OMP_TASK
:
2471 error_at (gimple_location (stmt
),
2472 "ordered region may not be closely nested inside "
2473 "of critical or explicit task region");
2475 case GIMPLE_OMP_FOR
:
2476 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2477 OMP_CLAUSE_ORDERED
) == NULL
)
2479 error_at (gimple_location (stmt
),
2480 "ordered region must be closely nested inside "
2481 "a loop region with an ordered clause");
2485 case GIMPLE_OMP_PARALLEL
:
2486 error_at (gimple_location (stmt
),
2487 "ordered region must be closely nested inside "
2488 "a loop region with an ordered clause");
2494 case GIMPLE_OMP_CRITICAL
:
2495 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2496 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_CRITICAL
2497 && (gimple_omp_critical_name (stmt
)
2498 == gimple_omp_critical_name (ctx
->stmt
)))
2500 error_at (gimple_location (stmt
),
2501 "critical region may not be nested inside a critical "
2502 "region with the same name");
2506 case GIMPLE_OMP_TEAMS
:
2508 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2509 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2511 error_at (gimple_location (stmt
),
2512 "teams construct not closely nested inside of target "
2517 case GIMPLE_OMP_TARGET
:
2518 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2519 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
2520 && gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_REGION
)
2523 switch (gimple_omp_target_kind (stmt
))
2525 case GF_OMP_TARGET_KIND_REGION
: name
= "target"; break;
2526 case GF_OMP_TARGET_KIND_DATA
: name
= "target data"; break;
2527 case GF_OMP_TARGET_KIND_UPDATE
: name
= "target update"; break;
2528 default: gcc_unreachable ();
2530 warning_at (gimple_location (stmt
), 0,
2531 "%s construct inside of target region", name
);
2541 /* Helper function scan_omp.
2543 Callback for walk_tree or operators in walk_gimple_stmt used to
2544 scan for OpenMP directives in TP. */
2547 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
2549 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
2550 omp_context
*ctx
= (omp_context
*) wi
->info
;
2553 switch (TREE_CODE (t
))
2560 *tp
= remap_decl (t
, &ctx
->cb
);
2564 if (ctx
&& TYPE_P (t
))
2565 *tp
= remap_type (t
, &ctx
->cb
);
2566 else if (!DECL_P (t
))
2571 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
2572 if (tem
!= TREE_TYPE (t
))
2574 if (TREE_CODE (t
) == INTEGER_CST
)
2575 *tp
= wide_int_to_tree (tem
, t
);
2577 TREE_TYPE (t
) = tem
;
2587 /* Return true if FNDECL is a setjmp or a longjmp. */
2590 setjmp_or_longjmp_p (const_tree fndecl
)
2592 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
2593 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
2594 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
2597 tree declname
= DECL_NAME (fndecl
);
2600 const char *name
= IDENTIFIER_POINTER (declname
);
2601 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
2605 /* Helper function for scan_omp.
2607 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2608 the current statement in GSI. */
2611 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
2612 struct walk_stmt_info
*wi
)
2614 gimple stmt
= gsi_stmt (*gsi
);
2615 omp_context
*ctx
= (omp_context
*) wi
->info
;
2617 if (gimple_has_location (stmt
))
2618 input_location
= gimple_location (stmt
);
2620 /* Check the OpenMP nesting restrictions. */
2621 bool remove
= false;
2622 if (is_gimple_omp (stmt
))
2623 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2624 else if (is_gimple_call (stmt
))
2626 tree fndecl
= gimple_call_fndecl (stmt
);
2629 if (setjmp_or_longjmp_p (fndecl
)
2631 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2632 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2635 error_at (gimple_location (stmt
),
2636 "setjmp/longjmp inside simd construct");
2638 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
2639 switch (DECL_FUNCTION_CODE (fndecl
))
2641 case BUILT_IN_GOMP_BARRIER
:
2642 case BUILT_IN_GOMP_CANCEL
:
2643 case BUILT_IN_GOMP_CANCELLATION_POINT
:
2644 case BUILT_IN_GOMP_TASKYIELD
:
2645 case BUILT_IN_GOMP_TASKWAIT
:
2646 case BUILT_IN_GOMP_TASKGROUP_START
:
2647 case BUILT_IN_GOMP_TASKGROUP_END
:
2648 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2657 stmt
= gimple_build_nop ();
2658 gsi_replace (gsi
, stmt
, false);
2661 *handled_ops_p
= true;
2663 switch (gimple_code (stmt
))
2665 case GIMPLE_OMP_PARALLEL
:
2666 taskreg_nesting_level
++;
2667 scan_omp_parallel (gsi
, ctx
);
2668 taskreg_nesting_level
--;
2671 case GIMPLE_OMP_TASK
:
2672 taskreg_nesting_level
++;
2673 scan_omp_task (gsi
, ctx
);
2674 taskreg_nesting_level
--;
2677 case GIMPLE_OMP_FOR
:
2678 scan_omp_for (stmt
, ctx
);
2681 case GIMPLE_OMP_SECTIONS
:
2682 scan_omp_sections (stmt
, ctx
);
2685 case GIMPLE_OMP_SINGLE
:
2686 scan_omp_single (stmt
, ctx
);
2689 case GIMPLE_OMP_SECTION
:
2690 case GIMPLE_OMP_MASTER
:
2691 case GIMPLE_OMP_TASKGROUP
:
2692 case GIMPLE_OMP_ORDERED
:
2693 case GIMPLE_OMP_CRITICAL
:
2694 ctx
= new_omp_context (stmt
, ctx
);
2695 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2698 case GIMPLE_OMP_TARGET
:
2699 scan_omp_target (stmt
, ctx
);
2702 case GIMPLE_OMP_TEAMS
:
2703 scan_omp_teams (stmt
, ctx
);
2710 *handled_ops_p
= false;
2712 for (var
= gimple_bind_vars (stmt
); var
; var
= DECL_CHAIN (var
))
2713 insert_decl_map (&ctx
->cb
, var
, var
);
2717 *handled_ops_p
= false;
2725 /* Scan all the statements starting at the current statement. CTX
2726 contains context information about the OpenMP directives and
2727 clauses found during the scan. */
2730 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
2732 location_t saved_location
;
2733 struct walk_stmt_info wi
;
2735 memset (&wi
, 0, sizeof (wi
));
2737 wi
.want_locations
= true;
2739 saved_location
= input_location
;
2740 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
2741 input_location
= saved_location
;
2744 /* Re-gimplification and code generation routines. */
2746 /* Build a call to GOMP_barrier. */
2749 build_omp_barrier (tree lhs
)
2751 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
2752 : BUILT_IN_GOMP_BARRIER
);
2753 gimple g
= gimple_build_call (fndecl
, 0);
2755 gimple_call_set_lhs (g
, lhs
);
2759 /* If a context was created for STMT when it was scanned, return it. */
2761 static omp_context
*
2762 maybe_lookup_ctx (gimple stmt
)
2765 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
2766 return n
? (omp_context
*) n
->value
: NULL
;
2770 /* Find the mapping for DECL in CTX or the immediately enclosing
2771 context that has a mapping for DECL.
2773 If CTX is a nested parallel directive, we may have to use the decl
2774 mappings created in CTX's parent context. Suppose that we have the
2775 following parallel nesting (variable UIDs showed for clarity):
2778 #omp parallel shared(iD.1562) -> outer parallel
2779 iD.1562 = iD.1562 + 1;
2781 #omp parallel shared (iD.1562) -> inner parallel
2782 iD.1562 = iD.1562 - 1;
2784 Each parallel structure will create a distinct .omp_data_s structure
2785 for copying iD.1562 in/out of the directive:
2787 outer parallel .omp_data_s.1.i -> iD.1562
2788 inner parallel .omp_data_s.2.i -> iD.1562
2790 A shared variable mapping will produce a copy-out operation before
2791 the parallel directive and a copy-in operation after it. So, in
2792 this case we would have:
2795 .omp_data_o.1.i = iD.1562;
2796 #omp parallel shared(iD.1562) -> outer parallel
2797 .omp_data_i.1 = &.omp_data_o.1
2798 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2800 .omp_data_o.2.i = iD.1562; -> **
2801 #omp parallel shared(iD.1562) -> inner parallel
2802 .omp_data_i.2 = &.omp_data_o.2
2803 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2806 ** This is a problem. The symbol iD.1562 cannot be referenced
2807 inside the body of the outer parallel region. But since we are
2808 emitting this copy operation while expanding the inner parallel
2809 directive, we need to access the CTX structure of the outer
2810 parallel directive to get the correct mapping:
2812 .omp_data_o.2.i = .omp_data_i.1->i
2814 Since there may be other workshare or parallel directives enclosing
2815 the parallel directive, it may be necessary to walk up the context
2816 parent chain. This is not a problem in general because nested
2817 parallelism happens only rarely. */
2820 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2825 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2826 t
= maybe_lookup_decl (decl
, up
);
2828 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
2830 return t
? t
: decl
;
2834 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2835 in outer contexts. */
2838 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2843 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2844 t
= maybe_lookup_decl (decl
, up
);
2846 return t
? t
: decl
;
2850 /* Construct the initialization value for reduction CLAUSE. */
2853 omp_reduction_init (tree clause
, tree type
)
2855 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
2856 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
2863 case TRUTH_ORIF_EXPR
:
2864 case TRUTH_XOR_EXPR
:
2866 return build_zero_cst (type
);
2869 case TRUTH_AND_EXPR
:
2870 case TRUTH_ANDIF_EXPR
:
2872 return fold_convert_loc (loc
, type
, integer_one_node
);
2875 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
2878 if (SCALAR_FLOAT_TYPE_P (type
))
2880 REAL_VALUE_TYPE max
, min
;
2881 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2884 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
2887 real_maxval (&min
, 1, TYPE_MODE (type
));
2888 return build_real (type
, min
);
2892 gcc_assert (INTEGRAL_TYPE_P (type
));
2893 return TYPE_MIN_VALUE (type
);
2897 if (SCALAR_FLOAT_TYPE_P (type
))
2899 REAL_VALUE_TYPE max
;
2900 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2903 real_maxval (&max
, 0, TYPE_MODE (type
));
2904 return build_real (type
, max
);
2908 gcc_assert (INTEGRAL_TYPE_P (type
));
2909 return TYPE_MAX_VALUE (type
);
2917 /* Return alignment to be assumed for var in CLAUSE, which should be
2918 OMP_CLAUSE_ALIGNED. */
2921 omp_clause_aligned_alignment (tree clause
)
2923 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
2924 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
2926 /* Otherwise return implementation defined alignment. */
2927 unsigned int al
= 1;
2928 enum machine_mode mode
, vmode
;
2929 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2931 vs
= 1 << floor_log2 (vs
);
2932 static enum mode_class classes
[]
2933 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
2934 for (int i
= 0; i
< 4; i
+= 2)
2935 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
2937 mode
= GET_MODE_WIDER_MODE (mode
))
2939 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
2940 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
2943 && GET_MODE_SIZE (vmode
) < vs
2944 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
2945 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
2947 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
2948 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
2950 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
2951 / GET_MODE_SIZE (mode
));
2952 if (TYPE_MODE (type
) != vmode
)
2954 if (TYPE_ALIGN_UNIT (type
) > al
)
2955 al
= TYPE_ALIGN_UNIT (type
);
2957 return build_int_cst (integer_type_node
, al
);
2960 /* Return maximum possible vectorization factor for the target. */
2967 || !flag_tree_loop_optimize
2968 || (!flag_tree_loop_vectorize
2969 && (global_options_set
.x_flag_tree_loop_vectorize
2970 || global_options_set
.x_flag_tree_vectorize
)))
2973 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2976 vs
= 1 << floor_log2 (vs
);
2979 enum machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
2980 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
2981 return GET_MODE_NUNITS (vqimode
);
2985 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
2989 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
2990 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
2994 max_vf
= omp_max_vf ();
2997 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2998 OMP_CLAUSE_SAFELEN
);
2999 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3001 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3003 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3007 idx
= create_tmp_var (unsigned_type_node
, NULL
);
3008 lane
= create_tmp_var (unsigned_type_node
, NULL
);
3014 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3015 tree avar
= create_tmp_var_raw (atype
, NULL
);
3016 if (TREE_ADDRESSABLE (new_var
))
3017 TREE_ADDRESSABLE (avar
) = 1;
3018 DECL_ATTRIBUTES (avar
)
3019 = tree_cons (get_identifier ("omp simd array"), NULL
,
3020 DECL_ATTRIBUTES (avar
));
3021 gimple_add_tmp_var (avar
);
3022 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3023 NULL_TREE
, NULL_TREE
);
3024 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3025 NULL_TREE
, NULL_TREE
);
3026 if (DECL_P (new_var
))
3028 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3029 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3034 /* Helper function of lower_rec_input_clauses. For a reference
3035 in simd reduction, add an underlying variable it will reference. */
3038 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3040 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3041 if (TREE_CONSTANT (z
))
3043 const char *name
= NULL
;
3044 if (DECL_NAME (new_vard
))
3045 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3047 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3048 gimple_add_tmp_var (z
);
3049 TREE_ADDRESSABLE (z
) = 1;
3050 z
= build_fold_addr_expr_loc (loc
, z
);
3051 gimplify_assign (new_vard
, z
, ilist
);
3055 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3056 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3057 private variables. Initialization statements go in ILIST, while calls
3058 to destructors go in DLIST. */
3061 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3062 omp_context
*ctx
, struct omp_for_data
*fd
)
3064 tree c
, dtor
, copyin_seq
, x
, ptr
;
3065 bool copyin_by_ref
= false;
3066 bool lastprivate_firstprivate
= false;
3067 bool reduction_omp_orig_ref
= false;
3069 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3070 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
3072 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3073 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3074 gimple_seq llist
[2] = { NULL
, NULL
};
3078 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3079 with data sharing clauses referencing variable sized vars. That
3080 is unnecessarily hard to support and very unlikely to result in
3081 vectorized code anyway. */
3083 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3084 switch (OMP_CLAUSE_CODE (c
))
3086 case OMP_CLAUSE_REDUCTION
:
3087 case OMP_CLAUSE_PRIVATE
:
3088 case OMP_CLAUSE_FIRSTPRIVATE
:
3089 case OMP_CLAUSE_LASTPRIVATE
:
3090 case OMP_CLAUSE_LINEAR
:
3091 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3098 /* Do all the fixed sized types in the first pass, and the variable sized
3099 types in the second pass. This makes sure that the scalar arguments to
3100 the variable sized types are processed before we use them in the
3101 variable sized operations. */
3102 for (pass
= 0; pass
< 2; ++pass
)
3104 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3106 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3109 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3113 case OMP_CLAUSE_PRIVATE
:
3114 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3117 case OMP_CLAUSE_SHARED
:
3118 /* Ignore shared directives in teams construct. */
3119 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3121 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3123 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3126 case OMP_CLAUSE_FIRSTPRIVATE
:
3127 case OMP_CLAUSE_COPYIN
:
3128 case OMP_CLAUSE_LINEAR
:
3130 case OMP_CLAUSE_REDUCTION
:
3131 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3132 reduction_omp_orig_ref
= true;
3134 case OMP_CLAUSE__LOOPTEMP_
:
3135 /* Handle _looptemp_ clauses only on parallel. */
3139 case OMP_CLAUSE_LASTPRIVATE
:
3140 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3142 lastprivate_firstprivate
= true;
3146 /* Even without corresponding firstprivate, if
3147 decl is Fortran allocatable, it needs outer var
3150 && lang_hooks
.decls
.omp_private_outer_ref
3151 (OMP_CLAUSE_DECL (c
)))
3152 lastprivate_firstprivate
= true;
3154 case OMP_CLAUSE_ALIGNED
:
3157 var
= OMP_CLAUSE_DECL (c
);
3158 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3159 && !is_global_var (var
))
3161 new_var
= maybe_lookup_decl (var
, ctx
);
3162 if (new_var
== NULL_TREE
)
3163 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3164 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3165 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3166 omp_clause_aligned_alignment (c
));
3167 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3168 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3169 gimplify_and_add (x
, ilist
);
3171 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3172 && is_global_var (var
))
3174 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3175 new_var
= lookup_decl (var
, ctx
);
3176 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3177 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3178 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3179 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3180 omp_clause_aligned_alignment (c
));
3181 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3182 x
= create_tmp_var (ptype
, NULL
);
3183 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3184 gimplify_and_add (t
, ilist
);
3185 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3186 SET_DECL_VALUE_EXPR (new_var
, t
);
3187 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3194 new_var
= var
= OMP_CLAUSE_DECL (c
);
3195 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3196 new_var
= lookup_decl (var
, ctx
);
3198 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3203 else if (is_variable_sized (var
))
3205 /* For variable sized types, we need to allocate the
3206 actual storage here. Call alloca and store the
3207 result in the pointer decl that we created elsewhere. */
3211 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3216 ptr
= DECL_VALUE_EXPR (new_var
);
3217 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3218 ptr
= TREE_OPERAND (ptr
, 0);
3219 gcc_assert (DECL_P (ptr
));
3220 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3222 /* void *tmp = __builtin_alloca */
3223 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3224 stmt
= gimple_build_call (atmp
, 1, x
);
3225 tmp
= create_tmp_var_raw (ptr_type_node
, NULL
);
3226 gimple_add_tmp_var (tmp
);
3227 gimple_call_set_lhs (stmt
, tmp
);
3229 gimple_seq_add_stmt (ilist
, stmt
);
3231 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3232 gimplify_assign (ptr
, x
, ilist
);
3235 else if (is_reference (var
))
3237 /* For references that are being privatized for Fortran,
3238 allocate new backing storage for the new pointer
3239 variable. This allows us to avoid changing all the
3240 code that expects a pointer to something that expects
3241 a direct variable. */
3245 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3246 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3248 x
= build_receiver_ref (var
, false, ctx
);
3249 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3251 else if (TREE_CONSTANT (x
))
3253 /* For reduction in SIMD loop, defer adding the
3254 initialization of the reference, because if we decide
3255 to use SIMD array for it, the initilization could cause
3257 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3261 const char *name
= NULL
;
3262 if (DECL_NAME (var
))
3263 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3265 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3267 gimple_add_tmp_var (x
);
3268 TREE_ADDRESSABLE (x
) = 1;
3269 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3274 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3275 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3280 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3281 gimplify_assign (new_var
, x
, ilist
);
3284 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3286 else if (c_kind
== OMP_CLAUSE_REDUCTION
3287 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3295 switch (OMP_CLAUSE_CODE (c
))
3297 case OMP_CLAUSE_SHARED
:
3298 /* Ignore shared directives in teams construct. */
3299 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3301 /* Shared global vars are just accessed directly. */
3302 if (is_global_var (new_var
))
3304 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3305 needs to be delayed until after fixup_child_record_type so
3306 that we get the correct type during the dereference. */
3307 by_ref
= use_pointer_for_field (var
, ctx
);
3308 x
= build_receiver_ref (var
, by_ref
, ctx
);
3309 SET_DECL_VALUE_EXPR (new_var
, x
);
3310 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3312 /* ??? If VAR is not passed by reference, and the variable
3313 hasn't been initialized yet, then we'll get a warning for
3314 the store into the omp_data_s structure. Ideally, we'd be
3315 able to notice this and not store anything at all, but
3316 we're generating code too early. Suppress the warning. */
3318 TREE_NO_WARNING (var
) = 1;
3321 case OMP_CLAUSE_LASTPRIVATE
:
3322 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3326 case OMP_CLAUSE_PRIVATE
:
3327 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3328 x
= build_outer_var_ref (var
, ctx
);
3329 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3331 if (is_task_ctx (ctx
))
3332 x
= build_receiver_ref (var
, false, ctx
);
3334 x
= build_outer_var_ref (var
, ctx
);
3340 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3343 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3344 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3345 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3346 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3347 idx
, lane
, ivar
, lvar
))
3350 x
= lang_hooks
.decls
.omp_clause_default_ctor
3351 (c
, unshare_expr (ivar
), x
);
3353 gimplify_and_add (x
, &llist
[0]);
3356 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3359 gimple_seq tseq
= NULL
;
3362 gimplify_stmt (&dtor
, &tseq
);
3363 gimple_seq_add_seq (&llist
[1], tseq
);
3370 gimplify_and_add (nx
, ilist
);
3374 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3377 gimple_seq tseq
= NULL
;
3380 gimplify_stmt (&dtor
, &tseq
);
3381 gimple_seq_add_seq (dlist
, tseq
);
3385 case OMP_CLAUSE_LINEAR
:
3386 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3387 goto do_firstprivate
;
3388 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3391 x
= build_outer_var_ref (var
, ctx
);
3394 case OMP_CLAUSE_FIRSTPRIVATE
:
3395 if (is_task_ctx (ctx
))
3397 if (is_reference (var
) || is_variable_sized (var
))
3399 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3401 || use_pointer_for_field (var
, NULL
))
3403 x
= build_receiver_ref (var
, false, ctx
);
3404 SET_DECL_VALUE_EXPR (new_var
, x
);
3405 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3410 x
= build_outer_var_ref (var
, ctx
);
3413 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3414 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3416 tree stept
= POINTER_TYPE_P (TREE_TYPE (x
))
3417 ? sizetype
: TREE_TYPE (x
);
3418 tree t
= fold_convert (stept
,
3419 OMP_CLAUSE_LINEAR_STEP (c
));
3420 tree c
= find_omp_clause (clauses
,
3421 OMP_CLAUSE__LOOPTEMP_
);
3423 tree l
= OMP_CLAUSE_DECL (c
);
3424 if (fd
->collapse
== 1)
3426 tree n1
= fd
->loop
.n1
;
3427 tree step
= fd
->loop
.step
;
3428 tree itype
= TREE_TYPE (l
);
3429 if (POINTER_TYPE_P (itype
))
3430 itype
= signed_type_for (itype
);
3431 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3432 if (TYPE_UNSIGNED (itype
)
3433 && fd
->loop
.cond_code
== GT_EXPR
)
3434 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3435 fold_build1 (NEGATE_EXPR
,
3437 fold_build1 (NEGATE_EXPR
,
3440 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3442 t
= fold_build2 (MULT_EXPR
, stept
,
3443 fold_convert (stept
, l
), t
);
3444 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3445 x
= fold_build2 (POINTER_PLUS_EXPR
,
3446 TREE_TYPE (x
), x
, t
);
3448 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3451 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3452 || TREE_ADDRESSABLE (new_var
))
3453 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3454 idx
, lane
, ivar
, lvar
))
3456 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3458 tree iv
= create_tmp_var (TREE_TYPE (new_var
), NULL
);
3459 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3460 gimplify_and_add (x
, ilist
);
3461 gimple_stmt_iterator gsi
3462 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3464 = gimple_build_assign (unshare_expr (lvar
), iv
);
3465 gsi_insert_before_without_update (&gsi
, g
,
3467 tree stept
= POINTER_TYPE_P (TREE_TYPE (iv
))
3468 ? sizetype
: TREE_TYPE (iv
);
3469 tree t
= fold_convert (stept
,
3470 OMP_CLAUSE_LINEAR_STEP (c
));
3471 enum tree_code code
= PLUS_EXPR
;
3472 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
3473 code
= POINTER_PLUS_EXPR
;
3474 g
= gimple_build_assign_with_ops (code
, iv
, iv
, t
);
3475 gsi_insert_before_without_update (&gsi
, g
,
3479 x
= lang_hooks
.decls
.omp_clause_copy_ctor
3480 (c
, unshare_expr (ivar
), x
);
3481 gimplify_and_add (x
, &llist
[0]);
3482 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3485 gimple_seq tseq
= NULL
;
3488 gimplify_stmt (&dtor
, &tseq
);
3489 gimple_seq_add_seq (&llist
[1], tseq
);
3494 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
3495 gimplify_and_add (x
, ilist
);
3498 case OMP_CLAUSE__LOOPTEMP_
:
3499 gcc_assert (is_parallel_ctx (ctx
));
3500 x
= build_outer_var_ref (var
, ctx
);
3501 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3502 gimplify_and_add (x
, ilist
);
3505 case OMP_CLAUSE_COPYIN
:
3506 by_ref
= use_pointer_for_field (var
, NULL
);
3507 x
= build_receiver_ref (var
, by_ref
, ctx
);
3508 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
3509 append_to_statement_list (x
, ©in_seq
);
3510 copyin_by_ref
|= by_ref
;
3513 case OMP_CLAUSE_REDUCTION
:
3514 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3516 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3518 x
= build_outer_var_ref (var
, ctx
);
3520 if (is_reference (var
)
3521 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3523 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3524 SET_DECL_VALUE_EXPR (placeholder
, x
);
3525 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3526 tree new_vard
= new_var
;
3527 if (is_reference (var
))
3529 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3530 new_vard
= TREE_OPERAND (new_var
, 0);
3531 gcc_assert (DECL_P (new_vard
));
3534 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3535 idx
, lane
, ivar
, lvar
))
3537 if (new_vard
== new_var
)
3539 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
3540 SET_DECL_VALUE_EXPR (new_var
, ivar
);
3544 SET_DECL_VALUE_EXPR (new_vard
,
3545 build_fold_addr_expr (ivar
));
3546 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3548 x
= lang_hooks
.decls
.omp_clause_default_ctor
3549 (c
, unshare_expr (ivar
),
3550 build_outer_var_ref (var
, ctx
));
3552 gimplify_and_add (x
, &llist
[0]);
3553 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3555 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3556 lower_omp (&tseq
, ctx
);
3557 gimple_seq_add_seq (&llist
[0], tseq
);
3559 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3560 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3561 lower_omp (&tseq
, ctx
);
3562 gimple_seq_add_seq (&llist
[1], tseq
);
3563 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3564 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3565 if (new_vard
== new_var
)
3566 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3568 SET_DECL_VALUE_EXPR (new_vard
,
3569 build_fold_addr_expr (lvar
));
3570 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3575 gimplify_stmt (&dtor
, &tseq
);
3576 gimple_seq_add_seq (&llist
[1], tseq
);
3580 /* If this is a reference to constant size reduction var
3581 with placeholder, we haven't emitted the initializer
3582 for it because it is undesirable if SIMD arrays are used.
3583 But if they aren't used, we need to emit the deferred
3584 initialization now. */
3585 else if (is_reference (var
) && is_simd
)
3586 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3587 x
= lang_hooks
.decls
.omp_clause_default_ctor
3588 (c
, unshare_expr (new_var
),
3589 build_outer_var_ref (var
, ctx
));
3591 gimplify_and_add (x
, ilist
);
3592 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3594 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3595 lower_omp (&tseq
, ctx
);
3596 gimple_seq_add_seq (ilist
, tseq
);
3598 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3601 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3602 lower_omp (&tseq
, ctx
);
3603 gimple_seq_add_seq (dlist
, tseq
);
3604 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3606 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3611 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
3612 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
3613 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3615 /* reduction(-:var) sums up the partial results, so it
3616 acts identically to reduction(+:var). */
3617 if (code
== MINUS_EXPR
)
3620 tree new_vard
= new_var
;
3621 if (is_simd
&& is_reference (var
))
3623 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3624 new_vard
= TREE_OPERAND (new_var
, 0);
3625 gcc_assert (DECL_P (new_vard
));
3628 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3629 idx
, lane
, ivar
, lvar
))
3631 tree ref
= build_outer_var_ref (var
, ctx
);
3633 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
3635 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
3636 ref
= build_outer_var_ref (var
, ctx
);
3637 gimplify_assign (ref
, x
, &llist
[1]);
3639 if (new_vard
!= new_var
)
3641 SET_DECL_VALUE_EXPR (new_vard
,
3642 build_fold_addr_expr (lvar
));
3643 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3648 if (is_reference (var
) && is_simd
)
3649 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3650 gimplify_assign (new_var
, x
, ilist
);
3653 tree ref
= build_outer_var_ref (var
, ctx
);
3655 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3656 ref
= build_outer_var_ref (var
, ctx
);
3657 gimplify_assign (ref
, x
, dlist
);
3671 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
3672 /* Don't want uninit warnings on simduid, it is always uninitialized,
3673 but we use it not for the value, but for the DECL_UID only. */
3674 TREE_NO_WARNING (uid
) = 1;
3676 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
3677 gimple_call_set_lhs (g
, lane
);
3678 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3679 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
3680 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
3681 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
3682 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3683 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3684 g
= gimple_build_assign_with_ops (INTEGER_CST
, lane
,
3685 build_int_cst (unsigned_type_node
, 0),
3687 gimple_seq_add_stmt (ilist
, g
);
3688 for (int i
= 0; i
< 2; i
++)
3691 tree vf
= create_tmp_var (unsigned_type_node
, NULL
);
3692 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
3693 gimple_call_set_lhs (g
, vf
);
3694 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
3695 gimple_seq_add_stmt (seq
, g
);
3696 tree t
= build_int_cst (unsigned_type_node
, 0);
3697 g
= gimple_build_assign_with_ops (INTEGER_CST
, idx
, t
, NULL_TREE
);
3698 gimple_seq_add_stmt (seq
, g
);
3699 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
3700 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
3701 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
3702 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
3703 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
3704 gimple_seq_add_seq (seq
, llist
[i
]);
3705 t
= build_int_cst (unsigned_type_node
, 1);
3706 g
= gimple_build_assign_with_ops (PLUS_EXPR
, idx
, idx
, t
);
3707 gimple_seq_add_stmt (seq
, g
);
3708 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
3709 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
3710 gimple_seq_add_stmt (seq
, g
);
3711 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
3715 /* The copyin sequence is not to be executed by the main thread, since
3716 that would result in self-copies. Perhaps not visible to scalars,
3717 but it certainly is to C++ operator=. */
3720 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
3722 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
3723 build_int_cst (TREE_TYPE (x
), 0));
3724 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
3725 gimplify_and_add (x
, ilist
);
3728 /* If any copyin variable is passed by reference, we must ensure the
3729 master thread doesn't modify it before it is copied over in all
3730 threads. Similarly for variables in both firstprivate and
3731 lastprivate clauses we need to ensure the lastprivate copying
3732 happens after firstprivate copying in all threads. And similarly
3733 for UDRs if initializer expression refers to omp_orig. */
3734 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
3736 /* Don't add any barrier for #pragma omp simd or
3737 #pragma omp distribute. */
3738 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3739 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
3740 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
3743 /* If max_vf is non-zero, then we can use only a vectorization factor
3744 up to the max_vf we chose. So stick it into the safelen clause. */
3747 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3748 OMP_CLAUSE_SAFELEN
);
3750 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
3751 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3754 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
3755 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
3757 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3758 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3764 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3765 both parallel and workshare constructs. PREDICATE may be NULL if it's
3769 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
3772 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
3773 bool par_clauses
= false;
3774 tree simduid
= NULL
, lastlane
= NULL
;
3776 /* Early exit if there are no lastprivate or linear clauses. */
3777 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
3778 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
3779 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
3780 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
3782 if (clauses
== NULL
)
3784 /* If this was a workshare clause, see if it had been combined
3785 with its parallel. In that case, look for the clauses on the
3786 parallel statement itself. */
3787 if (is_parallel_ctx (ctx
))
3791 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3794 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3795 OMP_CLAUSE_LASTPRIVATE
);
3796 if (clauses
== NULL
)
3804 tree label_true
, arm1
, arm2
;
3806 label
= create_artificial_label (UNKNOWN_LOCATION
);
3807 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
3808 arm1
= TREE_OPERAND (predicate
, 0);
3809 arm2
= TREE_OPERAND (predicate
, 1);
3810 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3811 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3812 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
3814 gimple_seq_add_stmt (stmt_list
, stmt
);
3815 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
3818 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3819 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3821 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
3823 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
3826 for (c
= clauses
; c
;)
3829 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3831 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3832 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3833 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
3835 var
= OMP_CLAUSE_DECL (c
);
3836 new_var
= lookup_decl (var
, ctx
);
3838 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
3840 tree val
= DECL_VALUE_EXPR (new_var
);
3841 if (TREE_CODE (val
) == ARRAY_REF
3842 && VAR_P (TREE_OPERAND (val
, 0))
3843 && lookup_attribute ("omp simd array",
3844 DECL_ATTRIBUTES (TREE_OPERAND (val
,
3847 if (lastlane
== NULL
)
3849 lastlane
= create_tmp_var (unsigned_type_node
, NULL
);
3851 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
3853 TREE_OPERAND (val
, 1));
3854 gimple_call_set_lhs (g
, lastlane
);
3855 gimple_seq_add_stmt (stmt_list
, g
);
3857 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
3858 TREE_OPERAND (val
, 0), lastlane
,
3859 NULL_TREE
, NULL_TREE
);
3863 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3864 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
3866 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
3867 gimple_seq_add_seq (stmt_list
,
3868 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
3869 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
3871 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3872 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
3874 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
3875 gimple_seq_add_seq (stmt_list
,
3876 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
3877 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
3880 x
= build_outer_var_ref (var
, ctx
);
3881 if (is_reference (var
))
3882 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3883 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
3884 gimplify_and_add (x
, stmt_list
);
3886 c
= OMP_CLAUSE_CHAIN (c
);
3887 if (c
== NULL
&& !par_clauses
)
3889 /* If this was a workshare clause, see if it had been combined
3890 with its parallel. In that case, continue looking for the
3891 clauses also on the parallel statement itself. */
3892 if (is_parallel_ctx (ctx
))
3896 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3899 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3900 OMP_CLAUSE_LASTPRIVATE
);
3906 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
3910 /* Generate code to implement the REDUCTION clauses. */
3913 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
3915 gimple_seq sub_seq
= NULL
;
3920 /* SIMD reductions are handled in lower_rec_input_clauses. */
3921 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3922 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3925 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
3926 update in that case, otherwise use a lock. */
3927 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
3928 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
3930 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3932 /* Never use OMP_ATOMIC for array reductions or UDRs. */
3942 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3944 tree var
, ref
, new_var
;
3945 enum tree_code code
;
3946 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3948 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
3951 var
= OMP_CLAUSE_DECL (c
);
3952 new_var
= lookup_decl (var
, ctx
);
3953 if (is_reference (var
))
3954 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3955 ref
= build_outer_var_ref (var
, ctx
);
3956 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3958 /* reduction(-:var) sums up the partial results, so it acts
3959 identically to reduction(+:var). */
3960 if (code
== MINUS_EXPR
)
3965 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
3967 addr
= save_expr (addr
);
3968 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
3969 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
3970 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
3971 gimplify_and_add (x
, stmt_seqp
);
3975 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3977 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3979 if (is_reference (var
)
3980 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3982 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
3983 SET_DECL_VALUE_EXPR (placeholder
, ref
);
3984 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3985 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
3986 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
3987 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3988 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
3992 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3993 ref
= build_outer_var_ref (var
, ctx
);
3994 gimplify_assign (ref
, x
, &sub_seq
);
3998 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4000 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4002 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4004 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4006 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4010 /* Generate code to implement the COPYPRIVATE clauses. */
4013 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4018 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4020 tree var
, new_var
, ref
, x
;
4022 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4024 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4027 var
= OMP_CLAUSE_DECL (c
);
4028 by_ref
= use_pointer_for_field (var
, NULL
);
4030 ref
= build_sender_ref (var
, ctx
);
4031 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4034 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4035 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4037 gimplify_assign (ref
, x
, slist
);
4039 ref
= build_receiver_ref (var
, false, ctx
);
4042 ref
= fold_convert_loc (clause_loc
,
4043 build_pointer_type (TREE_TYPE (new_var
)),
4045 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4047 if (is_reference (var
))
4049 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4050 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4051 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4053 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4054 gimplify_and_add (x
, rlist
);
4059 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4060 and REDUCTION from the sender (aka parent) side. */
4063 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4068 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4070 tree val
, ref
, x
, var
;
4071 bool by_ref
, do_in
= false, do_out
= false;
4072 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4074 switch (OMP_CLAUSE_CODE (c
))
4076 case OMP_CLAUSE_PRIVATE
:
4077 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4080 case OMP_CLAUSE_FIRSTPRIVATE
:
4081 case OMP_CLAUSE_COPYIN
:
4082 case OMP_CLAUSE_LASTPRIVATE
:
4083 case OMP_CLAUSE_REDUCTION
:
4084 case OMP_CLAUSE__LOOPTEMP_
:
4090 val
= OMP_CLAUSE_DECL (c
);
4091 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4093 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4094 && is_global_var (var
))
4096 if (is_variable_sized (val
))
4098 by_ref
= use_pointer_for_field (val
, NULL
);
4100 switch (OMP_CLAUSE_CODE (c
))
4102 case OMP_CLAUSE_PRIVATE
:
4103 case OMP_CLAUSE_FIRSTPRIVATE
:
4104 case OMP_CLAUSE_COPYIN
:
4105 case OMP_CLAUSE__LOOPTEMP_
:
4109 case OMP_CLAUSE_LASTPRIVATE
:
4110 if (by_ref
|| is_reference (val
))
4112 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4119 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4124 case OMP_CLAUSE_REDUCTION
:
4126 do_out
= !(by_ref
|| is_reference (val
));
4135 ref
= build_sender_ref (val
, ctx
);
4136 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4137 gimplify_assign (ref
, x
, ilist
);
4138 if (is_task_ctx (ctx
))
4139 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4144 ref
= build_sender_ref (val
, ctx
);
4145 gimplify_assign (var
, ref
, olist
);
4150 /* Generate code to implement SHARED from the sender (aka parent)
4151 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4152 list things that got automatically shared. */
4155 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4157 tree var
, ovar
, nvar
, f
, x
, record_type
;
4159 if (ctx
->record_type
== NULL
)
4162 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4163 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4165 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4166 nvar
= maybe_lookup_decl (ovar
, ctx
);
4167 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4170 /* If CTX is a nested parallel directive. Find the immediately
4171 enclosing parallel or workshare construct that contains a
4172 mapping for OVAR. */
4173 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4175 if (use_pointer_for_field (ovar
, ctx
))
4177 x
= build_sender_ref (ovar
, ctx
);
4178 var
= build_fold_addr_expr (var
);
4179 gimplify_assign (x
, var
, ilist
);
4183 x
= build_sender_ref (ovar
, ctx
);
4184 gimplify_assign (x
, var
, ilist
);
4186 if (!TREE_READONLY (var
)
4187 /* We don't need to receive a new reference to a result
4188 or parm decl. In fact we may not store to it as we will
4189 invalidate any pending RSO and generate wrong gimple
4191 && !((TREE_CODE (var
) == RESULT_DECL
4192 || TREE_CODE (var
) == PARM_DECL
)
4193 && DECL_BY_REFERENCE (var
)))
4195 x
= build_sender_ref (ovar
, ctx
);
4196 gimplify_assign (var
, x
, olist
);
4203 /* A convenience function to build an empty GIMPLE_COND with just the
4207 gimple_build_cond_empty (tree cond
)
4209 enum tree_code pred_code
;
4212 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4213 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4217 /* Build the function calls to GOMP_parallel_start etc to actually
4218 generate the parallel operation. REGION is the parallel region
4219 being expanded. BB is the block where to insert the code. WS_ARGS
4220 will be set if this is a call to a combined parallel+workshare
4221 construct, it contains the list of additional arguments needed by
4222 the workshare construct. */
4225 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4226 gimple entry_stmt
, vec
<tree
, va_gc
> *ws_args
)
4228 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4229 gimple_stmt_iterator gsi
;
4231 enum built_in_function start_ix
;
4233 location_t clause_loc
;
4234 vec
<tree
, va_gc
> *args
;
4236 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4238 /* Determine what flavor of GOMP_parallel we will be
4240 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4241 if (is_combined_parallel (region
))
4243 switch (region
->inner
->type
)
4245 case GIMPLE_OMP_FOR
:
4246 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4247 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4248 + (region
->inner
->sched_kind
4249 == OMP_CLAUSE_SCHEDULE_RUNTIME
4250 ? 3 : region
->inner
->sched_kind
));
4251 start_ix
= (enum built_in_function
)start_ix2
;
4253 case GIMPLE_OMP_SECTIONS
:
4254 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4261 /* By default, the value of NUM_THREADS is zero (selected at run time)
4262 and there is no conditional. */
4264 val
= build_int_cst (unsigned_type_node
, 0);
4265 flags
= build_int_cst (unsigned_type_node
, 0);
4267 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4269 cond
= OMP_CLAUSE_IF_EXPR (c
);
4271 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4274 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4275 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4278 clause_loc
= gimple_location (entry_stmt
);
4280 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4282 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4284 /* Ensure 'val' is of the correct type. */
4285 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4287 /* If we found the clause 'if (cond)', build either
4288 (cond != 0) or (cond ? val : 1u). */
4291 cond
= gimple_boolify (cond
);
4293 if (integer_zerop (val
))
4294 val
= fold_build2_loc (clause_loc
,
4295 EQ_EXPR
, unsigned_type_node
, cond
,
4296 build_int_cst (TREE_TYPE (cond
), 0));
4299 basic_block cond_bb
, then_bb
, else_bb
;
4300 edge e
, e_then
, e_else
;
4301 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4303 tmp_var
= create_tmp_var (TREE_TYPE (val
), NULL
);
4304 if (gimple_in_ssa_p (cfun
))
4306 tmp_then
= make_ssa_name (tmp_var
, NULL
);
4307 tmp_else
= make_ssa_name (tmp_var
, NULL
);
4308 tmp_join
= make_ssa_name (tmp_var
, NULL
);
4317 e
= split_block (bb
, NULL
);
4322 then_bb
= create_empty_bb (cond_bb
);
4323 else_bb
= create_empty_bb (then_bb
);
4324 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4325 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4327 stmt
= gimple_build_cond_empty (cond
);
4328 gsi
= gsi_start_bb (cond_bb
);
4329 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4331 gsi
= gsi_start_bb (then_bb
);
4332 stmt
= gimple_build_assign (tmp_then
, val
);
4333 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4335 gsi
= gsi_start_bb (else_bb
);
4336 stmt
= gimple_build_assign
4337 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4338 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4340 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4341 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4342 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4343 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4344 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4345 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4347 if (gimple_in_ssa_p (cfun
))
4349 gimple phi
= create_phi_node (tmp_join
, bb
);
4350 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4351 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4357 gsi
= gsi_start_bb (bb
);
4358 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4359 false, GSI_CONTINUE_LINKING
);
4362 gsi
= gsi_last_bb (bb
);
4363 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4365 t1
= null_pointer_node
;
4367 t1
= build_fold_addr_expr (t
);
4368 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4370 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4371 args
->quick_push (t2
);
4372 args
->quick_push (t1
);
4373 args
->quick_push (val
);
4375 args
->splice (*ws_args
);
4376 args
->quick_push (flags
);
4378 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4379 builtin_decl_explicit (start_ix
), args
);
4381 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4382 false, GSI_CONTINUE_LINKING
);
4386 /* Build the function call to GOMP_task to actually
4387 generate the task operation. BB is the block where to insert the code. */
4390 expand_task_call (basic_block bb
, gimple entry_stmt
)
4392 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
4393 gimple_stmt_iterator gsi
;
4394 location_t loc
= gimple_location (entry_stmt
);
4396 clauses
= gimple_omp_task_clauses (entry_stmt
);
4398 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4400 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
4402 cond
= boolean_true_node
;
4404 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
4405 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
4406 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
4407 flags
= build_int_cst (unsigned_type_node
,
4408 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
4410 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
4413 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
4414 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
4415 build_int_cst (unsigned_type_node
, 2),
4416 build_int_cst (unsigned_type_node
, 0));
4417 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
4420 depend
= OMP_CLAUSE_DECL (depend
);
4422 depend
= build_int_cst (ptr_type_node
, 0);
4424 gsi
= gsi_last_bb (bb
);
4425 t
= gimple_omp_task_data_arg (entry_stmt
);
4427 t2
= null_pointer_node
;
4429 t2
= build_fold_addr_expr_loc (loc
, t
);
4430 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
4431 t
= gimple_omp_task_copy_fn (entry_stmt
);
4433 t3
= null_pointer_node
;
4435 t3
= build_fold_addr_expr_loc (loc
, t
);
4437 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
4439 gimple_omp_task_arg_size (entry_stmt
),
4440 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
4443 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4444 false, GSI_CONTINUE_LINKING
);
4448 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4449 catch handler and return it. This prevents programs from violating the
4450 structured block semantics with throws. */
4453 maybe_catch_exception (gimple_seq body
)
4458 if (!flag_exceptions
)
4461 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
4462 decl
= lang_hooks
.eh_protect_cleanup_actions ();
4464 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
4466 g
= gimple_build_eh_must_not_throw (decl
);
4467 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
4470 return gimple_seq_alloc_with_stmt (g
);
4473 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4476 vec2chain (vec
<tree
, va_gc
> *v
)
4478 tree chain
= NULL_TREE
, t
;
4481 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
4483 DECL_CHAIN (t
) = chain
;
4491 /* Remove barriers in REGION->EXIT's block. Note that this is only
4492 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4493 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4494 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4498 remove_exit_barrier (struct omp_region
*region
)
4500 gimple_stmt_iterator gsi
;
4501 basic_block exit_bb
;
4505 int any_addressable_vars
= -1;
4507 exit_bb
= region
->exit
;
4509 /* If the parallel region doesn't return, we don't have REGION->EXIT
4514 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4515 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4516 statements that can appear in between are extremely limited -- no
4517 memory operations at all. Here, we allow nothing at all, so the
4518 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4519 gsi
= gsi_last_bb (exit_bb
);
4520 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4522 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
4525 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
4527 gsi
= gsi_last_bb (e
->src
);
4528 if (gsi_end_p (gsi
))
4530 stmt
= gsi_stmt (gsi
);
4531 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
4532 && !gimple_omp_return_nowait_p (stmt
))
4534 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4535 in many cases. If there could be tasks queued, the barrier
4536 might be needed to let the tasks run before some local
4537 variable of the parallel that the task uses as shared
4538 runs out of scope. The task can be spawned either
4539 from within current function (this would be easy to check)
4540 or from some function it calls and gets passed an address
4541 of such a variable. */
4542 if (any_addressable_vars
< 0)
4544 gimple parallel_stmt
= last_stmt (region
->entry
);
4545 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
4546 tree local_decls
, block
, decl
;
4549 any_addressable_vars
= 0;
4550 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
4551 if (TREE_ADDRESSABLE (decl
))
4553 any_addressable_vars
= 1;
4556 for (block
= gimple_block (stmt
);
4557 !any_addressable_vars
4559 && TREE_CODE (block
) == BLOCK
;
4560 block
= BLOCK_SUPERCONTEXT (block
))
4562 for (local_decls
= BLOCK_VARS (block
);
4564 local_decls
= DECL_CHAIN (local_decls
))
4565 if (TREE_ADDRESSABLE (local_decls
))
4567 any_addressable_vars
= 1;
4570 if (block
== gimple_block (parallel_stmt
))
4574 if (!any_addressable_vars
)
4575 gimple_omp_return_set_nowait (stmt
);
4581 remove_exit_barriers (struct omp_region
*region
)
4583 if (region
->type
== GIMPLE_OMP_PARALLEL
)
4584 remove_exit_barrier (region
);
4588 region
= region
->inner
;
4589 remove_exit_barriers (region
);
4590 while (region
->next
)
4592 region
= region
->next
;
4593 remove_exit_barriers (region
);
4598 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4599 calls. These can't be declared as const functions, but
4600 within one parallel body they are constant, so they can be
4601 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4602 which are declared const. Similarly for task body, except
4603 that in untied task omp_get_thread_num () can change at any task
4604 scheduling point. */
4607 optimize_omp_library_calls (gimple entry_stmt
)
4610 gimple_stmt_iterator gsi
;
4611 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4612 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
4613 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4614 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
4615 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
4616 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
4617 OMP_CLAUSE_UNTIED
) != NULL
);
4619 FOR_EACH_BB_FN (bb
, cfun
)
4620 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4622 gimple call
= gsi_stmt (gsi
);
4625 if (is_gimple_call (call
)
4626 && (decl
= gimple_call_fndecl (call
))
4627 && DECL_EXTERNAL (decl
)
4628 && TREE_PUBLIC (decl
)
4629 && DECL_INITIAL (decl
) == NULL
)
4633 if (DECL_NAME (decl
) == thr_num_id
)
4635 /* In #pragma omp task untied omp_get_thread_num () can change
4636 during the execution of the task region. */
4639 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4641 else if (DECL_NAME (decl
) == num_thr_id
)
4642 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4646 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
4647 || gimple_call_num_args (call
) != 0)
4650 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
4653 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
4654 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
4655 TREE_TYPE (TREE_TYPE (built_in
))))
4658 gimple_call_set_fndecl (call
, built_in
);
4663 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4667 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
4671 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4672 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
4675 if (TREE_CODE (t
) == ADDR_EXPR
)
4676 recompute_tree_invariant_for_addr_expr (t
);
4678 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
4682 /* Prepend TO = FROM assignment before *GSI_P. */
4685 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
4687 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
4688 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
4689 true, GSI_SAME_STMT
);
4690 gimple stmt
= gimple_build_assign (to
, from
);
4691 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
4692 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
4693 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
4695 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
4696 gimple_regimplify_operands (stmt
, &gsi
);
4700 /* Expand the OpenMP parallel or task directive starting at REGION. */
4703 expand_omp_taskreg (struct omp_region
*region
)
4705 basic_block entry_bb
, exit_bb
, new_bb
;
4706 struct function
*child_cfun
;
4707 tree child_fn
, block
, t
;
4708 gimple_stmt_iterator gsi
;
4709 gimple entry_stmt
, stmt
;
4711 vec
<tree
, va_gc
> *ws_args
;
4713 entry_stmt
= last_stmt (region
->entry
);
4714 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
4715 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
4717 entry_bb
= region
->entry
;
4718 exit_bb
= region
->exit
;
4720 if (is_combined_parallel (region
))
4721 ws_args
= region
->ws_args
;
4725 if (child_cfun
->cfg
)
4727 /* Due to inlining, it may happen that we have already outlined
4728 the region, in which case all we need to do is make the
4729 sub-graph unreachable and emit the parallel call. */
4730 edge entry_succ_e
, exit_succ_e
;
4732 entry_succ_e
= single_succ_edge (entry_bb
);
4734 gsi
= gsi_last_bb (entry_bb
);
4735 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
4736 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
4737 gsi_remove (&gsi
, true);
4742 exit_succ_e
= single_succ_edge (exit_bb
);
4743 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
4745 remove_edge_and_dominated_blocks (entry_succ_e
);
4749 unsigned srcidx
, dstidx
, num
;
4751 /* If the parallel region needs data sent from the parent
4752 function, then the very first statement (except possible
4753 tree profile counter updates) of the parallel body
4754 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4755 &.OMP_DATA_O is passed as an argument to the child function,
4756 we need to replace it with the argument as seen by the child
4759 In most cases, this will end up being the identity assignment
4760 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4761 a function call that has been inlined, the original PARM_DECL
4762 .OMP_DATA_I may have been converted into a different local
4763 variable. In which case, we need to keep the assignment. */
4764 if (gimple_omp_taskreg_data_arg (entry_stmt
))
4766 basic_block entry_succ_bb
= single_succ (entry_bb
);
4768 gimple parcopy_stmt
= NULL
;
4770 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
4774 gcc_assert (!gsi_end_p (gsi
));
4775 stmt
= gsi_stmt (gsi
);
4776 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
4779 if (gimple_num_ops (stmt
) == 2)
4781 tree arg
= gimple_assign_rhs1 (stmt
);
4783 /* We're ignore the subcode because we're
4784 effectively doing a STRIP_NOPS. */
4786 if (TREE_CODE (arg
) == ADDR_EXPR
4787 && TREE_OPERAND (arg
, 0)
4788 == gimple_omp_taskreg_data_arg (entry_stmt
))
4790 parcopy_stmt
= stmt
;
4796 gcc_assert (parcopy_stmt
!= NULL
);
4797 arg
= DECL_ARGUMENTS (child_fn
);
4799 if (!gimple_in_ssa_p (cfun
))
4801 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
4802 gsi_remove (&gsi
, true);
4805 /* ?? Is setting the subcode really necessary ?? */
4806 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
4807 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
4812 /* If we are in ssa form, we must load the value from the default
4813 definition of the argument. That should not be defined now,
4814 since the argument is not used uninitialized. */
4815 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
4816 narg
= make_ssa_name (arg
, gimple_build_nop ());
4817 set_ssa_default_def (cfun
, arg
, narg
);
4818 /* ?? Is setting the subcode really necessary ?? */
4819 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
4820 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
4821 update_stmt (parcopy_stmt
);
4825 /* Declare local variables needed in CHILD_CFUN. */
4826 block
= DECL_INITIAL (child_fn
);
4827 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
4828 /* The gimplifier could record temporaries in parallel/task block
4829 rather than in containing function's local_decls chain,
4830 which would mean cgraph missed finalizing them. Do it now. */
4831 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
4832 if (TREE_CODE (t
) == VAR_DECL
4834 && !DECL_EXTERNAL (t
))
4835 varpool_finalize_decl (t
);
4836 DECL_SAVED_TREE (child_fn
) = NULL
;
4837 /* We'll create a CFG for child_fn, so no gimple body is needed. */
4838 gimple_set_body (child_fn
, NULL
);
4839 TREE_USED (block
) = 1;
4841 /* Reset DECL_CONTEXT on function arguments. */
4842 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
4843 DECL_CONTEXT (t
) = child_fn
;
4845 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
4846 so that it can be moved to the child function. */
4847 gsi
= gsi_last_bb (entry_bb
);
4848 stmt
= gsi_stmt (gsi
);
4849 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
4850 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
4851 gsi_remove (&gsi
, true);
4852 e
= split_block (entry_bb
, stmt
);
4854 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
4856 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
4859 gsi
= gsi_last_bb (exit_bb
);
4860 gcc_assert (!gsi_end_p (gsi
)
4861 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4862 stmt
= gimple_build_return (NULL
);
4863 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
4864 gsi_remove (&gsi
, true);
4867 /* Move the parallel region into CHILD_CFUN. */
4869 if (gimple_in_ssa_p (cfun
))
4871 init_tree_ssa (child_cfun
);
4872 init_ssa_operands (child_cfun
);
4873 child_cfun
->gimple_df
->in_ssa_p
= true;
4877 block
= gimple_block (entry_stmt
);
4879 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
4881 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
4882 /* When the OMP expansion process cannot guarantee an up-to-date
4883 loop tree arrange for the child function to fixup loops. */
4884 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
4885 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
4887 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
4888 num
= vec_safe_length (child_cfun
->local_decls
);
4889 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
4891 t
= (*child_cfun
->local_decls
)[srcidx
];
4892 if (DECL_CONTEXT (t
) == cfun
->decl
)
4894 if (srcidx
!= dstidx
)
4895 (*child_cfun
->local_decls
)[dstidx
] = t
;
4899 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
4901 /* Inform the callgraph about the new function. */
4902 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
4903 cgraph_add_new_function (child_fn
, true);
4905 /* Fix the callgraph edges for child_cfun. Those for cfun will be
4906 fixed in a following pass. */
4907 push_cfun (child_cfun
);
4909 optimize_omp_library_calls (entry_stmt
);
4910 rebuild_cgraph_edges ();
4912 /* Some EH regions might become dead, see PR34608. If
4913 pass_cleanup_cfg isn't the first pass to happen with the
4914 new child, these dead EH edges might cause problems.
4915 Clean them up now. */
4916 if (flag_exceptions
)
4919 bool changed
= false;
4921 FOR_EACH_BB_FN (bb
, cfun
)
4922 changed
|= gimple_purge_dead_eh_edges (bb
);
4924 cleanup_tree_cfg ();
4926 if (gimple_in_ssa_p (cfun
))
4927 update_ssa (TODO_update_ssa
);
4931 /* Emit a library call to launch the children threads. */
4932 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
4933 expand_parallel_call (region
, new_bb
, entry_stmt
, ws_args
);
4935 expand_task_call (new_bb
, entry_stmt
);
4936 if (gimple_in_ssa_p (cfun
))
4937 update_ssa (TODO_update_ssa_only_virtuals
);
4941 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
4942 of the combined collapse > 1 loop constructs, generate code like:
4943 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
4948 count3 = (adj + N32 - N31) / STEP3;
4949 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
4954 count2 = (adj + N22 - N21) / STEP2;
4955 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
4960 count1 = (adj + N12 - N11) / STEP1;
4961 count = count1 * count2 * count3;
4962 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
4964 and set ZERO_ITER_BB to that bb. If this isn't the outermost
4965 of the combined loop constructs, just initialize COUNTS array
4966 from the _looptemp_ clauses. */
4968 /* NOTE: It *could* be better to moosh all of the BBs together,
4969 creating one larger BB with all the computation and the unexpected
4970 jump at the end. I.e.
4972 bool zero3, zero2, zero1, zero;
4975 count3 = (N32 - N31) /[cl] STEP3;
4977 count2 = (N22 - N21) /[cl] STEP2;
4979 count1 = (N12 - N11) /[cl] STEP1;
4980 zero = zero3 || zero2 || zero1;
4981 count = count1 * count2 * count3;
4982 if (__builtin_expect(zero, false)) goto zero_iter_bb;
4984 After all, we expect the zero=false, and thus we expect to have to
4985 evaluate all of the comparison expressions, so short-circuiting
4986 oughtn't be a win. Since the condition isn't protecting a
4987 denominator, we're not concerned about divide-by-zero, so we can
4988 fully evaluate count even if a numerator turned out to be wrong.
4990 It seems like putting this all together would create much better
4991 scheduling opportunities, and less pressure on the chip's branch
4995 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
4996 basic_block
&entry_bb
, tree
*counts
,
4997 basic_block
&zero_iter_bb
, int &first_zero_iter
,
4998 basic_block
&l2_dom_bb
)
5000 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5005 /* Collapsed loops need work for expansion into SSA form. */
5006 gcc_assert (!gimple_in_ssa_p (cfun
));
5008 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5009 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5011 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5012 isn't supposed to be handled, as the inner loop doesn't
5014 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5015 OMP_CLAUSE__LOOPTEMP_
);
5016 gcc_assert (innerc
);
5017 for (i
= 0; i
< fd
->collapse
; i
++)
5019 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5020 OMP_CLAUSE__LOOPTEMP_
);
5021 gcc_assert (innerc
);
5023 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5025 counts
[0] = NULL_TREE
;
5030 for (i
= 0; i
< fd
->collapse
; i
++)
5032 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5034 if (SSA_VAR_P (fd
->loop
.n2
)
5035 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5036 fold_convert (itype
, fd
->loops
[i
].n1
),
5037 fold_convert (itype
, fd
->loops
[i
].n2
)))
5038 == NULL_TREE
|| !integer_onep (t
)))
5041 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5042 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5043 true, GSI_SAME_STMT
);
5044 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5045 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5046 true, GSI_SAME_STMT
);
5047 stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5048 NULL_TREE
, NULL_TREE
);
5049 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5050 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5051 expand_omp_regimplify_p
, NULL
, NULL
)
5052 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5053 expand_omp_regimplify_p
, NULL
, NULL
))
5055 *gsi
= gsi_for_stmt (stmt
);
5056 gimple_regimplify_operands (stmt
, gsi
);
5058 e
= split_block (entry_bb
, stmt
);
5059 if (zero_iter_bb
== NULL
)
5061 first_zero_iter
= i
;
5062 zero_iter_bb
= create_empty_bb (entry_bb
);
5063 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5064 *gsi
= gsi_after_labels (zero_iter_bb
);
5065 stmt
= gimple_build_assign (fd
->loop
.n2
,
5066 build_zero_cst (type
));
5067 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5068 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5071 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5072 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5073 e
->flags
= EDGE_TRUE_VALUE
;
5074 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5075 if (l2_dom_bb
== NULL
)
5076 l2_dom_bb
= entry_bb
;
5078 *gsi
= gsi_last_bb (entry_bb
);
5081 if (POINTER_TYPE_P (itype
))
5082 itype
= signed_type_for (itype
);
5083 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5085 t
= fold_build2 (PLUS_EXPR
, itype
,
5086 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5087 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5088 fold_convert (itype
, fd
->loops
[i
].n2
));
5089 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5090 fold_convert (itype
, fd
->loops
[i
].n1
));
5091 /* ?? We could probably use CEIL_DIV_EXPR instead of
5092 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5093 generate the same code in the end because generically we
5094 don't know that the values involved must be negative for
5096 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5097 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5098 fold_build1 (NEGATE_EXPR
, itype
, t
),
5099 fold_build1 (NEGATE_EXPR
, itype
,
5100 fold_convert (itype
,
5101 fd
->loops
[i
].step
)));
5103 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5104 fold_convert (itype
, fd
->loops
[i
].step
));
5105 t
= fold_convert (type
, t
);
5106 if (TREE_CODE (t
) == INTEGER_CST
)
5110 counts
[i
] = create_tmp_reg (type
, ".count");
5111 expand_omp_build_assign (gsi
, counts
[i
], t
);
5113 if (SSA_VAR_P (fd
->loop
.n2
))
5118 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5119 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5125 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5127 V3 = N31 + (T % count3) * STEP3;
5129 V2 = N21 + (T % count2) * STEP2;
5131 V1 = N11 + T * STEP1;
5132 if this loop doesn't have an inner loop construct combined with it.
5133 If it does have an inner loop construct combined with it and the
5134 iteration count isn't known constant, store values from counts array
5135 into its _looptemp_ temporaries instead. */
5138 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5139 tree
*counts
, gimple inner_stmt
, tree startvar
)
5142 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5144 /* If fd->loop.n2 is constant, then no propagation of the counts
5145 is needed, they are constant. */
5146 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5149 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5150 ? gimple_omp_parallel_clauses (inner_stmt
)
5151 : gimple_omp_for_clauses (inner_stmt
);
5152 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5153 isn't supposed to be handled, as the inner loop doesn't
5155 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5156 gcc_assert (innerc
);
5157 for (i
= 0; i
< fd
->collapse
; i
++)
5159 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5160 OMP_CLAUSE__LOOPTEMP_
);
5161 gcc_assert (innerc
);
5164 tree tem
= OMP_CLAUSE_DECL (innerc
);
5165 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5166 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5167 false, GSI_CONTINUE_LINKING
);
5168 gimple stmt
= gimple_build_assign (tem
, t
);
5169 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5175 tree type
= TREE_TYPE (fd
->loop
.v
);
5176 tree tem
= create_tmp_reg (type
, ".tem");
5177 gimple stmt
= gimple_build_assign (tem
, startvar
);
5178 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5180 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5182 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5184 if (POINTER_TYPE_P (vtype
))
5185 itype
= signed_type_for (vtype
);
5187 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5190 t
= fold_convert (itype
, t
);
5191 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5192 fold_convert (itype
, fd
->loops
[i
].step
));
5193 if (POINTER_TYPE_P (vtype
))
5194 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5196 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5197 t
= force_gimple_operand_gsi (gsi
, t
,
5198 DECL_P (fd
->loops
[i
].v
)
5199 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5201 GSI_CONTINUE_LINKING
);
5202 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5203 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5206 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5207 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5208 false, GSI_CONTINUE_LINKING
);
5209 stmt
= gimple_build_assign (tem
, t
);
5210 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5216 /* Helper function for expand_omp_for_*. Generate code like:
5219 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5223 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5230 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5231 basic_block body_bb
)
5233 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5235 gimple_stmt_iterator gsi
;
5241 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5243 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5245 bb
= create_empty_bb (last_bb
);
5246 add_bb_to_loop (bb
, last_bb
->loop_father
);
5247 gsi
= gsi_start_bb (bb
);
5249 if (i
< fd
->collapse
- 1)
5251 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5252 e
->probability
= REG_BR_PROB_BASE
/ 8;
5254 t
= fd
->loops
[i
+ 1].n1
;
5255 t
= force_gimple_operand_gsi (&gsi
, t
,
5256 DECL_P (fd
->loops
[i
+ 1].v
)
5257 && TREE_ADDRESSABLE (fd
->loops
[i
5260 GSI_CONTINUE_LINKING
);
5261 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5262 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5267 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5269 if (POINTER_TYPE_P (vtype
))
5270 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5272 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5273 t
= force_gimple_operand_gsi (&gsi
, t
,
5274 DECL_P (fd
->loops
[i
].v
)
5275 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5276 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5277 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5278 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5282 t
= fd
->loops
[i
].n2
;
5283 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5284 false, GSI_CONTINUE_LINKING
);
5285 tree v
= fd
->loops
[i
].v
;
5286 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5287 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5288 false, GSI_CONTINUE_LINKING
);
5289 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5290 stmt
= gimple_build_cond_empty (t
);
5291 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5292 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5293 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5296 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5304 /* A subroutine of expand_omp_for. Generate code for a parallel
5305 loop with any schedule. Given parameters:
5307 for (V = N1; V cond N2; V += STEP) BODY;
5309 where COND is "<" or ">", we generate pseudocode
5311 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5312 if (more) goto L0; else goto L3;
5319 if (V cond iend) goto L1; else goto L2;
5321 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5324 If this is a combined omp parallel loop, instead of the call to
5325 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5326 If this is gimple_omp_for_combined_p loop, then instead of assigning
5327 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5328 inner GIMPLE_OMP_FOR and V += STEP; and
5329 if (V cond iend) goto L1; else goto L2; are removed.
5331 For collapsed loops, given parameters:
5333 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5334 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5335 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5338 we generate pseudocode
5340 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5345 count3 = (adj + N32 - N31) / STEP3;
5346 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5351 count2 = (adj + N22 - N21) / STEP2;
5352 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5357 count1 = (adj + N12 - N11) / STEP1;
5358 count = count1 * count2 * count3;
5363 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5364 if (more) goto L0; else goto L3;
5368 V3 = N31 + (T % count3) * STEP3;
5370 V2 = N21 + (T % count2) * STEP2;
5372 V1 = N11 + T * STEP1;
5377 if (V < iend) goto L10; else goto L2;
5380 if (V3 cond3 N32) goto L1; else goto L11;
5384 if (V2 cond2 N22) goto L1; else goto L12;
5390 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5396 expand_omp_for_generic (struct omp_region
*region
,
5397 struct omp_for_data
*fd
,
5398 enum built_in_function start_fn
,
5399 enum built_in_function next_fn
,
5402 tree type
, istart0
, iend0
, iend
;
5403 tree t
, vmain
, vback
, bias
= NULL_TREE
;
5404 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
5405 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
5406 gimple_stmt_iterator gsi
;
5408 bool in_combined_parallel
= is_combined_parallel (region
);
5409 bool broken_loop
= region
->cont
== NULL
;
5411 tree
*counts
= NULL
;
5414 gcc_assert (!broken_loop
|| !in_combined_parallel
);
5415 gcc_assert (fd
->iter_type
== long_integer_type_node
5416 || !in_combined_parallel
);
5418 type
= TREE_TYPE (fd
->loop
.v
);
5419 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
5420 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
5421 TREE_ADDRESSABLE (istart0
) = 1;
5422 TREE_ADDRESSABLE (iend0
) = 1;
5424 /* See if we need to bias by LLONG_MIN. */
5425 if (fd
->iter_type
== long_long_unsigned_type_node
5426 && TREE_CODE (type
) == INTEGER_TYPE
5427 && !TYPE_UNSIGNED (type
))
5431 if (fd
->loop
.cond_code
== LT_EXPR
)
5434 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5438 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5441 if (TREE_CODE (n1
) != INTEGER_CST
5442 || TREE_CODE (n2
) != INTEGER_CST
5443 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
5444 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
5447 entry_bb
= region
->entry
;
5448 cont_bb
= region
->cont
;
5450 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5451 gcc_assert (broken_loop
5452 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
5453 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5454 l1_bb
= single_succ (l0_bb
);
5457 l2_bb
= create_empty_bb (cont_bb
);
5458 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
5459 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5463 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5464 exit_bb
= region
->exit
;
5466 gsi
= gsi_last_bb (entry_bb
);
5468 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5469 if (fd
->collapse
> 1)
5471 int first_zero_iter
= -1;
5472 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
5474 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5475 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5476 zero_iter_bb
, first_zero_iter
,
5481 /* Some counts[i] vars might be uninitialized if
5482 some loop has zero iterations. But the body shouldn't
5483 be executed in that case, so just avoid uninit warnings. */
5484 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
5485 if (SSA_VAR_P (counts
[i
]))
5486 TREE_NO_WARNING (counts
[i
]) = 1;
5488 e
= split_block (entry_bb
, gsi_stmt (gsi
));
5490 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
5491 gsi
= gsi_last_bb (entry_bb
);
5492 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
5493 get_immediate_dominator (CDI_DOMINATORS
,
5497 if (in_combined_parallel
)
5499 /* In a combined parallel loop, emit a call to
5500 GOMP_loop_foo_next. */
5501 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5502 build_fold_addr_expr (istart0
),
5503 build_fold_addr_expr (iend0
));
5507 tree t0
, t1
, t2
, t3
, t4
;
5508 /* If this is not a combined parallel loop, emit a call to
5509 GOMP_loop_foo_start in ENTRY_BB. */
5510 t4
= build_fold_addr_expr (iend0
);
5511 t3
= build_fold_addr_expr (istart0
);
5512 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
5515 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5517 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5518 OMP_CLAUSE__LOOPTEMP_
);
5519 gcc_assert (innerc
);
5520 t0
= OMP_CLAUSE_DECL (innerc
);
5521 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5522 OMP_CLAUSE__LOOPTEMP_
);
5523 gcc_assert (innerc
);
5524 t1
= OMP_CLAUSE_DECL (innerc
);
5526 if (POINTER_TYPE_P (TREE_TYPE (t0
))
5527 && TYPE_PRECISION (TREE_TYPE (t0
))
5528 != TYPE_PRECISION (fd
->iter_type
))
5530 /* Avoid casting pointers to integer of a different size. */
5531 tree itype
= signed_type_for (type
);
5532 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
5533 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
5537 t1
= fold_convert (fd
->iter_type
, t1
);
5538 t0
= fold_convert (fd
->iter_type
, t0
);
5542 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
5543 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
5545 if (fd
->iter_type
== long_integer_type_node
)
5549 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5550 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5551 6, t0
, t1
, t2
, t
, t3
, t4
);
5554 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5555 5, t0
, t1
, t2
, t3
, t4
);
5563 /* The GOMP_loop_ull_*start functions have additional boolean
5564 argument, true for < loops and false for > loops.
5565 In Fortran, the C bool type can be different from
5566 boolean_type_node. */
5567 bfn_decl
= builtin_decl_explicit (start_fn
);
5568 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
5569 t5
= build_int_cst (c_bool_type
,
5570 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
5573 tree bfn_decl
= builtin_decl_explicit (start_fn
);
5574 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5575 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
5578 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5579 6, t5
, t0
, t1
, t2
, t3
, t4
);
5582 if (TREE_TYPE (t
) != boolean_type_node
)
5583 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5584 t
, build_int_cst (TREE_TYPE (t
), 0));
5585 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5586 true, GSI_SAME_STMT
);
5587 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5589 /* Remove the GIMPLE_OMP_FOR statement. */
5590 gsi_remove (&gsi
, true);
5592 /* Iteration setup for sequential loop goes in L0_BB. */
5593 tree startvar
= fd
->loop
.v
;
5594 tree endvar
= NULL_TREE
;
5596 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5598 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
5599 && gimple_omp_for_kind (inner_stmt
)
5600 == GF_OMP_FOR_KIND_SIMD
);
5601 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
5602 OMP_CLAUSE__LOOPTEMP_
);
5603 gcc_assert (innerc
);
5604 startvar
= OMP_CLAUSE_DECL (innerc
);
5605 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5606 OMP_CLAUSE__LOOPTEMP_
);
5607 gcc_assert (innerc
);
5608 endvar
= OMP_CLAUSE_DECL (innerc
);
5611 gsi
= gsi_start_bb (l0_bb
);
5614 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5615 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5616 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5617 t
= fold_convert (TREE_TYPE (startvar
), t
);
5618 t
= force_gimple_operand_gsi (&gsi
, t
,
5620 && TREE_ADDRESSABLE (startvar
),
5621 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5622 stmt
= gimple_build_assign (startvar
, t
);
5623 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5627 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5628 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5629 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5630 t
= fold_convert (TREE_TYPE (startvar
), t
);
5631 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5632 false, GSI_CONTINUE_LINKING
);
5635 stmt
= gimple_build_assign (endvar
, iend
);
5636 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5637 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
5638 stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
5640 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, iend
,
5642 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5644 if (fd
->collapse
> 1)
5645 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5649 /* Code to control the increment and predicate for the sequential
5650 loop goes in the CONT_BB. */
5651 gsi
= gsi_last_bb (cont_bb
);
5652 stmt
= gsi_stmt (gsi
);
5653 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5654 vmain
= gimple_omp_continue_control_use (stmt
);
5655 vback
= gimple_omp_continue_control_def (stmt
);
5657 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5659 if (POINTER_TYPE_P (type
))
5660 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
5662 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
5663 t
= force_gimple_operand_gsi (&gsi
, t
,
5665 && TREE_ADDRESSABLE (vback
),
5666 NULL_TREE
, true, GSI_SAME_STMT
);
5667 stmt
= gimple_build_assign (vback
, t
);
5668 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5670 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5671 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
5673 stmt
= gimple_build_cond_empty (t
);
5674 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5677 /* Remove GIMPLE_OMP_CONTINUE. */
5678 gsi_remove (&gsi
, true);
5680 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5681 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
5683 /* Emit code to get the next parallel iteration in L2_BB. */
5684 gsi
= gsi_start_bb (l2_bb
);
5686 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5687 build_fold_addr_expr (istart0
),
5688 build_fold_addr_expr (iend0
));
5689 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5690 false, GSI_CONTINUE_LINKING
);
5691 if (TREE_TYPE (t
) != boolean_type_node
)
5692 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5693 t
, build_int_cst (TREE_TYPE (t
), 0));
5694 stmt
= gimple_build_cond_empty (t
);
5695 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5698 /* Add the loop cleanup function. */
5699 gsi
= gsi_last_bb (exit_bb
);
5700 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5701 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
5702 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5703 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
5705 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
5706 stmt
= gimple_build_call (t
, 0);
5707 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5708 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
5709 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5710 gsi_remove (&gsi
, true);
5712 /* Connect the new blocks. */
5713 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
5714 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
5720 e
= find_edge (cont_bb
, l3_bb
);
5721 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
5723 phis
= phi_nodes (l3_bb
);
5724 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5726 gimple phi
= gsi_stmt (gsi
);
5727 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
5728 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
5732 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
5733 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
5734 e
= find_edge (cont_bb
, l1_bb
);
5735 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5740 else if (fd
->collapse
> 1)
5743 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5746 e
->flags
= EDGE_TRUE_VALUE
;
5749 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5750 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
5754 e
= find_edge (cont_bb
, l2_bb
);
5755 e
->flags
= EDGE_FALLTHRU
;
5757 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
5759 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
5760 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
5761 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
5762 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
5763 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
5764 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
5765 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
5766 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
5768 struct loop
*outer_loop
= alloc_loop ();
5769 outer_loop
->header
= l0_bb
;
5770 outer_loop
->latch
= l2_bb
;
5771 add_loop (outer_loop
, l0_bb
->loop_father
);
5773 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5775 struct loop
*loop
= alloc_loop ();
5776 loop
->header
= l1_bb
;
5777 /* The loop may have multiple latches. */
5778 add_loop (loop
, outer_loop
);
5784 /* A subroutine of expand_omp_for. Generate code for a parallel
5785 loop with static schedule and no specified chunk size. Given
5788 for (V = N1; V cond N2; V += STEP) BODY;
5790 where COND is "<" or ">", we generate pseudocode
5792 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5797 if ((__typeof (V)) -1 > 0 && cond is >)
5798 n = -(adj + N2 - N1) / -STEP;
5800 n = (adj + N2 - N1) / STEP;
5803 if (threadid < tt) goto L3; else goto L4;
5808 s0 = q * threadid + tt;
5811 if (s0 >= e0) goto L2; else goto L0;
5817 if (V cond e) goto L1;
5822 expand_omp_for_static_nochunk (struct omp_region
*region
,
5823 struct omp_for_data
*fd
,
5826 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
5827 tree type
, itype
, vmain
, vback
;
5828 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
5829 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
5831 gimple_stmt_iterator gsi
;
5834 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
5835 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
5836 bool broken_loop
= region
->cont
== NULL
;
5837 tree
*counts
= NULL
;
5840 itype
= type
= TREE_TYPE (fd
->loop
.v
);
5841 if (POINTER_TYPE_P (type
))
5842 itype
= signed_type_for (type
);
5844 entry_bb
= region
->entry
;
5845 cont_bb
= region
->cont
;
5846 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5847 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5848 gcc_assert (broken_loop
5849 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
5850 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5851 body_bb
= single_succ (seq_start_bb
);
5854 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
5855 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5857 exit_bb
= region
->exit
;
5859 /* Iteration space partitioning goes in ENTRY_BB. */
5860 gsi
= gsi_last_bb (entry_bb
);
5861 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5863 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
5865 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
5866 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
5869 if (fd
->collapse
> 1)
5871 int first_zero_iter
= -1;
5872 basic_block l2_dom_bb
= NULL
;
5874 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5875 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5876 fin_bb
, first_zero_iter
,
5880 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5881 t
= integer_one_node
;
5883 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
5884 fold_convert (type
, fd
->loop
.n1
),
5885 fold_convert (type
, fd
->loop
.n2
));
5886 if (fd
->collapse
== 1
5887 && TYPE_UNSIGNED (type
)
5888 && (t
== NULL_TREE
|| !integer_onep (t
)))
5890 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
5891 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
5892 true, GSI_SAME_STMT
);
5893 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
5894 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
5895 true, GSI_SAME_STMT
);
5896 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
5897 NULL_TREE
, NULL_TREE
);
5898 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5899 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5900 expand_omp_regimplify_p
, NULL
, NULL
)
5901 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5902 expand_omp_regimplify_p
, NULL
, NULL
))
5904 gsi
= gsi_for_stmt (stmt
);
5905 gimple_regimplify_operands (stmt
, &gsi
);
5907 ep
= split_block (entry_bb
, stmt
);
5908 ep
->flags
= EDGE_TRUE_VALUE
;
5909 entry_bb
= ep
->dest
;
5910 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
5911 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
5912 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5913 if (gimple_in_ssa_p (cfun
))
5915 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
5916 for (gsi
= gsi_start_phis (fin_bb
);
5917 !gsi_end_p (gsi
); gsi_next (&gsi
))
5919 gimple phi
= gsi_stmt (gsi
);
5920 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
5921 ep
, UNKNOWN_LOCATION
);
5924 gsi
= gsi_last_bb (entry_bb
);
5927 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
5928 t
= fold_convert (itype
, t
);
5929 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5930 true, GSI_SAME_STMT
);
5932 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
5933 t
= fold_convert (itype
, t
);
5934 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5935 true, GSI_SAME_STMT
);
5939 step
= fd
->loop
.step
;
5940 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5942 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5943 OMP_CLAUSE__LOOPTEMP_
);
5944 gcc_assert (innerc
);
5945 n1
= OMP_CLAUSE_DECL (innerc
);
5946 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5947 OMP_CLAUSE__LOOPTEMP_
);
5948 gcc_assert (innerc
);
5949 n2
= OMP_CLAUSE_DECL (innerc
);
5951 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
5952 true, NULL_TREE
, true, GSI_SAME_STMT
);
5953 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
5954 true, NULL_TREE
, true, GSI_SAME_STMT
);
5955 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
5956 true, NULL_TREE
, true, GSI_SAME_STMT
);
5958 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
5959 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
5960 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
5961 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
5962 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
5963 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5964 fold_build1 (NEGATE_EXPR
, itype
, t
),
5965 fold_build1 (NEGATE_EXPR
, itype
, step
));
5967 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
5968 t
= fold_convert (itype
, t
);
5969 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5971 q
= create_tmp_reg (itype
, "q");
5972 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
5973 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
5974 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
5976 tt
= create_tmp_reg (itype
, "tt");
5977 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
5978 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
5979 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
5981 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
5982 stmt
= gimple_build_cond_empty (t
);
5983 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5985 second_bb
= split_block (entry_bb
, stmt
)->dest
;
5986 gsi
= gsi_last_bb (second_bb
);
5987 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5989 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
5991 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, q
, q
,
5992 build_int_cst (itype
, 1));
5993 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5995 third_bb
= split_block (second_bb
, stmt
)->dest
;
5996 gsi
= gsi_last_bb (third_bb
);
5997 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5999 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6000 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6001 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6003 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6004 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6006 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6007 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6009 /* Remove the GIMPLE_OMP_FOR statement. */
6010 gsi_remove (&gsi
, true);
6012 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6013 gsi
= gsi_start_bb (seq_start_bb
);
6015 tree startvar
= fd
->loop
.v
;
6016 tree endvar
= NULL_TREE
;
6018 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6020 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6021 ? gimple_omp_parallel_clauses (inner_stmt
)
6022 : gimple_omp_for_clauses (inner_stmt
);
6023 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6024 gcc_assert (innerc
);
6025 startvar
= OMP_CLAUSE_DECL (innerc
);
6026 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6027 OMP_CLAUSE__LOOPTEMP_
);
6028 gcc_assert (innerc
);
6029 endvar
= OMP_CLAUSE_DECL (innerc
);
6031 t
= fold_convert (itype
, s0
);
6032 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6033 if (POINTER_TYPE_P (type
))
6034 t
= fold_build_pointer_plus (n1
, t
);
6036 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6037 t
= fold_convert (TREE_TYPE (startvar
), t
);
6038 t
= force_gimple_operand_gsi (&gsi
, t
,
6040 && TREE_ADDRESSABLE (startvar
),
6041 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6042 stmt
= gimple_build_assign (startvar
, t
);
6043 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6045 t
= fold_convert (itype
, e0
);
6046 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6047 if (POINTER_TYPE_P (type
))
6048 t
= fold_build_pointer_plus (n1
, t
);
6050 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6051 t
= fold_convert (TREE_TYPE (startvar
), t
);
6052 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6053 false, GSI_CONTINUE_LINKING
);
6056 stmt
= gimple_build_assign (endvar
, e
);
6057 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6058 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6059 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6061 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6063 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6065 if (fd
->collapse
> 1)
6066 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6070 /* The code controlling the sequential loop replaces the
6071 GIMPLE_OMP_CONTINUE. */
6072 gsi
= gsi_last_bb (cont_bb
);
6073 stmt
= gsi_stmt (gsi
);
6074 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6075 vmain
= gimple_omp_continue_control_use (stmt
);
6076 vback
= gimple_omp_continue_control_def (stmt
);
6078 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6080 if (POINTER_TYPE_P (type
))
6081 t
= fold_build_pointer_plus (vmain
, step
);
6083 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6084 t
= force_gimple_operand_gsi (&gsi
, t
,
6086 && TREE_ADDRESSABLE (vback
),
6087 NULL_TREE
, true, GSI_SAME_STMT
);
6088 stmt
= gimple_build_assign (vback
, t
);
6089 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6091 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6092 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6094 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6097 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6098 gsi_remove (&gsi
, true);
6100 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6101 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6104 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6105 gsi
= gsi_last_bb (exit_bb
);
6106 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6108 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6109 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6111 gsi_remove (&gsi
, true);
6113 /* Connect all the blocks. */
6114 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6115 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6116 ep
= find_edge (entry_bb
, second_bb
);
6117 ep
->flags
= EDGE_TRUE_VALUE
;
6118 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6119 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6120 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6124 ep
= find_edge (cont_bb
, body_bb
);
6125 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6130 else if (fd
->collapse
> 1)
6133 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6136 ep
->flags
= EDGE_TRUE_VALUE
;
6137 find_edge (cont_bb
, fin_bb
)->flags
6138 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6141 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6142 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6143 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6145 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6146 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6147 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6148 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6150 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6152 struct loop
*loop
= alloc_loop ();
6153 loop
->header
= body_bb
;
6154 if (collapse_bb
== NULL
)
6155 loop
->latch
= cont_bb
;
6156 add_loop (loop
, body_bb
->loop_father
);
6161 /* A subroutine of expand_omp_for. Generate code for a parallel
6162 loop with static schedule and a specified chunk size. Given
6165 for (V = N1; V cond N2; V += STEP) BODY;
6167 where COND is "<" or ">", we generate pseudocode
6169 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6174 if ((__typeof (V)) -1 > 0 && cond is >)
6175 n = -(adj + N2 - N1) / -STEP;
6177 n = (adj + N2 - N1) / STEP;
6179 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6180 here so that V is defined
6181 if the loop is not entered
6183 s0 = (trip * nthreads + threadid) * CHUNK;
6184 e0 = min(s0 + CHUNK, n);
6185 if (s0 < n) goto L1; else goto L4;
6192 if (V cond e) goto L2; else goto L3;
6200 expand_omp_for_static_chunk (struct omp_region
*region
,
6201 struct omp_for_data
*fd
, gimple inner_stmt
)
6203 tree n
, s0
, e0
, e
, t
;
6204 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6205 tree type
, itype
, vmain
, vback
, vextra
;
6206 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6207 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6208 gimple_stmt_iterator gsi
;
6211 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6212 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6213 bool broken_loop
= region
->cont
== NULL
;
6214 tree
*counts
= NULL
;
6217 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6218 if (POINTER_TYPE_P (type
))
6219 itype
= signed_type_for (type
);
6221 entry_bb
= region
->entry
;
6222 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6224 iter_part_bb
= se
->dest
;
6225 cont_bb
= region
->cont
;
6226 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6227 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6228 gcc_assert (broken_loop
6229 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6230 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6231 body_bb
= single_succ (seq_start_bb
);
6234 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6235 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6236 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6238 exit_bb
= region
->exit
;
6240 /* Trip and adjustment setup goes in ENTRY_BB. */
6241 gsi
= gsi_last_bb (entry_bb
);
6242 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6244 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6246 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6247 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6250 if (fd
->collapse
> 1)
6252 int first_zero_iter
= -1;
6253 basic_block l2_dom_bb
= NULL
;
6255 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6256 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6257 fin_bb
, first_zero_iter
,
6261 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6262 t
= integer_one_node
;
6264 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6265 fold_convert (type
, fd
->loop
.n1
),
6266 fold_convert (type
, fd
->loop
.n2
));
6267 if (fd
->collapse
== 1
6268 && TYPE_UNSIGNED (type
)
6269 && (t
== NULL_TREE
|| !integer_onep (t
)))
6271 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6272 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6273 true, GSI_SAME_STMT
);
6274 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6275 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6276 true, GSI_SAME_STMT
);
6277 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6278 NULL_TREE
, NULL_TREE
);
6279 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6280 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6281 expand_omp_regimplify_p
, NULL
, NULL
)
6282 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6283 expand_omp_regimplify_p
, NULL
, NULL
))
6285 gsi
= gsi_for_stmt (stmt
);
6286 gimple_regimplify_operands (stmt
, &gsi
);
6288 se
= split_block (entry_bb
, stmt
);
6289 se
->flags
= EDGE_TRUE_VALUE
;
6290 entry_bb
= se
->dest
;
6291 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6292 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6293 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6294 if (gimple_in_ssa_p (cfun
))
6296 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6297 for (gsi
= gsi_start_phis (fin_bb
);
6298 !gsi_end_p (gsi
); gsi_next (&gsi
))
6300 gimple phi
= gsi_stmt (gsi
);
6301 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6302 se
, UNKNOWN_LOCATION
);
6305 gsi
= gsi_last_bb (entry_bb
);
6308 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6309 t
= fold_convert (itype
, t
);
6310 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6311 true, GSI_SAME_STMT
);
6313 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6314 t
= fold_convert (itype
, t
);
6315 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6316 true, GSI_SAME_STMT
);
6320 step
= fd
->loop
.step
;
6321 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6323 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6324 OMP_CLAUSE__LOOPTEMP_
);
6325 gcc_assert (innerc
);
6326 n1
= OMP_CLAUSE_DECL (innerc
);
6327 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6328 OMP_CLAUSE__LOOPTEMP_
);
6329 gcc_assert (innerc
);
6330 n2
= OMP_CLAUSE_DECL (innerc
);
6332 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6333 true, NULL_TREE
, true, GSI_SAME_STMT
);
6334 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6335 true, NULL_TREE
, true, GSI_SAME_STMT
);
6336 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6337 true, NULL_TREE
, true, GSI_SAME_STMT
);
6339 = force_gimple_operand_gsi (&gsi
, fold_convert (itype
, fd
->chunk_size
),
6340 true, NULL_TREE
, true, GSI_SAME_STMT
);
6342 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6343 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6344 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6345 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6346 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6347 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6348 fold_build1 (NEGATE_EXPR
, itype
, t
),
6349 fold_build1 (NEGATE_EXPR
, itype
, step
));
6351 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6352 t
= fold_convert (itype
, t
);
6353 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6354 true, GSI_SAME_STMT
);
6356 trip_var
= create_tmp_reg (itype
, ".trip");
6357 if (gimple_in_ssa_p (cfun
))
6359 trip_init
= make_ssa_name (trip_var
, NULL
);
6360 trip_main
= make_ssa_name (trip_var
, NULL
);
6361 trip_back
= make_ssa_name (trip_var
, NULL
);
6365 trip_init
= trip_var
;
6366 trip_main
= trip_var
;
6367 trip_back
= trip_var
;
6370 stmt
= gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
6371 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6373 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
6374 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6375 if (POINTER_TYPE_P (type
))
6376 t
= fold_build_pointer_plus (n1
, t
);
6378 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6379 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6380 true, GSI_SAME_STMT
);
6382 /* Remove the GIMPLE_OMP_FOR. */
6383 gsi_remove (&gsi
, true);
6385 /* Iteration space partitioning goes in ITER_PART_BB. */
6386 gsi
= gsi_last_bb (iter_part_bb
);
6388 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
6389 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
6390 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
6391 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6392 false, GSI_CONTINUE_LINKING
);
6394 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
6395 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
6396 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6397 false, GSI_CONTINUE_LINKING
);
6399 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
6400 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
6402 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6403 gsi
= gsi_start_bb (seq_start_bb
);
6405 tree startvar
= fd
->loop
.v
;
6406 tree endvar
= NULL_TREE
;
6408 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6410 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6411 ? gimple_omp_parallel_clauses (inner_stmt
)
6412 : gimple_omp_for_clauses (inner_stmt
);
6413 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6414 gcc_assert (innerc
);
6415 startvar
= OMP_CLAUSE_DECL (innerc
);
6416 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6417 OMP_CLAUSE__LOOPTEMP_
);
6418 gcc_assert (innerc
);
6419 endvar
= OMP_CLAUSE_DECL (innerc
);
6422 t
= fold_convert (itype
, s0
);
6423 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6424 if (POINTER_TYPE_P (type
))
6425 t
= fold_build_pointer_plus (n1
, t
);
6427 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6428 t
= fold_convert (TREE_TYPE (startvar
), t
);
6429 t
= force_gimple_operand_gsi (&gsi
, t
,
6431 && TREE_ADDRESSABLE (startvar
),
6432 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6433 stmt
= gimple_build_assign (startvar
, t
);
6434 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6436 t
= fold_convert (itype
, e0
);
6437 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6438 if (POINTER_TYPE_P (type
))
6439 t
= fold_build_pointer_plus (n1
, t
);
6441 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6442 t
= fold_convert (TREE_TYPE (startvar
), t
);
6443 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6444 false, GSI_CONTINUE_LINKING
);
6447 stmt
= gimple_build_assign (endvar
, e
);
6448 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6449 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6450 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6452 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6454 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6456 if (fd
->collapse
> 1)
6457 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6461 /* The code controlling the sequential loop goes in CONT_BB,
6462 replacing the GIMPLE_OMP_CONTINUE. */
6463 gsi
= gsi_last_bb (cont_bb
);
6464 stmt
= gsi_stmt (gsi
);
6465 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6466 vmain
= gimple_omp_continue_control_use (stmt
);
6467 vback
= gimple_omp_continue_control_def (stmt
);
6469 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6471 if (POINTER_TYPE_P (type
))
6472 t
= fold_build_pointer_plus (vmain
, step
);
6474 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6475 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
6476 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6477 true, GSI_SAME_STMT
);
6478 stmt
= gimple_build_assign (vback
, t
);
6479 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6481 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6482 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6484 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6487 /* Remove GIMPLE_OMP_CONTINUE. */
6488 gsi_remove (&gsi
, true);
6490 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6491 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6493 /* Trip update code goes into TRIP_UPDATE_BB. */
6494 gsi
= gsi_start_bb (trip_update_bb
);
6496 t
= build_int_cst (itype
, 1);
6497 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
6498 stmt
= gimple_build_assign (trip_back
, t
);
6499 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6502 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6503 gsi
= gsi_last_bb (exit_bb
);
6504 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6506 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6507 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6509 gsi_remove (&gsi
, true);
6511 /* Connect the new blocks. */
6512 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
6513 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
6517 se
= find_edge (cont_bb
, body_bb
);
6518 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6523 else if (fd
->collapse
> 1)
6526 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6529 se
->flags
= EDGE_TRUE_VALUE
;
6530 find_edge (cont_bb
, trip_update_bb
)->flags
6531 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6533 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
6536 if (gimple_in_ssa_p (cfun
))
6538 gimple_stmt_iterator psi
;
6541 edge_var_map_vector
*head
;
6545 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
6547 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6548 remove arguments of the phi nodes in fin_bb. We need to create
6549 appropriate phi nodes in iter_part_bb instead. */
6550 se
= single_pred_edge (fin_bb
);
6551 re
= single_succ_edge (trip_update_bb
);
6552 head
= redirect_edge_var_map_vector (re
);
6553 ene
= single_succ_edge (entry_bb
);
6555 psi
= gsi_start_phis (fin_bb
);
6556 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
6557 gsi_next (&psi
), ++i
)
6560 source_location locus
;
6562 phi
= gsi_stmt (psi
);
6563 t
= gimple_phi_result (phi
);
6564 gcc_assert (t
== redirect_edge_var_map_result (vm
));
6565 nphi
= create_phi_node (t
, iter_part_bb
);
6567 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
6568 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
6570 /* A special case -- fd->loop.v is not yet computed in
6571 iter_part_bb, we need to use vextra instead. */
6572 if (t
== fd
->loop
.v
)
6574 add_phi_arg (nphi
, t
, ene
, locus
);
6575 locus
= redirect_edge_var_map_location (vm
);
6576 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
6578 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
6579 redirect_edge_var_map_clear (re
);
6582 psi
= gsi_start_phis (fin_bb
);
6583 if (gsi_end_p (psi
))
6585 remove_phi_node (&psi
, false);
6588 /* Make phi node for trip. */
6589 phi
= create_phi_node (trip_main
, iter_part_bb
);
6590 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
6592 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
6597 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
6598 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
6599 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
6600 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6601 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6602 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
6603 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
6604 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6605 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6609 struct loop
*trip_loop
= alloc_loop ();
6610 trip_loop
->header
= iter_part_bb
;
6611 trip_loop
->latch
= trip_update_bb
;
6612 add_loop (trip_loop
, iter_part_bb
->loop_father
);
6614 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6616 struct loop
*loop
= alloc_loop ();
6617 loop
->header
= body_bb
;
6618 if (collapse_bb
== NULL
)
6619 loop
->latch
= cont_bb
;
6620 add_loop (loop
, trip_loop
);
6626 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
6627 loop. Given parameters:
6629 for (V = N1; V cond N2; V += STEP) BODY;
6631 where COND is "<" or ">", we generate pseudocode
6639 if (V cond N2) goto L0; else goto L2;
6642 For collapsed loops, given parameters:
6644 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6645 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6646 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6649 we generate pseudocode
6655 count3 = (adj + N32 - N31) / STEP3;
6660 count2 = (adj + N22 - N21) / STEP2;
6665 count1 = (adj + N12 - N11) / STEP1;
6666 count = count1 * count2 * count3;
6676 V2 += (V3 cond3 N32) ? 0 : STEP2;
6677 V3 = (V3 cond3 N32) ? V3 : N31;
6678 V1 += (V2 cond2 N22) ? 0 : STEP1;
6679 V2 = (V2 cond2 N22) ? V2 : N21;
6681 if (V < count) goto L0; else goto L2;
6687 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
6690 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
6691 gimple_stmt_iterator gsi
;
6693 bool broken_loop
= region
->cont
== NULL
;
6695 tree
*counts
= NULL
;
6697 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6698 OMP_CLAUSE_SAFELEN
);
6699 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6700 OMP_CLAUSE__SIMDUID_
);
6703 type
= TREE_TYPE (fd
->loop
.v
);
6704 entry_bb
= region
->entry
;
6705 cont_bb
= region
->cont
;
6706 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6707 gcc_assert (broken_loop
6708 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6709 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
6712 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
6713 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6714 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
6715 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6719 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
6720 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
6721 l2_bb
= single_succ (l1_bb
);
6723 exit_bb
= region
->exit
;
6726 gsi
= gsi_last_bb (entry_bb
);
6728 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6729 /* Not needed in SSA form right now. */
6730 gcc_assert (!gimple_in_ssa_p (cfun
));
6731 if (fd
->collapse
> 1)
6733 int first_zero_iter
= -1;
6734 basic_block zero_iter_bb
= l2_bb
;
6736 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6737 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6738 zero_iter_bb
, first_zero_iter
,
6741 if (l2_dom_bb
== NULL
)
6746 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6748 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6749 OMP_CLAUSE__LOOPTEMP_
);
6750 gcc_assert (innerc
);
6751 n1
= OMP_CLAUSE_DECL (innerc
);
6752 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6753 OMP_CLAUSE__LOOPTEMP_
);
6754 gcc_assert (innerc
);
6755 n2
= OMP_CLAUSE_DECL (innerc
);
6756 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6757 fold_convert (type
, n1
));
6758 if (fd
->collapse
> 1)
6761 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
6767 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6768 fold_convert (type
, fd
->loop
.n1
));
6769 if (fd
->collapse
> 1)
6770 for (i
= 0; i
< fd
->collapse
; i
++)
6772 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6773 if (POINTER_TYPE_P (itype
))
6774 itype
= signed_type_for (itype
);
6775 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
6776 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6780 /* Remove the GIMPLE_OMP_FOR statement. */
6781 gsi_remove (&gsi
, true);
6785 /* Code to control the increment goes in the CONT_BB. */
6786 gsi
= gsi_last_bb (cont_bb
);
6787 stmt
= gsi_stmt (gsi
);
6788 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6790 if (POINTER_TYPE_P (type
))
6791 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
6793 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
6794 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
6796 if (fd
->collapse
> 1)
6798 i
= fd
->collapse
- 1;
6799 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
6801 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
6802 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
6806 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
6808 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
6811 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6813 for (i
= fd
->collapse
- 1; i
> 0; i
--)
6815 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6816 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
6817 if (POINTER_TYPE_P (itype2
))
6818 itype2
= signed_type_for (itype2
);
6819 t
= build3 (COND_EXPR
, itype2
,
6820 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6822 fold_convert (itype
, fd
->loops
[i
].n2
)),
6823 build_int_cst (itype2
, 0),
6824 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
6825 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
6826 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
6828 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
6829 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
6831 t
= build3 (COND_EXPR
, itype
,
6832 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6834 fold_convert (itype
, fd
->loops
[i
].n2
)),
6836 fold_convert (itype
, fd
->loops
[i
].n1
));
6837 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6841 /* Remove GIMPLE_OMP_CONTINUE. */
6842 gsi_remove (&gsi
, true);
6845 /* Emit the condition in L1_BB. */
6846 gsi
= gsi_start_bb (l1_bb
);
6848 t
= fold_convert (type
, n2
);
6849 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6850 false, GSI_CONTINUE_LINKING
);
6851 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
6852 stmt
= gimple_build_cond_empty (t
);
6853 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6854 if (walk_tree (gimple_cond_lhs_ptr (stmt
), expand_omp_regimplify_p
,
6856 || walk_tree (gimple_cond_rhs_ptr (stmt
), expand_omp_regimplify_p
,
6859 gsi
= gsi_for_stmt (stmt
);
6860 gimple_regimplify_operands (stmt
, &gsi
);
6863 /* Remove GIMPLE_OMP_RETURN. */
6864 gsi
= gsi_last_bb (exit_bb
);
6865 gsi_remove (&gsi
, true);
6867 /* Connect the new blocks. */
6868 remove_edge (FALLTHRU_EDGE (entry_bb
));
6872 remove_edge (BRANCH_EDGE (entry_bb
));
6873 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
6875 e
= BRANCH_EDGE (l1_bb
);
6876 ne
= FALLTHRU_EDGE (l1_bb
);
6877 e
->flags
= EDGE_TRUE_VALUE
;
6881 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6883 ne
= single_succ_edge (l1_bb
);
6884 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6887 ne
->flags
= EDGE_FALSE_VALUE
;
6888 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6889 ne
->probability
= REG_BR_PROB_BASE
/ 8;
6891 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
6892 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
6893 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
6897 struct loop
*loop
= alloc_loop ();
6898 loop
->header
= l1_bb
;
6899 loop
->latch
= cont_bb
;
6900 add_loop (loop
, l1_bb
->loop_father
);
6901 if (safelen
== NULL_TREE
)
6902 loop
->safelen
= INT_MAX
;
6905 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
6906 if (TREE_CODE (safelen
) != INTEGER_CST
)
6908 else if (!tree_fits_uhwi_p (safelen
)
6909 || tree_to_uhwi (safelen
) > INT_MAX
)
6910 loop
->safelen
= INT_MAX
;
6912 loop
->safelen
= tree_to_uhwi (safelen
);
6913 if (loop
->safelen
== 1)
6918 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
6919 cfun
->has_simduid_loops
= true;
6921 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
6923 if ((flag_tree_loop_vectorize
6924 || (!global_options_set
.x_flag_tree_loop_vectorize
6925 && !global_options_set
.x_flag_tree_vectorize
))
6926 && flag_tree_loop_optimize
6927 && loop
->safelen
> 1)
6929 loop
->force_vectorize
= true;
6930 cfun
->has_force_vectorize_loops
= true;
6936 /* Expand the OpenMP loop defined by REGION. */
6939 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
6941 struct omp_for_data fd
;
6942 struct omp_for_data_loop
*loops
;
6945 = (struct omp_for_data_loop
*)
6946 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
6947 * sizeof (struct omp_for_data_loop
));
6948 extract_omp_for_data (last_stmt (region
->entry
), &fd
, loops
);
6949 region
->sched_kind
= fd
.sched_kind
;
6951 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
6952 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6953 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6956 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
6957 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6958 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6961 /* If there isn't a continue then this is a degerate case where
6962 the introduction of abnormal edges during lowering will prevent
6963 original loops from being detected. Fix that up. */
6964 loops_state_set (LOOPS_NEED_FIXUP
);
6966 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
6967 expand_omp_simd (region
, &fd
);
6968 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
6969 && !fd
.have_ordered
)
6971 if (fd
.chunk_size
== NULL
)
6972 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
6974 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
6978 int fn_index
, start_ix
, next_ix
;
6980 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
6981 == GF_OMP_FOR_KIND_FOR
);
6982 if (fd
.chunk_size
== NULL
6983 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
6984 fd
.chunk_size
= integer_zero_node
;
6985 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6986 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
6987 ? 3 : fd
.sched_kind
;
6988 fn_index
+= fd
.have_ordered
* 4;
6989 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
6990 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
6991 if (fd
.iter_type
== long_long_unsigned_type_node
)
6993 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
6994 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
6995 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
6996 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
6998 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
6999 (enum built_in_function
) next_ix
, inner_stmt
);
7002 if (gimple_in_ssa_p (cfun
))
7003 update_ssa (TODO_update_ssa_only_virtuals
);
7007 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7009 v = GOMP_sections_start (n);
7026 v = GOMP_sections_next ();
7031 If this is a combined parallel sections, replace the call to
7032 GOMP_sections_start with call to GOMP_sections_next. */
7035 expand_omp_sections (struct omp_region
*region
)
7037 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7039 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7040 gimple_stmt_iterator si
, switch_si
;
7041 gimple sections_stmt
, stmt
, cont
;
7044 struct omp_region
*inner
;
7046 bool exit_reachable
= region
->cont
!= NULL
;
7048 gcc_assert (region
->exit
!= NULL
);
7049 entry_bb
= region
->entry
;
7050 l0_bb
= single_succ (entry_bb
);
7051 l1_bb
= region
->cont
;
7052 l2_bb
= region
->exit
;
7053 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7054 l2
= gimple_block_label (l2_bb
);
7057 /* This can happen if there are reductions. */
7058 len
= EDGE_COUNT (l0_bb
->succs
);
7059 gcc_assert (len
> 0);
7060 e
= EDGE_SUCC (l0_bb
, len
- 1);
7061 si
= gsi_last_bb (e
->dest
);
7064 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7065 l2
= gimple_block_label (e
->dest
);
7067 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7069 si
= gsi_last_bb (e
->dest
);
7071 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7073 l2
= gimple_block_label (e
->dest
);
7079 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7081 default_bb
= create_empty_bb (l0_bb
);
7083 /* We will build a switch() with enough cases for all the
7084 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7085 and a default case to abort if something goes wrong. */
7086 len
= EDGE_COUNT (l0_bb
->succs
);
7088 /* Use vec::quick_push on label_vec throughout, since we know the size
7090 auto_vec
<tree
> label_vec (len
);
7092 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7093 GIMPLE_OMP_SECTIONS statement. */
7094 si
= gsi_last_bb (entry_bb
);
7095 sections_stmt
= gsi_stmt (si
);
7096 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
7097 vin
= gimple_omp_sections_control (sections_stmt
);
7098 if (!is_combined_parallel (region
))
7100 /* If we are not inside a combined parallel+sections region,
7101 call GOMP_sections_start. */
7102 t
= build_int_cst (unsigned_type_node
, len
- 1);
7103 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
7104 stmt
= gimple_build_call (u
, 1, t
);
7108 /* Otherwise, call GOMP_sections_next. */
7109 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7110 stmt
= gimple_build_call (u
, 0);
7112 gimple_call_set_lhs (stmt
, vin
);
7113 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7114 gsi_remove (&si
, true);
7116 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
7118 switch_si
= gsi_last_bb (l0_bb
);
7119 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
7122 cont
= last_stmt (l1_bb
);
7123 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
7124 vmain
= gimple_omp_continue_control_use (cont
);
7125 vnext
= gimple_omp_continue_control_def (cont
);
7133 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
7134 label_vec
.quick_push (t
);
7137 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
7138 for (inner
= region
->inner
, casei
= 1;
7140 inner
= inner
->next
, i
++, casei
++)
7142 basic_block s_entry_bb
, s_exit_bb
;
7144 /* Skip optional reduction region. */
7145 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
7152 s_entry_bb
= inner
->entry
;
7153 s_exit_bb
= inner
->exit
;
7155 t
= gimple_block_label (s_entry_bb
);
7156 u
= build_int_cst (unsigned_type_node
, casei
);
7157 u
= build_case_label (u
, NULL
, t
);
7158 label_vec
.quick_push (u
);
7160 si
= gsi_last_bb (s_entry_bb
);
7161 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
7162 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
7163 gsi_remove (&si
, true);
7164 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
7166 if (s_exit_bb
== NULL
)
7169 si
= gsi_last_bb (s_exit_bb
);
7170 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7171 gsi_remove (&si
, true);
7173 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
7176 /* Error handling code goes in DEFAULT_BB. */
7177 t
= gimple_block_label (default_bb
);
7178 u
= build_case_label (NULL
, NULL
, t
);
7179 make_edge (l0_bb
, default_bb
, 0);
7180 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
7182 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
7183 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
7184 gsi_remove (&switch_si
, true);
7186 si
= gsi_start_bb (default_bb
);
7187 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
7188 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
7194 /* Code to get the next section goes in L1_BB. */
7195 si
= gsi_last_bb (l1_bb
);
7196 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
7198 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7199 stmt
= gimple_build_call (bfn_decl
, 0);
7200 gimple_call_set_lhs (stmt
, vnext
);
7201 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7202 gsi_remove (&si
, true);
7204 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
7207 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7208 si
= gsi_last_bb (l2_bb
);
7209 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
7210 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
7211 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
7212 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
7214 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
7215 stmt
= gimple_build_call (t
, 0);
7216 if (gimple_omp_return_lhs (gsi_stmt (si
)))
7217 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
7218 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7219 gsi_remove (&si
, true);
7221 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
7225 /* Expand code for an OpenMP single directive. We've already expanded
7226 much of the code, here we simply place the GOMP_barrier call. */
7229 expand_omp_single (struct omp_region
*region
)
7231 basic_block entry_bb
, exit_bb
;
7232 gimple_stmt_iterator si
;
7234 entry_bb
= region
->entry
;
7235 exit_bb
= region
->exit
;
7237 si
= gsi_last_bb (entry_bb
);
7238 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
7239 gsi_remove (&si
, true);
7240 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7242 si
= gsi_last_bb (exit_bb
);
7243 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
7245 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
7246 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
7248 gsi_remove (&si
, true);
7249 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7253 /* Generic expansion for OpenMP synchronization directives: master,
7254 ordered and critical. All we need to do here is remove the entry
7255 and exit markers for REGION. */
7258 expand_omp_synch (struct omp_region
*region
)
7260 basic_block entry_bb
, exit_bb
;
7261 gimple_stmt_iterator si
;
7263 entry_bb
= region
->entry
;
7264 exit_bb
= region
->exit
;
7266 si
= gsi_last_bb (entry_bb
);
7267 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
7268 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
7269 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
7270 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
7271 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
7272 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
7273 gsi_remove (&si
, true);
7274 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7278 si
= gsi_last_bb (exit_bb
);
7279 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7280 gsi_remove (&si
, true);
7281 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7285 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7286 operation as a normal volatile load. */
7289 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
7290 tree loaded_val
, int index
)
7292 enum built_in_function tmpbase
;
7293 gimple_stmt_iterator gsi
;
7294 basic_block store_bb
;
7297 tree decl
, call
, type
, itype
;
7299 gsi
= gsi_last_bb (load_bb
);
7300 stmt
= gsi_stmt (gsi
);
7301 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7302 loc
= gimple_location (stmt
);
7304 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7305 is smaller than word size, then expand_atomic_load assumes that the load
7306 is atomic. We could avoid the builtin entirely in this case. */
7308 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7309 decl
= builtin_decl_explicit (tmpbase
);
7310 if (decl
== NULL_TREE
)
7313 type
= TREE_TYPE (loaded_val
);
7314 itype
= TREE_TYPE (TREE_TYPE (decl
));
7316 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
7317 build_int_cst (NULL
,
7318 gimple_omp_atomic_seq_cst_p (stmt
)
7320 : MEMMODEL_RELAXED
));
7321 if (!useless_type_conversion_p (type
, itype
))
7322 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7323 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7325 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7326 gsi_remove (&gsi
, true);
7328 store_bb
= single_succ (load_bb
);
7329 gsi
= gsi_last_bb (store_bb
);
7330 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7331 gsi_remove (&gsi
, true);
7333 if (gimple_in_ssa_p (cfun
))
7334 update_ssa (TODO_update_ssa_no_phi
);
7339 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7340 operation as a normal volatile store. */
7343 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
7344 tree loaded_val
, tree stored_val
, int index
)
7346 enum built_in_function tmpbase
;
7347 gimple_stmt_iterator gsi
;
7348 basic_block store_bb
= single_succ (load_bb
);
7351 tree decl
, call
, type
, itype
;
7352 enum machine_mode imode
;
7355 gsi
= gsi_last_bb (load_bb
);
7356 stmt
= gsi_stmt (gsi
);
7357 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7359 /* If the load value is needed, then this isn't a store but an exchange. */
7360 exchange
= gimple_omp_atomic_need_value_p (stmt
);
7362 gsi
= gsi_last_bb (store_bb
);
7363 stmt
= gsi_stmt (gsi
);
7364 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
7365 loc
= gimple_location (stmt
);
7367 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7368 is smaller than word size, then expand_atomic_store assumes that the store
7369 is atomic. We could avoid the builtin entirely in this case. */
7371 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
7372 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
7373 decl
= builtin_decl_explicit (tmpbase
);
7374 if (decl
== NULL_TREE
)
7377 type
= TREE_TYPE (stored_val
);
7379 /* Dig out the type of the function's second argument. */
7380 itype
= TREE_TYPE (decl
);
7381 itype
= TYPE_ARG_TYPES (itype
);
7382 itype
= TREE_CHAIN (itype
);
7383 itype
= TREE_VALUE (itype
);
7384 imode
= TYPE_MODE (itype
);
7386 if (exchange
&& !can_atomic_exchange_p (imode
, true))
7389 if (!useless_type_conversion_p (itype
, type
))
7390 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
7391 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
7392 build_int_cst (NULL
,
7393 gimple_omp_atomic_seq_cst_p (stmt
)
7395 : MEMMODEL_RELAXED
));
7398 if (!useless_type_conversion_p (type
, itype
))
7399 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7400 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7403 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7404 gsi_remove (&gsi
, true);
7406 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7407 gsi
= gsi_last_bb (load_bb
);
7408 gsi_remove (&gsi
, true);
7410 if (gimple_in_ssa_p (cfun
))
7411 update_ssa (TODO_update_ssa_no_phi
);
7416 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7417 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7418 size of the data type, and thus usable to find the index of the builtin
7419 decl. Returns false if the expression is not of the proper form. */
7422 expand_omp_atomic_fetch_op (basic_block load_bb
,
7423 tree addr
, tree loaded_val
,
7424 tree stored_val
, int index
)
7426 enum built_in_function oldbase
, newbase
, tmpbase
;
7427 tree decl
, itype
, call
;
7429 basic_block store_bb
= single_succ (load_bb
);
7430 gimple_stmt_iterator gsi
;
7433 enum tree_code code
;
7434 bool need_old
, need_new
;
7435 enum machine_mode imode
;
7438 /* We expect to find the following sequences:
7441 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7444 val = tmp OP something; (or: something OP tmp)
7445 GIMPLE_OMP_STORE (val)
7447 ???FIXME: Allow a more flexible sequence.
7448 Perhaps use data flow to pick the statements.
7452 gsi
= gsi_after_labels (store_bb
);
7453 stmt
= gsi_stmt (gsi
);
7454 loc
= gimple_location (stmt
);
7455 if (!is_gimple_assign (stmt
))
7458 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
7460 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
7461 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
7462 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
7463 gcc_checking_assert (!need_old
|| !need_new
);
7465 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
7468 /* Check for one of the supported fetch-op operations. */
7469 code
= gimple_assign_rhs_code (stmt
);
7473 case POINTER_PLUS_EXPR
:
7474 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
7475 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
7478 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
7479 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
7482 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
7483 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
7486 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
7487 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
7490 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
7491 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
7497 /* Make sure the expression is of the proper form. */
7498 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
7499 rhs
= gimple_assign_rhs2 (stmt
);
7500 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
7501 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
7502 rhs
= gimple_assign_rhs1 (stmt
);
7506 tmpbase
= ((enum built_in_function
)
7507 ((need_new
? newbase
: oldbase
) + index
+ 1));
7508 decl
= builtin_decl_explicit (tmpbase
);
7509 if (decl
== NULL_TREE
)
7511 itype
= TREE_TYPE (TREE_TYPE (decl
));
7512 imode
= TYPE_MODE (itype
);
7514 /* We could test all of the various optabs involved, but the fact of the
7515 matter is that (with the exception of i486 vs i586 and xadd) all targets
7516 that support any atomic operaton optab also implements compare-and-swap.
7517 Let optabs.c take care of expanding any compare-and-swap loop. */
7518 if (!can_compare_and_swap_p (imode
, true))
7521 gsi
= gsi_last_bb (load_bb
);
7522 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7524 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7525 It only requires that the operation happen atomically. Thus we can
7526 use the RELAXED memory model. */
7527 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
7528 fold_convert_loc (loc
, itype
, rhs
),
7529 build_int_cst (NULL
,
7530 seq_cst
? MEMMODEL_SEQ_CST
7531 : MEMMODEL_RELAXED
));
7533 if (need_old
|| need_new
)
7535 lhs
= need_old
? loaded_val
: stored_val
;
7536 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
7537 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
7540 call
= fold_convert_loc (loc
, void_type_node
, call
);
7541 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7542 gsi_remove (&gsi
, true);
7544 gsi
= gsi_last_bb (store_bb
);
7545 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7546 gsi_remove (&gsi
, true);
7547 gsi
= gsi_last_bb (store_bb
);
7548 gsi_remove (&gsi
, true);
7550 if (gimple_in_ssa_p (cfun
))
7551 update_ssa (TODO_update_ssa_no_phi
);
7556 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7560 newval = rhs; // with oldval replacing *addr in rhs
7561 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7562 if (oldval != newval)
7565 INDEX is log2 of the size of the data type, and thus usable to find the
7566 index of the builtin decl. */
7569 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
7570 tree addr
, tree loaded_val
, tree stored_val
,
7573 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
7574 tree type
, itype
, cmpxchg
, iaddr
;
7575 gimple_stmt_iterator si
;
7576 basic_block loop_header
= single_succ (load_bb
);
7579 enum built_in_function fncode
;
7581 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7582 order to use the RELAXED memory model effectively. */
7583 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7585 cmpxchg
= builtin_decl_explicit (fncode
);
7586 if (cmpxchg
== NULL_TREE
)
7588 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7589 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
7591 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
7594 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7595 si
= gsi_last_bb (load_bb
);
7596 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7598 /* For floating-point values, we'll need to view-convert them to integers
7599 so that we can perform the atomic compare and swap. Simplify the
7600 following code by always setting up the "i"ntegral variables. */
7601 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
7605 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
7608 = force_gimple_operand_gsi (&si
,
7609 fold_convert (TREE_TYPE (iaddr
), addr
),
7610 false, NULL_TREE
, true, GSI_SAME_STMT
);
7611 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
7612 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7613 loadedi
= create_tmp_var (itype
, NULL
);
7614 if (gimple_in_ssa_p (cfun
))
7615 loadedi
= make_ssa_name (loadedi
, NULL
);
7620 loadedi
= loaded_val
;
7623 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7624 tree loaddecl
= builtin_decl_explicit (fncode
);
7627 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
7628 build_call_expr (loaddecl
, 2, iaddr
,
7629 build_int_cst (NULL_TREE
,
7630 MEMMODEL_RELAXED
)));
7632 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
7633 build_int_cst (TREE_TYPE (iaddr
), 0));
7636 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
7639 /* Move the value to the LOADEDI temporary. */
7640 if (gimple_in_ssa_p (cfun
))
7642 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
7643 phi
= create_phi_node (loadedi
, loop_header
);
7644 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
7648 gsi_insert_before (&si
,
7649 gimple_build_assign (loadedi
, initial
),
7651 if (loadedi
!= loaded_val
)
7653 gimple_stmt_iterator gsi2
;
7656 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
7657 gsi2
= gsi_start_bb (loop_header
);
7658 if (gimple_in_ssa_p (cfun
))
7661 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7662 true, GSI_SAME_STMT
);
7663 stmt
= gimple_build_assign (loaded_val
, x
);
7664 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
7668 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
7669 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7670 true, GSI_SAME_STMT
);
7673 gsi_remove (&si
, true);
7675 si
= gsi_last_bb (store_bb
);
7676 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7679 storedi
= stored_val
;
7682 force_gimple_operand_gsi (&si
,
7683 build1 (VIEW_CONVERT_EXPR
, itype
,
7684 stored_val
), true, NULL_TREE
, true,
7687 /* Build the compare&swap statement. */
7688 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
7689 new_storedi
= force_gimple_operand_gsi (&si
,
7690 fold_convert (TREE_TYPE (loadedi
),
7693 true, GSI_SAME_STMT
);
7695 if (gimple_in_ssa_p (cfun
))
7699 old_vali
= create_tmp_var (TREE_TYPE (loadedi
), NULL
);
7700 stmt
= gimple_build_assign (old_vali
, loadedi
);
7701 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7703 stmt
= gimple_build_assign (loadedi
, new_storedi
);
7704 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7707 /* Note that we always perform the comparison as an integer, even for
7708 floating point. This allows the atomic operation to properly
7709 succeed even with NaNs and -0.0. */
7710 stmt
= gimple_build_cond_empty
7711 (build2 (NE_EXPR
, boolean_type_node
,
7712 new_storedi
, old_vali
));
7713 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7716 e
= single_succ_edge (store_bb
);
7717 e
->flags
&= ~EDGE_FALLTHRU
;
7718 e
->flags
|= EDGE_FALSE_VALUE
;
7720 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
7722 /* Copy the new value to loadedi (we already did that before the condition
7723 if we are not in SSA). */
7724 if (gimple_in_ssa_p (cfun
))
7726 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
7727 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
7730 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
7731 gsi_remove (&si
, true);
7733 struct loop
*loop
= alloc_loop ();
7734 loop
->header
= loop_header
;
7735 loop
->latch
= store_bb
;
7736 add_loop (loop
, loop_header
->loop_father
);
7738 if (gimple_in_ssa_p (cfun
))
7739 update_ssa (TODO_update_ssa_no_phi
);
7744 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7746 GOMP_atomic_start ();
7750 The result is not globally atomic, but works so long as all parallel
7751 references are within #pragma omp atomic directives. According to
7752 responses received from omp@openmp.org, appears to be within spec.
7753 Which makes sense, since that's how several other compilers handle
7754 this situation as well.
7755 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
7756 expanding. STORED_VAL is the operand of the matching
7757 GIMPLE_OMP_ATOMIC_STORE.
7760 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
7764 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
7769 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
7770 tree addr
, tree loaded_val
, tree stored_val
)
7772 gimple_stmt_iterator si
;
7776 si
= gsi_last_bb (load_bb
);
7777 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7779 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
7780 t
= build_call_expr (t
, 0);
7781 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7783 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
7784 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7785 gsi_remove (&si
, true);
7787 si
= gsi_last_bb (store_bb
);
7788 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7790 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
7792 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7794 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
7795 t
= build_call_expr (t
, 0);
7796 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7797 gsi_remove (&si
, true);
7799 if (gimple_in_ssa_p (cfun
))
7800 update_ssa (TODO_update_ssa_no_phi
);
7804 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
7805 using expand_omp_atomic_fetch_op. If it failed, we try to
7806 call expand_omp_atomic_pipeline, and if it fails too, the
7807 ultimate fallback is wrapping the operation in a mutex
7808 (expand_omp_atomic_mutex). REGION is the atomic region built
7809 by build_omp_regions_1(). */
7812 expand_omp_atomic (struct omp_region
*region
)
7814 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
7815 gimple load
= last_stmt (load_bb
), store
= last_stmt (store_bb
);
7816 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
7817 tree addr
= gimple_omp_atomic_load_rhs (load
);
7818 tree stored_val
= gimple_omp_atomic_store_val (store
);
7819 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7820 HOST_WIDE_INT index
;
7822 /* Make sure the type is one of the supported sizes. */
7823 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
7824 index
= exact_log2 (index
);
7825 if (index
>= 0 && index
<= 4)
7827 unsigned int align
= TYPE_ALIGN_UNIT (type
);
7829 /* __sync builtins require strict data alignment. */
7830 if (exact_log2 (align
) >= index
)
7833 if (loaded_val
== stored_val
7834 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7835 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7836 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7837 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
7841 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7842 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7843 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7844 && store_bb
== single_succ (load_bb
)
7845 && first_stmt (store_bb
) == store
7846 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
7850 /* When possible, use specialized atomic update functions. */
7851 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
7852 && store_bb
== single_succ (load_bb
)
7853 && expand_omp_atomic_fetch_op (load_bb
, addr
,
7854 loaded_val
, stored_val
, index
))
7857 /* If we don't have specialized __sync builtins, try and implement
7858 as a compare and swap loop. */
7859 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
7860 loaded_val
, stored_val
, index
))
7865 /* The ultimate fallback is wrapping the operation in a mutex. */
7866 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
7870 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
7873 expand_omp_target (struct omp_region
*region
)
7875 basic_block entry_bb
, exit_bb
, new_bb
;
7876 struct function
*child_cfun
= NULL
;
7877 tree child_fn
= NULL_TREE
, block
, t
;
7878 gimple_stmt_iterator gsi
;
7879 gimple entry_stmt
, stmt
;
7882 entry_stmt
= last_stmt (region
->entry
);
7883 new_bb
= region
->entry
;
7884 int kind
= gimple_omp_target_kind (entry_stmt
);
7885 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7887 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
7888 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7891 entry_bb
= region
->entry
;
7892 exit_bb
= region
->exit
;
7894 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7896 unsigned srcidx
, dstidx
, num
;
7898 /* If the target region needs data sent from the parent
7899 function, then the very first statement (except possible
7900 tree profile counter updates) of the parallel body
7901 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7902 &.OMP_DATA_O is passed as an argument to the child function,
7903 we need to replace it with the argument as seen by the child
7906 In most cases, this will end up being the identity assignment
7907 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7908 a function call that has been inlined, the original PARM_DECL
7909 .OMP_DATA_I may have been converted into a different local
7910 variable. In which case, we need to keep the assignment. */
7911 if (gimple_omp_target_data_arg (entry_stmt
))
7913 basic_block entry_succ_bb
= single_succ (entry_bb
);
7914 gimple_stmt_iterator gsi
;
7916 gimple tgtcopy_stmt
= NULL
;
7918 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt
), 0);
7920 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
7922 gcc_assert (!gsi_end_p (gsi
));
7923 stmt
= gsi_stmt (gsi
);
7924 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
7927 if (gimple_num_ops (stmt
) == 2)
7929 tree arg
= gimple_assign_rhs1 (stmt
);
7931 /* We're ignoring the subcode because we're
7932 effectively doing a STRIP_NOPS. */
7934 if (TREE_CODE (arg
) == ADDR_EXPR
7935 && TREE_OPERAND (arg
, 0) == sender
)
7937 tgtcopy_stmt
= stmt
;
7943 gcc_assert (tgtcopy_stmt
!= NULL
);
7944 arg
= DECL_ARGUMENTS (child_fn
);
7946 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
7947 gsi_remove (&gsi
, true);
7950 /* Declare local variables needed in CHILD_CFUN. */
7951 block
= DECL_INITIAL (child_fn
);
7952 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
7953 /* The gimplifier could record temporaries in target block
7954 rather than in containing function's local_decls chain,
7955 which would mean cgraph missed finalizing them. Do it now. */
7956 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
7957 if (TREE_CODE (t
) == VAR_DECL
7959 && !DECL_EXTERNAL (t
))
7960 varpool_finalize_decl (t
);
7961 DECL_SAVED_TREE (child_fn
) = NULL
;
7962 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7963 gimple_set_body (child_fn
, NULL
);
7964 TREE_USED (block
) = 1;
7966 /* Reset DECL_CONTEXT on function arguments. */
7967 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
7968 DECL_CONTEXT (t
) = child_fn
;
7970 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
7971 so that it can be moved to the child function. */
7972 gsi
= gsi_last_bb (entry_bb
);
7973 stmt
= gsi_stmt (gsi
);
7974 gcc_assert (stmt
&& gimple_code (stmt
) == GIMPLE_OMP_TARGET
7975 && gimple_omp_target_kind (stmt
)
7976 == GF_OMP_TARGET_KIND_REGION
);
7977 gsi_remove (&gsi
, true);
7978 e
= split_block (entry_bb
, stmt
);
7980 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7982 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
7985 gsi
= gsi_last_bb (exit_bb
);
7986 gcc_assert (!gsi_end_p (gsi
)
7987 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
7988 stmt
= gimple_build_return (NULL
);
7989 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
7990 gsi_remove (&gsi
, true);
7993 /* Move the target region into CHILD_CFUN. */
7995 block
= gimple_block (entry_stmt
);
7997 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7999 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8000 /* When the OMP expansion process cannot guarantee an up-to-date
8001 loop tree arrange for the child function to fixup loops. */
8002 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8003 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8005 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8006 num
= vec_safe_length (child_cfun
->local_decls
);
8007 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8009 t
= (*child_cfun
->local_decls
)[srcidx
];
8010 if (DECL_CONTEXT (t
) == cfun
->decl
)
8012 if (srcidx
!= dstidx
)
8013 (*child_cfun
->local_decls
)[dstidx
] = t
;
8017 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8019 /* Inform the callgraph about the new function. */
8020 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
8021 cgraph_add_new_function (child_fn
, true);
8023 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8024 fixed in a following pass. */
8025 push_cfun (child_cfun
);
8026 rebuild_cgraph_edges ();
8028 /* Some EH regions might become dead, see PR34608. If
8029 pass_cleanup_cfg isn't the first pass to happen with the
8030 new child, these dead EH edges might cause problems.
8031 Clean them up now. */
8032 if (flag_exceptions
)
8035 bool changed
= false;
8037 FOR_EACH_BB_FN (bb
, cfun
)
8038 changed
|= gimple_purge_dead_eh_edges (bb
);
8040 cleanup_tree_cfg ();
8045 /* Emit a library call to launch the target region, or do data
8047 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8048 enum built_in_function start_ix
;
8049 location_t clause_loc
;
8051 clauses
= gimple_omp_target_clauses (entry_stmt
);
8053 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8054 start_ix
= BUILT_IN_GOMP_TARGET
;
8055 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
8056 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
8058 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
8060 /* By default, the value of DEVICE is -1 (let runtime library choose)
8061 and there is no conditional. */
8063 device
= build_int_cst (integer_type_node
, -1);
8065 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
8067 cond
= OMP_CLAUSE_IF_EXPR (c
);
8069 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
8072 device
= OMP_CLAUSE_DEVICE_ID (c
);
8073 clause_loc
= OMP_CLAUSE_LOCATION (c
);
8076 clause_loc
= gimple_location (entry_stmt
);
8078 /* Ensure 'device' is of the correct type. */
8079 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
8081 /* If we found the clause 'if (cond)', build
8082 (cond ? device : -2). */
8085 cond
= gimple_boolify (cond
);
8087 basic_block cond_bb
, then_bb
, else_bb
;
8091 tmp_var
= create_tmp_var (TREE_TYPE (device
), NULL
);
8092 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8094 gsi
= gsi_last_bb (new_bb
);
8096 e
= split_block (new_bb
, gsi_stmt (gsi
));
8099 e
= split_block (new_bb
, NULL
);
8104 then_bb
= create_empty_bb (cond_bb
);
8105 else_bb
= create_empty_bb (then_bb
);
8106 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
8107 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
8109 stmt
= gimple_build_cond_empty (cond
);
8110 gsi
= gsi_last_bb (cond_bb
);
8111 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8113 gsi
= gsi_start_bb (then_bb
);
8114 stmt
= gimple_build_assign (tmp_var
, device
);
8115 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8117 gsi
= gsi_start_bb (else_bb
);
8118 stmt
= gimple_build_assign (tmp_var
,
8119 build_int_cst (integer_type_node
, -2));
8120 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8122 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
8123 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
8124 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
8125 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
8126 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
8127 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
8132 gsi
= gsi_last_bb (new_bb
);
8133 t
= gimple_omp_target_data_arg (entry_stmt
);
8136 t1
= size_zero_node
;
8137 t2
= build_zero_cst (ptr_type_node
);
8143 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
8144 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
8145 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
8146 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
8147 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
8151 /* FIXME: This will be address of
8152 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
8153 symbol, as soon as the linker plugin is able to create it for us. */
8154 tree openmp_target
= build_zero_cst (ptr_type_node
);
8155 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8157 tree fnaddr
= build_fold_addr_expr (child_fn
);
8158 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 7,
8159 device
, fnaddr
, openmp_target
, t1
, t2
, t3
, t4
);
8162 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 6,
8163 device
, openmp_target
, t1
, t2
, t3
, t4
);
8164 gimple_set_location (g
, gimple_location (entry_stmt
));
8165 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8166 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8169 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
8170 gsi_remove (&gsi
, true);
8172 if (kind
== GF_OMP_TARGET_KIND_DATA
&& region
->exit
)
8174 gsi
= gsi_last_bb (region
->exit
);
8176 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
8177 gsi_remove (&gsi
, true);
8182 /* Expand the parallel region tree rooted at REGION. Expansion
8183 proceeds in depth-first order. Innermost regions are expanded
8184 first. This way, parallel regions that require a new function to
8185 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8186 internal dependencies in their body. */
8189 expand_omp (struct omp_region
*region
)
8193 location_t saved_location
;
8194 gimple inner_stmt
= NULL
;
8196 /* First, determine whether this is a combined parallel+workshare
8198 if (region
->type
== GIMPLE_OMP_PARALLEL
)
8199 determine_parallel_type (region
);
8201 if (region
->type
== GIMPLE_OMP_FOR
8202 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
8203 inner_stmt
= last_stmt (region
->inner
->entry
);
8206 expand_omp (region
->inner
);
8208 saved_location
= input_location
;
8209 if (gimple_has_location (last_stmt (region
->entry
)))
8210 input_location
= gimple_location (last_stmt (region
->entry
));
8212 switch (region
->type
)
8214 case GIMPLE_OMP_PARALLEL
:
8215 case GIMPLE_OMP_TASK
:
8216 expand_omp_taskreg (region
);
8219 case GIMPLE_OMP_FOR
:
8220 expand_omp_for (region
, inner_stmt
);
8223 case GIMPLE_OMP_SECTIONS
:
8224 expand_omp_sections (region
);
8227 case GIMPLE_OMP_SECTION
:
8228 /* Individual omp sections are handled together with their
8229 parent GIMPLE_OMP_SECTIONS region. */
8232 case GIMPLE_OMP_SINGLE
:
8233 expand_omp_single (region
);
8236 case GIMPLE_OMP_MASTER
:
8237 case GIMPLE_OMP_TASKGROUP
:
8238 case GIMPLE_OMP_ORDERED
:
8239 case GIMPLE_OMP_CRITICAL
:
8240 case GIMPLE_OMP_TEAMS
:
8241 expand_omp_synch (region
);
8244 case GIMPLE_OMP_ATOMIC_LOAD
:
8245 expand_omp_atomic (region
);
8248 case GIMPLE_OMP_TARGET
:
8249 expand_omp_target (region
);
8256 input_location
= saved_location
;
8257 region
= region
->next
;
8262 /* Helper for build_omp_regions. Scan the dominator tree starting at
8263 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8264 true, the function ends once a single tree is built (otherwise, whole
8265 forest of OMP constructs may be built). */
8268 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
8271 gimple_stmt_iterator gsi
;
8275 gsi
= gsi_last_bb (bb
);
8276 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
8278 struct omp_region
*region
;
8279 enum gimple_code code
;
8281 stmt
= gsi_stmt (gsi
);
8282 code
= gimple_code (stmt
);
8283 if (code
== GIMPLE_OMP_RETURN
)
8285 /* STMT is the return point out of region PARENT. Mark it
8286 as the exit point and make PARENT the immediately
8287 enclosing region. */
8288 gcc_assert (parent
);
8291 parent
= parent
->outer
;
8293 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
8295 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8296 GIMPLE_OMP_RETURN, but matches with
8297 GIMPLE_OMP_ATOMIC_LOAD. */
8298 gcc_assert (parent
);
8299 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
8302 parent
= parent
->outer
;
8305 else if (code
== GIMPLE_OMP_CONTINUE
)
8307 gcc_assert (parent
);
8310 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
8312 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8313 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8316 else if (code
== GIMPLE_OMP_TARGET
8317 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_UPDATE
)
8318 new_omp_region (bb
, code
, parent
);
8321 /* Otherwise, this directive becomes the parent for a new
8323 region
= new_omp_region (bb
, code
, parent
);
8328 if (single_tree
&& !parent
)
8331 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
8333 son
= next_dom_son (CDI_DOMINATORS
, son
))
8334 build_omp_regions_1 (son
, parent
, single_tree
);
8337 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8341 build_omp_regions_root (basic_block root
)
8343 gcc_assert (root_omp_region
== NULL
);
8344 build_omp_regions_1 (root
, NULL
, true);
8345 gcc_assert (root_omp_region
!= NULL
);
8348 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8351 omp_expand_local (basic_block head
)
8353 build_omp_regions_root (head
);
8354 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
8356 fprintf (dump_file
, "\nOMP region tree\n\n");
8357 dump_omp_region (dump_file
, root_omp_region
, 0);
8358 fprintf (dump_file
, "\n");
8361 remove_exit_barriers (root_omp_region
);
8362 expand_omp (root_omp_region
);
8364 free_omp_regions ();
8367 /* Scan the CFG and build a tree of OMP regions. Return the root of
8368 the OMP region tree. */
8371 build_omp_regions (void)
8373 gcc_assert (root_omp_region
== NULL
);
8374 calculate_dominance_info (CDI_DOMINATORS
);
8375 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
8378 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8381 execute_expand_omp (void)
8383 build_omp_regions ();
8385 if (!root_omp_region
)
8390 fprintf (dump_file
, "\nOMP region tree\n\n");
8391 dump_omp_region (dump_file
, root_omp_region
, 0);
8392 fprintf (dump_file
, "\n");
8395 remove_exit_barriers (root_omp_region
);
8397 expand_omp (root_omp_region
);
8399 cleanup_tree_cfg ();
8401 free_omp_regions ();
8406 /* OMP expansion -- the default pass, run before creation of SSA form. */
8410 const pass_data pass_data_expand_omp
=
8412 GIMPLE_PASS
, /* type */
8413 "ompexp", /* name */
8414 OPTGROUP_NONE
, /* optinfo_flags */
8415 true, /* has_execute */
8416 TV_NONE
, /* tv_id */
8417 PROP_gimple_any
, /* properties_required */
8418 0, /* properties_provided */
8419 0, /* properties_destroyed */
8420 0, /* todo_flags_start */
8421 0, /* todo_flags_finish */
8424 class pass_expand_omp
: public gimple_opt_pass
8427 pass_expand_omp (gcc::context
*ctxt
)
8428 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
8431 /* opt_pass methods: */
8432 virtual bool gate (function
*)
8434 return ((flag_openmp
!= 0 || flag_openmp_simd
!= 0
8435 || flag_cilkplus
!= 0) && !seen_error ());
8438 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
8440 }; // class pass_expand_omp
8445 make_pass_expand_omp (gcc::context
*ctxt
)
8447 return new pass_expand_omp (ctxt
);
8450 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8452 /* If ctx is a worksharing context inside of a cancellable parallel
8453 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8454 and conditional branch to parallel's cancel_label to handle
8455 cancellation in the implicit barrier. */
8458 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
8460 gimple omp_return
= gimple_seq_last_stmt (*body
);
8461 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
8462 if (gimple_omp_return_nowait_p (omp_return
))
8465 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
8466 && ctx
->outer
->cancellable
)
8468 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
8469 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
8470 tree lhs
= create_tmp_var (c_bool_type
, NULL
);
8471 gimple_omp_return_set_lhs (omp_return
, lhs
);
8472 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
8473 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
8474 fold_convert (c_bool_type
,
8475 boolean_false_node
),
8476 ctx
->outer
->cancel_label
, fallthru_label
);
8477 gimple_seq_add_stmt (body
, g
);
8478 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
8482 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8483 CTX is the enclosing OMP context for the current statement. */
8486 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8488 tree block
, control
;
8489 gimple_stmt_iterator tgsi
;
8490 gimple stmt
, new_stmt
, bind
, t
;
8491 gimple_seq ilist
, dlist
, olist
, new_body
;
8493 stmt
= gsi_stmt (*gsi_p
);
8495 push_gimplify_context ();
8499 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
8500 &ilist
, &dlist
, ctx
, NULL
);
8502 new_body
= gimple_omp_body (stmt
);
8503 gimple_omp_set_body (stmt
, NULL
);
8504 tgsi
= gsi_start (new_body
);
8505 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
8510 sec_start
= gsi_stmt (tgsi
);
8511 sctx
= maybe_lookup_ctx (sec_start
);
8514 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
8515 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
8516 GSI_CONTINUE_LINKING
);
8517 gimple_omp_set_body (sec_start
, NULL
);
8519 if (gsi_one_before_end_p (tgsi
))
8521 gimple_seq l
= NULL
;
8522 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
8524 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
8525 gimple_omp_section_set_last (sec_start
);
8528 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
8529 GSI_CONTINUE_LINKING
);
8532 block
= make_node (BLOCK
);
8533 bind
= gimple_build_bind (NULL
, new_body
, block
);
8536 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
8538 block
= make_node (BLOCK
);
8539 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8540 gsi_replace (gsi_p
, new_stmt
, true);
8542 pop_gimplify_context (new_stmt
);
8543 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
8544 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8545 if (BLOCK_VARS (block
))
8546 TREE_USED (block
) = 1;
8549 gimple_seq_add_seq (&new_body
, ilist
);
8550 gimple_seq_add_stmt (&new_body
, stmt
);
8551 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
8552 gimple_seq_add_stmt (&new_body
, bind
);
8554 control
= create_tmp_var (unsigned_type_node
, ".section");
8555 t
= gimple_build_omp_continue (control
, control
);
8556 gimple_omp_sections_set_control (stmt
, control
);
8557 gimple_seq_add_stmt (&new_body
, t
);
8559 gimple_seq_add_seq (&new_body
, olist
);
8560 if (ctx
->cancellable
)
8561 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
8562 gimple_seq_add_seq (&new_body
, dlist
);
8564 new_body
= maybe_catch_exception (new_body
);
8566 t
= gimple_build_omp_return
8567 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
8568 OMP_CLAUSE_NOWAIT
));
8569 gimple_seq_add_stmt (&new_body
, t
);
8570 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
8572 gimple_bind_set_body (new_stmt
, new_body
);
8576 /* A subroutine of lower_omp_single. Expand the simple form of
8577 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8579 if (GOMP_single_start ())
8581 [ GOMP_barrier (); ] -> unless 'nowait' is present.
8583 FIXME. It may be better to delay expanding the logic of this until
8584 pass_expand_omp. The expanded logic may make the job more difficult
8585 to a synchronization analysis pass. */
8588 lower_omp_single_simple (gimple single_stmt
, gimple_seq
*pre_p
)
8590 location_t loc
= gimple_location (single_stmt
);
8591 tree tlabel
= create_artificial_label (loc
);
8592 tree flabel
= create_artificial_label (loc
);
8596 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
8597 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)), NULL
);
8598 call
= gimple_build_call (decl
, 0);
8599 gimple_call_set_lhs (call
, lhs
);
8600 gimple_seq_add_stmt (pre_p
, call
);
8602 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
8603 fold_convert_loc (loc
, TREE_TYPE (lhs
),
8606 gimple_seq_add_stmt (pre_p
, cond
);
8607 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
8608 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8609 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
8613 /* A subroutine of lower_omp_single. Expand the simple form of
8614 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8616 #pragma omp single copyprivate (a, b, c)
8618 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8621 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8627 GOMP_single_copy_end (©out);
8638 FIXME. It may be better to delay expanding the logic of this until
8639 pass_expand_omp. The expanded logic may make the job more difficult
8640 to a synchronization analysis pass. */
8643 lower_omp_single_copy (gimple single_stmt
, gimple_seq
*pre_p
, omp_context
*ctx
)
8645 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
8646 gimple_seq copyin_seq
;
8647 location_t loc
= gimple_location (single_stmt
);
8649 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
8651 ptr_type
= build_pointer_type (ctx
->record_type
);
8652 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
8654 l0
= create_artificial_label (loc
);
8655 l1
= create_artificial_label (loc
);
8656 l2
= create_artificial_label (loc
);
8658 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
8659 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
8660 t
= fold_convert_loc (loc
, ptr_type
, t
);
8661 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
8663 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
8664 build_int_cst (ptr_type
, 0));
8665 t
= build3 (COND_EXPR
, void_type_node
, t
,
8666 build_and_jump (&l0
), build_and_jump (&l1
));
8667 gimplify_and_add (t
, pre_p
);
8669 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
8671 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8674 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
8677 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
8678 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
8679 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
8680 gimplify_and_add (t
, pre_p
);
8682 t
= build_and_jump (&l2
);
8683 gimplify_and_add (t
, pre_p
);
8685 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
8687 gimple_seq_add_seq (pre_p
, copyin_seq
);
8689 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
8693 /* Expand code for an OpenMP single directive. */
8696 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8699 gimple t
, bind
, single_stmt
= gsi_stmt (*gsi_p
);
8700 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
8702 push_gimplify_context ();
8704 block
= make_node (BLOCK
);
8705 bind
= gimple_build_bind (NULL
, NULL
, block
);
8706 gsi_replace (gsi_p
, bind
, true);
8709 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
8710 &bind_body
, &dlist
, ctx
, NULL
);
8711 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
8713 gimple_seq_add_stmt (&bind_body
, single_stmt
);
8715 if (ctx
->record_type
)
8716 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
8718 lower_omp_single_simple (single_stmt
, &bind_body
);
8720 gimple_omp_set_body (single_stmt
, NULL
);
8722 gimple_seq_add_seq (&bind_body
, dlist
);
8724 bind_body
= maybe_catch_exception (bind_body
);
8726 t
= gimple_build_omp_return
8727 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
8728 OMP_CLAUSE_NOWAIT
));
8729 gimple_seq_add_stmt (&bind_body_tail
, t
);
8730 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
8731 if (ctx
->record_type
)
8733 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
8734 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
8735 TREE_THIS_VOLATILE (clobber
) = 1;
8736 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
8737 clobber
), GSI_SAME_STMT
);
8739 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
8740 gimple_bind_set_body (bind
, bind_body
);
8742 pop_gimplify_context (bind
);
8744 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8745 BLOCK_VARS (block
) = ctx
->block_vars
;
8746 if (BLOCK_VARS (block
))
8747 TREE_USED (block
) = 1;
8751 /* Expand code for an OpenMP master directive. */
8754 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8756 tree block
, lab
= NULL
, x
, bfn_decl
;
8757 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8758 location_t loc
= gimple_location (stmt
);
8761 push_gimplify_context ();
8763 block
= make_node (BLOCK
);
8764 bind
= gimple_build_bind (NULL
, NULL
, block
);
8765 gsi_replace (gsi_p
, bind
, true);
8766 gimple_bind_add_stmt (bind
, stmt
);
8768 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
8769 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
8770 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
8771 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
8773 gimplify_and_add (x
, &tseq
);
8774 gimple_bind_add_seq (bind
, tseq
);
8776 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8777 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8778 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8779 gimple_omp_set_body (stmt
, NULL
);
8781 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
8783 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8785 pop_gimplify_context (bind
);
8787 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8788 BLOCK_VARS (block
) = ctx
->block_vars
;
8792 /* Expand code for an OpenMP taskgroup directive. */
8795 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8797 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8798 tree block
= make_node (BLOCK
);
8800 bind
= gimple_build_bind (NULL
, NULL
, block
);
8801 gsi_replace (gsi_p
, bind
, true);
8802 gimple_bind_add_stmt (bind
, stmt
);
8804 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
8806 gimple_bind_add_stmt (bind
, x
);
8808 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8809 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8810 gimple_omp_set_body (stmt
, NULL
);
8812 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8814 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8815 BLOCK_VARS (block
) = ctx
->block_vars
;
8819 /* Expand code for an OpenMP ordered directive. */
8822 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8825 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8827 push_gimplify_context ();
8829 block
= make_node (BLOCK
);
8830 bind
= gimple_build_bind (NULL
, NULL
, block
);
8831 gsi_replace (gsi_p
, bind
, true);
8832 gimple_bind_add_stmt (bind
, stmt
);
8834 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
8836 gimple_bind_add_stmt (bind
, x
);
8838 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8839 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8840 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8841 gimple_omp_set_body (stmt
, NULL
);
8843 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
8844 gimple_bind_add_stmt (bind
, x
);
8846 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8848 pop_gimplify_context (bind
);
8850 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8851 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8855 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
8856 substitution of a couple of function calls. But in the NAMED case,
8857 requires that languages coordinate a symbol name. It is therefore
8858 best put here in common code. */
8860 static GTY((param1_is (tree
), param2_is (tree
)))
8861 splay_tree critical_name_mutexes
;
8864 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8867 tree name
, lock
, unlock
;
8868 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8869 location_t loc
= gimple_location (stmt
);
8872 name
= gimple_omp_critical_name (stmt
);
8878 if (!critical_name_mutexes
)
8879 critical_name_mutexes
8880 = splay_tree_new_ggc (splay_tree_compare_pointers
,
8881 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s
,
8882 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s
);
8884 n
= splay_tree_lookup (critical_name_mutexes
, (splay_tree_key
) name
);
8889 decl
= create_tmp_var_raw (ptr_type_node
, NULL
);
8891 new_str
= ACONCAT ((".gomp_critical_user_",
8892 IDENTIFIER_POINTER (name
), NULL
));
8893 DECL_NAME (decl
) = get_identifier (new_str
);
8894 TREE_PUBLIC (decl
) = 1;
8895 TREE_STATIC (decl
) = 1;
8896 DECL_COMMON (decl
) = 1;
8897 DECL_ARTIFICIAL (decl
) = 1;
8898 DECL_IGNORED_P (decl
) = 1;
8899 varpool_finalize_decl (decl
);
8901 splay_tree_insert (critical_name_mutexes
, (splay_tree_key
) name
,
8902 (splay_tree_value
) decl
);
8905 decl
= (tree
) n
->value
;
8907 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
8908 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
8910 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
8911 unlock
= build_call_expr_loc (loc
, unlock
, 1,
8912 build_fold_addr_expr_loc (loc
, decl
));
8916 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
8917 lock
= build_call_expr_loc (loc
, lock
, 0);
8919 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
8920 unlock
= build_call_expr_loc (loc
, unlock
, 0);
8923 push_gimplify_context ();
8925 block
= make_node (BLOCK
);
8926 bind
= gimple_build_bind (NULL
, NULL
, block
);
8927 gsi_replace (gsi_p
, bind
, true);
8928 gimple_bind_add_stmt (bind
, stmt
);
8930 tbody
= gimple_bind_body (bind
);
8931 gimplify_and_add (lock
, &tbody
);
8932 gimple_bind_set_body (bind
, tbody
);
8934 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8935 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8936 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8937 gimple_omp_set_body (stmt
, NULL
);
8939 tbody
= gimple_bind_body (bind
);
8940 gimplify_and_add (unlock
, &tbody
);
8941 gimple_bind_set_body (bind
, tbody
);
8943 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8945 pop_gimplify_context (bind
);
8946 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8947 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8951 /* A subroutine of lower_omp_for. Generate code to emit the predicate
8952 for a lastprivate clause. Given a loop control predicate of (V
8953 cond N2), we gate the clause on (!(V cond N2)). The lowered form
8954 is appended to *DLIST, iterator initialization is appended to
8958 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
8959 gimple_seq
*dlist
, struct omp_context
*ctx
)
8961 tree clauses
, cond
, vinit
;
8962 enum tree_code cond_code
;
8965 cond_code
= fd
->loop
.cond_code
;
8966 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
8968 /* When possible, use a strict equality expression. This can let VRP
8969 type optimizations deduce the value and remove a copy. */
8970 if (tree_fits_shwi_p (fd
->loop
.step
))
8972 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
8973 if (step
== 1 || step
== -1)
8974 cond_code
= EQ_EXPR
;
8977 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
8979 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
8981 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
8982 if (!gimple_seq_empty_p (stmts
))
8984 gimple_seq_add_seq (&stmts
, *dlist
);
8987 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
8988 vinit
= fd
->loop
.n1
;
8989 if (cond_code
== EQ_EXPR
8990 && tree_fits_shwi_p (fd
->loop
.n2
)
8991 && ! integer_zerop (fd
->loop
.n2
))
8992 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
8994 vinit
= unshare_expr (vinit
);
8996 /* Initialize the iterator variable, so that threads that don't execute
8997 any iterations don't execute the lastprivate clauses by accident. */
8998 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
9003 /* Lower code for an OpenMP loop directive. */
9006 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9009 struct omp_for_data fd
, *fdp
= NULL
;
9010 gimple stmt
= gsi_stmt (*gsi_p
), new_stmt
;
9011 gimple_seq omp_for_body
, body
, dlist
;
9014 push_gimplify_context ();
9016 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
9018 block
= make_node (BLOCK
);
9019 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
9020 /* Replace at gsi right away, so that 'stmt' is no member
9021 of a sequence anymore as we're going to add to to a different
9023 gsi_replace (gsi_p
, new_stmt
, true);
9025 /* Move declaration of temporaries in the loop body before we make
9027 omp_for_body
= gimple_omp_body (stmt
);
9028 if (!gimple_seq_empty_p (omp_for_body
)
9029 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
9031 gimple inner_bind
= gimple_seq_first_stmt (omp_for_body
);
9032 tree vars
= gimple_bind_vars (inner_bind
);
9033 gimple_bind_append_vars (new_stmt
, vars
);
9034 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
9035 keep them on the inner_bind and it's block. */
9036 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
9037 if (gimple_bind_block (inner_bind
))
9038 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
9041 if (gimple_omp_for_combined_into_p (stmt
))
9043 extract_omp_for_data (stmt
, &fd
, NULL
);
9046 /* We need two temporaries with fd.loop.v type (istart/iend)
9047 and then (fd.collapse - 1) temporaries with the same
9048 type for count2 ... countN-1 vars if not constant. */
9050 tree type
= fd
.iter_type
;
9052 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
9053 count
+= fd
.collapse
- 1;
9054 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
9055 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
9059 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
9060 OMP_CLAUSE__LOOPTEMP_
);
9061 for (i
= 0; i
< count
; i
++)
9066 gcc_assert (outerc
);
9067 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
9068 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
9069 OMP_CLAUSE__LOOPTEMP_
);
9073 temp
= create_tmp_var (type
, NULL
);
9074 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
9076 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
9077 OMP_CLAUSE_DECL (*pc
) = temp
;
9078 pc
= &OMP_CLAUSE_CHAIN (*pc
);
9083 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
9086 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
9088 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
9090 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9092 /* Lower the header expressions. At this point, we can assume that
9093 the header is of the form:
9095 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
9097 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
9098 using the .omp_data_s mapping, if needed. */
9099 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
9101 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
9102 if (!is_gimple_min_invariant (*rhs_p
))
9103 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9105 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
9106 if (!is_gimple_min_invariant (*rhs_p
))
9107 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9109 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
9110 if (!is_gimple_min_invariant (*rhs_p
))
9111 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9114 /* Once lowered, extract the bounds and clauses. */
9115 extract_omp_for_data (stmt
, &fd
, NULL
);
9117 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
9119 gimple_seq_add_stmt (&body
, stmt
);
9120 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
9122 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
9125 /* After the loop, add exit clauses. */
9126 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
9128 if (ctx
->cancellable
)
9129 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
9131 gimple_seq_add_seq (&body
, dlist
);
9133 body
= maybe_catch_exception (body
);
9135 /* Region exit marker goes at the end of the loop body. */
9136 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
9137 maybe_add_implicit_barrier_cancel (ctx
, &body
);
9138 pop_gimplify_context (new_stmt
);
9140 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
9141 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
9142 if (BLOCK_VARS (block
))
9143 TREE_USED (block
) = 1;
9145 gimple_bind_set_body (new_stmt
, body
);
9146 gimple_omp_set_body (stmt
, NULL
);
9147 gimple_omp_for_set_pre_body (stmt
, NULL
);
9150 /* Callback for walk_stmts. Check if the current statement only contains
9151 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
9154 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
9155 bool *handled_ops_p
,
9156 struct walk_stmt_info
*wi
)
9158 int *info
= (int *) wi
->info
;
9159 gimple stmt
= gsi_stmt (*gsi_p
);
9161 *handled_ops_p
= true;
9162 switch (gimple_code (stmt
))
9166 case GIMPLE_OMP_FOR
:
9167 case GIMPLE_OMP_SECTIONS
:
9168 *info
= *info
== 0 ? 1 : -1;
9177 struct omp_taskcopy_context
9179 /* This field must be at the beginning, as we do "inheritance": Some
9180 callback functions for tree-inline.c (e.g., omp_copy_decl)
9181 receive a copy_body_data pointer that is up-casted to an
9182 omp_context pointer. */
9188 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
9190 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
9192 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
9193 return create_tmp_var (TREE_TYPE (var
), NULL
);
9199 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
9201 tree name
, new_fields
= NULL
, type
, f
;
9203 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
9204 name
= DECL_NAME (TYPE_NAME (orig_type
));
9205 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
9206 TYPE_DECL
, name
, type
);
9207 TYPE_NAME (type
) = name
;
9209 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
9211 tree new_f
= copy_node (f
);
9212 DECL_CONTEXT (new_f
) = type
;
9213 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
9214 TREE_CHAIN (new_f
) = new_fields
;
9215 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9216 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9217 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
9220 *pointer_map_insert (tcctx
->cb
.decl_map
, f
) = new_f
;
9222 TYPE_FIELDS (type
) = nreverse (new_fields
);
9227 /* Create task copyfn. */
9230 create_task_copyfn (gimple task_stmt
, omp_context
*ctx
)
9232 struct function
*child_cfun
;
9233 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
9234 tree record_type
, srecord_type
, bind
, list
;
9235 bool record_needs_remap
= false, srecord_needs_remap
= false;
9237 struct omp_taskcopy_context tcctx
;
9238 location_t loc
= gimple_location (task_stmt
);
9240 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
9241 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
9242 gcc_assert (child_cfun
->cfg
== NULL
);
9243 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
9245 /* Reset DECL_CONTEXT on function arguments. */
9246 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
9247 DECL_CONTEXT (t
) = child_fn
;
9249 /* Populate the function. */
9250 push_gimplify_context ();
9251 push_cfun (child_cfun
);
9253 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
9254 TREE_SIDE_EFFECTS (bind
) = 1;
9256 DECL_SAVED_TREE (child_fn
) = bind
;
9257 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
9259 /* Remap src and dst argument types if needed. */
9260 record_type
= ctx
->record_type
;
9261 srecord_type
= ctx
->srecord_type
;
9262 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
9263 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9265 record_needs_remap
= true;
9268 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
9269 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9271 srecord_needs_remap
= true;
9275 if (record_needs_remap
|| srecord_needs_remap
)
9277 memset (&tcctx
, '\0', sizeof (tcctx
));
9278 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
9279 tcctx
.cb
.dst_fn
= child_fn
;
9280 tcctx
.cb
.src_node
= cgraph_get_node (tcctx
.cb
.src_fn
);
9281 gcc_checking_assert (tcctx
.cb
.src_node
);
9282 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
9283 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
9284 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
9285 tcctx
.cb
.eh_lp_nr
= 0;
9286 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
9287 tcctx
.cb
.decl_map
= pointer_map_create ();
9290 if (record_needs_remap
)
9291 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
9292 if (srecord_needs_remap
)
9293 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
9296 tcctx
.cb
.decl_map
= NULL
;
9298 arg
= DECL_ARGUMENTS (child_fn
);
9299 TREE_TYPE (arg
) = build_pointer_type (record_type
);
9300 sarg
= DECL_CHAIN (arg
);
9301 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
9303 /* First pass: initialize temporaries used in record_type and srecord_type
9304 sizes and field offsets. */
9305 if (tcctx
.cb
.decl_map
)
9306 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9307 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9311 decl
= OMP_CLAUSE_DECL (c
);
9312 p
= (tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, decl
);
9315 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9316 sf
= (tree
) n
->value
;
9317 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9318 src
= build_simple_mem_ref_loc (loc
, sarg
);
9319 src
= omp_build_component_ref (src
, sf
);
9320 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
9321 append_to_statement_list (t
, &list
);
9324 /* Second pass: copy shared var pointers and copy construct non-VLA
9325 firstprivate vars. */
9326 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9327 switch (OMP_CLAUSE_CODE (c
))
9329 case OMP_CLAUSE_SHARED
:
9330 decl
= OMP_CLAUSE_DECL (c
);
9331 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9334 f
= (tree
) n
->value
;
9335 if (tcctx
.cb
.decl_map
)
9336 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9337 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9338 sf
= (tree
) n
->value
;
9339 if (tcctx
.cb
.decl_map
)
9340 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9341 src
= build_simple_mem_ref_loc (loc
, sarg
);
9342 src
= omp_build_component_ref (src
, sf
);
9343 dst
= build_simple_mem_ref_loc (loc
, arg
);
9344 dst
= omp_build_component_ref (dst
, f
);
9345 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9346 append_to_statement_list (t
, &list
);
9348 case OMP_CLAUSE_FIRSTPRIVATE
:
9349 decl
= OMP_CLAUSE_DECL (c
);
9350 if (is_variable_sized (decl
))
9352 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9355 f
= (tree
) n
->value
;
9356 if (tcctx
.cb
.decl_map
)
9357 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9358 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9361 sf
= (tree
) n
->value
;
9362 if (tcctx
.cb
.decl_map
)
9363 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9364 src
= build_simple_mem_ref_loc (loc
, sarg
);
9365 src
= omp_build_component_ref (src
, sf
);
9366 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
9367 src
= build_simple_mem_ref_loc (loc
, src
);
9371 dst
= build_simple_mem_ref_loc (loc
, arg
);
9372 dst
= omp_build_component_ref (dst
, f
);
9373 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9374 append_to_statement_list (t
, &list
);
9376 case OMP_CLAUSE_PRIVATE
:
9377 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
9379 decl
= OMP_CLAUSE_DECL (c
);
9380 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9381 f
= (tree
) n
->value
;
9382 if (tcctx
.cb
.decl_map
)
9383 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9384 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9387 sf
= (tree
) n
->value
;
9388 if (tcctx
.cb
.decl_map
)
9389 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9390 src
= build_simple_mem_ref_loc (loc
, sarg
);
9391 src
= omp_build_component_ref (src
, sf
);
9392 if (use_pointer_for_field (decl
, NULL
))
9393 src
= build_simple_mem_ref_loc (loc
, src
);
9397 dst
= build_simple_mem_ref_loc (loc
, arg
);
9398 dst
= omp_build_component_ref (dst
, f
);
9399 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9400 append_to_statement_list (t
, &list
);
9406 /* Last pass: handle VLA firstprivates. */
9407 if (tcctx
.cb
.decl_map
)
9408 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9409 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9413 decl
= OMP_CLAUSE_DECL (c
);
9414 if (!is_variable_sized (decl
))
9416 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9419 f
= (tree
) n
->value
;
9420 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9421 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
9422 ind
= DECL_VALUE_EXPR (decl
);
9423 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
9424 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
9425 n
= splay_tree_lookup (ctx
->sfield_map
,
9426 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9427 sf
= (tree
) n
->value
;
9428 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9429 src
= build_simple_mem_ref_loc (loc
, sarg
);
9430 src
= omp_build_component_ref (src
, sf
);
9431 src
= build_simple_mem_ref_loc (loc
, src
);
9432 dst
= build_simple_mem_ref_loc (loc
, arg
);
9433 dst
= omp_build_component_ref (dst
, f
);
9434 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9435 append_to_statement_list (t
, &list
);
9436 n
= splay_tree_lookup (ctx
->field_map
,
9437 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9438 df
= (tree
) n
->value
;
9439 df
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, df
);
9440 ptr
= build_simple_mem_ref_loc (loc
, arg
);
9441 ptr
= omp_build_component_ref (ptr
, df
);
9442 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
9443 build_fold_addr_expr_loc (loc
, dst
));
9444 append_to_statement_list (t
, &list
);
9447 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
9448 append_to_statement_list (t
, &list
);
9450 if (tcctx
.cb
.decl_map
)
9451 pointer_map_destroy (tcctx
.cb
.decl_map
);
9452 pop_gimplify_context (NULL
);
9453 BIND_EXPR_BODY (bind
) = list
;
9458 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
9462 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
9464 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
9466 gcc_assert (clauses
);
9467 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9468 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
9469 switch (OMP_CLAUSE_DEPEND_KIND (c
))
9471 case OMP_CLAUSE_DEPEND_IN
:
9474 case OMP_CLAUSE_DEPEND_OUT
:
9475 case OMP_CLAUSE_DEPEND_INOUT
:
9481 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
9482 tree array
= create_tmp_var (type
, NULL
);
9483 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
9485 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
9486 gimple_seq_add_stmt (iseq
, g
);
9487 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
9489 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
9490 gimple_seq_add_stmt (iseq
, g
);
9491 for (i
= 0; i
< 2; i
++)
9493 if ((i
? n_in
: n_out
) == 0)
9495 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9496 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
9497 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
9499 tree t
= OMP_CLAUSE_DECL (c
);
9500 t
= fold_convert (ptr_type_node
, t
);
9501 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
9502 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
9503 NULL_TREE
, NULL_TREE
);
9504 g
= gimple_build_assign (r
, t
);
9505 gimple_seq_add_stmt (iseq
, g
);
9508 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
9509 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
9510 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
9511 OMP_CLAUSE_CHAIN (c
) = *p
;
9513 tree clobber
= build_constructor (type
, NULL
);
9514 TREE_THIS_VOLATILE (clobber
) = 1;
9515 g
= gimple_build_assign (array
, clobber
);
9516 gimple_seq_add_stmt (oseq
, g
);
9519 /* Lower the OpenMP parallel or task directive in the current statement
9520 in GSI_P. CTX holds context information for the directive. */
9523 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9527 gimple stmt
= gsi_stmt (*gsi_p
);
9528 gimple par_bind
, bind
, dep_bind
= NULL
;
9529 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
9530 location_t loc
= gimple_location (stmt
);
9532 clauses
= gimple_omp_taskreg_clauses (stmt
);
9533 par_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9534 par_body
= gimple_bind_body (par_bind
);
9535 child_fn
= ctx
->cb
.dst_fn
;
9536 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
9537 && !gimple_omp_parallel_combined_p (stmt
))
9539 struct walk_stmt_info wi
;
9542 memset (&wi
, 0, sizeof (wi
));
9545 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
9547 gimple_omp_parallel_set_combined_p (stmt
, true);
9549 gimple_seq dep_ilist
= NULL
;
9550 gimple_seq dep_olist
= NULL
;
9551 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
9552 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
9554 push_gimplify_context ();
9555 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
9556 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
9559 if (ctx
->srecord_type
)
9560 create_task_copyfn (stmt
, ctx
);
9562 push_gimplify_context ();
9567 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
9568 lower_omp (&par_body
, ctx
);
9569 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
9570 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
9572 /* Declare all the variables created by mapping and the variables
9573 declared in the scope of the parallel body. */
9574 record_vars_into (ctx
->block_vars
, child_fn
);
9575 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
9577 if (ctx
->record_type
)
9580 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
9581 : ctx
->record_type
, ".omp_data_o");
9582 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9583 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9584 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
9589 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
9590 lower_send_shared_vars (&ilist
, &olist
, ctx
);
9592 if (ctx
->record_type
)
9594 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
9595 TREE_THIS_VOLATILE (clobber
) = 1;
9596 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9600 /* Once all the expansions are done, sequence all the different
9601 fragments inside gimple_omp_body. */
9605 if (ctx
->record_type
)
9607 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9608 /* fixup_child_record_type might have changed receiver_decl's type. */
9609 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9610 gimple_seq_add_stmt (&new_body
,
9611 gimple_build_assign (ctx
->receiver_decl
, t
));
9614 gimple_seq_add_seq (&new_body
, par_ilist
);
9615 gimple_seq_add_seq (&new_body
, par_body
);
9616 gimple_seq_add_seq (&new_body
, par_rlist
);
9617 if (ctx
->cancellable
)
9618 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
9619 gimple_seq_add_seq (&new_body
, par_olist
);
9620 new_body
= maybe_catch_exception (new_body
);
9621 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9622 gimple_omp_set_body (stmt
, new_body
);
9624 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
9625 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
9626 gimple_bind_add_seq (bind
, ilist
);
9627 gimple_bind_add_stmt (bind
, stmt
);
9628 gimple_bind_add_seq (bind
, olist
);
9630 pop_gimplify_context (NULL
);
9634 gimple_bind_add_seq (dep_bind
, dep_ilist
);
9635 gimple_bind_add_stmt (dep_bind
, bind
);
9636 gimple_bind_add_seq (dep_bind
, dep_olist
);
9637 pop_gimplify_context (dep_bind
);
9641 /* Lower the OpenMP target directive in the current statement
9642 in GSI_P. CTX holds context information for the directive. */
9645 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9648 tree child_fn
, t
, c
;
9649 gimple stmt
= gsi_stmt (*gsi_p
);
9650 gimple tgt_bind
= NULL
, bind
;
9651 gimple_seq tgt_body
= NULL
, olist
, ilist
, new_body
;
9652 location_t loc
= gimple_location (stmt
);
9653 int kind
= gimple_omp_target_kind (stmt
);
9654 unsigned int map_cnt
= 0;
9656 clauses
= gimple_omp_target_clauses (stmt
);
9657 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9659 tgt_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9660 tgt_body
= gimple_bind_body (tgt_bind
);
9662 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9663 tgt_body
= gimple_omp_body (stmt
);
9664 child_fn
= ctx
->cb
.dst_fn
;
9666 push_gimplify_context ();
9668 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9669 switch (OMP_CLAUSE_CODE (c
))
9675 case OMP_CLAUSE_MAP
:
9677 case OMP_CLAUSE_FROM
:
9678 var
= OMP_CLAUSE_DECL (c
);
9681 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
9682 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9688 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
9690 tree var2
= DECL_VALUE_EXPR (var
);
9691 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
9692 var2
= TREE_OPERAND (var2
, 0);
9693 gcc_assert (DECL_P (var2
));
9697 if (!maybe_lookup_field (var
, ctx
))
9700 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9702 x
= build_receiver_ref (var
, true, ctx
);
9703 tree new_var
= lookup_decl (var
, ctx
);
9704 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9705 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9706 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9707 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
9708 x
= build_simple_mem_ref (x
);
9709 SET_DECL_VALUE_EXPR (new_var
, x
);
9710 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
9715 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9717 target_nesting_level
++;
9718 lower_omp (&tgt_body
, ctx
);
9719 target_nesting_level
--;
9721 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9722 lower_omp (&tgt_body
, ctx
);
9724 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9726 /* Declare all the variables created by mapping and the variables
9727 declared in the scope of the target body. */
9728 record_vars_into (ctx
->block_vars
, child_fn
);
9729 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
9734 if (ctx
->record_type
)
9737 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
9738 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9739 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9740 t
= make_tree_vec (3);
9741 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
9743 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
9745 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
9746 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
9747 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
9749 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node
,
9752 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
9753 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
9754 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
9755 gimple_omp_target_set_data_arg (stmt
, t
);
9757 vec
<constructor_elt
, va_gc
> *vsize
;
9758 vec
<constructor_elt
, va_gc
> *vkind
;
9759 vec_alloc (vsize
, map_cnt
);
9760 vec_alloc (vkind
, map_cnt
);
9761 unsigned int map_idx
= 0;
9763 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9764 switch (OMP_CLAUSE_CODE (c
))
9770 case OMP_CLAUSE_MAP
:
9772 case OMP_CLAUSE_FROM
:
9774 ovar
= OMP_CLAUSE_DECL (c
);
9777 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9778 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9780 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
9781 == get_base_address (ovar
));
9782 nc
= OMP_CLAUSE_CHAIN (c
);
9783 ovar
= OMP_CLAUSE_DECL (nc
);
9787 tree x
= build_sender_ref (ovar
, ctx
);
9789 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
9790 gimplify_assign (x
, v
, &ilist
);
9796 if (DECL_SIZE (ovar
)
9797 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
9799 tree ovar2
= DECL_VALUE_EXPR (ovar
);
9800 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
9801 ovar2
= TREE_OPERAND (ovar2
, 0);
9802 gcc_assert (DECL_P (ovar2
));
9805 if (!maybe_lookup_field (ovar
, ctx
))
9811 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
9812 tree x
= build_sender_ref (ovar
, ctx
);
9813 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9814 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9815 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9816 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
9818 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9820 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)), NULL
);
9821 mark_addressable (avar
);
9822 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
9823 avar
= build_fold_addr_expr (avar
);
9824 gimplify_assign (x
, avar
, &ilist
);
9826 else if (is_gimple_reg (var
))
9828 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9829 tree avar
= create_tmp_var (TREE_TYPE (var
), NULL
);
9830 mark_addressable (avar
);
9831 if (OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_ALLOC
9832 && OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_FROM
)
9833 gimplify_assign (avar
, var
, &ilist
);
9834 avar
= build_fold_addr_expr (avar
);
9835 gimplify_assign (x
, avar
, &ilist
);
9836 if ((OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_FROM
9837 || OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_TOFROM
)
9838 && !TYPE_READONLY (TREE_TYPE (var
)))
9840 x
= build_sender_ref (ovar
, ctx
);
9841 x
= build_simple_mem_ref (x
);
9842 gimplify_assign (var
, x
, &olist
);
9847 var
= build_fold_addr_expr (var
);
9848 gimplify_assign (x
, var
, &ilist
);
9851 tree s
= OMP_CLAUSE_SIZE (c
);
9853 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
9854 s
= fold_convert (size_type_node
, s
);
9855 tree purpose
= size_int (map_idx
++);
9856 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
9857 if (TREE_CODE (s
) != INTEGER_CST
)
9858 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
9860 unsigned char tkind
= 0;
9861 switch (OMP_CLAUSE_CODE (c
))
9863 case OMP_CLAUSE_MAP
:
9864 tkind
= OMP_CLAUSE_MAP_KIND (c
);
9867 tkind
= OMP_CLAUSE_MAP_TO
;
9869 case OMP_CLAUSE_FROM
:
9870 tkind
= OMP_CLAUSE_MAP_FROM
;
9875 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
9876 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
9877 talign
= DECL_ALIGN_UNIT (ovar
);
9878 talign
= ceil_log2 (talign
);
9879 tkind
|= talign
<< 3;
9880 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
9881 build_int_cst (unsigned_char_type_node
,
9887 gcc_assert (map_idx
== map_cnt
);
9889 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
9890 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
9891 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
9892 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
9893 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
9895 gimple_seq initlist
= NULL
;
9896 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
9897 TREE_VEC_ELT (t
, 1)),
9898 &initlist
, true, NULL_TREE
);
9899 gimple_seq_add_seq (&ilist
, initlist
);
9901 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
9903 TREE_THIS_VOLATILE (clobber
) = 1;
9904 gimple_seq_add_stmt (&olist
,
9905 gimple_build_assign (TREE_VEC_ELT (t
, 1),
9909 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
9910 TREE_THIS_VOLATILE (clobber
) = 1;
9911 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9915 /* Once all the expansions are done, sequence all the different
9916 fragments inside gimple_omp_body. */
9920 if (ctx
->record_type
&& kind
== GF_OMP_TARGET_KIND_REGION
)
9922 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9923 /* fixup_child_record_type might have changed receiver_decl's type. */
9924 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9925 gimple_seq_add_stmt (&new_body
,
9926 gimple_build_assign (ctx
->receiver_decl
, t
));
9929 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9931 gimple_seq_add_seq (&new_body
, tgt_body
);
9932 new_body
= maybe_catch_exception (new_body
);
9934 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9935 new_body
= tgt_body
;
9936 if (kind
!= GF_OMP_TARGET_KIND_UPDATE
)
9938 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9939 gimple_omp_set_body (stmt
, new_body
);
9942 bind
= gimple_build_bind (NULL
, NULL
,
9943 tgt_bind
? gimple_bind_block (tgt_bind
)
9945 gsi_replace (gsi_p
, bind
, true);
9946 gimple_bind_add_seq (bind
, ilist
);
9947 gimple_bind_add_stmt (bind
, stmt
);
9948 gimple_bind_add_seq (bind
, olist
);
9950 pop_gimplify_context (NULL
);
9953 /* Expand code for an OpenMP teams directive. */
9956 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9958 gimple teams_stmt
= gsi_stmt (*gsi_p
);
9959 push_gimplify_context ();
9961 tree block
= make_node (BLOCK
);
9962 gimple bind
= gimple_build_bind (NULL
, NULL
, block
);
9963 gsi_replace (gsi_p
, bind
, true);
9964 gimple_seq bind_body
= NULL
;
9965 gimple_seq dlist
= NULL
;
9966 gimple_seq olist
= NULL
;
9968 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
9969 OMP_CLAUSE_NUM_TEAMS
);
9970 if (num_teams
== NULL_TREE
)
9971 num_teams
= build_int_cst (unsigned_type_node
, 0);
9974 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
9975 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
9976 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
9978 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
9979 OMP_CLAUSE_THREAD_LIMIT
);
9980 if (thread_limit
== NULL_TREE
)
9981 thread_limit
= build_int_cst (unsigned_type_node
, 0);
9984 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
9985 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
9986 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
9990 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
9991 &bind_body
, &dlist
, ctx
, NULL
);
9992 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
9993 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
9994 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
9996 location_t loc
= gimple_location (teams_stmt
);
9997 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
9998 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
9999 gimple_set_location (call
, loc
);
10000 gimple_seq_add_stmt (&bind_body
, call
);
10002 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
10003 gimple_omp_set_body (teams_stmt
, NULL
);
10004 gimple_seq_add_seq (&bind_body
, olist
);
10005 gimple_seq_add_seq (&bind_body
, dlist
);
10006 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
10007 gimple_bind_set_body (bind
, bind_body
);
10009 pop_gimplify_context (bind
);
10011 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10012 BLOCK_VARS (block
) = ctx
->block_vars
;
10013 if (BLOCK_VARS (block
))
10014 TREE_USED (block
) = 1;
10018 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
10019 regimplified. If DATA is non-NULL, lower_omp_1 is outside
10020 of OpenMP context, but with task_shared_vars set. */
10023 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
10028 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
10029 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
10032 if (task_shared_vars
10034 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
10037 /* If a global variable has been privatized, TREE_CONSTANT on
10038 ADDR_EXPR might be wrong. */
10039 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
10040 recompute_tree_invariant_for_addr_expr (t
);
10042 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
10047 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10049 gimple stmt
= gsi_stmt (*gsi_p
);
10050 struct walk_stmt_info wi
;
10052 if (gimple_has_location (stmt
))
10053 input_location
= gimple_location (stmt
);
10055 if (task_shared_vars
)
10056 memset (&wi
, '\0', sizeof (wi
));
10058 /* If we have issued syntax errors, avoid doing any heavy lifting.
10059 Just replace the OpenMP directives with a NOP to avoid
10060 confusing RTL expansion. */
10061 if (seen_error () && is_gimple_omp (stmt
))
10063 gsi_replace (gsi_p
, gimple_build_nop (), true);
10067 switch (gimple_code (stmt
))
10070 if ((ctx
|| task_shared_vars
)
10071 && (walk_tree (gimple_cond_lhs_ptr (stmt
), lower_omp_regimplify_p
,
10072 ctx
? NULL
: &wi
, NULL
)
10073 || walk_tree (gimple_cond_rhs_ptr (stmt
), lower_omp_regimplify_p
,
10074 ctx
? NULL
: &wi
, NULL
)))
10075 gimple_regimplify_operands (stmt
, gsi_p
);
10078 lower_omp (gimple_catch_handler_ptr (stmt
), ctx
);
10080 case GIMPLE_EH_FILTER
:
10081 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
10084 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
10085 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
10087 case GIMPLE_TRANSACTION
:
10088 lower_omp (gimple_transaction_body_ptr (stmt
), ctx
);
10091 lower_omp (gimple_bind_body_ptr (stmt
), ctx
);
10093 case GIMPLE_OMP_PARALLEL
:
10094 case GIMPLE_OMP_TASK
:
10095 ctx
= maybe_lookup_ctx (stmt
);
10097 if (ctx
->cancellable
)
10098 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10099 lower_omp_taskreg (gsi_p
, ctx
);
10101 case GIMPLE_OMP_FOR
:
10102 ctx
= maybe_lookup_ctx (stmt
);
10104 if (ctx
->cancellable
)
10105 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10106 lower_omp_for (gsi_p
, ctx
);
10108 case GIMPLE_OMP_SECTIONS
:
10109 ctx
= maybe_lookup_ctx (stmt
);
10111 if (ctx
->cancellable
)
10112 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10113 lower_omp_sections (gsi_p
, ctx
);
10115 case GIMPLE_OMP_SINGLE
:
10116 ctx
= maybe_lookup_ctx (stmt
);
10118 lower_omp_single (gsi_p
, ctx
);
10120 case GIMPLE_OMP_MASTER
:
10121 ctx
= maybe_lookup_ctx (stmt
);
10123 lower_omp_master (gsi_p
, ctx
);
10125 case GIMPLE_OMP_TASKGROUP
:
10126 ctx
= maybe_lookup_ctx (stmt
);
10128 lower_omp_taskgroup (gsi_p
, ctx
);
10130 case GIMPLE_OMP_ORDERED
:
10131 ctx
= maybe_lookup_ctx (stmt
);
10133 lower_omp_ordered (gsi_p
, ctx
);
10135 case GIMPLE_OMP_CRITICAL
:
10136 ctx
= maybe_lookup_ctx (stmt
);
10138 lower_omp_critical (gsi_p
, ctx
);
10140 case GIMPLE_OMP_ATOMIC_LOAD
:
10141 if ((ctx
|| task_shared_vars
)
10142 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt
),
10143 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
10144 gimple_regimplify_operands (stmt
, gsi_p
);
10146 case GIMPLE_OMP_TARGET
:
10147 ctx
= maybe_lookup_ctx (stmt
);
10149 lower_omp_target (gsi_p
, ctx
);
10151 case GIMPLE_OMP_TEAMS
:
10152 ctx
= maybe_lookup_ctx (stmt
);
10154 lower_omp_teams (gsi_p
, ctx
);
10158 fndecl
= gimple_call_fndecl (stmt
);
10160 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
10161 switch (DECL_FUNCTION_CODE (fndecl
))
10163 case BUILT_IN_GOMP_BARRIER
:
10167 case BUILT_IN_GOMP_CANCEL
:
10168 case BUILT_IN_GOMP_CANCELLATION_POINT
:
10171 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
10172 cctx
= cctx
->outer
;
10173 gcc_assert (gimple_call_lhs (stmt
) == NULL_TREE
);
10174 if (!cctx
->cancellable
)
10176 if (DECL_FUNCTION_CODE (fndecl
)
10177 == BUILT_IN_GOMP_CANCELLATION_POINT
)
10179 stmt
= gimple_build_nop ();
10180 gsi_replace (gsi_p
, stmt
, false);
10184 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
10186 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
10187 gimple_call_set_fndecl (stmt
, fndecl
);
10188 gimple_call_set_fntype (stmt
, TREE_TYPE (fndecl
));
10191 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)), NULL
);
10192 gimple_call_set_lhs (stmt
, lhs
);
10193 tree fallthru_label
;
10194 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
10196 g
= gimple_build_label (fallthru_label
);
10197 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10198 g
= gimple_build_cond (NE_EXPR
, lhs
,
10199 fold_convert (TREE_TYPE (lhs
),
10200 boolean_false_node
),
10201 cctx
->cancel_label
, fallthru_label
);
10202 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10209 if ((ctx
|| task_shared_vars
)
10210 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
10213 /* Just remove clobbers, this should happen only if we have
10214 "privatized" local addressable variables in SIMD regions,
10215 the clobber isn't needed in that case and gimplifying address
10216 of the ARRAY_REF into a pointer and creating MEM_REF based
10217 clobber would create worse code than we get with the clobber
10219 if (gimple_clobber_p (stmt
))
10221 gsi_replace (gsi_p
, gimple_build_nop (), true);
10224 gimple_regimplify_operands (stmt
, gsi_p
);
10231 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
10233 location_t saved_location
= input_location
;
10234 gimple_stmt_iterator gsi
;
10235 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10236 lower_omp_1 (&gsi
, ctx
);
10237 /* During gimplification, we have not always invoked fold_stmt
10238 (gimplify.c:maybe_fold_stmt); call it now. */
10239 if (target_nesting_level
)
10240 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10242 input_location
= saved_location
;
10245 /* Main entry point. */
10247 static unsigned int
10248 execute_lower_omp (void)
10252 /* This pass always runs, to provide PROP_gimple_lomp.
10253 But there is nothing to do unless -fopenmp is given. */
10254 if (flag_openmp
== 0 && flag_openmp_simd
== 0 && flag_cilkplus
== 0)
10257 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
10258 delete_omp_context
);
10260 body
= gimple_body (current_function_decl
);
10261 scan_omp (&body
, NULL
);
10262 gcc_assert (taskreg_nesting_level
== 0);
10264 if (all_contexts
->root
)
10266 if (task_shared_vars
)
10267 push_gimplify_context ();
10268 lower_omp (&body
, NULL
);
10269 if (task_shared_vars
)
10270 pop_gimplify_context (NULL
);
10275 splay_tree_delete (all_contexts
);
10276 all_contexts
= NULL
;
10278 BITMAP_FREE (task_shared_vars
);
10284 const pass_data pass_data_lower_omp
=
10286 GIMPLE_PASS
, /* type */
10287 "omplower", /* name */
10288 OPTGROUP_NONE
, /* optinfo_flags */
10289 true, /* has_execute */
10290 TV_NONE
, /* tv_id */
10291 PROP_gimple_any
, /* properties_required */
10292 PROP_gimple_lomp
, /* properties_provided */
10293 0, /* properties_destroyed */
10294 0, /* todo_flags_start */
10295 0, /* todo_flags_finish */
10298 class pass_lower_omp
: public gimple_opt_pass
10301 pass_lower_omp (gcc::context
*ctxt
)
10302 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
10305 /* opt_pass methods: */
10306 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
10308 }; // class pass_lower_omp
10310 } // anon namespace
10313 make_pass_lower_omp (gcc::context
*ctxt
)
10315 return new pass_lower_omp (ctxt
);
10318 /* The following is a utility to diagnose OpenMP structured block violations.
10319 It is not part of the "omplower" pass, as that's invoked too late. It
10320 should be invoked by the respective front ends after gimplification. */
10322 static splay_tree all_labels
;
10324 /* Check for mismatched contexts and generate an error if needed. Return
10325 true if an error is detected. */
10328 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
10329 gimple branch_ctx
, gimple label_ctx
)
10331 if (label_ctx
== branch_ctx
)
10336 Previously we kept track of the label's entire context in diagnose_sb_[12]
10337 so we could traverse it and issue a correct "exit" or "enter" error
10338 message upon a structured block violation.
10340 We built the context by building a list with tree_cons'ing, but there is
10341 no easy counterpart in gimple tuples. It seems like far too much work
10342 for issuing exit/enter error messages. If someone really misses the
10343 distinct error message... patches welcome.
10347 /* Try to avoid confusing the user by producing and error message
10348 with correct "exit" or "enter" verbiage. We prefer "exit"
10349 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10350 if (branch_ctx
== NULL
)
10356 if (TREE_VALUE (label_ctx
) == branch_ctx
)
10361 label_ctx
= TREE_CHAIN (label_ctx
);
10366 error ("invalid exit from OpenMP structured block");
10368 error ("invalid entry to OpenMP structured block");
10371 bool cilkplus_block
= false;
10375 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
10376 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
10378 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
10379 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
10380 cilkplus_block
= true;
10383 /* If it's obvious we have an invalid entry, be specific about the error. */
10384 if (branch_ctx
== NULL
)
10386 if (cilkplus_block
)
10387 error ("invalid entry to Cilk Plus structured block");
10389 error ("invalid entry to OpenMP structured block");
10393 /* Otherwise, be vague and lazy, but efficient. */
10394 if (cilkplus_block
)
10395 error ("invalid branch to/from a Cilk Plus structured block");
10397 error ("invalid branch to/from an OpenMP structured block");
10400 gsi_replace (gsi_p
, gimple_build_nop (), false);
10404 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10405 where each label is found. */
10408 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10409 struct walk_stmt_info
*wi
)
10411 gimple context
= (gimple
) wi
->info
;
10412 gimple inner_context
;
10413 gimple stmt
= gsi_stmt (*gsi_p
);
10415 *handled_ops_p
= true;
10417 switch (gimple_code (stmt
))
10421 case GIMPLE_OMP_PARALLEL
:
10422 case GIMPLE_OMP_TASK
:
10423 case GIMPLE_OMP_SECTIONS
:
10424 case GIMPLE_OMP_SINGLE
:
10425 case GIMPLE_OMP_SECTION
:
10426 case GIMPLE_OMP_MASTER
:
10427 case GIMPLE_OMP_ORDERED
:
10428 case GIMPLE_OMP_CRITICAL
:
10429 case GIMPLE_OMP_TARGET
:
10430 case GIMPLE_OMP_TEAMS
:
10431 case GIMPLE_OMP_TASKGROUP
:
10432 /* The minimal context here is just the current OMP construct. */
10433 inner_context
= stmt
;
10434 wi
->info
= inner_context
;
10435 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10436 wi
->info
= context
;
10439 case GIMPLE_OMP_FOR
:
10440 inner_context
= stmt
;
10441 wi
->info
= inner_context
;
10442 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10444 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
10445 diagnose_sb_1
, NULL
, wi
);
10446 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10447 wi
->info
= context
;
10451 splay_tree_insert (all_labels
, (splay_tree_key
) gimple_label_label (stmt
),
10452 (splay_tree_value
) context
);
10462 /* Pass 2: Check each branch and see if its context differs from that of
10463 the destination label's context. */
10466 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10467 struct walk_stmt_info
*wi
)
10469 gimple context
= (gimple
) wi
->info
;
10471 gimple stmt
= gsi_stmt (*gsi_p
);
10473 *handled_ops_p
= true;
10475 switch (gimple_code (stmt
))
10479 case GIMPLE_OMP_PARALLEL
:
10480 case GIMPLE_OMP_TASK
:
10481 case GIMPLE_OMP_SECTIONS
:
10482 case GIMPLE_OMP_SINGLE
:
10483 case GIMPLE_OMP_SECTION
:
10484 case GIMPLE_OMP_MASTER
:
10485 case GIMPLE_OMP_ORDERED
:
10486 case GIMPLE_OMP_CRITICAL
:
10487 case GIMPLE_OMP_TARGET
:
10488 case GIMPLE_OMP_TEAMS
:
10489 case GIMPLE_OMP_TASKGROUP
:
10491 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10492 wi
->info
= context
;
10495 case GIMPLE_OMP_FOR
:
10497 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10499 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
10500 diagnose_sb_2
, NULL
, wi
);
10501 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10502 wi
->info
= context
;
10507 tree lab
= gimple_cond_true_label (stmt
);
10510 n
= splay_tree_lookup (all_labels
,
10511 (splay_tree_key
) lab
);
10512 diagnose_sb_0 (gsi_p
, context
,
10513 n
? (gimple
) n
->value
: NULL
);
10515 lab
= gimple_cond_false_label (stmt
);
10518 n
= splay_tree_lookup (all_labels
,
10519 (splay_tree_key
) lab
);
10520 diagnose_sb_0 (gsi_p
, context
,
10521 n
? (gimple
) n
->value
: NULL
);
10528 tree lab
= gimple_goto_dest (stmt
);
10529 if (TREE_CODE (lab
) != LABEL_DECL
)
10532 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10533 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
10537 case GIMPLE_SWITCH
:
10540 for (i
= 0; i
< gimple_switch_num_labels (stmt
); ++i
)
10542 tree lab
= CASE_LABEL (gimple_switch_label (stmt
, i
));
10543 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10544 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
10550 case GIMPLE_RETURN
:
10551 diagnose_sb_0 (gsi_p
, context
, NULL
);
10561 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
10564 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
10567 gimple last
= last_stmt (bb
);
10568 enum gimple_code code
= gimple_code (last
);
10569 struct omp_region
*cur_region
= *region
;
10570 bool fallthru
= false;
10574 case GIMPLE_OMP_PARALLEL
:
10575 case GIMPLE_OMP_TASK
:
10576 case GIMPLE_OMP_FOR
:
10577 case GIMPLE_OMP_SINGLE
:
10578 case GIMPLE_OMP_TEAMS
:
10579 case GIMPLE_OMP_MASTER
:
10580 case GIMPLE_OMP_TASKGROUP
:
10581 case GIMPLE_OMP_ORDERED
:
10582 case GIMPLE_OMP_CRITICAL
:
10583 case GIMPLE_OMP_SECTION
:
10584 cur_region
= new_omp_region (bb
, code
, cur_region
);
10588 case GIMPLE_OMP_TARGET
:
10589 cur_region
= new_omp_region (bb
, code
, cur_region
);
10591 if (gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_UPDATE
)
10592 cur_region
= cur_region
->outer
;
10595 case GIMPLE_OMP_SECTIONS
:
10596 cur_region
= new_omp_region (bb
, code
, cur_region
);
10600 case GIMPLE_OMP_SECTIONS_SWITCH
:
10604 case GIMPLE_OMP_ATOMIC_LOAD
:
10605 case GIMPLE_OMP_ATOMIC_STORE
:
10609 case GIMPLE_OMP_RETURN
:
10610 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
10611 somewhere other than the next block. This will be
10613 cur_region
->exit
= bb
;
10614 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
10615 cur_region
= cur_region
->outer
;
10618 case GIMPLE_OMP_CONTINUE
:
10619 cur_region
->cont
= bb
;
10620 switch (cur_region
->type
)
10622 case GIMPLE_OMP_FOR
:
10623 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
10624 succs edges as abnormal to prevent splitting
10626 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
10627 /* Make the loopback edge. */
10628 make_edge (bb
, single_succ (cur_region
->entry
),
10631 /* Create an edge from GIMPLE_OMP_FOR to exit, which
10632 corresponds to the case that the body of the loop
10633 is not executed at all. */
10634 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
10635 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
10639 case GIMPLE_OMP_SECTIONS
:
10640 /* Wire up the edges into and out of the nested sections. */
10642 basic_block switch_bb
= single_succ (cur_region
->entry
);
10644 struct omp_region
*i
;
10645 for (i
= cur_region
->inner
; i
; i
= i
->next
)
10647 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
10648 make_edge (switch_bb
, i
->entry
, 0);
10649 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
10652 /* Make the loopback edge to the block with
10653 GIMPLE_OMP_SECTIONS_SWITCH. */
10654 make_edge (bb
, switch_bb
, 0);
10656 /* Make the edge from the switch to exit. */
10657 make_edge (switch_bb
, bb
->next_bb
, 0);
10663 gcc_unreachable ();
10668 gcc_unreachable ();
10671 if (*region
!= cur_region
)
10673 *region
= cur_region
;
10675 *region_idx
= cur_region
->entry
->index
;
10683 static unsigned int
10684 diagnose_omp_structured_block_errors (void)
10686 struct walk_stmt_info wi
;
10687 gimple_seq body
= gimple_body (current_function_decl
);
10689 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
10691 memset (&wi
, 0, sizeof (wi
));
10692 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
10694 memset (&wi
, 0, sizeof (wi
));
10695 wi
.want_locations
= true;
10696 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
10698 gimple_set_body (current_function_decl
, body
);
10700 splay_tree_delete (all_labels
);
10708 const pass_data pass_data_diagnose_omp_blocks
=
10710 GIMPLE_PASS
, /* type */
10711 "*diagnose_omp_blocks", /* name */
10712 OPTGROUP_NONE
, /* optinfo_flags */
10713 true, /* has_execute */
10714 TV_NONE
, /* tv_id */
10715 PROP_gimple_any
, /* properties_required */
10716 0, /* properties_provided */
10717 0, /* properties_destroyed */
10718 0, /* todo_flags_start */
10719 0, /* todo_flags_finish */
10722 class pass_diagnose_omp_blocks
: public gimple_opt_pass
10725 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10726 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
10729 /* opt_pass methods: */
10730 virtual bool gate (function
*) { return flag_openmp
|| flag_cilkplus
; }
10731 virtual unsigned int execute (function
*)
10733 return diagnose_omp_structured_block_errors ();
10736 }; // class pass_diagnose_omp_blocks
10738 } // anon namespace
10741 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10743 return new pass_diagnose_omp_blocks (ctxt
);
10746 /* SIMD clone supporting code. */
10748 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
10749 of arguments to reserve space for. */
10751 static struct cgraph_simd_clone
*
10752 simd_clone_struct_alloc (int nargs
)
10754 struct cgraph_simd_clone
*clone_info
;
10755 size_t len
= (sizeof (struct cgraph_simd_clone
)
10756 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
10757 clone_info
= (struct cgraph_simd_clone
*)
10758 ggc_internal_cleared_alloc (len
);
10762 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
10765 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
10766 struct cgraph_simd_clone
*from
)
10768 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
10769 + ((from
->nargs
- from
->inbranch
)
10770 * sizeof (struct cgraph_simd_clone_arg
))));
10773 /* Return vector of parameter types of function FNDECL. This uses
10774 TYPE_ARG_TYPES if available, otherwise falls back to types of
10775 DECL_ARGUMENTS types. */
10778 simd_clone_vector_of_formal_parm_types (tree fndecl
)
10780 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
10781 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
10782 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
10785 FOR_EACH_VEC_ELT (args
, i
, arg
)
10786 args
[i
] = TREE_TYPE (args
[i
]);
10790 /* Given a simd function in NODE, extract the simd specific
10791 information from the OMP clauses passed in CLAUSES, and return
10792 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
10793 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
10794 otherwise set to FALSE. */
10796 static struct cgraph_simd_clone
*
10797 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
10798 bool *inbranch_specified
)
10800 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
10803 *inbranch_specified
= false;
10805 n
= args
.length ();
10806 if (n
> 0 && args
.last () == void_type_node
)
10809 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
10810 be cloned have a distinctive artificial label in addition to "omp
10814 && lookup_attribute ("cilk simd function",
10815 DECL_ATTRIBUTES (node
->decl
)));
10817 /* Allocate one more than needed just in case this is an in-branch
10818 clone which will require a mask argument. */
10819 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
10820 clone_info
->nargs
= n
;
10821 clone_info
->cilk_elemental
= cilk_clone
;
10828 clauses
= TREE_VALUE (clauses
);
10829 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
10832 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
10834 switch (OMP_CLAUSE_CODE (t
))
10836 case OMP_CLAUSE_INBRANCH
:
10837 clone_info
->inbranch
= 1;
10838 *inbranch_specified
= true;
10840 case OMP_CLAUSE_NOTINBRANCH
:
10841 clone_info
->inbranch
= 0;
10842 *inbranch_specified
= true;
10844 case OMP_CLAUSE_SIMDLEN
:
10845 clone_info
->simdlen
10846 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
10848 case OMP_CLAUSE_LINEAR
:
10850 tree decl
= OMP_CLAUSE_DECL (t
);
10851 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
10852 int argno
= TREE_INT_CST_LOW (decl
);
10853 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
10855 clone_info
->args
[argno
].arg_type
10856 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
10857 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
10858 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
10859 && clone_info
->args
[argno
].linear_step
< n
);
10863 if (POINTER_TYPE_P (args
[argno
]))
10864 step
= fold_convert (ssizetype
, step
);
10865 if (!tree_fits_shwi_p (step
))
10867 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
10868 "ignoring large linear step");
10872 else if (integer_zerop (step
))
10874 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
10875 "ignoring zero linear step");
10881 clone_info
->args
[argno
].arg_type
10882 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
10883 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
10888 case OMP_CLAUSE_UNIFORM
:
10890 tree decl
= OMP_CLAUSE_DECL (t
);
10891 int argno
= tree_to_uhwi (decl
);
10892 clone_info
->args
[argno
].arg_type
10893 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
10896 case OMP_CLAUSE_ALIGNED
:
10898 tree decl
= OMP_CLAUSE_DECL (t
);
10899 int argno
= tree_to_uhwi (decl
);
10900 clone_info
->args
[argno
].alignment
10901 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
10912 /* Given a SIMD clone in NODE, calculate the characteristic data
10913 type and return the coresponding type. The characteristic data
10914 type is computed as described in the Intel Vector ABI. */
10917 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
10918 struct cgraph_simd_clone
*clone_info
)
10920 tree type
= integer_type_node
;
10921 tree fndecl
= node
->decl
;
10923 /* a) For non-void function, the characteristic data type is the
10925 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
10926 type
= TREE_TYPE (TREE_TYPE (fndecl
));
10928 /* b) If the function has any non-uniform, non-linear parameters,
10929 then the characteristic data type is the type of the first
10933 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
10934 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
10935 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
10943 /* c) If the characteristic data type determined by a) or b) above
10944 is struct, union, or class type which is pass-by-value (except
10945 for the type that maps to the built-in complex data type), the
10946 characteristic data type is int. */
10947 if (RECORD_OR_UNION_TYPE_P (type
)
10948 && !aggregate_value_p (type
, NULL
)
10949 && TREE_CODE (type
) != COMPLEX_TYPE
)
10950 return integer_type_node
;
10952 /* d) If none of the above three classes is applicable, the
10953 characteristic data type is int. */
10957 /* e) For Intel Xeon Phi native and offload compilation, if the
10958 resulting characteristic data type is 8-bit or 16-bit integer
10959 data type, the characteristic data type is int. */
10960 /* Well, we don't handle Xeon Phi yet. */
10964 simd_clone_mangle (struct cgraph_node
*node
,
10965 struct cgraph_simd_clone
*clone_info
)
10967 char vecsize_mangle
= clone_info
->vecsize_mangle
;
10968 char mask
= clone_info
->inbranch
? 'M' : 'N';
10969 unsigned int simdlen
= clone_info
->simdlen
;
10973 gcc_assert (vecsize_mangle
&& simdlen
);
10975 pp_string (&pp
, "_ZGV");
10976 pp_character (&pp
, vecsize_mangle
);
10977 pp_character (&pp
, mask
);
10978 pp_decimal_int (&pp
, simdlen
);
10980 for (n
= 0; n
< clone_info
->nargs
; ++n
)
10982 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
10984 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
10985 pp_character (&pp
, 'u');
10986 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
10988 gcc_assert (arg
.linear_step
!= 0);
10989 pp_character (&pp
, 'l');
10990 if (arg
.linear_step
> 1)
10991 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
10992 else if (arg
.linear_step
< 0)
10994 pp_character (&pp
, 'n');
10995 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
10999 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
11001 pp_character (&pp
, 's');
11002 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
11005 pp_character (&pp
, 'v');
11008 pp_character (&pp
, 'a');
11009 pp_decimal_int (&pp
, arg
.alignment
);
11013 pp_underscore (&pp
);
11015 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
)));
11016 const char *str
= pp_formatted_text (&pp
);
11018 /* If there already is a SIMD clone with the same mangled name, don't
11019 add another one. This can happen e.g. for
11020 #pragma omp declare simd
11021 #pragma omp declare simd simdlen(8)
11022 int foo (int, int);
11023 if the simdlen is assumed to be 8 for the first one, etc. */
11024 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
11025 clone
= clone
->simdclone
->next_clone
)
11026 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
11030 return get_identifier (str
);
11033 /* Create a simd clone of OLD_NODE and return it. */
11035 static struct cgraph_node
*
11036 simd_clone_create (struct cgraph_node
*old_node
)
11038 struct cgraph_node
*new_node
;
11039 if (old_node
->definition
)
11041 if (!cgraph_function_with_gimple_body_p (old_node
))
11043 cgraph_get_body (old_node
);
11044 new_node
= cgraph_function_versioning (old_node
, vNULL
, NULL
, NULL
,
11045 false, NULL
, NULL
, "simdclone");
11049 tree old_decl
= old_node
->decl
;
11050 tree new_decl
= copy_node (old_node
->decl
);
11051 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
11052 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
11053 SET_DECL_RTL (new_decl
, NULL
);
11054 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
11055 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
11057 = cgraph_copy_node_for_versioning (old_node
, new_decl
, vNULL
, NULL
);
11058 cgraph_call_function_insertion_hooks (new_node
);
11060 if (new_node
== NULL
)
11063 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
11065 /* The function cgraph_function_versioning () will force the new
11066 symbol local. Undo this, and inherit external visability from
11068 new_node
->local
.local
= old_node
->local
.local
;
11069 new_node
->externally_visible
= old_node
->externally_visible
;
11074 /* Adjust the return type of the given function to its appropriate
11075 vector counterpart. Returns a simd array to be used throughout the
11076 function as a return value. */
11079 simd_clone_adjust_return_type (struct cgraph_node
*node
)
11081 tree fndecl
= node
->decl
;
11082 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
11083 unsigned int veclen
;
11086 /* Adjust the function return type. */
11087 if (orig_rettype
== void_type_node
)
11089 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
11090 if (INTEGRAL_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
)))
11091 || POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
))))
11092 veclen
= node
->simdclone
->vecsize_int
;
11094 veclen
= node
->simdclone
->vecsize_float
;
11095 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl
))));
11096 if (veclen
> node
->simdclone
->simdlen
)
11097 veclen
= node
->simdclone
->simdlen
;
11098 if (veclen
== node
->simdclone
->simdlen
)
11099 TREE_TYPE (TREE_TYPE (fndecl
))
11100 = build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)),
11101 node
->simdclone
->simdlen
);
11104 t
= build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)), veclen
);
11105 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
11106 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
11108 if (!node
->definition
)
11111 t
= DECL_RESULT (fndecl
);
11112 /* Adjust the DECL_RESULT. */
11113 gcc_assert (TREE_TYPE (t
) != void_type_node
);
11114 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
11117 tree atype
= build_array_type_nelts (orig_rettype
,
11118 node
->simdclone
->simdlen
);
11119 if (veclen
!= node
->simdclone
->simdlen
)
11120 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
11122 /* Set up a SIMD array to use as the return value. */
11123 tree retval
= create_tmp_var_raw (atype
, "retval");
11124 gimple_add_tmp_var (retval
);
11128 /* Each vector argument has a corresponding array to be used locally
11129 as part of the eventual loop. Create such temporary array and
11132 PREFIX is the prefix to be used for the temporary.
11134 TYPE is the inner element type.
11136 SIMDLEN is the number of elements. */
11139 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
11141 tree atype
= build_array_type_nelts (type
, simdlen
);
11142 tree avar
= create_tmp_var_raw (atype
, prefix
);
11143 gimple_add_tmp_var (avar
);
11147 /* Modify the function argument types to their corresponding vector
11148 counterparts if appropriate. Also, create one array for each simd
11149 argument to be used locally when using the function arguments as
11152 NODE is the function whose arguments are to be adjusted.
11154 Returns an adjustment vector that will be filled describing how the
11155 argument types will be adjusted. */
11157 static ipa_parm_adjustment_vec
11158 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
11161 ipa_parm_adjustment_vec adjustments
;
11163 if (node
->definition
)
11164 args
= ipa_get_vector_of_formal_parms (node
->decl
);
11166 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
11167 adjustments
.create (args
.length ());
11168 unsigned i
, j
, veclen
;
11169 struct ipa_parm_adjustment adj
;
11170 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
11172 memset (&adj
, 0, sizeof (adj
));
11173 tree parm
= args
[i
];
11174 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
11175 adj
.base_index
= i
;
11178 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
11179 node
->simdclone
->args
[i
].orig_type
= parm_type
;
11181 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
11183 /* No adjustment necessary for scalar arguments. */
11184 adj
.op
= IPA_PARM_OP_COPY
;
11188 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
11189 veclen
= node
->simdclone
->vecsize_int
;
11191 veclen
= node
->simdclone
->vecsize_float
;
11192 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
11193 if (veclen
> node
->simdclone
->simdlen
)
11194 veclen
= node
->simdclone
->simdlen
;
11195 adj
.arg_prefix
= "simd";
11196 adj
.type
= build_vector_type (parm_type
, veclen
);
11197 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
11198 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11200 adjustments
.safe_push (adj
);
11203 memset (&adj
, 0, sizeof (adj
));
11204 adj
.op
= IPA_PARM_OP_NEW
;
11205 adj
.arg_prefix
= "simd";
11206 adj
.base_index
= i
;
11207 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
11211 if (node
->definition
)
11212 node
->simdclone
->args
[i
].simd_array
11213 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
11214 parm_type
, node
->simdclone
->simdlen
);
11216 adjustments
.safe_push (adj
);
11219 if (node
->simdclone
->inbranch
)
11222 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
11225 memset (&adj
, 0, sizeof (adj
));
11226 adj
.op
= IPA_PARM_OP_NEW
;
11227 adj
.arg_prefix
= "mask";
11229 adj
.base_index
= i
;
11230 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
11231 veclen
= node
->simdclone
->vecsize_int
;
11233 veclen
= node
->simdclone
->vecsize_float
;
11234 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
11235 if (veclen
> node
->simdclone
->simdlen
)
11236 veclen
= node
->simdclone
->simdlen
;
11237 adj
.type
= build_vector_type (base_type
, veclen
);
11238 adjustments
.safe_push (adj
);
11240 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11241 adjustments
.safe_push (adj
);
11243 /* We have previously allocated one extra entry for the mask. Use
11245 struct cgraph_simd_clone
*sc
= node
->simdclone
;
11247 if (node
->definition
)
11249 sc
->args
[i
].orig_arg
11250 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
11251 sc
->args
[i
].simd_array
11252 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
11254 sc
->args
[i
].orig_type
= base_type
;
11255 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
11258 if (node
->definition
)
11259 ipa_modify_formal_parameters (node
->decl
, adjustments
);
11262 tree new_arg_types
= NULL_TREE
, new_reversed
;
11263 bool last_parm_void
= false;
11264 if (args
.length () > 0 && args
.last () == void_type_node
)
11265 last_parm_void
= true;
11267 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
11268 j
= adjustments
.length ();
11269 for (i
= 0; i
< j
; i
++)
11271 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
11273 if (adj
->op
== IPA_PARM_OP_COPY
)
11274 ptype
= args
[adj
->base_index
];
11277 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
11279 new_reversed
= nreverse (new_arg_types
);
11280 if (last_parm_void
)
11283 TREE_CHAIN (new_arg_types
) = void_list_node
;
11285 new_reversed
= void_list_node
;
11288 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
11289 TYPE_ARG_TYPES (new_type
) = new_reversed
;
11290 TREE_TYPE (node
->decl
) = new_type
;
11292 adjustments
.release ();
11295 return adjustments
;
11298 /* Initialize and copy the function arguments in NODE to their
11299 corresponding local simd arrays. Returns a fresh gimple_seq with
11300 the instruction sequence generated. */
11303 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
11304 ipa_parm_adjustment_vec adjustments
)
11306 gimple_seq seq
= NULL
;
11307 unsigned i
= 0, j
= 0, k
;
11309 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
11311 arg
= DECL_CHAIN (arg
), i
++, j
++)
11313 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
11316 node
->simdclone
->args
[i
].vector_arg
= arg
;
11318 tree array
= node
->simdclone
->args
[i
].simd_array
;
11319 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
11321 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11322 tree ptr
= build_fold_addr_expr (array
);
11323 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11324 build_int_cst (ptype
, 0));
11325 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11326 gimplify_and_add (t
, &seq
);
11330 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
11331 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11332 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
11334 tree ptr
= build_fold_addr_expr (array
);
11338 arg
= DECL_CHAIN (arg
);
11342 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
11343 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11344 build_int_cst (ptype
, k
* elemsize
));
11345 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11346 gimplify_and_add (t
, &seq
);
11353 /* Callback info for ipa_simd_modify_stmt_ops below. */
11355 struct modify_stmt_info
{
11356 ipa_parm_adjustment_vec adjustments
;
11358 /* True if the parent statement was modified by
11359 ipa_simd_modify_stmt_ops. */
11363 /* Callback for walk_gimple_op.
11365 Adjust operands from a given statement as specified in the
11366 adjustments vector in the callback data. */
11369 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
11371 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
11372 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
11373 tree
*orig_tp
= tp
;
11374 if (TREE_CODE (*tp
) == ADDR_EXPR
)
11375 tp
= &TREE_OPERAND (*tp
, 0);
11376 struct ipa_parm_adjustment
*cand
= NULL
;
11377 if (TREE_CODE (*tp
) == PARM_DECL
)
11378 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
11382 *walk_subtrees
= 0;
11385 tree repl
= NULL_TREE
;
11387 repl
= unshare_expr (cand
->new_decl
);
11392 *walk_subtrees
= 0;
11393 bool modified
= info
->modified
;
11394 info
->modified
= false;
11395 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
11396 if (!info
->modified
)
11398 info
->modified
= modified
;
11401 info
->modified
= modified
;
11410 repl
= build_fold_addr_expr (repl
);
11412 = gimple_build_assign (make_ssa_name (TREE_TYPE (repl
), NULL
), repl
);
11413 repl
= gimple_assign_lhs (stmt
);
11414 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
11415 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11418 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
11420 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
11426 info
->modified
= true;
11430 /* Traverse the function body and perform all modifications as
11431 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
11432 modified such that the replacement/reduction value will now be an
11433 offset into the corresponding simd_array.
11435 This function will replace all function argument uses with their
11436 corresponding simd array elements, and ajust the return values
11440 ipa_simd_modify_function_body (struct cgraph_node
*node
,
11441 ipa_parm_adjustment_vec adjustments
,
11442 tree retval_array
, tree iter
)
11445 unsigned int i
, j
, l
;
11447 /* Re-use the adjustments array, but this time use it to replace
11448 every function argument use to an offset into the corresponding
11450 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
11452 if (!node
->simdclone
->args
[i
].vector_arg
)
11455 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
11456 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
11457 adjustments
[j
].new_decl
11458 = build4 (ARRAY_REF
,
11460 node
->simdclone
->args
[i
].simd_array
,
11462 NULL_TREE
, NULL_TREE
);
11463 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
11464 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
11465 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
11468 l
= adjustments
.length ();
11469 for (i
= 1; i
< num_ssa_names
; i
++)
11471 tree name
= ssa_name (i
);
11473 && SSA_NAME_VAR (name
)
11474 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
11476 for (j
= 0; j
< l
; j
++)
11477 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
11478 && adjustments
[j
].new_decl
)
11481 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
11484 = copy_var_decl (adjustments
[j
].base
,
11485 DECL_NAME (adjustments
[j
].base
),
11486 TREE_TYPE (adjustments
[j
].base
));
11487 adjustments
[j
].new_ssa_base
= base_var
;
11490 base_var
= adjustments
[j
].new_ssa_base
;
11491 if (SSA_NAME_IS_DEFAULT_DEF (name
))
11493 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11494 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
11495 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
11496 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
11497 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
11498 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
11499 gimple stmt
= gimple_build_assign (name
, new_decl
);
11500 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11503 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
11508 struct modify_stmt_info info
;
11509 info
.adjustments
= adjustments
;
11511 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
11513 gimple_stmt_iterator gsi
;
11515 gsi
= gsi_start_bb (bb
);
11516 while (!gsi_end_p (gsi
))
11518 gimple stmt
= gsi_stmt (gsi
);
11520 struct walk_stmt_info wi
;
11522 memset (&wi
, 0, sizeof (wi
));
11523 info
.modified
= false;
11525 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
11527 if (gimple_code (stmt
) == GIMPLE_RETURN
)
11529 tree retval
= gimple_return_retval (stmt
);
11532 gsi_remove (&gsi
, true);
11536 /* Replace `return foo' with `retval_array[iter] = foo'. */
11537 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
11538 retval_array
, iter
, NULL
, NULL
);
11539 stmt
= gimple_build_assign (ref
, retval
);
11540 gsi_replace (&gsi
, stmt
, true);
11541 info
.modified
= true;
11546 update_stmt (stmt
);
11547 if (maybe_clean_eh_stmt (stmt
))
11548 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
11555 /* Adjust the argument types in NODE to their appropriate vector
11559 simd_clone_adjust (struct cgraph_node
*node
)
11561 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
11563 targetm
.simd_clone
.adjust (node
);
11565 tree retval
= simd_clone_adjust_return_type (node
);
11566 ipa_parm_adjustment_vec adjustments
11567 = simd_clone_adjust_argument_types (node
);
11569 push_gimplify_context ();
11571 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
11573 /* Adjust all uses of vector arguments accordingly. Adjust all
11574 return values accordingly. */
11575 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
11576 tree iter1
= make_ssa_name (iter
, NULL
);
11577 tree iter2
= make_ssa_name (iter
, NULL
);
11578 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
11580 /* Initialize the iteration variable. */
11581 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11582 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
11583 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
11584 /* Insert the SIMD array and iv initialization at function
11586 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
11588 pop_gimplify_context (NULL
);
11590 /* Create a new BB right before the original exit BB, to hold the
11591 iteration increment and the condition/branch. */
11592 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
11593 basic_block incr_bb
= create_empty_bb (orig_exit
);
11594 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
11595 flag. Set it now to be a FALLTHRU_EDGE. */
11596 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
11597 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
11598 for (unsigned i
= 0;
11599 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
11601 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
11602 redirect_edge_succ (e
, incr_bb
);
11604 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
11605 e
->probability
= REG_BR_PROB_BASE
;
11606 gsi
= gsi_last_bb (incr_bb
);
11607 gimple g
= gimple_build_assign_with_ops (PLUS_EXPR
, iter2
, iter1
,
11608 build_int_cst (unsigned_type_node
,
11610 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11612 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
11613 struct loop
*loop
= alloc_loop ();
11614 cfun
->has_force_vectorize_loops
= true;
11615 loop
->safelen
= node
->simdclone
->simdlen
;
11616 loop
->force_vectorize
= true;
11617 loop
->header
= body_bb
;
11618 add_bb_to_loop (incr_bb
, loop
);
11620 /* Branch around the body if the mask applies. */
11621 if (node
->simdclone
->inbranch
)
11623 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
11625 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
11626 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)), NULL
);
11627 tree aref
= build4 (ARRAY_REF
,
11628 TREE_TYPE (TREE_TYPE (mask_array
)),
11631 g
= gimple_build_assign (mask
, aref
);
11632 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11633 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
11634 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
11636 aref
= build1 (VIEW_CONVERT_EXPR
,
11637 build_nonstandard_integer_type (bitsize
, 0), mask
);
11638 mask
= make_ssa_name (TREE_TYPE (aref
), NULL
);
11639 g
= gimple_build_assign (mask
, aref
);
11640 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11643 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
11645 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11646 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
11647 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
11650 /* Generate the condition. */
11651 g
= gimple_build_cond (LT_EXPR
,
11653 build_int_cst (unsigned_type_node
,
11654 node
->simdclone
->simdlen
),
11656 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11657 e
= split_block (incr_bb
, gsi_stmt (gsi
));
11658 basic_block latch_bb
= e
->dest
;
11659 basic_block new_exit_bb
= e
->dest
;
11660 new_exit_bb
= split_block (latch_bb
, NULL
)->dest
;
11661 loop
->latch
= latch_bb
;
11663 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
11665 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
11666 /* The successor of incr_bb is already pointing to latch_bb; just
11668 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
11669 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
11671 gimple phi
= create_phi_node (iter1
, body_bb
);
11672 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
11673 edge latch_edge
= single_succ_edge (latch_bb
);
11674 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
11676 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
11678 /* Generate the new return. */
11679 gsi
= gsi_last_bb (new_exit_bb
);
11681 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
11682 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
11683 retval
= TREE_OPERAND (retval
, 0);
11686 retval
= build1 (VIEW_CONVERT_EXPR
,
11687 TREE_TYPE (TREE_TYPE (node
->decl
)),
11689 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
11690 false, GSI_CONTINUE_LINKING
);
11692 g
= gimple_build_return (retval
);
11693 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11695 /* Handle aligned clauses by replacing default defs of the aligned
11696 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
11697 lhs. Handle linear by adding PHIs. */
11698 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
11699 if (node
->simdclone
->args
[i
].alignment
11700 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
11701 && (node
->simdclone
->args
[i
].alignment
11702 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
11703 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
11706 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
11707 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
11708 tree def
= ssa_default_def (cfun
, orig_arg
);
11709 if (def
&& !has_zero_uses (def
))
11711 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
11712 gimple_seq seq
= NULL
;
11713 bool need_cvt
= false;
11715 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
11717 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
11720 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
, NULL
);
11721 gimple_call_set_lhs (g
, t
);
11722 gimple_seq_add_stmt_without_update (&seq
, g
);
11725 t
= make_ssa_name (orig_arg
, NULL
);
11726 g
= gimple_build_assign_with_ops (NOP_EXPR
, t
,
11727 gimple_call_lhs (g
),
11729 gimple_seq_add_stmt_without_update (&seq
, g
);
11731 gsi_insert_seq_on_edge_immediate
11732 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
11734 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11735 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
11737 cgraph_create_edge (node
, cgraph_get_create_node (fn
),
11738 call
, entry_bb
->count
, freq
);
11740 imm_use_iterator iter
;
11741 use_operand_p use_p
;
11743 tree repl
= gimple_get_lhs (g
);
11744 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
11745 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
11748 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
11749 SET_USE (use_p
, repl
);
11752 else if (node
->simdclone
->args
[i
].arg_type
11753 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
11755 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
11756 tree def
= ssa_default_def (cfun
, orig_arg
);
11757 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11758 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
11759 if (def
&& !has_zero_uses (def
))
11761 iter1
= make_ssa_name (orig_arg
, NULL
);
11762 iter2
= make_ssa_name (orig_arg
, NULL
);
11763 phi
= create_phi_node (iter1
, body_bb
);
11764 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
11765 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
11766 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11767 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
11768 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11769 ? TREE_TYPE (orig_arg
) : sizetype
;
11771 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
11772 g
= gimple_build_assign_with_ops (code
, iter2
, iter1
, addcst
);
11773 gsi
= gsi_last_bb (incr_bb
);
11774 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
11776 imm_use_iterator iter
;
11777 use_operand_p use_p
;
11779 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
11780 if (use_stmt
== phi
)
11783 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
11784 SET_USE (use_p
, iter1
);
11788 calculate_dominance_info (CDI_DOMINATORS
);
11789 add_loop (loop
, loop
->header
->loop_father
);
11790 update_ssa (TODO_update_ssa
);
11795 /* If the function in NODE is tagged as an elemental SIMD function,
11796 create the appropriate SIMD clones. */
11799 expand_simd_clones (struct cgraph_node
*node
)
11801 tree attr
= lookup_attribute ("omp declare simd",
11802 DECL_ATTRIBUTES (node
->decl
));
11803 if (attr
== NULL_TREE
11804 || node
->global
.inlined_to
11805 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
11809 #pragma omp declare simd
11811 in C, there we don't know the argument types at all. */
11812 if (!node
->definition
11813 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
11818 /* Start with parsing the "omp declare simd" attribute(s). */
11819 bool inbranch_clause_specified
;
11820 struct cgraph_simd_clone
*clone_info
11821 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
11822 &inbranch_clause_specified
);
11823 if (clone_info
== NULL
)
11826 int orig_simdlen
= clone_info
->simdlen
;
11827 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
11828 /* The target can return 0 (no simd clones should be created),
11829 1 (just one ISA of simd clones should be created) or higher
11830 count of ISA variants. In that case, clone_info is initialized
11831 for the first ISA variant. */
11833 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
11838 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
11839 also create one inbranch and one !inbranch clone of it. */
11840 for (int i
= 0; i
< count
* 2; i
++)
11842 struct cgraph_simd_clone
*clone
= clone_info
;
11843 if (inbranch_clause_specified
&& (i
& 1) != 0)
11848 clone
= simd_clone_struct_alloc (clone_info
->nargs
11850 simd_clone_struct_copy (clone
, clone_info
);
11851 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
11852 and simd_clone_adjust_argument_types did to the first
11854 clone
->nargs
-= clone_info
->inbranch
;
11855 clone
->simdlen
= orig_simdlen
;
11856 /* And call the target hook again to get the right ISA. */
11857 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
11861 clone
->inbranch
= 1;
11864 /* simd_clone_mangle might fail if such a clone has been created
11866 tree id
= simd_clone_mangle (node
, clone
);
11867 if (id
== NULL_TREE
)
11870 /* Only when we are sure we want to create the clone actually
11871 clone the function (or definitions) or create another
11872 extern FUNCTION_DECL (for prototypes without definitions). */
11873 struct cgraph_node
*n
= simd_clone_create (node
);
11877 n
->simdclone
= clone
;
11878 clone
->origin
= node
;
11879 clone
->next_clone
= NULL
;
11880 if (node
->simd_clones
== NULL
)
11882 clone
->prev_clone
= n
;
11883 node
->simd_clones
= n
;
11887 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
11888 clone
->prev_clone
->simdclone
->next_clone
= n
;
11889 node
->simd_clones
->simdclone
->prev_clone
= n
;
11891 change_decl_assembler_name (n
->decl
, id
);
11892 /* And finally adjust the return type, parameters and for
11893 definitions also function body. */
11894 if (node
->definition
)
11895 simd_clone_adjust (n
);
11898 simd_clone_adjust_return_type (n
);
11899 simd_clone_adjust_argument_types (n
);
11903 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
11906 /* Entry point for IPA simd clone creation pass. */
11908 static unsigned int
11909 ipa_omp_simd_clone (void)
11911 struct cgraph_node
*node
;
11912 FOR_EACH_FUNCTION (node
)
11913 expand_simd_clones (node
);
11919 const pass_data pass_data_omp_simd_clone
=
11921 SIMPLE_IPA_PASS
, /* type */
11922 "simdclone", /* name */
11923 OPTGROUP_NONE
, /* optinfo_flags */
11924 true, /* has_execute */
11925 TV_NONE
, /* tv_id */
11926 ( PROP_ssa
| PROP_cfg
), /* properties_required */
11927 0, /* properties_provided */
11928 0, /* properties_destroyed */
11929 0, /* todo_flags_start */
11930 0, /* todo_flags_finish */
11933 class pass_omp_simd_clone
: public simple_ipa_opt_pass
11936 pass_omp_simd_clone(gcc::context
*ctxt
)
11937 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
11940 /* opt_pass methods: */
11941 virtual bool gate (function
*);
11942 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
11946 pass_omp_simd_clone::gate (function
*)
11948 return ((flag_openmp
|| flag_openmp_simd
11950 || (in_lto_p
&& !flag_wpa
))
11951 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
11954 } // anon namespace
11956 simple_ipa_opt_pass
*
11957 make_pass_omp_simd_clone (gcc::context
*ctxt
)
11959 return new pass_omp_simd_clone (ctxt
);
11962 #include "gt-omp-low.h"