1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2014 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
32 #include "pointer-set.h"
33 #include "basic-block.h"
34 #include "tree-ssa-alias.h"
35 #include "internal-fn.h"
36 #include "gimple-fold.h"
37 #include "gimple-expr.h"
41 #include "gimple-iterator.h"
42 #include "gimplify-me.h"
43 #include "gimple-walk.h"
44 #include "tree-iterator.h"
45 #include "tree-inline.h"
46 #include "langhooks.h"
47 #include "diagnostic-core.h"
48 #include "gimple-ssa.h"
51 #include "tree-phinodes.h"
52 #include "ssa-iterators.h"
53 #include "tree-ssanames.h"
54 #include "tree-into-ssa.h"
61 #include "tree-pass.h"
63 #include "splay-tree.h"
68 #include "gimple-low.h"
69 #include "tree-cfgcleanup.h"
70 #include "pretty-print.h"
72 #include "tree-nested.h"
76 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
77 phases. The first phase scans the function looking for OMP statements
78 and then for variables that must be replaced to satisfy data sharing
79 clauses. The second phase expands code for the constructs, as well as
80 re-gimplifying things when variables have been replaced with complex
83 Final code generation is done by pass_expand_omp. The flowgraph is
84 scanned for parallel regions which are then moved to a new
85 function, to be invoked by the thread library. */
87 /* Parallel region information. Every parallel and workshare
88 directive is enclosed between two markers, the OMP_* directive
89 and a corresponding OMP_RETURN statement. */
93 /* The enclosing region. */
94 struct omp_region
*outer
;
96 /* First child region. */
97 struct omp_region
*inner
;
99 /* Next peer region. */
100 struct omp_region
*next
;
102 /* Block containing the omp directive as its last stmt. */
105 /* Block containing the OMP_RETURN as its last stmt. */
108 /* Block containing the OMP_CONTINUE as its last stmt. */
111 /* If this is a combined parallel+workshare region, this is a list
112 of additional arguments needed by the combined parallel+workshare
114 vec
<tree
, va_gc
> *ws_args
;
116 /* The code for the omp directive of this region. */
117 enum gimple_code type
;
119 /* Schedule kind, only used for OMP_FOR type regions. */
120 enum omp_clause_schedule_kind sched_kind
;
122 /* True if this is a combined parallel+workshare region. */
123 bool is_combined_parallel
;
126 /* Context structure. Used to store information about each parallel
127 directive in the code. */
129 typedef struct omp_context
131 /* This field must be at the beginning, as we do "inheritance": Some
132 callback functions for tree-inline.c (e.g., omp_copy_decl)
133 receive a copy_body_data pointer that is up-casted to an
134 omp_context pointer. */
137 /* The tree of contexts corresponding to the encountered constructs. */
138 struct omp_context
*outer
;
141 /* Map variables to fields in a structure that allows communication
142 between sending and receiving threads. */
143 splay_tree field_map
;
148 /* These are used just by task contexts, if task firstprivate fn is
149 needed. srecord_type is used to communicate from the thread
150 that encountered the task construct to task firstprivate fn,
151 record_type is allocated by GOMP_task, initialized by task firstprivate
152 fn and passed to the task body fn. */
153 splay_tree sfield_map
;
156 /* A chain of variables to add to the top-level block surrounding the
157 construct. In the case of a parallel, this is in the child function. */
160 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
161 barriers should jump to during omplower pass. */
164 /* What to do with variables with implicitly determined sharing
166 enum omp_clause_default_kind default_kind
;
168 /* Nesting depth of this context. Used to beautify error messages re
169 invalid gotos. The outermost ctx is depth 1, with depth 0 being
170 reserved for the main body of the function. */
173 /* True if this parallel directive is nested within another. */
176 /* True if this construct can be cancelled. */
181 struct omp_for_data_loop
183 tree v
, n1
, n2
, step
;
184 enum tree_code cond_code
;
187 /* A structure describing the main elements of a parallel loop. */
191 struct omp_for_data_loop loop
;
196 bool have_nowait
, have_ordered
;
197 enum omp_clause_schedule_kind sched_kind
;
198 struct omp_for_data_loop
*loops
;
202 static splay_tree all_contexts
;
203 static int taskreg_nesting_level
;
204 static int target_nesting_level
;
205 static struct omp_region
*root_omp_region
;
206 static bitmap task_shared_vars
;
208 static void scan_omp (gimple_seq
*, omp_context
*);
209 static tree
scan_omp_1_op (tree
*, int *, void *);
211 #define WALK_SUBSTMTS \
215 case GIMPLE_EH_FILTER: \
216 case GIMPLE_TRANSACTION: \
217 /* The sub-statements for these should be walked. */ \
218 *handled_ops_p = false; \
221 /* Convenience function for calling scan_omp_1_op on tree operands. */
224 scan_omp_op (tree
*tp
, omp_context
*ctx
)
226 struct walk_stmt_info wi
;
228 memset (&wi
, 0, sizeof (wi
));
230 wi
.want_locations
= true;
232 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
235 static void lower_omp (gimple_seq
*, omp_context
*);
236 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
237 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
239 /* Find an OpenMP clause of type KIND within CLAUSES. */
242 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
244 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
245 if (OMP_CLAUSE_CODE (clauses
) == kind
)
251 /* Return true if CTX is for an omp parallel. */
254 is_parallel_ctx (omp_context
*ctx
)
256 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
260 /* Return true if CTX is for an omp task. */
263 is_task_ctx (omp_context
*ctx
)
265 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
269 /* Return true if CTX is for an omp parallel or omp task. */
272 is_taskreg_ctx (omp_context
*ctx
)
274 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
275 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
279 /* Return true if REGION is a combined parallel+workshare region. */
282 is_combined_parallel (struct omp_region
*region
)
284 return region
->is_combined_parallel
;
288 /* Extract the header elements of parallel loop FOR_STMT and store
292 extract_omp_for_data (gimple for_stmt
, struct omp_for_data
*fd
,
293 struct omp_for_data_loop
*loops
)
295 tree t
, var
, *collapse_iter
, *collapse_count
;
296 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
297 struct omp_for_data_loop
*loop
;
299 struct omp_for_data_loop dummy_loop
;
300 location_t loc
= gimple_location (for_stmt
);
301 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
302 bool distribute
= gimple_omp_for_kind (for_stmt
)
303 == GF_OMP_FOR_KIND_DISTRIBUTE
;
305 fd
->for_stmt
= for_stmt
;
307 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
308 if (fd
->collapse
> 1)
311 fd
->loops
= &fd
->loop
;
313 fd
->have_nowait
= distribute
|| simd
;
314 fd
->have_ordered
= false;
315 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
316 fd
->chunk_size
= NULL_TREE
;
317 collapse_iter
= NULL
;
318 collapse_count
= NULL
;
320 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
321 switch (OMP_CLAUSE_CODE (t
))
323 case OMP_CLAUSE_NOWAIT
:
324 fd
->have_nowait
= true;
326 case OMP_CLAUSE_ORDERED
:
327 fd
->have_ordered
= true;
329 case OMP_CLAUSE_SCHEDULE
:
330 gcc_assert (!distribute
);
331 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
332 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
334 case OMP_CLAUSE_DIST_SCHEDULE
:
335 gcc_assert (distribute
);
336 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
338 case OMP_CLAUSE_COLLAPSE
:
339 if (fd
->collapse
> 1)
341 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
342 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
348 /* FIXME: for now map schedule(auto) to schedule(static).
349 There should be analysis to determine whether all iterations
350 are approximately the same amount of work (then schedule(static)
351 is best) or if it varies (then schedule(dynamic,N) is better). */
352 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
354 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
355 gcc_assert (fd
->chunk_size
== NULL
);
357 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
358 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
359 gcc_assert (fd
->chunk_size
== NULL
);
360 else if (fd
->chunk_size
== NULL
)
362 /* We only need to compute a default chunk size for ordered
363 static loops and dynamic loops. */
364 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
366 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
367 ? integer_zero_node
: integer_one_node
;
370 for (i
= 0; i
< fd
->collapse
; i
++)
372 if (fd
->collapse
== 1)
374 else if (loops
!= NULL
)
379 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
380 gcc_assert (SSA_VAR_P (loop
->v
));
381 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
382 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
383 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
384 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
386 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
387 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
388 switch (loop
->cond_code
)
394 gcc_assert (gimple_omp_for_kind (for_stmt
)
395 == GF_OMP_FOR_KIND_CILKSIMD
);
398 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
399 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
401 loop
->n2
= fold_build2_loc (loc
,
402 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
403 build_int_cst (TREE_TYPE (loop
->n2
), 1));
404 loop
->cond_code
= LT_EXPR
;
407 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
408 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
410 loop
->n2
= fold_build2_loc (loc
,
411 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
412 build_int_cst (TREE_TYPE (loop
->n2
), 1));
413 loop
->cond_code
= GT_EXPR
;
419 t
= gimple_omp_for_incr (for_stmt
, i
);
420 gcc_assert (TREE_OPERAND (t
, 0) == var
);
421 switch (TREE_CODE (t
))
424 loop
->step
= TREE_OPERAND (t
, 1);
426 case POINTER_PLUS_EXPR
:
427 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
430 loop
->step
= TREE_OPERAND (t
, 1);
431 loop
->step
= fold_build1_loc (loc
,
432 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
440 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
441 && !fd
->have_ordered
))
443 if (fd
->collapse
== 1)
444 iter_type
= TREE_TYPE (loop
->v
);
446 || TYPE_PRECISION (iter_type
)
447 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
449 = build_nonstandard_integer_type
450 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
452 else if (iter_type
!= long_long_unsigned_type_node
)
454 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
455 iter_type
= long_long_unsigned_type_node
;
456 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
457 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
458 >= TYPE_PRECISION (iter_type
))
462 if (loop
->cond_code
== LT_EXPR
)
463 n
= fold_build2_loc (loc
,
464 PLUS_EXPR
, TREE_TYPE (loop
->v
),
465 loop
->n2
, loop
->step
);
468 if (TREE_CODE (n
) != INTEGER_CST
469 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
470 iter_type
= long_long_unsigned_type_node
;
472 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
473 > TYPE_PRECISION (iter_type
))
477 if (loop
->cond_code
== LT_EXPR
)
480 n2
= fold_build2_loc (loc
,
481 PLUS_EXPR
, TREE_TYPE (loop
->v
),
482 loop
->n2
, loop
->step
);
486 n1
= fold_build2_loc (loc
,
487 MINUS_EXPR
, TREE_TYPE (loop
->v
),
488 loop
->n2
, loop
->step
);
491 if (TREE_CODE (n1
) != INTEGER_CST
492 || TREE_CODE (n2
) != INTEGER_CST
493 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
494 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
495 iter_type
= long_long_unsigned_type_node
;
499 if (collapse_count
&& *collapse_count
== NULL
)
501 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
502 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
503 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
504 if (t
&& integer_zerop (t
))
505 count
= build_zero_cst (long_long_unsigned_type_node
);
506 else if ((i
== 0 || count
!= NULL_TREE
)
507 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
508 && TREE_CONSTANT (loop
->n1
)
509 && TREE_CONSTANT (loop
->n2
)
510 && TREE_CODE (loop
->step
) == INTEGER_CST
)
512 tree itype
= TREE_TYPE (loop
->v
);
514 if (POINTER_TYPE_P (itype
))
515 itype
= signed_type_for (itype
);
516 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
517 t
= fold_build2_loc (loc
,
519 fold_convert_loc (loc
, itype
, loop
->step
), t
);
520 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
521 fold_convert_loc (loc
, itype
, loop
->n2
));
522 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
523 fold_convert_loc (loc
, itype
, loop
->n1
));
524 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
525 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
526 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
527 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
528 fold_convert_loc (loc
, itype
,
531 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
532 fold_convert_loc (loc
, itype
, loop
->step
));
533 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
534 if (count
!= NULL_TREE
)
535 count
= fold_build2_loc (loc
,
536 MULT_EXPR
, long_long_unsigned_type_node
,
540 if (TREE_CODE (count
) != INTEGER_CST
)
543 else if (count
&& !integer_zerop (count
))
550 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
551 || fd
->have_ordered
))
553 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
554 iter_type
= long_long_unsigned_type_node
;
556 iter_type
= long_integer_type_node
;
558 else if (collapse_iter
&& *collapse_iter
!= NULL
)
559 iter_type
= TREE_TYPE (*collapse_iter
);
560 fd
->iter_type
= iter_type
;
561 if (collapse_iter
&& *collapse_iter
== NULL
)
562 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
563 if (collapse_count
&& *collapse_count
== NULL
)
566 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
568 *collapse_count
= create_tmp_var (iter_type
, ".count");
571 if (fd
->collapse
> 1)
573 fd
->loop
.v
= *collapse_iter
;
574 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
575 fd
->loop
.n2
= *collapse_count
;
576 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
577 fd
->loop
.cond_code
= LT_EXPR
;
582 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
583 is the immediate dominator of PAR_ENTRY_BB, return true if there
584 are no data dependencies that would prevent expanding the parallel
585 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
587 When expanding a combined parallel+workshare region, the call to
588 the child function may need additional arguments in the case of
589 GIMPLE_OMP_FOR regions. In some cases, these arguments are
590 computed out of variables passed in from the parent to the child
591 via 'struct .omp_data_s'. For instance:
593 #pragma omp parallel for schedule (guided, i * 4)
598 # BLOCK 2 (PAR_ENTRY_BB)
600 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
602 # BLOCK 3 (WS_ENTRY_BB)
603 .omp_data_i = &.omp_data_o;
604 D.1667 = .omp_data_i->i;
606 #pragma omp for schedule (guided, D.1598)
608 When we outline the parallel region, the call to the child function
609 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
610 that value is computed *after* the call site. So, in principle we
611 cannot do the transformation.
613 To see whether the code in WS_ENTRY_BB blocks the combined
614 parallel+workshare call, we collect all the variables used in the
615 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
616 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
619 FIXME. If we had the SSA form built at this point, we could merely
620 hoist the code in block 3 into block 2 and be done with it. But at
621 this point we don't have dataflow information and though we could
622 hack something up here, it is really not worth the aggravation. */
625 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
627 struct omp_for_data fd
;
628 gimple ws_stmt
= last_stmt (ws_entry_bb
);
630 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
633 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
635 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
637 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
639 if (fd
.iter_type
!= long_integer_type_node
)
642 /* FIXME. We give up too easily here. If any of these arguments
643 are not constants, they will likely involve variables that have
644 been mapped into fields of .omp_data_s for sharing with the child
645 function. With appropriate data flow, it would be possible to
647 if (!is_gimple_min_invariant (fd
.loop
.n1
)
648 || !is_gimple_min_invariant (fd
.loop
.n2
)
649 || !is_gimple_min_invariant (fd
.loop
.step
)
650 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
657 /* Collect additional arguments needed to emit a combined
658 parallel+workshare call. WS_STMT is the workshare directive being
661 static vec
<tree
, va_gc
> *
662 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
665 location_t loc
= gimple_location (ws_stmt
);
666 vec
<tree
, va_gc
> *ws_args
;
668 if (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
)
670 struct omp_for_data fd
;
673 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
677 if (gimple_omp_for_combined_into_p (ws_stmt
))
680 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
681 OMP_CLAUSE__LOOPTEMP_
);
683 n1
= OMP_CLAUSE_DECL (innerc
);
684 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
685 OMP_CLAUSE__LOOPTEMP_
);
687 n2
= OMP_CLAUSE_DECL (innerc
);
690 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
692 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
693 ws_args
->quick_push (t
);
695 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
696 ws_args
->quick_push (t
);
698 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
699 ws_args
->quick_push (t
);
703 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
704 ws_args
->quick_push (t
);
709 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
711 /* Number of sections is equal to the number of edges from the
712 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
713 the exit of the sections region. */
714 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
715 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
716 vec_alloc (ws_args
, 1);
717 ws_args
->quick_push (t
);
725 /* Discover whether REGION is a combined parallel+workshare region. */
728 determine_parallel_type (struct omp_region
*region
)
730 basic_block par_entry_bb
, par_exit_bb
;
731 basic_block ws_entry_bb
, ws_exit_bb
;
733 if (region
== NULL
|| region
->inner
== NULL
734 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
735 || region
->inner
->cont
== NULL
)
738 /* We only support parallel+for and parallel+sections. */
739 if (region
->type
!= GIMPLE_OMP_PARALLEL
740 || (region
->inner
->type
!= GIMPLE_OMP_FOR
741 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
744 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
745 WS_EXIT_BB -> PAR_EXIT_BB. */
746 par_entry_bb
= region
->entry
;
747 par_exit_bb
= region
->exit
;
748 ws_entry_bb
= region
->inner
->entry
;
749 ws_exit_bb
= region
->inner
->exit
;
751 if (single_succ (par_entry_bb
) == ws_entry_bb
752 && single_succ (ws_exit_bb
) == par_exit_bb
753 && workshare_safe_to_combine_p (ws_entry_bb
)
754 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
755 || (last_and_only_stmt (ws_entry_bb
)
756 && last_and_only_stmt (par_exit_bb
))))
758 gimple par_stmt
= last_stmt (par_entry_bb
);
759 gimple ws_stmt
= last_stmt (ws_entry_bb
);
761 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
763 /* If this is a combined parallel loop, we need to determine
764 whether or not to use the combined library calls. There
765 are two cases where we do not apply the transformation:
766 static loops and any kind of ordered loop. In the first
767 case, we already open code the loop so there is no need
768 to do anything else. In the latter case, the combined
769 parallel loop call would still need extra synchronization
770 to implement ordered semantics, so there would not be any
771 gain in using the combined call. */
772 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
773 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
775 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
776 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
778 region
->is_combined_parallel
= false;
779 region
->inner
->is_combined_parallel
= false;
784 region
->is_combined_parallel
= true;
785 region
->inner
->is_combined_parallel
= true;
786 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
791 /* Return true if EXPR is variable sized. */
794 is_variable_sized (const_tree expr
)
796 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
799 /* Return true if DECL is a reference type. */
802 is_reference (tree decl
)
804 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
807 /* Lookup variables in the decl or field splay trees. The "maybe" form
808 allows for the variable form to not have been entered, otherwise we
809 assert that the variable must have been entered. */
812 lookup_decl (tree var
, omp_context
*ctx
)
815 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
820 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
823 n
= (tree
*) pointer_map_contains (ctx
->cb
.decl_map
, var
);
824 return n
? *n
: NULL_TREE
;
828 lookup_field (tree var
, omp_context
*ctx
)
831 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
832 return (tree
) n
->value
;
836 lookup_sfield (tree var
, omp_context
*ctx
)
839 n
= splay_tree_lookup (ctx
->sfield_map
840 ? ctx
->sfield_map
: ctx
->field_map
,
841 (splay_tree_key
) var
);
842 return (tree
) n
->value
;
846 maybe_lookup_field (tree var
, omp_context
*ctx
)
849 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
850 return n
? (tree
) n
->value
: NULL_TREE
;
853 /* Return true if DECL should be copied by pointer. SHARED_CTX is
854 the parallel context if DECL is to be shared. */
857 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
859 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
862 /* We can only use copy-in/copy-out semantics for shared variables
863 when we know the value is not accessible from an outer scope. */
866 /* ??? Trivially accessible from anywhere. But why would we even
867 be passing an address in this case? Should we simply assert
868 this to be false, or should we have a cleanup pass that removes
869 these from the list of mappings? */
870 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
873 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
874 without analyzing the expression whether or not its location
875 is accessible to anyone else. In the case of nested parallel
876 regions it certainly may be. */
877 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
880 /* Do not use copy-in/copy-out for variables that have their
882 if (TREE_ADDRESSABLE (decl
))
885 /* lower_send_shared_vars only uses copy-in, but not copy-out
887 if (TREE_READONLY (decl
)
888 || ((TREE_CODE (decl
) == RESULT_DECL
889 || TREE_CODE (decl
) == PARM_DECL
)
890 && DECL_BY_REFERENCE (decl
)))
893 /* Disallow copy-in/out in nested parallel if
894 decl is shared in outer parallel, otherwise
895 each thread could store the shared variable
896 in its own copy-in location, making the
897 variable no longer really shared. */
898 if (shared_ctx
->is_nested
)
902 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
903 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
910 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
911 c
; c
= OMP_CLAUSE_CHAIN (c
))
912 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
913 && OMP_CLAUSE_DECL (c
) == decl
)
917 goto maybe_mark_addressable_and_ret
;
921 /* For tasks avoid using copy-in/out. As tasks can be
922 deferred or executed in different thread, when GOMP_task
923 returns, the task hasn't necessarily terminated. */
924 if (is_task_ctx (shared_ctx
))
927 maybe_mark_addressable_and_ret
:
928 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
929 if (is_gimple_reg (outer
))
931 /* Taking address of OUTER in lower_send_shared_vars
932 might need regimplification of everything that uses the
934 if (!task_shared_vars
)
935 task_shared_vars
= BITMAP_ALLOC (NULL
);
936 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
937 TREE_ADDRESSABLE (outer
) = 1;
946 /* Construct a new automatic decl similar to VAR. */
949 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
951 tree copy
= copy_var_decl (var
, name
, type
);
953 DECL_CONTEXT (copy
) = current_function_decl
;
954 DECL_CHAIN (copy
) = ctx
->block_vars
;
955 ctx
->block_vars
= copy
;
961 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
963 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
966 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
969 omp_build_component_ref (tree obj
, tree field
)
971 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
972 if (TREE_THIS_VOLATILE (field
))
973 TREE_THIS_VOLATILE (ret
) |= 1;
974 if (TREE_READONLY (field
))
975 TREE_READONLY (ret
) |= 1;
979 /* Build tree nodes to access the field for VAR on the receiver side. */
982 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
984 tree x
, field
= lookup_field (var
, ctx
);
986 /* If the receiver record type was remapped in the child function,
987 remap the field into the new record type. */
988 x
= maybe_lookup_field (field
, ctx
);
992 x
= build_simple_mem_ref (ctx
->receiver_decl
);
993 x
= omp_build_component_ref (x
, field
);
995 x
= build_simple_mem_ref (x
);
1000 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1001 of a parallel, this is a component reference; for workshare constructs
1002 this is some variable. */
1005 build_outer_var_ref (tree var
, omp_context
*ctx
)
1009 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1011 else if (is_variable_sized (var
))
1013 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1014 x
= build_outer_var_ref (x
, ctx
);
1015 x
= build_simple_mem_ref (x
);
1017 else if (is_taskreg_ctx (ctx
))
1019 bool by_ref
= use_pointer_for_field (var
, NULL
);
1020 x
= build_receiver_ref (var
, by_ref
, ctx
);
1022 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1023 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1025 /* #pragma omp simd isn't a worksharing construct, and can reference even
1026 private vars in its linear etc. clauses. */
1028 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1029 x
= lookup_decl (var
, ctx
->outer
);
1030 else if (ctx
->outer
)
1031 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1035 else if (ctx
->outer
)
1036 x
= lookup_decl (var
, ctx
->outer
);
1037 else if (is_reference (var
))
1038 /* This can happen with orphaned constructs. If var is reference, it is
1039 possible it is shared and as such valid. */
1044 if (is_reference (var
))
1045 x
= build_simple_mem_ref (x
);
1050 /* Build tree nodes to access the field for VAR on the sender side. */
1053 build_sender_ref (tree var
, omp_context
*ctx
)
1055 tree field
= lookup_sfield (var
, ctx
);
1056 return omp_build_component_ref (ctx
->sender_decl
, field
);
1059 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1062 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1064 tree field
, type
, sfield
= NULL_TREE
;
1066 gcc_assert ((mask
& 1) == 0
1067 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1068 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1069 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1071 type
= TREE_TYPE (var
);
1074 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1075 type
= build_pointer_type (build_pointer_type (type
));
1078 type
= build_pointer_type (type
);
1079 else if ((mask
& 3) == 1 && is_reference (var
))
1080 type
= TREE_TYPE (type
);
1082 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1083 FIELD_DECL
, DECL_NAME (var
), type
);
1085 /* Remember what variable this field was created for. This does have a
1086 side effect of making dwarf2out ignore this member, so for helpful
1087 debugging we clear it later in delete_omp_context. */
1088 DECL_ABSTRACT_ORIGIN (field
) = var
;
1089 if (type
== TREE_TYPE (var
))
1091 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1092 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1093 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1096 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1098 if ((mask
& 3) == 3)
1100 insert_field_into_struct (ctx
->record_type
, field
);
1101 if (ctx
->srecord_type
)
1103 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1104 FIELD_DECL
, DECL_NAME (var
), type
);
1105 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1106 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1107 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1108 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1109 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1114 if (ctx
->srecord_type
== NULL_TREE
)
1118 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1119 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1120 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1122 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1123 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1124 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1125 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1126 splay_tree_insert (ctx
->sfield_map
,
1127 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1128 (splay_tree_value
) sfield
);
1132 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1133 : ctx
->srecord_type
, field
);
1137 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1138 (splay_tree_value
) field
);
1139 if ((mask
& 2) && ctx
->sfield_map
)
1140 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1141 (splay_tree_value
) sfield
);
1145 install_var_local (tree var
, omp_context
*ctx
)
1147 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1148 insert_decl_map (&ctx
->cb
, var
, new_var
);
1152 /* Adjust the replacement for DECL in CTX for the new context. This means
1153 copying the DECL_VALUE_EXPR, and fixing up the type. */
1156 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1158 tree new_decl
, size
;
1160 new_decl
= lookup_decl (decl
, ctx
);
1162 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1164 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1165 && DECL_HAS_VALUE_EXPR_P (decl
))
1167 tree ve
= DECL_VALUE_EXPR (decl
);
1168 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1169 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1170 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1173 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1175 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1176 if (size
== error_mark_node
)
1177 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1178 DECL_SIZE (new_decl
) = size
;
1180 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1181 if (size
== error_mark_node
)
1182 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1183 DECL_SIZE_UNIT (new_decl
) = size
;
1187 /* The callback for remap_decl. Search all containing contexts for a
1188 mapping of the variable; this avoids having to duplicate the splay
1189 tree ahead of time. We know a mapping doesn't already exist in the
1190 given context. Create new mappings to implement default semantics. */
1193 omp_copy_decl (tree var
, copy_body_data
*cb
)
1195 omp_context
*ctx
= (omp_context
*) cb
;
1198 if (TREE_CODE (var
) == LABEL_DECL
)
1200 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1201 DECL_CONTEXT (new_var
) = current_function_decl
;
1202 insert_decl_map (&ctx
->cb
, var
, new_var
);
1206 while (!is_taskreg_ctx (ctx
))
1211 new_var
= maybe_lookup_decl (var
, ctx
);
1216 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1219 return error_mark_node
;
1223 /* Debugging dumps for parallel regions. */
1224 void dump_omp_region (FILE *, struct omp_region
*, int);
1225 void debug_omp_region (struct omp_region
*);
1226 void debug_all_omp_regions (void);
1228 /* Dump the parallel region tree rooted at REGION. */
1231 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1233 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1234 gimple_code_name
[region
->type
]);
1237 dump_omp_region (file
, region
->inner
, indent
+ 4);
1241 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1242 region
->cont
->index
);
1246 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1247 region
->exit
->index
);
1249 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1252 dump_omp_region (file
, region
->next
, indent
);
1256 debug_omp_region (struct omp_region
*region
)
1258 dump_omp_region (stderr
, region
, 0);
1262 debug_all_omp_regions (void)
1264 dump_omp_region (stderr
, root_omp_region
, 0);
1268 /* Create a new parallel region starting at STMT inside region PARENT. */
1270 static struct omp_region
*
1271 new_omp_region (basic_block bb
, enum gimple_code type
,
1272 struct omp_region
*parent
)
1274 struct omp_region
*region
= XCNEW (struct omp_region
);
1276 region
->outer
= parent
;
1278 region
->type
= type
;
1282 /* This is a nested region. Add it to the list of inner
1283 regions in PARENT. */
1284 region
->next
= parent
->inner
;
1285 parent
->inner
= region
;
1289 /* This is a toplevel region. Add it to the list of toplevel
1290 regions in ROOT_OMP_REGION. */
1291 region
->next
= root_omp_region
;
1292 root_omp_region
= region
;
1298 /* Release the memory associated with the region tree rooted at REGION. */
1301 free_omp_region_1 (struct omp_region
*region
)
1303 struct omp_region
*i
, *n
;
1305 for (i
= region
->inner
; i
; i
= n
)
1308 free_omp_region_1 (i
);
1314 /* Release the memory for the entire omp region tree. */
1317 free_omp_regions (void)
1319 struct omp_region
*r
, *n
;
1320 for (r
= root_omp_region
; r
; r
= n
)
1323 free_omp_region_1 (r
);
1325 root_omp_region
= NULL
;
1329 /* Create a new context, with OUTER_CTX being the surrounding context. */
1331 static omp_context
*
1332 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1334 omp_context
*ctx
= XCNEW (omp_context
);
1336 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1337 (splay_tree_value
) ctx
);
1342 ctx
->outer
= outer_ctx
;
1343 ctx
->cb
= outer_ctx
->cb
;
1344 ctx
->cb
.block
= NULL
;
1345 ctx
->depth
= outer_ctx
->depth
+ 1;
1349 ctx
->cb
.src_fn
= current_function_decl
;
1350 ctx
->cb
.dst_fn
= current_function_decl
;
1351 ctx
->cb
.src_node
= cgraph_get_node (current_function_decl
);
1352 gcc_checking_assert (ctx
->cb
.src_node
);
1353 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1354 ctx
->cb
.src_cfun
= cfun
;
1355 ctx
->cb
.copy_decl
= omp_copy_decl
;
1356 ctx
->cb
.eh_lp_nr
= 0;
1357 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1361 ctx
->cb
.decl_map
= pointer_map_create ();
1366 static gimple_seq
maybe_catch_exception (gimple_seq
);
1368 /* Finalize task copyfn. */
1371 finalize_task_copyfn (gimple task_stmt
)
1373 struct function
*child_cfun
;
1375 gimple_seq seq
= NULL
, new_seq
;
1378 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1379 if (child_fn
== NULL_TREE
)
1382 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1383 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1385 push_cfun (child_cfun
);
1386 bind
= gimplify_body (child_fn
, false);
1387 gimple_seq_add_stmt (&seq
, bind
);
1388 new_seq
= maybe_catch_exception (seq
);
1391 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1393 gimple_seq_add_stmt (&seq
, bind
);
1395 gimple_set_body (child_fn
, seq
);
1398 /* Inform the callgraph about the new function. */
1399 cgraph_add_new_function (child_fn
, false);
1402 /* Destroy a omp_context data structures. Called through the splay tree
1403 value delete callback. */
1406 delete_omp_context (splay_tree_value value
)
1408 omp_context
*ctx
= (omp_context
*) value
;
1410 pointer_map_destroy (ctx
->cb
.decl_map
);
1413 splay_tree_delete (ctx
->field_map
);
1414 if (ctx
->sfield_map
)
1415 splay_tree_delete (ctx
->sfield_map
);
1417 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1418 it produces corrupt debug information. */
1419 if (ctx
->record_type
)
1422 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1423 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1425 if (ctx
->srecord_type
)
1428 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1429 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1432 if (is_task_ctx (ctx
))
1433 finalize_task_copyfn (ctx
->stmt
);
1438 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1442 fixup_child_record_type (omp_context
*ctx
)
1444 tree f
, type
= ctx
->record_type
;
1446 /* ??? It isn't sufficient to just call remap_type here, because
1447 variably_modified_type_p doesn't work the way we expect for
1448 record types. Testing each field for whether it needs remapping
1449 and creating a new record by hand works, however. */
1450 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1451 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1455 tree name
, new_fields
= NULL
;
1457 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1458 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1459 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1460 TYPE_DECL
, name
, type
);
1461 TYPE_NAME (type
) = name
;
1463 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1465 tree new_f
= copy_node (f
);
1466 DECL_CONTEXT (new_f
) = type
;
1467 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1468 DECL_CHAIN (new_f
) = new_fields
;
1469 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1470 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1472 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1476 /* Arrange to be able to look up the receiver field
1477 given the sender field. */
1478 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1479 (splay_tree_value
) new_f
);
1481 TYPE_FIELDS (type
) = nreverse (new_fields
);
1485 TREE_TYPE (ctx
->receiver_decl
) = build_pointer_type (type
);
1488 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1489 specified by CLAUSES. */
1492 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1495 bool scan_array_reductions
= false;
1497 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1501 switch (OMP_CLAUSE_CODE (c
))
1503 case OMP_CLAUSE_PRIVATE
:
1504 decl
= OMP_CLAUSE_DECL (c
);
1505 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1507 else if (!is_variable_sized (decl
))
1508 install_var_local (decl
, ctx
);
1511 case OMP_CLAUSE_SHARED
:
1512 decl
= OMP_CLAUSE_DECL (c
);
1513 /* Ignore shared directives in teams construct. */
1514 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1516 /* Global variables don't need to be copied,
1517 the receiver side will use them directly. */
1518 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1519 if (is_global_var (odecl
))
1521 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1524 gcc_assert (is_taskreg_ctx (ctx
));
1525 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1526 || !is_variable_sized (decl
));
1527 /* Global variables don't need to be copied,
1528 the receiver side will use them directly. */
1529 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1531 by_ref
= use_pointer_for_field (decl
, ctx
);
1532 if (! TREE_READONLY (decl
)
1533 || TREE_ADDRESSABLE (decl
)
1535 || is_reference (decl
))
1537 install_var_field (decl
, by_ref
, 3, ctx
);
1538 install_var_local (decl
, ctx
);
1541 /* We don't need to copy const scalar vars back. */
1542 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1545 case OMP_CLAUSE_LASTPRIVATE
:
1546 /* Let the corresponding firstprivate clause create
1548 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1552 case OMP_CLAUSE_FIRSTPRIVATE
:
1553 case OMP_CLAUSE_REDUCTION
:
1554 case OMP_CLAUSE_LINEAR
:
1555 decl
= OMP_CLAUSE_DECL (c
);
1557 if (is_variable_sized (decl
))
1559 if (is_task_ctx (ctx
))
1560 install_var_field (decl
, false, 1, ctx
);
1563 else if (is_taskreg_ctx (ctx
))
1566 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1567 by_ref
= use_pointer_for_field (decl
, NULL
);
1569 if (is_task_ctx (ctx
)
1570 && (global
|| by_ref
|| is_reference (decl
)))
1572 install_var_field (decl
, false, 1, ctx
);
1574 install_var_field (decl
, by_ref
, 2, ctx
);
1577 install_var_field (decl
, by_ref
, 3, ctx
);
1579 install_var_local (decl
, ctx
);
1582 case OMP_CLAUSE__LOOPTEMP_
:
1583 gcc_assert (is_parallel_ctx (ctx
));
1584 decl
= OMP_CLAUSE_DECL (c
);
1585 install_var_field (decl
, false, 3, ctx
);
1586 install_var_local (decl
, ctx
);
1589 case OMP_CLAUSE_COPYPRIVATE
:
1590 case OMP_CLAUSE_COPYIN
:
1591 decl
= OMP_CLAUSE_DECL (c
);
1592 by_ref
= use_pointer_for_field (decl
, NULL
);
1593 install_var_field (decl
, by_ref
, 3, ctx
);
1596 case OMP_CLAUSE_DEFAULT
:
1597 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1600 case OMP_CLAUSE_FINAL
:
1602 case OMP_CLAUSE_NUM_THREADS
:
1603 case OMP_CLAUSE_NUM_TEAMS
:
1604 case OMP_CLAUSE_THREAD_LIMIT
:
1605 case OMP_CLAUSE_DEVICE
:
1606 case OMP_CLAUSE_SCHEDULE
:
1607 case OMP_CLAUSE_DIST_SCHEDULE
:
1608 case OMP_CLAUSE_DEPEND
:
1610 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1614 case OMP_CLAUSE_FROM
:
1615 case OMP_CLAUSE_MAP
:
1617 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1618 decl
= OMP_CLAUSE_DECL (c
);
1619 /* Global variables with "omp declare target" attribute
1620 don't need to be copied, the receiver side will use them
1622 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1624 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1625 && lookup_attribute ("omp declare target",
1626 DECL_ATTRIBUTES (decl
)))
1628 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1629 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1631 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1632 #pragma omp target data, there is nothing to map for
1634 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
1635 && !POINTER_TYPE_P (TREE_TYPE (decl
)))
1640 if (DECL_SIZE (decl
)
1641 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1643 tree decl2
= DECL_VALUE_EXPR (decl
);
1644 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1645 decl2
= TREE_OPERAND (decl2
, 0);
1646 gcc_assert (DECL_P (decl2
));
1647 install_var_field (decl2
, true, 3, ctx
);
1648 install_var_local (decl2
, ctx
);
1649 install_var_local (decl
, ctx
);
1653 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1654 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1655 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1656 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1657 install_var_field (decl
, true, 7, ctx
);
1659 install_var_field (decl
, true, 3, ctx
);
1660 if (gimple_omp_target_kind (ctx
->stmt
)
1661 == GF_OMP_TARGET_KIND_REGION
)
1662 install_var_local (decl
, ctx
);
1667 tree base
= get_base_address (decl
);
1668 tree nc
= OMP_CLAUSE_CHAIN (c
);
1671 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1672 && OMP_CLAUSE_DECL (nc
) == base
1673 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1674 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1676 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1677 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1683 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1684 decl
= OMP_CLAUSE_DECL (c
);
1686 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1687 (splay_tree_key
) decl
));
1689 = build_decl (OMP_CLAUSE_LOCATION (c
),
1690 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1691 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1692 insert_field_into_struct (ctx
->record_type
, field
);
1693 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1694 (splay_tree_value
) field
);
1699 case OMP_CLAUSE_NOWAIT
:
1700 case OMP_CLAUSE_ORDERED
:
1701 case OMP_CLAUSE_COLLAPSE
:
1702 case OMP_CLAUSE_UNTIED
:
1703 case OMP_CLAUSE_MERGEABLE
:
1704 case OMP_CLAUSE_PROC_BIND
:
1705 case OMP_CLAUSE_SAFELEN
:
1708 case OMP_CLAUSE_ALIGNED
:
1709 decl
= OMP_CLAUSE_DECL (c
);
1710 if (is_global_var (decl
)
1711 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1712 install_var_local (decl
, ctx
);
1720 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1722 switch (OMP_CLAUSE_CODE (c
))
1724 case OMP_CLAUSE_LASTPRIVATE
:
1725 /* Let the corresponding firstprivate clause create
1727 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1728 scan_array_reductions
= true;
1729 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1733 case OMP_CLAUSE_PRIVATE
:
1734 case OMP_CLAUSE_FIRSTPRIVATE
:
1735 case OMP_CLAUSE_REDUCTION
:
1736 case OMP_CLAUSE_LINEAR
:
1737 decl
= OMP_CLAUSE_DECL (c
);
1738 if (is_variable_sized (decl
))
1739 install_var_local (decl
, ctx
);
1740 fixup_remapped_decl (decl
, ctx
,
1741 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1742 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1743 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1744 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1745 scan_array_reductions
= true;
1746 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1747 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1748 scan_array_reductions
= true;
1751 case OMP_CLAUSE_SHARED
:
1752 /* Ignore shared directives in teams construct. */
1753 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1755 decl
= OMP_CLAUSE_DECL (c
);
1756 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1757 fixup_remapped_decl (decl
, ctx
, false);
1760 case OMP_CLAUSE_MAP
:
1761 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
)
1763 decl
= OMP_CLAUSE_DECL (c
);
1765 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1766 && lookup_attribute ("omp declare target",
1767 DECL_ATTRIBUTES (decl
)))
1771 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1772 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1773 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1775 tree new_decl
= lookup_decl (decl
, ctx
);
1776 TREE_TYPE (new_decl
)
1777 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1779 else if (DECL_SIZE (decl
)
1780 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1782 tree decl2
= DECL_VALUE_EXPR (decl
);
1783 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1784 decl2
= TREE_OPERAND (decl2
, 0);
1785 gcc_assert (DECL_P (decl2
));
1786 fixup_remapped_decl (decl2
, ctx
, false);
1787 fixup_remapped_decl (decl
, ctx
, true);
1790 fixup_remapped_decl (decl
, ctx
, false);
1794 case OMP_CLAUSE_COPYPRIVATE
:
1795 case OMP_CLAUSE_COPYIN
:
1796 case OMP_CLAUSE_DEFAULT
:
1798 case OMP_CLAUSE_NUM_THREADS
:
1799 case OMP_CLAUSE_NUM_TEAMS
:
1800 case OMP_CLAUSE_THREAD_LIMIT
:
1801 case OMP_CLAUSE_DEVICE
:
1802 case OMP_CLAUSE_SCHEDULE
:
1803 case OMP_CLAUSE_DIST_SCHEDULE
:
1804 case OMP_CLAUSE_NOWAIT
:
1805 case OMP_CLAUSE_ORDERED
:
1806 case OMP_CLAUSE_COLLAPSE
:
1807 case OMP_CLAUSE_UNTIED
:
1808 case OMP_CLAUSE_FINAL
:
1809 case OMP_CLAUSE_MERGEABLE
:
1810 case OMP_CLAUSE_PROC_BIND
:
1811 case OMP_CLAUSE_SAFELEN
:
1812 case OMP_CLAUSE_ALIGNED
:
1813 case OMP_CLAUSE_DEPEND
:
1814 case OMP_CLAUSE__LOOPTEMP_
:
1816 case OMP_CLAUSE_FROM
:
1824 if (scan_array_reductions
)
1825 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1826 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1827 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1829 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
1830 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
1832 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
1833 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1834 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
1835 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1836 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1837 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
1840 /* Create a new name for omp child function. Returns an identifier. */
1843 create_omp_child_function_name (bool task_copy
)
1845 return (clone_function_name (current_function_decl
,
1846 task_copy
? "_omp_cpyfn" : "_omp_fn"));
1849 /* Build a decl for the omp child function. It'll not contain a body
1850 yet, just the bare decl. */
1853 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
1855 tree decl
, type
, name
, t
;
1857 name
= create_omp_child_function_name (task_copy
);
1859 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1860 ptr_type_node
, NULL_TREE
);
1862 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
1864 decl
= build_decl (gimple_location (ctx
->stmt
),
1865 FUNCTION_DECL
, name
, type
);
1868 ctx
->cb
.dst_fn
= decl
;
1870 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
1872 TREE_STATIC (decl
) = 1;
1873 TREE_USED (decl
) = 1;
1874 DECL_ARTIFICIAL (decl
) = 1;
1875 DECL_NAMELESS (decl
) = 1;
1876 DECL_IGNORED_P (decl
) = 0;
1877 TREE_PUBLIC (decl
) = 0;
1878 DECL_UNINLINABLE (decl
) = 1;
1879 DECL_EXTERNAL (decl
) = 0;
1880 DECL_CONTEXT (decl
) = NULL_TREE
;
1881 DECL_INITIAL (decl
) = make_node (BLOCK
);
1882 bool target_p
= false;
1883 if (lookup_attribute ("omp declare target",
1884 DECL_ATTRIBUTES (current_function_decl
)))
1889 for (octx
= ctx
; octx
; octx
= octx
->outer
)
1890 if (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
1891 && gimple_omp_target_kind (octx
->stmt
)
1892 == GF_OMP_TARGET_KIND_REGION
)
1899 DECL_ATTRIBUTES (decl
)
1900 = tree_cons (get_identifier ("omp declare target"),
1901 NULL_TREE
, DECL_ATTRIBUTES (decl
));
1903 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1904 RESULT_DECL
, NULL_TREE
, void_type_node
);
1905 DECL_ARTIFICIAL (t
) = 1;
1906 DECL_IGNORED_P (t
) = 1;
1907 DECL_CONTEXT (t
) = decl
;
1908 DECL_RESULT (decl
) = t
;
1910 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1911 PARM_DECL
, get_identifier (".omp_data_i"), ptr_type_node
);
1912 DECL_ARTIFICIAL (t
) = 1;
1913 DECL_NAMELESS (t
) = 1;
1914 DECL_ARG_TYPE (t
) = ptr_type_node
;
1915 DECL_CONTEXT (t
) = current_function_decl
;
1917 DECL_ARGUMENTS (decl
) = t
;
1919 ctx
->receiver_decl
= t
;
1922 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1923 PARM_DECL
, get_identifier (".omp_data_o"),
1925 DECL_ARTIFICIAL (t
) = 1;
1926 DECL_NAMELESS (t
) = 1;
1927 DECL_ARG_TYPE (t
) = ptr_type_node
;
1928 DECL_CONTEXT (t
) = current_function_decl
;
1930 TREE_ADDRESSABLE (t
) = 1;
1931 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1932 DECL_ARGUMENTS (decl
) = t
;
1935 /* Allocate memory for the function structure. The call to
1936 allocate_struct_function clobbers CFUN, so we need to restore
1938 push_struct_function (decl
);
1939 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
1943 /* Callback for walk_gimple_seq. Check if combined parallel
1944 contains gimple_omp_for_combined_into_p OMP_FOR. */
1947 find_combined_for (gimple_stmt_iterator
*gsi_p
,
1948 bool *handled_ops_p
,
1949 struct walk_stmt_info
*wi
)
1951 gimple stmt
= gsi_stmt (*gsi_p
);
1953 *handled_ops_p
= true;
1954 switch (gimple_code (stmt
))
1958 case GIMPLE_OMP_FOR
:
1959 if (gimple_omp_for_combined_into_p (stmt
)
1960 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
1963 return integer_zero_node
;
1972 /* Scan an OpenMP parallel directive. */
1975 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
1979 gimple stmt
= gsi_stmt (*gsi
);
1981 /* Ignore parallel directives with empty bodies, unless there
1982 are copyin clauses. */
1984 && empty_body_p (gimple_omp_body (stmt
))
1985 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
1986 OMP_CLAUSE_COPYIN
) == NULL
)
1988 gsi_replace (gsi
, gimple_build_nop (), false);
1992 if (gimple_omp_parallel_combined_p (stmt
))
1995 struct walk_stmt_info wi
;
1997 memset (&wi
, 0, sizeof (wi
));
1999 walk_gimple_seq (gimple_omp_body (stmt
),
2000 find_combined_for
, NULL
, &wi
);
2001 for_stmt
= (gimple
) wi
.info
;
2004 struct omp_for_data fd
;
2005 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2006 /* We need two temporaries with fd.loop.v type (istart/iend)
2007 and then (fd.collapse - 1) temporaries with the same
2008 type for count2 ... countN-1 vars if not constant. */
2009 size_t count
= 2, i
;
2010 tree type
= fd
.iter_type
;
2012 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2013 count
+= fd
.collapse
- 1;
2014 for (i
= 0; i
< count
; i
++)
2016 tree temp
= create_tmp_var (type
, NULL
);
2017 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2018 OMP_CLAUSE__LOOPTEMP_
);
2019 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2020 OMP_CLAUSE_DECL (c
) = temp
;
2021 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2022 gimple_omp_parallel_set_clauses (stmt
, c
);
2027 ctx
= new_omp_context (stmt
, outer_ctx
);
2028 if (taskreg_nesting_level
> 1)
2029 ctx
->is_nested
= true;
2030 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2031 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2032 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2033 name
= create_tmp_var_name (".omp_data_s");
2034 name
= build_decl (gimple_location (stmt
),
2035 TYPE_DECL
, name
, ctx
->record_type
);
2036 DECL_ARTIFICIAL (name
) = 1;
2037 DECL_NAMELESS (name
) = 1;
2038 TYPE_NAME (ctx
->record_type
) = name
;
2039 create_omp_child_function (ctx
, false);
2040 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2042 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2043 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2045 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2046 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2049 layout_type (ctx
->record_type
);
2050 fixup_child_record_type (ctx
);
2054 /* Scan an OpenMP task directive. */
2057 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2061 gimple stmt
= gsi_stmt (*gsi
);
2062 location_t loc
= gimple_location (stmt
);
2064 /* Ignore task directives with empty bodies. */
2066 && empty_body_p (gimple_omp_body (stmt
)))
2068 gsi_replace (gsi
, gimple_build_nop (), false);
2072 ctx
= new_omp_context (stmt
, outer_ctx
);
2073 if (taskreg_nesting_level
> 1)
2074 ctx
->is_nested
= true;
2075 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2076 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2077 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2078 name
= create_tmp_var_name (".omp_data_s");
2079 name
= build_decl (gimple_location (stmt
),
2080 TYPE_DECL
, name
, ctx
->record_type
);
2081 DECL_ARTIFICIAL (name
) = 1;
2082 DECL_NAMELESS (name
) = 1;
2083 TYPE_NAME (ctx
->record_type
) = name
;
2084 create_omp_child_function (ctx
, false);
2085 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2087 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2089 if (ctx
->srecord_type
)
2091 name
= create_tmp_var_name (".omp_data_a");
2092 name
= build_decl (gimple_location (stmt
),
2093 TYPE_DECL
, name
, ctx
->srecord_type
);
2094 DECL_ARTIFICIAL (name
) = 1;
2095 DECL_NAMELESS (name
) = 1;
2096 TYPE_NAME (ctx
->srecord_type
) = name
;
2097 create_omp_child_function (ctx
, true);
2100 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2102 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2104 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2105 t
= build_int_cst (long_integer_type_node
, 0);
2106 gimple_omp_task_set_arg_size (stmt
, t
);
2107 t
= build_int_cst (long_integer_type_node
, 1);
2108 gimple_omp_task_set_arg_align (stmt
, t
);
2112 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2113 /* Move VLA fields to the end. */
2114 p
= &TYPE_FIELDS (ctx
->record_type
);
2116 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2117 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2120 *p
= TREE_CHAIN (*p
);
2121 TREE_CHAIN (*q
) = NULL_TREE
;
2122 q
= &TREE_CHAIN (*q
);
2125 p
= &DECL_CHAIN (*p
);
2127 layout_type (ctx
->record_type
);
2128 fixup_child_record_type (ctx
);
2129 if (ctx
->srecord_type
)
2130 layout_type (ctx
->srecord_type
);
2131 t
= fold_convert_loc (loc
, long_integer_type_node
,
2132 TYPE_SIZE_UNIT (ctx
->record_type
));
2133 gimple_omp_task_set_arg_size (stmt
, t
);
2134 t
= build_int_cst (long_integer_type_node
,
2135 TYPE_ALIGN_UNIT (ctx
->record_type
));
2136 gimple_omp_task_set_arg_align (stmt
, t
);
2141 /* Scan an OpenMP loop directive. */
2144 scan_omp_for (gimple stmt
, omp_context
*outer_ctx
)
2149 ctx
= new_omp_context (stmt
, outer_ctx
);
2151 scan_sharing_clauses (gimple_omp_for_clauses (stmt
), ctx
);
2153 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2154 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2156 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2157 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2158 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2159 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2161 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2164 /* Scan an OpenMP sections directive. */
2167 scan_omp_sections (gimple stmt
, omp_context
*outer_ctx
)
2171 ctx
= new_omp_context (stmt
, outer_ctx
);
2172 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2173 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2176 /* Scan an OpenMP single directive. */
2179 scan_omp_single (gimple stmt
, omp_context
*outer_ctx
)
2184 ctx
= new_omp_context (stmt
, outer_ctx
);
2185 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2186 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2187 name
= create_tmp_var_name (".omp_copy_s");
2188 name
= build_decl (gimple_location (stmt
),
2189 TYPE_DECL
, name
, ctx
->record_type
);
2190 TYPE_NAME (ctx
->record_type
) = name
;
2192 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2193 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2195 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2196 ctx
->record_type
= NULL
;
2198 layout_type (ctx
->record_type
);
2201 /* Scan an OpenMP target{, data, update} directive. */
2204 scan_omp_target (gimple stmt
, omp_context
*outer_ctx
)
2208 int kind
= gimple_omp_target_kind (stmt
);
2210 ctx
= new_omp_context (stmt
, outer_ctx
);
2211 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2212 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2213 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2214 name
= create_tmp_var_name (".omp_data_t");
2215 name
= build_decl (gimple_location (stmt
),
2216 TYPE_DECL
, name
, ctx
->record_type
);
2217 DECL_ARTIFICIAL (name
) = 1;
2218 DECL_NAMELESS (name
) = 1;
2219 TYPE_NAME (ctx
->record_type
) = name
;
2220 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2222 create_omp_child_function (ctx
, false);
2223 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2226 scan_sharing_clauses (gimple_omp_target_clauses (stmt
), ctx
);
2227 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2229 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2230 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2233 TYPE_FIELDS (ctx
->record_type
)
2234 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2235 #ifdef ENABLE_CHECKING
2237 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2238 for (field
= TYPE_FIELDS (ctx
->record_type
);
2240 field
= DECL_CHAIN (field
))
2241 gcc_assert (DECL_ALIGN (field
) == align
);
2243 layout_type (ctx
->record_type
);
2244 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2245 fixup_child_record_type (ctx
);
2249 /* Scan an OpenMP teams directive. */
2252 scan_omp_teams (gimple stmt
, omp_context
*outer_ctx
)
2254 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2255 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2256 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2259 /* Check OpenMP nesting restrictions. */
2261 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2265 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2266 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2268 error_at (gimple_location (stmt
),
2269 "OpenMP constructs may not be nested inside simd region");
2272 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2274 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2275 || (gimple_omp_for_kind (stmt
)
2276 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2277 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2279 error_at (gimple_location (stmt
),
2280 "only distribute or parallel constructs are allowed to "
2281 "be closely nested inside teams construct");
2286 switch (gimple_code (stmt
))
2288 case GIMPLE_OMP_FOR
:
2289 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
2291 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2293 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2295 error_at (gimple_location (stmt
),
2296 "distribute construct must be closely nested inside "
2304 if (is_gimple_call (stmt
)
2305 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2306 == BUILT_IN_GOMP_CANCEL
2307 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2308 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2310 const char *bad
= NULL
;
2311 const char *kind
= NULL
;
2314 error_at (gimple_location (stmt
), "orphaned %qs construct",
2315 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2316 == BUILT_IN_GOMP_CANCEL
2317 ? "#pragma omp cancel"
2318 : "#pragma omp cancellation point");
2321 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2322 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2326 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2327 bad
= "#pragma omp parallel";
2328 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2329 == BUILT_IN_GOMP_CANCEL
2330 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2331 ctx
->cancellable
= true;
2335 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2336 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2337 bad
= "#pragma omp for";
2338 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2339 == BUILT_IN_GOMP_CANCEL
2340 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2342 ctx
->cancellable
= true;
2343 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2345 warning_at (gimple_location (stmt
), 0,
2346 "%<#pragma omp cancel for%> inside "
2347 "%<nowait%> for construct");
2348 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2349 OMP_CLAUSE_ORDERED
))
2350 warning_at (gimple_location (stmt
), 0,
2351 "%<#pragma omp cancel for%> inside "
2352 "%<ordered%> for construct");
2357 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2358 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2359 bad
= "#pragma omp sections";
2360 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2361 == BUILT_IN_GOMP_CANCEL
2362 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2364 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2366 ctx
->cancellable
= true;
2367 if (find_omp_clause (gimple_omp_sections_clauses
2370 warning_at (gimple_location (stmt
), 0,
2371 "%<#pragma omp cancel sections%> inside "
2372 "%<nowait%> sections construct");
2376 gcc_assert (ctx
->outer
2377 && gimple_code (ctx
->outer
->stmt
)
2378 == GIMPLE_OMP_SECTIONS
);
2379 ctx
->outer
->cancellable
= true;
2380 if (find_omp_clause (gimple_omp_sections_clauses
2383 warning_at (gimple_location (stmt
), 0,
2384 "%<#pragma omp cancel sections%> inside "
2385 "%<nowait%> sections construct");
2391 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2392 bad
= "#pragma omp task";
2394 ctx
->cancellable
= true;
2398 error_at (gimple_location (stmt
), "invalid arguments");
2403 error_at (gimple_location (stmt
),
2404 "%<%s %s%> construct not closely nested inside of %qs",
2405 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2406 == BUILT_IN_GOMP_CANCEL
2407 ? "#pragma omp cancel"
2408 : "#pragma omp cancellation point", kind
, bad
);
2413 case GIMPLE_OMP_SECTIONS
:
2414 case GIMPLE_OMP_SINGLE
:
2415 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2416 switch (gimple_code (ctx
->stmt
))
2418 case GIMPLE_OMP_FOR
:
2419 case GIMPLE_OMP_SECTIONS
:
2420 case GIMPLE_OMP_SINGLE
:
2421 case GIMPLE_OMP_ORDERED
:
2422 case GIMPLE_OMP_MASTER
:
2423 case GIMPLE_OMP_TASK
:
2424 case GIMPLE_OMP_CRITICAL
:
2425 if (is_gimple_call (stmt
))
2427 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2428 != BUILT_IN_GOMP_BARRIER
)
2430 error_at (gimple_location (stmt
),
2431 "barrier region may not be closely nested inside "
2432 "of work-sharing, critical, ordered, master or "
2433 "explicit task region");
2436 error_at (gimple_location (stmt
),
2437 "work-sharing region may not be closely nested inside "
2438 "of work-sharing, critical, ordered, master or explicit "
2441 case GIMPLE_OMP_PARALLEL
:
2447 case GIMPLE_OMP_MASTER
:
2448 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2449 switch (gimple_code (ctx
->stmt
))
2451 case GIMPLE_OMP_FOR
:
2452 case GIMPLE_OMP_SECTIONS
:
2453 case GIMPLE_OMP_SINGLE
:
2454 case GIMPLE_OMP_TASK
:
2455 error_at (gimple_location (stmt
),
2456 "master region may not be closely nested inside "
2457 "of work-sharing or explicit task region");
2459 case GIMPLE_OMP_PARALLEL
:
2465 case GIMPLE_OMP_ORDERED
:
2466 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2467 switch (gimple_code (ctx
->stmt
))
2469 case GIMPLE_OMP_CRITICAL
:
2470 case GIMPLE_OMP_TASK
:
2471 error_at (gimple_location (stmt
),
2472 "ordered region may not be closely nested inside "
2473 "of critical or explicit task region");
2475 case GIMPLE_OMP_FOR
:
2476 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2477 OMP_CLAUSE_ORDERED
) == NULL
)
2479 error_at (gimple_location (stmt
),
2480 "ordered region must be closely nested inside "
2481 "a loop region with an ordered clause");
2485 case GIMPLE_OMP_PARALLEL
:
2486 error_at (gimple_location (stmt
),
2487 "ordered region must be closely nested inside "
2488 "a loop region with an ordered clause");
2494 case GIMPLE_OMP_CRITICAL
:
2495 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2496 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_CRITICAL
2497 && (gimple_omp_critical_name (stmt
)
2498 == gimple_omp_critical_name (ctx
->stmt
)))
2500 error_at (gimple_location (stmt
),
2501 "critical region may not be nested inside a critical "
2502 "region with the same name");
2506 case GIMPLE_OMP_TEAMS
:
2508 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2509 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2511 error_at (gimple_location (stmt
),
2512 "teams construct not closely nested inside of target "
2517 case GIMPLE_OMP_TARGET
:
2518 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2519 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
2520 && gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_REGION
)
2523 switch (gimple_omp_target_kind (stmt
))
2525 case GF_OMP_TARGET_KIND_REGION
: name
= "target"; break;
2526 case GF_OMP_TARGET_KIND_DATA
: name
= "target data"; break;
2527 case GF_OMP_TARGET_KIND_UPDATE
: name
= "target update"; break;
2528 default: gcc_unreachable ();
2530 warning_at (gimple_location (stmt
), 0,
2531 "%s construct inside of target region", name
);
2541 /* Helper function scan_omp.
2543 Callback for walk_tree or operators in walk_gimple_stmt used to
2544 scan for OpenMP directives in TP. */
2547 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
2549 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
2550 omp_context
*ctx
= (omp_context
*) wi
->info
;
2553 switch (TREE_CODE (t
))
2560 *tp
= remap_decl (t
, &ctx
->cb
);
2564 if (ctx
&& TYPE_P (t
))
2565 *tp
= remap_type (t
, &ctx
->cb
);
2566 else if (!DECL_P (t
))
2571 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
2572 if (tem
!= TREE_TYPE (t
))
2574 if (TREE_CODE (t
) == INTEGER_CST
)
2575 *tp
= wide_int_to_tree (tem
, t
);
2577 TREE_TYPE (t
) = tem
;
2587 /* Return true if FNDECL is a setjmp or a longjmp. */
2590 setjmp_or_longjmp_p (const_tree fndecl
)
2592 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
2593 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
2594 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
2597 tree declname
= DECL_NAME (fndecl
);
2600 const char *name
= IDENTIFIER_POINTER (declname
);
2601 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
2605 /* Helper function for scan_omp.
2607 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2608 the current statement in GSI. */
2611 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
2612 struct walk_stmt_info
*wi
)
2614 gimple stmt
= gsi_stmt (*gsi
);
2615 omp_context
*ctx
= (omp_context
*) wi
->info
;
2617 if (gimple_has_location (stmt
))
2618 input_location
= gimple_location (stmt
);
2620 /* Check the OpenMP nesting restrictions. */
2621 bool remove
= false;
2622 if (is_gimple_omp (stmt
))
2623 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2624 else if (is_gimple_call (stmt
))
2626 tree fndecl
= gimple_call_fndecl (stmt
);
2629 if (setjmp_or_longjmp_p (fndecl
)
2631 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2632 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2635 error_at (gimple_location (stmt
),
2636 "setjmp/longjmp inside simd construct");
2638 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
2639 switch (DECL_FUNCTION_CODE (fndecl
))
2641 case BUILT_IN_GOMP_BARRIER
:
2642 case BUILT_IN_GOMP_CANCEL
:
2643 case BUILT_IN_GOMP_CANCELLATION_POINT
:
2644 case BUILT_IN_GOMP_TASKYIELD
:
2645 case BUILT_IN_GOMP_TASKWAIT
:
2646 case BUILT_IN_GOMP_TASKGROUP_START
:
2647 case BUILT_IN_GOMP_TASKGROUP_END
:
2648 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2657 stmt
= gimple_build_nop ();
2658 gsi_replace (gsi
, stmt
, false);
2661 *handled_ops_p
= true;
2663 switch (gimple_code (stmt
))
2665 case GIMPLE_OMP_PARALLEL
:
2666 taskreg_nesting_level
++;
2667 scan_omp_parallel (gsi
, ctx
);
2668 taskreg_nesting_level
--;
2671 case GIMPLE_OMP_TASK
:
2672 taskreg_nesting_level
++;
2673 scan_omp_task (gsi
, ctx
);
2674 taskreg_nesting_level
--;
2677 case GIMPLE_OMP_FOR
:
2678 scan_omp_for (stmt
, ctx
);
2681 case GIMPLE_OMP_SECTIONS
:
2682 scan_omp_sections (stmt
, ctx
);
2685 case GIMPLE_OMP_SINGLE
:
2686 scan_omp_single (stmt
, ctx
);
2689 case GIMPLE_OMP_SECTION
:
2690 case GIMPLE_OMP_MASTER
:
2691 case GIMPLE_OMP_TASKGROUP
:
2692 case GIMPLE_OMP_ORDERED
:
2693 case GIMPLE_OMP_CRITICAL
:
2694 ctx
= new_omp_context (stmt
, ctx
);
2695 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2698 case GIMPLE_OMP_TARGET
:
2699 scan_omp_target (stmt
, ctx
);
2702 case GIMPLE_OMP_TEAMS
:
2703 scan_omp_teams (stmt
, ctx
);
2710 *handled_ops_p
= false;
2712 for (var
= gimple_bind_vars (stmt
); var
; var
= DECL_CHAIN (var
))
2713 insert_decl_map (&ctx
->cb
, var
, var
);
2717 *handled_ops_p
= false;
2725 /* Scan all the statements starting at the current statement. CTX
2726 contains context information about the OpenMP directives and
2727 clauses found during the scan. */
2730 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
2732 location_t saved_location
;
2733 struct walk_stmt_info wi
;
2735 memset (&wi
, 0, sizeof (wi
));
2737 wi
.want_locations
= true;
2739 saved_location
= input_location
;
2740 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
2741 input_location
= saved_location
;
2744 /* Re-gimplification and code generation routines. */
2746 /* Build a call to GOMP_barrier. */
2749 build_omp_barrier (tree lhs
)
2751 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
2752 : BUILT_IN_GOMP_BARRIER
);
2753 gimple g
= gimple_build_call (fndecl
, 0);
2755 gimple_call_set_lhs (g
, lhs
);
2759 /* If a context was created for STMT when it was scanned, return it. */
2761 static omp_context
*
2762 maybe_lookup_ctx (gimple stmt
)
2765 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
2766 return n
? (omp_context
*) n
->value
: NULL
;
2770 /* Find the mapping for DECL in CTX or the immediately enclosing
2771 context that has a mapping for DECL.
2773 If CTX is a nested parallel directive, we may have to use the decl
2774 mappings created in CTX's parent context. Suppose that we have the
2775 following parallel nesting (variable UIDs showed for clarity):
2778 #omp parallel shared(iD.1562) -> outer parallel
2779 iD.1562 = iD.1562 + 1;
2781 #omp parallel shared (iD.1562) -> inner parallel
2782 iD.1562 = iD.1562 - 1;
2784 Each parallel structure will create a distinct .omp_data_s structure
2785 for copying iD.1562 in/out of the directive:
2787 outer parallel .omp_data_s.1.i -> iD.1562
2788 inner parallel .omp_data_s.2.i -> iD.1562
2790 A shared variable mapping will produce a copy-out operation before
2791 the parallel directive and a copy-in operation after it. So, in
2792 this case we would have:
2795 .omp_data_o.1.i = iD.1562;
2796 #omp parallel shared(iD.1562) -> outer parallel
2797 .omp_data_i.1 = &.omp_data_o.1
2798 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2800 .omp_data_o.2.i = iD.1562; -> **
2801 #omp parallel shared(iD.1562) -> inner parallel
2802 .omp_data_i.2 = &.omp_data_o.2
2803 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2806 ** This is a problem. The symbol iD.1562 cannot be referenced
2807 inside the body of the outer parallel region. But since we are
2808 emitting this copy operation while expanding the inner parallel
2809 directive, we need to access the CTX structure of the outer
2810 parallel directive to get the correct mapping:
2812 .omp_data_o.2.i = .omp_data_i.1->i
2814 Since there may be other workshare or parallel directives enclosing
2815 the parallel directive, it may be necessary to walk up the context
2816 parent chain. This is not a problem in general because nested
2817 parallelism happens only rarely. */
2820 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2825 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2826 t
= maybe_lookup_decl (decl
, up
);
2828 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
2830 return t
? t
: decl
;
2834 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2835 in outer contexts. */
2838 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2843 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2844 t
= maybe_lookup_decl (decl
, up
);
2846 return t
? t
: decl
;
2850 /* Construct the initialization value for reduction CLAUSE. */
2853 omp_reduction_init (tree clause
, tree type
)
2855 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
2856 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
2863 case TRUTH_ORIF_EXPR
:
2864 case TRUTH_XOR_EXPR
:
2866 return build_zero_cst (type
);
2869 case TRUTH_AND_EXPR
:
2870 case TRUTH_ANDIF_EXPR
:
2872 return fold_convert_loc (loc
, type
, integer_one_node
);
2875 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
2878 if (SCALAR_FLOAT_TYPE_P (type
))
2880 REAL_VALUE_TYPE max
, min
;
2881 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2884 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
2887 real_maxval (&min
, 1, TYPE_MODE (type
));
2888 return build_real (type
, min
);
2892 gcc_assert (INTEGRAL_TYPE_P (type
));
2893 return TYPE_MIN_VALUE (type
);
2897 if (SCALAR_FLOAT_TYPE_P (type
))
2899 REAL_VALUE_TYPE max
;
2900 if (HONOR_INFINITIES (TYPE_MODE (type
)))
2903 real_maxval (&max
, 0, TYPE_MODE (type
));
2904 return build_real (type
, max
);
2908 gcc_assert (INTEGRAL_TYPE_P (type
));
2909 return TYPE_MAX_VALUE (type
);
2917 /* Return alignment to be assumed for var in CLAUSE, which should be
2918 OMP_CLAUSE_ALIGNED. */
2921 omp_clause_aligned_alignment (tree clause
)
2923 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
2924 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
2926 /* Otherwise return implementation defined alignment. */
2927 unsigned int al
= 1;
2928 enum machine_mode mode
, vmode
;
2929 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2931 vs
= 1 << floor_log2 (vs
);
2932 static enum mode_class classes
[]
2933 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
2934 for (int i
= 0; i
< 4; i
+= 2)
2935 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
2937 mode
= GET_MODE_WIDER_MODE (mode
))
2939 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
2940 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
2943 && GET_MODE_SIZE (vmode
) < vs
2944 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
2945 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
2947 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
2948 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
2950 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
2951 / GET_MODE_SIZE (mode
));
2952 if (TYPE_MODE (type
) != vmode
)
2954 if (TYPE_ALIGN_UNIT (type
) > al
)
2955 al
= TYPE_ALIGN_UNIT (type
);
2957 return build_int_cst (integer_type_node
, al
);
2960 /* Return maximum possible vectorization factor for the target. */
2967 || !flag_tree_loop_optimize
2968 || (!flag_tree_loop_vectorize
2969 && (global_options_set
.x_flag_tree_loop_vectorize
2970 || global_options_set
.x_flag_tree_vectorize
)))
2973 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
2976 vs
= 1 << floor_log2 (vs
);
2979 enum machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
2980 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
2981 return GET_MODE_NUNITS (vqimode
);
2985 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
2989 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
2990 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
2994 max_vf
= omp_max_vf ();
2997 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2998 OMP_CLAUSE_SAFELEN
);
2999 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3001 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3003 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3007 idx
= create_tmp_var (unsigned_type_node
, NULL
);
3008 lane
= create_tmp_var (unsigned_type_node
, NULL
);
3014 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3015 tree avar
= create_tmp_var_raw (atype
, NULL
);
3016 if (TREE_ADDRESSABLE (new_var
))
3017 TREE_ADDRESSABLE (avar
) = 1;
3018 DECL_ATTRIBUTES (avar
)
3019 = tree_cons (get_identifier ("omp simd array"), NULL
,
3020 DECL_ATTRIBUTES (avar
));
3021 gimple_add_tmp_var (avar
);
3022 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3023 NULL_TREE
, NULL_TREE
);
3024 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3025 NULL_TREE
, NULL_TREE
);
3026 if (DECL_P (new_var
))
3028 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3029 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3034 /* Helper function of lower_rec_input_clauses. For a reference
3035 in simd reduction, add an underlying variable it will reference. */
3038 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3040 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3041 if (TREE_CONSTANT (z
))
3043 const char *name
= NULL
;
3044 if (DECL_NAME (new_vard
))
3045 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3047 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3048 gimple_add_tmp_var (z
);
3049 TREE_ADDRESSABLE (z
) = 1;
3050 z
= build_fold_addr_expr_loc (loc
, z
);
3051 gimplify_assign (new_vard
, z
, ilist
);
3055 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3056 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3057 private variables. Initialization statements go in ILIST, while calls
3058 to destructors go in DLIST. */
3061 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3062 omp_context
*ctx
, struct omp_for_data
*fd
)
3064 tree c
, dtor
, copyin_seq
, x
, ptr
;
3065 bool copyin_by_ref
= false;
3066 bool lastprivate_firstprivate
= false;
3067 bool reduction_omp_orig_ref
= false;
3069 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3070 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
3072 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3073 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3074 gimple_seq llist
[2] = { NULL
, NULL
};
3078 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3079 with data sharing clauses referencing variable sized vars. That
3080 is unnecessarily hard to support and very unlikely to result in
3081 vectorized code anyway. */
3083 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3084 switch (OMP_CLAUSE_CODE (c
))
3086 case OMP_CLAUSE_REDUCTION
:
3087 case OMP_CLAUSE_PRIVATE
:
3088 case OMP_CLAUSE_FIRSTPRIVATE
:
3089 case OMP_CLAUSE_LASTPRIVATE
:
3090 case OMP_CLAUSE_LINEAR
:
3091 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3098 /* Do all the fixed sized types in the first pass, and the variable sized
3099 types in the second pass. This makes sure that the scalar arguments to
3100 the variable sized types are processed before we use them in the
3101 variable sized operations. */
3102 for (pass
= 0; pass
< 2; ++pass
)
3104 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3106 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3109 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3113 case OMP_CLAUSE_PRIVATE
:
3114 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3117 case OMP_CLAUSE_SHARED
:
3118 /* Ignore shared directives in teams construct. */
3119 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3121 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3123 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3126 case OMP_CLAUSE_FIRSTPRIVATE
:
3127 case OMP_CLAUSE_COPYIN
:
3128 case OMP_CLAUSE_LINEAR
:
3130 case OMP_CLAUSE_REDUCTION
:
3131 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3132 reduction_omp_orig_ref
= true;
3134 case OMP_CLAUSE__LOOPTEMP_
:
3135 /* Handle _looptemp_ clauses only on parallel. */
3139 case OMP_CLAUSE_LASTPRIVATE
:
3140 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3142 lastprivate_firstprivate
= true;
3146 /* Even without corresponding firstprivate, if
3147 decl is Fortran allocatable, it needs outer var
3150 && lang_hooks
.decls
.omp_private_outer_ref
3151 (OMP_CLAUSE_DECL (c
)))
3152 lastprivate_firstprivate
= true;
3154 case OMP_CLAUSE_ALIGNED
:
3157 var
= OMP_CLAUSE_DECL (c
);
3158 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3159 && !is_global_var (var
))
3161 new_var
= maybe_lookup_decl (var
, ctx
);
3162 if (new_var
== NULL_TREE
)
3163 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3164 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3165 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3166 omp_clause_aligned_alignment (c
));
3167 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3168 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3169 gimplify_and_add (x
, ilist
);
3171 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3172 && is_global_var (var
))
3174 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3175 new_var
= lookup_decl (var
, ctx
);
3176 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3177 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3178 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3179 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3180 omp_clause_aligned_alignment (c
));
3181 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3182 x
= create_tmp_var (ptype
, NULL
);
3183 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3184 gimplify_and_add (t
, ilist
);
3185 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3186 SET_DECL_VALUE_EXPR (new_var
, t
);
3187 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3194 new_var
= var
= OMP_CLAUSE_DECL (c
);
3195 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3196 new_var
= lookup_decl (var
, ctx
);
3198 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3203 else if (is_variable_sized (var
))
3205 /* For variable sized types, we need to allocate the
3206 actual storage here. Call alloca and store the
3207 result in the pointer decl that we created elsewhere. */
3211 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3216 ptr
= DECL_VALUE_EXPR (new_var
);
3217 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3218 ptr
= TREE_OPERAND (ptr
, 0);
3219 gcc_assert (DECL_P (ptr
));
3220 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3222 /* void *tmp = __builtin_alloca */
3223 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3224 stmt
= gimple_build_call (atmp
, 1, x
);
3225 tmp
= create_tmp_var_raw (ptr_type_node
, NULL
);
3226 gimple_add_tmp_var (tmp
);
3227 gimple_call_set_lhs (stmt
, tmp
);
3229 gimple_seq_add_stmt (ilist
, stmt
);
3231 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3232 gimplify_assign (ptr
, x
, ilist
);
3235 else if (is_reference (var
))
3237 /* For references that are being privatized for Fortran,
3238 allocate new backing storage for the new pointer
3239 variable. This allows us to avoid changing all the
3240 code that expects a pointer to something that expects
3241 a direct variable. */
3245 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3246 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3248 x
= build_receiver_ref (var
, false, ctx
);
3249 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3251 else if (TREE_CONSTANT (x
))
3253 /* For reduction in SIMD loop, defer adding the
3254 initialization of the reference, because if we decide
3255 to use SIMD array for it, the initilization could cause
3257 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3261 const char *name
= NULL
;
3262 if (DECL_NAME (var
))
3263 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3265 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3267 gimple_add_tmp_var (x
);
3268 TREE_ADDRESSABLE (x
) = 1;
3269 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3274 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3275 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3280 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3281 gimplify_assign (new_var
, x
, ilist
);
3284 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3286 else if (c_kind
== OMP_CLAUSE_REDUCTION
3287 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3295 switch (OMP_CLAUSE_CODE (c
))
3297 case OMP_CLAUSE_SHARED
:
3298 /* Ignore shared directives in teams construct. */
3299 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3301 /* Shared global vars are just accessed directly. */
3302 if (is_global_var (new_var
))
3304 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3305 needs to be delayed until after fixup_child_record_type so
3306 that we get the correct type during the dereference. */
3307 by_ref
= use_pointer_for_field (var
, ctx
);
3308 x
= build_receiver_ref (var
, by_ref
, ctx
);
3309 SET_DECL_VALUE_EXPR (new_var
, x
);
3310 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3312 /* ??? If VAR is not passed by reference, and the variable
3313 hasn't been initialized yet, then we'll get a warning for
3314 the store into the omp_data_s structure. Ideally, we'd be
3315 able to notice this and not store anything at all, but
3316 we're generating code too early. Suppress the warning. */
3318 TREE_NO_WARNING (var
) = 1;
3321 case OMP_CLAUSE_LASTPRIVATE
:
3322 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3326 case OMP_CLAUSE_PRIVATE
:
3327 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3328 x
= build_outer_var_ref (var
, ctx
);
3329 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3331 if (is_task_ctx (ctx
))
3332 x
= build_receiver_ref (var
, false, ctx
);
3334 x
= build_outer_var_ref (var
, ctx
);
3340 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3343 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3344 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3345 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3346 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3347 idx
, lane
, ivar
, lvar
))
3350 x
= lang_hooks
.decls
.omp_clause_default_ctor
3351 (c
, unshare_expr (ivar
), x
);
3353 gimplify_and_add (x
, &llist
[0]);
3356 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3359 gimple_seq tseq
= NULL
;
3362 gimplify_stmt (&dtor
, &tseq
);
3363 gimple_seq_add_seq (&llist
[1], tseq
);
3370 gimplify_and_add (nx
, ilist
);
3374 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3377 gimple_seq tseq
= NULL
;
3380 gimplify_stmt (&dtor
, &tseq
);
3381 gimple_seq_add_seq (dlist
, tseq
);
3385 case OMP_CLAUSE_LINEAR
:
3386 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3387 goto do_firstprivate
;
3388 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3391 x
= build_outer_var_ref (var
, ctx
);
3394 case OMP_CLAUSE_FIRSTPRIVATE
:
3395 if (is_task_ctx (ctx
))
3397 if (is_reference (var
) || is_variable_sized (var
))
3399 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3401 || use_pointer_for_field (var
, NULL
))
3403 x
= build_receiver_ref (var
, false, ctx
);
3404 SET_DECL_VALUE_EXPR (new_var
, x
);
3405 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3410 x
= build_outer_var_ref (var
, ctx
);
3413 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3414 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3416 tree stept
= POINTER_TYPE_P (TREE_TYPE (x
))
3417 ? sizetype
: TREE_TYPE (x
);
3418 tree t
= fold_convert (stept
,
3419 OMP_CLAUSE_LINEAR_STEP (c
));
3420 tree c
= find_omp_clause (clauses
,
3421 OMP_CLAUSE__LOOPTEMP_
);
3423 tree l
= OMP_CLAUSE_DECL (c
);
3424 tree n1
= fd
->loop
.n1
;
3425 tree step
= fd
->loop
.step
;
3426 tree itype
= TREE_TYPE (l
);
3427 if (POINTER_TYPE_P (itype
))
3428 itype
= signed_type_for (itype
);
3429 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3430 if (TYPE_UNSIGNED (itype
)
3431 && fd
->loop
.cond_code
== GT_EXPR
)
3432 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3433 fold_build1 (NEGATE_EXPR
, itype
, l
),
3434 fold_build1 (NEGATE_EXPR
,
3437 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3438 t
= fold_build2 (MULT_EXPR
, stept
,
3439 fold_convert (stept
, l
), t
);
3440 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3441 x
= fold_build2 (POINTER_PLUS_EXPR
,
3442 TREE_TYPE (x
), x
, t
);
3444 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3447 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3448 || TREE_ADDRESSABLE (new_var
))
3449 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3450 idx
, lane
, ivar
, lvar
))
3452 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3454 tree iv
= create_tmp_var (TREE_TYPE (new_var
), NULL
);
3455 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3456 gimplify_and_add (x
, ilist
);
3457 gimple_stmt_iterator gsi
3458 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3460 = gimple_build_assign (unshare_expr (lvar
), iv
);
3461 gsi_insert_before_without_update (&gsi
, g
,
3463 tree stept
= POINTER_TYPE_P (TREE_TYPE (iv
))
3464 ? sizetype
: TREE_TYPE (iv
);
3465 tree t
= fold_convert (stept
,
3466 OMP_CLAUSE_LINEAR_STEP (c
));
3467 enum tree_code code
= PLUS_EXPR
;
3468 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
3469 code
= POINTER_PLUS_EXPR
;
3470 g
= gimple_build_assign_with_ops (code
, iv
, iv
, t
);
3471 gsi_insert_before_without_update (&gsi
, g
,
3475 x
= lang_hooks
.decls
.omp_clause_copy_ctor
3476 (c
, unshare_expr (ivar
), x
);
3477 gimplify_and_add (x
, &llist
[0]);
3478 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3481 gimple_seq tseq
= NULL
;
3484 gimplify_stmt (&dtor
, &tseq
);
3485 gimple_seq_add_seq (&llist
[1], tseq
);
3490 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
3491 gimplify_and_add (x
, ilist
);
3494 case OMP_CLAUSE__LOOPTEMP_
:
3495 gcc_assert (is_parallel_ctx (ctx
));
3496 x
= build_outer_var_ref (var
, ctx
);
3497 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3498 gimplify_and_add (x
, ilist
);
3501 case OMP_CLAUSE_COPYIN
:
3502 by_ref
= use_pointer_for_field (var
, NULL
);
3503 x
= build_receiver_ref (var
, by_ref
, ctx
);
3504 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
3505 append_to_statement_list (x
, ©in_seq
);
3506 copyin_by_ref
|= by_ref
;
3509 case OMP_CLAUSE_REDUCTION
:
3510 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3512 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3514 x
= build_outer_var_ref (var
, ctx
);
3516 if (is_reference (var
)
3517 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3519 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3520 SET_DECL_VALUE_EXPR (placeholder
, x
);
3521 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3522 tree new_vard
= new_var
;
3523 if (is_reference (var
))
3525 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3526 new_vard
= TREE_OPERAND (new_var
, 0);
3527 gcc_assert (DECL_P (new_vard
));
3530 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3531 idx
, lane
, ivar
, lvar
))
3533 if (new_vard
== new_var
)
3535 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
3536 SET_DECL_VALUE_EXPR (new_var
, ivar
);
3540 SET_DECL_VALUE_EXPR (new_vard
,
3541 build_fold_addr_expr (ivar
));
3542 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3544 x
= lang_hooks
.decls
.omp_clause_default_ctor
3545 (c
, unshare_expr (ivar
),
3546 build_outer_var_ref (var
, ctx
));
3548 gimplify_and_add (x
, &llist
[0]);
3549 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3551 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3552 lower_omp (&tseq
, ctx
);
3553 gimple_seq_add_seq (&llist
[0], tseq
);
3555 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3556 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3557 lower_omp (&tseq
, ctx
);
3558 gimple_seq_add_seq (&llist
[1], tseq
);
3559 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3560 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3561 if (new_vard
== new_var
)
3562 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3564 SET_DECL_VALUE_EXPR (new_vard
,
3565 build_fold_addr_expr (lvar
));
3566 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3571 gimplify_stmt (&dtor
, &tseq
);
3572 gimple_seq_add_seq (&llist
[1], tseq
);
3576 /* If this is a reference to constant size reduction var
3577 with placeholder, we haven't emitted the initializer
3578 for it because it is undesirable if SIMD arrays are used.
3579 But if they aren't used, we need to emit the deferred
3580 initialization now. */
3581 else if (is_reference (var
) && is_simd
)
3582 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3583 x
= lang_hooks
.decls
.omp_clause_default_ctor
3584 (c
, unshare_expr (new_var
),
3585 build_outer_var_ref (var
, ctx
));
3587 gimplify_and_add (x
, ilist
);
3588 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3590 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3591 lower_omp (&tseq
, ctx
);
3592 gimple_seq_add_seq (ilist
, tseq
);
3594 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3597 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3598 lower_omp (&tseq
, ctx
);
3599 gimple_seq_add_seq (dlist
, tseq
);
3600 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3602 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3607 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
3608 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
3609 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3611 /* reduction(-:var) sums up the partial results, so it
3612 acts identically to reduction(+:var). */
3613 if (code
== MINUS_EXPR
)
3616 tree new_vard
= new_var
;
3617 if (is_simd
&& is_reference (var
))
3619 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3620 new_vard
= TREE_OPERAND (new_var
, 0);
3621 gcc_assert (DECL_P (new_vard
));
3624 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3625 idx
, lane
, ivar
, lvar
))
3627 tree ref
= build_outer_var_ref (var
, ctx
);
3629 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
3631 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
3632 ref
= build_outer_var_ref (var
, ctx
);
3633 gimplify_assign (ref
, x
, &llist
[1]);
3635 if (new_vard
!= new_var
)
3637 SET_DECL_VALUE_EXPR (new_vard
,
3638 build_fold_addr_expr (lvar
));
3639 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3644 if (is_reference (var
) && is_simd
)
3645 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3646 gimplify_assign (new_var
, x
, ilist
);
3649 tree ref
= build_outer_var_ref (var
, ctx
);
3651 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3652 ref
= build_outer_var_ref (var
, ctx
);
3653 gimplify_assign (ref
, x
, dlist
);
3667 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
3668 /* Don't want uninit warnings on simduid, it is always uninitialized,
3669 but we use it not for the value, but for the DECL_UID only. */
3670 TREE_NO_WARNING (uid
) = 1;
3672 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
3673 gimple_call_set_lhs (g
, lane
);
3674 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3675 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
3676 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
3677 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
3678 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3679 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3680 g
= gimple_build_assign_with_ops (INTEGER_CST
, lane
,
3681 build_int_cst (unsigned_type_node
, 0),
3683 gimple_seq_add_stmt (ilist
, g
);
3684 for (int i
= 0; i
< 2; i
++)
3687 tree vf
= create_tmp_var (unsigned_type_node
, NULL
);
3688 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
3689 gimple_call_set_lhs (g
, vf
);
3690 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
3691 gimple_seq_add_stmt (seq
, g
);
3692 tree t
= build_int_cst (unsigned_type_node
, 0);
3693 g
= gimple_build_assign_with_ops (INTEGER_CST
, idx
, t
, NULL_TREE
);
3694 gimple_seq_add_stmt (seq
, g
);
3695 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
3696 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
3697 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
3698 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
3699 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
3700 gimple_seq_add_seq (seq
, llist
[i
]);
3701 t
= build_int_cst (unsigned_type_node
, 1);
3702 g
= gimple_build_assign_with_ops (PLUS_EXPR
, idx
, idx
, t
);
3703 gimple_seq_add_stmt (seq
, g
);
3704 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
3705 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
3706 gimple_seq_add_stmt (seq
, g
);
3707 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
3711 /* The copyin sequence is not to be executed by the main thread, since
3712 that would result in self-copies. Perhaps not visible to scalars,
3713 but it certainly is to C++ operator=. */
3716 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
3718 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
3719 build_int_cst (TREE_TYPE (x
), 0));
3720 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
3721 gimplify_and_add (x
, ilist
);
3724 /* If any copyin variable is passed by reference, we must ensure the
3725 master thread doesn't modify it before it is copied over in all
3726 threads. Similarly for variables in both firstprivate and
3727 lastprivate clauses we need to ensure the lastprivate copying
3728 happens after firstprivate copying in all threads. And similarly
3729 for UDRs if initializer expression refers to omp_orig. */
3730 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
3732 /* Don't add any barrier for #pragma omp simd or
3733 #pragma omp distribute. */
3734 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3735 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
3736 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
3739 /* If max_vf is non-zero, then we can use only a vectorization factor
3740 up to the max_vf we chose. So stick it into the safelen clause. */
3743 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3744 OMP_CLAUSE_SAFELEN
);
3746 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
3747 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3750 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
3751 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
3753 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3754 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3760 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3761 both parallel and workshare constructs. PREDICATE may be NULL if it's
3765 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
3768 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
3769 bool par_clauses
= false;
3770 tree simduid
= NULL
, lastlane
= NULL
;
3772 /* Early exit if there are no lastprivate or linear clauses. */
3773 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
3774 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
3775 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
3776 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
3778 if (clauses
== NULL
)
3780 /* If this was a workshare clause, see if it had been combined
3781 with its parallel. In that case, look for the clauses on the
3782 parallel statement itself. */
3783 if (is_parallel_ctx (ctx
))
3787 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3790 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3791 OMP_CLAUSE_LASTPRIVATE
);
3792 if (clauses
== NULL
)
3800 tree label_true
, arm1
, arm2
;
3802 label
= create_artificial_label (UNKNOWN_LOCATION
);
3803 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
3804 arm1
= TREE_OPERAND (predicate
, 0);
3805 arm2
= TREE_OPERAND (predicate
, 1);
3806 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3807 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3808 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
3810 gimple_seq_add_stmt (stmt_list
, stmt
);
3811 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
3814 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3815 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3817 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
3819 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
3822 for (c
= clauses
; c
;)
3825 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3827 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3828 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3829 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
3831 var
= OMP_CLAUSE_DECL (c
);
3832 new_var
= lookup_decl (var
, ctx
);
3834 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
3836 tree val
= DECL_VALUE_EXPR (new_var
);
3837 if (TREE_CODE (val
) == ARRAY_REF
3838 && VAR_P (TREE_OPERAND (val
, 0))
3839 && lookup_attribute ("omp simd array",
3840 DECL_ATTRIBUTES (TREE_OPERAND (val
,
3843 if (lastlane
== NULL
)
3845 lastlane
= create_tmp_var (unsigned_type_node
, NULL
);
3847 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
3849 TREE_OPERAND (val
, 1));
3850 gimple_call_set_lhs (g
, lastlane
);
3851 gimple_seq_add_stmt (stmt_list
, g
);
3853 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
3854 TREE_OPERAND (val
, 0), lastlane
,
3855 NULL_TREE
, NULL_TREE
);
3859 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3860 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
3862 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
3863 gimple_seq_add_seq (stmt_list
,
3864 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
3865 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
3867 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3868 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
3870 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
3871 gimple_seq_add_seq (stmt_list
,
3872 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
3873 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
3876 x
= build_outer_var_ref (var
, ctx
);
3877 if (is_reference (var
))
3878 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3879 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
3880 gimplify_and_add (x
, stmt_list
);
3882 c
= OMP_CLAUSE_CHAIN (c
);
3883 if (c
== NULL
&& !par_clauses
)
3885 /* If this was a workshare clause, see if it had been combined
3886 with its parallel. In that case, continue looking for the
3887 clauses also on the parallel statement itself. */
3888 if (is_parallel_ctx (ctx
))
3892 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3895 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3896 OMP_CLAUSE_LASTPRIVATE
);
3902 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
3906 /* Generate code to implement the REDUCTION clauses. */
3909 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
3911 gimple_seq sub_seq
= NULL
;
3916 /* SIMD reductions are handled in lower_rec_input_clauses. */
3917 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3918 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3921 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
3922 update in that case, otherwise use a lock. */
3923 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
3924 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
3926 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3928 /* Never use OMP_ATOMIC for array reductions or UDRs. */
3938 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3940 tree var
, ref
, new_var
;
3941 enum tree_code code
;
3942 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3944 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
3947 var
= OMP_CLAUSE_DECL (c
);
3948 new_var
= lookup_decl (var
, ctx
);
3949 if (is_reference (var
))
3950 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3951 ref
= build_outer_var_ref (var
, ctx
);
3952 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3954 /* reduction(-:var) sums up the partial results, so it acts
3955 identically to reduction(+:var). */
3956 if (code
== MINUS_EXPR
)
3961 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
3963 addr
= save_expr (addr
);
3964 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
3965 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
3966 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
3967 gimplify_and_add (x
, stmt_seqp
);
3971 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3973 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3975 if (is_reference (var
)
3976 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3978 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
3979 SET_DECL_VALUE_EXPR (placeholder
, ref
);
3980 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3981 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
3982 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
3983 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3984 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
3988 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3989 ref
= build_outer_var_ref (var
, ctx
);
3990 gimplify_assign (ref
, x
, &sub_seq
);
3994 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
3996 gimple_seq_add_stmt (stmt_seqp
, stmt
);
3998 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4000 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4002 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4006 /* Generate code to implement the COPYPRIVATE clauses. */
4009 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4014 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4016 tree var
, new_var
, ref
, x
;
4018 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4020 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4023 var
= OMP_CLAUSE_DECL (c
);
4024 by_ref
= use_pointer_for_field (var
, NULL
);
4026 ref
= build_sender_ref (var
, ctx
);
4027 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4030 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4031 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4033 gimplify_assign (ref
, x
, slist
);
4035 ref
= build_receiver_ref (var
, false, ctx
);
4038 ref
= fold_convert_loc (clause_loc
,
4039 build_pointer_type (TREE_TYPE (new_var
)),
4041 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4043 if (is_reference (var
))
4045 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4046 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4047 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4049 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4050 gimplify_and_add (x
, rlist
);
4055 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4056 and REDUCTION from the sender (aka parent) side. */
4059 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4064 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4066 tree val
, ref
, x
, var
;
4067 bool by_ref
, do_in
= false, do_out
= false;
4068 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4070 switch (OMP_CLAUSE_CODE (c
))
4072 case OMP_CLAUSE_PRIVATE
:
4073 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4076 case OMP_CLAUSE_FIRSTPRIVATE
:
4077 case OMP_CLAUSE_COPYIN
:
4078 case OMP_CLAUSE_LASTPRIVATE
:
4079 case OMP_CLAUSE_REDUCTION
:
4080 case OMP_CLAUSE__LOOPTEMP_
:
4086 val
= OMP_CLAUSE_DECL (c
);
4087 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4089 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4090 && is_global_var (var
))
4092 if (is_variable_sized (val
))
4094 by_ref
= use_pointer_for_field (val
, NULL
);
4096 switch (OMP_CLAUSE_CODE (c
))
4098 case OMP_CLAUSE_PRIVATE
:
4099 case OMP_CLAUSE_FIRSTPRIVATE
:
4100 case OMP_CLAUSE_COPYIN
:
4101 case OMP_CLAUSE__LOOPTEMP_
:
4105 case OMP_CLAUSE_LASTPRIVATE
:
4106 if (by_ref
|| is_reference (val
))
4108 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4115 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4120 case OMP_CLAUSE_REDUCTION
:
4122 do_out
= !(by_ref
|| is_reference (val
));
4131 ref
= build_sender_ref (val
, ctx
);
4132 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4133 gimplify_assign (ref
, x
, ilist
);
4134 if (is_task_ctx (ctx
))
4135 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4140 ref
= build_sender_ref (val
, ctx
);
4141 gimplify_assign (var
, ref
, olist
);
4146 /* Generate code to implement SHARED from the sender (aka parent)
4147 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4148 list things that got automatically shared. */
4151 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4153 tree var
, ovar
, nvar
, f
, x
, record_type
;
4155 if (ctx
->record_type
== NULL
)
4158 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4159 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4161 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4162 nvar
= maybe_lookup_decl (ovar
, ctx
);
4163 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4166 /* If CTX is a nested parallel directive. Find the immediately
4167 enclosing parallel or workshare construct that contains a
4168 mapping for OVAR. */
4169 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4171 if (use_pointer_for_field (ovar
, ctx
))
4173 x
= build_sender_ref (ovar
, ctx
);
4174 var
= build_fold_addr_expr (var
);
4175 gimplify_assign (x
, var
, ilist
);
4179 x
= build_sender_ref (ovar
, ctx
);
4180 gimplify_assign (x
, var
, ilist
);
4182 if (!TREE_READONLY (var
)
4183 /* We don't need to receive a new reference to a result
4184 or parm decl. In fact we may not store to it as we will
4185 invalidate any pending RSO and generate wrong gimple
4187 && !((TREE_CODE (var
) == RESULT_DECL
4188 || TREE_CODE (var
) == PARM_DECL
)
4189 && DECL_BY_REFERENCE (var
)))
4191 x
= build_sender_ref (ovar
, ctx
);
4192 gimplify_assign (var
, x
, olist
);
4199 /* A convenience function to build an empty GIMPLE_COND with just the
4203 gimple_build_cond_empty (tree cond
)
4205 enum tree_code pred_code
;
4208 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4209 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4213 /* Build the function calls to GOMP_parallel_start etc to actually
4214 generate the parallel operation. REGION is the parallel region
4215 being expanded. BB is the block where to insert the code. WS_ARGS
4216 will be set if this is a call to a combined parallel+workshare
4217 construct, it contains the list of additional arguments needed by
4218 the workshare construct. */
4221 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4222 gimple entry_stmt
, vec
<tree
, va_gc
> *ws_args
)
4224 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4225 gimple_stmt_iterator gsi
;
4227 enum built_in_function start_ix
;
4229 location_t clause_loc
;
4230 vec
<tree
, va_gc
> *args
;
4232 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4234 /* Determine what flavor of GOMP_parallel we will be
4236 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4237 if (is_combined_parallel (region
))
4239 switch (region
->inner
->type
)
4241 case GIMPLE_OMP_FOR
:
4242 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4243 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4244 + (region
->inner
->sched_kind
4245 == OMP_CLAUSE_SCHEDULE_RUNTIME
4246 ? 3 : region
->inner
->sched_kind
));
4247 start_ix
= (enum built_in_function
)start_ix2
;
4249 case GIMPLE_OMP_SECTIONS
:
4250 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4257 /* By default, the value of NUM_THREADS is zero (selected at run time)
4258 and there is no conditional. */
4260 val
= build_int_cst (unsigned_type_node
, 0);
4261 flags
= build_int_cst (unsigned_type_node
, 0);
4263 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4265 cond
= OMP_CLAUSE_IF_EXPR (c
);
4267 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4270 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4271 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4274 clause_loc
= gimple_location (entry_stmt
);
4276 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4278 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4280 /* Ensure 'val' is of the correct type. */
4281 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4283 /* If we found the clause 'if (cond)', build either
4284 (cond != 0) or (cond ? val : 1u). */
4287 cond
= gimple_boolify (cond
);
4289 if (integer_zerop (val
))
4290 val
= fold_build2_loc (clause_loc
,
4291 EQ_EXPR
, unsigned_type_node
, cond
,
4292 build_int_cst (TREE_TYPE (cond
), 0));
4295 basic_block cond_bb
, then_bb
, else_bb
;
4296 edge e
, e_then
, e_else
;
4297 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4299 tmp_var
= create_tmp_var (TREE_TYPE (val
), NULL
);
4300 if (gimple_in_ssa_p (cfun
))
4302 tmp_then
= make_ssa_name (tmp_var
, NULL
);
4303 tmp_else
= make_ssa_name (tmp_var
, NULL
);
4304 tmp_join
= make_ssa_name (tmp_var
, NULL
);
4313 e
= split_block (bb
, NULL
);
4318 then_bb
= create_empty_bb (cond_bb
);
4319 else_bb
= create_empty_bb (then_bb
);
4320 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4321 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4323 stmt
= gimple_build_cond_empty (cond
);
4324 gsi
= gsi_start_bb (cond_bb
);
4325 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4327 gsi
= gsi_start_bb (then_bb
);
4328 stmt
= gimple_build_assign (tmp_then
, val
);
4329 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4331 gsi
= gsi_start_bb (else_bb
);
4332 stmt
= gimple_build_assign
4333 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4334 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4336 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4337 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4338 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4339 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4340 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4341 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4343 if (gimple_in_ssa_p (cfun
))
4345 gimple phi
= create_phi_node (tmp_join
, bb
);
4346 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4347 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4353 gsi
= gsi_start_bb (bb
);
4354 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4355 false, GSI_CONTINUE_LINKING
);
4358 gsi
= gsi_last_bb (bb
);
4359 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4361 t1
= null_pointer_node
;
4363 t1
= build_fold_addr_expr (t
);
4364 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4366 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4367 args
->quick_push (t2
);
4368 args
->quick_push (t1
);
4369 args
->quick_push (val
);
4371 args
->splice (*ws_args
);
4372 args
->quick_push (flags
);
4374 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4375 builtin_decl_explicit (start_ix
), args
);
4377 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4378 false, GSI_CONTINUE_LINKING
);
4382 /* Build the function call to GOMP_task to actually
4383 generate the task operation. BB is the block where to insert the code. */
4386 expand_task_call (basic_block bb
, gimple entry_stmt
)
4388 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
4389 gimple_stmt_iterator gsi
;
4390 location_t loc
= gimple_location (entry_stmt
);
4392 clauses
= gimple_omp_task_clauses (entry_stmt
);
4394 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4396 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
4398 cond
= boolean_true_node
;
4400 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
4401 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
4402 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
4403 flags
= build_int_cst (unsigned_type_node
,
4404 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
4406 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
4409 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
4410 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
4411 build_int_cst (unsigned_type_node
, 2),
4412 build_int_cst (unsigned_type_node
, 0));
4413 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
4416 depend
= OMP_CLAUSE_DECL (depend
);
4418 depend
= build_int_cst (ptr_type_node
, 0);
4420 gsi
= gsi_last_bb (bb
);
4421 t
= gimple_omp_task_data_arg (entry_stmt
);
4423 t2
= null_pointer_node
;
4425 t2
= build_fold_addr_expr_loc (loc
, t
);
4426 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
4427 t
= gimple_omp_task_copy_fn (entry_stmt
);
4429 t3
= null_pointer_node
;
4431 t3
= build_fold_addr_expr_loc (loc
, t
);
4433 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
4435 gimple_omp_task_arg_size (entry_stmt
),
4436 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
4439 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4440 false, GSI_CONTINUE_LINKING
);
4444 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4445 catch handler and return it. This prevents programs from violating the
4446 structured block semantics with throws. */
4449 maybe_catch_exception (gimple_seq body
)
4454 if (!flag_exceptions
)
4457 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
4458 decl
= lang_hooks
.eh_protect_cleanup_actions ();
4460 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
4462 g
= gimple_build_eh_must_not_throw (decl
);
4463 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
4466 return gimple_seq_alloc_with_stmt (g
);
4469 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4472 vec2chain (vec
<tree
, va_gc
> *v
)
4474 tree chain
= NULL_TREE
, t
;
4477 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
4479 DECL_CHAIN (t
) = chain
;
4487 /* Remove barriers in REGION->EXIT's block. Note that this is only
4488 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4489 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4490 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4494 remove_exit_barrier (struct omp_region
*region
)
4496 gimple_stmt_iterator gsi
;
4497 basic_block exit_bb
;
4501 int any_addressable_vars
= -1;
4503 exit_bb
= region
->exit
;
4505 /* If the parallel region doesn't return, we don't have REGION->EXIT
4510 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4511 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4512 statements that can appear in between are extremely limited -- no
4513 memory operations at all. Here, we allow nothing at all, so the
4514 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4515 gsi
= gsi_last_bb (exit_bb
);
4516 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4518 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
4521 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
4523 gsi
= gsi_last_bb (e
->src
);
4524 if (gsi_end_p (gsi
))
4526 stmt
= gsi_stmt (gsi
);
4527 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
4528 && !gimple_omp_return_nowait_p (stmt
))
4530 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4531 in many cases. If there could be tasks queued, the barrier
4532 might be needed to let the tasks run before some local
4533 variable of the parallel that the task uses as shared
4534 runs out of scope. The task can be spawned either
4535 from within current function (this would be easy to check)
4536 or from some function it calls and gets passed an address
4537 of such a variable. */
4538 if (any_addressable_vars
< 0)
4540 gimple parallel_stmt
= last_stmt (region
->entry
);
4541 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
4542 tree local_decls
, block
, decl
;
4545 any_addressable_vars
= 0;
4546 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
4547 if (TREE_ADDRESSABLE (decl
))
4549 any_addressable_vars
= 1;
4552 for (block
= gimple_block (stmt
);
4553 !any_addressable_vars
4555 && TREE_CODE (block
) == BLOCK
;
4556 block
= BLOCK_SUPERCONTEXT (block
))
4558 for (local_decls
= BLOCK_VARS (block
);
4560 local_decls
= DECL_CHAIN (local_decls
))
4561 if (TREE_ADDRESSABLE (local_decls
))
4563 any_addressable_vars
= 1;
4566 if (block
== gimple_block (parallel_stmt
))
4570 if (!any_addressable_vars
)
4571 gimple_omp_return_set_nowait (stmt
);
4577 remove_exit_barriers (struct omp_region
*region
)
4579 if (region
->type
== GIMPLE_OMP_PARALLEL
)
4580 remove_exit_barrier (region
);
4584 region
= region
->inner
;
4585 remove_exit_barriers (region
);
4586 while (region
->next
)
4588 region
= region
->next
;
4589 remove_exit_barriers (region
);
4594 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4595 calls. These can't be declared as const functions, but
4596 within one parallel body they are constant, so they can be
4597 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4598 which are declared const. Similarly for task body, except
4599 that in untied task omp_get_thread_num () can change at any task
4600 scheduling point. */
4603 optimize_omp_library_calls (gimple entry_stmt
)
4606 gimple_stmt_iterator gsi
;
4607 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4608 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
4609 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4610 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
4611 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
4612 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
4613 OMP_CLAUSE_UNTIED
) != NULL
);
4615 FOR_EACH_BB_FN (bb
, cfun
)
4616 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4618 gimple call
= gsi_stmt (gsi
);
4621 if (is_gimple_call (call
)
4622 && (decl
= gimple_call_fndecl (call
))
4623 && DECL_EXTERNAL (decl
)
4624 && TREE_PUBLIC (decl
)
4625 && DECL_INITIAL (decl
) == NULL
)
4629 if (DECL_NAME (decl
) == thr_num_id
)
4631 /* In #pragma omp task untied omp_get_thread_num () can change
4632 during the execution of the task region. */
4635 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4637 else if (DECL_NAME (decl
) == num_thr_id
)
4638 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4642 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
4643 || gimple_call_num_args (call
) != 0)
4646 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
4649 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
4650 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
4651 TREE_TYPE (TREE_TYPE (built_in
))))
4654 gimple_call_set_fndecl (call
, built_in
);
4659 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4663 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
4667 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4668 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
4671 if (TREE_CODE (t
) == ADDR_EXPR
)
4672 recompute_tree_invariant_for_addr_expr (t
);
4674 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
4678 /* Prepend TO = FROM assignment before *GSI_P. */
4681 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
4683 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
4684 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
4685 true, GSI_SAME_STMT
);
4686 gimple stmt
= gimple_build_assign (to
, from
);
4687 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
4688 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
4689 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
4691 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
4692 gimple_regimplify_operands (stmt
, &gsi
);
4696 /* Expand the OpenMP parallel or task directive starting at REGION. */
4699 expand_omp_taskreg (struct omp_region
*region
)
4701 basic_block entry_bb
, exit_bb
, new_bb
;
4702 struct function
*child_cfun
;
4703 tree child_fn
, block
, t
;
4704 gimple_stmt_iterator gsi
;
4705 gimple entry_stmt
, stmt
;
4707 vec
<tree
, va_gc
> *ws_args
;
4709 entry_stmt
= last_stmt (region
->entry
);
4710 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
4711 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
4713 entry_bb
= region
->entry
;
4714 exit_bb
= region
->exit
;
4716 if (is_combined_parallel (region
))
4717 ws_args
= region
->ws_args
;
4721 if (child_cfun
->cfg
)
4723 /* Due to inlining, it may happen that we have already outlined
4724 the region, in which case all we need to do is make the
4725 sub-graph unreachable and emit the parallel call. */
4726 edge entry_succ_e
, exit_succ_e
;
4728 entry_succ_e
= single_succ_edge (entry_bb
);
4730 gsi
= gsi_last_bb (entry_bb
);
4731 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
4732 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
4733 gsi_remove (&gsi
, true);
4738 exit_succ_e
= single_succ_edge (exit_bb
);
4739 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
4741 remove_edge_and_dominated_blocks (entry_succ_e
);
4745 unsigned srcidx
, dstidx
, num
;
4747 /* If the parallel region needs data sent from the parent
4748 function, then the very first statement (except possible
4749 tree profile counter updates) of the parallel body
4750 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4751 &.OMP_DATA_O is passed as an argument to the child function,
4752 we need to replace it with the argument as seen by the child
4755 In most cases, this will end up being the identity assignment
4756 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4757 a function call that has been inlined, the original PARM_DECL
4758 .OMP_DATA_I may have been converted into a different local
4759 variable. In which case, we need to keep the assignment. */
4760 if (gimple_omp_taskreg_data_arg (entry_stmt
))
4762 basic_block entry_succ_bb
= single_succ (entry_bb
);
4764 gimple parcopy_stmt
= NULL
;
4766 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
4770 gcc_assert (!gsi_end_p (gsi
));
4771 stmt
= gsi_stmt (gsi
);
4772 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
4775 if (gimple_num_ops (stmt
) == 2)
4777 tree arg
= gimple_assign_rhs1 (stmt
);
4779 /* We're ignore the subcode because we're
4780 effectively doing a STRIP_NOPS. */
4782 if (TREE_CODE (arg
) == ADDR_EXPR
4783 && TREE_OPERAND (arg
, 0)
4784 == gimple_omp_taskreg_data_arg (entry_stmt
))
4786 parcopy_stmt
= stmt
;
4792 gcc_assert (parcopy_stmt
!= NULL
);
4793 arg
= DECL_ARGUMENTS (child_fn
);
4795 if (!gimple_in_ssa_p (cfun
))
4797 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
4798 gsi_remove (&gsi
, true);
4801 /* ?? Is setting the subcode really necessary ?? */
4802 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
4803 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
4808 /* If we are in ssa form, we must load the value from the default
4809 definition of the argument. That should not be defined now,
4810 since the argument is not used uninitialized. */
4811 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
4812 narg
= make_ssa_name (arg
, gimple_build_nop ());
4813 set_ssa_default_def (cfun
, arg
, narg
);
4814 /* ?? Is setting the subcode really necessary ?? */
4815 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
4816 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
4817 update_stmt (parcopy_stmt
);
4821 /* Declare local variables needed in CHILD_CFUN. */
4822 block
= DECL_INITIAL (child_fn
);
4823 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
4824 /* The gimplifier could record temporaries in parallel/task block
4825 rather than in containing function's local_decls chain,
4826 which would mean cgraph missed finalizing them. Do it now. */
4827 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
4828 if (TREE_CODE (t
) == VAR_DECL
4830 && !DECL_EXTERNAL (t
))
4831 varpool_finalize_decl (t
);
4832 DECL_SAVED_TREE (child_fn
) = NULL
;
4833 /* We'll create a CFG for child_fn, so no gimple body is needed. */
4834 gimple_set_body (child_fn
, NULL
);
4835 TREE_USED (block
) = 1;
4837 /* Reset DECL_CONTEXT on function arguments. */
4838 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
4839 DECL_CONTEXT (t
) = child_fn
;
4841 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
4842 so that it can be moved to the child function. */
4843 gsi
= gsi_last_bb (entry_bb
);
4844 stmt
= gsi_stmt (gsi
);
4845 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
4846 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
4847 gsi_remove (&gsi
, true);
4848 e
= split_block (entry_bb
, stmt
);
4850 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
4852 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
4855 gsi
= gsi_last_bb (exit_bb
);
4856 gcc_assert (!gsi_end_p (gsi
)
4857 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4858 stmt
= gimple_build_return (NULL
);
4859 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
4860 gsi_remove (&gsi
, true);
4863 /* Move the parallel region into CHILD_CFUN. */
4865 if (gimple_in_ssa_p (cfun
))
4867 init_tree_ssa (child_cfun
);
4868 init_ssa_operands (child_cfun
);
4869 child_cfun
->gimple_df
->in_ssa_p
= true;
4873 block
= gimple_block (entry_stmt
);
4875 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
4877 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
4878 /* When the OMP expansion process cannot guarantee an up-to-date
4879 loop tree arrange for the child function to fixup loops. */
4880 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
4881 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
4883 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
4884 num
= vec_safe_length (child_cfun
->local_decls
);
4885 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
4887 t
= (*child_cfun
->local_decls
)[srcidx
];
4888 if (DECL_CONTEXT (t
) == cfun
->decl
)
4890 if (srcidx
!= dstidx
)
4891 (*child_cfun
->local_decls
)[dstidx
] = t
;
4895 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
4897 /* Inform the callgraph about the new function. */
4898 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
4899 cgraph_add_new_function (child_fn
, true);
4901 /* Fix the callgraph edges for child_cfun. Those for cfun will be
4902 fixed in a following pass. */
4903 push_cfun (child_cfun
);
4905 optimize_omp_library_calls (entry_stmt
);
4906 rebuild_cgraph_edges ();
4908 /* Some EH regions might become dead, see PR34608. If
4909 pass_cleanup_cfg isn't the first pass to happen with the
4910 new child, these dead EH edges might cause problems.
4911 Clean them up now. */
4912 if (flag_exceptions
)
4915 bool changed
= false;
4917 FOR_EACH_BB_FN (bb
, cfun
)
4918 changed
|= gimple_purge_dead_eh_edges (bb
);
4920 cleanup_tree_cfg ();
4922 if (gimple_in_ssa_p (cfun
))
4923 update_ssa (TODO_update_ssa
);
4927 /* Emit a library call to launch the children threads. */
4928 if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
4929 expand_parallel_call (region
, new_bb
, entry_stmt
, ws_args
);
4931 expand_task_call (new_bb
, entry_stmt
);
4932 if (gimple_in_ssa_p (cfun
))
4933 update_ssa (TODO_update_ssa_only_virtuals
);
4937 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
4938 of the combined collapse > 1 loop constructs, generate code like:
4939 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
4944 count3 = (adj + N32 - N31) / STEP3;
4945 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
4950 count2 = (adj + N22 - N21) / STEP2;
4951 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
4956 count1 = (adj + N12 - N11) / STEP1;
4957 count = count1 * count2 * count3;
4958 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
4960 and set ZERO_ITER_BB to that bb. If this isn't the outermost
4961 of the combined loop constructs, just initialize COUNTS array
4962 from the _looptemp_ clauses. */
4964 /* NOTE: It *could* be better to moosh all of the BBs together,
4965 creating one larger BB with all the computation and the unexpected
4966 jump at the end. I.e.
4968 bool zero3, zero2, zero1, zero;
4971 count3 = (N32 - N31) /[cl] STEP3;
4973 count2 = (N22 - N21) /[cl] STEP2;
4975 count1 = (N12 - N11) /[cl] STEP1;
4976 zero = zero3 || zero2 || zero1;
4977 count = count1 * count2 * count3;
4978 if (__builtin_expect(zero, false)) goto zero_iter_bb;
4980 After all, we expect the zero=false, and thus we expect to have to
4981 evaluate all of the comparison expressions, so short-circuiting
4982 oughtn't be a win. Since the condition isn't protecting a
4983 denominator, we're not concerned about divide-by-zero, so we can
4984 fully evaluate count even if a numerator turned out to be wrong.
4986 It seems like putting this all together would create much better
4987 scheduling opportunities, and less pressure on the chip's branch
4991 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
4992 basic_block
&entry_bb
, tree
*counts
,
4993 basic_block
&zero_iter_bb
, int &first_zero_iter
,
4994 basic_block
&l2_dom_bb
)
4996 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5001 /* Collapsed loops need work for expansion into SSA form. */
5002 gcc_assert (!gimple_in_ssa_p (cfun
));
5004 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5005 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5007 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5008 isn't supposed to be handled, as the inner loop doesn't
5010 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5011 OMP_CLAUSE__LOOPTEMP_
);
5012 gcc_assert (innerc
);
5013 for (i
= 0; i
< fd
->collapse
; i
++)
5015 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5016 OMP_CLAUSE__LOOPTEMP_
);
5017 gcc_assert (innerc
);
5019 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5021 counts
[0] = NULL_TREE
;
5026 for (i
= 0; i
< fd
->collapse
; i
++)
5028 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5030 if (SSA_VAR_P (fd
->loop
.n2
)
5031 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5032 fold_convert (itype
, fd
->loops
[i
].n1
),
5033 fold_convert (itype
, fd
->loops
[i
].n2
)))
5034 == NULL_TREE
|| !integer_onep (t
)))
5037 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5038 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5039 true, GSI_SAME_STMT
);
5040 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5041 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5042 true, GSI_SAME_STMT
);
5043 stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5044 NULL_TREE
, NULL_TREE
);
5045 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5046 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5047 expand_omp_regimplify_p
, NULL
, NULL
)
5048 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5049 expand_omp_regimplify_p
, NULL
, NULL
))
5051 *gsi
= gsi_for_stmt (stmt
);
5052 gimple_regimplify_operands (stmt
, gsi
);
5054 e
= split_block (entry_bb
, stmt
);
5055 if (zero_iter_bb
== NULL
)
5057 first_zero_iter
= i
;
5058 zero_iter_bb
= create_empty_bb (entry_bb
);
5059 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5060 *gsi
= gsi_after_labels (zero_iter_bb
);
5061 stmt
= gimple_build_assign (fd
->loop
.n2
,
5062 build_zero_cst (type
));
5063 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5064 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5067 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5068 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5069 e
->flags
= EDGE_TRUE_VALUE
;
5070 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5071 if (l2_dom_bb
== NULL
)
5072 l2_dom_bb
= entry_bb
;
5074 *gsi
= gsi_last_bb (entry_bb
);
5077 if (POINTER_TYPE_P (itype
))
5078 itype
= signed_type_for (itype
);
5079 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5081 t
= fold_build2 (PLUS_EXPR
, itype
,
5082 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5083 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5084 fold_convert (itype
, fd
->loops
[i
].n2
));
5085 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5086 fold_convert (itype
, fd
->loops
[i
].n1
));
5087 /* ?? We could probably use CEIL_DIV_EXPR instead of
5088 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5089 generate the same code in the end because generically we
5090 don't know that the values involved must be negative for
5092 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5093 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5094 fold_build1 (NEGATE_EXPR
, itype
, t
),
5095 fold_build1 (NEGATE_EXPR
, itype
,
5096 fold_convert (itype
,
5097 fd
->loops
[i
].step
)));
5099 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5100 fold_convert (itype
, fd
->loops
[i
].step
));
5101 t
= fold_convert (type
, t
);
5102 if (TREE_CODE (t
) == INTEGER_CST
)
5106 counts
[i
] = create_tmp_reg (type
, ".count");
5107 expand_omp_build_assign (gsi
, counts
[i
], t
);
5109 if (SSA_VAR_P (fd
->loop
.n2
))
5114 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5115 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5121 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5123 V3 = N31 + (T % count3) * STEP3;
5125 V2 = N21 + (T % count2) * STEP2;
5127 V1 = N11 + T * STEP1;
5128 if this loop doesn't have an inner loop construct combined with it.
5129 If it does have an inner loop construct combined with it and the
5130 iteration count isn't known constant, store values from counts array
5131 into its _looptemp_ temporaries instead. */
5134 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5135 tree
*counts
, gimple inner_stmt
, tree startvar
)
5138 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5140 /* If fd->loop.n2 is constant, then no propagation of the counts
5141 is needed, they are constant. */
5142 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5145 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5146 ? gimple_omp_parallel_clauses (inner_stmt
)
5147 : gimple_omp_for_clauses (inner_stmt
);
5148 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5149 isn't supposed to be handled, as the inner loop doesn't
5151 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5152 gcc_assert (innerc
);
5153 for (i
= 0; i
< fd
->collapse
; i
++)
5155 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5156 OMP_CLAUSE__LOOPTEMP_
);
5157 gcc_assert (innerc
);
5160 tree tem
= OMP_CLAUSE_DECL (innerc
);
5161 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5162 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5163 false, GSI_CONTINUE_LINKING
);
5164 gimple stmt
= gimple_build_assign (tem
, t
);
5165 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5171 tree type
= TREE_TYPE (fd
->loop
.v
);
5172 tree tem
= create_tmp_reg (type
, ".tem");
5173 gimple stmt
= gimple_build_assign (tem
, startvar
);
5174 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5176 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5178 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5180 if (POINTER_TYPE_P (vtype
))
5181 itype
= signed_type_for (vtype
);
5183 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5186 t
= fold_convert (itype
, t
);
5187 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5188 fold_convert (itype
, fd
->loops
[i
].step
));
5189 if (POINTER_TYPE_P (vtype
))
5190 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5192 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5193 t
= force_gimple_operand_gsi (gsi
, t
,
5194 DECL_P (fd
->loops
[i
].v
)
5195 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5197 GSI_CONTINUE_LINKING
);
5198 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5199 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5202 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5203 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5204 false, GSI_CONTINUE_LINKING
);
5205 stmt
= gimple_build_assign (tem
, t
);
5206 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5212 /* Helper function for expand_omp_for_*. Generate code like:
5215 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5219 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5226 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5227 basic_block body_bb
)
5229 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5231 gimple_stmt_iterator gsi
;
5237 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5239 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5241 bb
= create_empty_bb (last_bb
);
5242 add_bb_to_loop (bb
, last_bb
->loop_father
);
5243 gsi
= gsi_start_bb (bb
);
5245 if (i
< fd
->collapse
- 1)
5247 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5248 e
->probability
= REG_BR_PROB_BASE
/ 8;
5250 t
= fd
->loops
[i
+ 1].n1
;
5251 t
= force_gimple_operand_gsi (&gsi
, t
,
5252 DECL_P (fd
->loops
[i
+ 1].v
)
5253 && TREE_ADDRESSABLE (fd
->loops
[i
5256 GSI_CONTINUE_LINKING
);
5257 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5258 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5263 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5265 if (POINTER_TYPE_P (vtype
))
5266 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5268 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5269 t
= force_gimple_operand_gsi (&gsi
, t
,
5270 DECL_P (fd
->loops
[i
].v
)
5271 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5272 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5273 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5274 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5278 t
= fd
->loops
[i
].n2
;
5279 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5280 false, GSI_CONTINUE_LINKING
);
5281 tree v
= fd
->loops
[i
].v
;
5282 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5283 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5284 false, GSI_CONTINUE_LINKING
);
5285 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5286 stmt
= gimple_build_cond_empty (t
);
5287 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5288 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5289 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5292 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5300 /* A subroutine of expand_omp_for. Generate code for a parallel
5301 loop with any schedule. Given parameters:
5303 for (V = N1; V cond N2; V += STEP) BODY;
5305 where COND is "<" or ">", we generate pseudocode
5307 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5308 if (more) goto L0; else goto L3;
5315 if (V cond iend) goto L1; else goto L2;
5317 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5320 If this is a combined omp parallel loop, instead of the call to
5321 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5322 If this is gimple_omp_for_combined_p loop, then instead of assigning
5323 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5324 inner GIMPLE_OMP_FOR and V += STEP; and
5325 if (V cond iend) goto L1; else goto L2; are removed.
5327 For collapsed loops, given parameters:
5329 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5330 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5331 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5334 we generate pseudocode
5336 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5341 count3 = (adj + N32 - N31) / STEP3;
5342 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5347 count2 = (adj + N22 - N21) / STEP2;
5348 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5353 count1 = (adj + N12 - N11) / STEP1;
5354 count = count1 * count2 * count3;
5359 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5360 if (more) goto L0; else goto L3;
5364 V3 = N31 + (T % count3) * STEP3;
5366 V2 = N21 + (T % count2) * STEP2;
5368 V1 = N11 + T * STEP1;
5373 if (V < iend) goto L10; else goto L2;
5376 if (V3 cond3 N32) goto L1; else goto L11;
5380 if (V2 cond2 N22) goto L1; else goto L12;
5386 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5392 expand_omp_for_generic (struct omp_region
*region
,
5393 struct omp_for_data
*fd
,
5394 enum built_in_function start_fn
,
5395 enum built_in_function next_fn
,
5398 tree type
, istart0
, iend0
, iend
;
5399 tree t
, vmain
, vback
, bias
= NULL_TREE
;
5400 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
5401 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
5402 gimple_stmt_iterator gsi
;
5404 bool in_combined_parallel
= is_combined_parallel (region
);
5405 bool broken_loop
= region
->cont
== NULL
;
5407 tree
*counts
= NULL
;
5410 gcc_assert (!broken_loop
|| !in_combined_parallel
);
5411 gcc_assert (fd
->iter_type
== long_integer_type_node
5412 || !in_combined_parallel
);
5414 type
= TREE_TYPE (fd
->loop
.v
);
5415 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
5416 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
5417 TREE_ADDRESSABLE (istart0
) = 1;
5418 TREE_ADDRESSABLE (iend0
) = 1;
5420 /* See if we need to bias by LLONG_MIN. */
5421 if (fd
->iter_type
== long_long_unsigned_type_node
5422 && TREE_CODE (type
) == INTEGER_TYPE
5423 && !TYPE_UNSIGNED (type
))
5427 if (fd
->loop
.cond_code
== LT_EXPR
)
5430 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5434 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5437 if (TREE_CODE (n1
) != INTEGER_CST
5438 || TREE_CODE (n2
) != INTEGER_CST
5439 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
5440 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
5443 entry_bb
= region
->entry
;
5444 cont_bb
= region
->cont
;
5446 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5447 gcc_assert (broken_loop
5448 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
5449 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5450 l1_bb
= single_succ (l0_bb
);
5453 l2_bb
= create_empty_bb (cont_bb
);
5454 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
5455 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5459 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5460 exit_bb
= region
->exit
;
5462 gsi
= gsi_last_bb (entry_bb
);
5464 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5465 if (fd
->collapse
> 1)
5467 int first_zero_iter
= -1;
5468 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
5470 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5471 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5472 zero_iter_bb
, first_zero_iter
,
5477 /* Some counts[i] vars might be uninitialized if
5478 some loop has zero iterations. But the body shouldn't
5479 be executed in that case, so just avoid uninit warnings. */
5480 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
5481 if (SSA_VAR_P (counts
[i
]))
5482 TREE_NO_WARNING (counts
[i
]) = 1;
5484 e
= split_block (entry_bb
, gsi_stmt (gsi
));
5486 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
5487 gsi
= gsi_last_bb (entry_bb
);
5488 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
5489 get_immediate_dominator (CDI_DOMINATORS
,
5493 if (in_combined_parallel
)
5495 /* In a combined parallel loop, emit a call to
5496 GOMP_loop_foo_next. */
5497 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5498 build_fold_addr_expr (istart0
),
5499 build_fold_addr_expr (iend0
));
5503 tree t0
, t1
, t2
, t3
, t4
;
5504 /* If this is not a combined parallel loop, emit a call to
5505 GOMP_loop_foo_start in ENTRY_BB. */
5506 t4
= build_fold_addr_expr (iend0
);
5507 t3
= build_fold_addr_expr (istart0
);
5508 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
5511 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5513 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5514 OMP_CLAUSE__LOOPTEMP_
);
5515 gcc_assert (innerc
);
5516 t0
= OMP_CLAUSE_DECL (innerc
);
5517 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5518 OMP_CLAUSE__LOOPTEMP_
);
5519 gcc_assert (innerc
);
5520 t1
= OMP_CLAUSE_DECL (innerc
);
5522 if (POINTER_TYPE_P (TREE_TYPE (t0
))
5523 && TYPE_PRECISION (TREE_TYPE (t0
))
5524 != TYPE_PRECISION (fd
->iter_type
))
5526 /* Avoid casting pointers to integer of a different size. */
5527 tree itype
= signed_type_for (type
);
5528 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
5529 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
5533 t1
= fold_convert (fd
->iter_type
, t1
);
5534 t0
= fold_convert (fd
->iter_type
, t0
);
5538 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
5539 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
5541 if (fd
->iter_type
== long_integer_type_node
)
5545 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5546 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5547 6, t0
, t1
, t2
, t
, t3
, t4
);
5550 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5551 5, t0
, t1
, t2
, t3
, t4
);
5559 /* The GOMP_loop_ull_*start functions have additional boolean
5560 argument, true for < loops and false for > loops.
5561 In Fortran, the C bool type can be different from
5562 boolean_type_node. */
5563 bfn_decl
= builtin_decl_explicit (start_fn
);
5564 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
5565 t5
= build_int_cst (c_bool_type
,
5566 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
5569 tree bfn_decl
= builtin_decl_explicit (start_fn
);
5570 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5571 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
5574 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5575 6, t5
, t0
, t1
, t2
, t3
, t4
);
5578 if (TREE_TYPE (t
) != boolean_type_node
)
5579 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5580 t
, build_int_cst (TREE_TYPE (t
), 0));
5581 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5582 true, GSI_SAME_STMT
);
5583 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5585 /* Remove the GIMPLE_OMP_FOR statement. */
5586 gsi_remove (&gsi
, true);
5588 /* Iteration setup for sequential loop goes in L0_BB. */
5589 tree startvar
= fd
->loop
.v
;
5590 tree endvar
= NULL_TREE
;
5592 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5594 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
5595 && gimple_omp_for_kind (inner_stmt
)
5596 == GF_OMP_FOR_KIND_SIMD
);
5597 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
5598 OMP_CLAUSE__LOOPTEMP_
);
5599 gcc_assert (innerc
);
5600 startvar
= OMP_CLAUSE_DECL (innerc
);
5601 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5602 OMP_CLAUSE__LOOPTEMP_
);
5603 gcc_assert (innerc
);
5604 endvar
= OMP_CLAUSE_DECL (innerc
);
5607 gsi
= gsi_start_bb (l0_bb
);
5610 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5611 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5612 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5613 t
= fold_convert (TREE_TYPE (startvar
), t
);
5614 t
= force_gimple_operand_gsi (&gsi
, t
,
5616 && TREE_ADDRESSABLE (startvar
),
5617 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5618 stmt
= gimple_build_assign (startvar
, t
);
5619 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5623 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5624 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5625 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5626 t
= fold_convert (TREE_TYPE (startvar
), t
);
5627 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5628 false, GSI_CONTINUE_LINKING
);
5631 stmt
= gimple_build_assign (endvar
, iend
);
5632 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5633 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
5634 stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
5636 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, iend
,
5638 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5640 if (fd
->collapse
> 1)
5641 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5645 /* Code to control the increment and predicate for the sequential
5646 loop goes in the CONT_BB. */
5647 gsi
= gsi_last_bb (cont_bb
);
5648 stmt
= gsi_stmt (gsi
);
5649 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5650 vmain
= gimple_omp_continue_control_use (stmt
);
5651 vback
= gimple_omp_continue_control_def (stmt
);
5653 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5655 if (POINTER_TYPE_P (type
))
5656 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
5658 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
5659 t
= force_gimple_operand_gsi (&gsi
, t
,
5661 && TREE_ADDRESSABLE (vback
),
5662 NULL_TREE
, true, GSI_SAME_STMT
);
5663 stmt
= gimple_build_assign (vback
, t
);
5664 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5666 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5667 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
5669 stmt
= gimple_build_cond_empty (t
);
5670 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5673 /* Remove GIMPLE_OMP_CONTINUE. */
5674 gsi_remove (&gsi
, true);
5676 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5677 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
5679 /* Emit code to get the next parallel iteration in L2_BB. */
5680 gsi
= gsi_start_bb (l2_bb
);
5682 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5683 build_fold_addr_expr (istart0
),
5684 build_fold_addr_expr (iend0
));
5685 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5686 false, GSI_CONTINUE_LINKING
);
5687 if (TREE_TYPE (t
) != boolean_type_node
)
5688 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5689 t
, build_int_cst (TREE_TYPE (t
), 0));
5690 stmt
= gimple_build_cond_empty (t
);
5691 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5694 /* Add the loop cleanup function. */
5695 gsi
= gsi_last_bb (exit_bb
);
5696 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5697 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
5698 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5699 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
5701 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
5702 stmt
= gimple_build_call (t
, 0);
5703 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5704 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
5705 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5706 gsi_remove (&gsi
, true);
5708 /* Connect the new blocks. */
5709 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
5710 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
5716 e
= find_edge (cont_bb
, l3_bb
);
5717 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
5719 phis
= phi_nodes (l3_bb
);
5720 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5722 gimple phi
= gsi_stmt (gsi
);
5723 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
5724 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
5728 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
5729 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
5730 e
= find_edge (cont_bb
, l1_bb
);
5731 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5736 else if (fd
->collapse
> 1)
5739 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5742 e
->flags
= EDGE_TRUE_VALUE
;
5745 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5746 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
5750 e
= find_edge (cont_bb
, l2_bb
);
5751 e
->flags
= EDGE_FALLTHRU
;
5753 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
5755 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
5756 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
5757 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
5758 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
5759 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
5760 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
5761 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
5762 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
5764 struct loop
*outer_loop
= alloc_loop ();
5765 outer_loop
->header
= l0_bb
;
5766 outer_loop
->latch
= l2_bb
;
5767 add_loop (outer_loop
, l0_bb
->loop_father
);
5769 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5771 struct loop
*loop
= alloc_loop ();
5772 loop
->header
= l1_bb
;
5773 /* The loop may have multiple latches. */
5774 add_loop (loop
, outer_loop
);
5780 /* A subroutine of expand_omp_for. Generate code for a parallel
5781 loop with static schedule and no specified chunk size. Given
5784 for (V = N1; V cond N2; V += STEP) BODY;
5786 where COND is "<" or ">", we generate pseudocode
5788 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5793 if ((__typeof (V)) -1 > 0 && cond is >)
5794 n = -(adj + N2 - N1) / -STEP;
5796 n = (adj + N2 - N1) / STEP;
5799 if (threadid < tt) goto L3; else goto L4;
5804 s0 = q * threadid + tt;
5807 if (s0 >= e0) goto L2; else goto L0;
5813 if (V cond e) goto L1;
5818 expand_omp_for_static_nochunk (struct omp_region
*region
,
5819 struct omp_for_data
*fd
,
5822 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
5823 tree type
, itype
, vmain
, vback
;
5824 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
5825 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
5827 gimple_stmt_iterator gsi
;
5830 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
5831 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
5832 bool broken_loop
= region
->cont
== NULL
;
5833 tree
*counts
= NULL
;
5836 itype
= type
= TREE_TYPE (fd
->loop
.v
);
5837 if (POINTER_TYPE_P (type
))
5838 itype
= signed_type_for (type
);
5840 entry_bb
= region
->entry
;
5841 cont_bb
= region
->cont
;
5842 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5843 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5844 gcc_assert (broken_loop
5845 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
5846 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5847 body_bb
= single_succ (seq_start_bb
);
5850 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
5851 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5853 exit_bb
= region
->exit
;
5855 /* Iteration space partitioning goes in ENTRY_BB. */
5856 gsi
= gsi_last_bb (entry_bb
);
5857 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5859 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
5861 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
5862 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
5865 if (fd
->collapse
> 1)
5867 int first_zero_iter
= -1;
5868 basic_block l2_dom_bb
= NULL
;
5870 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5871 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5872 fin_bb
, first_zero_iter
,
5876 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5877 t
= integer_one_node
;
5879 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
5880 fold_convert (type
, fd
->loop
.n1
),
5881 fold_convert (type
, fd
->loop
.n2
));
5882 if (fd
->collapse
== 1
5883 && TYPE_UNSIGNED (type
)
5884 && (t
== NULL_TREE
|| !integer_onep (t
)))
5886 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
5887 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
5888 true, GSI_SAME_STMT
);
5889 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
5890 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
5891 true, GSI_SAME_STMT
);
5892 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
5893 NULL_TREE
, NULL_TREE
);
5894 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5895 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5896 expand_omp_regimplify_p
, NULL
, NULL
)
5897 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5898 expand_omp_regimplify_p
, NULL
, NULL
))
5900 gsi
= gsi_for_stmt (stmt
);
5901 gimple_regimplify_operands (stmt
, &gsi
);
5903 ep
= split_block (entry_bb
, stmt
);
5904 ep
->flags
= EDGE_TRUE_VALUE
;
5905 entry_bb
= ep
->dest
;
5906 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
5907 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
5908 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5909 if (gimple_in_ssa_p (cfun
))
5911 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
5912 for (gsi
= gsi_start_phis (fin_bb
);
5913 !gsi_end_p (gsi
); gsi_next (&gsi
))
5915 gimple phi
= gsi_stmt (gsi
);
5916 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
5917 ep
, UNKNOWN_LOCATION
);
5920 gsi
= gsi_last_bb (entry_bb
);
5923 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
5924 t
= fold_convert (itype
, t
);
5925 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5926 true, GSI_SAME_STMT
);
5928 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
5929 t
= fold_convert (itype
, t
);
5930 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5931 true, GSI_SAME_STMT
);
5935 step
= fd
->loop
.step
;
5936 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5938 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5939 OMP_CLAUSE__LOOPTEMP_
);
5940 gcc_assert (innerc
);
5941 n1
= OMP_CLAUSE_DECL (innerc
);
5942 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5943 OMP_CLAUSE__LOOPTEMP_
);
5944 gcc_assert (innerc
);
5945 n2
= OMP_CLAUSE_DECL (innerc
);
5947 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
5948 true, NULL_TREE
, true, GSI_SAME_STMT
);
5949 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
5950 true, NULL_TREE
, true, GSI_SAME_STMT
);
5951 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
5952 true, NULL_TREE
, true, GSI_SAME_STMT
);
5954 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
5955 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
5956 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
5957 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
5958 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
5959 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5960 fold_build1 (NEGATE_EXPR
, itype
, t
),
5961 fold_build1 (NEGATE_EXPR
, itype
, step
));
5963 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
5964 t
= fold_convert (itype
, t
);
5965 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5967 q
= create_tmp_reg (itype
, "q");
5968 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
5969 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
5970 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
5972 tt
= create_tmp_reg (itype
, "tt");
5973 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
5974 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
5975 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
5977 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
5978 stmt
= gimple_build_cond_empty (t
);
5979 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5981 second_bb
= split_block (entry_bb
, stmt
)->dest
;
5982 gsi
= gsi_last_bb (second_bb
);
5983 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5985 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
5987 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, q
, q
,
5988 build_int_cst (itype
, 1));
5989 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5991 third_bb
= split_block (second_bb
, stmt
)->dest
;
5992 gsi
= gsi_last_bb (third_bb
);
5993 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5995 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
5996 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
5997 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
5999 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6000 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6002 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6003 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6005 /* Remove the GIMPLE_OMP_FOR statement. */
6006 gsi_remove (&gsi
, true);
6008 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6009 gsi
= gsi_start_bb (seq_start_bb
);
6011 tree startvar
= fd
->loop
.v
;
6012 tree endvar
= NULL_TREE
;
6014 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6016 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6017 ? gimple_omp_parallel_clauses (inner_stmt
)
6018 : gimple_omp_for_clauses (inner_stmt
);
6019 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6020 gcc_assert (innerc
);
6021 startvar
= OMP_CLAUSE_DECL (innerc
);
6022 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6023 OMP_CLAUSE__LOOPTEMP_
);
6024 gcc_assert (innerc
);
6025 endvar
= OMP_CLAUSE_DECL (innerc
);
6027 t
= fold_convert (itype
, s0
);
6028 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6029 if (POINTER_TYPE_P (type
))
6030 t
= fold_build_pointer_plus (n1
, t
);
6032 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6033 t
= fold_convert (TREE_TYPE (startvar
), t
);
6034 t
= force_gimple_operand_gsi (&gsi
, t
,
6036 && TREE_ADDRESSABLE (startvar
),
6037 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6038 stmt
= gimple_build_assign (startvar
, t
);
6039 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6041 t
= fold_convert (itype
, e0
);
6042 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6043 if (POINTER_TYPE_P (type
))
6044 t
= fold_build_pointer_plus (n1
, t
);
6046 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6047 t
= fold_convert (TREE_TYPE (startvar
), t
);
6048 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6049 false, GSI_CONTINUE_LINKING
);
6052 stmt
= gimple_build_assign (endvar
, e
);
6053 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6054 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6055 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6057 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6059 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6061 if (fd
->collapse
> 1)
6062 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6066 /* The code controlling the sequential loop replaces the
6067 GIMPLE_OMP_CONTINUE. */
6068 gsi
= gsi_last_bb (cont_bb
);
6069 stmt
= gsi_stmt (gsi
);
6070 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6071 vmain
= gimple_omp_continue_control_use (stmt
);
6072 vback
= gimple_omp_continue_control_def (stmt
);
6074 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6076 if (POINTER_TYPE_P (type
))
6077 t
= fold_build_pointer_plus (vmain
, step
);
6079 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6080 t
= force_gimple_operand_gsi (&gsi
, t
,
6082 && TREE_ADDRESSABLE (vback
),
6083 NULL_TREE
, true, GSI_SAME_STMT
);
6084 stmt
= gimple_build_assign (vback
, t
);
6085 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6087 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6088 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6090 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6093 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6094 gsi_remove (&gsi
, true);
6096 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6097 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6100 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6101 gsi
= gsi_last_bb (exit_bb
);
6102 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6104 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6105 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6107 gsi_remove (&gsi
, true);
6109 /* Connect all the blocks. */
6110 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6111 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6112 ep
= find_edge (entry_bb
, second_bb
);
6113 ep
->flags
= EDGE_TRUE_VALUE
;
6114 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6115 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6116 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6120 ep
= find_edge (cont_bb
, body_bb
);
6121 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6126 else if (fd
->collapse
> 1)
6129 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6132 ep
->flags
= EDGE_TRUE_VALUE
;
6133 find_edge (cont_bb
, fin_bb
)->flags
6134 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6137 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6138 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6139 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6141 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6142 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6143 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6144 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6146 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6148 struct loop
*loop
= alloc_loop ();
6149 loop
->header
= body_bb
;
6150 if (collapse_bb
== NULL
)
6151 loop
->latch
= cont_bb
;
6152 add_loop (loop
, body_bb
->loop_father
);
6157 /* A subroutine of expand_omp_for. Generate code for a parallel
6158 loop with static schedule and a specified chunk size. Given
6161 for (V = N1; V cond N2; V += STEP) BODY;
6163 where COND is "<" or ">", we generate pseudocode
6165 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6170 if ((__typeof (V)) -1 > 0 && cond is >)
6171 n = -(adj + N2 - N1) / -STEP;
6173 n = (adj + N2 - N1) / STEP;
6175 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6176 here so that V is defined
6177 if the loop is not entered
6179 s0 = (trip * nthreads + threadid) * CHUNK;
6180 e0 = min(s0 + CHUNK, n);
6181 if (s0 < n) goto L1; else goto L4;
6188 if (V cond e) goto L2; else goto L3;
6196 expand_omp_for_static_chunk (struct omp_region
*region
,
6197 struct omp_for_data
*fd
, gimple inner_stmt
)
6199 tree n
, s0
, e0
, e
, t
;
6200 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6201 tree type
, itype
, vmain
, vback
, vextra
;
6202 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6203 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6204 gimple_stmt_iterator gsi
;
6207 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6208 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6209 bool broken_loop
= region
->cont
== NULL
;
6210 tree
*counts
= NULL
;
6213 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6214 if (POINTER_TYPE_P (type
))
6215 itype
= signed_type_for (type
);
6217 entry_bb
= region
->entry
;
6218 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6220 iter_part_bb
= se
->dest
;
6221 cont_bb
= region
->cont
;
6222 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6223 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6224 gcc_assert (broken_loop
6225 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6226 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6227 body_bb
= single_succ (seq_start_bb
);
6230 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6231 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6232 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6234 exit_bb
= region
->exit
;
6236 /* Trip and adjustment setup goes in ENTRY_BB. */
6237 gsi
= gsi_last_bb (entry_bb
);
6238 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6240 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6242 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6243 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6246 if (fd
->collapse
> 1)
6248 int first_zero_iter
= -1;
6249 basic_block l2_dom_bb
= NULL
;
6251 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6252 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6253 fin_bb
, first_zero_iter
,
6257 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6258 t
= integer_one_node
;
6260 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6261 fold_convert (type
, fd
->loop
.n1
),
6262 fold_convert (type
, fd
->loop
.n2
));
6263 if (fd
->collapse
== 1
6264 && TYPE_UNSIGNED (type
)
6265 && (t
== NULL_TREE
|| !integer_onep (t
)))
6267 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6268 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6269 true, GSI_SAME_STMT
);
6270 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6271 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6272 true, GSI_SAME_STMT
);
6273 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6274 NULL_TREE
, NULL_TREE
);
6275 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6276 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6277 expand_omp_regimplify_p
, NULL
, NULL
)
6278 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6279 expand_omp_regimplify_p
, NULL
, NULL
))
6281 gsi
= gsi_for_stmt (stmt
);
6282 gimple_regimplify_operands (stmt
, &gsi
);
6284 se
= split_block (entry_bb
, stmt
);
6285 se
->flags
= EDGE_TRUE_VALUE
;
6286 entry_bb
= se
->dest
;
6287 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6288 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6289 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6290 if (gimple_in_ssa_p (cfun
))
6292 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6293 for (gsi
= gsi_start_phis (fin_bb
);
6294 !gsi_end_p (gsi
); gsi_next (&gsi
))
6296 gimple phi
= gsi_stmt (gsi
);
6297 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6298 se
, UNKNOWN_LOCATION
);
6301 gsi
= gsi_last_bb (entry_bb
);
6304 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6305 t
= fold_convert (itype
, t
);
6306 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6307 true, GSI_SAME_STMT
);
6309 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6310 t
= fold_convert (itype
, t
);
6311 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6312 true, GSI_SAME_STMT
);
6316 step
= fd
->loop
.step
;
6317 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6319 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6320 OMP_CLAUSE__LOOPTEMP_
);
6321 gcc_assert (innerc
);
6322 n1
= OMP_CLAUSE_DECL (innerc
);
6323 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6324 OMP_CLAUSE__LOOPTEMP_
);
6325 gcc_assert (innerc
);
6326 n2
= OMP_CLAUSE_DECL (innerc
);
6328 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6329 true, NULL_TREE
, true, GSI_SAME_STMT
);
6330 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6331 true, NULL_TREE
, true, GSI_SAME_STMT
);
6332 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6333 true, NULL_TREE
, true, GSI_SAME_STMT
);
6335 = force_gimple_operand_gsi (&gsi
, fold_convert (itype
, fd
->chunk_size
),
6336 true, NULL_TREE
, true, GSI_SAME_STMT
);
6338 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6339 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6340 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6341 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6342 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6343 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6344 fold_build1 (NEGATE_EXPR
, itype
, t
),
6345 fold_build1 (NEGATE_EXPR
, itype
, step
));
6347 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6348 t
= fold_convert (itype
, t
);
6349 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6350 true, GSI_SAME_STMT
);
6352 trip_var
= create_tmp_reg (itype
, ".trip");
6353 if (gimple_in_ssa_p (cfun
))
6355 trip_init
= make_ssa_name (trip_var
, NULL
);
6356 trip_main
= make_ssa_name (trip_var
, NULL
);
6357 trip_back
= make_ssa_name (trip_var
, NULL
);
6361 trip_init
= trip_var
;
6362 trip_main
= trip_var
;
6363 trip_back
= trip_var
;
6366 stmt
= gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
6367 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6369 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
6370 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6371 if (POINTER_TYPE_P (type
))
6372 t
= fold_build_pointer_plus (n1
, t
);
6374 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6375 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6376 true, GSI_SAME_STMT
);
6378 /* Remove the GIMPLE_OMP_FOR. */
6379 gsi_remove (&gsi
, true);
6381 /* Iteration space partitioning goes in ITER_PART_BB. */
6382 gsi
= gsi_last_bb (iter_part_bb
);
6384 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
6385 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
6386 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
6387 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6388 false, GSI_CONTINUE_LINKING
);
6390 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
6391 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
6392 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6393 false, GSI_CONTINUE_LINKING
);
6395 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
6396 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
6398 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6399 gsi
= gsi_start_bb (seq_start_bb
);
6401 tree startvar
= fd
->loop
.v
;
6402 tree endvar
= NULL_TREE
;
6404 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6406 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6407 ? gimple_omp_parallel_clauses (inner_stmt
)
6408 : gimple_omp_for_clauses (inner_stmt
);
6409 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6410 gcc_assert (innerc
);
6411 startvar
= OMP_CLAUSE_DECL (innerc
);
6412 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6413 OMP_CLAUSE__LOOPTEMP_
);
6414 gcc_assert (innerc
);
6415 endvar
= OMP_CLAUSE_DECL (innerc
);
6418 t
= fold_convert (itype
, s0
);
6419 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6420 if (POINTER_TYPE_P (type
))
6421 t
= fold_build_pointer_plus (n1
, t
);
6423 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6424 t
= fold_convert (TREE_TYPE (startvar
), t
);
6425 t
= force_gimple_operand_gsi (&gsi
, t
,
6427 && TREE_ADDRESSABLE (startvar
),
6428 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6429 stmt
= gimple_build_assign (startvar
, t
);
6430 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6432 t
= fold_convert (itype
, e0
);
6433 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6434 if (POINTER_TYPE_P (type
))
6435 t
= fold_build_pointer_plus (n1
, t
);
6437 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6438 t
= fold_convert (TREE_TYPE (startvar
), t
);
6439 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6440 false, GSI_CONTINUE_LINKING
);
6443 stmt
= gimple_build_assign (endvar
, e
);
6444 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6445 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6446 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6448 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6450 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6452 if (fd
->collapse
> 1)
6453 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6457 /* The code controlling the sequential loop goes in CONT_BB,
6458 replacing the GIMPLE_OMP_CONTINUE. */
6459 gsi
= gsi_last_bb (cont_bb
);
6460 stmt
= gsi_stmt (gsi
);
6461 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6462 vmain
= gimple_omp_continue_control_use (stmt
);
6463 vback
= gimple_omp_continue_control_def (stmt
);
6465 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6467 if (POINTER_TYPE_P (type
))
6468 t
= fold_build_pointer_plus (vmain
, step
);
6470 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6471 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
6472 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6473 true, GSI_SAME_STMT
);
6474 stmt
= gimple_build_assign (vback
, t
);
6475 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6477 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6478 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6480 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6483 /* Remove GIMPLE_OMP_CONTINUE. */
6484 gsi_remove (&gsi
, true);
6486 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6487 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6489 /* Trip update code goes into TRIP_UPDATE_BB. */
6490 gsi
= gsi_start_bb (trip_update_bb
);
6492 t
= build_int_cst (itype
, 1);
6493 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
6494 stmt
= gimple_build_assign (trip_back
, t
);
6495 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6498 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6499 gsi
= gsi_last_bb (exit_bb
);
6500 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6502 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6503 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6505 gsi_remove (&gsi
, true);
6507 /* Connect the new blocks. */
6508 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
6509 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
6513 se
= find_edge (cont_bb
, body_bb
);
6514 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6519 else if (fd
->collapse
> 1)
6522 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6525 se
->flags
= EDGE_TRUE_VALUE
;
6526 find_edge (cont_bb
, trip_update_bb
)->flags
6527 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6529 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
6532 if (gimple_in_ssa_p (cfun
))
6534 gimple_stmt_iterator psi
;
6537 edge_var_map_vector
*head
;
6541 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
6543 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6544 remove arguments of the phi nodes in fin_bb. We need to create
6545 appropriate phi nodes in iter_part_bb instead. */
6546 se
= single_pred_edge (fin_bb
);
6547 re
= single_succ_edge (trip_update_bb
);
6548 head
= redirect_edge_var_map_vector (re
);
6549 ene
= single_succ_edge (entry_bb
);
6551 psi
= gsi_start_phis (fin_bb
);
6552 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
6553 gsi_next (&psi
), ++i
)
6556 source_location locus
;
6558 phi
= gsi_stmt (psi
);
6559 t
= gimple_phi_result (phi
);
6560 gcc_assert (t
== redirect_edge_var_map_result (vm
));
6561 nphi
= create_phi_node (t
, iter_part_bb
);
6563 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
6564 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
6566 /* A special case -- fd->loop.v is not yet computed in
6567 iter_part_bb, we need to use vextra instead. */
6568 if (t
== fd
->loop
.v
)
6570 add_phi_arg (nphi
, t
, ene
, locus
);
6571 locus
= redirect_edge_var_map_location (vm
);
6572 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
6574 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
6575 redirect_edge_var_map_clear (re
);
6578 psi
= gsi_start_phis (fin_bb
);
6579 if (gsi_end_p (psi
))
6581 remove_phi_node (&psi
, false);
6584 /* Make phi node for trip. */
6585 phi
= create_phi_node (trip_main
, iter_part_bb
);
6586 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
6588 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
6593 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
6594 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
6595 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
6596 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6597 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6598 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
6599 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
6600 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6601 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6605 struct loop
*trip_loop
= alloc_loop ();
6606 trip_loop
->header
= iter_part_bb
;
6607 trip_loop
->latch
= trip_update_bb
;
6608 add_loop (trip_loop
, iter_part_bb
->loop_father
);
6610 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6612 struct loop
*loop
= alloc_loop ();
6613 loop
->header
= body_bb
;
6614 if (collapse_bb
== NULL
)
6615 loop
->latch
= cont_bb
;
6616 add_loop (loop
, trip_loop
);
6622 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
6623 loop. Given parameters:
6625 for (V = N1; V cond N2; V += STEP) BODY;
6627 where COND is "<" or ">", we generate pseudocode
6635 if (V cond N2) goto L0; else goto L2;
6638 For collapsed loops, given parameters:
6640 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
6641 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
6642 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
6645 we generate pseudocode
6651 count3 = (adj + N32 - N31) / STEP3;
6656 count2 = (adj + N22 - N21) / STEP2;
6661 count1 = (adj + N12 - N11) / STEP1;
6662 count = count1 * count2 * count3;
6672 V2 += (V3 cond3 N32) ? 0 : STEP2;
6673 V3 = (V3 cond3 N32) ? V3 : N31;
6674 V1 += (V2 cond2 N22) ? 0 : STEP1;
6675 V2 = (V2 cond2 N22) ? V2 : N21;
6677 if (V < count) goto L0; else goto L2;
6683 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
6686 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
6687 gimple_stmt_iterator gsi
;
6689 bool broken_loop
= region
->cont
== NULL
;
6691 tree
*counts
= NULL
;
6693 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6694 OMP_CLAUSE_SAFELEN
);
6695 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6696 OMP_CLAUSE__SIMDUID_
);
6699 type
= TREE_TYPE (fd
->loop
.v
);
6700 entry_bb
= region
->entry
;
6701 cont_bb
= region
->cont
;
6702 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6703 gcc_assert (broken_loop
6704 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6705 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
6708 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
6709 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6710 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
6711 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6715 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
6716 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
6717 l2_bb
= single_succ (l1_bb
);
6719 exit_bb
= region
->exit
;
6722 gsi
= gsi_last_bb (entry_bb
);
6724 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6725 /* Not needed in SSA form right now. */
6726 gcc_assert (!gimple_in_ssa_p (cfun
));
6727 if (fd
->collapse
> 1)
6729 int first_zero_iter
= -1;
6730 basic_block zero_iter_bb
= l2_bb
;
6732 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6733 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6734 zero_iter_bb
, first_zero_iter
,
6737 if (l2_dom_bb
== NULL
)
6742 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6744 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6745 OMP_CLAUSE__LOOPTEMP_
);
6746 gcc_assert (innerc
);
6747 n1
= OMP_CLAUSE_DECL (innerc
);
6748 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6749 OMP_CLAUSE__LOOPTEMP_
);
6750 gcc_assert (innerc
);
6751 n2
= OMP_CLAUSE_DECL (innerc
);
6752 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6753 fold_convert (type
, n1
));
6754 if (fd
->collapse
> 1)
6757 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
6763 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
6764 fold_convert (type
, fd
->loop
.n1
));
6765 if (fd
->collapse
> 1)
6766 for (i
= 0; i
< fd
->collapse
; i
++)
6768 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6769 if (POINTER_TYPE_P (itype
))
6770 itype
= signed_type_for (itype
);
6771 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
6772 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6776 /* Remove the GIMPLE_OMP_FOR statement. */
6777 gsi_remove (&gsi
, true);
6781 /* Code to control the increment goes in the CONT_BB. */
6782 gsi
= gsi_last_bb (cont_bb
);
6783 stmt
= gsi_stmt (gsi
);
6784 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6786 if (POINTER_TYPE_P (type
))
6787 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
6789 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
6790 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
6792 if (fd
->collapse
> 1)
6794 i
= fd
->collapse
- 1;
6795 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
6797 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
6798 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
6802 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
6804 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
6807 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6809 for (i
= fd
->collapse
- 1; i
> 0; i
--)
6811 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
6812 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
6813 if (POINTER_TYPE_P (itype2
))
6814 itype2
= signed_type_for (itype2
);
6815 t
= build3 (COND_EXPR
, itype2
,
6816 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6818 fold_convert (itype
, fd
->loops
[i
].n2
)),
6819 build_int_cst (itype2
, 0),
6820 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
6821 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
6822 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
6824 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
6825 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
6827 t
= build3 (COND_EXPR
, itype
,
6828 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
6830 fold_convert (itype
, fd
->loops
[i
].n2
)),
6832 fold_convert (itype
, fd
->loops
[i
].n1
));
6833 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
6837 /* Remove GIMPLE_OMP_CONTINUE. */
6838 gsi_remove (&gsi
, true);
6841 /* Emit the condition in L1_BB. */
6842 gsi
= gsi_start_bb (l1_bb
);
6844 t
= fold_convert (type
, n2
);
6845 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6846 false, GSI_CONTINUE_LINKING
);
6847 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
6848 stmt
= gimple_build_cond_empty (t
);
6849 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6850 if (walk_tree (gimple_cond_lhs_ptr (stmt
), expand_omp_regimplify_p
,
6852 || walk_tree (gimple_cond_rhs_ptr (stmt
), expand_omp_regimplify_p
,
6855 gsi
= gsi_for_stmt (stmt
);
6856 gimple_regimplify_operands (stmt
, &gsi
);
6859 /* Remove GIMPLE_OMP_RETURN. */
6860 gsi
= gsi_last_bb (exit_bb
);
6861 gsi_remove (&gsi
, true);
6863 /* Connect the new blocks. */
6864 remove_edge (FALLTHRU_EDGE (entry_bb
));
6868 remove_edge (BRANCH_EDGE (entry_bb
));
6869 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
6871 e
= BRANCH_EDGE (l1_bb
);
6872 ne
= FALLTHRU_EDGE (l1_bb
);
6873 e
->flags
= EDGE_TRUE_VALUE
;
6877 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6879 ne
= single_succ_edge (l1_bb
);
6880 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6883 ne
->flags
= EDGE_FALSE_VALUE
;
6884 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6885 ne
->probability
= REG_BR_PROB_BASE
/ 8;
6887 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
6888 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
6889 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
6893 struct loop
*loop
= alloc_loop ();
6894 loop
->header
= l1_bb
;
6895 loop
->latch
= cont_bb
;
6896 add_loop (loop
, l1_bb
->loop_father
);
6897 if (safelen
== NULL_TREE
)
6898 loop
->safelen
= INT_MAX
;
6901 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
6902 if (TREE_CODE (safelen
) != INTEGER_CST
)
6904 else if (!tree_fits_uhwi_p (safelen
)
6905 || tree_to_uhwi (safelen
) > INT_MAX
)
6906 loop
->safelen
= INT_MAX
;
6908 loop
->safelen
= tree_to_uhwi (safelen
);
6909 if (loop
->safelen
== 1)
6914 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
6915 cfun
->has_simduid_loops
= true;
6917 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
6919 if ((flag_tree_loop_vectorize
6920 || (!global_options_set
.x_flag_tree_loop_vectorize
6921 && !global_options_set
.x_flag_tree_vectorize
))
6922 && flag_tree_loop_optimize
6923 && loop
->safelen
> 1)
6925 loop
->force_vectorize
= true;
6926 cfun
->has_force_vectorize_loops
= true;
6932 /* Expand the OpenMP loop defined by REGION. */
6935 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
6937 struct omp_for_data fd
;
6938 struct omp_for_data_loop
*loops
;
6941 = (struct omp_for_data_loop
*)
6942 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
6943 * sizeof (struct omp_for_data_loop
));
6944 extract_omp_for_data (last_stmt (region
->entry
), &fd
, loops
);
6945 region
->sched_kind
= fd
.sched_kind
;
6947 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
6948 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6949 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
6952 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
6953 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6954 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
6957 /* If there isn't a continue then this is a degerate case where
6958 the introduction of abnormal edges during lowering will prevent
6959 original loops from being detected. Fix that up. */
6960 loops_state_set (LOOPS_NEED_FIXUP
);
6962 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
6963 expand_omp_simd (region
, &fd
);
6964 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
6965 && !fd
.have_ordered
)
6967 if (fd
.chunk_size
== NULL
)
6968 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
6970 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
6974 int fn_index
, start_ix
, next_ix
;
6976 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
6977 == GF_OMP_FOR_KIND_FOR
);
6978 if (fd
.chunk_size
== NULL
6979 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
6980 fd
.chunk_size
= integer_zero_node
;
6981 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
6982 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
6983 ? 3 : fd
.sched_kind
;
6984 fn_index
+= fd
.have_ordered
* 4;
6985 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
6986 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
6987 if (fd
.iter_type
== long_long_unsigned_type_node
)
6989 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
6990 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
6991 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
6992 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
6994 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
6995 (enum built_in_function
) next_ix
, inner_stmt
);
6998 if (gimple_in_ssa_p (cfun
))
6999 update_ssa (TODO_update_ssa_only_virtuals
);
7003 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7005 v = GOMP_sections_start (n);
7022 v = GOMP_sections_next ();
7027 If this is a combined parallel sections, replace the call to
7028 GOMP_sections_start with call to GOMP_sections_next. */
7031 expand_omp_sections (struct omp_region
*region
)
7033 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7035 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7036 gimple_stmt_iterator si
, switch_si
;
7037 gimple sections_stmt
, stmt
, cont
;
7040 struct omp_region
*inner
;
7042 bool exit_reachable
= region
->cont
!= NULL
;
7044 gcc_assert (region
->exit
!= NULL
);
7045 entry_bb
= region
->entry
;
7046 l0_bb
= single_succ (entry_bb
);
7047 l1_bb
= region
->cont
;
7048 l2_bb
= region
->exit
;
7049 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7050 l2
= gimple_block_label (l2_bb
);
7053 /* This can happen if there are reductions. */
7054 len
= EDGE_COUNT (l0_bb
->succs
);
7055 gcc_assert (len
> 0);
7056 e
= EDGE_SUCC (l0_bb
, len
- 1);
7057 si
= gsi_last_bb (e
->dest
);
7060 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7061 l2
= gimple_block_label (e
->dest
);
7063 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7065 si
= gsi_last_bb (e
->dest
);
7067 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7069 l2
= gimple_block_label (e
->dest
);
7075 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7077 default_bb
= create_empty_bb (l0_bb
);
7079 /* We will build a switch() with enough cases for all the
7080 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7081 and a default case to abort if something goes wrong. */
7082 len
= EDGE_COUNT (l0_bb
->succs
);
7084 /* Use vec::quick_push on label_vec throughout, since we know the size
7086 auto_vec
<tree
> label_vec (len
);
7088 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7089 GIMPLE_OMP_SECTIONS statement. */
7090 si
= gsi_last_bb (entry_bb
);
7091 sections_stmt
= gsi_stmt (si
);
7092 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
7093 vin
= gimple_omp_sections_control (sections_stmt
);
7094 if (!is_combined_parallel (region
))
7096 /* If we are not inside a combined parallel+sections region,
7097 call GOMP_sections_start. */
7098 t
= build_int_cst (unsigned_type_node
, len
- 1);
7099 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
7100 stmt
= gimple_build_call (u
, 1, t
);
7104 /* Otherwise, call GOMP_sections_next. */
7105 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7106 stmt
= gimple_build_call (u
, 0);
7108 gimple_call_set_lhs (stmt
, vin
);
7109 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7110 gsi_remove (&si
, true);
7112 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
7114 switch_si
= gsi_last_bb (l0_bb
);
7115 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
7118 cont
= last_stmt (l1_bb
);
7119 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
7120 vmain
= gimple_omp_continue_control_use (cont
);
7121 vnext
= gimple_omp_continue_control_def (cont
);
7129 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
7130 label_vec
.quick_push (t
);
7133 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
7134 for (inner
= region
->inner
, casei
= 1;
7136 inner
= inner
->next
, i
++, casei
++)
7138 basic_block s_entry_bb
, s_exit_bb
;
7140 /* Skip optional reduction region. */
7141 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
7148 s_entry_bb
= inner
->entry
;
7149 s_exit_bb
= inner
->exit
;
7151 t
= gimple_block_label (s_entry_bb
);
7152 u
= build_int_cst (unsigned_type_node
, casei
);
7153 u
= build_case_label (u
, NULL
, t
);
7154 label_vec
.quick_push (u
);
7156 si
= gsi_last_bb (s_entry_bb
);
7157 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
7158 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
7159 gsi_remove (&si
, true);
7160 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
7162 if (s_exit_bb
== NULL
)
7165 si
= gsi_last_bb (s_exit_bb
);
7166 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7167 gsi_remove (&si
, true);
7169 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
7172 /* Error handling code goes in DEFAULT_BB. */
7173 t
= gimple_block_label (default_bb
);
7174 u
= build_case_label (NULL
, NULL
, t
);
7175 make_edge (l0_bb
, default_bb
, 0);
7176 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
7178 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
7179 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
7180 gsi_remove (&switch_si
, true);
7182 si
= gsi_start_bb (default_bb
);
7183 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
7184 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
7190 /* Code to get the next section goes in L1_BB. */
7191 si
= gsi_last_bb (l1_bb
);
7192 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
7194 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7195 stmt
= gimple_build_call (bfn_decl
, 0);
7196 gimple_call_set_lhs (stmt
, vnext
);
7197 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7198 gsi_remove (&si
, true);
7200 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
7203 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7204 si
= gsi_last_bb (l2_bb
);
7205 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
7206 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
7207 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
7208 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
7210 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
7211 stmt
= gimple_build_call (t
, 0);
7212 if (gimple_omp_return_lhs (gsi_stmt (si
)))
7213 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
7214 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7215 gsi_remove (&si
, true);
7217 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
7221 /* Expand code for an OpenMP single directive. We've already expanded
7222 much of the code, here we simply place the GOMP_barrier call. */
7225 expand_omp_single (struct omp_region
*region
)
7227 basic_block entry_bb
, exit_bb
;
7228 gimple_stmt_iterator si
;
7230 entry_bb
= region
->entry
;
7231 exit_bb
= region
->exit
;
7233 si
= gsi_last_bb (entry_bb
);
7234 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
7235 gsi_remove (&si
, true);
7236 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7238 si
= gsi_last_bb (exit_bb
);
7239 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
7241 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
7242 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
7244 gsi_remove (&si
, true);
7245 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7249 /* Generic expansion for OpenMP synchronization directives: master,
7250 ordered and critical. All we need to do here is remove the entry
7251 and exit markers for REGION. */
7254 expand_omp_synch (struct omp_region
*region
)
7256 basic_block entry_bb
, exit_bb
;
7257 gimple_stmt_iterator si
;
7259 entry_bb
= region
->entry
;
7260 exit_bb
= region
->exit
;
7262 si
= gsi_last_bb (entry_bb
);
7263 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
7264 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
7265 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
7266 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
7267 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
7268 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
7269 gsi_remove (&si
, true);
7270 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7274 si
= gsi_last_bb (exit_bb
);
7275 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7276 gsi_remove (&si
, true);
7277 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7281 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7282 operation as a normal volatile load. */
7285 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
7286 tree loaded_val
, int index
)
7288 enum built_in_function tmpbase
;
7289 gimple_stmt_iterator gsi
;
7290 basic_block store_bb
;
7293 tree decl
, call
, type
, itype
;
7295 gsi
= gsi_last_bb (load_bb
);
7296 stmt
= gsi_stmt (gsi
);
7297 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7298 loc
= gimple_location (stmt
);
7300 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7301 is smaller than word size, then expand_atomic_load assumes that the load
7302 is atomic. We could avoid the builtin entirely in this case. */
7304 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7305 decl
= builtin_decl_explicit (tmpbase
);
7306 if (decl
== NULL_TREE
)
7309 type
= TREE_TYPE (loaded_val
);
7310 itype
= TREE_TYPE (TREE_TYPE (decl
));
7312 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
7313 build_int_cst (NULL
,
7314 gimple_omp_atomic_seq_cst_p (stmt
)
7316 : MEMMODEL_RELAXED
));
7317 if (!useless_type_conversion_p (type
, itype
))
7318 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7319 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7321 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7322 gsi_remove (&gsi
, true);
7324 store_bb
= single_succ (load_bb
);
7325 gsi
= gsi_last_bb (store_bb
);
7326 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7327 gsi_remove (&gsi
, true);
7329 if (gimple_in_ssa_p (cfun
))
7330 update_ssa (TODO_update_ssa_no_phi
);
7335 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7336 operation as a normal volatile store. */
7339 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
7340 tree loaded_val
, tree stored_val
, int index
)
7342 enum built_in_function tmpbase
;
7343 gimple_stmt_iterator gsi
;
7344 basic_block store_bb
= single_succ (load_bb
);
7347 tree decl
, call
, type
, itype
;
7348 enum machine_mode imode
;
7351 gsi
= gsi_last_bb (load_bb
);
7352 stmt
= gsi_stmt (gsi
);
7353 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7355 /* If the load value is needed, then this isn't a store but an exchange. */
7356 exchange
= gimple_omp_atomic_need_value_p (stmt
);
7358 gsi
= gsi_last_bb (store_bb
);
7359 stmt
= gsi_stmt (gsi
);
7360 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
7361 loc
= gimple_location (stmt
);
7363 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7364 is smaller than word size, then expand_atomic_store assumes that the store
7365 is atomic. We could avoid the builtin entirely in this case. */
7367 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
7368 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
7369 decl
= builtin_decl_explicit (tmpbase
);
7370 if (decl
== NULL_TREE
)
7373 type
= TREE_TYPE (stored_val
);
7375 /* Dig out the type of the function's second argument. */
7376 itype
= TREE_TYPE (decl
);
7377 itype
= TYPE_ARG_TYPES (itype
);
7378 itype
= TREE_CHAIN (itype
);
7379 itype
= TREE_VALUE (itype
);
7380 imode
= TYPE_MODE (itype
);
7382 if (exchange
&& !can_atomic_exchange_p (imode
, true))
7385 if (!useless_type_conversion_p (itype
, type
))
7386 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
7387 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
7388 build_int_cst (NULL
,
7389 gimple_omp_atomic_seq_cst_p (stmt
)
7391 : MEMMODEL_RELAXED
));
7394 if (!useless_type_conversion_p (type
, itype
))
7395 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7396 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7399 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7400 gsi_remove (&gsi
, true);
7402 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7403 gsi
= gsi_last_bb (load_bb
);
7404 gsi_remove (&gsi
, true);
7406 if (gimple_in_ssa_p (cfun
))
7407 update_ssa (TODO_update_ssa_no_phi
);
7412 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7413 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7414 size of the data type, and thus usable to find the index of the builtin
7415 decl. Returns false if the expression is not of the proper form. */
7418 expand_omp_atomic_fetch_op (basic_block load_bb
,
7419 tree addr
, tree loaded_val
,
7420 tree stored_val
, int index
)
7422 enum built_in_function oldbase
, newbase
, tmpbase
;
7423 tree decl
, itype
, call
;
7425 basic_block store_bb
= single_succ (load_bb
);
7426 gimple_stmt_iterator gsi
;
7429 enum tree_code code
;
7430 bool need_old
, need_new
;
7431 enum machine_mode imode
;
7434 /* We expect to find the following sequences:
7437 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7440 val = tmp OP something; (or: something OP tmp)
7441 GIMPLE_OMP_STORE (val)
7443 ???FIXME: Allow a more flexible sequence.
7444 Perhaps use data flow to pick the statements.
7448 gsi
= gsi_after_labels (store_bb
);
7449 stmt
= gsi_stmt (gsi
);
7450 loc
= gimple_location (stmt
);
7451 if (!is_gimple_assign (stmt
))
7454 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
7456 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
7457 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
7458 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
7459 gcc_checking_assert (!need_old
|| !need_new
);
7461 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
7464 /* Check for one of the supported fetch-op operations. */
7465 code
= gimple_assign_rhs_code (stmt
);
7469 case POINTER_PLUS_EXPR
:
7470 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
7471 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
7474 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
7475 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
7478 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
7479 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
7482 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
7483 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
7486 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
7487 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
7493 /* Make sure the expression is of the proper form. */
7494 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
7495 rhs
= gimple_assign_rhs2 (stmt
);
7496 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
7497 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
7498 rhs
= gimple_assign_rhs1 (stmt
);
7502 tmpbase
= ((enum built_in_function
)
7503 ((need_new
? newbase
: oldbase
) + index
+ 1));
7504 decl
= builtin_decl_explicit (tmpbase
);
7505 if (decl
== NULL_TREE
)
7507 itype
= TREE_TYPE (TREE_TYPE (decl
));
7508 imode
= TYPE_MODE (itype
);
7510 /* We could test all of the various optabs involved, but the fact of the
7511 matter is that (with the exception of i486 vs i586 and xadd) all targets
7512 that support any atomic operaton optab also implements compare-and-swap.
7513 Let optabs.c take care of expanding any compare-and-swap loop. */
7514 if (!can_compare_and_swap_p (imode
, true))
7517 gsi
= gsi_last_bb (load_bb
);
7518 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7520 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7521 It only requires that the operation happen atomically. Thus we can
7522 use the RELAXED memory model. */
7523 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
7524 fold_convert_loc (loc
, itype
, rhs
),
7525 build_int_cst (NULL
,
7526 seq_cst
? MEMMODEL_SEQ_CST
7527 : MEMMODEL_RELAXED
));
7529 if (need_old
|| need_new
)
7531 lhs
= need_old
? loaded_val
: stored_val
;
7532 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
7533 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
7536 call
= fold_convert_loc (loc
, void_type_node
, call
);
7537 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7538 gsi_remove (&gsi
, true);
7540 gsi
= gsi_last_bb (store_bb
);
7541 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7542 gsi_remove (&gsi
, true);
7543 gsi
= gsi_last_bb (store_bb
);
7544 gsi_remove (&gsi
, true);
7546 if (gimple_in_ssa_p (cfun
))
7547 update_ssa (TODO_update_ssa_no_phi
);
7552 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7556 newval = rhs; // with oldval replacing *addr in rhs
7557 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7558 if (oldval != newval)
7561 INDEX is log2 of the size of the data type, and thus usable to find the
7562 index of the builtin decl. */
7565 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
7566 tree addr
, tree loaded_val
, tree stored_val
,
7569 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
7570 tree type
, itype
, cmpxchg
, iaddr
;
7571 gimple_stmt_iterator si
;
7572 basic_block loop_header
= single_succ (load_bb
);
7575 enum built_in_function fncode
;
7577 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7578 order to use the RELAXED memory model effectively. */
7579 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7581 cmpxchg
= builtin_decl_explicit (fncode
);
7582 if (cmpxchg
== NULL_TREE
)
7584 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7585 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
7587 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
7590 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7591 si
= gsi_last_bb (load_bb
);
7592 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7594 /* For floating-point values, we'll need to view-convert them to integers
7595 so that we can perform the atomic compare and swap. Simplify the
7596 following code by always setting up the "i"ntegral variables. */
7597 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
7601 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
7604 = force_gimple_operand_gsi (&si
,
7605 fold_convert (TREE_TYPE (iaddr
), addr
),
7606 false, NULL_TREE
, true, GSI_SAME_STMT
);
7607 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
7608 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7609 loadedi
= create_tmp_var (itype
, NULL
);
7610 if (gimple_in_ssa_p (cfun
))
7611 loadedi
= make_ssa_name (loadedi
, NULL
);
7616 loadedi
= loaded_val
;
7619 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7620 tree loaddecl
= builtin_decl_explicit (fncode
);
7623 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
7624 build_call_expr (loaddecl
, 2, iaddr
,
7625 build_int_cst (NULL_TREE
,
7626 MEMMODEL_RELAXED
)));
7628 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
7629 build_int_cst (TREE_TYPE (iaddr
), 0));
7632 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
7635 /* Move the value to the LOADEDI temporary. */
7636 if (gimple_in_ssa_p (cfun
))
7638 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
7639 phi
= create_phi_node (loadedi
, loop_header
);
7640 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
7644 gsi_insert_before (&si
,
7645 gimple_build_assign (loadedi
, initial
),
7647 if (loadedi
!= loaded_val
)
7649 gimple_stmt_iterator gsi2
;
7652 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
7653 gsi2
= gsi_start_bb (loop_header
);
7654 if (gimple_in_ssa_p (cfun
))
7657 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7658 true, GSI_SAME_STMT
);
7659 stmt
= gimple_build_assign (loaded_val
, x
);
7660 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
7664 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
7665 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
7666 true, GSI_SAME_STMT
);
7669 gsi_remove (&si
, true);
7671 si
= gsi_last_bb (store_bb
);
7672 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7675 storedi
= stored_val
;
7678 force_gimple_operand_gsi (&si
,
7679 build1 (VIEW_CONVERT_EXPR
, itype
,
7680 stored_val
), true, NULL_TREE
, true,
7683 /* Build the compare&swap statement. */
7684 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
7685 new_storedi
= force_gimple_operand_gsi (&si
,
7686 fold_convert (TREE_TYPE (loadedi
),
7689 true, GSI_SAME_STMT
);
7691 if (gimple_in_ssa_p (cfun
))
7695 old_vali
= create_tmp_var (TREE_TYPE (loadedi
), NULL
);
7696 stmt
= gimple_build_assign (old_vali
, loadedi
);
7697 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7699 stmt
= gimple_build_assign (loadedi
, new_storedi
);
7700 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7703 /* Note that we always perform the comparison as an integer, even for
7704 floating point. This allows the atomic operation to properly
7705 succeed even with NaNs and -0.0. */
7706 stmt
= gimple_build_cond_empty
7707 (build2 (NE_EXPR
, boolean_type_node
,
7708 new_storedi
, old_vali
));
7709 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7712 e
= single_succ_edge (store_bb
);
7713 e
->flags
&= ~EDGE_FALLTHRU
;
7714 e
->flags
|= EDGE_FALSE_VALUE
;
7716 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
7718 /* Copy the new value to loadedi (we already did that before the condition
7719 if we are not in SSA). */
7720 if (gimple_in_ssa_p (cfun
))
7722 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
7723 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
7726 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
7727 gsi_remove (&si
, true);
7729 struct loop
*loop
= alloc_loop ();
7730 loop
->header
= loop_header
;
7731 loop
->latch
= store_bb
;
7732 add_loop (loop
, loop_header
->loop_father
);
7734 if (gimple_in_ssa_p (cfun
))
7735 update_ssa (TODO_update_ssa_no_phi
);
7740 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7742 GOMP_atomic_start ();
7746 The result is not globally atomic, but works so long as all parallel
7747 references are within #pragma omp atomic directives. According to
7748 responses received from omp@openmp.org, appears to be within spec.
7749 Which makes sense, since that's how several other compilers handle
7750 this situation as well.
7751 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
7752 expanding. STORED_VAL is the operand of the matching
7753 GIMPLE_OMP_ATOMIC_STORE.
7756 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
7760 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
7765 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
7766 tree addr
, tree loaded_val
, tree stored_val
)
7768 gimple_stmt_iterator si
;
7772 si
= gsi_last_bb (load_bb
);
7773 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7775 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
7776 t
= build_call_expr (t
, 0);
7777 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7779 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
7780 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7781 gsi_remove (&si
, true);
7783 si
= gsi_last_bb (store_bb
);
7784 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
7786 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
7788 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7790 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
7791 t
= build_call_expr (t
, 0);
7792 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7793 gsi_remove (&si
, true);
7795 if (gimple_in_ssa_p (cfun
))
7796 update_ssa (TODO_update_ssa_no_phi
);
7800 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
7801 using expand_omp_atomic_fetch_op. If it failed, we try to
7802 call expand_omp_atomic_pipeline, and if it fails too, the
7803 ultimate fallback is wrapping the operation in a mutex
7804 (expand_omp_atomic_mutex). REGION is the atomic region built
7805 by build_omp_regions_1(). */
7808 expand_omp_atomic (struct omp_region
*region
)
7810 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
7811 gimple load
= last_stmt (load_bb
), store
= last_stmt (store_bb
);
7812 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
7813 tree addr
= gimple_omp_atomic_load_rhs (load
);
7814 tree stored_val
= gimple_omp_atomic_store_val (store
);
7815 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7816 HOST_WIDE_INT index
;
7818 /* Make sure the type is one of the supported sizes. */
7819 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
7820 index
= exact_log2 (index
);
7821 if (index
>= 0 && index
<= 4)
7823 unsigned int align
= TYPE_ALIGN_UNIT (type
);
7825 /* __sync builtins require strict data alignment. */
7826 if (exact_log2 (align
) >= index
)
7829 if (loaded_val
== stored_val
7830 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7831 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7832 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7833 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
7837 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
7838 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
7839 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
7840 && store_bb
== single_succ (load_bb
)
7841 && first_stmt (store_bb
) == store
7842 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
7846 /* When possible, use specialized atomic update functions. */
7847 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
7848 && store_bb
== single_succ (load_bb
)
7849 && expand_omp_atomic_fetch_op (load_bb
, addr
,
7850 loaded_val
, stored_val
, index
))
7853 /* If we don't have specialized __sync builtins, try and implement
7854 as a compare and swap loop. */
7855 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
7856 loaded_val
, stored_val
, index
))
7861 /* The ultimate fallback is wrapping the operation in a mutex. */
7862 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
7866 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
7869 expand_omp_target (struct omp_region
*region
)
7871 basic_block entry_bb
, exit_bb
, new_bb
;
7872 struct function
*child_cfun
= NULL
;
7873 tree child_fn
= NULL_TREE
, block
, t
;
7874 gimple_stmt_iterator gsi
;
7875 gimple entry_stmt
, stmt
;
7878 entry_stmt
= last_stmt (region
->entry
);
7879 new_bb
= region
->entry
;
7880 int kind
= gimple_omp_target_kind (entry_stmt
);
7881 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7883 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
7884 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
7887 entry_bb
= region
->entry
;
7888 exit_bb
= region
->exit
;
7890 if (kind
== GF_OMP_TARGET_KIND_REGION
)
7892 unsigned srcidx
, dstidx
, num
;
7894 /* If the target region needs data sent from the parent
7895 function, then the very first statement (except possible
7896 tree profile counter updates) of the parallel body
7897 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
7898 &.OMP_DATA_O is passed as an argument to the child function,
7899 we need to replace it with the argument as seen by the child
7902 In most cases, this will end up being the identity assignment
7903 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
7904 a function call that has been inlined, the original PARM_DECL
7905 .OMP_DATA_I may have been converted into a different local
7906 variable. In which case, we need to keep the assignment. */
7907 if (gimple_omp_target_data_arg (entry_stmt
))
7909 basic_block entry_succ_bb
= single_succ (entry_bb
);
7910 gimple_stmt_iterator gsi
;
7912 gimple tgtcopy_stmt
= NULL
;
7914 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt
), 0);
7916 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
7918 gcc_assert (!gsi_end_p (gsi
));
7919 stmt
= gsi_stmt (gsi
);
7920 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
7923 if (gimple_num_ops (stmt
) == 2)
7925 tree arg
= gimple_assign_rhs1 (stmt
);
7927 /* We're ignoring the subcode because we're
7928 effectively doing a STRIP_NOPS. */
7930 if (TREE_CODE (arg
) == ADDR_EXPR
7931 && TREE_OPERAND (arg
, 0) == sender
)
7933 tgtcopy_stmt
= stmt
;
7939 gcc_assert (tgtcopy_stmt
!= NULL
);
7940 arg
= DECL_ARGUMENTS (child_fn
);
7942 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
7943 gsi_remove (&gsi
, true);
7946 /* Declare local variables needed in CHILD_CFUN. */
7947 block
= DECL_INITIAL (child_fn
);
7948 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
7949 /* The gimplifier could record temporaries in target block
7950 rather than in containing function's local_decls chain,
7951 which would mean cgraph missed finalizing them. Do it now. */
7952 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
7953 if (TREE_CODE (t
) == VAR_DECL
7955 && !DECL_EXTERNAL (t
))
7956 varpool_finalize_decl (t
);
7957 DECL_SAVED_TREE (child_fn
) = NULL
;
7958 /* We'll create a CFG for child_fn, so no gimple body is needed. */
7959 gimple_set_body (child_fn
, NULL
);
7960 TREE_USED (block
) = 1;
7962 /* Reset DECL_CONTEXT on function arguments. */
7963 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
7964 DECL_CONTEXT (t
) = child_fn
;
7966 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
7967 so that it can be moved to the child function. */
7968 gsi
= gsi_last_bb (entry_bb
);
7969 stmt
= gsi_stmt (gsi
);
7970 gcc_assert (stmt
&& gimple_code (stmt
) == GIMPLE_OMP_TARGET
7971 && gimple_omp_target_kind (stmt
)
7972 == GF_OMP_TARGET_KIND_REGION
);
7973 gsi_remove (&gsi
, true);
7974 e
= split_block (entry_bb
, stmt
);
7976 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7978 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
7981 gsi
= gsi_last_bb (exit_bb
);
7982 gcc_assert (!gsi_end_p (gsi
)
7983 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
7984 stmt
= gimple_build_return (NULL
);
7985 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
7986 gsi_remove (&gsi
, true);
7989 /* Move the target region into CHILD_CFUN. */
7991 block
= gimple_block (entry_stmt
);
7993 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
7995 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
7996 /* When the OMP expansion process cannot guarantee an up-to-date
7997 loop tree arrange for the child function to fixup loops. */
7998 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
7999 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8001 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8002 num
= vec_safe_length (child_cfun
->local_decls
);
8003 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8005 t
= (*child_cfun
->local_decls
)[srcidx
];
8006 if (DECL_CONTEXT (t
) == cfun
->decl
)
8008 if (srcidx
!= dstidx
)
8009 (*child_cfun
->local_decls
)[dstidx
] = t
;
8013 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8015 /* Inform the callgraph about the new function. */
8016 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
8017 cgraph_add_new_function (child_fn
, true);
8019 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8020 fixed in a following pass. */
8021 push_cfun (child_cfun
);
8022 rebuild_cgraph_edges ();
8024 /* Some EH regions might become dead, see PR34608. If
8025 pass_cleanup_cfg isn't the first pass to happen with the
8026 new child, these dead EH edges might cause problems.
8027 Clean them up now. */
8028 if (flag_exceptions
)
8031 bool changed
= false;
8033 FOR_EACH_BB_FN (bb
, cfun
)
8034 changed
|= gimple_purge_dead_eh_edges (bb
);
8036 cleanup_tree_cfg ();
8041 /* Emit a library call to launch the target region, or do data
8043 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8044 enum built_in_function start_ix
;
8045 location_t clause_loc
;
8047 clauses
= gimple_omp_target_clauses (entry_stmt
);
8049 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8050 start_ix
= BUILT_IN_GOMP_TARGET
;
8051 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
8052 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
8054 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
8056 /* By default, the value of DEVICE is -1 (let runtime library choose)
8057 and there is no conditional. */
8059 device
= build_int_cst (integer_type_node
, -1);
8061 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
8063 cond
= OMP_CLAUSE_IF_EXPR (c
);
8065 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
8068 device
= OMP_CLAUSE_DEVICE_ID (c
);
8069 clause_loc
= OMP_CLAUSE_LOCATION (c
);
8072 clause_loc
= gimple_location (entry_stmt
);
8074 /* Ensure 'device' is of the correct type. */
8075 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
8077 /* If we found the clause 'if (cond)', build
8078 (cond ? device : -2). */
8081 cond
= gimple_boolify (cond
);
8083 basic_block cond_bb
, then_bb
, else_bb
;
8087 tmp_var
= create_tmp_var (TREE_TYPE (device
), NULL
);
8088 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8090 gsi
= gsi_last_bb (new_bb
);
8092 e
= split_block (new_bb
, gsi_stmt (gsi
));
8095 e
= split_block (new_bb
, NULL
);
8100 then_bb
= create_empty_bb (cond_bb
);
8101 else_bb
= create_empty_bb (then_bb
);
8102 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
8103 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
8105 stmt
= gimple_build_cond_empty (cond
);
8106 gsi
= gsi_last_bb (cond_bb
);
8107 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8109 gsi
= gsi_start_bb (then_bb
);
8110 stmt
= gimple_build_assign (tmp_var
, device
);
8111 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8113 gsi
= gsi_start_bb (else_bb
);
8114 stmt
= gimple_build_assign (tmp_var
,
8115 build_int_cst (integer_type_node
, -2));
8116 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8118 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
8119 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
8120 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
8121 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
8122 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
8123 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
8128 gsi
= gsi_last_bb (new_bb
);
8129 t
= gimple_omp_target_data_arg (entry_stmt
);
8132 t1
= size_zero_node
;
8133 t2
= build_zero_cst (ptr_type_node
);
8139 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
8140 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
8141 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
8142 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
8143 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
8147 /* FIXME: This will be address of
8148 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
8149 symbol, as soon as the linker plugin is able to create it for us. */
8150 tree openmp_target
= build_zero_cst (ptr_type_node
);
8151 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8153 tree fnaddr
= build_fold_addr_expr (child_fn
);
8154 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 7,
8155 device
, fnaddr
, openmp_target
, t1
, t2
, t3
, t4
);
8158 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 6,
8159 device
, openmp_target
, t1
, t2
, t3
, t4
);
8160 gimple_set_location (g
, gimple_location (entry_stmt
));
8161 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8162 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8165 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
8166 gsi_remove (&gsi
, true);
8168 if (kind
== GF_OMP_TARGET_KIND_DATA
&& region
->exit
)
8170 gsi
= gsi_last_bb (region
->exit
);
8172 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
8173 gsi_remove (&gsi
, true);
8178 /* Expand the parallel region tree rooted at REGION. Expansion
8179 proceeds in depth-first order. Innermost regions are expanded
8180 first. This way, parallel regions that require a new function to
8181 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8182 internal dependencies in their body. */
8185 expand_omp (struct omp_region
*region
)
8189 location_t saved_location
;
8190 gimple inner_stmt
= NULL
;
8192 /* First, determine whether this is a combined parallel+workshare
8194 if (region
->type
== GIMPLE_OMP_PARALLEL
)
8195 determine_parallel_type (region
);
8197 if (region
->type
== GIMPLE_OMP_FOR
8198 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
8199 inner_stmt
= last_stmt (region
->inner
->entry
);
8202 expand_omp (region
->inner
);
8204 saved_location
= input_location
;
8205 if (gimple_has_location (last_stmt (region
->entry
)))
8206 input_location
= gimple_location (last_stmt (region
->entry
));
8208 switch (region
->type
)
8210 case GIMPLE_OMP_PARALLEL
:
8211 case GIMPLE_OMP_TASK
:
8212 expand_omp_taskreg (region
);
8215 case GIMPLE_OMP_FOR
:
8216 expand_omp_for (region
, inner_stmt
);
8219 case GIMPLE_OMP_SECTIONS
:
8220 expand_omp_sections (region
);
8223 case GIMPLE_OMP_SECTION
:
8224 /* Individual omp sections are handled together with their
8225 parent GIMPLE_OMP_SECTIONS region. */
8228 case GIMPLE_OMP_SINGLE
:
8229 expand_omp_single (region
);
8232 case GIMPLE_OMP_MASTER
:
8233 case GIMPLE_OMP_TASKGROUP
:
8234 case GIMPLE_OMP_ORDERED
:
8235 case GIMPLE_OMP_CRITICAL
:
8236 case GIMPLE_OMP_TEAMS
:
8237 expand_omp_synch (region
);
8240 case GIMPLE_OMP_ATOMIC_LOAD
:
8241 expand_omp_atomic (region
);
8244 case GIMPLE_OMP_TARGET
:
8245 expand_omp_target (region
);
8252 input_location
= saved_location
;
8253 region
= region
->next
;
8258 /* Helper for build_omp_regions. Scan the dominator tree starting at
8259 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8260 true, the function ends once a single tree is built (otherwise, whole
8261 forest of OMP constructs may be built). */
8264 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
8267 gimple_stmt_iterator gsi
;
8271 gsi
= gsi_last_bb (bb
);
8272 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
8274 struct omp_region
*region
;
8275 enum gimple_code code
;
8277 stmt
= gsi_stmt (gsi
);
8278 code
= gimple_code (stmt
);
8279 if (code
== GIMPLE_OMP_RETURN
)
8281 /* STMT is the return point out of region PARENT. Mark it
8282 as the exit point and make PARENT the immediately
8283 enclosing region. */
8284 gcc_assert (parent
);
8287 parent
= parent
->outer
;
8289 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
8291 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8292 GIMPLE_OMP_RETURN, but matches with
8293 GIMPLE_OMP_ATOMIC_LOAD. */
8294 gcc_assert (parent
);
8295 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
8298 parent
= parent
->outer
;
8301 else if (code
== GIMPLE_OMP_CONTINUE
)
8303 gcc_assert (parent
);
8306 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
8308 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8309 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8312 else if (code
== GIMPLE_OMP_TARGET
8313 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_UPDATE
)
8314 new_omp_region (bb
, code
, parent
);
8317 /* Otherwise, this directive becomes the parent for a new
8319 region
= new_omp_region (bb
, code
, parent
);
8324 if (single_tree
&& !parent
)
8327 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
8329 son
= next_dom_son (CDI_DOMINATORS
, son
))
8330 build_omp_regions_1 (son
, parent
, single_tree
);
8333 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8337 build_omp_regions_root (basic_block root
)
8339 gcc_assert (root_omp_region
== NULL
);
8340 build_omp_regions_1 (root
, NULL
, true);
8341 gcc_assert (root_omp_region
!= NULL
);
8344 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8347 omp_expand_local (basic_block head
)
8349 build_omp_regions_root (head
);
8350 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
8352 fprintf (dump_file
, "\nOMP region tree\n\n");
8353 dump_omp_region (dump_file
, root_omp_region
, 0);
8354 fprintf (dump_file
, "\n");
8357 remove_exit_barriers (root_omp_region
);
8358 expand_omp (root_omp_region
);
8360 free_omp_regions ();
8363 /* Scan the CFG and build a tree of OMP regions. Return the root of
8364 the OMP region tree. */
8367 build_omp_regions (void)
8369 gcc_assert (root_omp_region
== NULL
);
8370 calculate_dominance_info (CDI_DOMINATORS
);
8371 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
8374 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8377 execute_expand_omp (void)
8379 build_omp_regions ();
8381 if (!root_omp_region
)
8386 fprintf (dump_file
, "\nOMP region tree\n\n");
8387 dump_omp_region (dump_file
, root_omp_region
, 0);
8388 fprintf (dump_file
, "\n");
8391 remove_exit_barriers (root_omp_region
);
8393 expand_omp (root_omp_region
);
8395 cleanup_tree_cfg ();
8397 free_omp_regions ();
8402 /* OMP expansion -- the default pass, run before creation of SSA form. */
8406 const pass_data pass_data_expand_omp
=
8408 GIMPLE_PASS
, /* type */
8409 "ompexp", /* name */
8410 OPTGROUP_NONE
, /* optinfo_flags */
8411 true, /* has_execute */
8412 TV_NONE
, /* tv_id */
8413 PROP_gimple_any
, /* properties_required */
8414 0, /* properties_provided */
8415 0, /* properties_destroyed */
8416 0, /* todo_flags_start */
8417 0, /* todo_flags_finish */
8420 class pass_expand_omp
: public gimple_opt_pass
8423 pass_expand_omp (gcc::context
*ctxt
)
8424 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
8427 /* opt_pass methods: */
8428 virtual bool gate (function
*)
8430 return ((flag_openmp
!= 0 || flag_openmp_simd
!= 0
8431 || flag_cilkplus
!= 0) && !seen_error ());
8434 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
8436 }; // class pass_expand_omp
8441 make_pass_expand_omp (gcc::context
*ctxt
)
8443 return new pass_expand_omp (ctxt
);
8446 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8448 /* If ctx is a worksharing context inside of a cancellable parallel
8449 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8450 and conditional branch to parallel's cancel_label to handle
8451 cancellation in the implicit barrier. */
8454 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
8456 gimple omp_return
= gimple_seq_last_stmt (*body
);
8457 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
8458 if (gimple_omp_return_nowait_p (omp_return
))
8461 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
8462 && ctx
->outer
->cancellable
)
8464 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
8465 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
8466 tree lhs
= create_tmp_var (c_bool_type
, NULL
);
8467 gimple_omp_return_set_lhs (omp_return
, lhs
);
8468 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
8469 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
8470 fold_convert (c_bool_type
,
8471 boolean_false_node
),
8472 ctx
->outer
->cancel_label
, fallthru_label
);
8473 gimple_seq_add_stmt (body
, g
);
8474 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
8478 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8479 CTX is the enclosing OMP context for the current statement. */
8482 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8484 tree block
, control
;
8485 gimple_stmt_iterator tgsi
;
8486 gimple stmt
, new_stmt
, bind
, t
;
8487 gimple_seq ilist
, dlist
, olist
, new_body
;
8489 stmt
= gsi_stmt (*gsi_p
);
8491 push_gimplify_context ();
8495 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
8496 &ilist
, &dlist
, ctx
, NULL
);
8498 new_body
= gimple_omp_body (stmt
);
8499 gimple_omp_set_body (stmt
, NULL
);
8500 tgsi
= gsi_start (new_body
);
8501 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
8506 sec_start
= gsi_stmt (tgsi
);
8507 sctx
= maybe_lookup_ctx (sec_start
);
8510 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
8511 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
8512 GSI_CONTINUE_LINKING
);
8513 gimple_omp_set_body (sec_start
, NULL
);
8515 if (gsi_one_before_end_p (tgsi
))
8517 gimple_seq l
= NULL
;
8518 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
8520 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
8521 gimple_omp_section_set_last (sec_start
);
8524 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
8525 GSI_CONTINUE_LINKING
);
8528 block
= make_node (BLOCK
);
8529 bind
= gimple_build_bind (NULL
, new_body
, block
);
8532 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
8534 block
= make_node (BLOCK
);
8535 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8536 gsi_replace (gsi_p
, new_stmt
, true);
8538 pop_gimplify_context (new_stmt
);
8539 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
8540 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8541 if (BLOCK_VARS (block
))
8542 TREE_USED (block
) = 1;
8545 gimple_seq_add_seq (&new_body
, ilist
);
8546 gimple_seq_add_stmt (&new_body
, stmt
);
8547 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
8548 gimple_seq_add_stmt (&new_body
, bind
);
8550 control
= create_tmp_var (unsigned_type_node
, ".section");
8551 t
= gimple_build_omp_continue (control
, control
);
8552 gimple_omp_sections_set_control (stmt
, control
);
8553 gimple_seq_add_stmt (&new_body
, t
);
8555 gimple_seq_add_seq (&new_body
, olist
);
8556 if (ctx
->cancellable
)
8557 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
8558 gimple_seq_add_seq (&new_body
, dlist
);
8560 new_body
= maybe_catch_exception (new_body
);
8562 t
= gimple_build_omp_return
8563 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
8564 OMP_CLAUSE_NOWAIT
));
8565 gimple_seq_add_stmt (&new_body
, t
);
8566 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
8568 gimple_bind_set_body (new_stmt
, new_body
);
8572 /* A subroutine of lower_omp_single. Expand the simple form of
8573 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8575 if (GOMP_single_start ())
8577 [ GOMP_barrier (); ] -> unless 'nowait' is present.
8579 FIXME. It may be better to delay expanding the logic of this until
8580 pass_expand_omp. The expanded logic may make the job more difficult
8581 to a synchronization analysis pass. */
8584 lower_omp_single_simple (gimple single_stmt
, gimple_seq
*pre_p
)
8586 location_t loc
= gimple_location (single_stmt
);
8587 tree tlabel
= create_artificial_label (loc
);
8588 tree flabel
= create_artificial_label (loc
);
8592 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
8593 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)), NULL
);
8594 call
= gimple_build_call (decl
, 0);
8595 gimple_call_set_lhs (call
, lhs
);
8596 gimple_seq_add_stmt (pre_p
, call
);
8598 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
8599 fold_convert_loc (loc
, TREE_TYPE (lhs
),
8602 gimple_seq_add_stmt (pre_p
, cond
);
8603 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
8604 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8605 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
8609 /* A subroutine of lower_omp_single. Expand the simple form of
8610 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8612 #pragma omp single copyprivate (a, b, c)
8614 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
8617 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
8623 GOMP_single_copy_end (©out);
8634 FIXME. It may be better to delay expanding the logic of this until
8635 pass_expand_omp. The expanded logic may make the job more difficult
8636 to a synchronization analysis pass. */
8639 lower_omp_single_copy (gimple single_stmt
, gimple_seq
*pre_p
, omp_context
*ctx
)
8641 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
8642 gimple_seq copyin_seq
;
8643 location_t loc
= gimple_location (single_stmt
);
8645 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
8647 ptr_type
= build_pointer_type (ctx
->record_type
);
8648 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
8650 l0
= create_artificial_label (loc
);
8651 l1
= create_artificial_label (loc
);
8652 l2
= create_artificial_label (loc
);
8654 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
8655 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
8656 t
= fold_convert_loc (loc
, ptr_type
, t
);
8657 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
8659 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
8660 build_int_cst (ptr_type
, 0));
8661 t
= build3 (COND_EXPR
, void_type_node
, t
,
8662 build_and_jump (&l0
), build_and_jump (&l1
));
8663 gimplify_and_add (t
, pre_p
);
8665 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
8667 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8670 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
8673 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
8674 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
8675 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
8676 gimplify_and_add (t
, pre_p
);
8678 t
= build_and_jump (&l2
);
8679 gimplify_and_add (t
, pre_p
);
8681 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
8683 gimple_seq_add_seq (pre_p
, copyin_seq
);
8685 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
8689 /* Expand code for an OpenMP single directive. */
8692 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8695 gimple t
, bind
, single_stmt
= gsi_stmt (*gsi_p
);
8696 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
8698 push_gimplify_context ();
8700 block
= make_node (BLOCK
);
8701 bind
= gimple_build_bind (NULL
, NULL
, block
);
8702 gsi_replace (gsi_p
, bind
, true);
8705 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
8706 &bind_body
, &dlist
, ctx
, NULL
);
8707 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
8709 gimple_seq_add_stmt (&bind_body
, single_stmt
);
8711 if (ctx
->record_type
)
8712 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
8714 lower_omp_single_simple (single_stmt
, &bind_body
);
8716 gimple_omp_set_body (single_stmt
, NULL
);
8718 gimple_seq_add_seq (&bind_body
, dlist
);
8720 bind_body
= maybe_catch_exception (bind_body
);
8722 t
= gimple_build_omp_return
8723 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
8724 OMP_CLAUSE_NOWAIT
));
8725 gimple_seq_add_stmt (&bind_body_tail
, t
);
8726 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
8727 if (ctx
->record_type
)
8729 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
8730 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
8731 TREE_THIS_VOLATILE (clobber
) = 1;
8732 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
8733 clobber
), GSI_SAME_STMT
);
8735 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
8736 gimple_bind_set_body (bind
, bind_body
);
8738 pop_gimplify_context (bind
);
8740 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8741 BLOCK_VARS (block
) = ctx
->block_vars
;
8742 if (BLOCK_VARS (block
))
8743 TREE_USED (block
) = 1;
8747 /* Expand code for an OpenMP master directive. */
8750 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8752 tree block
, lab
= NULL
, x
, bfn_decl
;
8753 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8754 location_t loc
= gimple_location (stmt
);
8757 push_gimplify_context ();
8759 block
= make_node (BLOCK
);
8760 bind
= gimple_build_bind (NULL
, NULL
, block
);
8761 gsi_replace (gsi_p
, bind
, true);
8762 gimple_bind_add_stmt (bind
, stmt
);
8764 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
8765 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
8766 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
8767 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
8769 gimplify_and_add (x
, &tseq
);
8770 gimple_bind_add_seq (bind
, tseq
);
8772 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8773 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8774 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8775 gimple_omp_set_body (stmt
, NULL
);
8777 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
8779 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8781 pop_gimplify_context (bind
);
8783 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8784 BLOCK_VARS (block
) = ctx
->block_vars
;
8788 /* Expand code for an OpenMP taskgroup directive. */
8791 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8793 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8794 tree block
= make_node (BLOCK
);
8796 bind
= gimple_build_bind (NULL
, NULL
, block
);
8797 gsi_replace (gsi_p
, bind
, true);
8798 gimple_bind_add_stmt (bind
, stmt
);
8800 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
8802 gimple_bind_add_stmt (bind
, x
);
8804 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8805 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8806 gimple_omp_set_body (stmt
, NULL
);
8808 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8810 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8811 BLOCK_VARS (block
) = ctx
->block_vars
;
8815 /* Expand code for an OpenMP ordered directive. */
8818 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8821 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
8823 push_gimplify_context ();
8825 block
= make_node (BLOCK
);
8826 bind
= gimple_build_bind (NULL
, NULL
, block
);
8827 gsi_replace (gsi_p
, bind
, true);
8828 gimple_bind_add_stmt (bind
, stmt
);
8830 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
8832 gimple_bind_add_stmt (bind
, x
);
8834 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8835 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8836 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8837 gimple_omp_set_body (stmt
, NULL
);
8839 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
8840 gimple_bind_add_stmt (bind
, x
);
8842 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8844 pop_gimplify_context (bind
);
8846 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8847 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8851 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
8852 substitution of a couple of function calls. But in the NAMED case,
8853 requires that languages coordinate a symbol name. It is therefore
8854 best put here in common code. */
8856 static GTY((param1_is (tree
), param2_is (tree
)))
8857 splay_tree critical_name_mutexes
;
8860 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8863 tree name
, lock
, unlock
;
8864 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
8865 location_t loc
= gimple_location (stmt
);
8868 name
= gimple_omp_critical_name (stmt
);
8874 if (!critical_name_mutexes
)
8875 critical_name_mutexes
8876 = splay_tree_new_ggc (splay_tree_compare_pointers
,
8877 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s
,
8878 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s
);
8880 n
= splay_tree_lookup (critical_name_mutexes
, (splay_tree_key
) name
);
8885 decl
= create_tmp_var_raw (ptr_type_node
, NULL
);
8887 new_str
= ACONCAT ((".gomp_critical_user_",
8888 IDENTIFIER_POINTER (name
), NULL
));
8889 DECL_NAME (decl
) = get_identifier (new_str
);
8890 TREE_PUBLIC (decl
) = 1;
8891 TREE_STATIC (decl
) = 1;
8892 DECL_COMMON (decl
) = 1;
8893 DECL_ARTIFICIAL (decl
) = 1;
8894 DECL_IGNORED_P (decl
) = 1;
8895 varpool_finalize_decl (decl
);
8897 splay_tree_insert (critical_name_mutexes
, (splay_tree_key
) name
,
8898 (splay_tree_value
) decl
);
8901 decl
= (tree
) n
->value
;
8903 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
8904 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
8906 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
8907 unlock
= build_call_expr_loc (loc
, unlock
, 1,
8908 build_fold_addr_expr_loc (loc
, decl
));
8912 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
8913 lock
= build_call_expr_loc (loc
, lock
, 0);
8915 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
8916 unlock
= build_call_expr_loc (loc
, unlock
, 0);
8919 push_gimplify_context ();
8921 block
= make_node (BLOCK
);
8922 bind
= gimple_build_bind (NULL
, NULL
, block
);
8923 gsi_replace (gsi_p
, bind
, true);
8924 gimple_bind_add_stmt (bind
, stmt
);
8926 tbody
= gimple_bind_body (bind
);
8927 gimplify_and_add (lock
, &tbody
);
8928 gimple_bind_set_body (bind
, tbody
);
8930 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
8931 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
8932 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
8933 gimple_omp_set_body (stmt
, NULL
);
8935 tbody
= gimple_bind_body (bind
);
8936 gimplify_and_add (unlock
, &tbody
);
8937 gimple_bind_set_body (bind
, tbody
);
8939 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
8941 pop_gimplify_context (bind
);
8942 gimple_bind_append_vars (bind
, ctx
->block_vars
);
8943 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8947 /* A subroutine of lower_omp_for. Generate code to emit the predicate
8948 for a lastprivate clause. Given a loop control predicate of (V
8949 cond N2), we gate the clause on (!(V cond N2)). The lowered form
8950 is appended to *DLIST, iterator initialization is appended to
8954 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
8955 gimple_seq
*dlist
, struct omp_context
*ctx
)
8957 tree clauses
, cond
, vinit
;
8958 enum tree_code cond_code
;
8961 cond_code
= fd
->loop
.cond_code
;
8962 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
8964 /* When possible, use a strict equality expression. This can let VRP
8965 type optimizations deduce the value and remove a copy. */
8966 if (tree_fits_shwi_p (fd
->loop
.step
))
8968 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
8969 if (step
== 1 || step
== -1)
8970 cond_code
= EQ_EXPR
;
8973 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
8975 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
8977 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
8978 if (!gimple_seq_empty_p (stmts
))
8980 gimple_seq_add_seq (&stmts
, *dlist
);
8983 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
8984 vinit
= fd
->loop
.n1
;
8985 if (cond_code
== EQ_EXPR
8986 && tree_fits_shwi_p (fd
->loop
.n2
)
8987 && ! integer_zerop (fd
->loop
.n2
))
8988 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
8990 vinit
= unshare_expr (vinit
);
8992 /* Initialize the iterator variable, so that threads that don't execute
8993 any iterations don't execute the lastprivate clauses by accident. */
8994 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
8999 /* Lower code for an OpenMP loop directive. */
9002 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9005 struct omp_for_data fd
, *fdp
= NULL
;
9006 gimple stmt
= gsi_stmt (*gsi_p
), new_stmt
;
9007 gimple_seq omp_for_body
, body
, dlist
;
9010 push_gimplify_context ();
9012 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
9014 block
= make_node (BLOCK
);
9015 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
9016 /* Replace at gsi right away, so that 'stmt' is no member
9017 of a sequence anymore as we're going to add to to a different
9019 gsi_replace (gsi_p
, new_stmt
, true);
9021 /* Move declaration of temporaries in the loop body before we make
9023 omp_for_body
= gimple_omp_body (stmt
);
9024 if (!gimple_seq_empty_p (omp_for_body
)
9025 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
9027 gimple inner_bind
= gimple_seq_first_stmt (omp_for_body
);
9028 tree vars
= gimple_bind_vars (inner_bind
);
9029 gimple_bind_append_vars (new_stmt
, vars
);
9030 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
9031 keep them on the inner_bind and it's block. */
9032 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
9033 if (gimple_bind_block (inner_bind
))
9034 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
9037 if (gimple_omp_for_combined_into_p (stmt
))
9039 extract_omp_for_data (stmt
, &fd
, NULL
);
9042 /* We need two temporaries with fd.loop.v type (istart/iend)
9043 and then (fd.collapse - 1) temporaries with the same
9044 type for count2 ... countN-1 vars if not constant. */
9046 tree type
= fd
.iter_type
;
9048 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
9049 count
+= fd
.collapse
- 1;
9050 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
9051 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
9055 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
9056 OMP_CLAUSE__LOOPTEMP_
);
9057 for (i
= 0; i
< count
; i
++)
9062 gcc_assert (outerc
);
9063 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
9064 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
9065 OMP_CLAUSE__LOOPTEMP_
);
9069 temp
= create_tmp_var (type
, NULL
);
9070 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
9072 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
9073 OMP_CLAUSE_DECL (*pc
) = temp
;
9074 pc
= &OMP_CLAUSE_CHAIN (*pc
);
9079 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
9082 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
9084 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
9086 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9088 /* Lower the header expressions. At this point, we can assume that
9089 the header is of the form:
9091 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
9093 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
9094 using the .omp_data_s mapping, if needed. */
9095 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
9097 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
9098 if (!is_gimple_min_invariant (*rhs_p
))
9099 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9101 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
9102 if (!is_gimple_min_invariant (*rhs_p
))
9103 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9105 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
9106 if (!is_gimple_min_invariant (*rhs_p
))
9107 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9110 /* Once lowered, extract the bounds and clauses. */
9111 extract_omp_for_data (stmt
, &fd
, NULL
);
9113 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
9115 gimple_seq_add_stmt (&body
, stmt
);
9116 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
9118 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
9121 /* After the loop, add exit clauses. */
9122 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
9124 if (ctx
->cancellable
)
9125 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
9127 gimple_seq_add_seq (&body
, dlist
);
9129 body
= maybe_catch_exception (body
);
9131 /* Region exit marker goes at the end of the loop body. */
9132 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
9133 maybe_add_implicit_barrier_cancel (ctx
, &body
);
9134 pop_gimplify_context (new_stmt
);
9136 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
9137 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
9138 if (BLOCK_VARS (block
))
9139 TREE_USED (block
) = 1;
9141 gimple_bind_set_body (new_stmt
, body
);
9142 gimple_omp_set_body (stmt
, NULL
);
9143 gimple_omp_for_set_pre_body (stmt
, NULL
);
9146 /* Callback for walk_stmts. Check if the current statement only contains
9147 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
9150 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
9151 bool *handled_ops_p
,
9152 struct walk_stmt_info
*wi
)
9154 int *info
= (int *) wi
->info
;
9155 gimple stmt
= gsi_stmt (*gsi_p
);
9157 *handled_ops_p
= true;
9158 switch (gimple_code (stmt
))
9162 case GIMPLE_OMP_FOR
:
9163 case GIMPLE_OMP_SECTIONS
:
9164 *info
= *info
== 0 ? 1 : -1;
9173 struct omp_taskcopy_context
9175 /* This field must be at the beginning, as we do "inheritance": Some
9176 callback functions for tree-inline.c (e.g., omp_copy_decl)
9177 receive a copy_body_data pointer that is up-casted to an
9178 omp_context pointer. */
9184 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
9186 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
9188 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
9189 return create_tmp_var (TREE_TYPE (var
), NULL
);
9195 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
9197 tree name
, new_fields
= NULL
, type
, f
;
9199 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
9200 name
= DECL_NAME (TYPE_NAME (orig_type
));
9201 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
9202 TYPE_DECL
, name
, type
);
9203 TYPE_NAME (type
) = name
;
9205 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
9207 tree new_f
= copy_node (f
);
9208 DECL_CONTEXT (new_f
) = type
;
9209 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
9210 TREE_CHAIN (new_f
) = new_fields
;
9211 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9212 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9213 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
9216 *pointer_map_insert (tcctx
->cb
.decl_map
, f
) = new_f
;
9218 TYPE_FIELDS (type
) = nreverse (new_fields
);
9223 /* Create task copyfn. */
9226 create_task_copyfn (gimple task_stmt
, omp_context
*ctx
)
9228 struct function
*child_cfun
;
9229 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
9230 tree record_type
, srecord_type
, bind
, list
;
9231 bool record_needs_remap
= false, srecord_needs_remap
= false;
9233 struct omp_taskcopy_context tcctx
;
9234 location_t loc
= gimple_location (task_stmt
);
9236 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
9237 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
9238 gcc_assert (child_cfun
->cfg
== NULL
);
9239 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
9241 /* Reset DECL_CONTEXT on function arguments. */
9242 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
9243 DECL_CONTEXT (t
) = child_fn
;
9245 /* Populate the function. */
9246 push_gimplify_context ();
9247 push_cfun (child_cfun
);
9249 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
9250 TREE_SIDE_EFFECTS (bind
) = 1;
9252 DECL_SAVED_TREE (child_fn
) = bind
;
9253 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
9255 /* Remap src and dst argument types if needed. */
9256 record_type
= ctx
->record_type
;
9257 srecord_type
= ctx
->srecord_type
;
9258 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
9259 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9261 record_needs_remap
= true;
9264 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
9265 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9267 srecord_needs_remap
= true;
9271 if (record_needs_remap
|| srecord_needs_remap
)
9273 memset (&tcctx
, '\0', sizeof (tcctx
));
9274 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
9275 tcctx
.cb
.dst_fn
= child_fn
;
9276 tcctx
.cb
.src_node
= cgraph_get_node (tcctx
.cb
.src_fn
);
9277 gcc_checking_assert (tcctx
.cb
.src_node
);
9278 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
9279 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
9280 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
9281 tcctx
.cb
.eh_lp_nr
= 0;
9282 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
9283 tcctx
.cb
.decl_map
= pointer_map_create ();
9286 if (record_needs_remap
)
9287 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
9288 if (srecord_needs_remap
)
9289 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
9292 tcctx
.cb
.decl_map
= NULL
;
9294 arg
= DECL_ARGUMENTS (child_fn
);
9295 TREE_TYPE (arg
) = build_pointer_type (record_type
);
9296 sarg
= DECL_CHAIN (arg
);
9297 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
9299 /* First pass: initialize temporaries used in record_type and srecord_type
9300 sizes and field offsets. */
9301 if (tcctx
.cb
.decl_map
)
9302 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9303 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9307 decl
= OMP_CLAUSE_DECL (c
);
9308 p
= (tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, decl
);
9311 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9312 sf
= (tree
) n
->value
;
9313 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9314 src
= build_simple_mem_ref_loc (loc
, sarg
);
9315 src
= omp_build_component_ref (src
, sf
);
9316 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
9317 append_to_statement_list (t
, &list
);
9320 /* Second pass: copy shared var pointers and copy construct non-VLA
9321 firstprivate vars. */
9322 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9323 switch (OMP_CLAUSE_CODE (c
))
9325 case OMP_CLAUSE_SHARED
:
9326 decl
= OMP_CLAUSE_DECL (c
);
9327 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9330 f
= (tree
) n
->value
;
9331 if (tcctx
.cb
.decl_map
)
9332 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9333 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9334 sf
= (tree
) n
->value
;
9335 if (tcctx
.cb
.decl_map
)
9336 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9337 src
= build_simple_mem_ref_loc (loc
, sarg
);
9338 src
= omp_build_component_ref (src
, sf
);
9339 dst
= build_simple_mem_ref_loc (loc
, arg
);
9340 dst
= omp_build_component_ref (dst
, f
);
9341 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9342 append_to_statement_list (t
, &list
);
9344 case OMP_CLAUSE_FIRSTPRIVATE
:
9345 decl
= OMP_CLAUSE_DECL (c
);
9346 if (is_variable_sized (decl
))
9348 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9351 f
= (tree
) n
->value
;
9352 if (tcctx
.cb
.decl_map
)
9353 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9354 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9357 sf
= (tree
) n
->value
;
9358 if (tcctx
.cb
.decl_map
)
9359 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9360 src
= build_simple_mem_ref_loc (loc
, sarg
);
9361 src
= omp_build_component_ref (src
, sf
);
9362 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
9363 src
= build_simple_mem_ref_loc (loc
, src
);
9367 dst
= build_simple_mem_ref_loc (loc
, arg
);
9368 dst
= omp_build_component_ref (dst
, f
);
9369 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9370 append_to_statement_list (t
, &list
);
9372 case OMP_CLAUSE_PRIVATE
:
9373 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
9375 decl
= OMP_CLAUSE_DECL (c
);
9376 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9377 f
= (tree
) n
->value
;
9378 if (tcctx
.cb
.decl_map
)
9379 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9380 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9383 sf
= (tree
) n
->value
;
9384 if (tcctx
.cb
.decl_map
)
9385 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9386 src
= build_simple_mem_ref_loc (loc
, sarg
);
9387 src
= omp_build_component_ref (src
, sf
);
9388 if (use_pointer_for_field (decl
, NULL
))
9389 src
= build_simple_mem_ref_loc (loc
, src
);
9393 dst
= build_simple_mem_ref_loc (loc
, arg
);
9394 dst
= omp_build_component_ref (dst
, f
);
9395 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9396 append_to_statement_list (t
, &list
);
9402 /* Last pass: handle VLA firstprivates. */
9403 if (tcctx
.cb
.decl_map
)
9404 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9405 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9409 decl
= OMP_CLAUSE_DECL (c
);
9410 if (!is_variable_sized (decl
))
9412 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9415 f
= (tree
) n
->value
;
9416 f
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, f
);
9417 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
9418 ind
= DECL_VALUE_EXPR (decl
);
9419 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
9420 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
9421 n
= splay_tree_lookup (ctx
->sfield_map
,
9422 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9423 sf
= (tree
) n
->value
;
9424 sf
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, sf
);
9425 src
= build_simple_mem_ref_loc (loc
, sarg
);
9426 src
= omp_build_component_ref (src
, sf
);
9427 src
= build_simple_mem_ref_loc (loc
, src
);
9428 dst
= build_simple_mem_ref_loc (loc
, arg
);
9429 dst
= omp_build_component_ref (dst
, f
);
9430 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9431 append_to_statement_list (t
, &list
);
9432 n
= splay_tree_lookup (ctx
->field_map
,
9433 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9434 df
= (tree
) n
->value
;
9435 df
= *(tree
*) pointer_map_contains (tcctx
.cb
.decl_map
, df
);
9436 ptr
= build_simple_mem_ref_loc (loc
, arg
);
9437 ptr
= omp_build_component_ref (ptr
, df
);
9438 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
9439 build_fold_addr_expr_loc (loc
, dst
));
9440 append_to_statement_list (t
, &list
);
9443 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
9444 append_to_statement_list (t
, &list
);
9446 if (tcctx
.cb
.decl_map
)
9447 pointer_map_destroy (tcctx
.cb
.decl_map
);
9448 pop_gimplify_context (NULL
);
9449 BIND_EXPR_BODY (bind
) = list
;
9454 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
9458 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
9460 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
9462 gcc_assert (clauses
);
9463 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9464 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
9465 switch (OMP_CLAUSE_DEPEND_KIND (c
))
9467 case OMP_CLAUSE_DEPEND_IN
:
9470 case OMP_CLAUSE_DEPEND_OUT
:
9471 case OMP_CLAUSE_DEPEND_INOUT
:
9477 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
9478 tree array
= create_tmp_var (type
, NULL
);
9479 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
9481 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
9482 gimple_seq_add_stmt (iseq
, g
);
9483 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
9485 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
9486 gimple_seq_add_stmt (iseq
, g
);
9487 for (i
= 0; i
< 2; i
++)
9489 if ((i
? n_in
: n_out
) == 0)
9491 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9492 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
9493 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
9495 tree t
= OMP_CLAUSE_DECL (c
);
9496 t
= fold_convert (ptr_type_node
, t
);
9497 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
9498 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
9499 NULL_TREE
, NULL_TREE
);
9500 g
= gimple_build_assign (r
, t
);
9501 gimple_seq_add_stmt (iseq
, g
);
9504 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
9505 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
9506 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
9507 OMP_CLAUSE_CHAIN (c
) = *p
;
9509 tree clobber
= build_constructor (type
, NULL
);
9510 TREE_THIS_VOLATILE (clobber
) = 1;
9511 g
= gimple_build_assign (array
, clobber
);
9512 gimple_seq_add_stmt (oseq
, g
);
9515 /* Lower the OpenMP parallel or task directive in the current statement
9516 in GSI_P. CTX holds context information for the directive. */
9519 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9523 gimple stmt
= gsi_stmt (*gsi_p
);
9524 gimple par_bind
, bind
, dep_bind
= NULL
;
9525 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
9526 location_t loc
= gimple_location (stmt
);
9528 clauses
= gimple_omp_taskreg_clauses (stmt
);
9529 par_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9530 par_body
= gimple_bind_body (par_bind
);
9531 child_fn
= ctx
->cb
.dst_fn
;
9532 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
9533 && !gimple_omp_parallel_combined_p (stmt
))
9535 struct walk_stmt_info wi
;
9538 memset (&wi
, 0, sizeof (wi
));
9541 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
9543 gimple_omp_parallel_set_combined_p (stmt
, true);
9545 gimple_seq dep_ilist
= NULL
;
9546 gimple_seq dep_olist
= NULL
;
9547 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
9548 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
9550 push_gimplify_context ();
9551 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
9552 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
9555 if (ctx
->srecord_type
)
9556 create_task_copyfn (stmt
, ctx
);
9558 push_gimplify_context ();
9563 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
9564 lower_omp (&par_body
, ctx
);
9565 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
9566 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
9568 /* Declare all the variables created by mapping and the variables
9569 declared in the scope of the parallel body. */
9570 record_vars_into (ctx
->block_vars
, child_fn
);
9571 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
9573 if (ctx
->record_type
)
9576 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
9577 : ctx
->record_type
, ".omp_data_o");
9578 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9579 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9580 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
9585 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
9586 lower_send_shared_vars (&ilist
, &olist
, ctx
);
9588 if (ctx
->record_type
)
9590 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
9591 TREE_THIS_VOLATILE (clobber
) = 1;
9592 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9596 /* Once all the expansions are done, sequence all the different
9597 fragments inside gimple_omp_body. */
9601 if (ctx
->record_type
)
9603 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9604 /* fixup_child_record_type might have changed receiver_decl's type. */
9605 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9606 gimple_seq_add_stmt (&new_body
,
9607 gimple_build_assign (ctx
->receiver_decl
, t
));
9610 gimple_seq_add_seq (&new_body
, par_ilist
);
9611 gimple_seq_add_seq (&new_body
, par_body
);
9612 gimple_seq_add_seq (&new_body
, par_rlist
);
9613 if (ctx
->cancellable
)
9614 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
9615 gimple_seq_add_seq (&new_body
, par_olist
);
9616 new_body
= maybe_catch_exception (new_body
);
9617 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9618 gimple_omp_set_body (stmt
, new_body
);
9620 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
9621 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
9622 gimple_bind_add_seq (bind
, ilist
);
9623 gimple_bind_add_stmt (bind
, stmt
);
9624 gimple_bind_add_seq (bind
, olist
);
9626 pop_gimplify_context (NULL
);
9630 gimple_bind_add_seq (dep_bind
, dep_ilist
);
9631 gimple_bind_add_stmt (dep_bind
, bind
);
9632 gimple_bind_add_seq (dep_bind
, dep_olist
);
9633 pop_gimplify_context (dep_bind
);
9637 /* Lower the OpenMP target directive in the current statement
9638 in GSI_P. CTX holds context information for the directive. */
9641 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9644 tree child_fn
, t
, c
;
9645 gimple stmt
= gsi_stmt (*gsi_p
);
9646 gimple tgt_bind
= NULL
, bind
;
9647 gimple_seq tgt_body
= NULL
, olist
, ilist
, new_body
;
9648 location_t loc
= gimple_location (stmt
);
9649 int kind
= gimple_omp_target_kind (stmt
);
9650 unsigned int map_cnt
= 0;
9652 clauses
= gimple_omp_target_clauses (stmt
);
9653 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9655 tgt_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9656 tgt_body
= gimple_bind_body (tgt_bind
);
9658 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9659 tgt_body
= gimple_omp_body (stmt
);
9660 child_fn
= ctx
->cb
.dst_fn
;
9662 push_gimplify_context ();
9664 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9665 switch (OMP_CLAUSE_CODE (c
))
9671 case OMP_CLAUSE_MAP
:
9673 case OMP_CLAUSE_FROM
:
9674 var
= OMP_CLAUSE_DECL (c
);
9677 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
9678 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9684 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
9686 tree var2
= DECL_VALUE_EXPR (var
);
9687 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
9688 var2
= TREE_OPERAND (var2
, 0);
9689 gcc_assert (DECL_P (var2
));
9693 if (!maybe_lookup_field (var
, ctx
))
9696 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9698 x
= build_receiver_ref (var
, true, ctx
);
9699 tree new_var
= lookup_decl (var
, ctx
);
9700 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9701 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9702 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9703 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
9704 x
= build_simple_mem_ref (x
);
9705 SET_DECL_VALUE_EXPR (new_var
, x
);
9706 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
9711 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9713 target_nesting_level
++;
9714 lower_omp (&tgt_body
, ctx
);
9715 target_nesting_level
--;
9717 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9718 lower_omp (&tgt_body
, ctx
);
9720 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9722 /* Declare all the variables created by mapping and the variables
9723 declared in the scope of the target body. */
9724 record_vars_into (ctx
->block_vars
, child_fn
);
9725 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
9730 if (ctx
->record_type
)
9733 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
9734 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9735 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9736 t
= make_tree_vec (3);
9737 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
9739 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
9741 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
9742 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
9743 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
9745 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node
,
9748 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
9749 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
9750 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
9751 gimple_omp_target_set_data_arg (stmt
, t
);
9753 vec
<constructor_elt
, va_gc
> *vsize
;
9754 vec
<constructor_elt
, va_gc
> *vkind
;
9755 vec_alloc (vsize
, map_cnt
);
9756 vec_alloc (vkind
, map_cnt
);
9757 unsigned int map_idx
= 0;
9759 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9760 switch (OMP_CLAUSE_CODE (c
))
9766 case OMP_CLAUSE_MAP
:
9768 case OMP_CLAUSE_FROM
:
9770 ovar
= OMP_CLAUSE_DECL (c
);
9773 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9774 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
9776 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
9777 == get_base_address (ovar
));
9778 nc
= OMP_CLAUSE_CHAIN (c
);
9779 ovar
= OMP_CLAUSE_DECL (nc
);
9783 tree x
= build_sender_ref (ovar
, ctx
);
9785 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
9786 gimplify_assign (x
, v
, &ilist
);
9792 if (DECL_SIZE (ovar
)
9793 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
9795 tree ovar2
= DECL_VALUE_EXPR (ovar
);
9796 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
9797 ovar2
= TREE_OPERAND (ovar2
, 0);
9798 gcc_assert (DECL_P (ovar2
));
9801 if (!maybe_lookup_field (ovar
, ctx
))
9807 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
9808 tree x
= build_sender_ref (ovar
, ctx
);
9809 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
9810 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
9811 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
9812 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
9814 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9816 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)), NULL
);
9817 mark_addressable (avar
);
9818 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
9819 avar
= build_fold_addr_expr (avar
);
9820 gimplify_assign (x
, avar
, &ilist
);
9822 else if (is_gimple_reg (var
))
9824 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
9825 tree avar
= create_tmp_var (TREE_TYPE (var
), NULL
);
9826 mark_addressable (avar
);
9827 if (OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_ALLOC
9828 && OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_FROM
)
9829 gimplify_assign (avar
, var
, &ilist
);
9830 avar
= build_fold_addr_expr (avar
);
9831 gimplify_assign (x
, avar
, &ilist
);
9832 if ((OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_FROM
9833 || OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_TOFROM
)
9834 && !TYPE_READONLY (TREE_TYPE (var
)))
9836 x
= build_sender_ref (ovar
, ctx
);
9837 x
= build_simple_mem_ref (x
);
9838 gimplify_assign (var
, x
, &olist
);
9843 var
= build_fold_addr_expr (var
);
9844 gimplify_assign (x
, var
, &ilist
);
9847 tree s
= OMP_CLAUSE_SIZE (c
);
9849 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
9850 s
= fold_convert (size_type_node
, s
);
9851 tree purpose
= size_int (map_idx
++);
9852 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
9853 if (TREE_CODE (s
) != INTEGER_CST
)
9854 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
9856 unsigned char tkind
= 0;
9857 switch (OMP_CLAUSE_CODE (c
))
9859 case OMP_CLAUSE_MAP
:
9860 tkind
= OMP_CLAUSE_MAP_KIND (c
);
9863 tkind
= OMP_CLAUSE_MAP_TO
;
9865 case OMP_CLAUSE_FROM
:
9866 tkind
= OMP_CLAUSE_MAP_FROM
;
9871 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
9872 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
9873 talign
= DECL_ALIGN_UNIT (ovar
);
9874 talign
= ceil_log2 (talign
);
9875 tkind
|= talign
<< 3;
9876 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
9877 build_int_cst (unsigned_char_type_node
,
9883 gcc_assert (map_idx
== map_cnt
);
9885 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
9886 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
9887 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
9888 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
9889 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
9891 gimple_seq initlist
= NULL
;
9892 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
9893 TREE_VEC_ELT (t
, 1)),
9894 &initlist
, true, NULL_TREE
);
9895 gimple_seq_add_seq (&ilist
, initlist
);
9897 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
9899 TREE_THIS_VOLATILE (clobber
) = 1;
9900 gimple_seq_add_stmt (&olist
,
9901 gimple_build_assign (TREE_VEC_ELT (t
, 1),
9905 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
9906 TREE_THIS_VOLATILE (clobber
) = 1;
9907 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9911 /* Once all the expansions are done, sequence all the different
9912 fragments inside gimple_omp_body. */
9916 if (ctx
->record_type
&& kind
== GF_OMP_TARGET_KIND_REGION
)
9918 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9919 /* fixup_child_record_type might have changed receiver_decl's type. */
9920 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9921 gimple_seq_add_stmt (&new_body
,
9922 gimple_build_assign (ctx
->receiver_decl
, t
));
9925 if (kind
== GF_OMP_TARGET_KIND_REGION
)
9927 gimple_seq_add_seq (&new_body
, tgt_body
);
9928 new_body
= maybe_catch_exception (new_body
);
9930 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
9931 new_body
= tgt_body
;
9932 if (kind
!= GF_OMP_TARGET_KIND_UPDATE
)
9934 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
9935 gimple_omp_set_body (stmt
, new_body
);
9938 bind
= gimple_build_bind (NULL
, NULL
,
9939 tgt_bind
? gimple_bind_block (tgt_bind
)
9941 gsi_replace (gsi_p
, bind
, true);
9942 gimple_bind_add_seq (bind
, ilist
);
9943 gimple_bind_add_stmt (bind
, stmt
);
9944 gimple_bind_add_seq (bind
, olist
);
9946 pop_gimplify_context (NULL
);
9949 /* Expand code for an OpenMP teams directive. */
9952 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9954 gimple teams_stmt
= gsi_stmt (*gsi_p
);
9955 push_gimplify_context ();
9957 tree block
= make_node (BLOCK
);
9958 gimple bind
= gimple_build_bind (NULL
, NULL
, block
);
9959 gsi_replace (gsi_p
, bind
, true);
9960 gimple_seq bind_body
= NULL
;
9961 gimple_seq dlist
= NULL
;
9962 gimple_seq olist
= NULL
;
9964 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
9965 OMP_CLAUSE_NUM_TEAMS
);
9966 if (num_teams
== NULL_TREE
)
9967 num_teams
= build_int_cst (unsigned_type_node
, 0);
9970 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
9971 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
9972 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
9974 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
9975 OMP_CLAUSE_THREAD_LIMIT
);
9976 if (thread_limit
== NULL_TREE
)
9977 thread_limit
= build_int_cst (unsigned_type_node
, 0);
9980 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
9981 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
9982 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
9986 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
9987 &bind_body
, &dlist
, ctx
, NULL
);
9988 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
9989 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
9990 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
9992 location_t loc
= gimple_location (teams_stmt
);
9993 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
9994 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
9995 gimple_set_location (call
, loc
);
9996 gimple_seq_add_stmt (&bind_body
, call
);
9998 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
9999 gimple_omp_set_body (teams_stmt
, NULL
);
10000 gimple_seq_add_seq (&bind_body
, olist
);
10001 gimple_seq_add_seq (&bind_body
, dlist
);
10002 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
10003 gimple_bind_set_body (bind
, bind_body
);
10005 pop_gimplify_context (bind
);
10007 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10008 BLOCK_VARS (block
) = ctx
->block_vars
;
10009 if (BLOCK_VARS (block
))
10010 TREE_USED (block
) = 1;
10014 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
10015 regimplified. If DATA is non-NULL, lower_omp_1 is outside
10016 of OpenMP context, but with task_shared_vars set. */
10019 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
10024 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
10025 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
10028 if (task_shared_vars
10030 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
10033 /* If a global variable has been privatized, TREE_CONSTANT on
10034 ADDR_EXPR might be wrong. */
10035 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
10036 recompute_tree_invariant_for_addr_expr (t
);
10038 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
10043 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10045 gimple stmt
= gsi_stmt (*gsi_p
);
10046 struct walk_stmt_info wi
;
10048 if (gimple_has_location (stmt
))
10049 input_location
= gimple_location (stmt
);
10051 if (task_shared_vars
)
10052 memset (&wi
, '\0', sizeof (wi
));
10054 /* If we have issued syntax errors, avoid doing any heavy lifting.
10055 Just replace the OpenMP directives with a NOP to avoid
10056 confusing RTL expansion. */
10057 if (seen_error () && is_gimple_omp (stmt
))
10059 gsi_replace (gsi_p
, gimple_build_nop (), true);
10063 switch (gimple_code (stmt
))
10066 if ((ctx
|| task_shared_vars
)
10067 && (walk_tree (gimple_cond_lhs_ptr (stmt
), lower_omp_regimplify_p
,
10068 ctx
? NULL
: &wi
, NULL
)
10069 || walk_tree (gimple_cond_rhs_ptr (stmt
), lower_omp_regimplify_p
,
10070 ctx
? NULL
: &wi
, NULL
)))
10071 gimple_regimplify_operands (stmt
, gsi_p
);
10074 lower_omp (gimple_catch_handler_ptr (stmt
), ctx
);
10076 case GIMPLE_EH_FILTER
:
10077 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
10080 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
10081 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
10083 case GIMPLE_TRANSACTION
:
10084 lower_omp (gimple_transaction_body_ptr (stmt
), ctx
);
10087 lower_omp (gimple_bind_body_ptr (stmt
), ctx
);
10089 case GIMPLE_OMP_PARALLEL
:
10090 case GIMPLE_OMP_TASK
:
10091 ctx
= maybe_lookup_ctx (stmt
);
10093 if (ctx
->cancellable
)
10094 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10095 lower_omp_taskreg (gsi_p
, ctx
);
10097 case GIMPLE_OMP_FOR
:
10098 ctx
= maybe_lookup_ctx (stmt
);
10100 if (ctx
->cancellable
)
10101 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10102 lower_omp_for (gsi_p
, ctx
);
10104 case GIMPLE_OMP_SECTIONS
:
10105 ctx
= maybe_lookup_ctx (stmt
);
10107 if (ctx
->cancellable
)
10108 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10109 lower_omp_sections (gsi_p
, ctx
);
10111 case GIMPLE_OMP_SINGLE
:
10112 ctx
= maybe_lookup_ctx (stmt
);
10114 lower_omp_single (gsi_p
, ctx
);
10116 case GIMPLE_OMP_MASTER
:
10117 ctx
= maybe_lookup_ctx (stmt
);
10119 lower_omp_master (gsi_p
, ctx
);
10121 case GIMPLE_OMP_TASKGROUP
:
10122 ctx
= maybe_lookup_ctx (stmt
);
10124 lower_omp_taskgroup (gsi_p
, ctx
);
10126 case GIMPLE_OMP_ORDERED
:
10127 ctx
= maybe_lookup_ctx (stmt
);
10129 lower_omp_ordered (gsi_p
, ctx
);
10131 case GIMPLE_OMP_CRITICAL
:
10132 ctx
= maybe_lookup_ctx (stmt
);
10134 lower_omp_critical (gsi_p
, ctx
);
10136 case GIMPLE_OMP_ATOMIC_LOAD
:
10137 if ((ctx
|| task_shared_vars
)
10138 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt
),
10139 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
10140 gimple_regimplify_operands (stmt
, gsi_p
);
10142 case GIMPLE_OMP_TARGET
:
10143 ctx
= maybe_lookup_ctx (stmt
);
10145 lower_omp_target (gsi_p
, ctx
);
10147 case GIMPLE_OMP_TEAMS
:
10148 ctx
= maybe_lookup_ctx (stmt
);
10150 lower_omp_teams (gsi_p
, ctx
);
10154 fndecl
= gimple_call_fndecl (stmt
);
10156 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
10157 switch (DECL_FUNCTION_CODE (fndecl
))
10159 case BUILT_IN_GOMP_BARRIER
:
10163 case BUILT_IN_GOMP_CANCEL
:
10164 case BUILT_IN_GOMP_CANCELLATION_POINT
:
10167 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
10168 cctx
= cctx
->outer
;
10169 gcc_assert (gimple_call_lhs (stmt
) == NULL_TREE
);
10170 if (!cctx
->cancellable
)
10172 if (DECL_FUNCTION_CODE (fndecl
)
10173 == BUILT_IN_GOMP_CANCELLATION_POINT
)
10175 stmt
= gimple_build_nop ();
10176 gsi_replace (gsi_p
, stmt
, false);
10180 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
10182 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
10183 gimple_call_set_fndecl (stmt
, fndecl
);
10184 gimple_call_set_fntype (stmt
, TREE_TYPE (fndecl
));
10187 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)), NULL
);
10188 gimple_call_set_lhs (stmt
, lhs
);
10189 tree fallthru_label
;
10190 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
10192 g
= gimple_build_label (fallthru_label
);
10193 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10194 g
= gimple_build_cond (NE_EXPR
, lhs
,
10195 fold_convert (TREE_TYPE (lhs
),
10196 boolean_false_node
),
10197 cctx
->cancel_label
, fallthru_label
);
10198 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10205 if ((ctx
|| task_shared_vars
)
10206 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
10209 /* Just remove clobbers, this should happen only if we have
10210 "privatized" local addressable variables in SIMD regions,
10211 the clobber isn't needed in that case and gimplifying address
10212 of the ARRAY_REF into a pointer and creating MEM_REF based
10213 clobber would create worse code than we get with the clobber
10215 if (gimple_clobber_p (stmt
))
10217 gsi_replace (gsi_p
, gimple_build_nop (), true);
10220 gimple_regimplify_operands (stmt
, gsi_p
);
10227 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
10229 location_t saved_location
= input_location
;
10230 gimple_stmt_iterator gsi
;
10231 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10232 lower_omp_1 (&gsi
, ctx
);
10233 /* During gimplification, we have not always invoked fold_stmt
10234 (gimplify.c:maybe_fold_stmt); call it now. */
10235 if (target_nesting_level
)
10236 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10238 input_location
= saved_location
;
10241 /* Main entry point. */
10243 static unsigned int
10244 execute_lower_omp (void)
10248 /* This pass always runs, to provide PROP_gimple_lomp.
10249 But there is nothing to do unless -fopenmp is given. */
10250 if (flag_openmp
== 0 && flag_openmp_simd
== 0 && flag_cilkplus
== 0)
10253 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
10254 delete_omp_context
);
10256 body
= gimple_body (current_function_decl
);
10257 scan_omp (&body
, NULL
);
10258 gcc_assert (taskreg_nesting_level
== 0);
10260 if (all_contexts
->root
)
10262 if (task_shared_vars
)
10263 push_gimplify_context ();
10264 lower_omp (&body
, NULL
);
10265 if (task_shared_vars
)
10266 pop_gimplify_context (NULL
);
10271 splay_tree_delete (all_contexts
);
10272 all_contexts
= NULL
;
10274 BITMAP_FREE (task_shared_vars
);
10280 const pass_data pass_data_lower_omp
=
10282 GIMPLE_PASS
, /* type */
10283 "omplower", /* name */
10284 OPTGROUP_NONE
, /* optinfo_flags */
10285 true, /* has_execute */
10286 TV_NONE
, /* tv_id */
10287 PROP_gimple_any
, /* properties_required */
10288 PROP_gimple_lomp
, /* properties_provided */
10289 0, /* properties_destroyed */
10290 0, /* todo_flags_start */
10291 0, /* todo_flags_finish */
10294 class pass_lower_omp
: public gimple_opt_pass
10297 pass_lower_omp (gcc::context
*ctxt
)
10298 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
10301 /* opt_pass methods: */
10302 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
10304 }; // class pass_lower_omp
10306 } // anon namespace
10309 make_pass_lower_omp (gcc::context
*ctxt
)
10311 return new pass_lower_omp (ctxt
);
10314 /* The following is a utility to diagnose OpenMP structured block violations.
10315 It is not part of the "omplower" pass, as that's invoked too late. It
10316 should be invoked by the respective front ends after gimplification. */
10318 static splay_tree all_labels
;
10320 /* Check for mismatched contexts and generate an error if needed. Return
10321 true if an error is detected. */
10324 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
10325 gimple branch_ctx
, gimple label_ctx
)
10327 if (label_ctx
== branch_ctx
)
10332 Previously we kept track of the label's entire context in diagnose_sb_[12]
10333 so we could traverse it and issue a correct "exit" or "enter" error
10334 message upon a structured block violation.
10336 We built the context by building a list with tree_cons'ing, but there is
10337 no easy counterpart in gimple tuples. It seems like far too much work
10338 for issuing exit/enter error messages. If someone really misses the
10339 distinct error message... patches welcome.
10343 /* Try to avoid confusing the user by producing and error message
10344 with correct "exit" or "enter" verbiage. We prefer "exit"
10345 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10346 if (branch_ctx
== NULL
)
10352 if (TREE_VALUE (label_ctx
) == branch_ctx
)
10357 label_ctx
= TREE_CHAIN (label_ctx
);
10362 error ("invalid exit from OpenMP structured block");
10364 error ("invalid entry to OpenMP structured block");
10367 bool cilkplus_block
= false;
10371 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
10372 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
10374 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
10375 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
10376 cilkplus_block
= true;
10379 /* If it's obvious we have an invalid entry, be specific about the error. */
10380 if (branch_ctx
== NULL
)
10382 if (cilkplus_block
)
10383 error ("invalid entry to Cilk Plus structured block");
10385 error ("invalid entry to OpenMP structured block");
10389 /* Otherwise, be vague and lazy, but efficient. */
10390 if (cilkplus_block
)
10391 error ("invalid branch to/from a Cilk Plus structured block");
10393 error ("invalid branch to/from an OpenMP structured block");
10396 gsi_replace (gsi_p
, gimple_build_nop (), false);
10400 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10401 where each label is found. */
10404 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10405 struct walk_stmt_info
*wi
)
10407 gimple context
= (gimple
) wi
->info
;
10408 gimple inner_context
;
10409 gimple stmt
= gsi_stmt (*gsi_p
);
10411 *handled_ops_p
= true;
10413 switch (gimple_code (stmt
))
10417 case GIMPLE_OMP_PARALLEL
:
10418 case GIMPLE_OMP_TASK
:
10419 case GIMPLE_OMP_SECTIONS
:
10420 case GIMPLE_OMP_SINGLE
:
10421 case GIMPLE_OMP_SECTION
:
10422 case GIMPLE_OMP_MASTER
:
10423 case GIMPLE_OMP_ORDERED
:
10424 case GIMPLE_OMP_CRITICAL
:
10425 case GIMPLE_OMP_TARGET
:
10426 case GIMPLE_OMP_TEAMS
:
10427 case GIMPLE_OMP_TASKGROUP
:
10428 /* The minimal context here is just the current OMP construct. */
10429 inner_context
= stmt
;
10430 wi
->info
= inner_context
;
10431 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10432 wi
->info
= context
;
10435 case GIMPLE_OMP_FOR
:
10436 inner_context
= stmt
;
10437 wi
->info
= inner_context
;
10438 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10440 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
10441 diagnose_sb_1
, NULL
, wi
);
10442 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10443 wi
->info
= context
;
10447 splay_tree_insert (all_labels
, (splay_tree_key
) gimple_label_label (stmt
),
10448 (splay_tree_value
) context
);
10458 /* Pass 2: Check each branch and see if its context differs from that of
10459 the destination label's context. */
10462 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10463 struct walk_stmt_info
*wi
)
10465 gimple context
= (gimple
) wi
->info
;
10467 gimple stmt
= gsi_stmt (*gsi_p
);
10469 *handled_ops_p
= true;
10471 switch (gimple_code (stmt
))
10475 case GIMPLE_OMP_PARALLEL
:
10476 case GIMPLE_OMP_TASK
:
10477 case GIMPLE_OMP_SECTIONS
:
10478 case GIMPLE_OMP_SINGLE
:
10479 case GIMPLE_OMP_SECTION
:
10480 case GIMPLE_OMP_MASTER
:
10481 case GIMPLE_OMP_ORDERED
:
10482 case GIMPLE_OMP_CRITICAL
:
10483 case GIMPLE_OMP_TARGET
:
10484 case GIMPLE_OMP_TEAMS
:
10485 case GIMPLE_OMP_TASKGROUP
:
10487 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10488 wi
->info
= context
;
10491 case GIMPLE_OMP_FOR
:
10493 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10495 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
10496 diagnose_sb_2
, NULL
, wi
);
10497 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10498 wi
->info
= context
;
10503 tree lab
= gimple_cond_true_label (stmt
);
10506 n
= splay_tree_lookup (all_labels
,
10507 (splay_tree_key
) lab
);
10508 diagnose_sb_0 (gsi_p
, context
,
10509 n
? (gimple
) n
->value
: NULL
);
10511 lab
= gimple_cond_false_label (stmt
);
10514 n
= splay_tree_lookup (all_labels
,
10515 (splay_tree_key
) lab
);
10516 diagnose_sb_0 (gsi_p
, context
,
10517 n
? (gimple
) n
->value
: NULL
);
10524 tree lab
= gimple_goto_dest (stmt
);
10525 if (TREE_CODE (lab
) != LABEL_DECL
)
10528 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10529 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
10533 case GIMPLE_SWITCH
:
10536 for (i
= 0; i
< gimple_switch_num_labels (stmt
); ++i
)
10538 tree lab
= CASE_LABEL (gimple_switch_label (stmt
, i
));
10539 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10540 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
10546 case GIMPLE_RETURN
:
10547 diagnose_sb_0 (gsi_p
, context
, NULL
);
10557 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
10560 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
10563 gimple last
= last_stmt (bb
);
10564 enum gimple_code code
= gimple_code (last
);
10565 struct omp_region
*cur_region
= *region
;
10566 bool fallthru
= false;
10570 case GIMPLE_OMP_PARALLEL
:
10571 case GIMPLE_OMP_TASK
:
10572 case GIMPLE_OMP_FOR
:
10573 case GIMPLE_OMP_SINGLE
:
10574 case GIMPLE_OMP_TEAMS
:
10575 case GIMPLE_OMP_MASTER
:
10576 case GIMPLE_OMP_TASKGROUP
:
10577 case GIMPLE_OMP_ORDERED
:
10578 case GIMPLE_OMP_CRITICAL
:
10579 case GIMPLE_OMP_SECTION
:
10580 cur_region
= new_omp_region (bb
, code
, cur_region
);
10584 case GIMPLE_OMP_TARGET
:
10585 cur_region
= new_omp_region (bb
, code
, cur_region
);
10587 if (gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_UPDATE
)
10588 cur_region
= cur_region
->outer
;
10591 case GIMPLE_OMP_SECTIONS
:
10592 cur_region
= new_omp_region (bb
, code
, cur_region
);
10596 case GIMPLE_OMP_SECTIONS_SWITCH
:
10600 case GIMPLE_OMP_ATOMIC_LOAD
:
10601 case GIMPLE_OMP_ATOMIC_STORE
:
10605 case GIMPLE_OMP_RETURN
:
10606 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
10607 somewhere other than the next block. This will be
10609 cur_region
->exit
= bb
;
10610 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
10611 cur_region
= cur_region
->outer
;
10614 case GIMPLE_OMP_CONTINUE
:
10615 cur_region
->cont
= bb
;
10616 switch (cur_region
->type
)
10618 case GIMPLE_OMP_FOR
:
10619 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
10620 succs edges as abnormal to prevent splitting
10622 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
10623 /* Make the loopback edge. */
10624 make_edge (bb
, single_succ (cur_region
->entry
),
10627 /* Create an edge from GIMPLE_OMP_FOR to exit, which
10628 corresponds to the case that the body of the loop
10629 is not executed at all. */
10630 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
10631 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
10635 case GIMPLE_OMP_SECTIONS
:
10636 /* Wire up the edges into and out of the nested sections. */
10638 basic_block switch_bb
= single_succ (cur_region
->entry
);
10640 struct omp_region
*i
;
10641 for (i
= cur_region
->inner
; i
; i
= i
->next
)
10643 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
10644 make_edge (switch_bb
, i
->entry
, 0);
10645 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
10648 /* Make the loopback edge to the block with
10649 GIMPLE_OMP_SECTIONS_SWITCH. */
10650 make_edge (bb
, switch_bb
, 0);
10652 /* Make the edge from the switch to exit. */
10653 make_edge (switch_bb
, bb
->next_bb
, 0);
10659 gcc_unreachable ();
10664 gcc_unreachable ();
10667 if (*region
!= cur_region
)
10669 *region
= cur_region
;
10671 *region_idx
= cur_region
->entry
->index
;
10679 static unsigned int
10680 diagnose_omp_structured_block_errors (void)
10682 struct walk_stmt_info wi
;
10683 gimple_seq body
= gimple_body (current_function_decl
);
10685 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
10687 memset (&wi
, 0, sizeof (wi
));
10688 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
10690 memset (&wi
, 0, sizeof (wi
));
10691 wi
.want_locations
= true;
10692 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
10694 gimple_set_body (current_function_decl
, body
);
10696 splay_tree_delete (all_labels
);
10704 const pass_data pass_data_diagnose_omp_blocks
=
10706 GIMPLE_PASS
, /* type */
10707 "*diagnose_omp_blocks", /* name */
10708 OPTGROUP_NONE
, /* optinfo_flags */
10709 true, /* has_execute */
10710 TV_NONE
, /* tv_id */
10711 PROP_gimple_any
, /* properties_required */
10712 0, /* properties_provided */
10713 0, /* properties_destroyed */
10714 0, /* todo_flags_start */
10715 0, /* todo_flags_finish */
10718 class pass_diagnose_omp_blocks
: public gimple_opt_pass
10721 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10722 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
10725 /* opt_pass methods: */
10726 virtual bool gate (function
*) { return flag_openmp
|| flag_cilkplus
; }
10727 virtual unsigned int execute (function
*)
10729 return diagnose_omp_structured_block_errors ();
10732 }; // class pass_diagnose_omp_blocks
10734 } // anon namespace
10737 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
10739 return new pass_diagnose_omp_blocks (ctxt
);
10742 /* SIMD clone supporting code. */
10744 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
10745 of arguments to reserve space for. */
10747 static struct cgraph_simd_clone
*
10748 simd_clone_struct_alloc (int nargs
)
10750 struct cgraph_simd_clone
*clone_info
;
10751 size_t len
= (sizeof (struct cgraph_simd_clone
)
10752 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
10753 clone_info
= (struct cgraph_simd_clone
*)
10754 ggc_internal_cleared_alloc (len
);
10758 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
10761 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
10762 struct cgraph_simd_clone
*from
)
10764 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
10765 + ((from
->nargs
- from
->inbranch
)
10766 * sizeof (struct cgraph_simd_clone_arg
))));
10769 /* Return vector of parameter types of function FNDECL. This uses
10770 TYPE_ARG_TYPES if available, otherwise falls back to types of
10771 DECL_ARGUMENTS types. */
10774 simd_clone_vector_of_formal_parm_types (tree fndecl
)
10776 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
10777 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
10778 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
10781 FOR_EACH_VEC_ELT (args
, i
, arg
)
10782 args
[i
] = TREE_TYPE (args
[i
]);
10786 /* Given a simd function in NODE, extract the simd specific
10787 information from the OMP clauses passed in CLAUSES, and return
10788 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
10789 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
10790 otherwise set to FALSE. */
10792 static struct cgraph_simd_clone
*
10793 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
10794 bool *inbranch_specified
)
10796 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
10799 *inbranch_specified
= false;
10801 n
= args
.length ();
10802 if (n
> 0 && args
.last () == void_type_node
)
10805 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
10806 be cloned have a distinctive artificial label in addition to "omp
10810 && lookup_attribute ("cilk simd function",
10811 DECL_ATTRIBUTES (node
->decl
)));
10813 /* Allocate one more than needed just in case this is an in-branch
10814 clone which will require a mask argument. */
10815 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
10816 clone_info
->nargs
= n
;
10817 clone_info
->cilk_elemental
= cilk_clone
;
10824 clauses
= TREE_VALUE (clauses
);
10825 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
10828 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
10830 switch (OMP_CLAUSE_CODE (t
))
10832 case OMP_CLAUSE_INBRANCH
:
10833 clone_info
->inbranch
= 1;
10834 *inbranch_specified
= true;
10836 case OMP_CLAUSE_NOTINBRANCH
:
10837 clone_info
->inbranch
= 0;
10838 *inbranch_specified
= true;
10840 case OMP_CLAUSE_SIMDLEN
:
10841 clone_info
->simdlen
10842 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
10844 case OMP_CLAUSE_LINEAR
:
10846 tree decl
= OMP_CLAUSE_DECL (t
);
10847 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
10848 int argno
= TREE_INT_CST_LOW (decl
);
10849 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
10851 clone_info
->args
[argno
].arg_type
10852 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
10853 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
10854 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
10855 && clone_info
->args
[argno
].linear_step
< n
);
10859 if (POINTER_TYPE_P (args
[argno
]))
10860 step
= fold_convert (ssizetype
, step
);
10861 if (!tree_fits_shwi_p (step
))
10863 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
10864 "ignoring large linear step");
10868 else if (integer_zerop (step
))
10870 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
10871 "ignoring zero linear step");
10877 clone_info
->args
[argno
].arg_type
10878 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
10879 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
10884 case OMP_CLAUSE_UNIFORM
:
10886 tree decl
= OMP_CLAUSE_DECL (t
);
10887 int argno
= tree_to_uhwi (decl
);
10888 clone_info
->args
[argno
].arg_type
10889 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
10892 case OMP_CLAUSE_ALIGNED
:
10894 tree decl
= OMP_CLAUSE_DECL (t
);
10895 int argno
= tree_to_uhwi (decl
);
10896 clone_info
->args
[argno
].alignment
10897 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
10908 /* Given a SIMD clone in NODE, calculate the characteristic data
10909 type and return the coresponding type. The characteristic data
10910 type is computed as described in the Intel Vector ABI. */
10913 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
10914 struct cgraph_simd_clone
*clone_info
)
10916 tree type
= integer_type_node
;
10917 tree fndecl
= node
->decl
;
10919 /* a) For non-void function, the characteristic data type is the
10921 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
10922 type
= TREE_TYPE (TREE_TYPE (fndecl
));
10924 /* b) If the function has any non-uniform, non-linear parameters,
10925 then the characteristic data type is the type of the first
10929 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
10930 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
10931 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
10939 /* c) If the characteristic data type determined by a) or b) above
10940 is struct, union, or class type which is pass-by-value (except
10941 for the type that maps to the built-in complex data type), the
10942 characteristic data type is int. */
10943 if (RECORD_OR_UNION_TYPE_P (type
)
10944 && !aggregate_value_p (type
, NULL
)
10945 && TREE_CODE (type
) != COMPLEX_TYPE
)
10946 return integer_type_node
;
10948 /* d) If none of the above three classes is applicable, the
10949 characteristic data type is int. */
10953 /* e) For Intel Xeon Phi native and offload compilation, if the
10954 resulting characteristic data type is 8-bit or 16-bit integer
10955 data type, the characteristic data type is int. */
10956 /* Well, we don't handle Xeon Phi yet. */
10960 simd_clone_mangle (struct cgraph_node
*node
,
10961 struct cgraph_simd_clone
*clone_info
)
10963 char vecsize_mangle
= clone_info
->vecsize_mangle
;
10964 char mask
= clone_info
->inbranch
? 'M' : 'N';
10965 unsigned int simdlen
= clone_info
->simdlen
;
10969 gcc_assert (vecsize_mangle
&& simdlen
);
10971 pp_string (&pp
, "_ZGV");
10972 pp_character (&pp
, vecsize_mangle
);
10973 pp_character (&pp
, mask
);
10974 pp_decimal_int (&pp
, simdlen
);
10976 for (n
= 0; n
< clone_info
->nargs
; ++n
)
10978 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
10980 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
10981 pp_character (&pp
, 'u');
10982 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
10984 gcc_assert (arg
.linear_step
!= 0);
10985 pp_character (&pp
, 'l');
10986 if (arg
.linear_step
> 1)
10987 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
10988 else if (arg
.linear_step
< 0)
10990 pp_character (&pp
, 'n');
10991 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
10995 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
10997 pp_character (&pp
, 's');
10998 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
11001 pp_character (&pp
, 'v');
11004 pp_character (&pp
, 'a');
11005 pp_decimal_int (&pp
, arg
.alignment
);
11009 pp_underscore (&pp
);
11011 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
)));
11012 const char *str
= pp_formatted_text (&pp
);
11014 /* If there already is a SIMD clone with the same mangled name, don't
11015 add another one. This can happen e.g. for
11016 #pragma omp declare simd
11017 #pragma omp declare simd simdlen(8)
11018 int foo (int, int);
11019 if the simdlen is assumed to be 8 for the first one, etc. */
11020 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
11021 clone
= clone
->simdclone
->next_clone
)
11022 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
11026 return get_identifier (str
);
11029 /* Create a simd clone of OLD_NODE and return it. */
11031 static struct cgraph_node
*
11032 simd_clone_create (struct cgraph_node
*old_node
)
11034 struct cgraph_node
*new_node
;
11035 if (old_node
->definition
)
11037 if (!cgraph_function_with_gimple_body_p (old_node
))
11039 cgraph_get_body (old_node
);
11040 new_node
= cgraph_function_versioning (old_node
, vNULL
, NULL
, NULL
,
11041 false, NULL
, NULL
, "simdclone");
11045 tree old_decl
= old_node
->decl
;
11046 tree new_decl
= copy_node (old_node
->decl
);
11047 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
11048 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
11049 SET_DECL_RTL (new_decl
, NULL
);
11050 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
11051 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
11053 = cgraph_copy_node_for_versioning (old_node
, new_decl
, vNULL
, NULL
);
11054 cgraph_call_function_insertion_hooks (new_node
);
11056 if (new_node
== NULL
)
11059 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
11061 /* The function cgraph_function_versioning () will force the new
11062 symbol local. Undo this, and inherit external visability from
11064 new_node
->local
.local
= old_node
->local
.local
;
11065 new_node
->externally_visible
= old_node
->externally_visible
;
11070 /* Adjust the return type of the given function to its appropriate
11071 vector counterpart. Returns a simd array to be used throughout the
11072 function as a return value. */
11075 simd_clone_adjust_return_type (struct cgraph_node
*node
)
11077 tree fndecl
= node
->decl
;
11078 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
11079 unsigned int veclen
;
11082 /* Adjust the function return type. */
11083 if (orig_rettype
== void_type_node
)
11085 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
11086 if (INTEGRAL_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
)))
11087 || POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
))))
11088 veclen
= node
->simdclone
->vecsize_int
;
11090 veclen
= node
->simdclone
->vecsize_float
;
11091 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl
))));
11092 if (veclen
> node
->simdclone
->simdlen
)
11093 veclen
= node
->simdclone
->simdlen
;
11094 if (veclen
== node
->simdclone
->simdlen
)
11095 TREE_TYPE (TREE_TYPE (fndecl
))
11096 = build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)),
11097 node
->simdclone
->simdlen
);
11100 t
= build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)), veclen
);
11101 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
11102 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
11104 if (!node
->definition
)
11107 t
= DECL_RESULT (fndecl
);
11108 /* Adjust the DECL_RESULT. */
11109 gcc_assert (TREE_TYPE (t
) != void_type_node
);
11110 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
11113 tree atype
= build_array_type_nelts (orig_rettype
,
11114 node
->simdclone
->simdlen
);
11115 if (veclen
!= node
->simdclone
->simdlen
)
11116 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
11118 /* Set up a SIMD array to use as the return value. */
11119 tree retval
= create_tmp_var_raw (atype
, "retval");
11120 gimple_add_tmp_var (retval
);
11124 /* Each vector argument has a corresponding array to be used locally
11125 as part of the eventual loop. Create such temporary array and
11128 PREFIX is the prefix to be used for the temporary.
11130 TYPE is the inner element type.
11132 SIMDLEN is the number of elements. */
11135 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
11137 tree atype
= build_array_type_nelts (type
, simdlen
);
11138 tree avar
= create_tmp_var_raw (atype
, prefix
);
11139 gimple_add_tmp_var (avar
);
11143 /* Modify the function argument types to their corresponding vector
11144 counterparts if appropriate. Also, create one array for each simd
11145 argument to be used locally when using the function arguments as
11148 NODE is the function whose arguments are to be adjusted.
11150 Returns an adjustment vector that will be filled describing how the
11151 argument types will be adjusted. */
11153 static ipa_parm_adjustment_vec
11154 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
11157 ipa_parm_adjustment_vec adjustments
;
11159 if (node
->definition
)
11160 args
= ipa_get_vector_of_formal_parms (node
->decl
);
11162 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
11163 adjustments
.create (args
.length ());
11164 unsigned i
, j
, veclen
;
11165 struct ipa_parm_adjustment adj
;
11166 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
11168 memset (&adj
, 0, sizeof (adj
));
11169 tree parm
= args
[i
];
11170 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
11171 adj
.base_index
= i
;
11174 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
11175 node
->simdclone
->args
[i
].orig_type
= parm_type
;
11177 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
11179 /* No adjustment necessary for scalar arguments. */
11180 adj
.op
= IPA_PARM_OP_COPY
;
11184 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
11185 veclen
= node
->simdclone
->vecsize_int
;
11187 veclen
= node
->simdclone
->vecsize_float
;
11188 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
11189 if (veclen
> node
->simdclone
->simdlen
)
11190 veclen
= node
->simdclone
->simdlen
;
11191 adj
.arg_prefix
= "simd";
11192 adj
.type
= build_vector_type (parm_type
, veclen
);
11193 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
11194 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11196 adjustments
.safe_push (adj
);
11199 memset (&adj
, 0, sizeof (adj
));
11200 adj
.op
= IPA_PARM_OP_NEW
;
11201 adj
.arg_prefix
= "simd";
11202 adj
.base_index
= i
;
11203 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
11207 if (node
->definition
)
11208 node
->simdclone
->args
[i
].simd_array
11209 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
11210 parm_type
, node
->simdclone
->simdlen
);
11212 adjustments
.safe_push (adj
);
11215 if (node
->simdclone
->inbranch
)
11218 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
11221 memset (&adj
, 0, sizeof (adj
));
11222 adj
.op
= IPA_PARM_OP_NEW
;
11223 adj
.arg_prefix
= "mask";
11225 adj
.base_index
= i
;
11226 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
11227 veclen
= node
->simdclone
->vecsize_int
;
11229 veclen
= node
->simdclone
->vecsize_float
;
11230 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
11231 if (veclen
> node
->simdclone
->simdlen
)
11232 veclen
= node
->simdclone
->simdlen
;
11233 adj
.type
= build_vector_type (base_type
, veclen
);
11234 adjustments
.safe_push (adj
);
11236 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11237 adjustments
.safe_push (adj
);
11239 /* We have previously allocated one extra entry for the mask. Use
11241 struct cgraph_simd_clone
*sc
= node
->simdclone
;
11243 if (node
->definition
)
11245 sc
->args
[i
].orig_arg
11246 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
11247 sc
->args
[i
].simd_array
11248 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
11250 sc
->args
[i
].orig_type
= base_type
;
11251 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
11254 if (node
->definition
)
11255 ipa_modify_formal_parameters (node
->decl
, adjustments
);
11258 tree new_arg_types
= NULL_TREE
, new_reversed
;
11259 bool last_parm_void
= false;
11260 if (args
.length () > 0 && args
.last () == void_type_node
)
11261 last_parm_void
= true;
11263 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
11264 j
= adjustments
.length ();
11265 for (i
= 0; i
< j
; i
++)
11267 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
11269 if (adj
->op
== IPA_PARM_OP_COPY
)
11270 ptype
= args
[adj
->base_index
];
11273 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
11275 new_reversed
= nreverse (new_arg_types
);
11276 if (last_parm_void
)
11279 TREE_CHAIN (new_arg_types
) = void_list_node
;
11281 new_reversed
= void_list_node
;
11284 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
11285 TYPE_ARG_TYPES (new_type
) = new_reversed
;
11286 TREE_TYPE (node
->decl
) = new_type
;
11288 adjustments
.release ();
11291 return adjustments
;
11294 /* Initialize and copy the function arguments in NODE to their
11295 corresponding local simd arrays. Returns a fresh gimple_seq with
11296 the instruction sequence generated. */
11299 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
11300 ipa_parm_adjustment_vec adjustments
)
11302 gimple_seq seq
= NULL
;
11303 unsigned i
= 0, j
= 0, k
;
11305 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
11307 arg
= DECL_CHAIN (arg
), i
++, j
++)
11309 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
11312 node
->simdclone
->args
[i
].vector_arg
= arg
;
11314 tree array
= node
->simdclone
->args
[i
].simd_array
;
11315 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
11317 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11318 tree ptr
= build_fold_addr_expr (array
);
11319 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11320 build_int_cst (ptype
, 0));
11321 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11322 gimplify_and_add (t
, &seq
);
11326 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
11327 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11328 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
11330 tree ptr
= build_fold_addr_expr (array
);
11334 arg
= DECL_CHAIN (arg
);
11338 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
11339 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11340 build_int_cst (ptype
, k
* elemsize
));
11341 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11342 gimplify_and_add (t
, &seq
);
11349 /* Callback info for ipa_simd_modify_stmt_ops below. */
11351 struct modify_stmt_info
{
11352 ipa_parm_adjustment_vec adjustments
;
11354 /* True if the parent statement was modified by
11355 ipa_simd_modify_stmt_ops. */
11359 /* Callback for walk_gimple_op.
11361 Adjust operands from a given statement as specified in the
11362 adjustments vector in the callback data. */
11365 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
11367 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
11368 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
11369 tree
*orig_tp
= tp
;
11370 if (TREE_CODE (*tp
) == ADDR_EXPR
)
11371 tp
= &TREE_OPERAND (*tp
, 0);
11372 struct ipa_parm_adjustment
*cand
= NULL
;
11373 if (TREE_CODE (*tp
) == PARM_DECL
)
11374 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
11378 *walk_subtrees
= 0;
11381 tree repl
= NULL_TREE
;
11383 repl
= unshare_expr (cand
->new_decl
);
11388 *walk_subtrees
= 0;
11389 bool modified
= info
->modified
;
11390 info
->modified
= false;
11391 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
11392 if (!info
->modified
)
11394 info
->modified
= modified
;
11397 info
->modified
= modified
;
11406 repl
= build_fold_addr_expr (repl
);
11408 = gimple_build_assign (make_ssa_name (TREE_TYPE (repl
), NULL
), repl
);
11409 repl
= gimple_assign_lhs (stmt
);
11410 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
11411 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11414 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
11416 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
11422 info
->modified
= true;
11426 /* Traverse the function body and perform all modifications as
11427 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
11428 modified such that the replacement/reduction value will now be an
11429 offset into the corresponding simd_array.
11431 This function will replace all function argument uses with their
11432 corresponding simd array elements, and ajust the return values
11436 ipa_simd_modify_function_body (struct cgraph_node
*node
,
11437 ipa_parm_adjustment_vec adjustments
,
11438 tree retval_array
, tree iter
)
11441 unsigned int i
, j
, l
;
11443 /* Re-use the adjustments array, but this time use it to replace
11444 every function argument use to an offset into the corresponding
11446 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
11448 if (!node
->simdclone
->args
[i
].vector_arg
)
11451 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
11452 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
11453 adjustments
[j
].new_decl
11454 = build4 (ARRAY_REF
,
11456 node
->simdclone
->args
[i
].simd_array
,
11458 NULL_TREE
, NULL_TREE
);
11459 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
11460 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
11461 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
11464 l
= adjustments
.length ();
11465 for (i
= 1; i
< num_ssa_names
; i
++)
11467 tree name
= ssa_name (i
);
11469 && SSA_NAME_VAR (name
)
11470 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
11472 for (j
= 0; j
< l
; j
++)
11473 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
11474 && adjustments
[j
].new_decl
)
11477 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
11480 = copy_var_decl (adjustments
[j
].base
,
11481 DECL_NAME (adjustments
[j
].base
),
11482 TREE_TYPE (adjustments
[j
].base
));
11483 adjustments
[j
].new_ssa_base
= base_var
;
11486 base_var
= adjustments
[j
].new_ssa_base
;
11487 if (SSA_NAME_IS_DEFAULT_DEF (name
))
11489 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11490 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
11491 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
11492 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
11493 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
11494 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
11495 gimple stmt
= gimple_build_assign (name
, new_decl
);
11496 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11499 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
11504 struct modify_stmt_info info
;
11505 info
.adjustments
= adjustments
;
11507 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
11509 gimple_stmt_iterator gsi
;
11511 gsi
= gsi_start_bb (bb
);
11512 while (!gsi_end_p (gsi
))
11514 gimple stmt
= gsi_stmt (gsi
);
11516 struct walk_stmt_info wi
;
11518 memset (&wi
, 0, sizeof (wi
));
11519 info
.modified
= false;
11521 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
11523 if (gimple_code (stmt
) == GIMPLE_RETURN
)
11525 tree retval
= gimple_return_retval (stmt
);
11528 gsi_remove (&gsi
, true);
11532 /* Replace `return foo' with `retval_array[iter] = foo'. */
11533 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
11534 retval_array
, iter
, NULL
, NULL
);
11535 stmt
= gimple_build_assign (ref
, retval
);
11536 gsi_replace (&gsi
, stmt
, true);
11537 info
.modified
= true;
11542 update_stmt (stmt
);
11543 if (maybe_clean_eh_stmt (stmt
))
11544 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
11551 /* Adjust the argument types in NODE to their appropriate vector
11555 simd_clone_adjust (struct cgraph_node
*node
)
11557 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
11559 targetm
.simd_clone
.adjust (node
);
11561 tree retval
= simd_clone_adjust_return_type (node
);
11562 ipa_parm_adjustment_vec adjustments
11563 = simd_clone_adjust_argument_types (node
);
11565 push_gimplify_context ();
11567 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
11569 /* Adjust all uses of vector arguments accordingly. Adjust all
11570 return values accordingly. */
11571 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
11572 tree iter1
= make_ssa_name (iter
, NULL
);
11573 tree iter2
= make_ssa_name (iter
, NULL
);
11574 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
11576 /* Initialize the iteration variable. */
11577 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11578 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
11579 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
11580 /* Insert the SIMD array and iv initialization at function
11582 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
11584 pop_gimplify_context (NULL
);
11586 /* Create a new BB right before the original exit BB, to hold the
11587 iteration increment and the condition/branch. */
11588 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
11589 basic_block incr_bb
= create_empty_bb (orig_exit
);
11590 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
11591 flag. Set it now to be a FALLTHRU_EDGE. */
11592 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
11593 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
11594 for (unsigned i
= 0;
11595 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
11597 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
11598 redirect_edge_succ (e
, incr_bb
);
11600 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
11601 e
->probability
= REG_BR_PROB_BASE
;
11602 gsi
= gsi_last_bb (incr_bb
);
11603 gimple g
= gimple_build_assign_with_ops (PLUS_EXPR
, iter2
, iter1
,
11604 build_int_cst (unsigned_type_node
,
11606 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11608 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
11609 struct loop
*loop
= alloc_loop ();
11610 cfun
->has_force_vectorize_loops
= true;
11611 loop
->safelen
= node
->simdclone
->simdlen
;
11612 loop
->force_vectorize
= true;
11613 loop
->header
= body_bb
;
11614 add_bb_to_loop (incr_bb
, loop
);
11616 /* Branch around the body if the mask applies. */
11617 if (node
->simdclone
->inbranch
)
11619 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
11621 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
11622 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)), NULL
);
11623 tree aref
= build4 (ARRAY_REF
,
11624 TREE_TYPE (TREE_TYPE (mask_array
)),
11627 g
= gimple_build_assign (mask
, aref
);
11628 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11629 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
11630 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
11632 aref
= build1 (VIEW_CONVERT_EXPR
,
11633 build_nonstandard_integer_type (bitsize
, 0), mask
);
11634 mask
= make_ssa_name (TREE_TYPE (aref
), NULL
);
11635 g
= gimple_build_assign (mask
, aref
);
11636 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11639 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
11641 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11642 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
11643 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
11646 /* Generate the condition. */
11647 g
= gimple_build_cond (LT_EXPR
,
11649 build_int_cst (unsigned_type_node
,
11650 node
->simdclone
->simdlen
),
11652 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11653 e
= split_block (incr_bb
, gsi_stmt (gsi
));
11654 basic_block latch_bb
= e
->dest
;
11655 basic_block new_exit_bb
= e
->dest
;
11656 new_exit_bb
= split_block (latch_bb
, NULL
)->dest
;
11657 loop
->latch
= latch_bb
;
11659 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
11661 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
11662 /* The successor of incr_bb is already pointing to latch_bb; just
11664 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
11665 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
11667 gimple phi
= create_phi_node (iter1
, body_bb
);
11668 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
11669 edge latch_edge
= single_succ_edge (latch_bb
);
11670 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
11672 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
11674 /* Generate the new return. */
11675 gsi
= gsi_last_bb (new_exit_bb
);
11677 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
11678 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
11679 retval
= TREE_OPERAND (retval
, 0);
11682 retval
= build1 (VIEW_CONVERT_EXPR
,
11683 TREE_TYPE (TREE_TYPE (node
->decl
)),
11685 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
11686 false, GSI_CONTINUE_LINKING
);
11688 g
= gimple_build_return (retval
);
11689 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
11691 /* Handle aligned clauses by replacing default defs of the aligned
11692 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
11693 lhs. Handle linear by adding PHIs. */
11694 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
11695 if (node
->simdclone
->args
[i
].alignment
11696 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
11697 && (node
->simdclone
->args
[i
].alignment
11698 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
11699 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
11702 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
11703 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
11704 tree def
= ssa_default_def (cfun
, orig_arg
);
11705 if (def
&& !has_zero_uses (def
))
11707 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
11708 gimple_seq seq
= NULL
;
11709 bool need_cvt
= false;
11711 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
11713 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
11716 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
, NULL
);
11717 gimple_call_set_lhs (g
, t
);
11718 gimple_seq_add_stmt_without_update (&seq
, g
);
11721 t
= make_ssa_name (orig_arg
, NULL
);
11722 g
= gimple_build_assign_with_ops (NOP_EXPR
, t
,
11723 gimple_call_lhs (g
),
11725 gimple_seq_add_stmt_without_update (&seq
, g
);
11727 gsi_insert_seq_on_edge_immediate
11728 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
11730 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11731 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
11733 cgraph_create_edge (node
, cgraph_get_create_node (fn
),
11734 call
, entry_bb
->count
, freq
);
11736 imm_use_iterator iter
;
11737 use_operand_p use_p
;
11739 tree repl
= gimple_get_lhs (g
);
11740 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
11741 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
11744 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
11745 SET_USE (use_p
, repl
);
11748 else if (node
->simdclone
->args
[i
].arg_type
11749 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
11751 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
11752 tree def
= ssa_default_def (cfun
, orig_arg
);
11753 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11754 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
11755 if (def
&& !has_zero_uses (def
))
11757 iter1
= make_ssa_name (orig_arg
, NULL
);
11758 iter2
= make_ssa_name (orig_arg
, NULL
);
11759 phi
= create_phi_node (iter1
, body_bb
);
11760 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
11761 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
11762 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11763 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
11764 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
11765 ? TREE_TYPE (orig_arg
) : sizetype
;
11767 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
11768 g
= gimple_build_assign_with_ops (code
, iter2
, iter1
, addcst
);
11769 gsi
= gsi_last_bb (incr_bb
);
11770 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
11772 imm_use_iterator iter
;
11773 use_operand_p use_p
;
11775 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
11776 if (use_stmt
== phi
)
11779 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
11780 SET_USE (use_p
, iter1
);
11784 calculate_dominance_info (CDI_DOMINATORS
);
11785 add_loop (loop
, loop
->header
->loop_father
);
11786 update_ssa (TODO_update_ssa
);
11791 /* If the function in NODE is tagged as an elemental SIMD function,
11792 create the appropriate SIMD clones. */
11795 expand_simd_clones (struct cgraph_node
*node
)
11797 tree attr
= lookup_attribute ("omp declare simd",
11798 DECL_ATTRIBUTES (node
->decl
));
11799 if (attr
== NULL_TREE
11800 || node
->global
.inlined_to
11801 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
11805 #pragma omp declare simd
11807 in C, there we don't know the argument types at all. */
11808 if (!node
->definition
11809 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
11814 /* Start with parsing the "omp declare simd" attribute(s). */
11815 bool inbranch_clause_specified
;
11816 struct cgraph_simd_clone
*clone_info
11817 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
11818 &inbranch_clause_specified
);
11819 if (clone_info
== NULL
)
11822 int orig_simdlen
= clone_info
->simdlen
;
11823 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
11824 /* The target can return 0 (no simd clones should be created),
11825 1 (just one ISA of simd clones should be created) or higher
11826 count of ISA variants. In that case, clone_info is initialized
11827 for the first ISA variant. */
11829 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
11834 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
11835 also create one inbranch and one !inbranch clone of it. */
11836 for (int i
= 0; i
< count
* 2; i
++)
11838 struct cgraph_simd_clone
*clone
= clone_info
;
11839 if (inbranch_clause_specified
&& (i
& 1) != 0)
11844 clone
= simd_clone_struct_alloc (clone_info
->nargs
11846 simd_clone_struct_copy (clone
, clone_info
);
11847 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
11848 and simd_clone_adjust_argument_types did to the first
11850 clone
->nargs
-= clone_info
->inbranch
;
11851 clone
->simdlen
= orig_simdlen
;
11852 /* And call the target hook again to get the right ISA. */
11853 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
11857 clone
->inbranch
= 1;
11860 /* simd_clone_mangle might fail if such a clone has been created
11862 tree id
= simd_clone_mangle (node
, clone
);
11863 if (id
== NULL_TREE
)
11866 /* Only when we are sure we want to create the clone actually
11867 clone the function (or definitions) or create another
11868 extern FUNCTION_DECL (for prototypes without definitions). */
11869 struct cgraph_node
*n
= simd_clone_create (node
);
11873 n
->simdclone
= clone
;
11874 clone
->origin
= node
;
11875 clone
->next_clone
= NULL
;
11876 if (node
->simd_clones
== NULL
)
11878 clone
->prev_clone
= n
;
11879 node
->simd_clones
= n
;
11883 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
11884 clone
->prev_clone
->simdclone
->next_clone
= n
;
11885 node
->simd_clones
->simdclone
->prev_clone
= n
;
11887 change_decl_assembler_name (n
->decl
, id
);
11888 /* And finally adjust the return type, parameters and for
11889 definitions also function body. */
11890 if (node
->definition
)
11891 simd_clone_adjust (n
);
11894 simd_clone_adjust_return_type (n
);
11895 simd_clone_adjust_argument_types (n
);
11899 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
11902 /* Entry point for IPA simd clone creation pass. */
11904 static unsigned int
11905 ipa_omp_simd_clone (void)
11907 struct cgraph_node
*node
;
11908 FOR_EACH_FUNCTION (node
)
11909 expand_simd_clones (node
);
11915 const pass_data pass_data_omp_simd_clone
=
11917 SIMPLE_IPA_PASS
, /* type */
11918 "simdclone", /* name */
11919 OPTGROUP_NONE
, /* optinfo_flags */
11920 true, /* has_execute */
11921 TV_NONE
, /* tv_id */
11922 ( PROP_ssa
| PROP_cfg
), /* properties_required */
11923 0, /* properties_provided */
11924 0, /* properties_destroyed */
11925 0, /* todo_flags_start */
11926 0, /* todo_flags_finish */
11929 class pass_omp_simd_clone
: public simple_ipa_opt_pass
11932 pass_omp_simd_clone(gcc::context
*ctxt
)
11933 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
11936 /* opt_pass methods: */
11937 virtual bool gate (function
*);
11938 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
11942 pass_omp_simd_clone::gate (function
*)
11944 return ((flag_openmp
|| flag_openmp_simd
11946 || (in_lto_p
&& !flag_wpa
))
11947 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
11950 } // anon namespace
11952 simple_ipa_opt_pass
*
11953 make_pass_omp_simd_clone (gcc::context
*ctxt
)
11955 return new pass_omp_simd_clone (ctxt
);
11958 #include "gt-omp-low.h"