1 /* Lowering pass for OpenMP directives. Converts OpenMP directives
2 into explicit calls to the runtime library (libgomp) and data
3 marshalling to implement data sharing and copying clauses.
4 Contributed by Diego Novillo <dnovillo@redhat.com>
6 Copyright (C) 2005-2014 Free Software Foundation, Inc.
8 This file is part of GCC.
10 GCC is free software; you can redistribute it and/or modify it under
11 the terms of the GNU General Public License as published by the Free
12 Software Foundation; either version 3, or (at your option) any later
15 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or
17 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
20 You should have received a copy of the GNU General Public License
21 along with GCC; see the file COPYING3. If not see
22 <http://www.gnu.org/licenses/>. */
26 #include "coretypes.h"
29 #include "stringpool.h"
30 #include "stor-layout.h"
37 #include "hard-reg-set.h"
40 #include "dominance.h"
43 #include "basic-block.h"
44 #include "tree-ssa-alias.h"
45 #include "internal-fn.h"
46 #include "gimple-fold.h"
47 #include "gimple-expr.h"
51 #include "gimple-iterator.h"
52 #include "gimplify-me.h"
53 #include "gimple-walk.h"
54 #include "tree-iterator.h"
55 #include "tree-inline.h"
56 #include "langhooks.h"
57 #include "diagnostic-core.h"
58 #include "gimple-ssa.h"
61 #include "tree-phinodes.h"
62 #include "ssa-iterators.h"
63 #include "tree-ssanames.h"
64 #include "tree-into-ssa.h"
70 #include "tree-pass.h"
72 #include "splay-tree.h"
77 #include "gimple-low.h"
78 #include "tree-cfgcleanup.h"
79 #include "pretty-print.h"
81 #include "tree-nested.h"
86 /* Lowering of OpenMP parallel and workshare constructs proceeds in two
87 phases. The first phase scans the function looking for OMP statements
88 and then for variables that must be replaced to satisfy data sharing
89 clauses. The second phase expands code for the constructs, as well as
90 re-gimplifying things when variables have been replaced with complex
93 Final code generation is done by pass_expand_omp. The flowgraph is
94 scanned for parallel regions which are then moved to a new
95 function, to be invoked by the thread library. */
97 /* Parallel region information. Every parallel and workshare
98 directive is enclosed between two markers, the OMP_* directive
99 and a corresponding OMP_RETURN statement. */
103 /* The enclosing region. */
104 struct omp_region
*outer
;
106 /* First child region. */
107 struct omp_region
*inner
;
109 /* Next peer region. */
110 struct omp_region
*next
;
112 /* Block containing the omp directive as its last stmt. */
115 /* Block containing the OMP_RETURN as its last stmt. */
118 /* Block containing the OMP_CONTINUE as its last stmt. */
121 /* If this is a combined parallel+workshare region, this is a list
122 of additional arguments needed by the combined parallel+workshare
124 vec
<tree
, va_gc
> *ws_args
;
126 /* The code for the omp directive of this region. */
127 enum gimple_code type
;
129 /* Schedule kind, only used for OMP_FOR type regions. */
130 enum omp_clause_schedule_kind sched_kind
;
132 /* True if this is a combined parallel+workshare region. */
133 bool is_combined_parallel
;
136 /* Context structure. Used to store information about each parallel
137 directive in the code. */
139 typedef struct omp_context
141 /* This field must be at the beginning, as we do "inheritance": Some
142 callback functions for tree-inline.c (e.g., omp_copy_decl)
143 receive a copy_body_data pointer that is up-casted to an
144 omp_context pointer. */
147 /* The tree of contexts corresponding to the encountered constructs. */
148 struct omp_context
*outer
;
151 /* Map variables to fields in a structure that allows communication
152 between sending and receiving threads. */
153 splay_tree field_map
;
158 /* These are used just by task contexts, if task firstprivate fn is
159 needed. srecord_type is used to communicate from the thread
160 that encountered the task construct to task firstprivate fn,
161 record_type is allocated by GOMP_task, initialized by task firstprivate
162 fn and passed to the task body fn. */
163 splay_tree sfield_map
;
166 /* A chain of variables to add to the top-level block surrounding the
167 construct. In the case of a parallel, this is in the child function. */
170 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit
171 barriers should jump to during omplower pass. */
174 /* What to do with variables with implicitly determined sharing
176 enum omp_clause_default_kind default_kind
;
178 /* Nesting depth of this context. Used to beautify error messages re
179 invalid gotos. The outermost ctx is depth 1, with depth 0 being
180 reserved for the main body of the function. */
183 /* True if this parallel directive is nested within another. */
186 /* True if this construct can be cancelled. */
191 struct omp_for_data_loop
193 tree v
, n1
, n2
, step
;
194 enum tree_code cond_code
;
197 /* A structure describing the main elements of a parallel loop. */
201 struct omp_for_data_loop loop
;
206 bool have_nowait
, have_ordered
;
207 enum omp_clause_schedule_kind sched_kind
;
208 struct omp_for_data_loop
*loops
;
212 static splay_tree all_contexts
;
213 static int taskreg_nesting_level
;
214 static int target_nesting_level
;
215 static struct omp_region
*root_omp_region
;
216 static bitmap task_shared_vars
;
217 static vec
<omp_context
*> taskreg_contexts
;
219 static void scan_omp (gimple_seq
*, omp_context
*);
220 static tree
scan_omp_1_op (tree
*, int *, void *);
222 #define WALK_SUBSTMTS \
226 case GIMPLE_EH_FILTER: \
227 case GIMPLE_TRANSACTION: \
228 /* The sub-statements for these should be walked. */ \
229 *handled_ops_p = false; \
232 /* Convenience function for calling scan_omp_1_op on tree operands. */
235 scan_omp_op (tree
*tp
, omp_context
*ctx
)
237 struct walk_stmt_info wi
;
239 memset (&wi
, 0, sizeof (wi
));
241 wi
.want_locations
= true;
243 return walk_tree (tp
, scan_omp_1_op
, &wi
, NULL
);
246 static void lower_omp (gimple_seq
*, omp_context
*);
247 static tree
lookup_decl_in_outer_ctx (tree
, omp_context
*);
248 static tree
maybe_lookup_decl_in_outer_ctx (tree
, omp_context
*);
250 /* Find an OpenMP clause of type KIND within CLAUSES. */
253 find_omp_clause (tree clauses
, enum omp_clause_code kind
)
255 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
256 if (OMP_CLAUSE_CODE (clauses
) == kind
)
262 /* Return true if CTX is for an omp parallel. */
265 is_parallel_ctx (omp_context
*ctx
)
267 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
;
271 /* Return true if CTX is for an omp task. */
274 is_task_ctx (omp_context
*ctx
)
276 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
280 /* Return true if CTX is for an omp parallel or omp task. */
283 is_taskreg_ctx (omp_context
*ctx
)
285 return gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
286 || gimple_code (ctx
->stmt
) == GIMPLE_OMP_TASK
;
290 /* Return true if REGION is a combined parallel+workshare region. */
293 is_combined_parallel (struct omp_region
*region
)
295 return region
->is_combined_parallel
;
299 /* Extract the header elements of parallel loop FOR_STMT and store
303 extract_omp_for_data (gimple for_stmt
, struct omp_for_data
*fd
,
304 struct omp_for_data_loop
*loops
)
306 tree t
, var
, *collapse_iter
, *collapse_count
;
307 tree count
= NULL_TREE
, iter_type
= long_integer_type_node
;
308 struct omp_for_data_loop
*loop
;
310 struct omp_for_data_loop dummy_loop
;
311 location_t loc
= gimple_location (for_stmt
);
312 bool simd
= gimple_omp_for_kind (for_stmt
) & GF_OMP_FOR_SIMD
;
313 bool distribute
= gimple_omp_for_kind (for_stmt
)
314 == GF_OMP_FOR_KIND_DISTRIBUTE
;
316 fd
->for_stmt
= for_stmt
;
318 fd
->collapse
= gimple_omp_for_collapse (for_stmt
);
319 if (fd
->collapse
> 1)
322 fd
->loops
= &fd
->loop
;
324 fd
->have_nowait
= distribute
|| simd
;
325 fd
->have_ordered
= false;
326 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
327 fd
->chunk_size
= NULL_TREE
;
328 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
329 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_CILKFOR
;
330 collapse_iter
= NULL
;
331 collapse_count
= NULL
;
333 for (t
= gimple_omp_for_clauses (for_stmt
); t
; t
= OMP_CLAUSE_CHAIN (t
))
334 switch (OMP_CLAUSE_CODE (t
))
336 case OMP_CLAUSE_NOWAIT
:
337 fd
->have_nowait
= true;
339 case OMP_CLAUSE_ORDERED
:
340 fd
->have_ordered
= true;
342 case OMP_CLAUSE_SCHEDULE
:
343 gcc_assert (!distribute
);
344 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_KIND (t
);
345 fd
->chunk_size
= OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t
);
347 case OMP_CLAUSE_DIST_SCHEDULE
:
348 gcc_assert (distribute
);
349 fd
->chunk_size
= OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t
);
351 case OMP_CLAUSE_COLLAPSE
:
352 if (fd
->collapse
> 1)
354 collapse_iter
= &OMP_CLAUSE_COLLAPSE_ITERVAR (t
);
355 collapse_count
= &OMP_CLAUSE_COLLAPSE_COUNT (t
);
362 /* FIXME: for now map schedule(auto) to schedule(static).
363 There should be analysis to determine whether all iterations
364 are approximately the same amount of work (then schedule(static)
365 is best) or if it varies (then schedule(dynamic,N) is better). */
366 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_AUTO
)
368 fd
->sched_kind
= OMP_CLAUSE_SCHEDULE_STATIC
;
369 gcc_assert (fd
->chunk_size
== NULL
);
371 gcc_assert (fd
->collapse
== 1 || collapse_iter
!= NULL
);
372 if (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
373 gcc_assert (fd
->chunk_size
== NULL
);
374 else if (fd
->chunk_size
== NULL
)
376 /* We only need to compute a default chunk size for ordered
377 static loops and dynamic loops. */
378 if (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
380 fd
->chunk_size
= (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
381 ? integer_zero_node
: integer_one_node
;
384 for (i
= 0; i
< fd
->collapse
; i
++)
386 if (fd
->collapse
== 1)
388 else if (loops
!= NULL
)
393 loop
->v
= gimple_omp_for_index (for_stmt
, i
);
394 gcc_assert (SSA_VAR_P (loop
->v
));
395 gcc_assert (TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
396 || TREE_CODE (TREE_TYPE (loop
->v
)) == POINTER_TYPE
);
397 var
= TREE_CODE (loop
->v
) == SSA_NAME
? SSA_NAME_VAR (loop
->v
) : loop
->v
;
398 loop
->n1
= gimple_omp_for_initial (for_stmt
, i
);
400 loop
->cond_code
= gimple_omp_for_cond (for_stmt
, i
);
401 loop
->n2
= gimple_omp_for_final (for_stmt
, i
);
402 switch (loop
->cond_code
)
408 gcc_assert (gimple_omp_for_kind (for_stmt
)
409 == GF_OMP_FOR_KIND_CILKSIMD
410 || (gimple_omp_for_kind (for_stmt
)
411 == GF_OMP_FOR_KIND_CILKFOR
));
414 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
415 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, 1);
417 loop
->n2
= fold_build2_loc (loc
,
418 PLUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
419 build_int_cst (TREE_TYPE (loop
->n2
), 1));
420 loop
->cond_code
= LT_EXPR
;
423 if (POINTER_TYPE_P (TREE_TYPE (loop
->n2
)))
424 loop
->n2
= fold_build_pointer_plus_hwi_loc (loc
, loop
->n2
, -1);
426 loop
->n2
= fold_build2_loc (loc
,
427 MINUS_EXPR
, TREE_TYPE (loop
->n2
), loop
->n2
,
428 build_int_cst (TREE_TYPE (loop
->n2
), 1));
429 loop
->cond_code
= GT_EXPR
;
435 t
= gimple_omp_for_incr (for_stmt
, i
);
436 gcc_assert (TREE_OPERAND (t
, 0) == var
);
437 switch (TREE_CODE (t
))
440 loop
->step
= TREE_OPERAND (t
, 1);
442 case POINTER_PLUS_EXPR
:
443 loop
->step
= fold_convert (ssizetype
, TREE_OPERAND (t
, 1));
446 loop
->step
= TREE_OPERAND (t
, 1);
447 loop
->step
= fold_build1_loc (loc
,
448 NEGATE_EXPR
, TREE_TYPE (loop
->step
),
456 || (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
457 && !fd
->have_ordered
))
459 if (fd
->collapse
== 1)
460 iter_type
= TREE_TYPE (loop
->v
);
462 || TYPE_PRECISION (iter_type
)
463 < TYPE_PRECISION (TREE_TYPE (loop
->v
)))
465 = build_nonstandard_integer_type
466 (TYPE_PRECISION (TREE_TYPE (loop
->v
)), 1);
468 else if (iter_type
!= long_long_unsigned_type_node
)
470 if (POINTER_TYPE_P (TREE_TYPE (loop
->v
)))
471 iter_type
= long_long_unsigned_type_node
;
472 else if (TYPE_UNSIGNED (TREE_TYPE (loop
->v
))
473 && TYPE_PRECISION (TREE_TYPE (loop
->v
))
474 >= TYPE_PRECISION (iter_type
))
478 if (loop
->cond_code
== LT_EXPR
)
479 n
= fold_build2_loc (loc
,
480 PLUS_EXPR
, TREE_TYPE (loop
->v
),
481 loop
->n2
, loop
->step
);
484 if (TREE_CODE (n
) != INTEGER_CST
485 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type
), n
))
486 iter_type
= long_long_unsigned_type_node
;
488 else if (TYPE_PRECISION (TREE_TYPE (loop
->v
))
489 > TYPE_PRECISION (iter_type
))
493 if (loop
->cond_code
== LT_EXPR
)
496 n2
= fold_build2_loc (loc
,
497 PLUS_EXPR
, TREE_TYPE (loop
->v
),
498 loop
->n2
, loop
->step
);
502 n1
= fold_build2_loc (loc
,
503 MINUS_EXPR
, TREE_TYPE (loop
->v
),
504 loop
->n2
, loop
->step
);
507 if (TREE_CODE (n1
) != INTEGER_CST
508 || TREE_CODE (n2
) != INTEGER_CST
509 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type
), n1
)
510 || !tree_int_cst_lt (n2
, TYPE_MAX_VALUE (iter_type
)))
511 iter_type
= long_long_unsigned_type_node
;
515 if (collapse_count
&& *collapse_count
== NULL
)
517 t
= fold_binary (loop
->cond_code
, boolean_type_node
,
518 fold_convert (TREE_TYPE (loop
->v
), loop
->n1
),
519 fold_convert (TREE_TYPE (loop
->v
), loop
->n2
));
520 if (t
&& integer_zerop (t
))
521 count
= build_zero_cst (long_long_unsigned_type_node
);
522 else if ((i
== 0 || count
!= NULL_TREE
)
523 && TREE_CODE (TREE_TYPE (loop
->v
)) == INTEGER_TYPE
524 && TREE_CONSTANT (loop
->n1
)
525 && TREE_CONSTANT (loop
->n2
)
526 && TREE_CODE (loop
->step
) == INTEGER_CST
)
528 tree itype
= TREE_TYPE (loop
->v
);
530 if (POINTER_TYPE_P (itype
))
531 itype
= signed_type_for (itype
);
532 t
= build_int_cst (itype
, (loop
->cond_code
== LT_EXPR
? -1 : 1));
533 t
= fold_build2_loc (loc
,
535 fold_convert_loc (loc
, itype
, loop
->step
), t
);
536 t
= fold_build2_loc (loc
, PLUS_EXPR
, itype
, t
,
537 fold_convert_loc (loc
, itype
, loop
->n2
));
538 t
= fold_build2_loc (loc
, MINUS_EXPR
, itype
, t
,
539 fold_convert_loc (loc
, itype
, loop
->n1
));
540 if (TYPE_UNSIGNED (itype
) && loop
->cond_code
== GT_EXPR
)
541 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
,
542 fold_build1_loc (loc
, NEGATE_EXPR
, itype
, t
),
543 fold_build1_loc (loc
, NEGATE_EXPR
, itype
,
544 fold_convert_loc (loc
, itype
,
547 t
= fold_build2_loc (loc
, TRUNC_DIV_EXPR
, itype
, t
,
548 fold_convert_loc (loc
, itype
, loop
->step
));
549 t
= fold_convert_loc (loc
, long_long_unsigned_type_node
, t
);
550 if (count
!= NULL_TREE
)
551 count
= fold_build2_loc (loc
,
552 MULT_EXPR
, long_long_unsigned_type_node
,
556 if (TREE_CODE (count
) != INTEGER_CST
)
559 else if (count
&& !integer_zerop (count
))
566 && (fd
->sched_kind
!= OMP_CLAUSE_SCHEDULE_STATIC
567 || fd
->have_ordered
))
569 if (!tree_int_cst_lt (count
, TYPE_MAX_VALUE (long_integer_type_node
)))
570 iter_type
= long_long_unsigned_type_node
;
572 iter_type
= long_integer_type_node
;
574 else if (collapse_iter
&& *collapse_iter
!= NULL
)
575 iter_type
= TREE_TYPE (*collapse_iter
);
576 fd
->iter_type
= iter_type
;
577 if (collapse_iter
&& *collapse_iter
== NULL
)
578 *collapse_iter
= create_tmp_var (iter_type
, ".iter");
579 if (collapse_count
&& *collapse_count
== NULL
)
582 *collapse_count
= fold_convert_loc (loc
, iter_type
, count
);
584 *collapse_count
= create_tmp_var (iter_type
, ".count");
587 if (fd
->collapse
> 1)
589 fd
->loop
.v
= *collapse_iter
;
590 fd
->loop
.n1
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
591 fd
->loop
.n2
= *collapse_count
;
592 fd
->loop
.step
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 1);
593 fd
->loop
.cond_code
= LT_EXPR
;
598 /* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB
599 is the immediate dominator of PAR_ENTRY_BB, return true if there
600 are no data dependencies that would prevent expanding the parallel
601 directive at PAR_ENTRY_BB as a combined parallel+workshare region.
603 When expanding a combined parallel+workshare region, the call to
604 the child function may need additional arguments in the case of
605 GIMPLE_OMP_FOR regions. In some cases, these arguments are
606 computed out of variables passed in from the parent to the child
607 via 'struct .omp_data_s'. For instance:
609 #pragma omp parallel for schedule (guided, i * 4)
614 # BLOCK 2 (PAR_ENTRY_BB)
616 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598)
618 # BLOCK 3 (WS_ENTRY_BB)
619 .omp_data_i = &.omp_data_o;
620 D.1667 = .omp_data_i->i;
622 #pragma omp for schedule (guided, D.1598)
624 When we outline the parallel region, the call to the child function
625 'bar.omp_fn.0' will need the value D.1598 in its argument list, but
626 that value is computed *after* the call site. So, in principle we
627 cannot do the transformation.
629 To see whether the code in WS_ENTRY_BB blocks the combined
630 parallel+workshare call, we collect all the variables used in the
631 GIMPLE_OMP_FOR header check whether they appear on the LHS of any
632 statement in WS_ENTRY_BB. If so, then we cannot emit the combined
635 FIXME. If we had the SSA form built at this point, we could merely
636 hoist the code in block 3 into block 2 and be done with it. But at
637 this point we don't have dataflow information and though we could
638 hack something up here, it is really not worth the aggravation. */
641 workshare_safe_to_combine_p (basic_block ws_entry_bb
)
643 struct omp_for_data fd
;
644 gimple ws_stmt
= last_stmt (ws_entry_bb
);
646 if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
649 gcc_assert (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
);
651 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
653 if (fd
.collapse
> 1 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
655 if (fd
.iter_type
!= long_integer_type_node
)
658 /* FIXME. We give up too easily here. If any of these arguments
659 are not constants, they will likely involve variables that have
660 been mapped into fields of .omp_data_s for sharing with the child
661 function. With appropriate data flow, it would be possible to
663 if (!is_gimple_min_invariant (fd
.loop
.n1
)
664 || !is_gimple_min_invariant (fd
.loop
.n2
)
665 || !is_gimple_min_invariant (fd
.loop
.step
)
666 || (fd
.chunk_size
&& !is_gimple_min_invariant (fd
.chunk_size
)))
673 /* Collect additional arguments needed to emit a combined
674 parallel+workshare call. WS_STMT is the workshare directive being
677 static vec
<tree
, va_gc
> *
678 get_ws_args_for (gimple par_stmt
, gimple ws_stmt
)
681 location_t loc
= gimple_location (ws_stmt
);
682 vec
<tree
, va_gc
> *ws_args
;
684 if (gimple_code (ws_stmt
) == GIMPLE_OMP_FOR
)
686 struct omp_for_data fd
;
689 extract_omp_for_data (ws_stmt
, &fd
, NULL
);
693 if (gimple_omp_for_combined_into_p (ws_stmt
))
696 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt
),
697 OMP_CLAUSE__LOOPTEMP_
);
699 n1
= OMP_CLAUSE_DECL (innerc
);
700 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
701 OMP_CLAUSE__LOOPTEMP_
);
703 n2
= OMP_CLAUSE_DECL (innerc
);
706 vec_alloc (ws_args
, 3 + (fd
.chunk_size
!= 0));
708 t
= fold_convert_loc (loc
, long_integer_type_node
, n1
);
709 ws_args
->quick_push (t
);
711 t
= fold_convert_loc (loc
, long_integer_type_node
, n2
);
712 ws_args
->quick_push (t
);
714 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.loop
.step
);
715 ws_args
->quick_push (t
);
719 t
= fold_convert_loc (loc
, long_integer_type_node
, fd
.chunk_size
);
720 ws_args
->quick_push (t
);
725 else if (gimple_code (ws_stmt
) == GIMPLE_OMP_SECTIONS
)
727 /* Number of sections is equal to the number of edges from the
728 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to
729 the exit of the sections region. */
730 basic_block bb
= single_succ (gimple_bb (ws_stmt
));
731 t
= build_int_cst (unsigned_type_node
, EDGE_COUNT (bb
->succs
) - 1);
732 vec_alloc (ws_args
, 1);
733 ws_args
->quick_push (t
);
741 /* Discover whether REGION is a combined parallel+workshare region. */
744 determine_parallel_type (struct omp_region
*region
)
746 basic_block par_entry_bb
, par_exit_bb
;
747 basic_block ws_entry_bb
, ws_exit_bb
;
749 if (region
== NULL
|| region
->inner
== NULL
750 || region
->exit
== NULL
|| region
->inner
->exit
== NULL
751 || region
->inner
->cont
== NULL
)
754 /* We only support parallel+for and parallel+sections. */
755 if (region
->type
!= GIMPLE_OMP_PARALLEL
756 || (region
->inner
->type
!= GIMPLE_OMP_FOR
757 && region
->inner
->type
!= GIMPLE_OMP_SECTIONS
))
760 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and
761 WS_EXIT_BB -> PAR_EXIT_BB. */
762 par_entry_bb
= region
->entry
;
763 par_exit_bb
= region
->exit
;
764 ws_entry_bb
= region
->inner
->entry
;
765 ws_exit_bb
= region
->inner
->exit
;
767 if (single_succ (par_entry_bb
) == ws_entry_bb
768 && single_succ (ws_exit_bb
) == par_exit_bb
769 && workshare_safe_to_combine_p (ws_entry_bb
)
770 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb
))
771 || (last_and_only_stmt (ws_entry_bb
)
772 && last_and_only_stmt (par_exit_bb
))))
774 gimple par_stmt
= last_stmt (par_entry_bb
);
775 gimple ws_stmt
= last_stmt (ws_entry_bb
);
777 if (region
->inner
->type
== GIMPLE_OMP_FOR
)
779 /* If this is a combined parallel loop, we need to determine
780 whether or not to use the combined library calls. There
781 are two cases where we do not apply the transformation:
782 static loops and any kind of ordered loop. In the first
783 case, we already open code the loop so there is no need
784 to do anything else. In the latter case, the combined
785 parallel loop call would still need extra synchronization
786 to implement ordered semantics, so there would not be any
787 gain in using the combined call. */
788 tree clauses
= gimple_omp_for_clauses (ws_stmt
);
789 tree c
= find_omp_clause (clauses
, OMP_CLAUSE_SCHEDULE
);
791 || OMP_CLAUSE_SCHEDULE_KIND (c
) == OMP_CLAUSE_SCHEDULE_STATIC
792 || find_omp_clause (clauses
, OMP_CLAUSE_ORDERED
))
794 region
->is_combined_parallel
= false;
795 region
->inner
->is_combined_parallel
= false;
800 region
->is_combined_parallel
= true;
801 region
->inner
->is_combined_parallel
= true;
802 region
->ws_args
= get_ws_args_for (par_stmt
, ws_stmt
);
807 /* Return true if EXPR is variable sized. */
810 is_variable_sized (const_tree expr
)
812 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr
)));
815 /* Return true if DECL is a reference type. */
818 is_reference (tree decl
)
820 return lang_hooks
.decls
.omp_privatize_by_reference (decl
);
823 /* Lookup variables in the decl or field splay trees. The "maybe" form
824 allows for the variable form to not have been entered, otherwise we
825 assert that the variable must have been entered. */
828 lookup_decl (tree var
, omp_context
*ctx
)
830 tree
*n
= ctx
->cb
.decl_map
->get (var
);
835 maybe_lookup_decl (const_tree var
, omp_context
*ctx
)
837 tree
*n
= ctx
->cb
.decl_map
->get (const_cast<tree
> (var
));
838 return n
? *n
: NULL_TREE
;
842 lookup_field (tree var
, omp_context
*ctx
)
845 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
846 return (tree
) n
->value
;
850 lookup_sfield (tree var
, omp_context
*ctx
)
853 n
= splay_tree_lookup (ctx
->sfield_map
854 ? ctx
->sfield_map
: ctx
->field_map
,
855 (splay_tree_key
) var
);
856 return (tree
) n
->value
;
860 maybe_lookup_field (tree var
, omp_context
*ctx
)
863 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
);
864 return n
? (tree
) n
->value
: NULL_TREE
;
867 /* Return true if DECL should be copied by pointer. SHARED_CTX is
868 the parallel context if DECL is to be shared. */
871 use_pointer_for_field (tree decl
, omp_context
*shared_ctx
)
873 if (AGGREGATE_TYPE_P (TREE_TYPE (decl
)))
876 /* We can only use copy-in/copy-out semantics for shared variables
877 when we know the value is not accessible from an outer scope. */
880 /* ??? Trivially accessible from anywhere. But why would we even
881 be passing an address in this case? Should we simply assert
882 this to be false, or should we have a cleanup pass that removes
883 these from the list of mappings? */
884 if (TREE_STATIC (decl
) || DECL_EXTERNAL (decl
))
887 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
888 without analyzing the expression whether or not its location
889 is accessible to anyone else. In the case of nested parallel
890 regions it certainly may be. */
891 if (TREE_CODE (decl
) != RESULT_DECL
&& DECL_HAS_VALUE_EXPR_P (decl
))
894 /* Do not use copy-in/copy-out for variables that have their
896 if (TREE_ADDRESSABLE (decl
))
899 /* lower_send_shared_vars only uses copy-in, but not copy-out
901 if (TREE_READONLY (decl
)
902 || ((TREE_CODE (decl
) == RESULT_DECL
903 || TREE_CODE (decl
) == PARM_DECL
)
904 && DECL_BY_REFERENCE (decl
)))
907 /* Disallow copy-in/out in nested parallel if
908 decl is shared in outer parallel, otherwise
909 each thread could store the shared variable
910 in its own copy-in location, making the
911 variable no longer really shared. */
912 if (shared_ctx
->is_nested
)
916 for (up
= shared_ctx
->outer
; up
; up
= up
->outer
)
917 if (is_taskreg_ctx (up
) && maybe_lookup_decl (decl
, up
))
924 for (c
= gimple_omp_taskreg_clauses (up
->stmt
);
925 c
; c
= OMP_CLAUSE_CHAIN (c
))
926 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
927 && OMP_CLAUSE_DECL (c
) == decl
)
931 goto maybe_mark_addressable_and_ret
;
935 /* For tasks avoid using copy-in/out. As tasks can be
936 deferred or executed in different thread, when GOMP_task
937 returns, the task hasn't necessarily terminated. */
938 if (is_task_ctx (shared_ctx
))
941 maybe_mark_addressable_and_ret
:
942 outer
= maybe_lookup_decl_in_outer_ctx (decl
, shared_ctx
);
943 if (is_gimple_reg (outer
))
945 /* Taking address of OUTER in lower_send_shared_vars
946 might need regimplification of everything that uses the
948 if (!task_shared_vars
)
949 task_shared_vars
= BITMAP_ALLOC (NULL
);
950 bitmap_set_bit (task_shared_vars
, DECL_UID (outer
));
951 TREE_ADDRESSABLE (outer
) = 1;
960 /* Construct a new automatic decl similar to VAR. */
963 omp_copy_decl_2 (tree var
, tree name
, tree type
, omp_context
*ctx
)
965 tree copy
= copy_var_decl (var
, name
, type
);
967 DECL_CONTEXT (copy
) = current_function_decl
;
968 DECL_CHAIN (copy
) = ctx
->block_vars
;
969 ctx
->block_vars
= copy
;
975 omp_copy_decl_1 (tree var
, omp_context
*ctx
)
977 return omp_copy_decl_2 (var
, DECL_NAME (var
), TREE_TYPE (var
), ctx
);
980 /* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it
983 omp_build_component_ref (tree obj
, tree field
)
985 tree ret
= build3 (COMPONENT_REF
, TREE_TYPE (field
), obj
, field
, NULL
);
986 if (TREE_THIS_VOLATILE (field
))
987 TREE_THIS_VOLATILE (ret
) |= 1;
988 if (TREE_READONLY (field
))
989 TREE_READONLY (ret
) |= 1;
993 /* Build tree nodes to access the field for VAR on the receiver side. */
996 build_receiver_ref (tree var
, bool by_ref
, omp_context
*ctx
)
998 tree x
, field
= lookup_field (var
, ctx
);
1000 /* If the receiver record type was remapped in the child function,
1001 remap the field into the new record type. */
1002 x
= maybe_lookup_field (field
, ctx
);
1006 x
= build_simple_mem_ref (ctx
->receiver_decl
);
1007 x
= omp_build_component_ref (x
, field
);
1009 x
= build_simple_mem_ref (x
);
1014 /* Build tree nodes to access VAR in the scope outer to CTX. In the case
1015 of a parallel, this is a component reference; for workshare constructs
1016 this is some variable. */
1019 build_outer_var_ref (tree var
, omp_context
*ctx
)
1023 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
, ctx
)))
1025 else if (is_variable_sized (var
))
1027 x
= TREE_OPERAND (DECL_VALUE_EXPR (var
), 0);
1028 x
= build_outer_var_ref (x
, ctx
);
1029 x
= build_simple_mem_ref (x
);
1031 else if (is_taskreg_ctx (ctx
))
1033 bool by_ref
= use_pointer_for_field (var
, NULL
);
1034 x
= build_receiver_ref (var
, by_ref
, ctx
);
1036 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
1037 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
1039 /* #pragma omp simd isn't a worksharing construct, and can reference even
1040 private vars in its linear etc. clauses. */
1042 if (ctx
->outer
&& is_taskreg_ctx (ctx
))
1043 x
= lookup_decl (var
, ctx
->outer
);
1044 else if (ctx
->outer
)
1045 x
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
1049 else if (ctx
->outer
)
1050 x
= lookup_decl (var
, ctx
->outer
);
1051 else if (is_reference (var
))
1052 /* This can happen with orphaned constructs. If var is reference, it is
1053 possible it is shared and as such valid. */
1058 if (is_reference (var
))
1059 x
= build_simple_mem_ref (x
);
1064 /* Build tree nodes to access the field for VAR on the sender side. */
1067 build_sender_ref (tree var
, omp_context
*ctx
)
1069 tree field
= lookup_sfield (var
, ctx
);
1070 return omp_build_component_ref (ctx
->sender_decl
, field
);
1073 /* Add a new field for VAR inside the structure CTX->SENDER_DECL. */
1076 install_var_field (tree var
, bool by_ref
, int mask
, omp_context
*ctx
)
1078 tree field
, type
, sfield
= NULL_TREE
;
1080 gcc_assert ((mask
& 1) == 0
1081 || !splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) var
));
1082 gcc_assert ((mask
& 2) == 0 || !ctx
->sfield_map
1083 || !splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) var
));
1085 type
= TREE_TYPE (var
);
1088 gcc_assert (TREE_CODE (type
) == ARRAY_TYPE
);
1089 type
= build_pointer_type (build_pointer_type (type
));
1092 type
= build_pointer_type (type
);
1093 else if ((mask
& 3) == 1 && is_reference (var
))
1094 type
= TREE_TYPE (type
);
1096 field
= build_decl (DECL_SOURCE_LOCATION (var
),
1097 FIELD_DECL
, DECL_NAME (var
), type
);
1099 /* Remember what variable this field was created for. This does have a
1100 side effect of making dwarf2out ignore this member, so for helpful
1101 debugging we clear it later in delete_omp_context. */
1102 DECL_ABSTRACT_ORIGIN (field
) = var
;
1103 if (type
== TREE_TYPE (var
))
1105 DECL_ALIGN (field
) = DECL_ALIGN (var
);
1106 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1107 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1110 DECL_ALIGN (field
) = TYPE_ALIGN (type
);
1112 if ((mask
& 3) == 3)
1114 insert_field_into_struct (ctx
->record_type
, field
);
1115 if (ctx
->srecord_type
)
1117 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1118 FIELD_DECL
, DECL_NAME (var
), type
);
1119 DECL_ABSTRACT_ORIGIN (sfield
) = var
;
1120 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
1121 DECL_USER_ALIGN (sfield
) = DECL_USER_ALIGN (field
);
1122 TREE_THIS_VOLATILE (sfield
) = TREE_THIS_VOLATILE (field
);
1123 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1128 if (ctx
->srecord_type
== NULL_TREE
)
1132 ctx
->srecord_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1133 ctx
->sfield_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
1134 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= TREE_CHAIN (t
))
1136 sfield
= build_decl (DECL_SOURCE_LOCATION (var
),
1137 FIELD_DECL
, DECL_NAME (t
), TREE_TYPE (t
));
1138 DECL_ABSTRACT_ORIGIN (sfield
) = DECL_ABSTRACT_ORIGIN (t
);
1139 insert_field_into_struct (ctx
->srecord_type
, sfield
);
1140 splay_tree_insert (ctx
->sfield_map
,
1141 (splay_tree_key
) DECL_ABSTRACT_ORIGIN (t
),
1142 (splay_tree_value
) sfield
);
1146 insert_field_into_struct ((mask
& 1) ? ctx
->record_type
1147 : ctx
->srecord_type
, field
);
1151 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) var
,
1152 (splay_tree_value
) field
);
1153 if ((mask
& 2) && ctx
->sfield_map
)
1154 splay_tree_insert (ctx
->sfield_map
, (splay_tree_key
) var
,
1155 (splay_tree_value
) sfield
);
1159 install_var_local (tree var
, omp_context
*ctx
)
1161 tree new_var
= omp_copy_decl_1 (var
, ctx
);
1162 insert_decl_map (&ctx
->cb
, var
, new_var
);
1166 /* Adjust the replacement for DECL in CTX for the new context. This means
1167 copying the DECL_VALUE_EXPR, and fixing up the type. */
1170 fixup_remapped_decl (tree decl
, omp_context
*ctx
, bool private_debug
)
1172 tree new_decl
, size
;
1174 new_decl
= lookup_decl (decl
, ctx
);
1176 TREE_TYPE (new_decl
) = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1178 if ((!TREE_CONSTANT (DECL_SIZE (new_decl
)) || private_debug
)
1179 && DECL_HAS_VALUE_EXPR_P (decl
))
1181 tree ve
= DECL_VALUE_EXPR (decl
);
1182 walk_tree (&ve
, copy_tree_body_r
, &ctx
->cb
, NULL
);
1183 SET_DECL_VALUE_EXPR (new_decl
, ve
);
1184 DECL_HAS_VALUE_EXPR_P (new_decl
) = 1;
1187 if (!TREE_CONSTANT (DECL_SIZE (new_decl
)))
1189 size
= remap_decl (DECL_SIZE (decl
), &ctx
->cb
);
1190 if (size
== error_mark_node
)
1191 size
= TYPE_SIZE (TREE_TYPE (new_decl
));
1192 DECL_SIZE (new_decl
) = size
;
1194 size
= remap_decl (DECL_SIZE_UNIT (decl
), &ctx
->cb
);
1195 if (size
== error_mark_node
)
1196 size
= TYPE_SIZE_UNIT (TREE_TYPE (new_decl
));
1197 DECL_SIZE_UNIT (new_decl
) = size
;
1201 /* The callback for remap_decl. Search all containing contexts for a
1202 mapping of the variable; this avoids having to duplicate the splay
1203 tree ahead of time. We know a mapping doesn't already exist in the
1204 given context. Create new mappings to implement default semantics. */
1207 omp_copy_decl (tree var
, copy_body_data
*cb
)
1209 omp_context
*ctx
= (omp_context
*) cb
;
1212 if (TREE_CODE (var
) == LABEL_DECL
)
1214 new_var
= create_artificial_label (DECL_SOURCE_LOCATION (var
));
1215 DECL_CONTEXT (new_var
) = current_function_decl
;
1216 insert_decl_map (&ctx
->cb
, var
, new_var
);
1220 while (!is_taskreg_ctx (ctx
))
1225 new_var
= maybe_lookup_decl (var
, ctx
);
1230 if (is_global_var (var
) || decl_function_context (var
) != ctx
->cb
.src_fn
)
1233 return error_mark_node
;
1237 /* Debugging dumps for parallel regions. */
1238 void dump_omp_region (FILE *, struct omp_region
*, int);
1239 void debug_omp_region (struct omp_region
*);
1240 void debug_all_omp_regions (void);
1242 /* Dump the parallel region tree rooted at REGION. */
1245 dump_omp_region (FILE *file
, struct omp_region
*region
, int indent
)
1247 fprintf (file
, "%*sbb %d: %s\n", indent
, "", region
->entry
->index
,
1248 gimple_code_name
[region
->type
]);
1251 dump_omp_region (file
, region
->inner
, indent
+ 4);
1255 fprintf (file
, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent
, "",
1256 region
->cont
->index
);
1260 fprintf (file
, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent
, "",
1261 region
->exit
->index
);
1263 fprintf (file
, "%*s[no exit marker]\n", indent
, "");
1266 dump_omp_region (file
, region
->next
, indent
);
1270 debug_omp_region (struct omp_region
*region
)
1272 dump_omp_region (stderr
, region
, 0);
1276 debug_all_omp_regions (void)
1278 dump_omp_region (stderr
, root_omp_region
, 0);
1282 /* Create a new parallel region starting at STMT inside region PARENT. */
1284 static struct omp_region
*
1285 new_omp_region (basic_block bb
, enum gimple_code type
,
1286 struct omp_region
*parent
)
1288 struct omp_region
*region
= XCNEW (struct omp_region
);
1290 region
->outer
= parent
;
1292 region
->type
= type
;
1296 /* This is a nested region. Add it to the list of inner
1297 regions in PARENT. */
1298 region
->next
= parent
->inner
;
1299 parent
->inner
= region
;
1303 /* This is a toplevel region. Add it to the list of toplevel
1304 regions in ROOT_OMP_REGION. */
1305 region
->next
= root_omp_region
;
1306 root_omp_region
= region
;
1312 /* Release the memory associated with the region tree rooted at REGION. */
1315 free_omp_region_1 (struct omp_region
*region
)
1317 struct omp_region
*i
, *n
;
1319 for (i
= region
->inner
; i
; i
= n
)
1322 free_omp_region_1 (i
);
1328 /* Release the memory for the entire omp region tree. */
1331 free_omp_regions (void)
1333 struct omp_region
*r
, *n
;
1334 for (r
= root_omp_region
; r
; r
= n
)
1337 free_omp_region_1 (r
);
1339 root_omp_region
= NULL
;
1343 /* Create a new context, with OUTER_CTX being the surrounding context. */
1345 static omp_context
*
1346 new_omp_context (gimple stmt
, omp_context
*outer_ctx
)
1348 omp_context
*ctx
= XCNEW (omp_context
);
1350 splay_tree_insert (all_contexts
, (splay_tree_key
) stmt
,
1351 (splay_tree_value
) ctx
);
1356 ctx
->outer
= outer_ctx
;
1357 ctx
->cb
= outer_ctx
->cb
;
1358 ctx
->cb
.block
= NULL
;
1359 ctx
->depth
= outer_ctx
->depth
+ 1;
1363 ctx
->cb
.src_fn
= current_function_decl
;
1364 ctx
->cb
.dst_fn
= current_function_decl
;
1365 ctx
->cb
.src_node
= cgraph_node::get (current_function_decl
);
1366 gcc_checking_assert (ctx
->cb
.src_node
);
1367 ctx
->cb
.dst_node
= ctx
->cb
.src_node
;
1368 ctx
->cb
.src_cfun
= cfun
;
1369 ctx
->cb
.copy_decl
= omp_copy_decl
;
1370 ctx
->cb
.eh_lp_nr
= 0;
1371 ctx
->cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
1375 ctx
->cb
.decl_map
= new hash_map
<tree
, tree
>;
1380 static gimple_seq
maybe_catch_exception (gimple_seq
);
1382 /* Finalize task copyfn. */
1385 finalize_task_copyfn (gimple task_stmt
)
1387 struct function
*child_cfun
;
1389 gimple_seq seq
= NULL
, new_seq
;
1392 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
1393 if (child_fn
== NULL_TREE
)
1396 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
1397 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
1399 push_cfun (child_cfun
);
1400 bind
= gimplify_body (child_fn
, false);
1401 gimple_seq_add_stmt (&seq
, bind
);
1402 new_seq
= maybe_catch_exception (seq
);
1405 bind
= gimple_build_bind (NULL
, new_seq
, NULL
);
1407 gimple_seq_add_stmt (&seq
, bind
);
1409 gimple_set_body (child_fn
, seq
);
1412 /* Inform the callgraph about the new function. */
1413 cgraph_node::add_new_function (child_fn
, false);
1416 /* Destroy a omp_context data structures. Called through the splay tree
1417 value delete callback. */
1420 delete_omp_context (splay_tree_value value
)
1422 omp_context
*ctx
= (omp_context
*) value
;
1424 delete ctx
->cb
.decl_map
;
1427 splay_tree_delete (ctx
->field_map
);
1428 if (ctx
->sfield_map
)
1429 splay_tree_delete (ctx
->sfield_map
);
1431 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before
1432 it produces corrupt debug information. */
1433 if (ctx
->record_type
)
1436 for (t
= TYPE_FIELDS (ctx
->record_type
); t
; t
= DECL_CHAIN (t
))
1437 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1439 if (ctx
->srecord_type
)
1442 for (t
= TYPE_FIELDS (ctx
->srecord_type
); t
; t
= DECL_CHAIN (t
))
1443 DECL_ABSTRACT_ORIGIN (t
) = NULL
;
1446 if (is_task_ctx (ctx
))
1447 finalize_task_copyfn (ctx
->stmt
);
1452 /* Fix up RECEIVER_DECL with a type that has been remapped to the child
1456 fixup_child_record_type (omp_context
*ctx
)
1458 tree f
, type
= ctx
->record_type
;
1460 /* ??? It isn't sufficient to just call remap_type here, because
1461 variably_modified_type_p doesn't work the way we expect for
1462 record types. Testing each field for whether it needs remapping
1463 and creating a new record by hand works, however. */
1464 for (f
= TYPE_FIELDS (type
); f
; f
= DECL_CHAIN (f
))
1465 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
1469 tree name
, new_fields
= NULL
;
1471 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1472 name
= DECL_NAME (TYPE_NAME (ctx
->record_type
));
1473 name
= build_decl (DECL_SOURCE_LOCATION (ctx
->receiver_decl
),
1474 TYPE_DECL
, name
, type
);
1475 TYPE_NAME (type
) = name
;
1477 for (f
= TYPE_FIELDS (ctx
->record_type
); f
; f
= DECL_CHAIN (f
))
1479 tree new_f
= copy_node (f
);
1480 DECL_CONTEXT (new_f
) = type
;
1481 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &ctx
->cb
);
1482 DECL_CHAIN (new_f
) = new_fields
;
1483 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &ctx
->cb
, NULL
);
1484 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
,
1486 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
1490 /* Arrange to be able to look up the receiver field
1491 given the sender field. */
1492 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) f
,
1493 (splay_tree_value
) new_f
);
1495 TYPE_FIELDS (type
) = nreverse (new_fields
);
1499 TREE_TYPE (ctx
->receiver_decl
) = build_pointer_type (type
);
1502 /* Instantiate decls as necessary in CTX to satisfy the data sharing
1503 specified by CLAUSES. */
1506 scan_sharing_clauses (tree clauses
, omp_context
*ctx
)
1509 bool scan_array_reductions
= false;
1511 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1515 switch (OMP_CLAUSE_CODE (c
))
1517 case OMP_CLAUSE_PRIVATE
:
1518 decl
= OMP_CLAUSE_DECL (c
);
1519 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
1521 else if (!is_variable_sized (decl
))
1522 install_var_local (decl
, ctx
);
1525 case OMP_CLAUSE_SHARED
:
1526 decl
= OMP_CLAUSE_DECL (c
);
1527 /* Ignore shared directives in teams construct. */
1528 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1530 /* Global variables don't need to be copied,
1531 the receiver side will use them directly. */
1532 tree odecl
= maybe_lookup_decl_in_outer_ctx (decl
, ctx
);
1533 if (is_global_var (odecl
))
1535 insert_decl_map (&ctx
->cb
, decl
, odecl
);
1538 gcc_assert (is_taskreg_ctx (ctx
));
1539 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl
))
1540 || !is_variable_sized (decl
));
1541 /* Global variables don't need to be copied,
1542 the receiver side will use them directly. */
1543 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1545 by_ref
= use_pointer_for_field (decl
, ctx
);
1546 if (! TREE_READONLY (decl
)
1547 || TREE_ADDRESSABLE (decl
)
1549 || is_reference (decl
))
1551 install_var_field (decl
, by_ref
, 3, ctx
);
1552 install_var_local (decl
, ctx
);
1555 /* We don't need to copy const scalar vars back. */
1556 OMP_CLAUSE_SET_CODE (c
, OMP_CLAUSE_FIRSTPRIVATE
);
1559 case OMP_CLAUSE_LASTPRIVATE
:
1560 /* Let the corresponding firstprivate clause create
1562 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1566 case OMP_CLAUSE_FIRSTPRIVATE
:
1567 case OMP_CLAUSE_REDUCTION
:
1568 case OMP_CLAUSE_LINEAR
:
1569 decl
= OMP_CLAUSE_DECL (c
);
1571 if (is_variable_sized (decl
))
1573 if (is_task_ctx (ctx
))
1574 install_var_field (decl
, false, 1, ctx
);
1577 else if (is_taskreg_ctx (ctx
))
1580 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
));
1581 by_ref
= use_pointer_for_field (decl
, NULL
);
1583 if (is_task_ctx (ctx
)
1584 && (global
|| by_ref
|| is_reference (decl
)))
1586 install_var_field (decl
, false, 1, ctx
);
1588 install_var_field (decl
, by_ref
, 2, ctx
);
1591 install_var_field (decl
, by_ref
, 3, ctx
);
1593 install_var_local (decl
, ctx
);
1596 case OMP_CLAUSE__LOOPTEMP_
:
1597 gcc_assert (is_parallel_ctx (ctx
));
1598 decl
= OMP_CLAUSE_DECL (c
);
1599 install_var_field (decl
, false, 3, ctx
);
1600 install_var_local (decl
, ctx
);
1603 case OMP_CLAUSE_COPYPRIVATE
:
1604 case OMP_CLAUSE_COPYIN
:
1605 decl
= OMP_CLAUSE_DECL (c
);
1606 by_ref
= use_pointer_for_field (decl
, NULL
);
1607 install_var_field (decl
, by_ref
, 3, ctx
);
1610 case OMP_CLAUSE_DEFAULT
:
1611 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_KIND (c
);
1614 case OMP_CLAUSE_FINAL
:
1616 case OMP_CLAUSE_NUM_THREADS
:
1617 case OMP_CLAUSE_NUM_TEAMS
:
1618 case OMP_CLAUSE_THREAD_LIMIT
:
1619 case OMP_CLAUSE_DEVICE
:
1620 case OMP_CLAUSE_SCHEDULE
:
1621 case OMP_CLAUSE_DIST_SCHEDULE
:
1622 case OMP_CLAUSE_DEPEND
:
1623 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1625 scan_omp_op (&OMP_CLAUSE_OPERAND (c
, 0), ctx
->outer
);
1629 case OMP_CLAUSE_FROM
:
1630 case OMP_CLAUSE_MAP
:
1632 scan_omp_op (&OMP_CLAUSE_SIZE (c
), ctx
->outer
);
1633 decl
= OMP_CLAUSE_DECL (c
);
1634 /* Global variables with "omp declare target" attribute
1635 don't need to be copied, the receiver side will use them
1637 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1639 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1640 && lookup_attribute ("omp declare target",
1641 DECL_ATTRIBUTES (decl
)))
1643 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1644 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
)
1646 /* Ignore OMP_CLAUSE_MAP_POINTER kind for arrays in
1647 #pragma omp target data, there is nothing to map for
1649 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
1650 && !POINTER_TYPE_P (TREE_TYPE (decl
)))
1655 if (DECL_SIZE (decl
)
1656 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1658 tree decl2
= DECL_VALUE_EXPR (decl
);
1659 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1660 decl2
= TREE_OPERAND (decl2
, 0);
1661 gcc_assert (DECL_P (decl2
));
1662 install_var_field (decl2
, true, 3, ctx
);
1663 install_var_local (decl2
, ctx
);
1664 install_var_local (decl
, ctx
);
1668 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
1669 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1670 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
1671 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1672 install_var_field (decl
, true, 7, ctx
);
1674 install_var_field (decl
, true, 3, ctx
);
1675 if (gimple_omp_target_kind (ctx
->stmt
)
1676 == GF_OMP_TARGET_KIND_REGION
)
1677 install_var_local (decl
, ctx
);
1682 tree base
= get_base_address (decl
);
1683 tree nc
= OMP_CLAUSE_CHAIN (c
);
1686 && OMP_CLAUSE_CODE (nc
) == OMP_CLAUSE_MAP
1687 && OMP_CLAUSE_DECL (nc
) == base
1688 && OMP_CLAUSE_MAP_KIND (nc
) == OMP_CLAUSE_MAP_POINTER
1689 && integer_zerop (OMP_CLAUSE_SIZE (nc
)))
1691 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
) = 1;
1692 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc
) = 1;
1698 scan_omp_op (&OMP_CLAUSE_DECL (c
), ctx
->outer
);
1699 decl
= OMP_CLAUSE_DECL (c
);
1701 gcc_assert (!splay_tree_lookup (ctx
->field_map
,
1702 (splay_tree_key
) decl
));
1704 = build_decl (OMP_CLAUSE_LOCATION (c
),
1705 FIELD_DECL
, NULL_TREE
, ptr_type_node
);
1706 DECL_ALIGN (field
) = TYPE_ALIGN (ptr_type_node
);
1707 insert_field_into_struct (ctx
->record_type
, field
);
1708 splay_tree_insert (ctx
->field_map
, (splay_tree_key
) decl
,
1709 (splay_tree_value
) field
);
1714 case OMP_CLAUSE_NOWAIT
:
1715 case OMP_CLAUSE_ORDERED
:
1716 case OMP_CLAUSE_COLLAPSE
:
1717 case OMP_CLAUSE_UNTIED
:
1718 case OMP_CLAUSE_MERGEABLE
:
1719 case OMP_CLAUSE_PROC_BIND
:
1720 case OMP_CLAUSE_SAFELEN
:
1723 case OMP_CLAUSE_ALIGNED
:
1724 decl
= OMP_CLAUSE_DECL (c
);
1725 if (is_global_var (decl
)
1726 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
)
1727 install_var_local (decl
, ctx
);
1735 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1737 switch (OMP_CLAUSE_CODE (c
))
1739 case OMP_CLAUSE_LASTPRIVATE
:
1740 /* Let the corresponding firstprivate clause create
1742 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1743 scan_array_reductions
= true;
1744 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
1748 case OMP_CLAUSE_PRIVATE
:
1749 case OMP_CLAUSE_FIRSTPRIVATE
:
1750 case OMP_CLAUSE_REDUCTION
:
1751 case OMP_CLAUSE_LINEAR
:
1752 decl
= OMP_CLAUSE_DECL (c
);
1753 if (is_variable_sized (decl
))
1754 install_var_local (decl
, ctx
);
1755 fixup_remapped_decl (decl
, ctx
,
1756 OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_PRIVATE
1757 && OMP_CLAUSE_PRIVATE_DEBUG (c
));
1758 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1759 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1760 scan_array_reductions
= true;
1761 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1762 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1763 scan_array_reductions
= true;
1766 case OMP_CLAUSE_SHARED
:
1767 /* Ignore shared directives in teams construct. */
1768 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
1770 decl
= OMP_CLAUSE_DECL (c
);
1771 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
1772 fixup_remapped_decl (decl
, ctx
, false);
1775 case OMP_CLAUSE_MAP
:
1776 if (gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_DATA
)
1778 decl
= OMP_CLAUSE_DECL (c
);
1780 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
))
1781 && lookup_attribute ("omp declare target",
1782 DECL_ATTRIBUTES (decl
)))
1786 if (OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
1787 && TREE_CODE (TREE_TYPE (decl
)) == ARRAY_TYPE
1788 && !COMPLETE_TYPE_P (TREE_TYPE (decl
)))
1790 tree new_decl
= lookup_decl (decl
, ctx
);
1791 TREE_TYPE (new_decl
)
1792 = remap_type (TREE_TYPE (decl
), &ctx
->cb
);
1794 else if (DECL_SIZE (decl
)
1795 && TREE_CODE (DECL_SIZE (decl
)) != INTEGER_CST
)
1797 tree decl2
= DECL_VALUE_EXPR (decl
);
1798 gcc_assert (TREE_CODE (decl2
) == INDIRECT_REF
);
1799 decl2
= TREE_OPERAND (decl2
, 0);
1800 gcc_assert (DECL_P (decl2
));
1801 fixup_remapped_decl (decl2
, ctx
, false);
1802 fixup_remapped_decl (decl
, ctx
, true);
1805 fixup_remapped_decl (decl
, ctx
, false);
1809 case OMP_CLAUSE_COPYPRIVATE
:
1810 case OMP_CLAUSE_COPYIN
:
1811 case OMP_CLAUSE_DEFAULT
:
1813 case OMP_CLAUSE_NUM_THREADS
:
1814 case OMP_CLAUSE_NUM_TEAMS
:
1815 case OMP_CLAUSE_THREAD_LIMIT
:
1816 case OMP_CLAUSE_DEVICE
:
1817 case OMP_CLAUSE_SCHEDULE
:
1818 case OMP_CLAUSE_DIST_SCHEDULE
:
1819 case OMP_CLAUSE_NOWAIT
:
1820 case OMP_CLAUSE_ORDERED
:
1821 case OMP_CLAUSE_COLLAPSE
:
1822 case OMP_CLAUSE_UNTIED
:
1823 case OMP_CLAUSE_FINAL
:
1824 case OMP_CLAUSE_MERGEABLE
:
1825 case OMP_CLAUSE_PROC_BIND
:
1826 case OMP_CLAUSE_SAFELEN
:
1827 case OMP_CLAUSE_ALIGNED
:
1828 case OMP_CLAUSE_DEPEND
:
1829 case OMP_CLAUSE__LOOPTEMP_
:
1831 case OMP_CLAUSE_FROM
:
1832 case OMP_CLAUSE__CILK_FOR_COUNT_
:
1840 if (scan_array_reductions
)
1841 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
1842 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
1843 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
1845 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
), ctx
);
1846 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
1848 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
1849 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
1850 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
1851 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
1852 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
1853 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
1856 /* Create a new name for omp child function. Returns an identifier. If
1857 IS_CILK_FOR is true then the suffix for the child function is
1861 create_omp_child_function_name (bool task_copy
, bool is_cilk_for
)
1864 return clone_function_name (current_function_decl
, "_cilk_for_fn");
1865 return clone_function_name (current_function_decl
,
1866 task_copy
? "_omp_cpyfn" : "_omp_fn");
1869 /* Returns the type of the induction variable for the child function for
1870 _Cilk_for and the types for _high and _low variables based on TYPE. */
1873 cilk_for_check_loop_diff_type (tree type
)
1875 if (TYPE_PRECISION (type
) <= TYPE_PRECISION (uint32_type_node
))
1877 if (TYPE_UNSIGNED (type
))
1878 return uint32_type_node
;
1880 return integer_type_node
;
1884 if (TYPE_UNSIGNED (type
))
1885 return uint64_type_node
;
1887 return long_long_integer_type_node
;
1891 /* Build a decl for the omp child function. It'll not contain a body
1892 yet, just the bare decl. */
1895 create_omp_child_function (omp_context
*ctx
, bool task_copy
)
1897 tree decl
, type
, name
, t
;
1900 = (flag_cilkplus
&& gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
1901 ? find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
1902 OMP_CLAUSE__CILK_FOR_COUNT_
) : NULL_TREE
;
1903 tree cilk_var_type
= NULL_TREE
;
1905 name
= create_omp_child_function_name (task_copy
,
1906 cilk_for_count
!= NULL_TREE
);
1908 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1909 ptr_type_node
, NULL_TREE
);
1910 else if (cilk_for_count
)
1912 type
= TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count
, 0));
1913 cilk_var_type
= cilk_for_check_loop_diff_type (type
);
1914 type
= build_function_type_list (void_type_node
, ptr_type_node
,
1915 cilk_var_type
, cilk_var_type
, NULL_TREE
);
1918 type
= build_function_type_list (void_type_node
, ptr_type_node
, NULL_TREE
);
1920 decl
= build_decl (gimple_location (ctx
->stmt
), FUNCTION_DECL
, name
, type
);
1923 ctx
->cb
.dst_fn
= decl
;
1925 gimple_omp_task_set_copy_fn (ctx
->stmt
, decl
);
1927 TREE_STATIC (decl
) = 1;
1928 TREE_USED (decl
) = 1;
1929 DECL_ARTIFICIAL (decl
) = 1;
1930 DECL_IGNORED_P (decl
) = 0;
1931 TREE_PUBLIC (decl
) = 0;
1932 DECL_UNINLINABLE (decl
) = 1;
1933 DECL_EXTERNAL (decl
) = 0;
1934 DECL_CONTEXT (decl
) = NULL_TREE
;
1935 DECL_INITIAL (decl
) = make_node (BLOCK
);
1936 bool target_p
= false;
1937 if (lookup_attribute ("omp declare target",
1938 DECL_ATTRIBUTES (current_function_decl
)))
1943 for (octx
= ctx
; octx
; octx
= octx
->outer
)
1944 if (gimple_code (octx
->stmt
) == GIMPLE_OMP_TARGET
1945 && gimple_omp_target_kind (octx
->stmt
)
1946 == GF_OMP_TARGET_KIND_REGION
)
1953 DECL_ATTRIBUTES (decl
)
1954 = tree_cons (get_identifier ("omp declare target"),
1955 NULL_TREE
, DECL_ATTRIBUTES (decl
));
1957 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1958 RESULT_DECL
, NULL_TREE
, void_type_node
);
1959 DECL_ARTIFICIAL (t
) = 1;
1960 DECL_IGNORED_P (t
) = 1;
1961 DECL_CONTEXT (t
) = decl
;
1962 DECL_RESULT (decl
) = t
;
1964 /* _Cilk_for's child function requires two extra parameters called
1965 __low and __high that are set the by Cilk runtime when it calls this
1969 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1970 PARM_DECL
, get_identifier ("__high"), cilk_var_type
);
1971 DECL_ARTIFICIAL (t
) = 1;
1972 DECL_NAMELESS (t
) = 1;
1973 DECL_ARG_TYPE (t
) = ptr_type_node
;
1974 DECL_CONTEXT (t
) = current_function_decl
;
1976 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1977 DECL_ARGUMENTS (decl
) = t
;
1979 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
1980 PARM_DECL
, get_identifier ("__low"), cilk_var_type
);
1981 DECL_ARTIFICIAL (t
) = 1;
1982 DECL_NAMELESS (t
) = 1;
1983 DECL_ARG_TYPE (t
) = ptr_type_node
;
1984 DECL_CONTEXT (t
) = current_function_decl
;
1986 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
1987 DECL_ARGUMENTS (decl
) = t
;
1990 tree data_name
= get_identifier (".omp_data_i");
1991 t
= build_decl (DECL_SOURCE_LOCATION (decl
), PARM_DECL
, data_name
,
1993 DECL_ARTIFICIAL (t
) = 1;
1994 DECL_NAMELESS (t
) = 1;
1995 DECL_ARG_TYPE (t
) = ptr_type_node
;
1996 DECL_CONTEXT (t
) = current_function_decl
;
1999 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2000 DECL_ARGUMENTS (decl
) = t
;
2002 ctx
->receiver_decl
= t
;
2005 t
= build_decl (DECL_SOURCE_LOCATION (decl
),
2006 PARM_DECL
, get_identifier (".omp_data_o"),
2008 DECL_ARTIFICIAL (t
) = 1;
2009 DECL_NAMELESS (t
) = 1;
2010 DECL_ARG_TYPE (t
) = ptr_type_node
;
2011 DECL_CONTEXT (t
) = current_function_decl
;
2013 TREE_ADDRESSABLE (t
) = 1;
2014 DECL_CHAIN (t
) = DECL_ARGUMENTS (decl
);
2015 DECL_ARGUMENTS (decl
) = t
;
2018 /* Allocate memory for the function structure. The call to
2019 allocate_struct_function clobbers CFUN, so we need to restore
2021 push_struct_function (decl
);
2022 cfun
->function_end_locus
= gimple_location (ctx
->stmt
);
2026 /* Callback for walk_gimple_seq. Check if combined parallel
2027 contains gimple_omp_for_combined_into_p OMP_FOR. */
2030 find_combined_for (gimple_stmt_iterator
*gsi_p
,
2031 bool *handled_ops_p
,
2032 struct walk_stmt_info
*wi
)
2034 gimple stmt
= gsi_stmt (*gsi_p
);
2036 *handled_ops_p
= true;
2037 switch (gimple_code (stmt
))
2041 case GIMPLE_OMP_FOR
:
2042 if (gimple_omp_for_combined_into_p (stmt
)
2043 && gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
)
2046 return integer_zero_node
;
2055 /* Scan an OpenMP parallel directive. */
2058 scan_omp_parallel (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2062 gimple stmt
= gsi_stmt (*gsi
);
2064 /* Ignore parallel directives with empty bodies, unless there
2065 are copyin clauses. */
2067 && empty_body_p (gimple_omp_body (stmt
))
2068 && find_omp_clause (gimple_omp_parallel_clauses (stmt
),
2069 OMP_CLAUSE_COPYIN
) == NULL
)
2071 gsi_replace (gsi
, gimple_build_nop (), false);
2075 if (gimple_omp_parallel_combined_p (stmt
))
2078 struct walk_stmt_info wi
;
2080 memset (&wi
, 0, sizeof (wi
));
2082 walk_gimple_seq (gimple_omp_body (stmt
),
2083 find_combined_for
, NULL
, &wi
);
2084 for_stmt
= (gimple
) wi
.info
;
2087 struct omp_for_data fd
;
2088 extract_omp_for_data (for_stmt
, &fd
, NULL
);
2089 /* We need two temporaries with fd.loop.v type (istart/iend)
2090 and then (fd.collapse - 1) temporaries with the same
2091 type for count2 ... countN-1 vars if not constant. */
2092 size_t count
= 2, i
;
2093 tree type
= fd
.iter_type
;
2095 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
2096 count
+= fd
.collapse
- 1;
2097 for (i
= 0; i
< count
; i
++)
2099 tree temp
= create_tmp_var (type
, NULL
);
2100 tree c
= build_omp_clause (UNKNOWN_LOCATION
,
2101 OMP_CLAUSE__LOOPTEMP_
);
2102 insert_decl_map (&outer_ctx
->cb
, temp
, temp
);
2103 OMP_CLAUSE_DECL (c
) = temp
;
2104 OMP_CLAUSE_CHAIN (c
) = gimple_omp_parallel_clauses (stmt
);
2105 gimple_omp_parallel_set_clauses (stmt
, c
);
2110 ctx
= new_omp_context (stmt
, outer_ctx
);
2111 taskreg_contexts
.safe_push (ctx
);
2112 if (taskreg_nesting_level
> 1)
2113 ctx
->is_nested
= true;
2114 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2115 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2116 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2117 name
= create_tmp_var_name (".omp_data_s");
2118 name
= build_decl (gimple_location (stmt
),
2119 TYPE_DECL
, name
, ctx
->record_type
);
2120 DECL_ARTIFICIAL (name
) = 1;
2121 DECL_NAMELESS (name
) = 1;
2122 TYPE_NAME (ctx
->record_type
) = name
;
2123 create_omp_child_function (ctx
, false);
2124 gimple_omp_parallel_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2126 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt
), ctx
);
2127 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2129 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2130 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2133 /* Scan an OpenMP task directive. */
2136 scan_omp_task (gimple_stmt_iterator
*gsi
, omp_context
*outer_ctx
)
2140 gimple stmt
= gsi_stmt (*gsi
);
2142 /* Ignore task directives with empty bodies. */
2144 && empty_body_p (gimple_omp_body (stmt
)))
2146 gsi_replace (gsi
, gimple_build_nop (), false);
2150 ctx
= new_omp_context (stmt
, outer_ctx
);
2151 taskreg_contexts
.safe_push (ctx
);
2152 if (taskreg_nesting_level
> 1)
2153 ctx
->is_nested
= true;
2154 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2155 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2156 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2157 name
= create_tmp_var_name (".omp_data_s");
2158 name
= build_decl (gimple_location (stmt
),
2159 TYPE_DECL
, name
, ctx
->record_type
);
2160 DECL_ARTIFICIAL (name
) = 1;
2161 DECL_NAMELESS (name
) = 1;
2162 TYPE_NAME (ctx
->record_type
) = name
;
2163 create_omp_child_function (ctx
, false);
2164 gimple_omp_task_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2166 scan_sharing_clauses (gimple_omp_task_clauses (stmt
), ctx
);
2168 if (ctx
->srecord_type
)
2170 name
= create_tmp_var_name (".omp_data_a");
2171 name
= build_decl (gimple_location (stmt
),
2172 TYPE_DECL
, name
, ctx
->srecord_type
);
2173 DECL_ARTIFICIAL (name
) = 1;
2174 DECL_NAMELESS (name
) = 1;
2175 TYPE_NAME (ctx
->srecord_type
) = name
;
2176 create_omp_child_function (ctx
, true);
2179 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2181 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2183 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2184 t
= build_int_cst (long_integer_type_node
, 0);
2185 gimple_omp_task_set_arg_size (stmt
, t
);
2186 t
= build_int_cst (long_integer_type_node
, 1);
2187 gimple_omp_task_set_arg_align (stmt
, t
);
2192 /* If any decls have been made addressable during scan_omp,
2193 adjust their fields if needed, and layout record types
2194 of parallel/task constructs. */
2197 finish_taskreg_scan (omp_context
*ctx
)
2199 if (ctx
->record_type
== NULL_TREE
)
2202 /* If any task_shared_vars were needed, verify all
2203 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
2204 statements if use_pointer_for_field hasn't changed
2205 because of that. If it did, update field types now. */
2206 if (task_shared_vars
)
2210 for (c
= gimple_omp_taskreg_clauses (ctx
->stmt
);
2211 c
; c
= OMP_CLAUSE_CHAIN (c
))
2212 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_SHARED
)
2214 tree decl
= OMP_CLAUSE_DECL (c
);
2216 /* Global variables don't need to be copied,
2217 the receiver side will use them directly. */
2218 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl
, ctx
)))
2220 if (!bitmap_bit_p (task_shared_vars
, DECL_UID (decl
))
2221 || !use_pointer_for_field (decl
, ctx
))
2223 tree field
= lookup_field (decl
, ctx
);
2224 if (TREE_CODE (TREE_TYPE (field
)) == POINTER_TYPE
2225 && TREE_TYPE (TREE_TYPE (field
)) == TREE_TYPE (decl
))
2227 TREE_TYPE (field
) = build_pointer_type (TREE_TYPE (decl
));
2228 TREE_THIS_VOLATILE (field
) = 0;
2229 DECL_USER_ALIGN (field
) = 0;
2230 DECL_ALIGN (field
) = TYPE_ALIGN (TREE_TYPE (field
));
2231 if (TYPE_ALIGN (ctx
->record_type
) < DECL_ALIGN (field
))
2232 TYPE_ALIGN (ctx
->record_type
) = DECL_ALIGN (field
);
2233 if (ctx
->srecord_type
)
2235 tree sfield
= lookup_sfield (decl
, ctx
);
2236 TREE_TYPE (sfield
) = TREE_TYPE (field
);
2237 TREE_THIS_VOLATILE (sfield
) = 0;
2238 DECL_USER_ALIGN (sfield
) = 0;
2239 DECL_ALIGN (sfield
) = DECL_ALIGN (field
);
2240 if (TYPE_ALIGN (ctx
->srecord_type
) < DECL_ALIGN (sfield
))
2241 TYPE_ALIGN (ctx
->srecord_type
) = DECL_ALIGN (sfield
);
2246 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_PARALLEL
)
2248 layout_type (ctx
->record_type
);
2249 fixup_child_record_type (ctx
);
2253 location_t loc
= gimple_location (ctx
->stmt
);
2254 tree
*p
, vla_fields
= NULL_TREE
, *q
= &vla_fields
;
2255 /* Move VLA fields to the end. */
2256 p
= &TYPE_FIELDS (ctx
->record_type
);
2258 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p
))
2259 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p
))))
2262 *p
= TREE_CHAIN (*p
);
2263 TREE_CHAIN (*q
) = NULL_TREE
;
2264 q
= &TREE_CHAIN (*q
);
2267 p
= &DECL_CHAIN (*p
);
2269 layout_type (ctx
->record_type
);
2270 fixup_child_record_type (ctx
);
2271 if (ctx
->srecord_type
)
2272 layout_type (ctx
->srecord_type
);
2273 tree t
= fold_convert_loc (loc
, long_integer_type_node
,
2274 TYPE_SIZE_UNIT (ctx
->record_type
));
2275 gimple_omp_task_set_arg_size (ctx
->stmt
, t
);
2276 t
= build_int_cst (long_integer_type_node
,
2277 TYPE_ALIGN_UNIT (ctx
->record_type
));
2278 gimple_omp_task_set_arg_align (ctx
->stmt
, t
);
2283 /* Scan an OpenMP loop directive. */
2286 scan_omp_for (gimple stmt
, omp_context
*outer_ctx
)
2291 ctx
= new_omp_context (stmt
, outer_ctx
);
2293 scan_sharing_clauses (gimple_omp_for_clauses (stmt
), ctx
);
2295 scan_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
2296 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
2298 scan_omp_op (gimple_omp_for_index_ptr (stmt
, i
), ctx
);
2299 scan_omp_op (gimple_omp_for_initial_ptr (stmt
, i
), ctx
);
2300 scan_omp_op (gimple_omp_for_final_ptr (stmt
, i
), ctx
);
2301 scan_omp_op (gimple_omp_for_incr_ptr (stmt
, i
), ctx
);
2303 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2306 /* Scan an OpenMP sections directive. */
2309 scan_omp_sections (gimple stmt
, omp_context
*outer_ctx
)
2313 ctx
= new_omp_context (stmt
, outer_ctx
);
2314 scan_sharing_clauses (gimple_omp_sections_clauses (stmt
), ctx
);
2315 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2318 /* Scan an OpenMP single directive. */
2321 scan_omp_single (gimple stmt
, omp_context
*outer_ctx
)
2326 ctx
= new_omp_context (stmt
, outer_ctx
);
2327 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2328 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2329 name
= create_tmp_var_name (".omp_copy_s");
2330 name
= build_decl (gimple_location (stmt
),
2331 TYPE_DECL
, name
, ctx
->record_type
);
2332 TYPE_NAME (ctx
->record_type
) = name
;
2334 scan_sharing_clauses (gimple_omp_single_clauses (stmt
), ctx
);
2335 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2337 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2338 ctx
->record_type
= NULL
;
2340 layout_type (ctx
->record_type
);
2343 /* Scan an OpenMP target{, data, update} directive. */
2346 scan_omp_target (gimple stmt
, omp_context
*outer_ctx
)
2350 int kind
= gimple_omp_target_kind (stmt
);
2352 ctx
= new_omp_context (stmt
, outer_ctx
);
2353 ctx
->field_map
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
2354 ctx
->default_kind
= OMP_CLAUSE_DEFAULT_SHARED
;
2355 ctx
->record_type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
2356 name
= create_tmp_var_name (".omp_data_t");
2357 name
= build_decl (gimple_location (stmt
),
2358 TYPE_DECL
, name
, ctx
->record_type
);
2359 DECL_ARTIFICIAL (name
) = 1;
2360 DECL_NAMELESS (name
) = 1;
2361 TYPE_NAME (ctx
->record_type
) = name
;
2362 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2364 create_omp_child_function (ctx
, false);
2365 gimple_omp_target_set_child_fn (stmt
, ctx
->cb
.dst_fn
);
2368 scan_sharing_clauses (gimple_omp_target_clauses (stmt
), ctx
);
2369 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2371 if (TYPE_FIELDS (ctx
->record_type
) == NULL
)
2372 ctx
->record_type
= ctx
->receiver_decl
= NULL
;
2375 TYPE_FIELDS (ctx
->record_type
)
2376 = nreverse (TYPE_FIELDS (ctx
->record_type
));
2377 #ifdef ENABLE_CHECKING
2379 unsigned int align
= DECL_ALIGN (TYPE_FIELDS (ctx
->record_type
));
2380 for (field
= TYPE_FIELDS (ctx
->record_type
);
2382 field
= DECL_CHAIN (field
))
2383 gcc_assert (DECL_ALIGN (field
) == align
);
2385 layout_type (ctx
->record_type
);
2386 if (kind
== GF_OMP_TARGET_KIND_REGION
)
2387 fixup_child_record_type (ctx
);
2391 /* Scan an OpenMP teams directive. */
2394 scan_omp_teams (gimple stmt
, omp_context
*outer_ctx
)
2396 omp_context
*ctx
= new_omp_context (stmt
, outer_ctx
);
2397 scan_sharing_clauses (gimple_omp_teams_clauses (stmt
), ctx
);
2398 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2401 /* Check OpenMP nesting restrictions. */
2403 check_omp_nesting_restrictions (gimple stmt
, omp_context
*ctx
)
2407 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2408 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2410 error_at (gimple_location (stmt
),
2411 "OpenMP constructs may not be nested inside simd region");
2414 else if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
2416 if ((gimple_code (stmt
) != GIMPLE_OMP_FOR
2417 || (gimple_omp_for_kind (stmt
)
2418 != GF_OMP_FOR_KIND_DISTRIBUTE
))
2419 && gimple_code (stmt
) != GIMPLE_OMP_PARALLEL
)
2421 error_at (gimple_location (stmt
),
2422 "only distribute or parallel constructs are allowed to "
2423 "be closely nested inside teams construct");
2428 switch (gimple_code (stmt
))
2430 case GIMPLE_OMP_FOR
:
2431 if (gimple_omp_for_kind (stmt
) & GF_OMP_FOR_SIMD
)
2433 if (gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
2435 if (ctx
!= NULL
&& gimple_code (ctx
->stmt
) != GIMPLE_OMP_TEAMS
)
2437 error_at (gimple_location (stmt
),
2438 "distribute construct must be closely nested inside "
2446 if (is_gimple_call (stmt
)
2447 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2448 == BUILT_IN_GOMP_CANCEL
2449 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2450 == BUILT_IN_GOMP_CANCELLATION_POINT
))
2452 const char *bad
= NULL
;
2453 const char *kind
= NULL
;
2456 error_at (gimple_location (stmt
), "orphaned %qs construct",
2457 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2458 == BUILT_IN_GOMP_CANCEL
2459 ? "#pragma omp cancel"
2460 : "#pragma omp cancellation point");
2463 switch (tree_fits_shwi_p (gimple_call_arg (stmt
, 0))
2464 ? tree_to_shwi (gimple_call_arg (stmt
, 0))
2468 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_PARALLEL
)
2469 bad
= "#pragma omp parallel";
2470 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2471 == BUILT_IN_GOMP_CANCEL
2472 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2473 ctx
->cancellable
= true;
2477 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
2478 || gimple_omp_for_kind (ctx
->stmt
) != GF_OMP_FOR_KIND_FOR
)
2479 bad
= "#pragma omp for";
2480 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2481 == BUILT_IN_GOMP_CANCEL
2482 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2484 ctx
->cancellable
= true;
2485 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2487 warning_at (gimple_location (stmt
), 0,
2488 "%<#pragma omp cancel for%> inside "
2489 "%<nowait%> for construct");
2490 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2491 OMP_CLAUSE_ORDERED
))
2492 warning_at (gimple_location (stmt
), 0,
2493 "%<#pragma omp cancel for%> inside "
2494 "%<ordered%> for construct");
2499 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTIONS
2500 && gimple_code (ctx
->stmt
) != GIMPLE_OMP_SECTION
)
2501 bad
= "#pragma omp sections";
2502 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2503 == BUILT_IN_GOMP_CANCEL
2504 && !integer_zerop (gimple_call_arg (stmt
, 1)))
2506 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_SECTIONS
)
2508 ctx
->cancellable
= true;
2509 if (find_omp_clause (gimple_omp_sections_clauses
2512 warning_at (gimple_location (stmt
), 0,
2513 "%<#pragma omp cancel sections%> inside "
2514 "%<nowait%> sections construct");
2518 gcc_assert (ctx
->outer
2519 && gimple_code (ctx
->outer
->stmt
)
2520 == GIMPLE_OMP_SECTIONS
);
2521 ctx
->outer
->cancellable
= true;
2522 if (find_omp_clause (gimple_omp_sections_clauses
2525 warning_at (gimple_location (stmt
), 0,
2526 "%<#pragma omp cancel sections%> inside "
2527 "%<nowait%> sections construct");
2533 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_TASK
)
2534 bad
= "#pragma omp task";
2536 ctx
->cancellable
= true;
2540 error_at (gimple_location (stmt
), "invalid arguments");
2545 error_at (gimple_location (stmt
),
2546 "%<%s %s%> construct not closely nested inside of %qs",
2547 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2548 == BUILT_IN_GOMP_CANCEL
2549 ? "#pragma omp cancel"
2550 : "#pragma omp cancellation point", kind
, bad
);
2555 case GIMPLE_OMP_SECTIONS
:
2556 case GIMPLE_OMP_SINGLE
:
2557 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2558 switch (gimple_code (ctx
->stmt
))
2560 case GIMPLE_OMP_FOR
:
2561 case GIMPLE_OMP_SECTIONS
:
2562 case GIMPLE_OMP_SINGLE
:
2563 case GIMPLE_OMP_ORDERED
:
2564 case GIMPLE_OMP_MASTER
:
2565 case GIMPLE_OMP_TASK
:
2566 case GIMPLE_OMP_CRITICAL
:
2567 if (is_gimple_call (stmt
))
2569 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
))
2570 != BUILT_IN_GOMP_BARRIER
)
2572 error_at (gimple_location (stmt
),
2573 "barrier region may not be closely nested inside "
2574 "of work-sharing, critical, ordered, master or "
2575 "explicit task region");
2578 error_at (gimple_location (stmt
),
2579 "work-sharing region may not be closely nested inside "
2580 "of work-sharing, critical, ordered, master or explicit "
2583 case GIMPLE_OMP_PARALLEL
:
2589 case GIMPLE_OMP_MASTER
:
2590 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2591 switch (gimple_code (ctx
->stmt
))
2593 case GIMPLE_OMP_FOR
:
2594 case GIMPLE_OMP_SECTIONS
:
2595 case GIMPLE_OMP_SINGLE
:
2596 case GIMPLE_OMP_TASK
:
2597 error_at (gimple_location (stmt
),
2598 "master region may not be closely nested inside "
2599 "of work-sharing or explicit task region");
2601 case GIMPLE_OMP_PARALLEL
:
2607 case GIMPLE_OMP_ORDERED
:
2608 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2609 switch (gimple_code (ctx
->stmt
))
2611 case GIMPLE_OMP_CRITICAL
:
2612 case GIMPLE_OMP_TASK
:
2613 error_at (gimple_location (stmt
),
2614 "ordered region may not be closely nested inside "
2615 "of critical or explicit task region");
2617 case GIMPLE_OMP_FOR
:
2618 if (find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
2619 OMP_CLAUSE_ORDERED
) == NULL
)
2621 error_at (gimple_location (stmt
),
2622 "ordered region must be closely nested inside "
2623 "a loop region with an ordered clause");
2627 case GIMPLE_OMP_PARALLEL
:
2628 error_at (gimple_location (stmt
),
2629 "ordered region must be closely nested inside "
2630 "a loop region with an ordered clause");
2636 case GIMPLE_OMP_CRITICAL
:
2637 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2638 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_CRITICAL
2639 && (gimple_omp_critical_name (stmt
)
2640 == gimple_omp_critical_name (ctx
->stmt
)))
2642 error_at (gimple_location (stmt
),
2643 "critical region may not be nested inside a critical "
2644 "region with the same name");
2648 case GIMPLE_OMP_TEAMS
:
2650 || gimple_code (ctx
->stmt
) != GIMPLE_OMP_TARGET
2651 || gimple_omp_target_kind (ctx
->stmt
) != GF_OMP_TARGET_KIND_REGION
)
2653 error_at (gimple_location (stmt
),
2654 "teams construct not closely nested inside of target "
2659 case GIMPLE_OMP_TARGET
:
2660 for (; ctx
!= NULL
; ctx
= ctx
->outer
)
2661 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TARGET
2662 && gimple_omp_target_kind (ctx
->stmt
) == GF_OMP_TARGET_KIND_REGION
)
2665 switch (gimple_omp_target_kind (stmt
))
2667 case GF_OMP_TARGET_KIND_REGION
: name
= "target"; break;
2668 case GF_OMP_TARGET_KIND_DATA
: name
= "target data"; break;
2669 case GF_OMP_TARGET_KIND_UPDATE
: name
= "target update"; break;
2670 default: gcc_unreachable ();
2672 warning_at (gimple_location (stmt
), 0,
2673 "%s construct inside of target region", name
);
2683 /* Helper function scan_omp.
2685 Callback for walk_tree or operators in walk_gimple_stmt used to
2686 scan for OpenMP directives in TP. */
2689 scan_omp_1_op (tree
*tp
, int *walk_subtrees
, void *data
)
2691 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
2692 omp_context
*ctx
= (omp_context
*) wi
->info
;
2695 switch (TREE_CODE (t
))
2702 *tp
= remap_decl (t
, &ctx
->cb
);
2706 if (ctx
&& TYPE_P (t
))
2707 *tp
= remap_type (t
, &ctx
->cb
);
2708 else if (!DECL_P (t
))
2713 tree tem
= remap_type (TREE_TYPE (t
), &ctx
->cb
);
2714 if (tem
!= TREE_TYPE (t
))
2716 if (TREE_CODE (t
) == INTEGER_CST
)
2717 *tp
= wide_int_to_tree (tem
, t
);
2719 TREE_TYPE (t
) = tem
;
2729 /* Return true if FNDECL is a setjmp or a longjmp. */
2732 setjmp_or_longjmp_p (const_tree fndecl
)
2734 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
2735 && (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_SETJMP
2736 || DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_LONGJMP
))
2739 tree declname
= DECL_NAME (fndecl
);
2742 const char *name
= IDENTIFIER_POINTER (declname
);
2743 return !strcmp (name
, "setjmp") || !strcmp (name
, "longjmp");
2747 /* Helper function for scan_omp.
2749 Callback for walk_gimple_stmt used to scan for OpenMP directives in
2750 the current statement in GSI. */
2753 scan_omp_1_stmt (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
2754 struct walk_stmt_info
*wi
)
2756 gimple stmt
= gsi_stmt (*gsi
);
2757 omp_context
*ctx
= (omp_context
*) wi
->info
;
2759 if (gimple_has_location (stmt
))
2760 input_location
= gimple_location (stmt
);
2762 /* Check the OpenMP nesting restrictions. */
2763 bool remove
= false;
2764 if (is_gimple_omp (stmt
))
2765 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2766 else if (is_gimple_call (stmt
))
2768 tree fndecl
= gimple_call_fndecl (stmt
);
2771 if (setjmp_or_longjmp_p (fndecl
)
2773 && gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
2774 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
2777 error_at (gimple_location (stmt
),
2778 "setjmp/longjmp inside simd construct");
2780 else if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
2781 switch (DECL_FUNCTION_CODE (fndecl
))
2783 case BUILT_IN_GOMP_BARRIER
:
2784 case BUILT_IN_GOMP_CANCEL
:
2785 case BUILT_IN_GOMP_CANCELLATION_POINT
:
2786 case BUILT_IN_GOMP_TASKYIELD
:
2787 case BUILT_IN_GOMP_TASKWAIT
:
2788 case BUILT_IN_GOMP_TASKGROUP_START
:
2789 case BUILT_IN_GOMP_TASKGROUP_END
:
2790 remove
= !check_omp_nesting_restrictions (stmt
, ctx
);
2799 stmt
= gimple_build_nop ();
2800 gsi_replace (gsi
, stmt
, false);
2803 *handled_ops_p
= true;
2805 switch (gimple_code (stmt
))
2807 case GIMPLE_OMP_PARALLEL
:
2808 taskreg_nesting_level
++;
2809 scan_omp_parallel (gsi
, ctx
);
2810 taskreg_nesting_level
--;
2813 case GIMPLE_OMP_TASK
:
2814 taskreg_nesting_level
++;
2815 scan_omp_task (gsi
, ctx
);
2816 taskreg_nesting_level
--;
2819 case GIMPLE_OMP_FOR
:
2820 scan_omp_for (stmt
, ctx
);
2823 case GIMPLE_OMP_SECTIONS
:
2824 scan_omp_sections (stmt
, ctx
);
2827 case GIMPLE_OMP_SINGLE
:
2828 scan_omp_single (stmt
, ctx
);
2831 case GIMPLE_OMP_SECTION
:
2832 case GIMPLE_OMP_MASTER
:
2833 case GIMPLE_OMP_TASKGROUP
:
2834 case GIMPLE_OMP_ORDERED
:
2835 case GIMPLE_OMP_CRITICAL
:
2836 ctx
= new_omp_context (stmt
, ctx
);
2837 scan_omp (gimple_omp_body_ptr (stmt
), ctx
);
2840 case GIMPLE_OMP_TARGET
:
2841 scan_omp_target (stmt
, ctx
);
2844 case GIMPLE_OMP_TEAMS
:
2845 scan_omp_teams (stmt
, ctx
);
2852 *handled_ops_p
= false;
2854 for (var
= gimple_bind_vars (stmt
); var
; var
= DECL_CHAIN (var
))
2855 insert_decl_map (&ctx
->cb
, var
, var
);
2859 *handled_ops_p
= false;
2867 /* Scan all the statements starting at the current statement. CTX
2868 contains context information about the OpenMP directives and
2869 clauses found during the scan. */
2872 scan_omp (gimple_seq
*body_p
, omp_context
*ctx
)
2874 location_t saved_location
;
2875 struct walk_stmt_info wi
;
2877 memset (&wi
, 0, sizeof (wi
));
2879 wi
.want_locations
= true;
2881 saved_location
= input_location
;
2882 walk_gimple_seq_mod (body_p
, scan_omp_1_stmt
, scan_omp_1_op
, &wi
);
2883 input_location
= saved_location
;
2886 /* Re-gimplification and code generation routines. */
2888 /* Build a call to GOMP_barrier. */
2891 build_omp_barrier (tree lhs
)
2893 tree fndecl
= builtin_decl_explicit (lhs
? BUILT_IN_GOMP_BARRIER_CANCEL
2894 : BUILT_IN_GOMP_BARRIER
);
2895 gimple g
= gimple_build_call (fndecl
, 0);
2897 gimple_call_set_lhs (g
, lhs
);
2901 /* If a context was created for STMT when it was scanned, return it. */
2903 static omp_context
*
2904 maybe_lookup_ctx (gimple stmt
)
2907 n
= splay_tree_lookup (all_contexts
, (splay_tree_key
) stmt
);
2908 return n
? (omp_context
*) n
->value
: NULL
;
2912 /* Find the mapping for DECL in CTX or the immediately enclosing
2913 context that has a mapping for DECL.
2915 If CTX is a nested parallel directive, we may have to use the decl
2916 mappings created in CTX's parent context. Suppose that we have the
2917 following parallel nesting (variable UIDs showed for clarity):
2920 #omp parallel shared(iD.1562) -> outer parallel
2921 iD.1562 = iD.1562 + 1;
2923 #omp parallel shared (iD.1562) -> inner parallel
2924 iD.1562 = iD.1562 - 1;
2926 Each parallel structure will create a distinct .omp_data_s structure
2927 for copying iD.1562 in/out of the directive:
2929 outer parallel .omp_data_s.1.i -> iD.1562
2930 inner parallel .omp_data_s.2.i -> iD.1562
2932 A shared variable mapping will produce a copy-out operation before
2933 the parallel directive and a copy-in operation after it. So, in
2934 this case we would have:
2937 .omp_data_o.1.i = iD.1562;
2938 #omp parallel shared(iD.1562) -> outer parallel
2939 .omp_data_i.1 = &.omp_data_o.1
2940 .omp_data_i.1->i = .omp_data_i.1->i + 1;
2942 .omp_data_o.2.i = iD.1562; -> **
2943 #omp parallel shared(iD.1562) -> inner parallel
2944 .omp_data_i.2 = &.omp_data_o.2
2945 .omp_data_i.2->i = .omp_data_i.2->i - 1;
2948 ** This is a problem. The symbol iD.1562 cannot be referenced
2949 inside the body of the outer parallel region. But since we are
2950 emitting this copy operation while expanding the inner parallel
2951 directive, we need to access the CTX structure of the outer
2952 parallel directive to get the correct mapping:
2954 .omp_data_o.2.i = .omp_data_i.1->i
2956 Since there may be other workshare or parallel directives enclosing
2957 the parallel directive, it may be necessary to walk up the context
2958 parent chain. This is not a problem in general because nested
2959 parallelism happens only rarely. */
2962 lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2967 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2968 t
= maybe_lookup_decl (decl
, up
);
2970 gcc_assert (!ctx
->is_nested
|| t
|| is_global_var (decl
));
2972 return t
? t
: decl
;
2976 /* Similar to lookup_decl_in_outer_ctx, but return DECL if not found
2977 in outer contexts. */
2980 maybe_lookup_decl_in_outer_ctx (tree decl
, omp_context
*ctx
)
2985 for (up
= ctx
->outer
, t
= NULL
; up
&& t
== NULL
; up
= up
->outer
)
2986 t
= maybe_lookup_decl (decl
, up
);
2988 return t
? t
: decl
;
2992 /* Construct the initialization value for reduction CLAUSE. */
2995 omp_reduction_init (tree clause
, tree type
)
2997 location_t loc
= OMP_CLAUSE_LOCATION (clause
);
2998 switch (OMP_CLAUSE_REDUCTION_CODE (clause
))
3005 case TRUTH_ORIF_EXPR
:
3006 case TRUTH_XOR_EXPR
:
3008 return build_zero_cst (type
);
3011 case TRUTH_AND_EXPR
:
3012 case TRUTH_ANDIF_EXPR
:
3014 return fold_convert_loc (loc
, type
, integer_one_node
);
3017 return fold_convert_loc (loc
, type
, integer_minus_one_node
);
3020 if (SCALAR_FLOAT_TYPE_P (type
))
3022 REAL_VALUE_TYPE max
, min
;
3023 if (HONOR_INFINITIES (TYPE_MODE (type
)))
3026 real_arithmetic (&min
, NEGATE_EXPR
, &max
, NULL
);
3029 real_maxval (&min
, 1, TYPE_MODE (type
));
3030 return build_real (type
, min
);
3034 gcc_assert (INTEGRAL_TYPE_P (type
));
3035 return TYPE_MIN_VALUE (type
);
3039 if (SCALAR_FLOAT_TYPE_P (type
))
3041 REAL_VALUE_TYPE max
;
3042 if (HONOR_INFINITIES (TYPE_MODE (type
)))
3045 real_maxval (&max
, 0, TYPE_MODE (type
));
3046 return build_real (type
, max
);
3050 gcc_assert (INTEGRAL_TYPE_P (type
));
3051 return TYPE_MAX_VALUE (type
);
3059 /* Return alignment to be assumed for var in CLAUSE, which should be
3060 OMP_CLAUSE_ALIGNED. */
3063 omp_clause_aligned_alignment (tree clause
)
3065 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
))
3066 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause
);
3068 /* Otherwise return implementation defined alignment. */
3069 unsigned int al
= 1;
3070 enum machine_mode mode
, vmode
;
3071 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3073 vs
= 1 << floor_log2 (vs
);
3074 static enum mode_class classes
[]
3075 = { MODE_INT
, MODE_VECTOR_INT
, MODE_FLOAT
, MODE_VECTOR_FLOAT
};
3076 for (int i
= 0; i
< 4; i
+= 2)
3077 for (mode
= GET_CLASS_NARROWEST_MODE (classes
[i
]);
3079 mode
= GET_MODE_WIDER_MODE (mode
))
3081 vmode
= targetm
.vectorize
.preferred_simd_mode (mode
);
3082 if (GET_MODE_CLASS (vmode
) != classes
[i
+ 1])
3085 && GET_MODE_SIZE (vmode
) < vs
3086 && GET_MODE_2XWIDER_MODE (vmode
) != VOIDmode
)
3087 vmode
= GET_MODE_2XWIDER_MODE (vmode
);
3089 tree type
= lang_hooks
.types
.type_for_mode (mode
, 1);
3090 if (type
== NULL_TREE
|| TYPE_MODE (type
) != mode
)
3092 type
= build_vector_type (type
, GET_MODE_SIZE (vmode
)
3093 / GET_MODE_SIZE (mode
));
3094 if (TYPE_MODE (type
) != vmode
)
3096 if (TYPE_ALIGN_UNIT (type
) > al
)
3097 al
= TYPE_ALIGN_UNIT (type
);
3099 return build_int_cst (integer_type_node
, al
);
3102 /* Return maximum possible vectorization factor for the target. */
3109 || !flag_tree_loop_optimize
3110 || (!flag_tree_loop_vectorize
3111 && (global_options_set
.x_flag_tree_loop_vectorize
3112 || global_options_set
.x_flag_tree_vectorize
)))
3115 int vs
= targetm
.vectorize
.autovectorize_vector_sizes ();
3118 vs
= 1 << floor_log2 (vs
);
3121 enum machine_mode vqimode
= targetm
.vectorize
.preferred_simd_mode (QImode
);
3122 if (GET_MODE_CLASS (vqimode
) == MODE_VECTOR_INT
)
3123 return GET_MODE_NUNITS (vqimode
);
3127 /* Helper function of lower_rec_input_clauses, used for #pragma omp simd
3131 lower_rec_simd_input_clauses (tree new_var
, omp_context
*ctx
, int &max_vf
,
3132 tree
&idx
, tree
&lane
, tree
&ivar
, tree
&lvar
)
3136 max_vf
= omp_max_vf ();
3139 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3140 OMP_CLAUSE_SAFELEN
);
3141 if (c
&& TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) != INTEGER_CST
)
3143 else if (c
&& compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3145 max_vf
= tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c
));
3149 idx
= create_tmp_var (unsigned_type_node
, NULL
);
3150 lane
= create_tmp_var (unsigned_type_node
, NULL
);
3156 tree atype
= build_array_type_nelts (TREE_TYPE (new_var
), max_vf
);
3157 tree avar
= create_tmp_var_raw (atype
, NULL
);
3158 if (TREE_ADDRESSABLE (new_var
))
3159 TREE_ADDRESSABLE (avar
) = 1;
3160 DECL_ATTRIBUTES (avar
)
3161 = tree_cons (get_identifier ("omp simd array"), NULL
,
3162 DECL_ATTRIBUTES (avar
));
3163 gimple_add_tmp_var (avar
);
3164 ivar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, idx
,
3165 NULL_TREE
, NULL_TREE
);
3166 lvar
= build4 (ARRAY_REF
, TREE_TYPE (new_var
), avar
, lane
,
3167 NULL_TREE
, NULL_TREE
);
3168 if (DECL_P (new_var
))
3170 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3171 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3176 /* Helper function of lower_rec_input_clauses. For a reference
3177 in simd reduction, add an underlying variable it will reference. */
3180 handle_simd_reference (location_t loc
, tree new_vard
, gimple_seq
*ilist
)
3182 tree z
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard
)));
3183 if (TREE_CONSTANT (z
))
3185 const char *name
= NULL
;
3186 if (DECL_NAME (new_vard
))
3187 name
= IDENTIFIER_POINTER (DECL_NAME (new_vard
));
3189 z
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard
)), name
);
3190 gimple_add_tmp_var (z
);
3191 TREE_ADDRESSABLE (z
) = 1;
3192 z
= build_fold_addr_expr_loc (loc
, z
);
3193 gimplify_assign (new_vard
, z
, ilist
);
3197 /* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
3198 from the receiver (aka child) side and initializers for REFERENCE_TYPE
3199 private variables. Initialization statements go in ILIST, while calls
3200 to destructors go in DLIST. */
3203 lower_rec_input_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*dlist
,
3204 omp_context
*ctx
, struct omp_for_data
*fd
)
3206 tree c
, dtor
, copyin_seq
, x
, ptr
;
3207 bool copyin_by_ref
= false;
3208 bool lastprivate_firstprivate
= false;
3209 bool reduction_omp_orig_ref
= false;
3211 bool is_simd
= (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3212 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
);
3214 tree lane
= NULL_TREE
, idx
= NULL_TREE
;
3215 tree ivar
= NULL_TREE
, lvar
= NULL_TREE
;
3216 gimple_seq llist
[2] = { NULL
, NULL
};
3220 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops
3221 with data sharing clauses referencing variable sized vars. That
3222 is unnecessarily hard to support and very unlikely to result in
3223 vectorized code anyway. */
3225 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3226 switch (OMP_CLAUSE_CODE (c
))
3228 case OMP_CLAUSE_LINEAR
:
3229 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3232 case OMP_CLAUSE_REDUCTION
:
3233 case OMP_CLAUSE_PRIVATE
:
3234 case OMP_CLAUSE_FIRSTPRIVATE
:
3235 case OMP_CLAUSE_LASTPRIVATE
:
3236 if (is_variable_sized (OMP_CLAUSE_DECL (c
)))
3243 /* Do all the fixed sized types in the first pass, and the variable sized
3244 types in the second pass. This makes sure that the scalar arguments to
3245 the variable sized types are processed before we use them in the
3246 variable sized operations. */
3247 for (pass
= 0; pass
< 2; ++pass
)
3249 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
3251 enum omp_clause_code c_kind
= OMP_CLAUSE_CODE (c
);
3254 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3258 case OMP_CLAUSE_PRIVATE
:
3259 if (OMP_CLAUSE_PRIVATE_DEBUG (c
))
3262 case OMP_CLAUSE_SHARED
:
3263 /* Ignore shared directives in teams construct. */
3264 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3266 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c
), ctx
) == NULL
)
3268 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c
)));
3271 case OMP_CLAUSE_FIRSTPRIVATE
:
3272 case OMP_CLAUSE_COPYIN
:
3273 case OMP_CLAUSE_LINEAR
:
3275 case OMP_CLAUSE_REDUCTION
:
3276 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c
))
3277 reduction_omp_orig_ref
= true;
3279 case OMP_CLAUSE__LOOPTEMP_
:
3280 /* Handle _looptemp_ clauses only on parallel. */
3284 case OMP_CLAUSE_LASTPRIVATE
:
3285 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3287 lastprivate_firstprivate
= true;
3291 /* Even without corresponding firstprivate, if
3292 decl is Fortran allocatable, it needs outer var
3295 && lang_hooks
.decls
.omp_private_outer_ref
3296 (OMP_CLAUSE_DECL (c
)))
3297 lastprivate_firstprivate
= true;
3299 case OMP_CLAUSE_ALIGNED
:
3302 var
= OMP_CLAUSE_DECL (c
);
3303 if (TREE_CODE (TREE_TYPE (var
)) == POINTER_TYPE
3304 && !is_global_var (var
))
3306 new_var
= maybe_lookup_decl (var
, ctx
);
3307 if (new_var
== NULL_TREE
)
3308 new_var
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3309 x
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3310 x
= build_call_expr_loc (clause_loc
, x
, 2, new_var
,
3311 omp_clause_aligned_alignment (c
));
3312 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3313 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3314 gimplify_and_add (x
, ilist
);
3316 else if (TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
3317 && is_global_var (var
))
3319 tree ptype
= build_pointer_type (TREE_TYPE (var
)), t
, t2
;
3320 new_var
= lookup_decl (var
, ctx
);
3321 t
= maybe_lookup_decl_in_outer_ctx (var
, ctx
);
3322 t
= build_fold_addr_expr_loc (clause_loc
, t
);
3323 t2
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
3324 t
= build_call_expr_loc (clause_loc
, t2
, 2, t
,
3325 omp_clause_aligned_alignment (c
));
3326 t
= fold_convert_loc (clause_loc
, ptype
, t
);
3327 x
= create_tmp_var (ptype
, NULL
);
3328 t
= build2 (MODIFY_EXPR
, ptype
, x
, t
);
3329 gimplify_and_add (t
, ilist
);
3330 t
= build_simple_mem_ref_loc (clause_loc
, x
);
3331 SET_DECL_VALUE_EXPR (new_var
, t
);
3332 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3339 new_var
= var
= OMP_CLAUSE_DECL (c
);
3340 if (c_kind
!= OMP_CLAUSE_COPYIN
)
3341 new_var
= lookup_decl (var
, ctx
);
3343 if (c_kind
== OMP_CLAUSE_SHARED
|| c_kind
== OMP_CLAUSE_COPYIN
)
3348 else if (is_variable_sized (var
))
3350 /* For variable sized types, we need to allocate the
3351 actual storage here. Call alloca and store the
3352 result in the pointer decl that we created elsewhere. */
3356 if (c_kind
!= OMP_CLAUSE_FIRSTPRIVATE
|| !is_task_ctx (ctx
))
3361 ptr
= DECL_VALUE_EXPR (new_var
);
3362 gcc_assert (TREE_CODE (ptr
) == INDIRECT_REF
);
3363 ptr
= TREE_OPERAND (ptr
, 0);
3364 gcc_assert (DECL_P (ptr
));
3365 x
= TYPE_SIZE_UNIT (TREE_TYPE (new_var
));
3367 /* void *tmp = __builtin_alloca */
3368 atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3369 stmt
= gimple_build_call (atmp
, 1, x
);
3370 tmp
= create_tmp_var_raw (ptr_type_node
, NULL
);
3371 gimple_add_tmp_var (tmp
);
3372 gimple_call_set_lhs (stmt
, tmp
);
3374 gimple_seq_add_stmt (ilist
, stmt
);
3376 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ptr
), tmp
);
3377 gimplify_assign (ptr
, x
, ilist
);
3380 else if (is_reference (var
))
3382 /* For references that are being privatized for Fortran,
3383 allocate new backing storage for the new pointer
3384 variable. This allows us to avoid changing all the
3385 code that expects a pointer to something that expects
3386 a direct variable. */
3390 x
= TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var
)));
3391 if (c_kind
== OMP_CLAUSE_FIRSTPRIVATE
&& is_task_ctx (ctx
))
3393 x
= build_receiver_ref (var
, false, ctx
);
3394 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3396 else if (TREE_CONSTANT (x
))
3398 /* For reduction in SIMD loop, defer adding the
3399 initialization of the reference, because if we decide
3400 to use SIMD array for it, the initilization could cause
3402 if (c_kind
== OMP_CLAUSE_REDUCTION
&& is_simd
)
3406 const char *name
= NULL
;
3407 if (DECL_NAME (var
))
3408 name
= IDENTIFIER_POINTER (DECL_NAME (new_var
));
3410 x
= create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var
)),
3412 gimple_add_tmp_var (x
);
3413 TREE_ADDRESSABLE (x
) = 1;
3414 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3419 tree atmp
= builtin_decl_explicit (BUILT_IN_ALLOCA
);
3420 x
= build_call_expr_loc (clause_loc
, atmp
, 1, x
);
3425 x
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), x
);
3426 gimplify_assign (new_var
, x
, ilist
);
3429 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
3431 else if (c_kind
== OMP_CLAUSE_REDUCTION
3432 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3440 switch (OMP_CLAUSE_CODE (c
))
3442 case OMP_CLAUSE_SHARED
:
3443 /* Ignore shared directives in teams construct. */
3444 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_TEAMS
)
3446 /* Shared global vars are just accessed directly. */
3447 if (is_global_var (new_var
))
3449 /* Set up the DECL_VALUE_EXPR for shared variables now. This
3450 needs to be delayed until after fixup_child_record_type so
3451 that we get the correct type during the dereference. */
3452 by_ref
= use_pointer_for_field (var
, ctx
);
3453 x
= build_receiver_ref (var
, by_ref
, ctx
);
3454 SET_DECL_VALUE_EXPR (new_var
, x
);
3455 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3457 /* ??? If VAR is not passed by reference, and the variable
3458 hasn't been initialized yet, then we'll get a warning for
3459 the store into the omp_data_s structure. Ideally, we'd be
3460 able to notice this and not store anything at all, but
3461 we're generating code too early. Suppress the warning. */
3463 TREE_NO_WARNING (var
) = 1;
3466 case OMP_CLAUSE_LASTPRIVATE
:
3467 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
3471 case OMP_CLAUSE_PRIVATE
:
3472 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_PRIVATE
)
3473 x
= build_outer_var_ref (var
, ctx
);
3474 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
3476 if (is_task_ctx (ctx
))
3477 x
= build_receiver_ref (var
, false, ctx
);
3479 x
= build_outer_var_ref (var
, ctx
);
3485 nx
= lang_hooks
.decls
.omp_clause_default_ctor (c
, new_var
, x
);
3488 tree y
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3489 if ((TREE_ADDRESSABLE (new_var
) || nx
|| y
3490 || OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
)
3491 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3492 idx
, lane
, ivar
, lvar
))
3495 x
= lang_hooks
.decls
.omp_clause_default_ctor
3496 (c
, unshare_expr (ivar
), x
);
3498 gimplify_and_add (x
, &llist
[0]);
3501 y
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3504 gimple_seq tseq
= NULL
;
3507 gimplify_stmt (&dtor
, &tseq
);
3508 gimple_seq_add_seq (&llist
[1], tseq
);
3515 gimplify_and_add (nx
, ilist
);
3519 x
= lang_hooks
.decls
.omp_clause_dtor (c
, new_var
);
3522 gimple_seq tseq
= NULL
;
3525 gimplify_stmt (&dtor
, &tseq
);
3526 gimple_seq_add_seq (dlist
, tseq
);
3530 case OMP_CLAUSE_LINEAR
:
3531 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c
))
3532 goto do_firstprivate
;
3533 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c
))
3536 x
= build_outer_var_ref (var
, ctx
);
3539 case OMP_CLAUSE_FIRSTPRIVATE
:
3540 if (is_task_ctx (ctx
))
3542 if (is_reference (var
) || is_variable_sized (var
))
3544 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var
,
3546 || use_pointer_for_field (var
, NULL
))
3548 x
= build_receiver_ref (var
, false, ctx
);
3549 SET_DECL_VALUE_EXPR (new_var
, x
);
3550 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
3555 x
= build_outer_var_ref (var
, ctx
);
3558 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3559 && gimple_omp_for_combined_into_p (ctx
->stmt
))
3561 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3562 tree stept
= TREE_TYPE (t
);
3563 tree ct
= find_omp_clause (clauses
,
3564 OMP_CLAUSE__LOOPTEMP_
);
3566 tree l
= OMP_CLAUSE_DECL (ct
);
3567 tree n1
= fd
->loop
.n1
;
3568 tree step
= fd
->loop
.step
;
3569 tree itype
= TREE_TYPE (l
);
3570 if (POINTER_TYPE_P (itype
))
3571 itype
= signed_type_for (itype
);
3572 l
= fold_build2 (MINUS_EXPR
, itype
, l
, n1
);
3573 if (TYPE_UNSIGNED (itype
)
3574 && fd
->loop
.cond_code
== GT_EXPR
)
3575 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
3576 fold_build1 (NEGATE_EXPR
, itype
, l
),
3577 fold_build1 (NEGATE_EXPR
,
3580 l
= fold_build2 (TRUNC_DIV_EXPR
, itype
, l
, step
);
3581 t
= fold_build2 (MULT_EXPR
, stept
,
3582 fold_convert (stept
, l
), t
);
3584 if (OMP_CLAUSE_LINEAR_ARRAY (c
))
3586 x
= lang_hooks
.decls
.omp_clause_linear_ctor
3588 gimplify_and_add (x
, ilist
);
3592 if (POINTER_TYPE_P (TREE_TYPE (x
)))
3593 x
= fold_build2 (POINTER_PLUS_EXPR
,
3594 TREE_TYPE (x
), x
, t
);
3596 x
= fold_build2 (PLUS_EXPR
, TREE_TYPE (x
), x
, t
);
3599 if ((OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_LINEAR
3600 || TREE_ADDRESSABLE (new_var
))
3601 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3602 idx
, lane
, ivar
, lvar
))
3604 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
)
3606 tree iv
= create_tmp_var (TREE_TYPE (new_var
), NULL
);
3607 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, iv
, x
);
3608 gimplify_and_add (x
, ilist
);
3609 gimple_stmt_iterator gsi
3610 = gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3612 = gimple_build_assign (unshare_expr (lvar
), iv
);
3613 gsi_insert_before_without_update (&gsi
, g
,
3615 tree t
= OMP_CLAUSE_LINEAR_STEP (c
);
3616 enum tree_code code
= PLUS_EXPR
;
3617 if (POINTER_TYPE_P (TREE_TYPE (new_var
)))
3618 code
= POINTER_PLUS_EXPR
;
3619 g
= gimple_build_assign_with_ops (code
, iv
, iv
, t
);
3620 gsi_insert_before_without_update (&gsi
, g
,
3624 x
= lang_hooks
.decls
.omp_clause_copy_ctor
3625 (c
, unshare_expr (ivar
), x
);
3626 gimplify_and_add (x
, &llist
[0]);
3627 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3630 gimple_seq tseq
= NULL
;
3633 gimplify_stmt (&dtor
, &tseq
);
3634 gimple_seq_add_seq (&llist
[1], tseq
);
3639 x
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, new_var
, x
);
3640 gimplify_and_add (x
, ilist
);
3643 case OMP_CLAUSE__LOOPTEMP_
:
3644 gcc_assert (is_parallel_ctx (ctx
));
3645 x
= build_outer_var_ref (var
, ctx
);
3646 x
= build2 (MODIFY_EXPR
, TREE_TYPE (new_var
), new_var
, x
);
3647 gimplify_and_add (x
, ilist
);
3650 case OMP_CLAUSE_COPYIN
:
3651 by_ref
= use_pointer_for_field (var
, NULL
);
3652 x
= build_receiver_ref (var
, by_ref
, ctx
);
3653 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, x
);
3654 append_to_statement_list (x
, ©in_seq
);
3655 copyin_by_ref
|= by_ref
;
3658 case OMP_CLAUSE_REDUCTION
:
3659 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
3661 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
3663 x
= build_outer_var_ref (var
, ctx
);
3665 if (is_reference (var
)
3666 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
3668 x
= build_fold_addr_expr_loc (clause_loc
, x
);
3669 SET_DECL_VALUE_EXPR (placeholder
, x
);
3670 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
3671 tree new_vard
= new_var
;
3672 if (is_reference (var
))
3674 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3675 new_vard
= TREE_OPERAND (new_var
, 0);
3676 gcc_assert (DECL_P (new_vard
));
3679 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3680 idx
, lane
, ivar
, lvar
))
3682 if (new_vard
== new_var
)
3684 gcc_assert (DECL_VALUE_EXPR (new_var
) == lvar
);
3685 SET_DECL_VALUE_EXPR (new_var
, ivar
);
3689 SET_DECL_VALUE_EXPR (new_vard
,
3690 build_fold_addr_expr (ivar
));
3691 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3693 x
= lang_hooks
.decls
.omp_clause_default_ctor
3694 (c
, unshare_expr (ivar
),
3695 build_outer_var_ref (var
, ctx
));
3697 gimplify_and_add (x
, &llist
[0]);
3698 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3700 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3701 lower_omp (&tseq
, ctx
);
3702 gimple_seq_add_seq (&llist
[0], tseq
);
3704 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3705 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3706 lower_omp (&tseq
, ctx
);
3707 gimple_seq_add_seq (&llist
[1], tseq
);
3708 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3709 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3710 if (new_vard
== new_var
)
3711 SET_DECL_VALUE_EXPR (new_var
, lvar
);
3713 SET_DECL_VALUE_EXPR (new_vard
,
3714 build_fold_addr_expr (lvar
));
3715 x
= lang_hooks
.decls
.omp_clause_dtor (c
, ivar
);
3720 gimplify_stmt (&dtor
, &tseq
);
3721 gimple_seq_add_seq (&llist
[1], tseq
);
3725 /* If this is a reference to constant size reduction var
3726 with placeholder, we haven't emitted the initializer
3727 for it because it is undesirable if SIMD arrays are used.
3728 But if they aren't used, we need to emit the deferred
3729 initialization now. */
3730 else if (is_reference (var
) && is_simd
)
3731 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3732 x
= lang_hooks
.decls
.omp_clause_default_ctor
3733 (c
, unshare_expr (new_var
),
3734 build_outer_var_ref (var
, ctx
));
3736 gimplify_and_add (x
, ilist
);
3737 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
))
3739 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
);
3740 lower_omp (&tseq
, ctx
);
3741 gimple_seq_add_seq (ilist
, tseq
);
3743 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c
) = NULL
;
3746 tseq
= OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
);
3747 lower_omp (&tseq
, ctx
);
3748 gimple_seq_add_seq (dlist
, tseq
);
3749 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
3751 DECL_HAS_VALUE_EXPR_P (placeholder
) = 0;
3756 x
= omp_reduction_init (c
, TREE_TYPE (new_var
));
3757 gcc_assert (TREE_CODE (TREE_TYPE (new_var
)) != ARRAY_TYPE
);
3758 enum tree_code code
= OMP_CLAUSE_REDUCTION_CODE (c
);
3760 /* reduction(-:var) sums up the partial results, so it
3761 acts identically to reduction(+:var). */
3762 if (code
== MINUS_EXPR
)
3765 tree new_vard
= new_var
;
3766 if (is_simd
&& is_reference (var
))
3768 gcc_assert (TREE_CODE (new_var
) == MEM_REF
);
3769 new_vard
= TREE_OPERAND (new_var
, 0);
3770 gcc_assert (DECL_P (new_vard
));
3773 && lower_rec_simd_input_clauses (new_var
, ctx
, max_vf
,
3774 idx
, lane
, ivar
, lvar
))
3776 tree ref
= build_outer_var_ref (var
, ctx
);
3778 gimplify_assign (unshare_expr (ivar
), x
, &llist
[0]);
3780 x
= build2 (code
, TREE_TYPE (ref
), ref
, ivar
);
3781 ref
= build_outer_var_ref (var
, ctx
);
3782 gimplify_assign (ref
, x
, &llist
[1]);
3784 if (new_vard
!= new_var
)
3786 SET_DECL_VALUE_EXPR (new_vard
,
3787 build_fold_addr_expr (lvar
));
3788 DECL_HAS_VALUE_EXPR_P (new_vard
) = 1;
3793 if (is_reference (var
) && is_simd
)
3794 handle_simd_reference (clause_loc
, new_vard
, ilist
);
3795 gimplify_assign (new_var
, x
, ilist
);
3798 tree ref
= build_outer_var_ref (var
, ctx
);
3800 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
3801 ref
= build_outer_var_ref (var
, ctx
);
3802 gimplify_assign (ref
, x
, dlist
);
3816 tree uid
= create_tmp_var (ptr_type_node
, "simduid");
3817 /* Don't want uninit warnings on simduid, it is always uninitialized,
3818 but we use it not for the value, but for the DECL_UID only. */
3819 TREE_NO_WARNING (uid
) = 1;
3821 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE
, 1, uid
);
3822 gimple_call_set_lhs (g
, lane
);
3823 gimple_stmt_iterator gsi
= gsi_start_1 (gimple_omp_body_ptr (ctx
->stmt
));
3824 gsi_insert_before_without_update (&gsi
, g
, GSI_SAME_STMT
);
3825 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__SIMDUID_
);
3826 OMP_CLAUSE__SIMDUID__DECL (c
) = uid
;
3827 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3828 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3829 g
= gimple_build_assign_with_ops (INTEGER_CST
, lane
,
3830 build_int_cst (unsigned_type_node
, 0),
3832 gimple_seq_add_stmt (ilist
, g
);
3833 for (int i
= 0; i
< 2; i
++)
3836 tree vf
= create_tmp_var (unsigned_type_node
, NULL
);
3837 g
= gimple_build_call_internal (IFN_GOMP_SIMD_VF
, 1, uid
);
3838 gimple_call_set_lhs (g
, vf
);
3839 gimple_seq
*seq
= i
== 0 ? ilist
: dlist
;
3840 gimple_seq_add_stmt (seq
, g
);
3841 tree t
= build_int_cst (unsigned_type_node
, 0);
3842 g
= gimple_build_assign_with_ops (INTEGER_CST
, idx
, t
, NULL_TREE
);
3843 gimple_seq_add_stmt (seq
, g
);
3844 tree body
= create_artificial_label (UNKNOWN_LOCATION
);
3845 tree header
= create_artificial_label (UNKNOWN_LOCATION
);
3846 tree end
= create_artificial_label (UNKNOWN_LOCATION
);
3847 gimple_seq_add_stmt (seq
, gimple_build_goto (header
));
3848 gimple_seq_add_stmt (seq
, gimple_build_label (body
));
3849 gimple_seq_add_seq (seq
, llist
[i
]);
3850 t
= build_int_cst (unsigned_type_node
, 1);
3851 g
= gimple_build_assign_with_ops (PLUS_EXPR
, idx
, idx
, t
);
3852 gimple_seq_add_stmt (seq
, g
);
3853 gimple_seq_add_stmt (seq
, gimple_build_label (header
));
3854 g
= gimple_build_cond (LT_EXPR
, idx
, vf
, body
, end
);
3855 gimple_seq_add_stmt (seq
, g
);
3856 gimple_seq_add_stmt (seq
, gimple_build_label (end
));
3860 /* The copyin sequence is not to be executed by the main thread, since
3861 that would result in self-copies. Perhaps not visible to scalars,
3862 but it certainly is to C++ operator=. */
3865 x
= build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
),
3867 x
= build2 (NE_EXPR
, boolean_type_node
, x
,
3868 build_int_cst (TREE_TYPE (x
), 0));
3869 x
= build3 (COND_EXPR
, void_type_node
, x
, copyin_seq
, NULL
);
3870 gimplify_and_add (x
, ilist
);
3873 /* If any copyin variable is passed by reference, we must ensure the
3874 master thread doesn't modify it before it is copied over in all
3875 threads. Similarly for variables in both firstprivate and
3876 lastprivate clauses we need to ensure the lastprivate copying
3877 happens after firstprivate copying in all threads. And similarly
3878 for UDRs if initializer expression refers to omp_orig. */
3879 if (copyin_by_ref
|| lastprivate_firstprivate
|| reduction_omp_orig_ref
)
3881 /* Don't add any barrier for #pragma omp simd or
3882 #pragma omp distribute. */
3883 if (gimple_code (ctx
->stmt
) != GIMPLE_OMP_FOR
3884 || gimple_omp_for_kind (ctx
->stmt
) == GF_OMP_FOR_KIND_FOR
)
3885 gimple_seq_add_stmt (ilist
, build_omp_barrier (NULL_TREE
));
3888 /* If max_vf is non-zero, then we can use only a vectorization factor
3889 up to the max_vf we chose. So stick it into the safelen clause. */
3892 tree c
= find_omp_clause (gimple_omp_for_clauses (ctx
->stmt
),
3893 OMP_CLAUSE_SAFELEN
);
3895 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c
)) == INTEGER_CST
3896 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c
),
3899 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_SAFELEN
);
3900 OMP_CLAUSE_SAFELEN_EXPR (c
) = build_int_cst (integer_type_node
,
3902 OMP_CLAUSE_CHAIN (c
) = gimple_omp_for_clauses (ctx
->stmt
);
3903 gimple_omp_for_set_clauses (ctx
->stmt
, c
);
3909 /* Generate code to implement the LASTPRIVATE clauses. This is used for
3910 both parallel and workshare constructs. PREDICATE may be NULL if it's
3914 lower_lastprivate_clauses (tree clauses
, tree predicate
, gimple_seq
*stmt_list
,
3917 tree x
, c
, label
= NULL
, orig_clauses
= clauses
;
3918 bool par_clauses
= false;
3919 tree simduid
= NULL
, lastlane
= NULL
;
3921 /* Early exit if there are no lastprivate or linear clauses. */
3922 for (; clauses
; clauses
= OMP_CLAUSE_CHAIN (clauses
))
3923 if (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LASTPRIVATE
3924 || (OMP_CLAUSE_CODE (clauses
) == OMP_CLAUSE_LINEAR
3925 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses
)))
3927 if (clauses
== NULL
)
3929 /* If this was a workshare clause, see if it had been combined
3930 with its parallel. In that case, look for the clauses on the
3931 parallel statement itself. */
3932 if (is_parallel_ctx (ctx
))
3936 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
3939 clauses
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
3940 OMP_CLAUSE_LASTPRIVATE
);
3941 if (clauses
== NULL
)
3949 tree label_true
, arm1
, arm2
;
3951 label
= create_artificial_label (UNKNOWN_LOCATION
);
3952 label_true
= create_artificial_label (UNKNOWN_LOCATION
);
3953 arm1
= TREE_OPERAND (predicate
, 0);
3954 arm2
= TREE_OPERAND (predicate
, 1);
3955 gimplify_expr (&arm1
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3956 gimplify_expr (&arm2
, stmt_list
, NULL
, is_gimple_val
, fb_rvalue
);
3957 stmt
= gimple_build_cond (TREE_CODE (predicate
), arm1
, arm2
,
3959 gimple_seq_add_stmt (stmt_list
, stmt
);
3960 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label_true
));
3963 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
3964 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
3966 simduid
= find_omp_clause (orig_clauses
, OMP_CLAUSE__SIMDUID_
);
3968 simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
3971 for (c
= clauses
; c
;)
3974 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
3976 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
3977 || (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
3978 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c
)))
3980 var
= OMP_CLAUSE_DECL (c
);
3981 new_var
= lookup_decl (var
, ctx
);
3983 if (simduid
&& DECL_HAS_VALUE_EXPR_P (new_var
))
3985 tree val
= DECL_VALUE_EXPR (new_var
);
3986 if (TREE_CODE (val
) == ARRAY_REF
3987 && VAR_P (TREE_OPERAND (val
, 0))
3988 && lookup_attribute ("omp simd array",
3989 DECL_ATTRIBUTES (TREE_OPERAND (val
,
3992 if (lastlane
== NULL
)
3994 lastlane
= create_tmp_var (unsigned_type_node
, NULL
);
3996 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE
,
3998 TREE_OPERAND (val
, 1));
3999 gimple_call_set_lhs (g
, lastlane
);
4000 gimple_seq_add_stmt (stmt_list
, g
);
4002 new_var
= build4 (ARRAY_REF
, TREE_TYPE (val
),
4003 TREE_OPERAND (val
, 0), lastlane
,
4004 NULL_TREE
, NULL_TREE
);
4008 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LASTPRIVATE
4009 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
))
4011 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
), ctx
);
4012 gimple_seq_add_seq (stmt_list
,
4013 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
));
4014 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c
) = NULL
;
4016 else if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_LINEAR
4017 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
))
4019 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
), ctx
);
4020 gimple_seq_add_seq (stmt_list
,
4021 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
));
4022 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c
) = NULL
;
4025 x
= build_outer_var_ref (var
, ctx
);
4026 if (is_reference (var
))
4027 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4028 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, x
, new_var
);
4029 gimplify_and_add (x
, stmt_list
);
4031 c
= OMP_CLAUSE_CHAIN (c
);
4032 if (c
== NULL
&& !par_clauses
)
4034 /* If this was a workshare clause, see if it had been combined
4035 with its parallel. In that case, continue looking for the
4036 clauses also on the parallel statement itself. */
4037 if (is_parallel_ctx (ctx
))
4041 if (ctx
== NULL
|| !is_parallel_ctx (ctx
))
4044 c
= find_omp_clause (gimple_omp_parallel_clauses (ctx
->stmt
),
4045 OMP_CLAUSE_LASTPRIVATE
);
4051 gimple_seq_add_stmt (stmt_list
, gimple_build_label (label
));
4055 /* Generate code to implement the REDUCTION clauses. */
4058 lower_reduction_clauses (tree clauses
, gimple_seq
*stmt_seqp
, omp_context
*ctx
)
4060 gimple_seq sub_seq
= NULL
;
4065 /* SIMD reductions are handled in lower_rec_input_clauses. */
4066 if (gimple_code (ctx
->stmt
) == GIMPLE_OMP_FOR
4067 && gimple_omp_for_kind (ctx
->stmt
) & GF_OMP_FOR_SIMD
)
4070 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC
4071 update in that case, otherwise use a lock. */
4072 for (c
= clauses
; c
&& count
< 2; c
= OMP_CLAUSE_CHAIN (c
))
4073 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_REDUCTION
)
4075 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4077 /* Never use OMP_ATOMIC for array reductions or UDRs. */
4087 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4089 tree var
, ref
, new_var
;
4090 enum tree_code code
;
4091 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4093 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_REDUCTION
)
4096 var
= OMP_CLAUSE_DECL (c
);
4097 new_var
= lookup_decl (var
, ctx
);
4098 if (is_reference (var
))
4099 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4100 ref
= build_outer_var_ref (var
, ctx
);
4101 code
= OMP_CLAUSE_REDUCTION_CODE (c
);
4103 /* reduction(-:var) sums up the partial results, so it acts
4104 identically to reduction(+:var). */
4105 if (code
== MINUS_EXPR
)
4110 tree addr
= build_fold_addr_expr_loc (clause_loc
, ref
);
4112 addr
= save_expr (addr
);
4113 ref
= build1 (INDIRECT_REF
, TREE_TYPE (TREE_TYPE (addr
)), addr
);
4114 x
= fold_build2_loc (clause_loc
, code
, TREE_TYPE (ref
), ref
, new_var
);
4115 x
= build2 (OMP_ATOMIC
, void_type_node
, addr
, x
);
4116 gimplify_and_add (x
, stmt_seqp
);
4120 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
))
4122 tree placeholder
= OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
);
4124 if (is_reference (var
)
4125 && !useless_type_conversion_p (TREE_TYPE (placeholder
),
4127 ref
= build_fold_addr_expr_loc (clause_loc
, ref
);
4128 SET_DECL_VALUE_EXPR (placeholder
, ref
);
4129 DECL_HAS_VALUE_EXPR_P (placeholder
) = 1;
4130 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
), ctx
);
4131 gimple_seq_add_seq (&sub_seq
, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
));
4132 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c
) = NULL
;
4133 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c
) = NULL
;
4137 x
= build2 (code
, TREE_TYPE (ref
), ref
, new_var
);
4138 ref
= build_outer_var_ref (var
, ctx
);
4139 gimplify_assign (ref
, x
, &sub_seq
);
4143 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
),
4145 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4147 gimple_seq_add_seq (stmt_seqp
, sub_seq
);
4149 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
),
4151 gimple_seq_add_stmt (stmt_seqp
, stmt
);
4155 /* Generate code to implement the COPYPRIVATE clauses. */
4158 lower_copyprivate_clauses (tree clauses
, gimple_seq
*slist
, gimple_seq
*rlist
,
4163 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4165 tree var
, new_var
, ref
, x
;
4167 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4169 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYPRIVATE
)
4172 var
= OMP_CLAUSE_DECL (c
);
4173 by_ref
= use_pointer_for_field (var
, NULL
);
4175 ref
= build_sender_ref (var
, ctx
);
4176 x
= new_var
= lookup_decl_in_outer_ctx (var
, ctx
);
4179 x
= build_fold_addr_expr_loc (clause_loc
, new_var
);
4180 x
= fold_convert_loc (clause_loc
, TREE_TYPE (ref
), x
);
4182 gimplify_assign (ref
, x
, slist
);
4184 ref
= build_receiver_ref (var
, false, ctx
);
4187 ref
= fold_convert_loc (clause_loc
,
4188 build_pointer_type (TREE_TYPE (new_var
)),
4190 ref
= build_fold_indirect_ref_loc (clause_loc
, ref
);
4192 if (is_reference (var
))
4194 ref
= fold_convert_loc (clause_loc
, TREE_TYPE (new_var
), ref
);
4195 ref
= build_simple_mem_ref_loc (clause_loc
, ref
);
4196 new_var
= build_simple_mem_ref_loc (clause_loc
, new_var
);
4198 x
= lang_hooks
.decls
.omp_clause_assign_op (c
, new_var
, ref
);
4199 gimplify_and_add (x
, rlist
);
4204 /* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE,
4205 and REDUCTION from the sender (aka parent) side. */
4208 lower_send_clauses (tree clauses
, gimple_seq
*ilist
, gimple_seq
*olist
,
4213 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
4215 tree val
, ref
, x
, var
;
4216 bool by_ref
, do_in
= false, do_out
= false;
4217 location_t clause_loc
= OMP_CLAUSE_LOCATION (c
);
4219 switch (OMP_CLAUSE_CODE (c
))
4221 case OMP_CLAUSE_PRIVATE
:
4222 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
4225 case OMP_CLAUSE_FIRSTPRIVATE
:
4226 case OMP_CLAUSE_COPYIN
:
4227 case OMP_CLAUSE_LASTPRIVATE
:
4228 case OMP_CLAUSE_REDUCTION
:
4229 case OMP_CLAUSE__LOOPTEMP_
:
4235 val
= OMP_CLAUSE_DECL (c
);
4236 var
= lookup_decl_in_outer_ctx (val
, ctx
);
4238 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_COPYIN
4239 && is_global_var (var
))
4241 if (is_variable_sized (val
))
4243 by_ref
= use_pointer_for_field (val
, NULL
);
4245 switch (OMP_CLAUSE_CODE (c
))
4247 case OMP_CLAUSE_PRIVATE
:
4248 case OMP_CLAUSE_FIRSTPRIVATE
:
4249 case OMP_CLAUSE_COPYIN
:
4250 case OMP_CLAUSE__LOOPTEMP_
:
4254 case OMP_CLAUSE_LASTPRIVATE
:
4255 if (by_ref
|| is_reference (val
))
4257 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c
))
4264 if (lang_hooks
.decls
.omp_private_outer_ref (val
))
4269 case OMP_CLAUSE_REDUCTION
:
4271 do_out
= !(by_ref
|| is_reference (val
));
4280 ref
= build_sender_ref (val
, ctx
);
4281 x
= by_ref
? build_fold_addr_expr_loc (clause_loc
, var
) : var
;
4282 gimplify_assign (ref
, x
, ilist
);
4283 if (is_task_ctx (ctx
))
4284 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref
, 1)) = NULL
;
4289 ref
= build_sender_ref (val
, ctx
);
4290 gimplify_assign (var
, ref
, olist
);
4295 /* Generate code to implement SHARED from the sender (aka parent)
4296 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't
4297 list things that got automatically shared. */
4300 lower_send_shared_vars (gimple_seq
*ilist
, gimple_seq
*olist
, omp_context
*ctx
)
4302 tree var
, ovar
, nvar
, f
, x
, record_type
;
4304 if (ctx
->record_type
== NULL
)
4307 record_type
= ctx
->srecord_type
? ctx
->srecord_type
: ctx
->record_type
;
4308 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
4310 ovar
= DECL_ABSTRACT_ORIGIN (f
);
4311 nvar
= maybe_lookup_decl (ovar
, ctx
);
4312 if (!nvar
|| !DECL_HAS_VALUE_EXPR_P (nvar
))
4315 /* If CTX is a nested parallel directive. Find the immediately
4316 enclosing parallel or workshare construct that contains a
4317 mapping for OVAR. */
4318 var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
4320 if (use_pointer_for_field (ovar
, ctx
))
4322 x
= build_sender_ref (ovar
, ctx
);
4323 var
= build_fold_addr_expr (var
);
4324 gimplify_assign (x
, var
, ilist
);
4328 x
= build_sender_ref (ovar
, ctx
);
4329 gimplify_assign (x
, var
, ilist
);
4331 if (!TREE_READONLY (var
)
4332 /* We don't need to receive a new reference to a result
4333 or parm decl. In fact we may not store to it as we will
4334 invalidate any pending RSO and generate wrong gimple
4336 && !((TREE_CODE (var
) == RESULT_DECL
4337 || TREE_CODE (var
) == PARM_DECL
)
4338 && DECL_BY_REFERENCE (var
)))
4340 x
= build_sender_ref (ovar
, ctx
);
4341 gimplify_assign (var
, x
, olist
);
4348 /* A convenience function to build an empty GIMPLE_COND with just the
4352 gimple_build_cond_empty (tree cond
)
4354 enum tree_code pred_code
;
4357 gimple_cond_get_ops_from_tree (cond
, &pred_code
, &lhs
, &rhs
);
4358 return gimple_build_cond (pred_code
, lhs
, rhs
, NULL_TREE
, NULL_TREE
);
4362 /* Build the function calls to GOMP_parallel_start etc to actually
4363 generate the parallel operation. REGION is the parallel region
4364 being expanded. BB is the block where to insert the code. WS_ARGS
4365 will be set if this is a call to a combined parallel+workshare
4366 construct, it contains the list of additional arguments needed by
4367 the workshare construct. */
4370 expand_parallel_call (struct omp_region
*region
, basic_block bb
,
4371 gimple entry_stmt
, vec
<tree
, va_gc
> *ws_args
)
4373 tree t
, t1
, t2
, val
, cond
, c
, clauses
, flags
;
4374 gimple_stmt_iterator gsi
;
4376 enum built_in_function start_ix
;
4378 location_t clause_loc
;
4379 vec
<tree
, va_gc
> *args
;
4381 clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4383 /* Determine what flavor of GOMP_parallel we will be
4385 start_ix
= BUILT_IN_GOMP_PARALLEL
;
4386 if (is_combined_parallel (region
))
4388 switch (region
->inner
->type
)
4390 case GIMPLE_OMP_FOR
:
4391 gcc_assert (region
->inner
->sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
4392 start_ix2
= ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC
4393 + (region
->inner
->sched_kind
4394 == OMP_CLAUSE_SCHEDULE_RUNTIME
4395 ? 3 : region
->inner
->sched_kind
));
4396 start_ix
= (enum built_in_function
)start_ix2
;
4398 case GIMPLE_OMP_SECTIONS
:
4399 start_ix
= BUILT_IN_GOMP_PARALLEL_SECTIONS
;
4406 /* By default, the value of NUM_THREADS is zero (selected at run time)
4407 and there is no conditional. */
4409 val
= build_int_cst (unsigned_type_node
, 0);
4410 flags
= build_int_cst (unsigned_type_node
, 0);
4412 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4414 cond
= OMP_CLAUSE_IF_EXPR (c
);
4416 c
= find_omp_clause (clauses
, OMP_CLAUSE_NUM_THREADS
);
4419 val
= OMP_CLAUSE_NUM_THREADS_EXPR (c
);
4420 clause_loc
= OMP_CLAUSE_LOCATION (c
);
4423 clause_loc
= gimple_location (entry_stmt
);
4425 c
= find_omp_clause (clauses
, OMP_CLAUSE_PROC_BIND
);
4427 flags
= build_int_cst (unsigned_type_node
, OMP_CLAUSE_PROC_BIND_KIND (c
));
4429 /* Ensure 'val' is of the correct type. */
4430 val
= fold_convert_loc (clause_loc
, unsigned_type_node
, val
);
4432 /* If we found the clause 'if (cond)', build either
4433 (cond != 0) or (cond ? val : 1u). */
4436 cond
= gimple_boolify (cond
);
4438 if (integer_zerop (val
))
4439 val
= fold_build2_loc (clause_loc
,
4440 EQ_EXPR
, unsigned_type_node
, cond
,
4441 build_int_cst (TREE_TYPE (cond
), 0));
4444 basic_block cond_bb
, then_bb
, else_bb
;
4445 edge e
, e_then
, e_else
;
4446 tree tmp_then
, tmp_else
, tmp_join
, tmp_var
;
4448 tmp_var
= create_tmp_var (TREE_TYPE (val
), NULL
);
4449 if (gimple_in_ssa_p (cfun
))
4451 tmp_then
= make_ssa_name (tmp_var
, NULL
);
4452 tmp_else
= make_ssa_name (tmp_var
, NULL
);
4453 tmp_join
= make_ssa_name (tmp_var
, NULL
);
4462 e
= split_block (bb
, NULL
);
4467 then_bb
= create_empty_bb (cond_bb
);
4468 else_bb
= create_empty_bb (then_bb
);
4469 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
4470 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
4472 stmt
= gimple_build_cond_empty (cond
);
4473 gsi
= gsi_start_bb (cond_bb
);
4474 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4476 gsi
= gsi_start_bb (then_bb
);
4477 stmt
= gimple_build_assign (tmp_then
, val
);
4478 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4480 gsi
= gsi_start_bb (else_bb
);
4481 stmt
= gimple_build_assign
4482 (tmp_else
, build_int_cst (unsigned_type_node
, 1));
4483 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
4485 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
4486 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
4487 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
4488 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
4489 e_then
= make_edge (then_bb
, bb
, EDGE_FALLTHRU
);
4490 e_else
= make_edge (else_bb
, bb
, EDGE_FALLTHRU
);
4492 if (gimple_in_ssa_p (cfun
))
4494 gimple phi
= create_phi_node (tmp_join
, bb
);
4495 add_phi_arg (phi
, tmp_then
, e_then
, UNKNOWN_LOCATION
);
4496 add_phi_arg (phi
, tmp_else
, e_else
, UNKNOWN_LOCATION
);
4502 gsi
= gsi_start_bb (bb
);
4503 val
= force_gimple_operand_gsi (&gsi
, val
, true, NULL_TREE
,
4504 false, GSI_CONTINUE_LINKING
);
4507 gsi
= gsi_last_bb (bb
);
4508 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4510 t1
= null_pointer_node
;
4512 t1
= build_fold_addr_expr (t
);
4513 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4515 vec_alloc (args
, 4 + vec_safe_length (ws_args
));
4516 args
->quick_push (t2
);
4517 args
->quick_push (t1
);
4518 args
->quick_push (val
);
4520 args
->splice (*ws_args
);
4521 args
->quick_push (flags
);
4523 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
,
4524 builtin_decl_explicit (start_ix
), args
);
4526 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4527 false, GSI_CONTINUE_LINKING
);
4530 /* Insert a function call whose name is FUNC_NAME with the information from
4531 ENTRY_STMT into the basic_block BB. */
4534 expand_cilk_for_call (basic_block bb
, gimple entry_stmt
,
4535 vec
<tree
, va_gc
> *ws_args
)
4538 gimple_stmt_iterator gsi
;
4539 vec
<tree
, va_gc
> *args
;
4541 gcc_assert (vec_safe_length (ws_args
) == 2);
4542 tree func_name
= (*ws_args
)[0];
4543 tree grain
= (*ws_args
)[1];
4545 tree clauses
= gimple_omp_parallel_clauses (entry_stmt
);
4546 tree count
= find_omp_clause (clauses
, OMP_CLAUSE__CILK_FOR_COUNT_
);
4547 gcc_assert (count
!= NULL_TREE
);
4548 count
= OMP_CLAUSE_OPERAND (count
, 0);
4550 gsi
= gsi_last_bb (bb
);
4551 t
= gimple_omp_parallel_data_arg (entry_stmt
);
4553 t1
= null_pointer_node
;
4555 t1
= build_fold_addr_expr (t
);
4556 t2
= build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt
));
4558 vec_alloc (args
, 4);
4559 args
->quick_push (t2
);
4560 args
->quick_push (t1
);
4561 args
->quick_push (count
);
4562 args
->quick_push (grain
);
4563 t
= build_call_expr_loc_vec (UNKNOWN_LOCATION
, func_name
, args
);
4565 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, false,
4566 GSI_CONTINUE_LINKING
);
4569 /* Build the function call to GOMP_task to actually
4570 generate the task operation. BB is the block where to insert the code. */
4573 expand_task_call (basic_block bb
, gimple entry_stmt
)
4575 tree t
, t1
, t2
, t3
, flags
, cond
, c
, c2
, clauses
, depend
;
4576 gimple_stmt_iterator gsi
;
4577 location_t loc
= gimple_location (entry_stmt
);
4579 clauses
= gimple_omp_task_clauses (entry_stmt
);
4581 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
4583 cond
= gimple_boolify (OMP_CLAUSE_IF_EXPR (c
));
4585 cond
= boolean_true_node
;
4587 c
= find_omp_clause (clauses
, OMP_CLAUSE_UNTIED
);
4588 c2
= find_omp_clause (clauses
, OMP_CLAUSE_MERGEABLE
);
4589 depend
= find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
);
4590 flags
= build_int_cst (unsigned_type_node
,
4591 (c
? 1 : 0) + (c2
? 4 : 0) + (depend
? 8 : 0));
4593 c
= find_omp_clause (clauses
, OMP_CLAUSE_FINAL
);
4596 c
= gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c
));
4597 c
= fold_build3_loc (loc
, COND_EXPR
, unsigned_type_node
, c
,
4598 build_int_cst (unsigned_type_node
, 2),
4599 build_int_cst (unsigned_type_node
, 0));
4600 flags
= fold_build2_loc (loc
, PLUS_EXPR
, unsigned_type_node
, flags
, c
);
4603 depend
= OMP_CLAUSE_DECL (depend
);
4605 depend
= build_int_cst (ptr_type_node
, 0);
4607 gsi
= gsi_last_bb (bb
);
4608 t
= gimple_omp_task_data_arg (entry_stmt
);
4610 t2
= null_pointer_node
;
4612 t2
= build_fold_addr_expr_loc (loc
, t
);
4613 t1
= build_fold_addr_expr_loc (loc
, gimple_omp_task_child_fn (entry_stmt
));
4614 t
= gimple_omp_task_copy_fn (entry_stmt
);
4616 t3
= null_pointer_node
;
4618 t3
= build_fold_addr_expr_loc (loc
, t
);
4620 t
= build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK
),
4622 gimple_omp_task_arg_size (entry_stmt
),
4623 gimple_omp_task_arg_align (entry_stmt
), cond
, flags
,
4626 force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
4627 false, GSI_CONTINUE_LINKING
);
4631 /* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW
4632 catch handler and return it. This prevents programs from violating the
4633 structured block semantics with throws. */
4636 maybe_catch_exception (gimple_seq body
)
4641 if (!flag_exceptions
)
4644 if (lang_hooks
.eh_protect_cleanup_actions
!= NULL
)
4645 decl
= lang_hooks
.eh_protect_cleanup_actions ();
4647 decl
= builtin_decl_explicit (BUILT_IN_TRAP
);
4649 g
= gimple_build_eh_must_not_throw (decl
);
4650 g
= gimple_build_try (body
, gimple_seq_alloc_with_stmt (g
),
4653 return gimple_seq_alloc_with_stmt (g
);
4656 /* Chain all the DECLs in LIST by their TREE_CHAIN fields. */
4659 vec2chain (vec
<tree
, va_gc
> *v
)
4661 tree chain
= NULL_TREE
, t
;
4664 FOR_EACH_VEC_SAFE_ELT_REVERSE (v
, ix
, t
)
4666 DECL_CHAIN (t
) = chain
;
4674 /* Remove barriers in REGION->EXIT's block. Note that this is only
4675 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region
4676 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that
4677 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be
4681 remove_exit_barrier (struct omp_region
*region
)
4683 gimple_stmt_iterator gsi
;
4684 basic_block exit_bb
;
4688 int any_addressable_vars
= -1;
4690 exit_bb
= region
->exit
;
4692 /* If the parallel region doesn't return, we don't have REGION->EXIT
4697 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The
4698 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of
4699 statements that can appear in between are extremely limited -- no
4700 memory operations at all. Here, we allow nothing at all, so the
4701 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */
4702 gsi
= gsi_last_bb (exit_bb
);
4703 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
4705 if (!gsi_end_p (gsi
) && gimple_code (gsi_stmt (gsi
)) != GIMPLE_LABEL
)
4708 FOR_EACH_EDGE (e
, ei
, exit_bb
->preds
)
4710 gsi
= gsi_last_bb (e
->src
);
4711 if (gsi_end_p (gsi
))
4713 stmt
= gsi_stmt (gsi
);
4714 if (gimple_code (stmt
) == GIMPLE_OMP_RETURN
4715 && !gimple_omp_return_nowait_p (stmt
))
4717 /* OpenMP 3.0 tasks unfortunately prevent this optimization
4718 in many cases. If there could be tasks queued, the barrier
4719 might be needed to let the tasks run before some local
4720 variable of the parallel that the task uses as shared
4721 runs out of scope. The task can be spawned either
4722 from within current function (this would be easy to check)
4723 or from some function it calls and gets passed an address
4724 of such a variable. */
4725 if (any_addressable_vars
< 0)
4727 gimple parallel_stmt
= last_stmt (region
->entry
);
4728 tree child_fun
= gimple_omp_parallel_child_fn (parallel_stmt
);
4729 tree local_decls
, block
, decl
;
4732 any_addressable_vars
= 0;
4733 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun
), ix
, decl
)
4734 if (TREE_ADDRESSABLE (decl
))
4736 any_addressable_vars
= 1;
4739 for (block
= gimple_block (stmt
);
4740 !any_addressable_vars
4742 && TREE_CODE (block
) == BLOCK
;
4743 block
= BLOCK_SUPERCONTEXT (block
))
4745 for (local_decls
= BLOCK_VARS (block
);
4747 local_decls
= DECL_CHAIN (local_decls
))
4748 if (TREE_ADDRESSABLE (local_decls
))
4750 any_addressable_vars
= 1;
4753 if (block
== gimple_block (parallel_stmt
))
4757 if (!any_addressable_vars
)
4758 gimple_omp_return_set_nowait (stmt
);
4764 remove_exit_barriers (struct omp_region
*region
)
4766 if (region
->type
== GIMPLE_OMP_PARALLEL
)
4767 remove_exit_barrier (region
);
4771 region
= region
->inner
;
4772 remove_exit_barriers (region
);
4773 while (region
->next
)
4775 region
= region
->next
;
4776 remove_exit_barriers (region
);
4781 /* Optimize omp_get_thread_num () and omp_get_num_threads ()
4782 calls. These can't be declared as const functions, but
4783 within one parallel body they are constant, so they can be
4784 transformed there into __builtin_omp_get_{thread_num,num_threads} ()
4785 which are declared const. Similarly for task body, except
4786 that in untied task omp_get_thread_num () can change at any task
4787 scheduling point. */
4790 optimize_omp_library_calls (gimple entry_stmt
)
4793 gimple_stmt_iterator gsi
;
4794 tree thr_num_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4795 tree thr_num_id
= DECL_ASSEMBLER_NAME (thr_num_tree
);
4796 tree num_thr_tree
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4797 tree num_thr_id
= DECL_ASSEMBLER_NAME (num_thr_tree
);
4798 bool untied_task
= (gimple_code (entry_stmt
) == GIMPLE_OMP_TASK
4799 && find_omp_clause (gimple_omp_task_clauses (entry_stmt
),
4800 OMP_CLAUSE_UNTIED
) != NULL
);
4802 FOR_EACH_BB_FN (bb
, cfun
)
4803 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4805 gimple call
= gsi_stmt (gsi
);
4808 if (is_gimple_call (call
)
4809 && (decl
= gimple_call_fndecl (call
))
4810 && DECL_EXTERNAL (decl
)
4811 && TREE_PUBLIC (decl
)
4812 && DECL_INITIAL (decl
) == NULL
)
4816 if (DECL_NAME (decl
) == thr_num_id
)
4818 /* In #pragma omp task untied omp_get_thread_num () can change
4819 during the execution of the task region. */
4822 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
4824 else if (DECL_NAME (decl
) == num_thr_id
)
4825 built_in
= builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS
);
4829 if (DECL_ASSEMBLER_NAME (decl
) != DECL_ASSEMBLER_NAME (built_in
)
4830 || gimple_call_num_args (call
) != 0)
4833 if (flag_exceptions
&& !TREE_NOTHROW (decl
))
4836 if (TREE_CODE (TREE_TYPE (decl
)) != FUNCTION_TYPE
4837 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl
)),
4838 TREE_TYPE (TREE_TYPE (built_in
))))
4841 gimple_call_set_fndecl (call
, built_in
);
4846 /* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be
4850 expand_omp_regimplify_p (tree
*tp
, int *walk_subtrees
, void *)
4854 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
4855 if (TREE_CODE (t
) == VAR_DECL
&& DECL_HAS_VALUE_EXPR_P (t
))
4858 if (TREE_CODE (t
) == ADDR_EXPR
)
4859 recompute_tree_invariant_for_addr_expr (t
);
4861 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
4865 /* Prepend TO = FROM assignment before *GSI_P. */
4868 expand_omp_build_assign (gimple_stmt_iterator
*gsi_p
, tree to
, tree from
)
4870 bool simple_p
= DECL_P (to
) && TREE_ADDRESSABLE (to
);
4871 from
= force_gimple_operand_gsi (gsi_p
, from
, simple_p
, NULL_TREE
,
4872 true, GSI_SAME_STMT
);
4873 gimple stmt
= gimple_build_assign (to
, from
);
4874 gsi_insert_before (gsi_p
, stmt
, GSI_SAME_STMT
);
4875 if (walk_tree (&from
, expand_omp_regimplify_p
, NULL
, NULL
)
4876 || walk_tree (&to
, expand_omp_regimplify_p
, NULL
, NULL
))
4878 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
4879 gimple_regimplify_operands (stmt
, &gsi
);
4883 /* Expand the OpenMP parallel or task directive starting at REGION. */
4886 expand_omp_taskreg (struct omp_region
*region
)
4888 basic_block entry_bb
, exit_bb
, new_bb
;
4889 struct function
*child_cfun
;
4890 tree child_fn
, block
, t
;
4891 gimple_stmt_iterator gsi
;
4892 gimple entry_stmt
, stmt
;
4894 vec
<tree
, va_gc
> *ws_args
;
4896 entry_stmt
= last_stmt (region
->entry
);
4897 child_fn
= gimple_omp_taskreg_child_fn (entry_stmt
);
4898 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
4900 entry_bb
= region
->entry
;
4901 exit_bb
= region
->exit
;
4905 && gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
4906 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt
),
4907 OMP_CLAUSE__CILK_FOR_COUNT_
) != NULL_TREE
);
4910 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for,
4911 and the inner statement contains the name of the built-in function
4913 ws_args
= region
->inner
->ws_args
;
4914 else if (is_combined_parallel (region
))
4915 ws_args
= region
->ws_args
;
4919 if (child_cfun
->cfg
)
4921 /* Due to inlining, it may happen that we have already outlined
4922 the region, in which case all we need to do is make the
4923 sub-graph unreachable and emit the parallel call. */
4924 edge entry_succ_e
, exit_succ_e
;
4926 entry_succ_e
= single_succ_edge (entry_bb
);
4928 gsi
= gsi_last_bb (entry_bb
);
4929 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_PARALLEL
4930 || gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_TASK
);
4931 gsi_remove (&gsi
, true);
4936 exit_succ_e
= single_succ_edge (exit_bb
);
4937 make_edge (new_bb
, exit_succ_e
->dest
, EDGE_FALLTHRU
);
4939 remove_edge_and_dominated_blocks (entry_succ_e
);
4943 unsigned srcidx
, dstidx
, num
;
4945 /* If the parallel region needs data sent from the parent
4946 function, then the very first statement (except possible
4947 tree profile counter updates) of the parallel body
4948 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
4949 &.OMP_DATA_O is passed as an argument to the child function,
4950 we need to replace it with the argument as seen by the child
4953 In most cases, this will end up being the identity assignment
4954 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
4955 a function call that has been inlined, the original PARM_DECL
4956 .OMP_DATA_I may have been converted into a different local
4957 variable. In which case, we need to keep the assignment. */
4958 if (gimple_omp_taskreg_data_arg (entry_stmt
))
4960 basic_block entry_succ_bb
= single_succ (entry_bb
);
4962 gimple parcopy_stmt
= NULL
;
4964 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
4968 gcc_assert (!gsi_end_p (gsi
));
4969 stmt
= gsi_stmt (gsi
);
4970 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
4973 if (gimple_num_ops (stmt
) == 2)
4975 tree arg
= gimple_assign_rhs1 (stmt
);
4977 /* We're ignore the subcode because we're
4978 effectively doing a STRIP_NOPS. */
4980 if (TREE_CODE (arg
) == ADDR_EXPR
4981 && TREE_OPERAND (arg
, 0)
4982 == gimple_omp_taskreg_data_arg (entry_stmt
))
4984 parcopy_stmt
= stmt
;
4990 gcc_assert (parcopy_stmt
!= NULL
);
4991 arg
= DECL_ARGUMENTS (child_fn
);
4993 if (!gimple_in_ssa_p (cfun
))
4995 if (gimple_assign_lhs (parcopy_stmt
) == arg
)
4996 gsi_remove (&gsi
, true);
4999 /* ?? Is setting the subcode really necessary ?? */
5000 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (arg
));
5001 gimple_assign_set_rhs1 (parcopy_stmt
, arg
);
5006 /* If we are in ssa form, we must load the value from the default
5007 definition of the argument. That should not be defined now,
5008 since the argument is not used uninitialized. */
5009 gcc_assert (ssa_default_def (cfun
, arg
) == NULL
);
5010 narg
= make_ssa_name (arg
, gimple_build_nop ());
5011 set_ssa_default_def (cfun
, arg
, narg
);
5012 /* ?? Is setting the subcode really necessary ?? */
5013 gimple_omp_set_subcode (parcopy_stmt
, TREE_CODE (narg
));
5014 gimple_assign_set_rhs1 (parcopy_stmt
, narg
);
5015 update_stmt (parcopy_stmt
);
5019 /* Declare local variables needed in CHILD_CFUN. */
5020 block
= DECL_INITIAL (child_fn
);
5021 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
5022 /* The gimplifier could record temporaries in parallel/task block
5023 rather than in containing function's local_decls chain,
5024 which would mean cgraph missed finalizing them. Do it now. */
5025 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
5026 if (TREE_CODE (t
) == VAR_DECL
5028 && !DECL_EXTERNAL (t
))
5029 varpool_node::finalize_decl (t
);
5030 DECL_SAVED_TREE (child_fn
) = NULL
;
5031 /* We'll create a CFG for child_fn, so no gimple body is needed. */
5032 gimple_set_body (child_fn
, NULL
);
5033 TREE_USED (block
) = 1;
5035 /* Reset DECL_CONTEXT on function arguments. */
5036 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
5037 DECL_CONTEXT (t
) = child_fn
;
5039 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK,
5040 so that it can be moved to the child function. */
5041 gsi
= gsi_last_bb (entry_bb
);
5042 stmt
= gsi_stmt (gsi
);
5043 gcc_assert (stmt
&& (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
5044 || gimple_code (stmt
) == GIMPLE_OMP_TASK
));
5045 gsi_remove (&gsi
, true);
5046 e
= split_block (entry_bb
, stmt
);
5048 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
5050 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
5053 gsi
= gsi_last_bb (exit_bb
);
5054 gcc_assert (!gsi_end_p (gsi
)
5055 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
5056 stmt
= gimple_build_return (NULL
);
5057 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5058 gsi_remove (&gsi
, true);
5061 /* Move the parallel region into CHILD_CFUN. */
5063 if (gimple_in_ssa_p (cfun
))
5065 init_tree_ssa (child_cfun
);
5066 init_ssa_operands (child_cfun
);
5067 child_cfun
->gimple_df
->in_ssa_p
= true;
5071 block
= gimple_block (entry_stmt
);
5073 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
5075 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
5076 /* When the OMP expansion process cannot guarantee an up-to-date
5077 loop tree arrange for the child function to fixup loops. */
5078 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
5079 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
5081 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
5082 num
= vec_safe_length (child_cfun
->local_decls
);
5083 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
5085 t
= (*child_cfun
->local_decls
)[srcidx
];
5086 if (DECL_CONTEXT (t
) == cfun
->decl
)
5088 if (srcidx
!= dstidx
)
5089 (*child_cfun
->local_decls
)[dstidx
] = t
;
5093 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
5095 /* Inform the callgraph about the new function. */
5096 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
5097 cgraph_node::add_new_function (child_fn
, true);
5099 /* Fix the callgraph edges for child_cfun. Those for cfun will be
5100 fixed in a following pass. */
5101 push_cfun (child_cfun
);
5103 optimize_omp_library_calls (entry_stmt
);
5104 cgraph_edge::rebuild_edges ();
5106 /* Some EH regions might become dead, see PR34608. If
5107 pass_cleanup_cfg isn't the first pass to happen with the
5108 new child, these dead EH edges might cause problems.
5109 Clean them up now. */
5110 if (flag_exceptions
)
5113 bool changed
= false;
5115 FOR_EACH_BB_FN (bb
, cfun
)
5116 changed
|= gimple_purge_dead_eh_edges (bb
);
5118 cleanup_tree_cfg ();
5120 if (gimple_in_ssa_p (cfun
))
5121 update_ssa (TODO_update_ssa
);
5125 /* Emit a library call to launch the children threads. */
5127 expand_cilk_for_call (new_bb
, entry_stmt
, ws_args
);
5128 else if (gimple_code (entry_stmt
) == GIMPLE_OMP_PARALLEL
)
5129 expand_parallel_call (region
, new_bb
, entry_stmt
, ws_args
);
5131 expand_task_call (new_bb
, entry_stmt
);
5132 if (gimple_in_ssa_p (cfun
))
5133 update_ssa (TODO_update_ssa_only_virtuals
);
5137 /* Helper function for expand_omp_{for_*,simd}. If this is the outermost
5138 of the combined collapse > 1 loop constructs, generate code like:
5139 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB;
5144 count3 = (adj + N32 - N31) / STEP3;
5145 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB;
5150 count2 = (adj + N22 - N21) / STEP2;
5151 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB;
5156 count1 = (adj + N12 - N11) / STEP1;
5157 count = count1 * count2 * count3;
5158 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does:
5160 and set ZERO_ITER_BB to that bb. If this isn't the outermost
5161 of the combined loop constructs, just initialize COUNTS array
5162 from the _looptemp_ clauses. */
5164 /* NOTE: It *could* be better to moosh all of the BBs together,
5165 creating one larger BB with all the computation and the unexpected
5166 jump at the end. I.e.
5168 bool zero3, zero2, zero1, zero;
5171 count3 = (N32 - N31) /[cl] STEP3;
5173 count2 = (N22 - N21) /[cl] STEP2;
5175 count1 = (N12 - N11) /[cl] STEP1;
5176 zero = zero3 || zero2 || zero1;
5177 count = count1 * count2 * count3;
5178 if (__builtin_expect(zero, false)) goto zero_iter_bb;
5180 After all, we expect the zero=false, and thus we expect to have to
5181 evaluate all of the comparison expressions, so short-circuiting
5182 oughtn't be a win. Since the condition isn't protecting a
5183 denominator, we're not concerned about divide-by-zero, so we can
5184 fully evaluate count even if a numerator turned out to be wrong.
5186 It seems like putting this all together would create much better
5187 scheduling opportunities, and less pressure on the chip's branch
5191 expand_omp_for_init_counts (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5192 basic_block
&entry_bb
, tree
*counts
,
5193 basic_block
&zero_iter_bb
, int &first_zero_iter
,
5194 basic_block
&l2_dom_bb
)
5196 tree t
, type
= TREE_TYPE (fd
->loop
.v
);
5201 /* Collapsed loops need work for expansion into SSA form. */
5202 gcc_assert (!gimple_in_ssa_p (cfun
));
5204 if (gimple_omp_for_combined_into_p (fd
->for_stmt
)
5205 && TREE_CODE (fd
->loop
.n2
) != INTEGER_CST
)
5207 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5208 isn't supposed to be handled, as the inner loop doesn't
5210 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5211 OMP_CLAUSE__LOOPTEMP_
);
5212 gcc_assert (innerc
);
5213 for (i
= 0; i
< fd
->collapse
; i
++)
5215 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5216 OMP_CLAUSE__LOOPTEMP_
);
5217 gcc_assert (innerc
);
5219 counts
[i
] = OMP_CLAUSE_DECL (innerc
);
5221 counts
[0] = NULL_TREE
;
5226 for (i
= 0; i
< fd
->collapse
; i
++)
5228 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
5230 if (SSA_VAR_P (fd
->loop
.n2
)
5231 && ((t
= fold_binary (fd
->loops
[i
].cond_code
, boolean_type_node
,
5232 fold_convert (itype
, fd
->loops
[i
].n1
),
5233 fold_convert (itype
, fd
->loops
[i
].n2
)))
5234 == NULL_TREE
|| !integer_onep (t
)))
5237 n1
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n1
));
5238 n1
= force_gimple_operand_gsi (gsi
, n1
, true, NULL_TREE
,
5239 true, GSI_SAME_STMT
);
5240 n2
= fold_convert (itype
, unshare_expr (fd
->loops
[i
].n2
));
5241 n2
= force_gimple_operand_gsi (gsi
, n2
, true, NULL_TREE
,
5242 true, GSI_SAME_STMT
);
5243 stmt
= gimple_build_cond (fd
->loops
[i
].cond_code
, n1
, n2
,
5244 NULL_TREE
, NULL_TREE
);
5245 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5246 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
5247 expand_omp_regimplify_p
, NULL
, NULL
)
5248 || walk_tree (gimple_cond_rhs_ptr (stmt
),
5249 expand_omp_regimplify_p
, NULL
, NULL
))
5251 *gsi
= gsi_for_stmt (stmt
);
5252 gimple_regimplify_operands (stmt
, gsi
);
5254 e
= split_block (entry_bb
, stmt
);
5255 if (zero_iter_bb
== NULL
)
5257 first_zero_iter
= i
;
5258 zero_iter_bb
= create_empty_bb (entry_bb
);
5259 add_bb_to_loop (zero_iter_bb
, entry_bb
->loop_father
);
5260 *gsi
= gsi_after_labels (zero_iter_bb
);
5261 stmt
= gimple_build_assign (fd
->loop
.n2
,
5262 build_zero_cst (type
));
5263 gsi_insert_before (gsi
, stmt
, GSI_SAME_STMT
);
5264 set_immediate_dominator (CDI_DOMINATORS
, zero_iter_bb
,
5267 ne
= make_edge (entry_bb
, zero_iter_bb
, EDGE_FALSE_VALUE
);
5268 ne
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
5269 e
->flags
= EDGE_TRUE_VALUE
;
5270 e
->probability
= REG_BR_PROB_BASE
- ne
->probability
;
5271 if (l2_dom_bb
== NULL
)
5272 l2_dom_bb
= entry_bb
;
5274 *gsi
= gsi_last_bb (entry_bb
);
5277 if (POINTER_TYPE_P (itype
))
5278 itype
= signed_type_for (itype
);
5279 t
= build_int_cst (itype
, (fd
->loops
[i
].cond_code
== LT_EXPR
5281 t
= fold_build2 (PLUS_EXPR
, itype
,
5282 fold_convert (itype
, fd
->loops
[i
].step
), t
);
5283 t
= fold_build2 (PLUS_EXPR
, itype
, t
,
5284 fold_convert (itype
, fd
->loops
[i
].n2
));
5285 t
= fold_build2 (MINUS_EXPR
, itype
, t
,
5286 fold_convert (itype
, fd
->loops
[i
].n1
));
5287 /* ?? We could probably use CEIL_DIV_EXPR instead of
5288 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't
5289 generate the same code in the end because generically we
5290 don't know that the values involved must be negative for
5292 if (TYPE_UNSIGNED (itype
) && fd
->loops
[i
].cond_code
== GT_EXPR
)
5293 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
5294 fold_build1 (NEGATE_EXPR
, itype
, t
),
5295 fold_build1 (NEGATE_EXPR
, itype
,
5296 fold_convert (itype
,
5297 fd
->loops
[i
].step
)));
5299 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
,
5300 fold_convert (itype
, fd
->loops
[i
].step
));
5301 t
= fold_convert (type
, t
);
5302 if (TREE_CODE (t
) == INTEGER_CST
)
5306 counts
[i
] = create_tmp_reg (type
, ".count");
5307 expand_omp_build_assign (gsi
, counts
[i
], t
);
5309 if (SSA_VAR_P (fd
->loop
.n2
))
5314 t
= fold_build2 (MULT_EXPR
, type
, fd
->loop
.n2
, counts
[i
]);
5315 expand_omp_build_assign (gsi
, fd
->loop
.n2
, t
);
5321 /* Helper function for expand_omp_{for_*,simd}. Generate code like:
5323 V3 = N31 + (T % count3) * STEP3;
5325 V2 = N21 + (T % count2) * STEP2;
5327 V1 = N11 + T * STEP1;
5328 if this loop doesn't have an inner loop construct combined with it.
5329 If it does have an inner loop construct combined with it and the
5330 iteration count isn't known constant, store values from counts array
5331 into its _looptemp_ temporaries instead. */
5334 expand_omp_for_init_vars (struct omp_for_data
*fd
, gimple_stmt_iterator
*gsi
,
5335 tree
*counts
, gimple inner_stmt
, tree startvar
)
5338 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5340 /* If fd->loop.n2 is constant, then no propagation of the counts
5341 is needed, they are constant. */
5342 if (TREE_CODE (fd
->loop
.n2
) == INTEGER_CST
)
5345 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
5346 ? gimple_omp_parallel_clauses (inner_stmt
)
5347 : gimple_omp_for_clauses (inner_stmt
);
5348 /* First two _looptemp_ clauses are for istart/iend, counts[0]
5349 isn't supposed to be handled, as the inner loop doesn't
5351 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
5352 gcc_assert (innerc
);
5353 for (i
= 0; i
< fd
->collapse
; i
++)
5355 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5356 OMP_CLAUSE__LOOPTEMP_
);
5357 gcc_assert (innerc
);
5360 tree tem
= OMP_CLAUSE_DECL (innerc
);
5361 tree t
= fold_convert (TREE_TYPE (tem
), counts
[i
]);
5362 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5363 false, GSI_CONTINUE_LINKING
);
5364 gimple stmt
= gimple_build_assign (tem
, t
);
5365 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5371 tree type
= TREE_TYPE (fd
->loop
.v
);
5372 tree tem
= create_tmp_reg (type
, ".tem");
5373 gimple stmt
= gimple_build_assign (tem
, startvar
);
5374 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5376 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5378 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
), itype
, t
;
5380 if (POINTER_TYPE_P (vtype
))
5381 itype
= signed_type_for (vtype
);
5383 t
= fold_build2 (TRUNC_MOD_EXPR
, type
, tem
, counts
[i
]);
5386 t
= fold_convert (itype
, t
);
5387 t
= fold_build2 (MULT_EXPR
, itype
, t
,
5388 fold_convert (itype
, fd
->loops
[i
].step
));
5389 if (POINTER_TYPE_P (vtype
))
5390 t
= fold_build_pointer_plus (fd
->loops
[i
].n1
, t
);
5392 t
= fold_build2 (PLUS_EXPR
, itype
, fd
->loops
[i
].n1
, t
);
5393 t
= force_gimple_operand_gsi (gsi
, t
,
5394 DECL_P (fd
->loops
[i
].v
)
5395 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5397 GSI_CONTINUE_LINKING
);
5398 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5399 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5402 t
= fold_build2 (TRUNC_DIV_EXPR
, type
, tem
, counts
[i
]);
5403 t
= force_gimple_operand_gsi (gsi
, t
, false, NULL_TREE
,
5404 false, GSI_CONTINUE_LINKING
);
5405 stmt
= gimple_build_assign (tem
, t
);
5406 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
5412 /* Helper function for expand_omp_for_*. Generate code like:
5415 if (V3 cond3 N32) goto BODY_BB; else goto L11;
5419 if (V2 cond2 N22) goto BODY_BB; else goto L12;
5426 extract_omp_for_update_vars (struct omp_for_data
*fd
, basic_block cont_bb
,
5427 basic_block body_bb
)
5429 basic_block last_bb
, bb
, collapse_bb
= NULL
;
5431 gimple_stmt_iterator gsi
;
5437 for (i
= fd
->collapse
- 1; i
>= 0; i
--)
5439 tree vtype
= TREE_TYPE (fd
->loops
[i
].v
);
5441 bb
= create_empty_bb (last_bb
);
5442 add_bb_to_loop (bb
, last_bb
->loop_father
);
5443 gsi
= gsi_start_bb (bb
);
5445 if (i
< fd
->collapse
- 1)
5447 e
= make_edge (last_bb
, bb
, EDGE_FALSE_VALUE
);
5448 e
->probability
= REG_BR_PROB_BASE
/ 8;
5450 t
= fd
->loops
[i
+ 1].n1
;
5451 t
= force_gimple_operand_gsi (&gsi
, t
,
5452 DECL_P (fd
->loops
[i
+ 1].v
)
5453 && TREE_ADDRESSABLE (fd
->loops
[i
5456 GSI_CONTINUE_LINKING
);
5457 stmt
= gimple_build_assign (fd
->loops
[i
+ 1].v
, t
);
5458 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5463 set_immediate_dominator (CDI_DOMINATORS
, bb
, last_bb
);
5465 if (POINTER_TYPE_P (vtype
))
5466 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5468 t
= fold_build2 (PLUS_EXPR
, vtype
, fd
->loops
[i
].v
, fd
->loops
[i
].step
);
5469 t
= force_gimple_operand_gsi (&gsi
, t
,
5470 DECL_P (fd
->loops
[i
].v
)
5471 && TREE_ADDRESSABLE (fd
->loops
[i
].v
),
5472 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5473 stmt
= gimple_build_assign (fd
->loops
[i
].v
, t
);
5474 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5478 t
= fd
->loops
[i
].n2
;
5479 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5480 false, GSI_CONTINUE_LINKING
);
5481 tree v
= fd
->loops
[i
].v
;
5482 if (DECL_P (v
) && TREE_ADDRESSABLE (v
))
5483 v
= force_gimple_operand_gsi (&gsi
, v
, true, NULL_TREE
,
5484 false, GSI_CONTINUE_LINKING
);
5485 t
= fold_build2 (fd
->loops
[i
].cond_code
, boolean_type_node
, v
, t
);
5486 stmt
= gimple_build_cond_empty (t
);
5487 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5488 e
= make_edge (bb
, body_bb
, EDGE_TRUE_VALUE
);
5489 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5492 make_edge (bb
, body_bb
, EDGE_FALLTHRU
);
5500 /* A subroutine of expand_omp_for. Generate code for a parallel
5501 loop with any schedule. Given parameters:
5503 for (V = N1; V cond N2; V += STEP) BODY;
5505 where COND is "<" or ">", we generate pseudocode
5507 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0);
5508 if (more) goto L0; else goto L3;
5515 if (V cond iend) goto L1; else goto L2;
5517 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5520 If this is a combined omp parallel loop, instead of the call to
5521 GOMP_loop_foo_start, we call GOMP_loop_foo_next.
5522 If this is gimple_omp_for_combined_p loop, then instead of assigning
5523 V and iend in L0 we assign the first two _looptemp_ clause decls of the
5524 inner GIMPLE_OMP_FOR and V += STEP; and
5525 if (V cond iend) goto L1; else goto L2; are removed.
5527 For collapsed loops, given parameters:
5529 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
5530 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
5531 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
5534 we generate pseudocode
5536 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0;
5541 count3 = (adj + N32 - N31) / STEP3;
5542 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0;
5547 count2 = (adj + N22 - N21) / STEP2;
5548 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0;
5553 count1 = (adj + N12 - N11) / STEP1;
5554 count = count1 * count2 * count3;
5559 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0);
5560 if (more) goto L0; else goto L3;
5564 V3 = N31 + (T % count3) * STEP3;
5566 V2 = N21 + (T % count2) * STEP2;
5568 V1 = N11 + T * STEP1;
5573 if (V < iend) goto L10; else goto L2;
5576 if (V3 cond3 N32) goto L1; else goto L11;
5580 if (V2 cond2 N22) goto L1; else goto L12;
5586 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3;
5592 expand_omp_for_generic (struct omp_region
*region
,
5593 struct omp_for_data
*fd
,
5594 enum built_in_function start_fn
,
5595 enum built_in_function next_fn
,
5598 tree type
, istart0
, iend0
, iend
;
5599 tree t
, vmain
, vback
, bias
= NULL_TREE
;
5600 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, collapse_bb
;
5601 basic_block l2_bb
= NULL
, l3_bb
= NULL
;
5602 gimple_stmt_iterator gsi
;
5604 bool in_combined_parallel
= is_combined_parallel (region
);
5605 bool broken_loop
= region
->cont
== NULL
;
5607 tree
*counts
= NULL
;
5610 gcc_assert (!broken_loop
|| !in_combined_parallel
);
5611 gcc_assert (fd
->iter_type
== long_integer_type_node
5612 || !in_combined_parallel
);
5614 type
= TREE_TYPE (fd
->loop
.v
);
5615 istart0
= create_tmp_var (fd
->iter_type
, ".istart0");
5616 iend0
= create_tmp_var (fd
->iter_type
, ".iend0");
5617 TREE_ADDRESSABLE (istart0
) = 1;
5618 TREE_ADDRESSABLE (iend0
) = 1;
5620 /* See if we need to bias by LLONG_MIN. */
5621 if (fd
->iter_type
== long_long_unsigned_type_node
5622 && TREE_CODE (type
) == INTEGER_TYPE
5623 && !TYPE_UNSIGNED (type
))
5627 if (fd
->loop
.cond_code
== LT_EXPR
)
5630 n2
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5634 n1
= fold_build2 (MINUS_EXPR
, type
, fd
->loop
.n2
, fd
->loop
.step
);
5637 if (TREE_CODE (n1
) != INTEGER_CST
5638 || TREE_CODE (n2
) != INTEGER_CST
5639 || ((tree_int_cst_sgn (n1
) < 0) ^ (tree_int_cst_sgn (n2
) < 0)))
5640 bias
= fold_convert (fd
->iter_type
, TYPE_MIN_VALUE (type
));
5643 entry_bb
= region
->entry
;
5644 cont_bb
= region
->cont
;
5646 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
5647 gcc_assert (broken_loop
5648 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
5649 l0_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
5650 l1_bb
= single_succ (l0_bb
);
5653 l2_bb
= create_empty_bb (cont_bb
);
5654 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l1_bb
);
5655 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
5659 l3_bb
= BRANCH_EDGE (entry_bb
)->dest
;
5660 exit_bb
= region
->exit
;
5662 gsi
= gsi_last_bb (entry_bb
);
5664 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
5665 if (fd
->collapse
> 1)
5667 int first_zero_iter
= -1;
5668 basic_block zero_iter_bb
= NULL
, l2_dom_bb
= NULL
;
5670 counts
= XALLOCAVEC (tree
, fd
->collapse
);
5671 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
5672 zero_iter_bb
, first_zero_iter
,
5677 /* Some counts[i] vars might be uninitialized if
5678 some loop has zero iterations. But the body shouldn't
5679 be executed in that case, so just avoid uninit warnings. */
5680 for (i
= first_zero_iter
; i
< fd
->collapse
; i
++)
5681 if (SSA_VAR_P (counts
[i
]))
5682 TREE_NO_WARNING (counts
[i
]) = 1;
5684 e
= split_block (entry_bb
, gsi_stmt (gsi
));
5686 make_edge (zero_iter_bb
, entry_bb
, EDGE_FALLTHRU
);
5687 gsi
= gsi_last_bb (entry_bb
);
5688 set_immediate_dominator (CDI_DOMINATORS
, entry_bb
,
5689 get_immediate_dominator (CDI_DOMINATORS
,
5693 if (in_combined_parallel
)
5695 /* In a combined parallel loop, emit a call to
5696 GOMP_loop_foo_next. */
5697 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5698 build_fold_addr_expr (istart0
),
5699 build_fold_addr_expr (iend0
));
5703 tree t0
, t1
, t2
, t3
, t4
;
5704 /* If this is not a combined parallel loop, emit a call to
5705 GOMP_loop_foo_start in ENTRY_BB. */
5706 t4
= build_fold_addr_expr (iend0
);
5707 t3
= build_fold_addr_expr (istart0
);
5708 t2
= fold_convert (fd
->iter_type
, fd
->loop
.step
);
5711 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
5713 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
5714 OMP_CLAUSE__LOOPTEMP_
);
5715 gcc_assert (innerc
);
5716 t0
= OMP_CLAUSE_DECL (innerc
);
5717 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5718 OMP_CLAUSE__LOOPTEMP_
);
5719 gcc_assert (innerc
);
5720 t1
= OMP_CLAUSE_DECL (innerc
);
5722 if (POINTER_TYPE_P (TREE_TYPE (t0
))
5723 && TYPE_PRECISION (TREE_TYPE (t0
))
5724 != TYPE_PRECISION (fd
->iter_type
))
5726 /* Avoid casting pointers to integer of a different size. */
5727 tree itype
= signed_type_for (type
);
5728 t1
= fold_convert (fd
->iter_type
, fold_convert (itype
, t1
));
5729 t0
= fold_convert (fd
->iter_type
, fold_convert (itype
, t0
));
5733 t1
= fold_convert (fd
->iter_type
, t1
);
5734 t0
= fold_convert (fd
->iter_type
, t0
);
5738 t1
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t1
, bias
);
5739 t0
= fold_build2 (PLUS_EXPR
, fd
->iter_type
, t0
, bias
);
5741 if (fd
->iter_type
== long_integer_type_node
)
5745 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5746 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5747 6, t0
, t1
, t2
, t
, t3
, t4
);
5750 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5751 5, t0
, t1
, t2
, t3
, t4
);
5759 /* The GOMP_loop_ull_*start functions have additional boolean
5760 argument, true for < loops and false for > loops.
5761 In Fortran, the C bool type can be different from
5762 boolean_type_node. */
5763 bfn_decl
= builtin_decl_explicit (start_fn
);
5764 c_bool_type
= TREE_TYPE (TREE_TYPE (bfn_decl
));
5765 t5
= build_int_cst (c_bool_type
,
5766 fd
->loop
.cond_code
== LT_EXPR
? 1 : 0);
5769 tree bfn_decl
= builtin_decl_explicit (start_fn
);
5770 t
= fold_convert (fd
->iter_type
, fd
->chunk_size
);
5771 t
= build_call_expr (bfn_decl
, 7, t5
, t0
, t1
, t2
, t
, t3
, t4
);
5774 t
= build_call_expr (builtin_decl_explicit (start_fn
),
5775 6, t5
, t0
, t1
, t2
, t3
, t4
);
5778 if (TREE_TYPE (t
) != boolean_type_node
)
5779 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5780 t
, build_int_cst (TREE_TYPE (t
), 0));
5781 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5782 true, GSI_SAME_STMT
);
5783 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
5785 /* Remove the GIMPLE_OMP_FOR statement. */
5786 gsi_remove (&gsi
, true);
5788 /* Iteration setup for sequential loop goes in L0_BB. */
5789 tree startvar
= fd
->loop
.v
;
5790 tree endvar
= NULL_TREE
;
5792 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5794 gcc_assert (gimple_code (inner_stmt
) == GIMPLE_OMP_FOR
5795 && gimple_omp_for_kind (inner_stmt
)
5796 == GF_OMP_FOR_KIND_SIMD
);
5797 tree innerc
= find_omp_clause (gimple_omp_for_clauses (inner_stmt
),
5798 OMP_CLAUSE__LOOPTEMP_
);
5799 gcc_assert (innerc
);
5800 startvar
= OMP_CLAUSE_DECL (innerc
);
5801 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
5802 OMP_CLAUSE__LOOPTEMP_
);
5803 gcc_assert (innerc
);
5804 endvar
= OMP_CLAUSE_DECL (innerc
);
5807 gsi
= gsi_start_bb (l0_bb
);
5810 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5811 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5812 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5813 t
= fold_convert (TREE_TYPE (startvar
), t
);
5814 t
= force_gimple_operand_gsi (&gsi
, t
,
5816 && TREE_ADDRESSABLE (startvar
),
5817 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
5818 stmt
= gimple_build_assign (startvar
, t
);
5819 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5823 t
= fold_build2 (MINUS_EXPR
, fd
->iter_type
, t
, bias
);
5824 if (POINTER_TYPE_P (TREE_TYPE (startvar
)))
5825 t
= fold_convert (signed_type_for (TREE_TYPE (startvar
)), t
);
5826 t
= fold_convert (TREE_TYPE (startvar
), t
);
5827 iend
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5828 false, GSI_CONTINUE_LINKING
);
5831 stmt
= gimple_build_assign (endvar
, iend
);
5832 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5833 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (iend
)))
5834 stmt
= gimple_build_assign (fd
->loop
.v
, iend
);
5836 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, iend
,
5838 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5840 if (fd
->collapse
> 1)
5841 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
5845 /* Code to control the increment and predicate for the sequential
5846 loop goes in the CONT_BB. */
5847 gsi
= gsi_last_bb (cont_bb
);
5848 stmt
= gsi_stmt (gsi
);
5849 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
5850 vmain
= gimple_omp_continue_control_use (stmt
);
5851 vback
= gimple_omp_continue_control_def (stmt
);
5853 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5855 if (POINTER_TYPE_P (type
))
5856 t
= fold_build_pointer_plus (vmain
, fd
->loop
.step
);
5858 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, fd
->loop
.step
);
5859 t
= force_gimple_operand_gsi (&gsi
, t
,
5861 && TREE_ADDRESSABLE (vback
),
5862 NULL_TREE
, true, GSI_SAME_STMT
);
5863 stmt
= gimple_build_assign (vback
, t
);
5864 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5866 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
5867 DECL_P (vback
) && TREE_ADDRESSABLE (vback
) ? t
: vback
,
5869 stmt
= gimple_build_cond_empty (t
);
5870 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
5873 /* Remove GIMPLE_OMP_CONTINUE. */
5874 gsi_remove (&gsi
, true);
5876 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
5877 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, l1_bb
);
5879 /* Emit code to get the next parallel iteration in L2_BB. */
5880 gsi
= gsi_start_bb (l2_bb
);
5882 t
= build_call_expr (builtin_decl_explicit (next_fn
), 2,
5883 build_fold_addr_expr (istart0
),
5884 build_fold_addr_expr (iend0
));
5885 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
5886 false, GSI_CONTINUE_LINKING
);
5887 if (TREE_TYPE (t
) != boolean_type_node
)
5888 t
= fold_build2 (NE_EXPR
, boolean_type_node
,
5889 t
, build_int_cst (TREE_TYPE (t
), 0));
5890 stmt
= gimple_build_cond_empty (t
);
5891 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
5894 /* Add the loop cleanup function. */
5895 gsi
= gsi_last_bb (exit_bb
);
5896 if (gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
5897 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT
);
5898 else if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5899 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL
);
5901 t
= builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END
);
5902 stmt
= gimple_build_call (t
, 0);
5903 if (gimple_omp_return_lhs (gsi_stmt (gsi
)))
5904 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (gsi
)));
5905 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
5906 gsi_remove (&gsi
, true);
5908 /* Connect the new blocks. */
5909 find_edge (entry_bb
, l0_bb
)->flags
= EDGE_TRUE_VALUE
;
5910 find_edge (entry_bb
, l3_bb
)->flags
= EDGE_FALSE_VALUE
;
5916 e
= find_edge (cont_bb
, l3_bb
);
5917 ne
= make_edge (l2_bb
, l3_bb
, EDGE_FALSE_VALUE
);
5919 phis
= phi_nodes (l3_bb
);
5920 for (gsi
= gsi_start (phis
); !gsi_end_p (gsi
); gsi_next (&gsi
))
5922 gimple phi
= gsi_stmt (gsi
);
5923 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, ne
),
5924 PHI_ARG_DEF_FROM_EDGE (phi
, e
));
5928 make_edge (cont_bb
, l2_bb
, EDGE_FALSE_VALUE
);
5929 add_bb_to_loop (l2_bb
, cont_bb
->loop_father
);
5930 e
= find_edge (cont_bb
, l1_bb
);
5931 if (gimple_omp_for_combined_p (fd
->for_stmt
))
5936 else if (fd
->collapse
> 1)
5939 e
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
5942 e
->flags
= EDGE_TRUE_VALUE
;
5945 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
5946 find_edge (cont_bb
, l2_bb
)->probability
= REG_BR_PROB_BASE
/ 8;
5950 e
= find_edge (cont_bb
, l2_bb
);
5951 e
->flags
= EDGE_FALLTHRU
;
5953 make_edge (l2_bb
, l0_bb
, EDGE_TRUE_VALUE
);
5955 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
,
5956 recompute_dominator (CDI_DOMINATORS
, l2_bb
));
5957 set_immediate_dominator (CDI_DOMINATORS
, l3_bb
,
5958 recompute_dominator (CDI_DOMINATORS
, l3_bb
));
5959 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
,
5960 recompute_dominator (CDI_DOMINATORS
, l0_bb
));
5961 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
,
5962 recompute_dominator (CDI_DOMINATORS
, l1_bb
));
5964 struct loop
*outer_loop
= alloc_loop ();
5965 outer_loop
->header
= l0_bb
;
5966 outer_loop
->latch
= l2_bb
;
5967 add_loop (outer_loop
, l0_bb
->loop_father
);
5969 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
5971 struct loop
*loop
= alloc_loop ();
5972 loop
->header
= l1_bb
;
5973 /* The loop may have multiple latches. */
5974 add_loop (loop
, outer_loop
);
5980 /* A subroutine of expand_omp_for. Generate code for a parallel
5981 loop with static schedule and no specified chunk size. Given
5984 for (V = N1; V cond N2; V += STEP) BODY;
5986 where COND is "<" or ">", we generate pseudocode
5988 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
5993 if ((__typeof (V)) -1 > 0 && cond is >)
5994 n = -(adj + N2 - N1) / -STEP;
5996 n = (adj + N2 - N1) / STEP;
5999 if (threadid < tt) goto L3; else goto L4;
6004 s0 = q * threadid + tt;
6007 if (s0 >= e0) goto L2; else goto L0;
6013 if (V cond e) goto L1;
6018 expand_omp_for_static_nochunk (struct omp_region
*region
,
6019 struct omp_for_data
*fd
,
6022 tree n
, q
, s0
, e0
, e
, t
, tt
, nthreads
, threadid
;
6023 tree type
, itype
, vmain
, vback
;
6024 basic_block entry_bb
, second_bb
, third_bb
, exit_bb
, seq_start_bb
;
6025 basic_block body_bb
, cont_bb
, collapse_bb
= NULL
;
6027 gimple_stmt_iterator gsi
;
6030 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6031 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6032 bool broken_loop
= region
->cont
== NULL
;
6033 tree
*counts
= NULL
;
6036 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6037 if (POINTER_TYPE_P (type
))
6038 itype
= signed_type_for (type
);
6040 entry_bb
= region
->entry
;
6041 cont_bb
= region
->cont
;
6042 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6043 fin_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6044 gcc_assert (broken_loop
6045 || (fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
));
6046 seq_start_bb
= split_edge (FALLTHRU_EDGE (entry_bb
));
6047 body_bb
= single_succ (seq_start_bb
);
6050 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6051 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6053 exit_bb
= region
->exit
;
6055 /* Iteration space partitioning goes in ENTRY_BB. */
6056 gsi
= gsi_last_bb (entry_bb
);
6057 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6059 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6061 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6062 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6065 if (fd
->collapse
> 1)
6067 int first_zero_iter
= -1;
6068 basic_block l2_dom_bb
= NULL
;
6070 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6071 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6072 fin_bb
, first_zero_iter
,
6076 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6077 t
= integer_one_node
;
6079 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6080 fold_convert (type
, fd
->loop
.n1
),
6081 fold_convert (type
, fd
->loop
.n2
));
6082 if (fd
->collapse
== 1
6083 && TYPE_UNSIGNED (type
)
6084 && (t
== NULL_TREE
|| !integer_onep (t
)))
6086 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6087 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6088 true, GSI_SAME_STMT
);
6089 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6090 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6091 true, GSI_SAME_STMT
);
6092 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6093 NULL_TREE
, NULL_TREE
);
6094 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6095 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6096 expand_omp_regimplify_p
, NULL
, NULL
)
6097 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6098 expand_omp_regimplify_p
, NULL
, NULL
))
6100 gsi
= gsi_for_stmt (stmt
);
6101 gimple_regimplify_operands (stmt
, &gsi
);
6103 ep
= split_block (entry_bb
, stmt
);
6104 ep
->flags
= EDGE_TRUE_VALUE
;
6105 entry_bb
= ep
->dest
;
6106 ep
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6107 ep
= make_edge (ep
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6108 ep
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6109 if (gimple_in_ssa_p (cfun
))
6111 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6112 for (gsi
= gsi_start_phis (fin_bb
);
6113 !gsi_end_p (gsi
); gsi_next (&gsi
))
6115 gimple phi
= gsi_stmt (gsi
);
6116 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6117 ep
, UNKNOWN_LOCATION
);
6120 gsi
= gsi_last_bb (entry_bb
);
6123 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6124 t
= fold_convert (itype
, t
);
6125 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6126 true, GSI_SAME_STMT
);
6128 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6129 t
= fold_convert (itype
, t
);
6130 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6131 true, GSI_SAME_STMT
);
6135 step
= fd
->loop
.step
;
6136 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6138 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6139 OMP_CLAUSE__LOOPTEMP_
);
6140 gcc_assert (innerc
);
6141 n1
= OMP_CLAUSE_DECL (innerc
);
6142 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6143 OMP_CLAUSE__LOOPTEMP_
);
6144 gcc_assert (innerc
);
6145 n2
= OMP_CLAUSE_DECL (innerc
);
6147 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6148 true, NULL_TREE
, true, GSI_SAME_STMT
);
6149 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6150 true, NULL_TREE
, true, GSI_SAME_STMT
);
6151 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6152 true, NULL_TREE
, true, GSI_SAME_STMT
);
6154 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6155 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6156 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6157 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6158 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6159 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6160 fold_build1 (NEGATE_EXPR
, itype
, t
),
6161 fold_build1 (NEGATE_EXPR
, itype
, step
));
6163 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6164 t
= fold_convert (itype
, t
);
6165 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6167 q
= create_tmp_reg (itype
, "q");
6168 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, n
, nthreads
);
6169 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6170 gsi_insert_before (&gsi
, gimple_build_assign (q
, t
), GSI_SAME_STMT
);
6172 tt
= create_tmp_reg (itype
, "tt");
6173 t
= fold_build2 (TRUNC_MOD_EXPR
, itype
, n
, nthreads
);
6174 t
= force_gimple_operand_gsi (&gsi
, t
, false, NULL_TREE
, true, GSI_SAME_STMT
);
6175 gsi_insert_before (&gsi
, gimple_build_assign (tt
, t
), GSI_SAME_STMT
);
6177 t
= build2 (LT_EXPR
, boolean_type_node
, threadid
, tt
);
6178 stmt
= gimple_build_cond_empty (t
);
6179 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6181 second_bb
= split_block (entry_bb
, stmt
)->dest
;
6182 gsi
= gsi_last_bb (second_bb
);
6183 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6185 gsi_insert_before (&gsi
, gimple_build_assign (tt
, build_int_cst (itype
, 0)),
6187 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, q
, q
,
6188 build_int_cst (itype
, 1));
6189 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6191 third_bb
= split_block (second_bb
, stmt
)->dest
;
6192 gsi
= gsi_last_bb (third_bb
);
6193 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6195 t
= build2 (MULT_EXPR
, itype
, q
, threadid
);
6196 t
= build2 (PLUS_EXPR
, itype
, t
, tt
);
6197 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6199 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, q
);
6200 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
6202 t
= build2 (GE_EXPR
, boolean_type_node
, s0
, e0
);
6203 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6205 /* Remove the GIMPLE_OMP_FOR statement. */
6206 gsi_remove (&gsi
, true);
6208 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6209 gsi
= gsi_start_bb (seq_start_bb
);
6211 tree startvar
= fd
->loop
.v
;
6212 tree endvar
= NULL_TREE
;
6214 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6216 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6217 ? gimple_omp_parallel_clauses (inner_stmt
)
6218 : gimple_omp_for_clauses (inner_stmt
);
6219 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6220 gcc_assert (innerc
);
6221 startvar
= OMP_CLAUSE_DECL (innerc
);
6222 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6223 OMP_CLAUSE__LOOPTEMP_
);
6224 gcc_assert (innerc
);
6225 endvar
= OMP_CLAUSE_DECL (innerc
);
6227 t
= fold_convert (itype
, s0
);
6228 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6229 if (POINTER_TYPE_P (type
))
6230 t
= fold_build_pointer_plus (n1
, t
);
6232 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6233 t
= fold_convert (TREE_TYPE (startvar
), t
);
6234 t
= force_gimple_operand_gsi (&gsi
, t
,
6236 && TREE_ADDRESSABLE (startvar
),
6237 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6238 stmt
= gimple_build_assign (startvar
, t
);
6239 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6241 t
= fold_convert (itype
, e0
);
6242 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6243 if (POINTER_TYPE_P (type
))
6244 t
= fold_build_pointer_plus (n1
, t
);
6246 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6247 t
= fold_convert (TREE_TYPE (startvar
), t
);
6248 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6249 false, GSI_CONTINUE_LINKING
);
6252 stmt
= gimple_build_assign (endvar
, e
);
6253 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6254 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6255 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6257 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6259 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6261 if (fd
->collapse
> 1)
6262 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6266 /* The code controlling the sequential loop replaces the
6267 GIMPLE_OMP_CONTINUE. */
6268 gsi
= gsi_last_bb (cont_bb
);
6269 stmt
= gsi_stmt (gsi
);
6270 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6271 vmain
= gimple_omp_continue_control_use (stmt
);
6272 vback
= gimple_omp_continue_control_def (stmt
);
6274 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6276 if (POINTER_TYPE_P (type
))
6277 t
= fold_build_pointer_plus (vmain
, step
);
6279 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6280 t
= force_gimple_operand_gsi (&gsi
, t
,
6282 && TREE_ADDRESSABLE (vback
),
6283 NULL_TREE
, true, GSI_SAME_STMT
);
6284 stmt
= gimple_build_assign (vback
, t
);
6285 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6287 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6288 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6290 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6293 /* Remove the GIMPLE_OMP_CONTINUE statement. */
6294 gsi_remove (&gsi
, true);
6296 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6297 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6300 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6301 gsi
= gsi_last_bb (exit_bb
);
6302 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6304 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6305 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6307 gsi_remove (&gsi
, true);
6309 /* Connect all the blocks. */
6310 ep
= make_edge (entry_bb
, third_bb
, EDGE_FALSE_VALUE
);
6311 ep
->probability
= REG_BR_PROB_BASE
/ 4 * 3;
6312 ep
= find_edge (entry_bb
, second_bb
);
6313 ep
->flags
= EDGE_TRUE_VALUE
;
6314 ep
->probability
= REG_BR_PROB_BASE
/ 4;
6315 find_edge (third_bb
, seq_start_bb
)->flags
= EDGE_FALSE_VALUE
;
6316 find_edge (third_bb
, fin_bb
)->flags
= EDGE_TRUE_VALUE
;
6320 ep
= find_edge (cont_bb
, body_bb
);
6321 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6326 else if (fd
->collapse
> 1)
6329 ep
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6332 ep
->flags
= EDGE_TRUE_VALUE
;
6333 find_edge (cont_bb
, fin_bb
)->flags
6334 = ep
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6337 set_immediate_dominator (CDI_DOMINATORS
, second_bb
, entry_bb
);
6338 set_immediate_dominator (CDI_DOMINATORS
, third_bb
, entry_bb
);
6339 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
, third_bb
);
6341 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6342 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6343 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6344 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6346 if (!broken_loop
&& !gimple_omp_for_combined_p (fd
->for_stmt
))
6348 struct loop
*loop
= alloc_loop ();
6349 loop
->header
= body_bb
;
6350 if (collapse_bb
== NULL
)
6351 loop
->latch
= cont_bb
;
6352 add_loop (loop
, body_bb
->loop_father
);
6357 /* A subroutine of expand_omp_for. Generate code for a parallel
6358 loop with static schedule and a specified chunk size. Given
6361 for (V = N1; V cond N2; V += STEP) BODY;
6363 where COND is "<" or ">", we generate pseudocode
6365 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2;
6370 if ((__typeof (V)) -1 > 0 && cond is >)
6371 n = -(adj + N2 - N1) / -STEP;
6373 n = (adj + N2 - N1) / STEP;
6375 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is
6376 here so that V is defined
6377 if the loop is not entered
6379 s0 = (trip * nthreads + threadid) * CHUNK;
6380 e0 = min(s0 + CHUNK, n);
6381 if (s0 < n) goto L1; else goto L4;
6388 if (V cond e) goto L2; else goto L3;
6396 expand_omp_for_static_chunk (struct omp_region
*region
,
6397 struct omp_for_data
*fd
, gimple inner_stmt
)
6399 tree n
, s0
, e0
, e
, t
;
6400 tree trip_var
, trip_init
, trip_main
, trip_back
, nthreads
, threadid
;
6401 tree type
, itype
, vmain
, vback
, vextra
;
6402 basic_block entry_bb
, exit_bb
, body_bb
, seq_start_bb
, iter_part_bb
;
6403 basic_block trip_update_bb
= NULL
, cont_bb
, collapse_bb
= NULL
, fin_bb
;
6404 gimple_stmt_iterator gsi
;
6407 enum built_in_function get_num_threads
= BUILT_IN_OMP_GET_NUM_THREADS
;
6408 enum built_in_function get_thread_num
= BUILT_IN_OMP_GET_THREAD_NUM
;
6409 bool broken_loop
= region
->cont
== NULL
;
6410 tree
*counts
= NULL
;
6413 itype
= type
= TREE_TYPE (fd
->loop
.v
);
6414 if (POINTER_TYPE_P (type
))
6415 itype
= signed_type_for (type
);
6417 entry_bb
= region
->entry
;
6418 se
= split_block (entry_bb
, last_stmt (entry_bb
));
6420 iter_part_bb
= se
->dest
;
6421 cont_bb
= region
->cont
;
6422 gcc_assert (EDGE_COUNT (iter_part_bb
->succs
) == 2);
6423 fin_bb
= BRANCH_EDGE (iter_part_bb
)->dest
;
6424 gcc_assert (broken_loop
6425 || fin_bb
== FALLTHRU_EDGE (cont_bb
)->dest
);
6426 seq_start_bb
= split_edge (FALLTHRU_EDGE (iter_part_bb
));
6427 body_bb
= single_succ (seq_start_bb
);
6430 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== body_bb
);
6431 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6432 trip_update_bb
= split_edge (FALLTHRU_EDGE (cont_bb
));
6434 exit_bb
= region
->exit
;
6436 /* Trip and adjustment setup goes in ENTRY_BB. */
6437 gsi
= gsi_last_bb (entry_bb
);
6438 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6440 if (gimple_omp_for_kind (fd
->for_stmt
) == GF_OMP_FOR_KIND_DISTRIBUTE
)
6442 get_num_threads
= BUILT_IN_OMP_GET_NUM_TEAMS
;
6443 get_thread_num
= BUILT_IN_OMP_GET_TEAM_NUM
;
6446 if (fd
->collapse
> 1)
6448 int first_zero_iter
= -1;
6449 basic_block l2_dom_bb
= NULL
;
6451 counts
= XALLOCAVEC (tree
, fd
->collapse
);
6452 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
6453 fin_bb
, first_zero_iter
,
6457 else if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6458 t
= integer_one_node
;
6460 t
= fold_binary (fd
->loop
.cond_code
, boolean_type_node
,
6461 fold_convert (type
, fd
->loop
.n1
),
6462 fold_convert (type
, fd
->loop
.n2
));
6463 if (fd
->collapse
== 1
6464 && TYPE_UNSIGNED (type
)
6465 && (t
== NULL_TREE
|| !integer_onep (t
)))
6467 n1
= fold_convert (type
, unshare_expr (fd
->loop
.n1
));
6468 n1
= force_gimple_operand_gsi (&gsi
, n1
, true, NULL_TREE
,
6469 true, GSI_SAME_STMT
);
6470 n2
= fold_convert (type
, unshare_expr (fd
->loop
.n2
));
6471 n2
= force_gimple_operand_gsi (&gsi
, n2
, true, NULL_TREE
,
6472 true, GSI_SAME_STMT
);
6473 stmt
= gimple_build_cond (fd
->loop
.cond_code
, n1
, n2
,
6474 NULL_TREE
, NULL_TREE
);
6475 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6476 if (walk_tree (gimple_cond_lhs_ptr (stmt
),
6477 expand_omp_regimplify_p
, NULL
, NULL
)
6478 || walk_tree (gimple_cond_rhs_ptr (stmt
),
6479 expand_omp_regimplify_p
, NULL
, NULL
))
6481 gsi
= gsi_for_stmt (stmt
);
6482 gimple_regimplify_operands (stmt
, &gsi
);
6484 se
= split_block (entry_bb
, stmt
);
6485 se
->flags
= EDGE_TRUE_VALUE
;
6486 entry_bb
= se
->dest
;
6487 se
->probability
= REG_BR_PROB_BASE
- (REG_BR_PROB_BASE
/ 2000 - 1);
6488 se
= make_edge (se
->src
, fin_bb
, EDGE_FALSE_VALUE
);
6489 se
->probability
= REG_BR_PROB_BASE
/ 2000 - 1;
6490 if (gimple_in_ssa_p (cfun
))
6492 int dest_idx
= find_edge (entry_bb
, fin_bb
)->dest_idx
;
6493 for (gsi
= gsi_start_phis (fin_bb
);
6494 !gsi_end_p (gsi
); gsi_next (&gsi
))
6496 gimple phi
= gsi_stmt (gsi
);
6497 add_phi_arg (phi
, gimple_phi_arg_def (phi
, dest_idx
),
6498 se
, UNKNOWN_LOCATION
);
6501 gsi
= gsi_last_bb (entry_bb
);
6504 t
= build_call_expr (builtin_decl_explicit (get_num_threads
), 0);
6505 t
= fold_convert (itype
, t
);
6506 nthreads
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6507 true, GSI_SAME_STMT
);
6509 t
= build_call_expr (builtin_decl_explicit (get_thread_num
), 0);
6510 t
= fold_convert (itype
, t
);
6511 threadid
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6512 true, GSI_SAME_STMT
);
6516 step
= fd
->loop
.step
;
6517 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
6519 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
6520 OMP_CLAUSE__LOOPTEMP_
);
6521 gcc_assert (innerc
);
6522 n1
= OMP_CLAUSE_DECL (innerc
);
6523 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6524 OMP_CLAUSE__LOOPTEMP_
);
6525 gcc_assert (innerc
);
6526 n2
= OMP_CLAUSE_DECL (innerc
);
6528 n1
= force_gimple_operand_gsi (&gsi
, fold_convert (type
, n1
),
6529 true, NULL_TREE
, true, GSI_SAME_STMT
);
6530 n2
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, n2
),
6531 true, NULL_TREE
, true, GSI_SAME_STMT
);
6532 step
= force_gimple_operand_gsi (&gsi
, fold_convert (itype
, step
),
6533 true, NULL_TREE
, true, GSI_SAME_STMT
);
6535 = force_gimple_operand_gsi (&gsi
, fold_convert (itype
, fd
->chunk_size
),
6536 true, NULL_TREE
, true, GSI_SAME_STMT
);
6538 t
= build_int_cst (itype
, (fd
->loop
.cond_code
== LT_EXPR
? -1 : 1));
6539 t
= fold_build2 (PLUS_EXPR
, itype
, step
, t
);
6540 t
= fold_build2 (PLUS_EXPR
, itype
, t
, n2
);
6541 t
= fold_build2 (MINUS_EXPR
, itype
, t
, fold_convert (itype
, n1
));
6542 if (TYPE_UNSIGNED (itype
) && fd
->loop
.cond_code
== GT_EXPR
)
6543 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
,
6544 fold_build1 (NEGATE_EXPR
, itype
, t
),
6545 fold_build1 (NEGATE_EXPR
, itype
, step
));
6547 t
= fold_build2 (TRUNC_DIV_EXPR
, itype
, t
, step
);
6548 t
= fold_convert (itype
, t
);
6549 n
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6550 true, GSI_SAME_STMT
);
6552 trip_var
= create_tmp_reg (itype
, ".trip");
6553 if (gimple_in_ssa_p (cfun
))
6555 trip_init
= make_ssa_name (trip_var
, NULL
);
6556 trip_main
= make_ssa_name (trip_var
, NULL
);
6557 trip_back
= make_ssa_name (trip_var
, NULL
);
6561 trip_init
= trip_var
;
6562 trip_main
= trip_var
;
6563 trip_back
= trip_var
;
6566 stmt
= gimple_build_assign (trip_init
, build_int_cst (itype
, 0));
6567 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6569 t
= fold_build2 (MULT_EXPR
, itype
, threadid
, fd
->chunk_size
);
6570 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6571 if (POINTER_TYPE_P (type
))
6572 t
= fold_build_pointer_plus (n1
, t
);
6574 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6575 vextra
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6576 true, GSI_SAME_STMT
);
6578 /* Remove the GIMPLE_OMP_FOR. */
6579 gsi_remove (&gsi
, true);
6581 /* Iteration space partitioning goes in ITER_PART_BB. */
6582 gsi
= gsi_last_bb (iter_part_bb
);
6584 t
= fold_build2 (MULT_EXPR
, itype
, trip_main
, nthreads
);
6585 t
= fold_build2 (PLUS_EXPR
, itype
, t
, threadid
);
6586 t
= fold_build2 (MULT_EXPR
, itype
, t
, fd
->chunk_size
);
6587 s0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6588 false, GSI_CONTINUE_LINKING
);
6590 t
= fold_build2 (PLUS_EXPR
, itype
, s0
, fd
->chunk_size
);
6591 t
= fold_build2 (MIN_EXPR
, itype
, t
, n
);
6592 e0
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6593 false, GSI_CONTINUE_LINKING
);
6595 t
= build2 (LT_EXPR
, boolean_type_node
, s0
, n
);
6596 gsi_insert_after (&gsi
, gimple_build_cond_empty (t
), GSI_CONTINUE_LINKING
);
6598 /* Setup code for sequential iteration goes in SEQ_START_BB. */
6599 gsi
= gsi_start_bb (seq_start_bb
);
6601 tree startvar
= fd
->loop
.v
;
6602 tree endvar
= NULL_TREE
;
6604 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6606 tree clauses
= gimple_code (inner_stmt
) == GIMPLE_OMP_PARALLEL
6607 ? gimple_omp_parallel_clauses (inner_stmt
)
6608 : gimple_omp_for_clauses (inner_stmt
);
6609 tree innerc
= find_omp_clause (clauses
, OMP_CLAUSE__LOOPTEMP_
);
6610 gcc_assert (innerc
);
6611 startvar
= OMP_CLAUSE_DECL (innerc
);
6612 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
6613 OMP_CLAUSE__LOOPTEMP_
);
6614 gcc_assert (innerc
);
6615 endvar
= OMP_CLAUSE_DECL (innerc
);
6618 t
= fold_convert (itype
, s0
);
6619 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6620 if (POINTER_TYPE_P (type
))
6621 t
= fold_build_pointer_plus (n1
, t
);
6623 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6624 t
= fold_convert (TREE_TYPE (startvar
), t
);
6625 t
= force_gimple_operand_gsi (&gsi
, t
,
6627 && TREE_ADDRESSABLE (startvar
),
6628 NULL_TREE
, false, GSI_CONTINUE_LINKING
);
6629 stmt
= gimple_build_assign (startvar
, t
);
6630 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6632 t
= fold_convert (itype
, e0
);
6633 t
= fold_build2 (MULT_EXPR
, itype
, t
, step
);
6634 if (POINTER_TYPE_P (type
))
6635 t
= fold_build_pointer_plus (n1
, t
);
6637 t
= fold_build2 (PLUS_EXPR
, type
, t
, n1
);
6638 t
= fold_convert (TREE_TYPE (startvar
), t
);
6639 e
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6640 false, GSI_CONTINUE_LINKING
);
6643 stmt
= gimple_build_assign (endvar
, e
);
6644 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6645 if (useless_type_conversion_p (TREE_TYPE (fd
->loop
.v
), TREE_TYPE (e
)))
6646 stmt
= gimple_build_assign (fd
->loop
.v
, e
);
6648 stmt
= gimple_build_assign_with_ops (NOP_EXPR
, fd
->loop
.v
, e
,
6650 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6652 if (fd
->collapse
> 1)
6653 expand_omp_for_init_vars (fd
, &gsi
, counts
, inner_stmt
, startvar
);
6657 /* The code controlling the sequential loop goes in CONT_BB,
6658 replacing the GIMPLE_OMP_CONTINUE. */
6659 gsi
= gsi_last_bb (cont_bb
);
6660 stmt
= gsi_stmt (gsi
);
6661 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6662 vmain
= gimple_omp_continue_control_use (stmt
);
6663 vback
= gimple_omp_continue_control_def (stmt
);
6665 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6667 if (POINTER_TYPE_P (type
))
6668 t
= fold_build_pointer_plus (vmain
, step
);
6670 t
= fold_build2 (PLUS_EXPR
, type
, vmain
, step
);
6671 if (DECL_P (vback
) && TREE_ADDRESSABLE (vback
))
6672 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
6673 true, GSI_SAME_STMT
);
6674 stmt
= gimple_build_assign (vback
, t
);
6675 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6677 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
,
6678 DECL_P (vback
) && TREE_ADDRESSABLE (vback
)
6680 gsi_insert_before (&gsi
, gimple_build_cond_empty (t
), GSI_SAME_STMT
);
6683 /* Remove GIMPLE_OMP_CONTINUE. */
6684 gsi_remove (&gsi
, true);
6686 if (fd
->collapse
> 1 && !gimple_omp_for_combined_p (fd
->for_stmt
))
6687 collapse_bb
= extract_omp_for_update_vars (fd
, cont_bb
, body_bb
);
6689 /* Trip update code goes into TRIP_UPDATE_BB. */
6690 gsi
= gsi_start_bb (trip_update_bb
);
6692 t
= build_int_cst (itype
, 1);
6693 t
= build2 (PLUS_EXPR
, itype
, trip_main
, t
);
6694 stmt
= gimple_build_assign (trip_back
, t
);
6695 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
6698 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */
6699 gsi
= gsi_last_bb (exit_bb
);
6700 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi
)))
6702 t
= gimple_omp_return_lhs (gsi_stmt (gsi
));
6703 gsi_insert_after (&gsi
, build_omp_barrier (t
), GSI_SAME_STMT
);
6705 gsi_remove (&gsi
, true);
6707 /* Connect the new blocks. */
6708 find_edge (iter_part_bb
, seq_start_bb
)->flags
= EDGE_TRUE_VALUE
;
6709 find_edge (iter_part_bb
, fin_bb
)->flags
= EDGE_FALSE_VALUE
;
6713 se
= find_edge (cont_bb
, body_bb
);
6714 if (gimple_omp_for_combined_p (fd
->for_stmt
))
6719 else if (fd
->collapse
> 1)
6722 se
= make_edge (cont_bb
, collapse_bb
, EDGE_TRUE_VALUE
);
6725 se
->flags
= EDGE_TRUE_VALUE
;
6726 find_edge (cont_bb
, trip_update_bb
)->flags
6727 = se
? EDGE_FALSE_VALUE
: EDGE_FALLTHRU
;
6729 redirect_edge_and_branch (single_succ_edge (trip_update_bb
), iter_part_bb
);
6732 if (gimple_in_ssa_p (cfun
))
6734 gimple_stmt_iterator psi
;
6740 gcc_assert (fd
->collapse
== 1 && !broken_loop
);
6742 /* When we redirect the edge from trip_update_bb to iter_part_bb, we
6743 remove arguments of the phi nodes in fin_bb. We need to create
6744 appropriate phi nodes in iter_part_bb instead. */
6745 se
= single_pred_edge (fin_bb
);
6746 re
= single_succ_edge (trip_update_bb
);
6747 vec
<edge_var_map
> *head
= redirect_edge_var_map_vector (re
);
6748 ene
= single_succ_edge (entry_bb
);
6750 psi
= gsi_start_phis (fin_bb
);
6751 for (i
= 0; !gsi_end_p (psi
) && head
->iterate (i
, &vm
);
6752 gsi_next (&psi
), ++i
)
6755 source_location locus
;
6757 phi
= gsi_stmt (psi
);
6758 t
= gimple_phi_result (phi
);
6759 gcc_assert (t
== redirect_edge_var_map_result (vm
));
6760 nphi
= create_phi_node (t
, iter_part_bb
);
6762 t
= PHI_ARG_DEF_FROM_EDGE (phi
, se
);
6763 locus
= gimple_phi_arg_location_from_edge (phi
, se
);
6765 /* A special case -- fd->loop.v is not yet computed in
6766 iter_part_bb, we need to use vextra instead. */
6767 if (t
== fd
->loop
.v
)
6769 add_phi_arg (nphi
, t
, ene
, locus
);
6770 locus
= redirect_edge_var_map_location (vm
);
6771 add_phi_arg (nphi
, redirect_edge_var_map_def (vm
), re
, locus
);
6773 gcc_assert (!gsi_end_p (psi
) && i
== head
->length ());
6774 redirect_edge_var_map_clear (re
);
6777 psi
= gsi_start_phis (fin_bb
);
6778 if (gsi_end_p (psi
))
6780 remove_phi_node (&psi
, false);
6783 /* Make phi node for trip. */
6784 phi
= create_phi_node (trip_main
, iter_part_bb
);
6785 add_phi_arg (phi
, trip_back
, single_succ_edge (trip_update_bb
),
6787 add_phi_arg (phi
, trip_init
, single_succ_edge (entry_bb
),
6792 set_immediate_dominator (CDI_DOMINATORS
, trip_update_bb
, cont_bb
);
6793 set_immediate_dominator (CDI_DOMINATORS
, iter_part_bb
,
6794 recompute_dominator (CDI_DOMINATORS
, iter_part_bb
));
6795 set_immediate_dominator (CDI_DOMINATORS
, fin_bb
,
6796 recompute_dominator (CDI_DOMINATORS
, fin_bb
));
6797 set_immediate_dominator (CDI_DOMINATORS
, seq_start_bb
,
6798 recompute_dominator (CDI_DOMINATORS
, seq_start_bb
));
6799 set_immediate_dominator (CDI_DOMINATORS
, body_bb
,
6800 recompute_dominator (CDI_DOMINATORS
, body_bb
));
6804 struct loop
*trip_loop
= alloc_loop ();
6805 trip_loop
->header
= iter_part_bb
;
6806 trip_loop
->latch
= trip_update_bb
;
6807 add_loop (trip_loop
, iter_part_bb
->loop_father
);
6809 if (!gimple_omp_for_combined_p (fd
->for_stmt
))
6811 struct loop
*loop
= alloc_loop ();
6812 loop
->header
= body_bb
;
6813 if (collapse_bb
== NULL
)
6814 loop
->latch
= cont_bb
;
6815 add_loop (loop
, trip_loop
);
6820 /* A subroutine of expand_omp_for. Generate code for _Cilk_for loop.
6822 for (V = N1; V cond N2; V += STEP) BODY;
6824 where COND is "<" or ">" or "!=", we generate pseudocode
6826 for (ind_var = low; ind_var < high; ind_var++)
6828 V = n1 + (ind_var * STEP)
6833 In the above pseudocode, low and high are function parameters of the
6834 child function. In the function below, we are inserting a temp.
6835 variable that will be making a call to two OMP functions that will not be
6836 found in the body of _Cilk_for (since OMP_FOR cannot be mixed
6837 with _Cilk_for). These functions are replaced with low and high
6838 by the function that handles taskreg. */
6842 expand_cilk_for (struct omp_region
*region
, struct omp_for_data
*fd
)
6844 bool broken_loop
= region
->cont
== NULL
;
6845 basic_block entry_bb
= region
->entry
;
6846 basic_block cont_bb
= region
->cont
;
6848 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
6849 gcc_assert (broken_loop
6850 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
6851 basic_block l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
6852 basic_block l1_bb
, l2_bb
;
6856 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
6857 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
6858 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
6859 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
6863 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
6864 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
6865 l2_bb
= single_succ (l1_bb
);
6867 basic_block exit_bb
= region
->exit
;
6868 basic_block l2_dom_bb
= NULL
;
6870 gimple_stmt_iterator gsi
= gsi_last_bb (entry_bb
);
6872 /* Below statements until the "tree high_val = ..." are pseudo statements
6873 used to pass information to be used by expand_omp_taskreg.
6874 low_val and high_val will be replaced by the __low and __high
6875 parameter from the child function.
6877 The call_exprs part is a place-holder, it is mainly used
6878 to distinctly identify to the top-level part that this is
6879 where we should put low and high (reasoning given in header
6883 = gimple_omp_parallel_child_fn (last_stmt (region
->outer
->entry
));
6884 tree t
, low_val
= NULL_TREE
, high_val
= NULL_TREE
;
6885 for (t
= DECL_ARGUMENTS (child_fndecl
); t
; t
= TREE_CHAIN (t
))
6887 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__high"))
6889 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t
)), "__low"))
6892 gcc_assert (low_val
&& high_val
);
6894 tree type
= TREE_TYPE (low_val
);
6895 tree ind_var
= create_tmp_reg (type
, "__cilk_ind_var");
6896 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
6898 /* Not needed in SSA form right now. */
6899 gcc_assert (!gimple_in_ssa_p (cfun
));
6900 if (l2_dom_bb
== NULL
)
6906 gimple stmt
= gimple_build_assign (ind_var
, n1
);
6908 /* Replace the GIMPLE_OMP_FOR statement. */
6909 gsi_replace (&gsi
, stmt
, true);
6913 /* Code to control the increment goes in the CONT_BB. */
6914 gsi
= gsi_last_bb (cont_bb
);
6915 stmt
= gsi_stmt (gsi
);
6916 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
6917 stmt
= gimple_build_assign_with_ops (PLUS_EXPR
, ind_var
, ind_var
,
6918 build_one_cst (type
));
6920 /* Replace GIMPLE_OMP_CONTINUE. */
6921 gsi_replace (&gsi
, stmt
, true);
6924 /* Emit the condition in L1_BB. */
6925 gsi
= gsi_after_labels (l1_bb
);
6926 t
= fold_build2 (MULT_EXPR
, TREE_TYPE (fd
->loop
.step
),
6927 fold_convert (TREE_TYPE (fd
->loop
.step
), ind_var
),
6929 if (POINTER_TYPE_P (TREE_TYPE (fd
->loop
.n1
)))
6930 t
= fold_build2 (POINTER_PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
6931 fd
->loop
.n1
, fold_convert (sizetype
, t
));
6933 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loop
.n1
),
6934 fd
->loop
.n1
, fold_convert (TREE_TYPE (fd
->loop
.n1
), t
));
6935 t
= fold_convert (TREE_TYPE (fd
->loop
.v
), t
);
6936 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
6938 /* The condition is always '<' since the runtime will fill in the low
6940 stmt
= gimple_build_cond (LT_EXPR
, ind_var
, n2
, NULL_TREE
, NULL_TREE
);
6941 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
6943 /* Remove GIMPLE_OMP_RETURN. */
6944 gsi
= gsi_last_bb (exit_bb
);
6945 gsi_remove (&gsi
, true);
6947 /* Connect the new blocks. */
6948 remove_edge (FALLTHRU_EDGE (entry_bb
));
6953 remove_edge (BRANCH_EDGE (entry_bb
));
6954 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
6956 e
= BRANCH_EDGE (l1_bb
);
6957 ne
= FALLTHRU_EDGE (l1_bb
);
6958 e
->flags
= EDGE_TRUE_VALUE
;
6962 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
6964 ne
= single_succ_edge (l1_bb
);
6965 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
6968 ne
->flags
= EDGE_FALSE_VALUE
;
6969 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
6970 ne
->probability
= REG_BR_PROB_BASE
/ 8;
6972 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
6973 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
6974 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
6978 struct loop
*loop
= alloc_loop ();
6979 loop
->header
= l1_bb
;
6980 loop
->latch
= cont_bb
;
6981 add_loop (loop
, l1_bb
->loop_father
);
6982 loop
->safelen
= INT_MAX
;
6985 /* Pick the correct library function based on the precision of the
6986 induction variable type. */
6987 tree lib_fun
= NULL_TREE
;
6988 if (TYPE_PRECISION (type
) == 32)
6989 lib_fun
= cilk_for_32_fndecl
;
6990 else if (TYPE_PRECISION (type
) == 64)
6991 lib_fun
= cilk_for_64_fndecl
;
6995 gcc_assert (fd
->sched_kind
== OMP_CLAUSE_SCHEDULE_CILKFOR
);
6997 /* WS_ARGS contains the library function flavor to call:
6998 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the
6999 user-defined grain value. If the user does not define one, then zero
7000 is passed in by the parser. */
7001 vec_alloc (region
->ws_args
, 2);
7002 region
->ws_args
->quick_push (lib_fun
);
7003 region
->ws_args
->quick_push (fd
->chunk_size
);
7006 /* A subroutine of expand_omp_for. Generate code for a simd non-worksharing
7007 loop. Given parameters:
7009 for (V = N1; V cond N2; V += STEP) BODY;
7011 where COND is "<" or ">", we generate pseudocode
7019 if (V cond N2) goto L0; else goto L2;
7022 For collapsed loops, given parameters:
7024 for (V1 = N11; V1 cond1 N12; V1 += STEP1)
7025 for (V2 = N21; V2 cond2 N22; V2 += STEP2)
7026 for (V3 = N31; V3 cond3 N32; V3 += STEP3)
7029 we generate pseudocode
7035 count3 = (adj + N32 - N31) / STEP3;
7040 count2 = (adj + N22 - N21) / STEP2;
7045 count1 = (adj + N12 - N11) / STEP1;
7046 count = count1 * count2 * count3;
7056 V2 += (V3 cond3 N32) ? 0 : STEP2;
7057 V3 = (V3 cond3 N32) ? V3 : N31;
7058 V1 += (V2 cond2 N22) ? 0 : STEP1;
7059 V2 = (V2 cond2 N22) ? V2 : N21;
7061 if (V < count) goto L0; else goto L2;
7067 expand_omp_simd (struct omp_region
*region
, struct omp_for_data
*fd
)
7070 basic_block entry_bb
, cont_bb
, exit_bb
, l0_bb
, l1_bb
, l2_bb
, l2_dom_bb
;
7071 gimple_stmt_iterator gsi
;
7073 bool broken_loop
= region
->cont
== NULL
;
7075 tree
*counts
= NULL
;
7077 tree safelen
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7078 OMP_CLAUSE_SAFELEN
);
7079 tree simduid
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7080 OMP_CLAUSE__SIMDUID_
);
7083 type
= TREE_TYPE (fd
->loop
.v
);
7084 entry_bb
= region
->entry
;
7085 cont_bb
= region
->cont
;
7086 gcc_assert (EDGE_COUNT (entry_bb
->succs
) == 2);
7087 gcc_assert (broken_loop
7088 || BRANCH_EDGE (entry_bb
)->dest
== FALLTHRU_EDGE (cont_bb
)->dest
);
7089 l0_bb
= FALLTHRU_EDGE (entry_bb
)->dest
;
7092 gcc_assert (BRANCH_EDGE (cont_bb
)->dest
== l0_bb
);
7093 gcc_assert (EDGE_COUNT (cont_bb
->succs
) == 2);
7094 l1_bb
= split_block (cont_bb
, last_stmt (cont_bb
))->dest
;
7095 l2_bb
= BRANCH_EDGE (entry_bb
)->dest
;
7099 BRANCH_EDGE (entry_bb
)->flags
&= ~EDGE_ABNORMAL
;
7100 l1_bb
= split_edge (BRANCH_EDGE (entry_bb
));
7101 l2_bb
= single_succ (l1_bb
);
7103 exit_bb
= region
->exit
;
7106 gsi
= gsi_last_bb (entry_bb
);
7108 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_FOR
);
7109 /* Not needed in SSA form right now. */
7110 gcc_assert (!gimple_in_ssa_p (cfun
));
7111 if (fd
->collapse
> 1)
7113 int first_zero_iter
= -1;
7114 basic_block zero_iter_bb
= l2_bb
;
7116 counts
= XALLOCAVEC (tree
, fd
->collapse
);
7117 expand_omp_for_init_counts (fd
, &gsi
, entry_bb
, counts
,
7118 zero_iter_bb
, first_zero_iter
,
7121 if (l2_dom_bb
== NULL
)
7126 if (gimple_omp_for_combined_into_p (fd
->for_stmt
))
7128 tree innerc
= find_omp_clause (gimple_omp_for_clauses (fd
->for_stmt
),
7129 OMP_CLAUSE__LOOPTEMP_
);
7130 gcc_assert (innerc
);
7131 n1
= OMP_CLAUSE_DECL (innerc
);
7132 innerc
= find_omp_clause (OMP_CLAUSE_CHAIN (innerc
),
7133 OMP_CLAUSE__LOOPTEMP_
);
7134 gcc_assert (innerc
);
7135 n2
= OMP_CLAUSE_DECL (innerc
);
7136 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7137 fold_convert (type
, n1
));
7138 if (fd
->collapse
> 1)
7141 expand_omp_for_init_vars (fd
, &gsi
, counts
, NULL
, n1
);
7147 expand_omp_build_assign (&gsi
, fd
->loop
.v
,
7148 fold_convert (type
, fd
->loop
.n1
));
7149 if (fd
->collapse
> 1)
7150 for (i
= 0; i
< fd
->collapse
; i
++)
7152 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7153 if (POINTER_TYPE_P (itype
))
7154 itype
= signed_type_for (itype
);
7155 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
), fd
->loops
[i
].n1
);
7156 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7160 /* Remove the GIMPLE_OMP_FOR statement. */
7161 gsi_remove (&gsi
, true);
7165 /* Code to control the increment goes in the CONT_BB. */
7166 gsi
= gsi_last_bb (cont_bb
);
7167 stmt
= gsi_stmt (gsi
);
7168 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_CONTINUE
);
7170 if (POINTER_TYPE_P (type
))
7171 t
= fold_build_pointer_plus (fd
->loop
.v
, fd
->loop
.step
);
7173 t
= fold_build2 (PLUS_EXPR
, type
, fd
->loop
.v
, fd
->loop
.step
);
7174 expand_omp_build_assign (&gsi
, fd
->loop
.v
, t
);
7176 if (fd
->collapse
> 1)
7178 i
= fd
->collapse
- 1;
7179 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
].v
)))
7181 t
= fold_convert (sizetype
, fd
->loops
[i
].step
);
7182 t
= fold_build_pointer_plus (fd
->loops
[i
].v
, t
);
7186 t
= fold_convert (TREE_TYPE (fd
->loops
[i
].v
),
7188 t
= fold_build2 (PLUS_EXPR
, TREE_TYPE (fd
->loops
[i
].v
),
7191 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7193 for (i
= fd
->collapse
- 1; i
> 0; i
--)
7195 tree itype
= TREE_TYPE (fd
->loops
[i
].v
);
7196 tree itype2
= TREE_TYPE (fd
->loops
[i
- 1].v
);
7197 if (POINTER_TYPE_P (itype2
))
7198 itype2
= signed_type_for (itype2
);
7199 t
= build3 (COND_EXPR
, itype2
,
7200 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7202 fold_convert (itype
, fd
->loops
[i
].n2
)),
7203 build_int_cst (itype2
, 0),
7204 fold_convert (itype2
, fd
->loops
[i
- 1].step
));
7205 if (POINTER_TYPE_P (TREE_TYPE (fd
->loops
[i
- 1].v
)))
7206 t
= fold_build_pointer_plus (fd
->loops
[i
- 1].v
, t
);
7208 t
= fold_build2 (PLUS_EXPR
, itype2
, fd
->loops
[i
- 1].v
, t
);
7209 expand_omp_build_assign (&gsi
, fd
->loops
[i
- 1].v
, t
);
7211 t
= build3 (COND_EXPR
, itype
,
7212 build2 (fd
->loops
[i
].cond_code
, boolean_type_node
,
7214 fold_convert (itype
, fd
->loops
[i
].n2
)),
7216 fold_convert (itype
, fd
->loops
[i
].n1
));
7217 expand_omp_build_assign (&gsi
, fd
->loops
[i
].v
, t
);
7221 /* Remove GIMPLE_OMP_CONTINUE. */
7222 gsi_remove (&gsi
, true);
7225 /* Emit the condition in L1_BB. */
7226 gsi
= gsi_start_bb (l1_bb
);
7228 t
= fold_convert (type
, n2
);
7229 t
= force_gimple_operand_gsi (&gsi
, t
, true, NULL_TREE
,
7230 false, GSI_CONTINUE_LINKING
);
7231 t
= build2 (fd
->loop
.cond_code
, boolean_type_node
, fd
->loop
.v
, t
);
7232 stmt
= gimple_build_cond_empty (t
);
7233 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
7234 if (walk_tree (gimple_cond_lhs_ptr (stmt
), expand_omp_regimplify_p
,
7236 || walk_tree (gimple_cond_rhs_ptr (stmt
), expand_omp_regimplify_p
,
7239 gsi
= gsi_for_stmt (stmt
);
7240 gimple_regimplify_operands (stmt
, &gsi
);
7243 /* Remove GIMPLE_OMP_RETURN. */
7244 gsi
= gsi_last_bb (exit_bb
);
7245 gsi_remove (&gsi
, true);
7247 /* Connect the new blocks. */
7248 remove_edge (FALLTHRU_EDGE (entry_bb
));
7252 remove_edge (BRANCH_EDGE (entry_bb
));
7253 make_edge (entry_bb
, l1_bb
, EDGE_FALLTHRU
);
7255 e
= BRANCH_EDGE (l1_bb
);
7256 ne
= FALLTHRU_EDGE (l1_bb
);
7257 e
->flags
= EDGE_TRUE_VALUE
;
7261 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7263 ne
= single_succ_edge (l1_bb
);
7264 e
= make_edge (l1_bb
, l0_bb
, EDGE_TRUE_VALUE
);
7267 ne
->flags
= EDGE_FALSE_VALUE
;
7268 e
->probability
= REG_BR_PROB_BASE
* 7 / 8;
7269 ne
->probability
= REG_BR_PROB_BASE
/ 8;
7271 set_immediate_dominator (CDI_DOMINATORS
, l1_bb
, entry_bb
);
7272 set_immediate_dominator (CDI_DOMINATORS
, l2_bb
, l2_dom_bb
);
7273 set_immediate_dominator (CDI_DOMINATORS
, l0_bb
, l1_bb
);
7277 struct loop
*loop
= alloc_loop ();
7278 loop
->header
= l1_bb
;
7279 loop
->latch
= cont_bb
;
7280 add_loop (loop
, l1_bb
->loop_father
);
7281 if (safelen
== NULL_TREE
)
7282 loop
->safelen
= INT_MAX
;
7285 safelen
= OMP_CLAUSE_SAFELEN_EXPR (safelen
);
7286 if (TREE_CODE (safelen
) != INTEGER_CST
)
7288 else if (!tree_fits_uhwi_p (safelen
)
7289 || tree_to_uhwi (safelen
) > INT_MAX
)
7290 loop
->safelen
= INT_MAX
;
7292 loop
->safelen
= tree_to_uhwi (safelen
);
7293 if (loop
->safelen
== 1)
7298 loop
->simduid
= OMP_CLAUSE__SIMDUID__DECL (simduid
);
7299 cfun
->has_simduid_loops
= true;
7301 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
7303 if ((flag_tree_loop_vectorize
7304 || (!global_options_set
.x_flag_tree_loop_vectorize
7305 && !global_options_set
.x_flag_tree_vectorize
))
7306 && flag_tree_loop_optimize
7307 && loop
->safelen
> 1)
7309 loop
->force_vectorize
= true;
7310 cfun
->has_force_vectorize_loops
= true;
7316 /* Expand the OpenMP loop defined by REGION. */
7319 expand_omp_for (struct omp_region
*region
, gimple inner_stmt
)
7321 struct omp_for_data fd
;
7322 struct omp_for_data_loop
*loops
;
7325 = (struct omp_for_data_loop
*)
7326 alloca (gimple_omp_for_collapse (last_stmt (region
->entry
))
7327 * sizeof (struct omp_for_data_loop
));
7328 extract_omp_for_data (last_stmt (region
->entry
), &fd
, loops
);
7329 region
->sched_kind
= fd
.sched_kind
;
7331 gcc_assert (EDGE_COUNT (region
->entry
->succs
) == 2);
7332 BRANCH_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7333 FALLTHRU_EDGE (region
->entry
)->flags
&= ~EDGE_ABNORMAL
;
7336 gcc_assert (EDGE_COUNT (region
->cont
->succs
) == 2);
7337 BRANCH_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7338 FALLTHRU_EDGE (region
->cont
)->flags
&= ~EDGE_ABNORMAL
;
7341 /* If there isn't a continue then this is a degerate case where
7342 the introduction of abnormal edges during lowering will prevent
7343 original loops from being detected. Fix that up. */
7344 loops_state_set (LOOPS_NEED_FIXUP
);
7346 if (gimple_omp_for_kind (fd
.for_stmt
) & GF_OMP_FOR_SIMD
)
7347 expand_omp_simd (region
, &fd
);
7348 else if (gimple_omp_for_kind (fd
.for_stmt
) == GF_OMP_FOR_KIND_CILKFOR
)
7349 expand_cilk_for (region
, &fd
);
7350 else if (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
7351 && !fd
.have_ordered
)
7353 if (fd
.chunk_size
== NULL
)
7354 expand_omp_for_static_nochunk (region
, &fd
, inner_stmt
);
7356 expand_omp_for_static_chunk (region
, &fd
, inner_stmt
);
7360 int fn_index
, start_ix
, next_ix
;
7362 gcc_assert (gimple_omp_for_kind (fd
.for_stmt
)
7363 == GF_OMP_FOR_KIND_FOR
);
7364 if (fd
.chunk_size
== NULL
7365 && fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_STATIC
)
7366 fd
.chunk_size
= integer_zero_node
;
7367 gcc_assert (fd
.sched_kind
!= OMP_CLAUSE_SCHEDULE_AUTO
);
7368 fn_index
= (fd
.sched_kind
== OMP_CLAUSE_SCHEDULE_RUNTIME
)
7369 ? 3 : fd
.sched_kind
;
7370 fn_index
+= fd
.have_ordered
* 4;
7371 start_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_START
) + fn_index
;
7372 next_ix
= ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
) + fn_index
;
7373 if (fd
.iter_type
== long_long_unsigned_type_node
)
7375 start_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START
7376 - (int)BUILT_IN_GOMP_LOOP_STATIC_START
);
7377 next_ix
+= ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT
7378 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT
);
7380 expand_omp_for_generic (region
, &fd
, (enum built_in_function
) start_ix
,
7381 (enum built_in_function
) next_ix
, inner_stmt
);
7384 if (gimple_in_ssa_p (cfun
))
7385 update_ssa (TODO_update_ssa_only_virtuals
);
7389 /* Expand code for an OpenMP sections directive. In pseudo code, we generate
7391 v = GOMP_sections_start (n);
7408 v = GOMP_sections_next ();
7413 If this is a combined parallel sections, replace the call to
7414 GOMP_sections_start with call to GOMP_sections_next. */
7417 expand_omp_sections (struct omp_region
*region
)
7419 tree t
, u
, vin
= NULL
, vmain
, vnext
, l2
;
7421 basic_block entry_bb
, l0_bb
, l1_bb
, l2_bb
, default_bb
;
7422 gimple_stmt_iterator si
, switch_si
;
7423 gimple sections_stmt
, stmt
, cont
;
7426 struct omp_region
*inner
;
7428 bool exit_reachable
= region
->cont
!= NULL
;
7430 gcc_assert (region
->exit
!= NULL
);
7431 entry_bb
= region
->entry
;
7432 l0_bb
= single_succ (entry_bb
);
7433 l1_bb
= region
->cont
;
7434 l2_bb
= region
->exit
;
7435 if (single_pred_p (l2_bb
) && single_pred (l2_bb
) == l0_bb
)
7436 l2
= gimple_block_label (l2_bb
);
7439 /* This can happen if there are reductions. */
7440 len
= EDGE_COUNT (l0_bb
->succs
);
7441 gcc_assert (len
> 0);
7442 e
= EDGE_SUCC (l0_bb
, len
- 1);
7443 si
= gsi_last_bb (e
->dest
);
7446 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7447 l2
= gimple_block_label (e
->dest
);
7449 FOR_EACH_EDGE (e
, ei
, l0_bb
->succs
)
7451 si
= gsi_last_bb (e
->dest
);
7453 || gimple_code (gsi_stmt (si
)) != GIMPLE_OMP_SECTION
)
7455 l2
= gimple_block_label (e
->dest
);
7461 default_bb
= create_empty_bb (l1_bb
->prev_bb
);
7463 default_bb
= create_empty_bb (l0_bb
);
7465 /* We will build a switch() with enough cases for all the
7466 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work
7467 and a default case to abort if something goes wrong. */
7468 len
= EDGE_COUNT (l0_bb
->succs
);
7470 /* Use vec::quick_push on label_vec throughout, since we know the size
7472 auto_vec
<tree
> label_vec (len
);
7474 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the
7475 GIMPLE_OMP_SECTIONS statement. */
7476 si
= gsi_last_bb (entry_bb
);
7477 sections_stmt
= gsi_stmt (si
);
7478 gcc_assert (gimple_code (sections_stmt
) == GIMPLE_OMP_SECTIONS
);
7479 vin
= gimple_omp_sections_control (sections_stmt
);
7480 if (!is_combined_parallel (region
))
7482 /* If we are not inside a combined parallel+sections region,
7483 call GOMP_sections_start. */
7484 t
= build_int_cst (unsigned_type_node
, len
- 1);
7485 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START
);
7486 stmt
= gimple_build_call (u
, 1, t
);
7490 /* Otherwise, call GOMP_sections_next. */
7491 u
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7492 stmt
= gimple_build_call (u
, 0);
7494 gimple_call_set_lhs (stmt
, vin
);
7495 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7496 gsi_remove (&si
, true);
7498 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in
7500 switch_si
= gsi_last_bb (l0_bb
);
7501 gcc_assert (gimple_code (gsi_stmt (switch_si
)) == GIMPLE_OMP_SECTIONS_SWITCH
);
7504 cont
= last_stmt (l1_bb
);
7505 gcc_assert (gimple_code (cont
) == GIMPLE_OMP_CONTINUE
);
7506 vmain
= gimple_omp_continue_control_use (cont
);
7507 vnext
= gimple_omp_continue_control_def (cont
);
7515 t
= build_case_label (build_int_cst (unsigned_type_node
, 0), NULL
, l2
);
7516 label_vec
.quick_push (t
);
7519 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */
7520 for (inner
= region
->inner
, casei
= 1;
7522 inner
= inner
->next
, i
++, casei
++)
7524 basic_block s_entry_bb
, s_exit_bb
;
7526 /* Skip optional reduction region. */
7527 if (inner
->type
== GIMPLE_OMP_ATOMIC_LOAD
)
7534 s_entry_bb
= inner
->entry
;
7535 s_exit_bb
= inner
->exit
;
7537 t
= gimple_block_label (s_entry_bb
);
7538 u
= build_int_cst (unsigned_type_node
, casei
);
7539 u
= build_case_label (u
, NULL
, t
);
7540 label_vec
.quick_push (u
);
7542 si
= gsi_last_bb (s_entry_bb
);
7543 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SECTION
);
7544 gcc_assert (i
< len
|| gimple_omp_section_last_p (gsi_stmt (si
)));
7545 gsi_remove (&si
, true);
7546 single_succ_edge (s_entry_bb
)->flags
= EDGE_FALLTHRU
;
7548 if (s_exit_bb
== NULL
)
7551 si
= gsi_last_bb (s_exit_bb
);
7552 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7553 gsi_remove (&si
, true);
7555 single_succ_edge (s_exit_bb
)->flags
= EDGE_FALLTHRU
;
7558 /* Error handling code goes in DEFAULT_BB. */
7559 t
= gimple_block_label (default_bb
);
7560 u
= build_case_label (NULL
, NULL
, t
);
7561 make_edge (l0_bb
, default_bb
, 0);
7562 add_bb_to_loop (default_bb
, current_loops
->tree_root
);
7564 stmt
= gimple_build_switch (vmain
, u
, label_vec
);
7565 gsi_insert_after (&switch_si
, stmt
, GSI_SAME_STMT
);
7566 gsi_remove (&switch_si
, true);
7568 si
= gsi_start_bb (default_bb
);
7569 stmt
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP
), 0);
7570 gsi_insert_after (&si
, stmt
, GSI_CONTINUE_LINKING
);
7576 /* Code to get the next section goes in L1_BB. */
7577 si
= gsi_last_bb (l1_bb
);
7578 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CONTINUE
);
7580 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT
);
7581 stmt
= gimple_build_call (bfn_decl
, 0);
7582 gimple_call_set_lhs (stmt
, vnext
);
7583 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7584 gsi_remove (&si
, true);
7586 single_succ_edge (l1_bb
)->flags
= EDGE_FALLTHRU
;
7589 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */
7590 si
= gsi_last_bb (l2_bb
);
7591 if (gimple_omp_return_nowait_p (gsi_stmt (si
)))
7592 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT
);
7593 else if (gimple_omp_return_lhs (gsi_stmt (si
)))
7594 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL
);
7596 t
= builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END
);
7597 stmt
= gimple_build_call (t
, 0);
7598 if (gimple_omp_return_lhs (gsi_stmt (si
)))
7599 gimple_call_set_lhs (stmt
, gimple_omp_return_lhs (gsi_stmt (si
)));
7600 gsi_insert_after (&si
, stmt
, GSI_SAME_STMT
);
7601 gsi_remove (&si
, true);
7603 set_immediate_dominator (CDI_DOMINATORS
, default_bb
, l0_bb
);
7607 /* Expand code for an OpenMP single directive. We've already expanded
7608 much of the code, here we simply place the GOMP_barrier call. */
7611 expand_omp_single (struct omp_region
*region
)
7613 basic_block entry_bb
, exit_bb
;
7614 gimple_stmt_iterator si
;
7616 entry_bb
= region
->entry
;
7617 exit_bb
= region
->exit
;
7619 si
= gsi_last_bb (entry_bb
);
7620 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
);
7621 gsi_remove (&si
, true);
7622 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7624 si
= gsi_last_bb (exit_bb
);
7625 if (!gimple_omp_return_nowait_p (gsi_stmt (si
)))
7627 tree t
= gimple_omp_return_lhs (gsi_stmt (si
));
7628 gsi_insert_after (&si
, build_omp_barrier (t
), GSI_SAME_STMT
);
7630 gsi_remove (&si
, true);
7631 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7635 /* Generic expansion for OpenMP synchronization directives: master,
7636 ordered and critical. All we need to do here is remove the entry
7637 and exit markers for REGION. */
7640 expand_omp_synch (struct omp_region
*region
)
7642 basic_block entry_bb
, exit_bb
;
7643 gimple_stmt_iterator si
;
7645 entry_bb
= region
->entry
;
7646 exit_bb
= region
->exit
;
7648 si
= gsi_last_bb (entry_bb
);
7649 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_SINGLE
7650 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_MASTER
7651 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TASKGROUP
7652 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ORDERED
7653 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_CRITICAL
7654 || gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_TEAMS
);
7655 gsi_remove (&si
, true);
7656 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
7660 si
= gsi_last_bb (exit_bb
);
7661 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_RETURN
);
7662 gsi_remove (&si
, true);
7663 single_succ_edge (exit_bb
)->flags
= EDGE_FALLTHRU
;
7667 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7668 operation as a normal volatile load. */
7671 expand_omp_atomic_load (basic_block load_bb
, tree addr
,
7672 tree loaded_val
, int index
)
7674 enum built_in_function tmpbase
;
7675 gimple_stmt_iterator gsi
;
7676 basic_block store_bb
;
7679 tree decl
, call
, type
, itype
;
7681 gsi
= gsi_last_bb (load_bb
);
7682 stmt
= gsi_stmt (gsi
);
7683 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7684 loc
= gimple_location (stmt
);
7686 /* ??? If the target does not implement atomic_load_optab[mode], and mode
7687 is smaller than word size, then expand_atomic_load assumes that the load
7688 is atomic. We could avoid the builtin entirely in this case. */
7690 tmpbase
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
7691 decl
= builtin_decl_explicit (tmpbase
);
7692 if (decl
== NULL_TREE
)
7695 type
= TREE_TYPE (loaded_val
);
7696 itype
= TREE_TYPE (TREE_TYPE (decl
));
7698 call
= build_call_expr_loc (loc
, decl
, 2, addr
,
7699 build_int_cst (NULL
,
7700 gimple_omp_atomic_seq_cst_p (stmt
)
7702 : MEMMODEL_RELAXED
));
7703 if (!useless_type_conversion_p (type
, itype
))
7704 call
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7705 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7707 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7708 gsi_remove (&gsi
, true);
7710 store_bb
= single_succ (load_bb
);
7711 gsi
= gsi_last_bb (store_bb
);
7712 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7713 gsi_remove (&gsi
, true);
7715 if (gimple_in_ssa_p (cfun
))
7716 update_ssa (TODO_update_ssa_no_phi
);
7721 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7722 operation as a normal volatile store. */
7725 expand_omp_atomic_store (basic_block load_bb
, tree addr
,
7726 tree loaded_val
, tree stored_val
, int index
)
7728 enum built_in_function tmpbase
;
7729 gimple_stmt_iterator gsi
;
7730 basic_block store_bb
= single_succ (load_bb
);
7733 tree decl
, call
, type
, itype
;
7734 enum machine_mode imode
;
7737 gsi
= gsi_last_bb (load_bb
);
7738 stmt
= gsi_stmt (gsi
);
7739 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_LOAD
);
7741 /* If the load value is needed, then this isn't a store but an exchange. */
7742 exchange
= gimple_omp_atomic_need_value_p (stmt
);
7744 gsi
= gsi_last_bb (store_bb
);
7745 stmt
= gsi_stmt (gsi
);
7746 gcc_assert (gimple_code (stmt
) == GIMPLE_OMP_ATOMIC_STORE
);
7747 loc
= gimple_location (stmt
);
7749 /* ??? If the target does not implement atomic_store_optab[mode], and mode
7750 is smaller than word size, then expand_atomic_store assumes that the store
7751 is atomic. We could avoid the builtin entirely in this case. */
7753 tmpbase
= (exchange
? BUILT_IN_ATOMIC_EXCHANGE_N
: BUILT_IN_ATOMIC_STORE_N
);
7754 tmpbase
= (enum built_in_function
) ((int) tmpbase
+ index
+ 1);
7755 decl
= builtin_decl_explicit (tmpbase
);
7756 if (decl
== NULL_TREE
)
7759 type
= TREE_TYPE (stored_val
);
7761 /* Dig out the type of the function's second argument. */
7762 itype
= TREE_TYPE (decl
);
7763 itype
= TYPE_ARG_TYPES (itype
);
7764 itype
= TREE_CHAIN (itype
);
7765 itype
= TREE_VALUE (itype
);
7766 imode
= TYPE_MODE (itype
);
7768 if (exchange
&& !can_atomic_exchange_p (imode
, true))
7771 if (!useless_type_conversion_p (itype
, type
))
7772 stored_val
= fold_build1_loc (loc
, VIEW_CONVERT_EXPR
, itype
, stored_val
);
7773 call
= build_call_expr_loc (loc
, decl
, 3, addr
, stored_val
,
7774 build_int_cst (NULL
,
7775 gimple_omp_atomic_seq_cst_p (stmt
)
7777 : MEMMODEL_RELAXED
));
7780 if (!useless_type_conversion_p (type
, itype
))
7781 call
= build1_loc (loc
, VIEW_CONVERT_EXPR
, type
, call
);
7782 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, loaded_val
, call
);
7785 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7786 gsi_remove (&gsi
, true);
7788 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */
7789 gsi
= gsi_last_bb (load_bb
);
7790 gsi_remove (&gsi
, true);
7792 if (gimple_in_ssa_p (cfun
))
7793 update_ssa (TODO_update_ssa_no_phi
);
7798 /* A subroutine of expand_omp_atomic. Attempt to implement the atomic
7799 operation as a __atomic_fetch_op builtin. INDEX is log2 of the
7800 size of the data type, and thus usable to find the index of the builtin
7801 decl. Returns false if the expression is not of the proper form. */
7804 expand_omp_atomic_fetch_op (basic_block load_bb
,
7805 tree addr
, tree loaded_val
,
7806 tree stored_val
, int index
)
7808 enum built_in_function oldbase
, newbase
, tmpbase
;
7809 tree decl
, itype
, call
;
7811 basic_block store_bb
= single_succ (load_bb
);
7812 gimple_stmt_iterator gsi
;
7815 enum tree_code code
;
7816 bool need_old
, need_new
;
7817 enum machine_mode imode
;
7820 /* We expect to find the following sequences:
7823 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem)
7826 val = tmp OP something; (or: something OP tmp)
7827 GIMPLE_OMP_STORE (val)
7829 ???FIXME: Allow a more flexible sequence.
7830 Perhaps use data flow to pick the statements.
7834 gsi
= gsi_after_labels (store_bb
);
7835 stmt
= gsi_stmt (gsi
);
7836 loc
= gimple_location (stmt
);
7837 if (!is_gimple_assign (stmt
))
7840 if (gimple_code (gsi_stmt (gsi
)) != GIMPLE_OMP_ATOMIC_STORE
)
7842 need_new
= gimple_omp_atomic_need_value_p (gsi_stmt (gsi
));
7843 need_old
= gimple_omp_atomic_need_value_p (last_stmt (load_bb
));
7844 seq_cst
= gimple_omp_atomic_seq_cst_p (last_stmt (load_bb
));
7845 gcc_checking_assert (!need_old
|| !need_new
);
7847 if (!operand_equal_p (gimple_assign_lhs (stmt
), stored_val
, 0))
7850 /* Check for one of the supported fetch-op operations. */
7851 code
= gimple_assign_rhs_code (stmt
);
7855 case POINTER_PLUS_EXPR
:
7856 oldbase
= BUILT_IN_ATOMIC_FETCH_ADD_N
;
7857 newbase
= BUILT_IN_ATOMIC_ADD_FETCH_N
;
7860 oldbase
= BUILT_IN_ATOMIC_FETCH_SUB_N
;
7861 newbase
= BUILT_IN_ATOMIC_SUB_FETCH_N
;
7864 oldbase
= BUILT_IN_ATOMIC_FETCH_AND_N
;
7865 newbase
= BUILT_IN_ATOMIC_AND_FETCH_N
;
7868 oldbase
= BUILT_IN_ATOMIC_FETCH_OR_N
;
7869 newbase
= BUILT_IN_ATOMIC_OR_FETCH_N
;
7872 oldbase
= BUILT_IN_ATOMIC_FETCH_XOR_N
;
7873 newbase
= BUILT_IN_ATOMIC_XOR_FETCH_N
;
7879 /* Make sure the expression is of the proper form. */
7880 if (operand_equal_p (gimple_assign_rhs1 (stmt
), loaded_val
, 0))
7881 rhs
= gimple_assign_rhs2 (stmt
);
7882 else if (commutative_tree_code (gimple_assign_rhs_code (stmt
))
7883 && operand_equal_p (gimple_assign_rhs2 (stmt
), loaded_val
, 0))
7884 rhs
= gimple_assign_rhs1 (stmt
);
7888 tmpbase
= ((enum built_in_function
)
7889 ((need_new
? newbase
: oldbase
) + index
+ 1));
7890 decl
= builtin_decl_explicit (tmpbase
);
7891 if (decl
== NULL_TREE
)
7893 itype
= TREE_TYPE (TREE_TYPE (decl
));
7894 imode
= TYPE_MODE (itype
);
7896 /* We could test all of the various optabs involved, but the fact of the
7897 matter is that (with the exception of i486 vs i586 and xadd) all targets
7898 that support any atomic operaton optab also implements compare-and-swap.
7899 Let optabs.c take care of expanding any compare-and-swap loop. */
7900 if (!can_compare_and_swap_p (imode
, true))
7903 gsi
= gsi_last_bb (load_bb
);
7904 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7906 /* OpenMP does not imply any barrier-like semantics on its atomic ops.
7907 It only requires that the operation happen atomically. Thus we can
7908 use the RELAXED memory model. */
7909 call
= build_call_expr_loc (loc
, decl
, 3, addr
,
7910 fold_convert_loc (loc
, itype
, rhs
),
7911 build_int_cst (NULL
,
7912 seq_cst
? MEMMODEL_SEQ_CST
7913 : MEMMODEL_RELAXED
));
7915 if (need_old
|| need_new
)
7917 lhs
= need_old
? loaded_val
: stored_val
;
7918 call
= fold_convert_loc (loc
, TREE_TYPE (lhs
), call
);
7919 call
= build2_loc (loc
, MODIFY_EXPR
, void_type_node
, lhs
, call
);
7922 call
= fold_convert_loc (loc
, void_type_node
, call
);
7923 force_gimple_operand_gsi (&gsi
, call
, true, NULL_TREE
, true, GSI_SAME_STMT
);
7924 gsi_remove (&gsi
, true);
7926 gsi
= gsi_last_bb (store_bb
);
7927 gcc_assert (gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_ATOMIC_STORE
);
7928 gsi_remove (&gsi
, true);
7929 gsi
= gsi_last_bb (store_bb
);
7930 gsi_remove (&gsi
, true);
7932 if (gimple_in_ssa_p (cfun
))
7933 update_ssa (TODO_update_ssa_no_phi
);
7938 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
7942 newval = rhs; // with oldval replacing *addr in rhs
7943 oldval = __sync_val_compare_and_swap (addr, oldval, newval);
7944 if (oldval != newval)
7947 INDEX is log2 of the size of the data type, and thus usable to find the
7948 index of the builtin decl. */
7951 expand_omp_atomic_pipeline (basic_block load_bb
, basic_block store_bb
,
7952 tree addr
, tree loaded_val
, tree stored_val
,
7955 tree loadedi
, storedi
, initial
, new_storedi
, old_vali
;
7956 tree type
, itype
, cmpxchg
, iaddr
;
7957 gimple_stmt_iterator si
;
7958 basic_block loop_header
= single_succ (load_bb
);
7961 enum built_in_function fncode
;
7963 /* ??? We need a non-pointer interface to __atomic_compare_exchange in
7964 order to use the RELAXED memory model effectively. */
7965 fncode
= (enum built_in_function
)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N
7967 cmpxchg
= builtin_decl_explicit (fncode
);
7968 if (cmpxchg
== NULL_TREE
)
7970 type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
7971 itype
= TREE_TYPE (TREE_TYPE (cmpxchg
));
7973 if (!can_compare_and_swap_p (TYPE_MODE (itype
), true))
7976 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */
7977 si
= gsi_last_bb (load_bb
);
7978 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
7980 /* For floating-point values, we'll need to view-convert them to integers
7981 so that we can perform the atomic compare and swap. Simplify the
7982 following code by always setting up the "i"ntegral variables. */
7983 if (!INTEGRAL_TYPE_P (type
) && !POINTER_TYPE_P (type
))
7987 iaddr
= create_tmp_reg (build_pointer_type_for_mode (itype
, ptr_mode
,
7990 = force_gimple_operand_gsi (&si
,
7991 fold_convert (TREE_TYPE (iaddr
), addr
),
7992 false, NULL_TREE
, true, GSI_SAME_STMT
);
7993 stmt
= gimple_build_assign (iaddr
, iaddr_val
);
7994 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
7995 loadedi
= create_tmp_var (itype
, NULL
);
7996 if (gimple_in_ssa_p (cfun
))
7997 loadedi
= make_ssa_name (loadedi
, NULL
);
8002 loadedi
= loaded_val
;
8005 fncode
= (enum built_in_function
) (BUILT_IN_ATOMIC_LOAD_N
+ index
+ 1);
8006 tree loaddecl
= builtin_decl_explicit (fncode
);
8009 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr
)),
8010 build_call_expr (loaddecl
, 2, iaddr
,
8011 build_int_cst (NULL_TREE
,
8012 MEMMODEL_RELAXED
)));
8014 initial
= build2 (MEM_REF
, TREE_TYPE (TREE_TYPE (iaddr
)), iaddr
,
8015 build_int_cst (TREE_TYPE (iaddr
), 0));
8018 = force_gimple_operand_gsi (&si
, initial
, true, NULL_TREE
, true,
8021 /* Move the value to the LOADEDI temporary. */
8022 if (gimple_in_ssa_p (cfun
))
8024 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header
)));
8025 phi
= create_phi_node (loadedi
, loop_header
);
8026 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, single_succ_edge (load_bb
)),
8030 gsi_insert_before (&si
,
8031 gimple_build_assign (loadedi
, initial
),
8033 if (loadedi
!= loaded_val
)
8035 gimple_stmt_iterator gsi2
;
8038 x
= build1 (VIEW_CONVERT_EXPR
, type
, loadedi
);
8039 gsi2
= gsi_start_bb (loop_header
);
8040 if (gimple_in_ssa_p (cfun
))
8043 x
= force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8044 true, GSI_SAME_STMT
);
8045 stmt
= gimple_build_assign (loaded_val
, x
);
8046 gsi_insert_before (&gsi2
, stmt
, GSI_SAME_STMT
);
8050 x
= build2 (MODIFY_EXPR
, TREE_TYPE (loaded_val
), loaded_val
, x
);
8051 force_gimple_operand_gsi (&gsi2
, x
, true, NULL_TREE
,
8052 true, GSI_SAME_STMT
);
8055 gsi_remove (&si
, true);
8057 si
= gsi_last_bb (store_bb
);
8058 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8061 storedi
= stored_val
;
8064 force_gimple_operand_gsi (&si
,
8065 build1 (VIEW_CONVERT_EXPR
, itype
,
8066 stored_val
), true, NULL_TREE
, true,
8069 /* Build the compare&swap statement. */
8070 new_storedi
= build_call_expr (cmpxchg
, 3, iaddr
, loadedi
, storedi
);
8071 new_storedi
= force_gimple_operand_gsi (&si
,
8072 fold_convert (TREE_TYPE (loadedi
),
8075 true, GSI_SAME_STMT
);
8077 if (gimple_in_ssa_p (cfun
))
8081 old_vali
= create_tmp_var (TREE_TYPE (loadedi
), NULL
);
8082 stmt
= gimple_build_assign (old_vali
, loadedi
);
8083 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8085 stmt
= gimple_build_assign (loadedi
, new_storedi
);
8086 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8089 /* Note that we always perform the comparison as an integer, even for
8090 floating point. This allows the atomic operation to properly
8091 succeed even with NaNs and -0.0. */
8092 stmt
= gimple_build_cond_empty
8093 (build2 (NE_EXPR
, boolean_type_node
,
8094 new_storedi
, old_vali
));
8095 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8098 e
= single_succ_edge (store_bb
);
8099 e
->flags
&= ~EDGE_FALLTHRU
;
8100 e
->flags
|= EDGE_FALSE_VALUE
;
8102 e
= make_edge (store_bb
, loop_header
, EDGE_TRUE_VALUE
);
8104 /* Copy the new value to loadedi (we already did that before the condition
8105 if we are not in SSA). */
8106 if (gimple_in_ssa_p (cfun
))
8108 phi
= gimple_seq_first_stmt (phi_nodes (loop_header
));
8109 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi
, e
), new_storedi
);
8112 /* Remove GIMPLE_OMP_ATOMIC_STORE. */
8113 gsi_remove (&si
, true);
8115 struct loop
*loop
= alloc_loop ();
8116 loop
->header
= loop_header
;
8117 loop
->latch
= store_bb
;
8118 add_loop (loop
, loop_header
->loop_father
);
8120 if (gimple_in_ssa_p (cfun
))
8121 update_ssa (TODO_update_ssa_no_phi
);
8126 /* A subroutine of expand_omp_atomic. Implement the atomic operation as:
8128 GOMP_atomic_start ();
8132 The result is not globally atomic, but works so long as all parallel
8133 references are within #pragma omp atomic directives. According to
8134 responses received from omp@openmp.org, appears to be within spec.
8135 Which makes sense, since that's how several other compilers handle
8136 this situation as well.
8137 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're
8138 expanding. STORED_VAL is the operand of the matching
8139 GIMPLE_OMP_ATOMIC_STORE.
8142 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with
8146 GIMPLE_OMP_ATOMIC_STORE (stored_val) with
8151 expand_omp_atomic_mutex (basic_block load_bb
, basic_block store_bb
,
8152 tree addr
, tree loaded_val
, tree stored_val
)
8154 gimple_stmt_iterator si
;
8158 si
= gsi_last_bb (load_bb
);
8159 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_LOAD
);
8161 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START
);
8162 t
= build_call_expr (t
, 0);
8163 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8165 stmt
= gimple_build_assign (loaded_val
, build_simple_mem_ref (addr
));
8166 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8167 gsi_remove (&si
, true);
8169 si
= gsi_last_bb (store_bb
);
8170 gcc_assert (gimple_code (gsi_stmt (si
)) == GIMPLE_OMP_ATOMIC_STORE
);
8172 stmt
= gimple_build_assign (build_simple_mem_ref (unshare_expr (addr
)),
8174 gsi_insert_before (&si
, stmt
, GSI_SAME_STMT
);
8176 t
= builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END
);
8177 t
= build_call_expr (t
, 0);
8178 force_gimple_operand_gsi (&si
, t
, true, NULL_TREE
, true, GSI_SAME_STMT
);
8179 gsi_remove (&si
, true);
8181 if (gimple_in_ssa_p (cfun
))
8182 update_ssa (TODO_update_ssa_no_phi
);
8186 /* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand
8187 using expand_omp_atomic_fetch_op. If it failed, we try to
8188 call expand_omp_atomic_pipeline, and if it fails too, the
8189 ultimate fallback is wrapping the operation in a mutex
8190 (expand_omp_atomic_mutex). REGION is the atomic region built
8191 by build_omp_regions_1(). */
8194 expand_omp_atomic (struct omp_region
*region
)
8196 basic_block load_bb
= region
->entry
, store_bb
= region
->exit
;
8197 gimple load
= last_stmt (load_bb
), store
= last_stmt (store_bb
);
8198 tree loaded_val
= gimple_omp_atomic_load_lhs (load
);
8199 tree addr
= gimple_omp_atomic_load_rhs (load
);
8200 tree stored_val
= gimple_omp_atomic_store_val (store
);
8201 tree type
= TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr
)));
8202 HOST_WIDE_INT index
;
8204 /* Make sure the type is one of the supported sizes. */
8205 index
= tree_to_uhwi (TYPE_SIZE_UNIT (type
));
8206 index
= exact_log2 (index
);
8207 if (index
>= 0 && index
<= 4)
8209 unsigned int align
= TYPE_ALIGN_UNIT (type
);
8211 /* __sync builtins require strict data alignment. */
8212 if (exact_log2 (align
) >= index
)
8215 if (loaded_val
== stored_val
8216 && (GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8217 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8218 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8219 && expand_omp_atomic_load (load_bb
, addr
, loaded_val
, index
))
8223 if ((GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_INT
8224 || GET_MODE_CLASS (TYPE_MODE (type
)) == MODE_FLOAT
)
8225 && GET_MODE_BITSIZE (TYPE_MODE (type
)) <= BITS_PER_WORD
8226 && store_bb
== single_succ (load_bb
)
8227 && first_stmt (store_bb
) == store
8228 && expand_omp_atomic_store (load_bb
, addr
, loaded_val
,
8232 /* When possible, use specialized atomic update functions. */
8233 if ((INTEGRAL_TYPE_P (type
) || POINTER_TYPE_P (type
))
8234 && store_bb
== single_succ (load_bb
)
8235 && expand_omp_atomic_fetch_op (load_bb
, addr
,
8236 loaded_val
, stored_val
, index
))
8239 /* If we don't have specialized __sync builtins, try and implement
8240 as a compare and swap loop. */
8241 if (expand_omp_atomic_pipeline (load_bb
, store_bb
, addr
,
8242 loaded_val
, stored_val
, index
))
8247 /* The ultimate fallback is wrapping the operation in a mutex. */
8248 expand_omp_atomic_mutex (load_bb
, store_bb
, addr
, loaded_val
, stored_val
);
8252 /* Expand the OpenMP target{, data, update} directive starting at REGION. */
8255 expand_omp_target (struct omp_region
*region
)
8257 basic_block entry_bb
, exit_bb
, new_bb
;
8258 struct function
*child_cfun
= NULL
;
8259 tree child_fn
= NULL_TREE
, block
, t
;
8260 gimple_stmt_iterator gsi
;
8261 gimple entry_stmt
, stmt
;
8264 entry_stmt
= last_stmt (region
->entry
);
8265 new_bb
= region
->entry
;
8266 int kind
= gimple_omp_target_kind (entry_stmt
);
8267 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8269 child_fn
= gimple_omp_target_child_fn (entry_stmt
);
8270 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
8273 entry_bb
= region
->entry
;
8274 exit_bb
= region
->exit
;
8276 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8278 unsigned srcidx
, dstidx
, num
;
8280 /* If the target region needs data sent from the parent
8281 function, then the very first statement (except possible
8282 tree profile counter updates) of the parallel body
8283 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since
8284 &.OMP_DATA_O is passed as an argument to the child function,
8285 we need to replace it with the argument as seen by the child
8288 In most cases, this will end up being the identity assignment
8289 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had
8290 a function call that has been inlined, the original PARM_DECL
8291 .OMP_DATA_I may have been converted into a different local
8292 variable. In which case, we need to keep the assignment. */
8293 if (gimple_omp_target_data_arg (entry_stmt
))
8295 basic_block entry_succ_bb
= single_succ (entry_bb
);
8296 gimple_stmt_iterator gsi
;
8298 gimple tgtcopy_stmt
= NULL
;
8300 = TREE_VEC_ELT (gimple_omp_target_data_arg (entry_stmt
), 0);
8302 for (gsi
= gsi_start_bb (entry_succ_bb
); ; gsi_next (&gsi
))
8304 gcc_assert (!gsi_end_p (gsi
));
8305 stmt
= gsi_stmt (gsi
);
8306 if (gimple_code (stmt
) != GIMPLE_ASSIGN
)
8309 if (gimple_num_ops (stmt
) == 2)
8311 tree arg
= gimple_assign_rhs1 (stmt
);
8313 /* We're ignoring the subcode because we're
8314 effectively doing a STRIP_NOPS. */
8316 if (TREE_CODE (arg
) == ADDR_EXPR
8317 && TREE_OPERAND (arg
, 0) == sender
)
8319 tgtcopy_stmt
= stmt
;
8325 gcc_assert (tgtcopy_stmt
!= NULL
);
8326 arg
= DECL_ARGUMENTS (child_fn
);
8328 gcc_assert (gimple_assign_lhs (tgtcopy_stmt
) == arg
);
8329 gsi_remove (&gsi
, true);
8332 /* Declare local variables needed in CHILD_CFUN. */
8333 block
= DECL_INITIAL (child_fn
);
8334 BLOCK_VARS (block
) = vec2chain (child_cfun
->local_decls
);
8335 /* The gimplifier could record temporaries in target block
8336 rather than in containing function's local_decls chain,
8337 which would mean cgraph missed finalizing them. Do it now. */
8338 for (t
= BLOCK_VARS (block
); t
; t
= DECL_CHAIN (t
))
8339 if (TREE_CODE (t
) == VAR_DECL
8341 && !DECL_EXTERNAL (t
))
8342 varpool_node::finalize_decl (t
);
8343 DECL_SAVED_TREE (child_fn
) = NULL
;
8344 /* We'll create a CFG for child_fn, so no gimple body is needed. */
8345 gimple_set_body (child_fn
, NULL
);
8346 TREE_USED (block
) = 1;
8348 /* Reset DECL_CONTEXT on function arguments. */
8349 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
8350 DECL_CONTEXT (t
) = child_fn
;
8352 /* Split ENTRY_BB at GIMPLE_OMP_TARGET,
8353 so that it can be moved to the child function. */
8354 gsi
= gsi_last_bb (entry_bb
);
8355 stmt
= gsi_stmt (gsi
);
8356 gcc_assert (stmt
&& gimple_code (stmt
) == GIMPLE_OMP_TARGET
8357 && gimple_omp_target_kind (stmt
)
8358 == GF_OMP_TARGET_KIND_REGION
);
8359 gsi_remove (&gsi
, true);
8360 e
= split_block (entry_bb
, stmt
);
8362 single_succ_edge (entry_bb
)->flags
= EDGE_FALLTHRU
;
8364 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */
8367 gsi
= gsi_last_bb (exit_bb
);
8368 gcc_assert (!gsi_end_p (gsi
)
8369 && gimple_code (gsi_stmt (gsi
)) == GIMPLE_OMP_RETURN
);
8370 stmt
= gimple_build_return (NULL
);
8371 gsi_insert_after (&gsi
, stmt
, GSI_SAME_STMT
);
8372 gsi_remove (&gsi
, true);
8375 /* Move the target region into CHILD_CFUN. */
8377 block
= gimple_block (entry_stmt
);
8379 new_bb
= move_sese_region_to_fn (child_cfun
, entry_bb
, exit_bb
, block
);
8381 single_succ_edge (new_bb
)->flags
= EDGE_FALLTHRU
;
8382 /* When the OMP expansion process cannot guarantee an up-to-date
8383 loop tree arrange for the child function to fixup loops. */
8384 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP
))
8385 child_cfun
->x_current_loops
->state
|= LOOPS_NEED_FIXUP
;
8387 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */
8388 num
= vec_safe_length (child_cfun
->local_decls
);
8389 for (srcidx
= 0, dstidx
= 0; srcidx
< num
; srcidx
++)
8391 t
= (*child_cfun
->local_decls
)[srcidx
];
8392 if (DECL_CONTEXT (t
) == cfun
->decl
)
8394 if (srcidx
!= dstidx
)
8395 (*child_cfun
->local_decls
)[dstidx
] = t
;
8399 vec_safe_truncate (child_cfun
->local_decls
, dstidx
);
8401 /* Inform the callgraph about the new function. */
8402 DECL_STRUCT_FUNCTION (child_fn
)->curr_properties
= cfun
->curr_properties
;
8403 cgraph_node::add_new_function (child_fn
, true);
8405 /* Fix the callgraph edges for child_cfun. Those for cfun will be
8406 fixed in a following pass. */
8407 push_cfun (child_cfun
);
8408 cgraph_edge::rebuild_edges ();
8410 /* Some EH regions might become dead, see PR34608. If
8411 pass_cleanup_cfg isn't the first pass to happen with the
8412 new child, these dead EH edges might cause problems.
8413 Clean them up now. */
8414 if (flag_exceptions
)
8417 bool changed
= false;
8419 FOR_EACH_BB_FN (bb
, cfun
)
8420 changed
|= gimple_purge_dead_eh_edges (bb
);
8422 cleanup_tree_cfg ();
8427 /* Emit a library call to launch the target region, or do data
8429 tree t1
, t2
, t3
, t4
, device
, cond
, c
, clauses
;
8430 enum built_in_function start_ix
;
8431 location_t clause_loc
;
8433 clauses
= gimple_omp_target_clauses (entry_stmt
);
8435 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8436 start_ix
= BUILT_IN_GOMP_TARGET
;
8437 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
8438 start_ix
= BUILT_IN_GOMP_TARGET_DATA
;
8440 start_ix
= BUILT_IN_GOMP_TARGET_UPDATE
;
8442 /* By default, the value of DEVICE is -1 (let runtime library choose)
8443 and there is no conditional. */
8445 device
= build_int_cst (integer_type_node
, -1);
8447 c
= find_omp_clause (clauses
, OMP_CLAUSE_IF
);
8449 cond
= OMP_CLAUSE_IF_EXPR (c
);
8451 c
= find_omp_clause (clauses
, OMP_CLAUSE_DEVICE
);
8454 device
= OMP_CLAUSE_DEVICE_ID (c
);
8455 clause_loc
= OMP_CLAUSE_LOCATION (c
);
8458 clause_loc
= gimple_location (entry_stmt
);
8460 /* Ensure 'device' is of the correct type. */
8461 device
= fold_convert_loc (clause_loc
, integer_type_node
, device
);
8463 /* If we found the clause 'if (cond)', build
8464 (cond ? device : -2). */
8467 cond
= gimple_boolify (cond
);
8469 basic_block cond_bb
, then_bb
, else_bb
;
8473 tmp_var
= create_tmp_var (TREE_TYPE (device
), NULL
);
8474 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8476 gsi
= gsi_last_bb (new_bb
);
8478 e
= split_block (new_bb
, gsi_stmt (gsi
));
8481 e
= split_block (new_bb
, NULL
);
8486 then_bb
= create_empty_bb (cond_bb
);
8487 else_bb
= create_empty_bb (then_bb
);
8488 set_immediate_dominator (CDI_DOMINATORS
, then_bb
, cond_bb
);
8489 set_immediate_dominator (CDI_DOMINATORS
, else_bb
, cond_bb
);
8491 stmt
= gimple_build_cond_empty (cond
);
8492 gsi
= gsi_last_bb (cond_bb
);
8493 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8495 gsi
= gsi_start_bb (then_bb
);
8496 stmt
= gimple_build_assign (tmp_var
, device
);
8497 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8499 gsi
= gsi_start_bb (else_bb
);
8500 stmt
= gimple_build_assign (tmp_var
,
8501 build_int_cst (integer_type_node
, -2));
8502 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
8504 make_edge (cond_bb
, then_bb
, EDGE_TRUE_VALUE
);
8505 make_edge (cond_bb
, else_bb
, EDGE_FALSE_VALUE
);
8506 add_bb_to_loop (then_bb
, cond_bb
->loop_father
);
8507 add_bb_to_loop (else_bb
, cond_bb
->loop_father
);
8508 make_edge (then_bb
, new_bb
, EDGE_FALLTHRU
);
8509 make_edge (else_bb
, new_bb
, EDGE_FALLTHRU
);
8514 gsi
= gsi_last_bb (new_bb
);
8515 t
= gimple_omp_target_data_arg (entry_stmt
);
8518 t1
= size_zero_node
;
8519 t2
= build_zero_cst (ptr_type_node
);
8525 t1
= TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t
, 1))));
8526 t1
= size_binop (PLUS_EXPR
, t1
, size_int (1));
8527 t2
= build_fold_addr_expr (TREE_VEC_ELT (t
, 0));
8528 t3
= build_fold_addr_expr (TREE_VEC_ELT (t
, 1));
8529 t4
= build_fold_addr_expr (TREE_VEC_ELT (t
, 2));
8533 /* FIXME: This will be address of
8534 extern char __OPENMP_TARGET__[] __attribute__((visibility ("hidden")))
8535 symbol, as soon as the linker plugin is able to create it for us. */
8536 tree openmp_target
= build_zero_cst (ptr_type_node
);
8537 if (kind
== GF_OMP_TARGET_KIND_REGION
)
8539 tree fnaddr
= build_fold_addr_expr (child_fn
);
8540 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 7,
8541 device
, fnaddr
, openmp_target
, t1
, t2
, t3
, t4
);
8544 g
= gimple_build_call (builtin_decl_explicit (start_ix
), 6,
8545 device
, openmp_target
, t1
, t2
, t3
, t4
);
8546 gimple_set_location (g
, gimple_location (entry_stmt
));
8547 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
8548 if (kind
!= GF_OMP_TARGET_KIND_REGION
)
8551 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_TARGET
);
8552 gsi_remove (&gsi
, true);
8554 if (kind
== GF_OMP_TARGET_KIND_DATA
&& region
->exit
)
8556 gsi
= gsi_last_bb (region
->exit
);
8558 gcc_assert (g
&& gimple_code (g
) == GIMPLE_OMP_RETURN
);
8559 gsi_remove (&gsi
, true);
8564 /* Expand the parallel region tree rooted at REGION. Expansion
8565 proceeds in depth-first order. Innermost regions are expanded
8566 first. This way, parallel regions that require a new function to
8567 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any
8568 internal dependencies in their body. */
8571 expand_omp (struct omp_region
*region
)
8575 location_t saved_location
;
8576 gimple inner_stmt
= NULL
;
8578 /* First, determine whether this is a combined parallel+workshare
8580 if (region
->type
== GIMPLE_OMP_PARALLEL
)
8581 determine_parallel_type (region
);
8583 if (region
->type
== GIMPLE_OMP_FOR
8584 && gimple_omp_for_combined_p (last_stmt (region
->entry
)))
8585 inner_stmt
= last_stmt (region
->inner
->entry
);
8588 expand_omp (region
->inner
);
8590 saved_location
= input_location
;
8591 if (gimple_has_location (last_stmt (region
->entry
)))
8592 input_location
= gimple_location (last_stmt (region
->entry
));
8594 switch (region
->type
)
8596 case GIMPLE_OMP_PARALLEL
:
8597 case GIMPLE_OMP_TASK
:
8598 expand_omp_taskreg (region
);
8601 case GIMPLE_OMP_FOR
:
8602 expand_omp_for (region
, inner_stmt
);
8605 case GIMPLE_OMP_SECTIONS
:
8606 expand_omp_sections (region
);
8609 case GIMPLE_OMP_SECTION
:
8610 /* Individual omp sections are handled together with their
8611 parent GIMPLE_OMP_SECTIONS region. */
8614 case GIMPLE_OMP_SINGLE
:
8615 expand_omp_single (region
);
8618 case GIMPLE_OMP_MASTER
:
8619 case GIMPLE_OMP_TASKGROUP
:
8620 case GIMPLE_OMP_ORDERED
:
8621 case GIMPLE_OMP_CRITICAL
:
8622 case GIMPLE_OMP_TEAMS
:
8623 expand_omp_synch (region
);
8626 case GIMPLE_OMP_ATOMIC_LOAD
:
8627 expand_omp_atomic (region
);
8630 case GIMPLE_OMP_TARGET
:
8631 expand_omp_target (region
);
8638 input_location
= saved_location
;
8639 region
= region
->next
;
8644 /* Helper for build_omp_regions. Scan the dominator tree starting at
8645 block BB. PARENT is the region that contains BB. If SINGLE_TREE is
8646 true, the function ends once a single tree is built (otherwise, whole
8647 forest of OMP constructs may be built). */
8650 build_omp_regions_1 (basic_block bb
, struct omp_region
*parent
,
8653 gimple_stmt_iterator gsi
;
8657 gsi
= gsi_last_bb (bb
);
8658 if (!gsi_end_p (gsi
) && is_gimple_omp (gsi_stmt (gsi
)))
8660 struct omp_region
*region
;
8661 enum gimple_code code
;
8663 stmt
= gsi_stmt (gsi
);
8664 code
= gimple_code (stmt
);
8665 if (code
== GIMPLE_OMP_RETURN
)
8667 /* STMT is the return point out of region PARENT. Mark it
8668 as the exit point and make PARENT the immediately
8669 enclosing region. */
8670 gcc_assert (parent
);
8673 parent
= parent
->outer
;
8675 else if (code
== GIMPLE_OMP_ATOMIC_STORE
)
8677 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to
8678 GIMPLE_OMP_RETURN, but matches with
8679 GIMPLE_OMP_ATOMIC_LOAD. */
8680 gcc_assert (parent
);
8681 gcc_assert (parent
->type
== GIMPLE_OMP_ATOMIC_LOAD
);
8684 parent
= parent
->outer
;
8687 else if (code
== GIMPLE_OMP_CONTINUE
)
8689 gcc_assert (parent
);
8692 else if (code
== GIMPLE_OMP_SECTIONS_SWITCH
)
8694 /* GIMPLE_OMP_SECTIONS_SWITCH is part of
8695 GIMPLE_OMP_SECTIONS, and we do nothing for it. */
8698 else if (code
== GIMPLE_OMP_TARGET
8699 && gimple_omp_target_kind (stmt
) == GF_OMP_TARGET_KIND_UPDATE
)
8700 new_omp_region (bb
, code
, parent
);
8703 /* Otherwise, this directive becomes the parent for a new
8705 region
= new_omp_region (bb
, code
, parent
);
8710 if (single_tree
&& !parent
)
8713 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
8715 son
= next_dom_son (CDI_DOMINATORS
, son
))
8716 build_omp_regions_1 (son
, parent
, single_tree
);
8719 /* Builds the tree of OMP regions rooted at ROOT, storing it to
8723 build_omp_regions_root (basic_block root
)
8725 gcc_assert (root_omp_region
== NULL
);
8726 build_omp_regions_1 (root
, NULL
, true);
8727 gcc_assert (root_omp_region
!= NULL
);
8730 /* Expands omp construct (and its subconstructs) starting in HEAD. */
8733 omp_expand_local (basic_block head
)
8735 build_omp_regions_root (head
);
8736 if (dump_file
&& (dump_flags
& TDF_DETAILS
))
8738 fprintf (dump_file
, "\nOMP region tree\n\n");
8739 dump_omp_region (dump_file
, root_omp_region
, 0);
8740 fprintf (dump_file
, "\n");
8743 remove_exit_barriers (root_omp_region
);
8744 expand_omp (root_omp_region
);
8746 free_omp_regions ();
8749 /* Scan the CFG and build a tree of OMP regions. Return the root of
8750 the OMP region tree. */
8753 build_omp_regions (void)
8755 gcc_assert (root_omp_region
== NULL
);
8756 calculate_dominance_info (CDI_DOMINATORS
);
8757 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun
), NULL
, false);
8760 /* Main entry point for expanding OMP-GIMPLE into runtime calls. */
8763 execute_expand_omp (void)
8765 build_omp_regions ();
8767 if (!root_omp_region
)
8772 fprintf (dump_file
, "\nOMP region tree\n\n");
8773 dump_omp_region (dump_file
, root_omp_region
, 0);
8774 fprintf (dump_file
, "\n");
8777 remove_exit_barriers (root_omp_region
);
8779 expand_omp (root_omp_region
);
8781 cleanup_tree_cfg ();
8783 free_omp_regions ();
8788 /* OMP expansion -- the default pass, run before creation of SSA form. */
8792 const pass_data pass_data_expand_omp
=
8794 GIMPLE_PASS
, /* type */
8795 "ompexp", /* name */
8796 OPTGROUP_NONE
, /* optinfo_flags */
8797 TV_NONE
, /* tv_id */
8798 PROP_gimple_any
, /* properties_required */
8799 0, /* properties_provided */
8800 0, /* properties_destroyed */
8801 0, /* todo_flags_start */
8802 0, /* todo_flags_finish */
8805 class pass_expand_omp
: public gimple_opt_pass
8808 pass_expand_omp (gcc::context
*ctxt
)
8809 : gimple_opt_pass (pass_data_expand_omp
, ctxt
)
8812 /* opt_pass methods: */
8813 virtual bool gate (function
*)
8815 return ((flag_openmp
!= 0 || flag_openmp_simd
!= 0
8816 || flag_cilkplus
!= 0) && !seen_error ());
8819 virtual unsigned int execute (function
*) { return execute_expand_omp (); }
8821 }; // class pass_expand_omp
8826 make_pass_expand_omp (gcc::context
*ctxt
)
8828 return new pass_expand_omp (ctxt
);
8831 /* Routines to lower OpenMP directives into OMP-GIMPLE. */
8833 /* If ctx is a worksharing context inside of a cancellable parallel
8834 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN
8835 and conditional branch to parallel's cancel_label to handle
8836 cancellation in the implicit barrier. */
8839 maybe_add_implicit_barrier_cancel (omp_context
*ctx
, gimple_seq
*body
)
8841 gimple omp_return
= gimple_seq_last_stmt (*body
);
8842 gcc_assert (gimple_code (omp_return
) == GIMPLE_OMP_RETURN
);
8843 if (gimple_omp_return_nowait_p (omp_return
))
8846 && gimple_code (ctx
->outer
->stmt
) == GIMPLE_OMP_PARALLEL
8847 && ctx
->outer
->cancellable
)
8849 tree fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_CANCEL
);
8850 tree c_bool_type
= TREE_TYPE (TREE_TYPE (fndecl
));
8851 tree lhs
= create_tmp_var (c_bool_type
, NULL
);
8852 gimple_omp_return_set_lhs (omp_return
, lhs
);
8853 tree fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
8854 gimple g
= gimple_build_cond (NE_EXPR
, lhs
,
8855 fold_convert (c_bool_type
,
8856 boolean_false_node
),
8857 ctx
->outer
->cancel_label
, fallthru_label
);
8858 gimple_seq_add_stmt (body
, g
);
8859 gimple_seq_add_stmt (body
, gimple_build_label (fallthru_label
));
8863 /* Lower the OpenMP sections directive in the current statement in GSI_P.
8864 CTX is the enclosing OMP context for the current statement. */
8867 lower_omp_sections (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
8869 tree block
, control
;
8870 gimple_stmt_iterator tgsi
;
8871 gimple stmt
, new_stmt
, bind
, t
;
8872 gimple_seq ilist
, dlist
, olist
, new_body
;
8874 stmt
= gsi_stmt (*gsi_p
);
8876 push_gimplify_context ();
8880 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt
),
8881 &ilist
, &dlist
, ctx
, NULL
);
8883 new_body
= gimple_omp_body (stmt
);
8884 gimple_omp_set_body (stmt
, NULL
);
8885 tgsi
= gsi_start (new_body
);
8886 for (; !gsi_end_p (tgsi
); gsi_next (&tgsi
))
8891 sec_start
= gsi_stmt (tgsi
);
8892 sctx
= maybe_lookup_ctx (sec_start
);
8895 lower_omp (gimple_omp_body_ptr (sec_start
), sctx
);
8896 gsi_insert_seq_after (&tgsi
, gimple_omp_body (sec_start
),
8897 GSI_CONTINUE_LINKING
);
8898 gimple_omp_set_body (sec_start
, NULL
);
8900 if (gsi_one_before_end_p (tgsi
))
8902 gimple_seq l
= NULL
;
8903 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt
), NULL
,
8905 gsi_insert_seq_after (&tgsi
, l
, GSI_CONTINUE_LINKING
);
8906 gimple_omp_section_set_last (sec_start
);
8909 gsi_insert_after (&tgsi
, gimple_build_omp_return (false),
8910 GSI_CONTINUE_LINKING
);
8913 block
= make_node (BLOCK
);
8914 bind
= gimple_build_bind (NULL
, new_body
, block
);
8917 lower_reduction_clauses (gimple_omp_sections_clauses (stmt
), &olist
, ctx
);
8919 block
= make_node (BLOCK
);
8920 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
8921 gsi_replace (gsi_p
, new_stmt
, true);
8923 pop_gimplify_context (new_stmt
);
8924 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
8925 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
8926 if (BLOCK_VARS (block
))
8927 TREE_USED (block
) = 1;
8930 gimple_seq_add_seq (&new_body
, ilist
);
8931 gimple_seq_add_stmt (&new_body
, stmt
);
8932 gimple_seq_add_stmt (&new_body
, gimple_build_omp_sections_switch ());
8933 gimple_seq_add_stmt (&new_body
, bind
);
8935 control
= create_tmp_var (unsigned_type_node
, ".section");
8936 t
= gimple_build_omp_continue (control
, control
);
8937 gimple_omp_sections_set_control (stmt
, control
);
8938 gimple_seq_add_stmt (&new_body
, t
);
8940 gimple_seq_add_seq (&new_body
, olist
);
8941 if (ctx
->cancellable
)
8942 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
8943 gimple_seq_add_seq (&new_body
, dlist
);
8945 new_body
= maybe_catch_exception (new_body
);
8947 t
= gimple_build_omp_return
8948 (!!find_omp_clause (gimple_omp_sections_clauses (stmt
),
8949 OMP_CLAUSE_NOWAIT
));
8950 gimple_seq_add_stmt (&new_body
, t
);
8951 maybe_add_implicit_barrier_cancel (ctx
, &new_body
);
8953 gimple_bind_set_body (new_stmt
, new_body
);
8957 /* A subroutine of lower_omp_single. Expand the simple form of
8958 a GIMPLE_OMP_SINGLE, without a copyprivate clause:
8960 if (GOMP_single_start ())
8962 [ GOMP_barrier (); ] -> unless 'nowait' is present.
8964 FIXME. It may be better to delay expanding the logic of this until
8965 pass_expand_omp. The expanded logic may make the job more difficult
8966 to a synchronization analysis pass. */
8969 lower_omp_single_simple (gimple single_stmt
, gimple_seq
*pre_p
)
8971 location_t loc
= gimple_location (single_stmt
);
8972 tree tlabel
= create_artificial_label (loc
);
8973 tree flabel
= create_artificial_label (loc
);
8977 decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START
);
8978 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (decl
)), NULL
);
8979 call
= gimple_build_call (decl
, 0);
8980 gimple_call_set_lhs (call
, lhs
);
8981 gimple_seq_add_stmt (pre_p
, call
);
8983 cond
= gimple_build_cond (EQ_EXPR
, lhs
,
8984 fold_convert_loc (loc
, TREE_TYPE (lhs
),
8987 gimple_seq_add_stmt (pre_p
, cond
);
8988 gimple_seq_add_stmt (pre_p
, gimple_build_label (tlabel
));
8989 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
8990 gimple_seq_add_stmt (pre_p
, gimple_build_label (flabel
));
8994 /* A subroutine of lower_omp_single. Expand the simple form of
8995 a GIMPLE_OMP_SINGLE, with a copyprivate clause:
8997 #pragma omp single copyprivate (a, b, c)
8999 Create a new structure to hold copies of 'a', 'b' and 'c' and emit:
9002 if ((copyout_p = GOMP_single_copy_start ()) == NULL)
9008 GOMP_single_copy_end (©out);
9019 FIXME. It may be better to delay expanding the logic of this until
9020 pass_expand_omp. The expanded logic may make the job more difficult
9021 to a synchronization analysis pass. */
9024 lower_omp_single_copy (gimple single_stmt
, gimple_seq
*pre_p
, omp_context
*ctx
)
9026 tree ptr_type
, t
, l0
, l1
, l2
, bfn_decl
;
9027 gimple_seq copyin_seq
;
9028 location_t loc
= gimple_location (single_stmt
);
9030 ctx
->sender_decl
= create_tmp_var (ctx
->record_type
, ".omp_copy_o");
9032 ptr_type
= build_pointer_type (ctx
->record_type
);
9033 ctx
->receiver_decl
= create_tmp_var (ptr_type
, ".omp_copy_i");
9035 l0
= create_artificial_label (loc
);
9036 l1
= create_artificial_label (loc
);
9037 l2
= create_artificial_label (loc
);
9039 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START
);
9040 t
= build_call_expr_loc (loc
, bfn_decl
, 0);
9041 t
= fold_convert_loc (loc
, ptr_type
, t
);
9042 gimplify_assign (ctx
->receiver_decl
, t
, pre_p
);
9044 t
= build2 (EQ_EXPR
, boolean_type_node
, ctx
->receiver_decl
,
9045 build_int_cst (ptr_type
, 0));
9046 t
= build3 (COND_EXPR
, void_type_node
, t
,
9047 build_and_jump (&l0
), build_and_jump (&l1
));
9048 gimplify_and_add (t
, pre_p
);
9050 gimple_seq_add_stmt (pre_p
, gimple_build_label (l0
));
9052 gimple_seq_add_seq (pre_p
, gimple_omp_body (single_stmt
));
9055 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt
), pre_p
,
9058 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9059 bfn_decl
= builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END
);
9060 t
= build_call_expr_loc (loc
, bfn_decl
, 1, t
);
9061 gimplify_and_add (t
, pre_p
);
9063 t
= build_and_jump (&l2
);
9064 gimplify_and_add (t
, pre_p
);
9066 gimple_seq_add_stmt (pre_p
, gimple_build_label (l1
));
9068 gimple_seq_add_seq (pre_p
, copyin_seq
);
9070 gimple_seq_add_stmt (pre_p
, gimple_build_label (l2
));
9074 /* Expand code for an OpenMP single directive. */
9077 lower_omp_single (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9080 gimple t
, bind
, single_stmt
= gsi_stmt (*gsi_p
);
9081 gimple_seq bind_body
, bind_body_tail
= NULL
, dlist
;
9083 push_gimplify_context ();
9085 block
= make_node (BLOCK
);
9086 bind
= gimple_build_bind (NULL
, NULL
, block
);
9087 gsi_replace (gsi_p
, bind
, true);
9090 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt
),
9091 &bind_body
, &dlist
, ctx
, NULL
);
9092 lower_omp (gimple_omp_body_ptr (single_stmt
), ctx
);
9094 gimple_seq_add_stmt (&bind_body
, single_stmt
);
9096 if (ctx
->record_type
)
9097 lower_omp_single_copy (single_stmt
, &bind_body
, ctx
);
9099 lower_omp_single_simple (single_stmt
, &bind_body
);
9101 gimple_omp_set_body (single_stmt
, NULL
);
9103 gimple_seq_add_seq (&bind_body
, dlist
);
9105 bind_body
= maybe_catch_exception (bind_body
);
9107 t
= gimple_build_omp_return
9108 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt
),
9109 OMP_CLAUSE_NOWAIT
));
9110 gimple_seq_add_stmt (&bind_body_tail
, t
);
9111 maybe_add_implicit_barrier_cancel (ctx
, &bind_body_tail
);
9112 if (ctx
->record_type
)
9114 gimple_stmt_iterator gsi
= gsi_start (bind_body_tail
);
9115 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
9116 TREE_THIS_VOLATILE (clobber
) = 1;
9117 gsi_insert_after (&gsi
, gimple_build_assign (ctx
->sender_decl
,
9118 clobber
), GSI_SAME_STMT
);
9120 gimple_seq_add_seq (&bind_body
, bind_body_tail
);
9121 gimple_bind_set_body (bind
, bind_body
);
9123 pop_gimplify_context (bind
);
9125 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9126 BLOCK_VARS (block
) = ctx
->block_vars
;
9127 if (BLOCK_VARS (block
))
9128 TREE_USED (block
) = 1;
9132 /* Expand code for an OpenMP master directive. */
9135 lower_omp_master (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9137 tree block
, lab
= NULL
, x
, bfn_decl
;
9138 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
9139 location_t loc
= gimple_location (stmt
);
9142 push_gimplify_context ();
9144 block
= make_node (BLOCK
);
9145 bind
= gimple_build_bind (NULL
, NULL
, block
);
9146 gsi_replace (gsi_p
, bind
, true);
9147 gimple_bind_add_stmt (bind
, stmt
);
9149 bfn_decl
= builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM
);
9150 x
= build_call_expr_loc (loc
, bfn_decl
, 0);
9151 x
= build2 (EQ_EXPR
, boolean_type_node
, x
, integer_zero_node
);
9152 x
= build3 (COND_EXPR
, void_type_node
, x
, NULL
, build_and_jump (&lab
));
9154 gimplify_and_add (x
, &tseq
);
9155 gimple_bind_add_seq (bind
, tseq
);
9157 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9158 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
9159 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9160 gimple_omp_set_body (stmt
, NULL
);
9162 gimple_bind_add_stmt (bind
, gimple_build_label (lab
));
9164 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9166 pop_gimplify_context (bind
);
9168 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9169 BLOCK_VARS (block
) = ctx
->block_vars
;
9173 /* Expand code for an OpenMP taskgroup directive. */
9176 lower_omp_taskgroup (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9178 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
9179 tree block
= make_node (BLOCK
);
9181 bind
= gimple_build_bind (NULL
, NULL
, block
);
9182 gsi_replace (gsi_p
, bind
, true);
9183 gimple_bind_add_stmt (bind
, stmt
);
9185 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START
),
9187 gimple_bind_add_stmt (bind
, x
);
9189 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9190 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9191 gimple_omp_set_body (stmt
, NULL
);
9193 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9195 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9196 BLOCK_VARS (block
) = ctx
->block_vars
;
9200 /* Expand code for an OpenMP ordered directive. */
9203 lower_omp_ordered (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9206 gimple stmt
= gsi_stmt (*gsi_p
), bind
, x
;
9208 push_gimplify_context ();
9210 block
= make_node (BLOCK
);
9211 bind
= gimple_build_bind (NULL
, NULL
, block
);
9212 gsi_replace (gsi_p
, bind
, true);
9213 gimple_bind_add_stmt (bind
, stmt
);
9215 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START
),
9217 gimple_bind_add_stmt (bind
, x
);
9219 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9220 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
9221 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9222 gimple_omp_set_body (stmt
, NULL
);
9224 x
= gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END
), 0);
9225 gimple_bind_add_stmt (bind
, x
);
9227 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9229 pop_gimplify_context (bind
);
9231 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9232 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
9236 /* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
9237 substitution of a couple of function calls. But in the NAMED case,
9238 requires that languages coordinate a symbol name. It is therefore
9239 best put here in common code. */
9241 static GTY((param1_is (tree
), param2_is (tree
)))
9242 splay_tree critical_name_mutexes
;
9245 lower_omp_critical (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9248 tree name
, lock
, unlock
;
9249 gimple stmt
= gsi_stmt (*gsi_p
), bind
;
9250 location_t loc
= gimple_location (stmt
);
9253 name
= gimple_omp_critical_name (stmt
);
9259 if (!critical_name_mutexes
)
9260 critical_name_mutexes
9261 = splay_tree_new_ggc (splay_tree_compare_pointers
,
9262 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_s
,
9263 ggc_alloc_splay_tree_tree_node_tree_node_splay_tree_node_s
);
9265 n
= splay_tree_lookup (critical_name_mutexes
, (splay_tree_key
) name
);
9270 decl
= create_tmp_var_raw (ptr_type_node
, NULL
);
9272 new_str
= ACONCAT ((".gomp_critical_user_",
9273 IDENTIFIER_POINTER (name
), NULL
));
9274 DECL_NAME (decl
) = get_identifier (new_str
);
9275 TREE_PUBLIC (decl
) = 1;
9276 TREE_STATIC (decl
) = 1;
9277 DECL_COMMON (decl
) = 1;
9278 DECL_ARTIFICIAL (decl
) = 1;
9279 DECL_IGNORED_P (decl
) = 1;
9280 varpool_node::finalize_decl (decl
);
9282 splay_tree_insert (critical_name_mutexes
, (splay_tree_key
) name
,
9283 (splay_tree_value
) decl
);
9286 decl
= (tree
) n
->value
;
9288 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START
);
9289 lock
= build_call_expr_loc (loc
, lock
, 1, build_fold_addr_expr_loc (loc
, decl
));
9291 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END
);
9292 unlock
= build_call_expr_loc (loc
, unlock
, 1,
9293 build_fold_addr_expr_loc (loc
, decl
));
9297 lock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START
);
9298 lock
= build_call_expr_loc (loc
, lock
, 0);
9300 unlock
= builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END
);
9301 unlock
= build_call_expr_loc (loc
, unlock
, 0);
9304 push_gimplify_context ();
9306 block
= make_node (BLOCK
);
9307 bind
= gimple_build_bind (NULL
, NULL
, block
);
9308 gsi_replace (gsi_p
, bind
, true);
9309 gimple_bind_add_stmt (bind
, stmt
);
9311 tbody
= gimple_bind_body (bind
);
9312 gimplify_and_add (lock
, &tbody
);
9313 gimple_bind_set_body (bind
, tbody
);
9315 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9316 gimple_omp_set_body (stmt
, maybe_catch_exception (gimple_omp_body (stmt
)));
9317 gimple_bind_add_seq (bind
, gimple_omp_body (stmt
));
9318 gimple_omp_set_body (stmt
, NULL
);
9320 tbody
= gimple_bind_body (bind
);
9321 gimplify_and_add (unlock
, &tbody
);
9322 gimple_bind_set_body (bind
, tbody
);
9324 gimple_bind_add_stmt (bind
, gimple_build_omp_return (true));
9326 pop_gimplify_context (bind
);
9327 gimple_bind_append_vars (bind
, ctx
->block_vars
);
9328 BLOCK_VARS (block
) = gimple_bind_vars (bind
);
9332 /* A subroutine of lower_omp_for. Generate code to emit the predicate
9333 for a lastprivate clause. Given a loop control predicate of (V
9334 cond N2), we gate the clause on (!(V cond N2)). The lowered form
9335 is appended to *DLIST, iterator initialization is appended to
9339 lower_omp_for_lastprivate (struct omp_for_data
*fd
, gimple_seq
*body_p
,
9340 gimple_seq
*dlist
, struct omp_context
*ctx
)
9342 tree clauses
, cond
, vinit
;
9343 enum tree_code cond_code
;
9346 cond_code
= fd
->loop
.cond_code
;
9347 cond_code
= cond_code
== LT_EXPR
? GE_EXPR
: LE_EXPR
;
9349 /* When possible, use a strict equality expression. This can let VRP
9350 type optimizations deduce the value and remove a copy. */
9351 if (tree_fits_shwi_p (fd
->loop
.step
))
9353 HOST_WIDE_INT step
= tree_to_shwi (fd
->loop
.step
);
9354 if (step
== 1 || step
== -1)
9355 cond_code
= EQ_EXPR
;
9358 cond
= build2 (cond_code
, boolean_type_node
, fd
->loop
.v
, fd
->loop
.n2
);
9360 clauses
= gimple_omp_for_clauses (fd
->for_stmt
);
9362 lower_lastprivate_clauses (clauses
, cond
, &stmts
, ctx
);
9363 if (!gimple_seq_empty_p (stmts
))
9365 gimple_seq_add_seq (&stmts
, *dlist
);
9368 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
9369 vinit
= fd
->loop
.n1
;
9370 if (cond_code
== EQ_EXPR
9371 && tree_fits_shwi_p (fd
->loop
.n2
)
9372 && ! integer_zerop (fd
->loop
.n2
))
9373 vinit
= build_int_cst (TREE_TYPE (fd
->loop
.v
), 0);
9375 vinit
= unshare_expr (vinit
);
9377 /* Initialize the iterator variable, so that threads that don't execute
9378 any iterations don't execute the lastprivate clauses by accident. */
9379 gimplify_assign (fd
->loop
.v
, vinit
, body_p
);
9384 /* Lower code for an OpenMP loop directive. */
9387 lower_omp_for (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9390 struct omp_for_data fd
, *fdp
= NULL
;
9391 gimple stmt
= gsi_stmt (*gsi_p
), new_stmt
;
9392 gimple_seq omp_for_body
, body
, dlist
;
9395 push_gimplify_context ();
9397 lower_omp (gimple_omp_for_pre_body_ptr (stmt
), ctx
);
9399 block
= make_node (BLOCK
);
9400 new_stmt
= gimple_build_bind (NULL
, NULL
, block
);
9401 /* Replace at gsi right away, so that 'stmt' is no member
9402 of a sequence anymore as we're going to add to to a different
9404 gsi_replace (gsi_p
, new_stmt
, true);
9406 /* Move declaration of temporaries in the loop body before we make
9408 omp_for_body
= gimple_omp_body (stmt
);
9409 if (!gimple_seq_empty_p (omp_for_body
)
9410 && gimple_code (gimple_seq_first_stmt (omp_for_body
)) == GIMPLE_BIND
)
9412 gimple inner_bind
= gimple_seq_first_stmt (omp_for_body
);
9413 tree vars
= gimple_bind_vars (inner_bind
);
9414 gimple_bind_append_vars (new_stmt
, vars
);
9415 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't
9416 keep them on the inner_bind and it's block. */
9417 gimple_bind_set_vars (inner_bind
, NULL_TREE
);
9418 if (gimple_bind_block (inner_bind
))
9419 BLOCK_VARS (gimple_bind_block (inner_bind
)) = NULL_TREE
;
9422 if (gimple_omp_for_combined_into_p (stmt
))
9424 extract_omp_for_data (stmt
, &fd
, NULL
);
9427 /* We need two temporaries with fd.loop.v type (istart/iend)
9428 and then (fd.collapse - 1) temporaries with the same
9429 type for count2 ... countN-1 vars if not constant. */
9431 tree type
= fd
.iter_type
;
9433 && TREE_CODE (fd
.loop
.n2
) != INTEGER_CST
)
9434 count
+= fd
.collapse
- 1;
9435 bool parallel_for
= gimple_omp_for_kind (stmt
) == GF_OMP_FOR_KIND_FOR
;
9436 tree outerc
= NULL
, *pc
= gimple_omp_for_clauses_ptr (stmt
);
9440 = find_omp_clause (gimple_omp_parallel_clauses (ctx
->outer
->stmt
),
9441 OMP_CLAUSE__LOOPTEMP_
);
9442 for (i
= 0; i
< count
; i
++)
9447 gcc_assert (outerc
);
9448 temp
= lookup_decl (OMP_CLAUSE_DECL (outerc
), ctx
->outer
);
9449 outerc
= find_omp_clause (OMP_CLAUSE_CHAIN (outerc
),
9450 OMP_CLAUSE__LOOPTEMP_
);
9454 temp
= create_tmp_var (type
, NULL
);
9455 insert_decl_map (&ctx
->outer
->cb
, temp
, temp
);
9457 *pc
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE__LOOPTEMP_
);
9458 OMP_CLAUSE_DECL (*pc
) = temp
;
9459 pc
= &OMP_CLAUSE_CHAIN (*pc
);
9464 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
9467 lower_rec_input_clauses (gimple_omp_for_clauses (stmt
), &body
, &dlist
, ctx
,
9469 gimple_seq_add_seq (&body
, gimple_omp_for_pre_body (stmt
));
9471 lower_omp (gimple_omp_body_ptr (stmt
), ctx
);
9473 /* Lower the header expressions. At this point, we can assume that
9474 the header is of the form:
9476 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3)
9478 We just need to make sure that VAL1, VAL2 and VAL3 are lowered
9479 using the .omp_data_s mapping, if needed. */
9480 for (i
= 0; i
< gimple_omp_for_collapse (stmt
); i
++)
9482 rhs_p
= gimple_omp_for_initial_ptr (stmt
, i
);
9483 if (!is_gimple_min_invariant (*rhs_p
))
9484 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9486 rhs_p
= gimple_omp_for_final_ptr (stmt
, i
);
9487 if (!is_gimple_min_invariant (*rhs_p
))
9488 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9490 rhs_p
= &TREE_OPERAND (gimple_omp_for_incr (stmt
, i
), 1);
9491 if (!is_gimple_min_invariant (*rhs_p
))
9492 *rhs_p
= get_formal_tmp_var (*rhs_p
, &body
);
9495 /* Once lowered, extract the bounds and clauses. */
9496 extract_omp_for_data (stmt
, &fd
, NULL
);
9498 lower_omp_for_lastprivate (&fd
, &body
, &dlist
, ctx
);
9500 gimple_seq_add_stmt (&body
, stmt
);
9501 gimple_seq_add_seq (&body
, gimple_omp_body (stmt
));
9503 gimple_seq_add_stmt (&body
, gimple_build_omp_continue (fd
.loop
.v
,
9506 /* After the loop, add exit clauses. */
9507 lower_reduction_clauses (gimple_omp_for_clauses (stmt
), &body
, ctx
);
9509 if (ctx
->cancellable
)
9510 gimple_seq_add_stmt (&body
, gimple_build_label (ctx
->cancel_label
));
9512 gimple_seq_add_seq (&body
, dlist
);
9514 body
= maybe_catch_exception (body
);
9516 /* Region exit marker goes at the end of the loop body. */
9517 gimple_seq_add_stmt (&body
, gimple_build_omp_return (fd
.have_nowait
));
9518 maybe_add_implicit_barrier_cancel (ctx
, &body
);
9519 pop_gimplify_context (new_stmt
);
9521 gimple_bind_append_vars (new_stmt
, ctx
->block_vars
);
9522 BLOCK_VARS (block
) = gimple_bind_vars (new_stmt
);
9523 if (BLOCK_VARS (block
))
9524 TREE_USED (block
) = 1;
9526 gimple_bind_set_body (new_stmt
, body
);
9527 gimple_omp_set_body (stmt
, NULL
);
9528 gimple_omp_for_set_pre_body (stmt
, NULL
);
9531 /* Callback for walk_stmts. Check if the current statement only contains
9532 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */
9535 check_combined_parallel (gimple_stmt_iterator
*gsi_p
,
9536 bool *handled_ops_p
,
9537 struct walk_stmt_info
*wi
)
9539 int *info
= (int *) wi
->info
;
9540 gimple stmt
= gsi_stmt (*gsi_p
);
9542 *handled_ops_p
= true;
9543 switch (gimple_code (stmt
))
9547 case GIMPLE_OMP_FOR
:
9548 case GIMPLE_OMP_SECTIONS
:
9549 *info
= *info
== 0 ? 1 : -1;
9558 struct omp_taskcopy_context
9560 /* This field must be at the beginning, as we do "inheritance": Some
9561 callback functions for tree-inline.c (e.g., omp_copy_decl)
9562 receive a copy_body_data pointer that is up-casted to an
9563 omp_context pointer. */
9569 task_copyfn_copy_decl (tree var
, copy_body_data
*cb
)
9571 struct omp_taskcopy_context
*tcctx
= (struct omp_taskcopy_context
*) cb
;
9573 if (splay_tree_lookup (tcctx
->ctx
->sfield_map
, (splay_tree_key
) var
))
9574 return create_tmp_var (TREE_TYPE (var
), NULL
);
9580 task_copyfn_remap_type (struct omp_taskcopy_context
*tcctx
, tree orig_type
)
9582 tree name
, new_fields
= NULL
, type
, f
;
9584 type
= lang_hooks
.types
.make_type (RECORD_TYPE
);
9585 name
= DECL_NAME (TYPE_NAME (orig_type
));
9586 name
= build_decl (gimple_location (tcctx
->ctx
->stmt
),
9587 TYPE_DECL
, name
, type
);
9588 TYPE_NAME (type
) = name
;
9590 for (f
= TYPE_FIELDS (orig_type
); f
; f
= TREE_CHAIN (f
))
9592 tree new_f
= copy_node (f
);
9593 DECL_CONTEXT (new_f
) = type
;
9594 TREE_TYPE (new_f
) = remap_type (TREE_TYPE (f
), &tcctx
->cb
);
9595 TREE_CHAIN (new_f
) = new_fields
;
9596 walk_tree (&DECL_SIZE (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9597 walk_tree (&DECL_SIZE_UNIT (new_f
), copy_tree_body_r
, &tcctx
->cb
, NULL
);
9598 walk_tree (&DECL_FIELD_OFFSET (new_f
), copy_tree_body_r
,
9601 tcctx
->cb
.decl_map
->put (f
, new_f
);
9603 TYPE_FIELDS (type
) = nreverse (new_fields
);
9608 /* Create task copyfn. */
9611 create_task_copyfn (gimple task_stmt
, omp_context
*ctx
)
9613 struct function
*child_cfun
;
9614 tree child_fn
, t
, c
, src
, dst
, f
, sf
, arg
, sarg
, decl
;
9615 tree record_type
, srecord_type
, bind
, list
;
9616 bool record_needs_remap
= false, srecord_needs_remap
= false;
9618 struct omp_taskcopy_context tcctx
;
9619 location_t loc
= gimple_location (task_stmt
);
9621 child_fn
= gimple_omp_task_copy_fn (task_stmt
);
9622 child_cfun
= DECL_STRUCT_FUNCTION (child_fn
);
9623 gcc_assert (child_cfun
->cfg
== NULL
);
9624 DECL_SAVED_TREE (child_fn
) = alloc_stmt_list ();
9626 /* Reset DECL_CONTEXT on function arguments. */
9627 for (t
= DECL_ARGUMENTS (child_fn
); t
; t
= DECL_CHAIN (t
))
9628 DECL_CONTEXT (t
) = child_fn
;
9630 /* Populate the function. */
9631 push_gimplify_context ();
9632 push_cfun (child_cfun
);
9634 bind
= build3 (BIND_EXPR
, void_type_node
, NULL
, NULL
, NULL
);
9635 TREE_SIDE_EFFECTS (bind
) = 1;
9637 DECL_SAVED_TREE (child_fn
) = bind
;
9638 DECL_SOURCE_LOCATION (child_fn
) = gimple_location (task_stmt
);
9640 /* Remap src and dst argument types if needed. */
9641 record_type
= ctx
->record_type
;
9642 srecord_type
= ctx
->srecord_type
;
9643 for (f
= TYPE_FIELDS (record_type
); f
; f
= DECL_CHAIN (f
))
9644 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9646 record_needs_remap
= true;
9649 for (f
= TYPE_FIELDS (srecord_type
); f
; f
= DECL_CHAIN (f
))
9650 if (variably_modified_type_p (TREE_TYPE (f
), ctx
->cb
.src_fn
))
9652 srecord_needs_remap
= true;
9656 if (record_needs_remap
|| srecord_needs_remap
)
9658 memset (&tcctx
, '\0', sizeof (tcctx
));
9659 tcctx
.cb
.src_fn
= ctx
->cb
.src_fn
;
9660 tcctx
.cb
.dst_fn
= child_fn
;
9661 tcctx
.cb
.src_node
= cgraph_node::get (tcctx
.cb
.src_fn
);
9662 gcc_checking_assert (tcctx
.cb
.src_node
);
9663 tcctx
.cb
.dst_node
= tcctx
.cb
.src_node
;
9664 tcctx
.cb
.src_cfun
= ctx
->cb
.src_cfun
;
9665 tcctx
.cb
.copy_decl
= task_copyfn_copy_decl
;
9666 tcctx
.cb
.eh_lp_nr
= 0;
9667 tcctx
.cb
.transform_call_graph_edges
= CB_CGE_MOVE
;
9668 tcctx
.cb
.decl_map
= new hash_map
<tree
, tree
>;
9671 if (record_needs_remap
)
9672 record_type
= task_copyfn_remap_type (&tcctx
, record_type
);
9673 if (srecord_needs_remap
)
9674 srecord_type
= task_copyfn_remap_type (&tcctx
, srecord_type
);
9677 tcctx
.cb
.decl_map
= NULL
;
9679 arg
= DECL_ARGUMENTS (child_fn
);
9680 TREE_TYPE (arg
) = build_pointer_type (record_type
);
9681 sarg
= DECL_CHAIN (arg
);
9682 TREE_TYPE (sarg
) = build_pointer_type (srecord_type
);
9684 /* First pass: initialize temporaries used in record_type and srecord_type
9685 sizes and field offsets. */
9686 if (tcctx
.cb
.decl_map
)
9687 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9688 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9692 decl
= OMP_CLAUSE_DECL (c
);
9693 p
= tcctx
.cb
.decl_map
->get (decl
);
9696 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9697 sf
= (tree
) n
->value
;
9698 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9699 src
= build_simple_mem_ref_loc (loc
, sarg
);
9700 src
= omp_build_component_ref (src
, sf
);
9701 t
= build2 (MODIFY_EXPR
, TREE_TYPE (*p
), *p
, src
);
9702 append_to_statement_list (t
, &list
);
9705 /* Second pass: copy shared var pointers and copy construct non-VLA
9706 firstprivate vars. */
9707 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9708 switch (OMP_CLAUSE_CODE (c
))
9710 case OMP_CLAUSE_SHARED
:
9711 decl
= OMP_CLAUSE_DECL (c
);
9712 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9715 f
= (tree
) n
->value
;
9716 if (tcctx
.cb
.decl_map
)
9717 f
= *tcctx
.cb
.decl_map
->get (f
);
9718 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9719 sf
= (tree
) n
->value
;
9720 if (tcctx
.cb
.decl_map
)
9721 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9722 src
= build_simple_mem_ref_loc (loc
, sarg
);
9723 src
= omp_build_component_ref (src
, sf
);
9724 dst
= build_simple_mem_ref_loc (loc
, arg
);
9725 dst
= omp_build_component_ref (dst
, f
);
9726 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9727 append_to_statement_list (t
, &list
);
9729 case OMP_CLAUSE_FIRSTPRIVATE
:
9730 decl
= OMP_CLAUSE_DECL (c
);
9731 if (is_variable_sized (decl
))
9733 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9736 f
= (tree
) n
->value
;
9737 if (tcctx
.cb
.decl_map
)
9738 f
= *tcctx
.cb
.decl_map
->get (f
);
9739 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9742 sf
= (tree
) n
->value
;
9743 if (tcctx
.cb
.decl_map
)
9744 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9745 src
= build_simple_mem_ref_loc (loc
, sarg
);
9746 src
= omp_build_component_ref (src
, sf
);
9747 if (use_pointer_for_field (decl
, NULL
) || is_reference (decl
))
9748 src
= build_simple_mem_ref_loc (loc
, src
);
9752 dst
= build_simple_mem_ref_loc (loc
, arg
);
9753 dst
= omp_build_component_ref (dst
, f
);
9754 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9755 append_to_statement_list (t
, &list
);
9757 case OMP_CLAUSE_PRIVATE
:
9758 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c
))
9760 decl
= OMP_CLAUSE_DECL (c
);
9761 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9762 f
= (tree
) n
->value
;
9763 if (tcctx
.cb
.decl_map
)
9764 f
= *tcctx
.cb
.decl_map
->get (f
);
9765 n
= splay_tree_lookup (ctx
->sfield_map
, (splay_tree_key
) decl
);
9768 sf
= (tree
) n
->value
;
9769 if (tcctx
.cb
.decl_map
)
9770 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9771 src
= build_simple_mem_ref_loc (loc
, sarg
);
9772 src
= omp_build_component_ref (src
, sf
);
9773 if (use_pointer_for_field (decl
, NULL
))
9774 src
= build_simple_mem_ref_loc (loc
, src
);
9778 dst
= build_simple_mem_ref_loc (loc
, arg
);
9779 dst
= omp_build_component_ref (dst
, f
);
9780 t
= build2 (MODIFY_EXPR
, TREE_TYPE (dst
), dst
, src
);
9781 append_to_statement_list (t
, &list
);
9787 /* Last pass: handle VLA firstprivates. */
9788 if (tcctx
.cb
.decl_map
)
9789 for (c
= gimple_omp_task_clauses (task_stmt
); c
; c
= OMP_CLAUSE_CHAIN (c
))
9790 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_FIRSTPRIVATE
)
9794 decl
= OMP_CLAUSE_DECL (c
);
9795 if (!is_variable_sized (decl
))
9797 n
= splay_tree_lookup (ctx
->field_map
, (splay_tree_key
) decl
);
9800 f
= (tree
) n
->value
;
9801 f
= *tcctx
.cb
.decl_map
->get (f
);
9802 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl
));
9803 ind
= DECL_VALUE_EXPR (decl
);
9804 gcc_assert (TREE_CODE (ind
) == INDIRECT_REF
);
9805 gcc_assert (DECL_P (TREE_OPERAND (ind
, 0)));
9806 n
= splay_tree_lookup (ctx
->sfield_map
,
9807 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9808 sf
= (tree
) n
->value
;
9809 sf
= *tcctx
.cb
.decl_map
->get (sf
);
9810 src
= build_simple_mem_ref_loc (loc
, sarg
);
9811 src
= omp_build_component_ref (src
, sf
);
9812 src
= build_simple_mem_ref_loc (loc
, src
);
9813 dst
= build_simple_mem_ref_loc (loc
, arg
);
9814 dst
= omp_build_component_ref (dst
, f
);
9815 t
= lang_hooks
.decls
.omp_clause_copy_ctor (c
, dst
, src
);
9816 append_to_statement_list (t
, &list
);
9817 n
= splay_tree_lookup (ctx
->field_map
,
9818 (splay_tree_key
) TREE_OPERAND (ind
, 0));
9819 df
= (tree
) n
->value
;
9820 df
= *tcctx
.cb
.decl_map
->get (df
);
9821 ptr
= build_simple_mem_ref_loc (loc
, arg
);
9822 ptr
= omp_build_component_ref (ptr
, df
);
9823 t
= build2 (MODIFY_EXPR
, TREE_TYPE (ptr
), ptr
,
9824 build_fold_addr_expr_loc (loc
, dst
));
9825 append_to_statement_list (t
, &list
);
9828 t
= build1 (RETURN_EXPR
, void_type_node
, NULL
);
9829 append_to_statement_list (t
, &list
);
9831 if (tcctx
.cb
.decl_map
)
9832 delete tcctx
.cb
.decl_map
;
9833 pop_gimplify_context (NULL
);
9834 BIND_EXPR_BODY (bind
) = list
;
9839 lower_depend_clauses (gimple stmt
, gimple_seq
*iseq
, gimple_seq
*oseq
)
9843 size_t n_in
= 0, n_out
= 0, idx
= 2, i
;
9845 clauses
= find_omp_clause (gimple_omp_task_clauses (stmt
),
9847 gcc_assert (clauses
);
9848 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9849 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
)
9850 switch (OMP_CLAUSE_DEPEND_KIND (c
))
9852 case OMP_CLAUSE_DEPEND_IN
:
9855 case OMP_CLAUSE_DEPEND_OUT
:
9856 case OMP_CLAUSE_DEPEND_INOUT
:
9862 tree type
= build_array_type_nelts (ptr_type_node
, n_in
+ n_out
+ 2);
9863 tree array
= create_tmp_var (type
, NULL
);
9864 tree r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (0), NULL_TREE
,
9866 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_in
+ n_out
));
9867 gimple_seq_add_stmt (iseq
, g
);
9868 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (1), NULL_TREE
,
9870 g
= gimple_build_assign (r
, build_int_cst (ptr_type_node
, n_out
));
9871 gimple_seq_add_stmt (iseq
, g
);
9872 for (i
= 0; i
< 2; i
++)
9874 if ((i
? n_in
: n_out
) == 0)
9876 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
9877 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_DEPEND
9878 && ((OMP_CLAUSE_DEPEND_KIND (c
) != OMP_CLAUSE_DEPEND_IN
) ^ i
))
9880 tree t
= OMP_CLAUSE_DECL (c
);
9881 t
= fold_convert (ptr_type_node
, t
);
9882 gimplify_expr (&t
, iseq
, NULL
, is_gimple_val
, fb_rvalue
);
9883 r
= build4 (ARRAY_REF
, ptr_type_node
, array
, size_int (idx
++),
9884 NULL_TREE
, NULL_TREE
);
9885 g
= gimple_build_assign (r
, t
);
9886 gimple_seq_add_stmt (iseq
, g
);
9889 tree
*p
= gimple_omp_task_clauses_ptr (stmt
);
9890 c
= build_omp_clause (UNKNOWN_LOCATION
, OMP_CLAUSE_DEPEND
);
9891 OMP_CLAUSE_DECL (c
) = build_fold_addr_expr (array
);
9892 OMP_CLAUSE_CHAIN (c
) = *p
;
9894 tree clobber
= build_constructor (type
, NULL
);
9895 TREE_THIS_VOLATILE (clobber
) = 1;
9896 g
= gimple_build_assign (array
, clobber
);
9897 gimple_seq_add_stmt (oseq
, g
);
9900 /* Lower the OpenMP parallel or task directive in the current statement
9901 in GSI_P. CTX holds context information for the directive. */
9904 lower_omp_taskreg (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
9908 gimple stmt
= gsi_stmt (*gsi_p
);
9909 gimple par_bind
, bind
, dep_bind
= NULL
;
9910 gimple_seq par_body
, olist
, ilist
, par_olist
, par_rlist
, par_ilist
, new_body
;
9911 location_t loc
= gimple_location (stmt
);
9913 clauses
= gimple_omp_taskreg_clauses (stmt
);
9914 par_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
9915 par_body
= gimple_bind_body (par_bind
);
9916 child_fn
= ctx
->cb
.dst_fn
;
9917 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
9918 && !gimple_omp_parallel_combined_p (stmt
))
9920 struct walk_stmt_info wi
;
9923 memset (&wi
, 0, sizeof (wi
));
9926 walk_gimple_seq (par_body
, check_combined_parallel
, NULL
, &wi
);
9928 gimple_omp_parallel_set_combined_p (stmt
, true);
9930 gimple_seq dep_ilist
= NULL
;
9931 gimple_seq dep_olist
= NULL
;
9932 if (gimple_code (stmt
) == GIMPLE_OMP_TASK
9933 && find_omp_clause (clauses
, OMP_CLAUSE_DEPEND
))
9935 push_gimplify_context ();
9936 dep_bind
= gimple_build_bind (NULL
, NULL
, make_node (BLOCK
));
9937 lower_depend_clauses (stmt
, &dep_ilist
, &dep_olist
);
9940 if (ctx
->srecord_type
)
9941 create_task_copyfn (stmt
, ctx
);
9943 push_gimplify_context ();
9948 lower_rec_input_clauses (clauses
, &par_ilist
, &par_olist
, ctx
, NULL
);
9949 lower_omp (&par_body
, ctx
);
9950 if (gimple_code (stmt
) == GIMPLE_OMP_PARALLEL
)
9951 lower_reduction_clauses (clauses
, &par_rlist
, ctx
);
9953 /* Declare all the variables created by mapping and the variables
9954 declared in the scope of the parallel body. */
9955 record_vars_into (ctx
->block_vars
, child_fn
);
9956 record_vars_into (gimple_bind_vars (par_bind
), child_fn
);
9958 if (ctx
->record_type
)
9961 = create_tmp_var (ctx
->srecord_type
? ctx
->srecord_type
9962 : ctx
->record_type
, ".omp_data_o");
9963 DECL_NAMELESS (ctx
->sender_decl
) = 1;
9964 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
9965 gimple_omp_taskreg_set_data_arg (stmt
, ctx
->sender_decl
);
9970 lower_send_clauses (clauses
, &ilist
, &olist
, ctx
);
9971 lower_send_shared_vars (&ilist
, &olist
, ctx
);
9973 if (ctx
->record_type
)
9975 tree clobber
= build_constructor (TREE_TYPE (ctx
->sender_decl
), NULL
);
9976 TREE_THIS_VOLATILE (clobber
) = 1;
9977 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
9981 /* Once all the expansions are done, sequence all the different
9982 fragments inside gimple_omp_body. */
9986 if (ctx
->record_type
)
9988 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
9989 /* fixup_child_record_type might have changed receiver_decl's type. */
9990 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
9991 gimple_seq_add_stmt (&new_body
,
9992 gimple_build_assign (ctx
->receiver_decl
, t
));
9995 gimple_seq_add_seq (&new_body
, par_ilist
);
9996 gimple_seq_add_seq (&new_body
, par_body
);
9997 gimple_seq_add_seq (&new_body
, par_rlist
);
9998 if (ctx
->cancellable
)
9999 gimple_seq_add_stmt (&new_body
, gimple_build_label (ctx
->cancel_label
));
10000 gimple_seq_add_seq (&new_body
, par_olist
);
10001 new_body
= maybe_catch_exception (new_body
);
10002 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
10003 gimple_omp_set_body (stmt
, new_body
);
10005 bind
= gimple_build_bind (NULL
, NULL
, gimple_bind_block (par_bind
));
10006 gsi_replace (gsi_p
, dep_bind
? dep_bind
: bind
, true);
10007 gimple_bind_add_seq (bind
, ilist
);
10008 gimple_bind_add_stmt (bind
, stmt
);
10009 gimple_bind_add_seq (bind
, olist
);
10011 pop_gimplify_context (NULL
);
10015 gimple_bind_add_seq (dep_bind
, dep_ilist
);
10016 gimple_bind_add_stmt (dep_bind
, bind
);
10017 gimple_bind_add_seq (dep_bind
, dep_olist
);
10018 pop_gimplify_context (dep_bind
);
10022 /* Lower the OpenMP target directive in the current statement
10023 in GSI_P. CTX holds context information for the directive. */
10026 lower_omp_target (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10029 tree child_fn
, t
, c
;
10030 gimple stmt
= gsi_stmt (*gsi_p
);
10031 gimple tgt_bind
= NULL
, bind
;
10032 gimple_seq tgt_body
= NULL
, olist
, ilist
, new_body
;
10033 location_t loc
= gimple_location (stmt
);
10034 int kind
= gimple_omp_target_kind (stmt
);
10035 unsigned int map_cnt
= 0;
10037 clauses
= gimple_omp_target_clauses (stmt
);
10038 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10040 tgt_bind
= gimple_seq_first_stmt (gimple_omp_body (stmt
));
10041 tgt_body
= gimple_bind_body (tgt_bind
);
10043 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
10044 tgt_body
= gimple_omp_body (stmt
);
10045 child_fn
= ctx
->cb
.dst_fn
;
10047 push_gimplify_context ();
10049 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
10050 switch (OMP_CLAUSE_CODE (c
))
10056 case OMP_CLAUSE_MAP
:
10057 case OMP_CLAUSE_TO
:
10058 case OMP_CLAUSE_FROM
:
10059 var
= OMP_CLAUSE_DECL (c
);
10062 if (OMP_CLAUSE_CODE (c
) != OMP_CLAUSE_MAP
10063 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
10068 if (DECL_SIZE (var
)
10069 && TREE_CODE (DECL_SIZE (var
)) != INTEGER_CST
)
10071 tree var2
= DECL_VALUE_EXPR (var
);
10072 gcc_assert (TREE_CODE (var2
) == INDIRECT_REF
);
10073 var2
= TREE_OPERAND (var2
, 0);
10074 gcc_assert (DECL_P (var2
));
10078 if (!maybe_lookup_field (var
, ctx
))
10081 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10083 x
= build_receiver_ref (var
, true, ctx
);
10084 tree new_var
= lookup_decl (var
, ctx
);
10085 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
10086 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
10087 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
10088 && TREE_CODE (TREE_TYPE (var
)) == ARRAY_TYPE
)
10089 x
= build_simple_mem_ref (x
);
10090 SET_DECL_VALUE_EXPR (new_var
, x
);
10091 DECL_HAS_VALUE_EXPR_P (new_var
) = 1;
10096 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10098 target_nesting_level
++;
10099 lower_omp (&tgt_body
, ctx
);
10100 target_nesting_level
--;
10102 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
10103 lower_omp (&tgt_body
, ctx
);
10105 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10107 /* Declare all the variables created by mapping and the variables
10108 declared in the scope of the target body. */
10109 record_vars_into (ctx
->block_vars
, child_fn
);
10110 record_vars_into (gimple_bind_vars (tgt_bind
), child_fn
);
10115 if (ctx
->record_type
)
10118 = create_tmp_var (ctx
->record_type
, ".omp_data_arr");
10119 DECL_NAMELESS (ctx
->sender_decl
) = 1;
10120 TREE_ADDRESSABLE (ctx
->sender_decl
) = 1;
10121 t
= make_tree_vec (3);
10122 TREE_VEC_ELT (t
, 0) = ctx
->sender_decl
;
10123 TREE_VEC_ELT (t
, 1)
10124 = create_tmp_var (build_array_type_nelts (size_type_node
, map_cnt
),
10125 ".omp_data_sizes");
10126 DECL_NAMELESS (TREE_VEC_ELT (t
, 1)) = 1;
10127 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 1)) = 1;
10128 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 1;
10129 TREE_VEC_ELT (t
, 2)
10130 = create_tmp_var (build_array_type_nelts (unsigned_char_type_node
,
10132 ".omp_data_kinds");
10133 DECL_NAMELESS (TREE_VEC_ELT (t
, 2)) = 1;
10134 TREE_ADDRESSABLE (TREE_VEC_ELT (t
, 2)) = 1;
10135 TREE_STATIC (TREE_VEC_ELT (t
, 2)) = 1;
10136 gimple_omp_target_set_data_arg (stmt
, t
);
10138 vec
<constructor_elt
, va_gc
> *vsize
;
10139 vec
<constructor_elt
, va_gc
> *vkind
;
10140 vec_alloc (vsize
, map_cnt
);
10141 vec_alloc (vkind
, map_cnt
);
10142 unsigned int map_idx
= 0;
10144 for (c
= clauses
; c
; c
= OMP_CLAUSE_CHAIN (c
))
10145 switch (OMP_CLAUSE_CODE (c
))
10151 case OMP_CLAUSE_MAP
:
10152 case OMP_CLAUSE_TO
:
10153 case OMP_CLAUSE_FROM
:
10155 ovar
= OMP_CLAUSE_DECL (c
);
10156 if (!DECL_P (ovar
))
10158 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
10159 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
))
10161 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c
))
10162 == get_base_address (ovar
));
10163 nc
= OMP_CLAUSE_CHAIN (c
);
10164 ovar
= OMP_CLAUSE_DECL (nc
);
10168 tree x
= build_sender_ref (ovar
, ctx
);
10170 = build_fold_addr_expr_with_type (ovar
, ptr_type_node
);
10171 gimplify_assign (x
, v
, &ilist
);
10177 if (DECL_SIZE (ovar
)
10178 && TREE_CODE (DECL_SIZE (ovar
)) != INTEGER_CST
)
10180 tree ovar2
= DECL_VALUE_EXPR (ovar
);
10181 gcc_assert (TREE_CODE (ovar2
) == INDIRECT_REF
);
10182 ovar2
= TREE_OPERAND (ovar2
, 0);
10183 gcc_assert (DECL_P (ovar2
));
10186 if (!maybe_lookup_field (ovar
, ctx
))
10190 unsigned int talign
= TYPE_ALIGN_UNIT (TREE_TYPE (ovar
));
10191 if (DECL_P (ovar
) && DECL_ALIGN_UNIT (ovar
) > talign
)
10192 talign
= DECL_ALIGN_UNIT (ovar
);
10195 tree var
= lookup_decl_in_outer_ctx (ovar
, ctx
);
10196 tree x
= build_sender_ref (ovar
, ctx
);
10197 if (OMP_CLAUSE_CODE (c
) == OMP_CLAUSE_MAP
10198 && OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_POINTER
10199 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c
)
10200 && TREE_CODE (TREE_TYPE (ovar
)) == ARRAY_TYPE
)
10202 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
10204 = create_tmp_var (TREE_TYPE (TREE_TYPE (x
)), NULL
);
10205 mark_addressable (avar
);
10206 gimplify_assign (avar
, build_fold_addr_expr (var
), &ilist
);
10207 talign
= DECL_ALIGN_UNIT (avar
);
10208 avar
= build_fold_addr_expr (avar
);
10209 gimplify_assign (x
, avar
, &ilist
);
10211 else if (is_gimple_reg (var
))
10213 gcc_assert (kind
== GF_OMP_TARGET_KIND_REGION
);
10214 tree avar
= create_tmp_var (TREE_TYPE (var
), NULL
);
10215 mark_addressable (avar
);
10216 if (OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_ALLOC
10217 && OMP_CLAUSE_MAP_KIND (c
) != OMP_CLAUSE_MAP_FROM
)
10218 gimplify_assign (avar
, var
, &ilist
);
10219 avar
= build_fold_addr_expr (avar
);
10220 gimplify_assign (x
, avar
, &ilist
);
10221 if ((OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_FROM
10222 || OMP_CLAUSE_MAP_KIND (c
) == OMP_CLAUSE_MAP_TOFROM
)
10223 && !TYPE_READONLY (TREE_TYPE (var
)))
10225 x
= build_sender_ref (ovar
, ctx
);
10226 x
= build_simple_mem_ref (x
);
10227 gimplify_assign (var
, x
, &olist
);
10232 var
= build_fold_addr_expr (var
);
10233 gimplify_assign (x
, var
, &ilist
);
10236 tree s
= OMP_CLAUSE_SIZE (c
);
10237 if (s
== NULL_TREE
)
10238 s
= TYPE_SIZE_UNIT (TREE_TYPE (ovar
));
10239 s
= fold_convert (size_type_node
, s
);
10240 tree purpose
= size_int (map_idx
++);
10241 CONSTRUCTOR_APPEND_ELT (vsize
, purpose
, s
);
10242 if (TREE_CODE (s
) != INTEGER_CST
)
10243 TREE_STATIC (TREE_VEC_ELT (t
, 1)) = 0;
10245 unsigned char tkind
= 0;
10246 switch (OMP_CLAUSE_CODE (c
))
10248 case OMP_CLAUSE_MAP
:
10249 tkind
= OMP_CLAUSE_MAP_KIND (c
);
10251 case OMP_CLAUSE_TO
:
10252 tkind
= OMP_CLAUSE_MAP_TO
;
10254 case OMP_CLAUSE_FROM
:
10255 tkind
= OMP_CLAUSE_MAP_FROM
;
10258 gcc_unreachable ();
10260 talign
= ceil_log2 (talign
);
10261 tkind
|= talign
<< 3;
10262 CONSTRUCTOR_APPEND_ELT (vkind
, purpose
,
10263 build_int_cst (unsigned_char_type_node
,
10269 gcc_assert (map_idx
== map_cnt
);
10271 DECL_INITIAL (TREE_VEC_ELT (t
, 1))
10272 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)), vsize
);
10273 DECL_INITIAL (TREE_VEC_ELT (t
, 2))
10274 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 2)), vkind
);
10275 if (!TREE_STATIC (TREE_VEC_ELT (t
, 1)))
10277 gimple_seq initlist
= NULL
;
10278 force_gimple_operand (build1 (DECL_EXPR
, void_type_node
,
10279 TREE_VEC_ELT (t
, 1)),
10280 &initlist
, true, NULL_TREE
);
10281 gimple_seq_add_seq (&ilist
, initlist
);
10283 tree clobber
= build_constructor (TREE_TYPE (TREE_VEC_ELT (t
, 1)),
10285 TREE_THIS_VOLATILE (clobber
) = 1;
10286 gimple_seq_add_stmt (&olist
,
10287 gimple_build_assign (TREE_VEC_ELT (t
, 1),
10291 tree clobber
= build_constructor (ctx
->record_type
, NULL
);
10292 TREE_THIS_VOLATILE (clobber
) = 1;
10293 gimple_seq_add_stmt (&olist
, gimple_build_assign (ctx
->sender_decl
,
10297 /* Once all the expansions are done, sequence all the different
10298 fragments inside gimple_omp_body. */
10302 if (ctx
->record_type
&& kind
== GF_OMP_TARGET_KIND_REGION
)
10304 t
= build_fold_addr_expr_loc (loc
, ctx
->sender_decl
);
10305 /* fixup_child_record_type might have changed receiver_decl's type. */
10306 t
= fold_convert_loc (loc
, TREE_TYPE (ctx
->receiver_decl
), t
);
10307 gimple_seq_add_stmt (&new_body
,
10308 gimple_build_assign (ctx
->receiver_decl
, t
));
10311 if (kind
== GF_OMP_TARGET_KIND_REGION
)
10313 gimple_seq_add_seq (&new_body
, tgt_body
);
10314 new_body
= maybe_catch_exception (new_body
);
10316 else if (kind
== GF_OMP_TARGET_KIND_DATA
)
10317 new_body
= tgt_body
;
10318 if (kind
!= GF_OMP_TARGET_KIND_UPDATE
)
10320 gimple_seq_add_stmt (&new_body
, gimple_build_omp_return (false));
10321 gimple_omp_set_body (stmt
, new_body
);
10324 bind
= gimple_build_bind (NULL
, NULL
,
10325 tgt_bind
? gimple_bind_block (tgt_bind
)
10327 gsi_replace (gsi_p
, bind
, true);
10328 gimple_bind_add_seq (bind
, ilist
);
10329 gimple_bind_add_stmt (bind
, stmt
);
10330 gimple_bind_add_seq (bind
, olist
);
10332 pop_gimplify_context (NULL
);
10335 /* Expand code for an OpenMP teams directive. */
10338 lower_omp_teams (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10340 gimple teams_stmt
= gsi_stmt (*gsi_p
);
10341 push_gimplify_context ();
10343 tree block
= make_node (BLOCK
);
10344 gimple bind
= gimple_build_bind (NULL
, NULL
, block
);
10345 gsi_replace (gsi_p
, bind
, true);
10346 gimple_seq bind_body
= NULL
;
10347 gimple_seq dlist
= NULL
;
10348 gimple_seq olist
= NULL
;
10350 tree num_teams
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
10351 OMP_CLAUSE_NUM_TEAMS
);
10352 if (num_teams
== NULL_TREE
)
10353 num_teams
= build_int_cst (unsigned_type_node
, 0);
10356 num_teams
= OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams
);
10357 num_teams
= fold_convert (unsigned_type_node
, num_teams
);
10358 gimplify_expr (&num_teams
, &bind_body
, NULL
, is_gimple_val
, fb_rvalue
);
10360 tree thread_limit
= find_omp_clause (gimple_omp_teams_clauses (teams_stmt
),
10361 OMP_CLAUSE_THREAD_LIMIT
);
10362 if (thread_limit
== NULL_TREE
)
10363 thread_limit
= build_int_cst (unsigned_type_node
, 0);
10366 thread_limit
= OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit
);
10367 thread_limit
= fold_convert (unsigned_type_node
, thread_limit
);
10368 gimplify_expr (&thread_limit
, &bind_body
, NULL
, is_gimple_val
,
10372 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt
),
10373 &bind_body
, &dlist
, ctx
, NULL
);
10374 lower_omp (gimple_omp_body_ptr (teams_stmt
), ctx
);
10375 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt
), &olist
, ctx
);
10376 gimple_seq_add_stmt (&bind_body
, teams_stmt
);
10378 location_t loc
= gimple_location (teams_stmt
);
10379 tree decl
= builtin_decl_explicit (BUILT_IN_GOMP_TEAMS
);
10380 gimple call
= gimple_build_call (decl
, 2, num_teams
, thread_limit
);
10381 gimple_set_location (call
, loc
);
10382 gimple_seq_add_stmt (&bind_body
, call
);
10384 gimple_seq_add_seq (&bind_body
, gimple_omp_body (teams_stmt
));
10385 gimple_omp_set_body (teams_stmt
, NULL
);
10386 gimple_seq_add_seq (&bind_body
, olist
);
10387 gimple_seq_add_seq (&bind_body
, dlist
);
10388 gimple_seq_add_stmt (&bind_body
, gimple_build_omp_return (true));
10389 gimple_bind_set_body (bind
, bind_body
);
10391 pop_gimplify_context (bind
);
10393 gimple_bind_append_vars (bind
, ctx
->block_vars
);
10394 BLOCK_VARS (block
) = ctx
->block_vars
;
10395 if (BLOCK_VARS (block
))
10396 TREE_USED (block
) = 1;
10400 /* Callback for lower_omp_1. Return non-NULL if *tp needs to be
10401 regimplified. If DATA is non-NULL, lower_omp_1 is outside
10402 of OpenMP context, but with task_shared_vars set. */
10405 lower_omp_regimplify_p (tree
*tp
, int *walk_subtrees
,
10410 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */
10411 if (TREE_CODE (t
) == VAR_DECL
&& data
== NULL
&& DECL_HAS_VALUE_EXPR_P (t
))
10414 if (task_shared_vars
10416 && bitmap_bit_p (task_shared_vars
, DECL_UID (t
)))
10419 /* If a global variable has been privatized, TREE_CONSTANT on
10420 ADDR_EXPR might be wrong. */
10421 if (data
== NULL
&& TREE_CODE (t
) == ADDR_EXPR
)
10422 recompute_tree_invariant_for_addr_expr (t
);
10424 *walk_subtrees
= !TYPE_P (t
) && !DECL_P (t
);
10429 lower_omp_1 (gimple_stmt_iterator
*gsi_p
, omp_context
*ctx
)
10431 gimple stmt
= gsi_stmt (*gsi_p
);
10432 struct walk_stmt_info wi
;
10434 if (gimple_has_location (stmt
))
10435 input_location
= gimple_location (stmt
);
10437 if (task_shared_vars
)
10438 memset (&wi
, '\0', sizeof (wi
));
10440 /* If we have issued syntax errors, avoid doing any heavy lifting.
10441 Just replace the OpenMP directives with a NOP to avoid
10442 confusing RTL expansion. */
10443 if (seen_error () && is_gimple_omp (stmt
))
10445 gsi_replace (gsi_p
, gimple_build_nop (), true);
10449 switch (gimple_code (stmt
))
10452 if ((ctx
|| task_shared_vars
)
10453 && (walk_tree (gimple_cond_lhs_ptr (stmt
), lower_omp_regimplify_p
,
10454 ctx
? NULL
: &wi
, NULL
)
10455 || walk_tree (gimple_cond_rhs_ptr (stmt
), lower_omp_regimplify_p
,
10456 ctx
? NULL
: &wi
, NULL
)))
10457 gimple_regimplify_operands (stmt
, gsi_p
);
10460 lower_omp (gimple_catch_handler_ptr (stmt
), ctx
);
10462 case GIMPLE_EH_FILTER
:
10463 lower_omp (gimple_eh_filter_failure_ptr (stmt
), ctx
);
10466 lower_omp (gimple_try_eval_ptr (stmt
), ctx
);
10467 lower_omp (gimple_try_cleanup_ptr (stmt
), ctx
);
10469 case GIMPLE_TRANSACTION
:
10470 lower_omp (gimple_transaction_body_ptr (stmt
), ctx
);
10473 lower_omp (gimple_bind_body_ptr (stmt
), ctx
);
10475 case GIMPLE_OMP_PARALLEL
:
10476 case GIMPLE_OMP_TASK
:
10477 ctx
= maybe_lookup_ctx (stmt
);
10479 if (ctx
->cancellable
)
10480 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10481 lower_omp_taskreg (gsi_p
, ctx
);
10483 case GIMPLE_OMP_FOR
:
10484 ctx
= maybe_lookup_ctx (stmt
);
10486 if (ctx
->cancellable
)
10487 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10488 lower_omp_for (gsi_p
, ctx
);
10490 case GIMPLE_OMP_SECTIONS
:
10491 ctx
= maybe_lookup_ctx (stmt
);
10493 if (ctx
->cancellable
)
10494 ctx
->cancel_label
= create_artificial_label (UNKNOWN_LOCATION
);
10495 lower_omp_sections (gsi_p
, ctx
);
10497 case GIMPLE_OMP_SINGLE
:
10498 ctx
= maybe_lookup_ctx (stmt
);
10500 lower_omp_single (gsi_p
, ctx
);
10502 case GIMPLE_OMP_MASTER
:
10503 ctx
= maybe_lookup_ctx (stmt
);
10505 lower_omp_master (gsi_p
, ctx
);
10507 case GIMPLE_OMP_TASKGROUP
:
10508 ctx
= maybe_lookup_ctx (stmt
);
10510 lower_omp_taskgroup (gsi_p
, ctx
);
10512 case GIMPLE_OMP_ORDERED
:
10513 ctx
= maybe_lookup_ctx (stmt
);
10515 lower_omp_ordered (gsi_p
, ctx
);
10517 case GIMPLE_OMP_CRITICAL
:
10518 ctx
= maybe_lookup_ctx (stmt
);
10520 lower_omp_critical (gsi_p
, ctx
);
10522 case GIMPLE_OMP_ATOMIC_LOAD
:
10523 if ((ctx
|| task_shared_vars
)
10524 && walk_tree (gimple_omp_atomic_load_rhs_ptr (stmt
),
10525 lower_omp_regimplify_p
, ctx
? NULL
: &wi
, NULL
))
10526 gimple_regimplify_operands (stmt
, gsi_p
);
10528 case GIMPLE_OMP_TARGET
:
10529 ctx
= maybe_lookup_ctx (stmt
);
10531 lower_omp_target (gsi_p
, ctx
);
10533 case GIMPLE_OMP_TEAMS
:
10534 ctx
= maybe_lookup_ctx (stmt
);
10536 lower_omp_teams (gsi_p
, ctx
);
10540 fndecl
= gimple_call_fndecl (stmt
);
10542 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
10543 switch (DECL_FUNCTION_CODE (fndecl
))
10545 case BUILT_IN_GOMP_BARRIER
:
10549 case BUILT_IN_GOMP_CANCEL
:
10550 case BUILT_IN_GOMP_CANCELLATION_POINT
:
10553 if (gimple_code (cctx
->stmt
) == GIMPLE_OMP_SECTION
)
10554 cctx
= cctx
->outer
;
10555 gcc_assert (gimple_call_lhs (stmt
) == NULL_TREE
);
10556 if (!cctx
->cancellable
)
10558 if (DECL_FUNCTION_CODE (fndecl
)
10559 == BUILT_IN_GOMP_CANCELLATION_POINT
)
10561 stmt
= gimple_build_nop ();
10562 gsi_replace (gsi_p
, stmt
, false);
10566 if (DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_GOMP_BARRIER
)
10568 fndecl
= builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL
);
10569 gimple_call_set_fndecl (stmt
, fndecl
);
10570 gimple_call_set_fntype (stmt
, TREE_TYPE (fndecl
));
10573 lhs
= create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl
)), NULL
);
10574 gimple_call_set_lhs (stmt
, lhs
);
10575 tree fallthru_label
;
10576 fallthru_label
= create_artificial_label (UNKNOWN_LOCATION
);
10578 g
= gimple_build_label (fallthru_label
);
10579 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10580 g
= gimple_build_cond (NE_EXPR
, lhs
,
10581 fold_convert (TREE_TYPE (lhs
),
10582 boolean_false_node
),
10583 cctx
->cancel_label
, fallthru_label
);
10584 gsi_insert_after (gsi_p
, g
, GSI_SAME_STMT
);
10591 if ((ctx
|| task_shared_vars
)
10592 && walk_gimple_op (stmt
, lower_omp_regimplify_p
,
10595 /* Just remove clobbers, this should happen only if we have
10596 "privatized" local addressable variables in SIMD regions,
10597 the clobber isn't needed in that case and gimplifying address
10598 of the ARRAY_REF into a pointer and creating MEM_REF based
10599 clobber would create worse code than we get with the clobber
10601 if (gimple_clobber_p (stmt
))
10603 gsi_replace (gsi_p
, gimple_build_nop (), true);
10606 gimple_regimplify_operands (stmt
, gsi_p
);
10613 lower_omp (gimple_seq
*body
, omp_context
*ctx
)
10615 location_t saved_location
= input_location
;
10616 gimple_stmt_iterator gsi
;
10617 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10618 lower_omp_1 (&gsi
, ctx
);
10619 /* During gimplification, we have not always invoked fold_stmt
10620 (gimplify.c:maybe_fold_stmt); call it now. */
10621 if (target_nesting_level
)
10622 for (gsi
= gsi_start (*body
); !gsi_end_p (gsi
); gsi_next (&gsi
))
10624 input_location
= saved_location
;
10627 /* Main entry point. */
10629 static unsigned int
10630 execute_lower_omp (void)
10636 /* This pass always runs, to provide PROP_gimple_lomp.
10637 But there is nothing to do unless -fopenmp is given. */
10638 if (flag_openmp
== 0 && flag_openmp_simd
== 0 && flag_cilkplus
== 0)
10641 all_contexts
= splay_tree_new (splay_tree_compare_pointers
, 0,
10642 delete_omp_context
);
10644 body
= gimple_body (current_function_decl
);
10645 scan_omp (&body
, NULL
);
10646 gcc_assert (taskreg_nesting_level
== 0);
10647 FOR_EACH_VEC_ELT (taskreg_contexts
, i
, ctx
)
10648 finish_taskreg_scan (ctx
);
10649 taskreg_contexts
.release ();
10651 if (all_contexts
->root
)
10653 if (task_shared_vars
)
10654 push_gimplify_context ();
10655 lower_omp (&body
, NULL
);
10656 if (task_shared_vars
)
10657 pop_gimplify_context (NULL
);
10662 splay_tree_delete (all_contexts
);
10663 all_contexts
= NULL
;
10665 BITMAP_FREE (task_shared_vars
);
10671 const pass_data pass_data_lower_omp
=
10673 GIMPLE_PASS
, /* type */
10674 "omplower", /* name */
10675 OPTGROUP_NONE
, /* optinfo_flags */
10676 TV_NONE
, /* tv_id */
10677 PROP_gimple_any
, /* properties_required */
10678 PROP_gimple_lomp
, /* properties_provided */
10679 0, /* properties_destroyed */
10680 0, /* todo_flags_start */
10681 0, /* todo_flags_finish */
10684 class pass_lower_omp
: public gimple_opt_pass
10687 pass_lower_omp (gcc::context
*ctxt
)
10688 : gimple_opt_pass (pass_data_lower_omp
, ctxt
)
10691 /* opt_pass methods: */
10692 virtual unsigned int execute (function
*) { return execute_lower_omp (); }
10694 }; // class pass_lower_omp
10696 } // anon namespace
10699 make_pass_lower_omp (gcc::context
*ctxt
)
10701 return new pass_lower_omp (ctxt
);
10704 /* The following is a utility to diagnose OpenMP structured block violations.
10705 It is not part of the "omplower" pass, as that's invoked too late. It
10706 should be invoked by the respective front ends after gimplification. */
10708 static splay_tree all_labels
;
10710 /* Check for mismatched contexts and generate an error if needed. Return
10711 true if an error is detected. */
10714 diagnose_sb_0 (gimple_stmt_iterator
*gsi_p
,
10715 gimple branch_ctx
, gimple label_ctx
)
10717 if (label_ctx
== branch_ctx
)
10722 Previously we kept track of the label's entire context in diagnose_sb_[12]
10723 so we could traverse it and issue a correct "exit" or "enter" error
10724 message upon a structured block violation.
10726 We built the context by building a list with tree_cons'ing, but there is
10727 no easy counterpart in gimple tuples. It seems like far too much work
10728 for issuing exit/enter error messages. If someone really misses the
10729 distinct error message... patches welcome.
10733 /* Try to avoid confusing the user by producing and error message
10734 with correct "exit" or "enter" verbiage. We prefer "exit"
10735 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */
10736 if (branch_ctx
== NULL
)
10742 if (TREE_VALUE (label_ctx
) == branch_ctx
)
10747 label_ctx
= TREE_CHAIN (label_ctx
);
10752 error ("invalid exit from OpenMP structured block");
10754 error ("invalid entry to OpenMP structured block");
10757 bool cilkplus_block
= false;
10761 && gimple_code (branch_ctx
) == GIMPLE_OMP_FOR
10762 && gimple_omp_for_kind (branch_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
)
10764 && gimple_code (label_ctx
) == GIMPLE_OMP_FOR
10765 && gimple_omp_for_kind (label_ctx
) == GF_OMP_FOR_KIND_CILKSIMD
))
10766 cilkplus_block
= true;
10769 /* If it's obvious we have an invalid entry, be specific about the error. */
10770 if (branch_ctx
== NULL
)
10772 if (cilkplus_block
)
10773 error ("invalid entry to Cilk Plus structured block");
10775 error ("invalid entry to OpenMP structured block");
10779 /* Otherwise, be vague and lazy, but efficient. */
10780 if (cilkplus_block
)
10781 error ("invalid branch to/from a Cilk Plus structured block");
10783 error ("invalid branch to/from an OpenMP structured block");
10786 gsi_replace (gsi_p
, gimple_build_nop (), false);
10790 /* Pass 1: Create a minimal tree of OpenMP structured blocks, and record
10791 where each label is found. */
10794 diagnose_sb_1 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10795 struct walk_stmt_info
*wi
)
10797 gimple context
= (gimple
) wi
->info
;
10798 gimple inner_context
;
10799 gimple stmt
= gsi_stmt (*gsi_p
);
10801 *handled_ops_p
= true;
10803 switch (gimple_code (stmt
))
10807 case GIMPLE_OMP_PARALLEL
:
10808 case GIMPLE_OMP_TASK
:
10809 case GIMPLE_OMP_SECTIONS
:
10810 case GIMPLE_OMP_SINGLE
:
10811 case GIMPLE_OMP_SECTION
:
10812 case GIMPLE_OMP_MASTER
:
10813 case GIMPLE_OMP_ORDERED
:
10814 case GIMPLE_OMP_CRITICAL
:
10815 case GIMPLE_OMP_TARGET
:
10816 case GIMPLE_OMP_TEAMS
:
10817 case GIMPLE_OMP_TASKGROUP
:
10818 /* The minimal context here is just the current OMP construct. */
10819 inner_context
= stmt
;
10820 wi
->info
= inner_context
;
10821 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10822 wi
->info
= context
;
10825 case GIMPLE_OMP_FOR
:
10826 inner_context
= stmt
;
10827 wi
->info
= inner_context
;
10828 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10830 walk_gimple_seq (gimple_omp_for_pre_body (stmt
),
10831 diagnose_sb_1
, NULL
, wi
);
10832 walk_gimple_seq (gimple_omp_body (stmt
), diagnose_sb_1
, NULL
, wi
);
10833 wi
->info
= context
;
10837 splay_tree_insert (all_labels
, (splay_tree_key
) gimple_label_label (stmt
),
10838 (splay_tree_value
) context
);
10848 /* Pass 2: Check each branch and see if its context differs from that of
10849 the destination label's context. */
10852 diagnose_sb_2 (gimple_stmt_iterator
*gsi_p
, bool *handled_ops_p
,
10853 struct walk_stmt_info
*wi
)
10855 gimple context
= (gimple
) wi
->info
;
10857 gimple stmt
= gsi_stmt (*gsi_p
);
10859 *handled_ops_p
= true;
10861 switch (gimple_code (stmt
))
10865 case GIMPLE_OMP_PARALLEL
:
10866 case GIMPLE_OMP_TASK
:
10867 case GIMPLE_OMP_SECTIONS
:
10868 case GIMPLE_OMP_SINGLE
:
10869 case GIMPLE_OMP_SECTION
:
10870 case GIMPLE_OMP_MASTER
:
10871 case GIMPLE_OMP_ORDERED
:
10872 case GIMPLE_OMP_CRITICAL
:
10873 case GIMPLE_OMP_TARGET
:
10874 case GIMPLE_OMP_TEAMS
:
10875 case GIMPLE_OMP_TASKGROUP
:
10877 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10878 wi
->info
= context
;
10881 case GIMPLE_OMP_FOR
:
10883 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to
10885 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt
),
10886 diagnose_sb_2
, NULL
, wi
);
10887 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt
), diagnose_sb_2
, NULL
, wi
);
10888 wi
->info
= context
;
10893 tree lab
= gimple_cond_true_label (stmt
);
10896 n
= splay_tree_lookup (all_labels
,
10897 (splay_tree_key
) lab
);
10898 diagnose_sb_0 (gsi_p
, context
,
10899 n
? (gimple
) n
->value
: NULL
);
10901 lab
= gimple_cond_false_label (stmt
);
10904 n
= splay_tree_lookup (all_labels
,
10905 (splay_tree_key
) lab
);
10906 diagnose_sb_0 (gsi_p
, context
,
10907 n
? (gimple
) n
->value
: NULL
);
10914 tree lab
= gimple_goto_dest (stmt
);
10915 if (TREE_CODE (lab
) != LABEL_DECL
)
10918 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10919 diagnose_sb_0 (gsi_p
, context
, n
? (gimple
) n
->value
: NULL
);
10923 case GIMPLE_SWITCH
:
10926 for (i
= 0; i
< gimple_switch_num_labels (stmt
); ++i
)
10928 tree lab
= CASE_LABEL (gimple_switch_label (stmt
, i
));
10929 n
= splay_tree_lookup (all_labels
, (splay_tree_key
) lab
);
10930 if (n
&& diagnose_sb_0 (gsi_p
, context
, (gimple
) n
->value
))
10936 case GIMPLE_RETURN
:
10937 diagnose_sb_0 (gsi_p
, context
, NULL
);
10947 /* Called from tree-cfg.c::make_edges to create cfg edges for all GIMPLE_OMP
10950 make_gimple_omp_edges (basic_block bb
, struct omp_region
**region
,
10953 gimple last
= last_stmt (bb
);
10954 enum gimple_code code
= gimple_code (last
);
10955 struct omp_region
*cur_region
= *region
;
10956 bool fallthru
= false;
10960 case GIMPLE_OMP_PARALLEL
:
10961 case GIMPLE_OMP_TASK
:
10962 case GIMPLE_OMP_FOR
:
10963 case GIMPLE_OMP_SINGLE
:
10964 case GIMPLE_OMP_TEAMS
:
10965 case GIMPLE_OMP_MASTER
:
10966 case GIMPLE_OMP_TASKGROUP
:
10967 case GIMPLE_OMP_ORDERED
:
10968 case GIMPLE_OMP_CRITICAL
:
10969 case GIMPLE_OMP_SECTION
:
10970 cur_region
= new_omp_region (bb
, code
, cur_region
);
10974 case GIMPLE_OMP_TARGET
:
10975 cur_region
= new_omp_region (bb
, code
, cur_region
);
10977 if (gimple_omp_target_kind (last
) == GF_OMP_TARGET_KIND_UPDATE
)
10978 cur_region
= cur_region
->outer
;
10981 case GIMPLE_OMP_SECTIONS
:
10982 cur_region
= new_omp_region (bb
, code
, cur_region
);
10986 case GIMPLE_OMP_SECTIONS_SWITCH
:
10990 case GIMPLE_OMP_ATOMIC_LOAD
:
10991 case GIMPLE_OMP_ATOMIC_STORE
:
10995 case GIMPLE_OMP_RETURN
:
10996 /* In the case of a GIMPLE_OMP_SECTION, the edge will go
10997 somewhere other than the next block. This will be
10999 cur_region
->exit
= bb
;
11000 fallthru
= cur_region
->type
!= GIMPLE_OMP_SECTION
;
11001 cur_region
= cur_region
->outer
;
11004 case GIMPLE_OMP_CONTINUE
:
11005 cur_region
->cont
= bb
;
11006 switch (cur_region
->type
)
11008 case GIMPLE_OMP_FOR
:
11009 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE
11010 succs edges as abnormal to prevent splitting
11012 single_succ_edge (cur_region
->entry
)->flags
|= EDGE_ABNORMAL
;
11013 /* Make the loopback edge. */
11014 make_edge (bb
, single_succ (cur_region
->entry
),
11017 /* Create an edge from GIMPLE_OMP_FOR to exit, which
11018 corresponds to the case that the body of the loop
11019 is not executed at all. */
11020 make_edge (cur_region
->entry
, bb
->next_bb
, EDGE_ABNORMAL
);
11021 make_edge (bb
, bb
->next_bb
, EDGE_FALLTHRU
| EDGE_ABNORMAL
);
11025 case GIMPLE_OMP_SECTIONS
:
11026 /* Wire up the edges into and out of the nested sections. */
11028 basic_block switch_bb
= single_succ (cur_region
->entry
);
11030 struct omp_region
*i
;
11031 for (i
= cur_region
->inner
; i
; i
= i
->next
)
11033 gcc_assert (i
->type
== GIMPLE_OMP_SECTION
);
11034 make_edge (switch_bb
, i
->entry
, 0);
11035 make_edge (i
->exit
, bb
, EDGE_FALLTHRU
);
11038 /* Make the loopback edge to the block with
11039 GIMPLE_OMP_SECTIONS_SWITCH. */
11040 make_edge (bb
, switch_bb
, 0);
11042 /* Make the edge from the switch to exit. */
11043 make_edge (switch_bb
, bb
->next_bb
, 0);
11049 gcc_unreachable ();
11054 gcc_unreachable ();
11057 if (*region
!= cur_region
)
11059 *region
= cur_region
;
11061 *region_idx
= cur_region
->entry
->index
;
11069 static unsigned int
11070 diagnose_omp_structured_block_errors (void)
11072 struct walk_stmt_info wi
;
11073 gimple_seq body
= gimple_body (current_function_decl
);
11075 all_labels
= splay_tree_new (splay_tree_compare_pointers
, 0, 0);
11077 memset (&wi
, 0, sizeof (wi
));
11078 walk_gimple_seq (body
, diagnose_sb_1
, NULL
, &wi
);
11080 memset (&wi
, 0, sizeof (wi
));
11081 wi
.want_locations
= true;
11082 walk_gimple_seq_mod (&body
, diagnose_sb_2
, NULL
, &wi
);
11084 gimple_set_body (current_function_decl
, body
);
11086 splay_tree_delete (all_labels
);
11094 const pass_data pass_data_diagnose_omp_blocks
=
11096 GIMPLE_PASS
, /* type */
11097 "*diagnose_omp_blocks", /* name */
11098 OPTGROUP_NONE
, /* optinfo_flags */
11099 TV_NONE
, /* tv_id */
11100 PROP_gimple_any
, /* properties_required */
11101 0, /* properties_provided */
11102 0, /* properties_destroyed */
11103 0, /* todo_flags_start */
11104 0, /* todo_flags_finish */
11107 class pass_diagnose_omp_blocks
: public gimple_opt_pass
11110 pass_diagnose_omp_blocks (gcc::context
*ctxt
)
11111 : gimple_opt_pass (pass_data_diagnose_omp_blocks
, ctxt
)
11114 /* opt_pass methods: */
11115 virtual bool gate (function
*) { return flag_openmp
|| flag_cilkplus
; }
11116 virtual unsigned int execute (function
*)
11118 return diagnose_omp_structured_block_errors ();
11121 }; // class pass_diagnose_omp_blocks
11123 } // anon namespace
11126 make_pass_diagnose_omp_blocks (gcc::context
*ctxt
)
11128 return new pass_diagnose_omp_blocks (ctxt
);
11131 /* SIMD clone supporting code. */
11133 /* Allocate a fresh `simd_clone' and return it. NARGS is the number
11134 of arguments to reserve space for. */
11136 static struct cgraph_simd_clone
*
11137 simd_clone_struct_alloc (int nargs
)
11139 struct cgraph_simd_clone
*clone_info
;
11140 size_t len
= (sizeof (struct cgraph_simd_clone
)
11141 + nargs
* sizeof (struct cgraph_simd_clone_arg
));
11142 clone_info
= (struct cgraph_simd_clone
*)
11143 ggc_internal_cleared_alloc (len
);
11147 /* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */
11150 simd_clone_struct_copy (struct cgraph_simd_clone
*to
,
11151 struct cgraph_simd_clone
*from
)
11153 memcpy (to
, from
, (sizeof (struct cgraph_simd_clone
)
11154 + ((from
->nargs
- from
->inbranch
)
11155 * sizeof (struct cgraph_simd_clone_arg
))));
11158 /* Return vector of parameter types of function FNDECL. This uses
11159 TYPE_ARG_TYPES if available, otherwise falls back to types of
11160 DECL_ARGUMENTS types. */
11163 simd_clone_vector_of_formal_parm_types (tree fndecl
)
11165 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl
)))
11166 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl
));
11167 vec
<tree
> args
= ipa_get_vector_of_formal_parms (fndecl
);
11170 FOR_EACH_VEC_ELT (args
, i
, arg
)
11171 args
[i
] = TREE_TYPE (args
[i
]);
11175 /* Given a simd function in NODE, extract the simd specific
11176 information from the OMP clauses passed in CLAUSES, and return
11177 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED
11178 is set to TRUE if the `inbranch' or `notinbranch' clause specified,
11179 otherwise set to FALSE. */
11181 static struct cgraph_simd_clone
*
11182 simd_clone_clauses_extract (struct cgraph_node
*node
, tree clauses
,
11183 bool *inbranch_specified
)
11185 vec
<tree
> args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
11188 *inbranch_specified
= false;
11190 n
= args
.length ();
11191 if (n
> 0 && args
.last () == void_type_node
)
11194 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to
11195 be cloned have a distinctive artificial label in addition to "omp
11199 && lookup_attribute ("cilk simd function",
11200 DECL_ATTRIBUTES (node
->decl
)));
11202 /* Allocate one more than needed just in case this is an in-branch
11203 clone which will require a mask argument. */
11204 struct cgraph_simd_clone
*clone_info
= simd_clone_struct_alloc (n
+ 1);
11205 clone_info
->nargs
= n
;
11206 clone_info
->cilk_elemental
= cilk_clone
;
11213 clauses
= TREE_VALUE (clauses
);
11214 if (!clauses
|| TREE_CODE (clauses
) != OMP_CLAUSE
)
11217 for (t
= clauses
; t
; t
= OMP_CLAUSE_CHAIN (t
))
11219 switch (OMP_CLAUSE_CODE (t
))
11221 case OMP_CLAUSE_INBRANCH
:
11222 clone_info
->inbranch
= 1;
11223 *inbranch_specified
= true;
11225 case OMP_CLAUSE_NOTINBRANCH
:
11226 clone_info
->inbranch
= 0;
11227 *inbranch_specified
= true;
11229 case OMP_CLAUSE_SIMDLEN
:
11230 clone_info
->simdlen
11231 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t
));
11233 case OMP_CLAUSE_LINEAR
:
11235 tree decl
= OMP_CLAUSE_DECL (t
);
11236 tree step
= OMP_CLAUSE_LINEAR_STEP (t
);
11237 int argno
= TREE_INT_CST_LOW (decl
);
11238 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t
))
11240 clone_info
->args
[argno
].arg_type
11241 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
;
11242 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
11243 gcc_assert (clone_info
->args
[argno
].linear_step
>= 0
11244 && clone_info
->args
[argno
].linear_step
< n
);
11248 if (POINTER_TYPE_P (args
[argno
]))
11249 step
= fold_convert (ssizetype
, step
);
11250 if (!tree_fits_shwi_p (step
))
11252 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
11253 "ignoring large linear step");
11257 else if (integer_zerop (step
))
11259 warning_at (OMP_CLAUSE_LOCATION (t
), 0,
11260 "ignoring zero linear step");
11266 clone_info
->args
[argno
].arg_type
11267 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
;
11268 clone_info
->args
[argno
].linear_step
= tree_to_shwi (step
);
11273 case OMP_CLAUSE_UNIFORM
:
11275 tree decl
= OMP_CLAUSE_DECL (t
);
11276 int argno
= tree_to_uhwi (decl
);
11277 clone_info
->args
[argno
].arg_type
11278 = SIMD_CLONE_ARG_TYPE_UNIFORM
;
11281 case OMP_CLAUSE_ALIGNED
:
11283 tree decl
= OMP_CLAUSE_DECL (t
);
11284 int argno
= tree_to_uhwi (decl
);
11285 clone_info
->args
[argno
].alignment
11286 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t
));
11297 /* Given a SIMD clone in NODE, calculate the characteristic data
11298 type and return the coresponding type. The characteristic data
11299 type is computed as described in the Intel Vector ABI. */
11302 simd_clone_compute_base_data_type (struct cgraph_node
*node
,
11303 struct cgraph_simd_clone
*clone_info
)
11305 tree type
= integer_type_node
;
11306 tree fndecl
= node
->decl
;
11308 /* a) For non-void function, the characteristic data type is the
11310 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl
))) != VOID_TYPE
)
11311 type
= TREE_TYPE (TREE_TYPE (fndecl
));
11313 /* b) If the function has any non-uniform, non-linear parameters,
11314 then the characteristic data type is the type of the first
11318 vec
<tree
> map
= simd_clone_vector_of_formal_parm_types (fndecl
);
11319 for (unsigned int i
= 0; i
< clone_info
->nargs
; ++i
)
11320 if (clone_info
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_VECTOR
)
11328 /* c) If the characteristic data type determined by a) or b) above
11329 is struct, union, or class type which is pass-by-value (except
11330 for the type that maps to the built-in complex data type), the
11331 characteristic data type is int. */
11332 if (RECORD_OR_UNION_TYPE_P (type
)
11333 && !aggregate_value_p (type
, NULL
)
11334 && TREE_CODE (type
) != COMPLEX_TYPE
)
11335 return integer_type_node
;
11337 /* d) If none of the above three classes is applicable, the
11338 characteristic data type is int. */
11342 /* e) For Intel Xeon Phi native and offload compilation, if the
11343 resulting characteristic data type is 8-bit or 16-bit integer
11344 data type, the characteristic data type is int. */
11345 /* Well, we don't handle Xeon Phi yet. */
11349 simd_clone_mangle (struct cgraph_node
*node
,
11350 struct cgraph_simd_clone
*clone_info
)
11352 char vecsize_mangle
= clone_info
->vecsize_mangle
;
11353 char mask
= clone_info
->inbranch
? 'M' : 'N';
11354 unsigned int simdlen
= clone_info
->simdlen
;
11358 gcc_assert (vecsize_mangle
&& simdlen
);
11360 pp_string (&pp
, "_ZGV");
11361 pp_character (&pp
, vecsize_mangle
);
11362 pp_character (&pp
, mask
);
11363 pp_decimal_int (&pp
, simdlen
);
11365 for (n
= 0; n
< clone_info
->nargs
; ++n
)
11367 struct cgraph_simd_clone_arg arg
= clone_info
->args
[n
];
11369 if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
)
11370 pp_character (&pp
, 'u');
11371 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
11373 gcc_assert (arg
.linear_step
!= 0);
11374 pp_character (&pp
, 'l');
11375 if (arg
.linear_step
> 1)
11376 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
11377 else if (arg
.linear_step
< 0)
11379 pp_character (&pp
, 'n');
11380 pp_unsigned_wide_integer (&pp
, (-(unsigned HOST_WIDE_INT
)
11384 else if (arg
.arg_type
== SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP
)
11386 pp_character (&pp
, 's');
11387 pp_unsigned_wide_integer (&pp
, arg
.linear_step
);
11390 pp_character (&pp
, 'v');
11393 pp_character (&pp
, 'a');
11394 pp_decimal_int (&pp
, arg
.alignment
);
11398 pp_underscore (&pp
);
11400 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node
->decl
)));
11401 const char *str
= pp_formatted_text (&pp
);
11403 /* If there already is a SIMD clone with the same mangled name, don't
11404 add another one. This can happen e.g. for
11405 #pragma omp declare simd
11406 #pragma omp declare simd simdlen(8)
11407 int foo (int, int);
11408 if the simdlen is assumed to be 8 for the first one, etc. */
11409 for (struct cgraph_node
*clone
= node
->simd_clones
; clone
;
11410 clone
= clone
->simdclone
->next_clone
)
11411 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone
->decl
)),
11415 return get_identifier (str
);
11418 /* Create a simd clone of OLD_NODE and return it. */
11420 static struct cgraph_node
*
11421 simd_clone_create (struct cgraph_node
*old_node
)
11423 struct cgraph_node
*new_node
;
11424 if (old_node
->definition
)
11426 if (!old_node
->has_gimple_body_p ())
11428 old_node
->get_body ();
11429 new_node
= old_node
->create_version_clone_with_body (vNULL
, NULL
, NULL
,
11435 tree old_decl
= old_node
->decl
;
11436 tree new_decl
= copy_node (old_node
->decl
);
11437 DECL_NAME (new_decl
) = clone_function_name (old_decl
, "simdclone");
11438 SET_DECL_ASSEMBLER_NAME (new_decl
, DECL_NAME (new_decl
));
11439 SET_DECL_RTL (new_decl
, NULL
);
11440 DECL_STATIC_CONSTRUCTOR (new_decl
) = 0;
11441 DECL_STATIC_DESTRUCTOR (new_decl
) = 0;
11442 new_node
= old_node
->create_version_clone (new_decl
, vNULL
, NULL
);
11443 symtab
->call_cgraph_insertion_hooks (new_node
);
11445 if (new_node
== NULL
)
11448 TREE_PUBLIC (new_node
->decl
) = TREE_PUBLIC (old_node
->decl
);
11450 /* The function cgraph_function_versioning () will force the new
11451 symbol local. Undo this, and inherit external visability from
11453 new_node
->local
.local
= old_node
->local
.local
;
11454 new_node
->externally_visible
= old_node
->externally_visible
;
11459 /* Adjust the return type of the given function to its appropriate
11460 vector counterpart. Returns a simd array to be used throughout the
11461 function as a return value. */
11464 simd_clone_adjust_return_type (struct cgraph_node
*node
)
11466 tree fndecl
= node
->decl
;
11467 tree orig_rettype
= TREE_TYPE (TREE_TYPE (fndecl
));
11468 unsigned int veclen
;
11471 /* Adjust the function return type. */
11472 if (orig_rettype
== void_type_node
)
11474 TREE_TYPE (fndecl
) = build_distinct_type_copy (TREE_TYPE (fndecl
));
11475 if (INTEGRAL_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
)))
11476 || POINTER_TYPE_P (TREE_TYPE (TREE_TYPE (fndecl
))))
11477 veclen
= node
->simdclone
->vecsize_int
;
11479 veclen
= node
->simdclone
->vecsize_float
;
11480 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (fndecl
))));
11481 if (veclen
> node
->simdclone
->simdlen
)
11482 veclen
= node
->simdclone
->simdlen
;
11483 if (veclen
== node
->simdclone
->simdlen
)
11484 TREE_TYPE (TREE_TYPE (fndecl
))
11485 = build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)),
11486 node
->simdclone
->simdlen
);
11489 t
= build_vector_type (TREE_TYPE (TREE_TYPE (fndecl
)), veclen
);
11490 t
= build_array_type_nelts (t
, node
->simdclone
->simdlen
/ veclen
);
11491 TREE_TYPE (TREE_TYPE (fndecl
)) = t
;
11493 if (!node
->definition
)
11496 t
= DECL_RESULT (fndecl
);
11497 /* Adjust the DECL_RESULT. */
11498 gcc_assert (TREE_TYPE (t
) != void_type_node
);
11499 TREE_TYPE (t
) = TREE_TYPE (TREE_TYPE (fndecl
));
11502 tree atype
= build_array_type_nelts (orig_rettype
,
11503 node
->simdclone
->simdlen
);
11504 if (veclen
!= node
->simdclone
->simdlen
)
11505 return build1 (VIEW_CONVERT_EXPR
, atype
, t
);
11507 /* Set up a SIMD array to use as the return value. */
11508 tree retval
= create_tmp_var_raw (atype
, "retval");
11509 gimple_add_tmp_var (retval
);
11513 /* Each vector argument has a corresponding array to be used locally
11514 as part of the eventual loop. Create such temporary array and
11517 PREFIX is the prefix to be used for the temporary.
11519 TYPE is the inner element type.
11521 SIMDLEN is the number of elements. */
11524 create_tmp_simd_array (const char *prefix
, tree type
, int simdlen
)
11526 tree atype
= build_array_type_nelts (type
, simdlen
);
11527 tree avar
= create_tmp_var_raw (atype
, prefix
);
11528 gimple_add_tmp_var (avar
);
11532 /* Modify the function argument types to their corresponding vector
11533 counterparts if appropriate. Also, create one array for each simd
11534 argument to be used locally when using the function arguments as
11537 NODE is the function whose arguments are to be adjusted.
11539 Returns an adjustment vector that will be filled describing how the
11540 argument types will be adjusted. */
11542 static ipa_parm_adjustment_vec
11543 simd_clone_adjust_argument_types (struct cgraph_node
*node
)
11546 ipa_parm_adjustment_vec adjustments
;
11548 if (node
->definition
)
11549 args
= ipa_get_vector_of_formal_parms (node
->decl
);
11551 args
= simd_clone_vector_of_formal_parm_types (node
->decl
);
11552 adjustments
.create (args
.length ());
11553 unsigned i
, j
, veclen
;
11554 struct ipa_parm_adjustment adj
;
11555 for (i
= 0; i
< node
->simdclone
->nargs
; ++i
)
11557 memset (&adj
, 0, sizeof (adj
));
11558 tree parm
= args
[i
];
11559 tree parm_type
= node
->definition
? TREE_TYPE (parm
) : parm
;
11560 adj
.base_index
= i
;
11563 node
->simdclone
->args
[i
].orig_arg
= node
->definition
? parm
: NULL_TREE
;
11564 node
->simdclone
->args
[i
].orig_type
= parm_type
;
11566 if (node
->simdclone
->args
[i
].arg_type
!= SIMD_CLONE_ARG_TYPE_VECTOR
)
11568 /* No adjustment necessary for scalar arguments. */
11569 adj
.op
= IPA_PARM_OP_COPY
;
11573 if (INTEGRAL_TYPE_P (parm_type
) || POINTER_TYPE_P (parm_type
))
11574 veclen
= node
->simdclone
->vecsize_int
;
11576 veclen
= node
->simdclone
->vecsize_float
;
11577 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (parm_type
));
11578 if (veclen
> node
->simdclone
->simdlen
)
11579 veclen
= node
->simdclone
->simdlen
;
11580 adj
.arg_prefix
= "simd";
11581 adj
.type
= build_vector_type (parm_type
, veclen
);
11582 node
->simdclone
->args
[i
].vector_type
= adj
.type
;
11583 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11585 adjustments
.safe_push (adj
);
11588 memset (&adj
, 0, sizeof (adj
));
11589 adj
.op
= IPA_PARM_OP_NEW
;
11590 adj
.arg_prefix
= "simd";
11591 adj
.base_index
= i
;
11592 adj
.type
= node
->simdclone
->args
[i
].vector_type
;
11596 if (node
->definition
)
11597 node
->simdclone
->args
[i
].simd_array
11598 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm
)),
11599 parm_type
, node
->simdclone
->simdlen
);
11601 adjustments
.safe_push (adj
);
11604 if (node
->simdclone
->inbranch
)
11607 = simd_clone_compute_base_data_type (node
->simdclone
->origin
,
11610 memset (&adj
, 0, sizeof (adj
));
11611 adj
.op
= IPA_PARM_OP_NEW
;
11612 adj
.arg_prefix
= "mask";
11614 adj
.base_index
= i
;
11615 if (INTEGRAL_TYPE_P (base_type
) || POINTER_TYPE_P (base_type
))
11616 veclen
= node
->simdclone
->vecsize_int
;
11618 veclen
= node
->simdclone
->vecsize_float
;
11619 veclen
/= GET_MODE_BITSIZE (TYPE_MODE (base_type
));
11620 if (veclen
> node
->simdclone
->simdlen
)
11621 veclen
= node
->simdclone
->simdlen
;
11622 adj
.type
= build_vector_type (base_type
, veclen
);
11623 adjustments
.safe_push (adj
);
11625 for (j
= veclen
; j
< node
->simdclone
->simdlen
; j
+= veclen
)
11626 adjustments
.safe_push (adj
);
11628 /* We have previously allocated one extra entry for the mask. Use
11630 struct cgraph_simd_clone
*sc
= node
->simdclone
;
11632 if (node
->definition
)
11634 sc
->args
[i
].orig_arg
11635 = build_decl (UNKNOWN_LOCATION
, PARM_DECL
, NULL
, base_type
);
11636 sc
->args
[i
].simd_array
11637 = create_tmp_simd_array ("mask", base_type
, sc
->simdlen
);
11639 sc
->args
[i
].orig_type
= base_type
;
11640 sc
->args
[i
].arg_type
= SIMD_CLONE_ARG_TYPE_MASK
;
11643 if (node
->definition
)
11644 ipa_modify_formal_parameters (node
->decl
, adjustments
);
11647 tree new_arg_types
= NULL_TREE
, new_reversed
;
11648 bool last_parm_void
= false;
11649 if (args
.length () > 0 && args
.last () == void_type_node
)
11650 last_parm_void
= true;
11652 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)));
11653 j
= adjustments
.length ();
11654 for (i
= 0; i
< j
; i
++)
11656 struct ipa_parm_adjustment
*adj
= &adjustments
[i
];
11658 if (adj
->op
== IPA_PARM_OP_COPY
)
11659 ptype
= args
[adj
->base_index
];
11662 new_arg_types
= tree_cons (NULL_TREE
, ptype
, new_arg_types
);
11664 new_reversed
= nreverse (new_arg_types
);
11665 if (last_parm_void
)
11668 TREE_CHAIN (new_arg_types
) = void_list_node
;
11670 new_reversed
= void_list_node
;
11673 tree new_type
= build_distinct_type_copy (TREE_TYPE (node
->decl
));
11674 TYPE_ARG_TYPES (new_type
) = new_reversed
;
11675 TREE_TYPE (node
->decl
) = new_type
;
11677 adjustments
.release ();
11680 return adjustments
;
11683 /* Initialize and copy the function arguments in NODE to their
11684 corresponding local simd arrays. Returns a fresh gimple_seq with
11685 the instruction sequence generated. */
11688 simd_clone_init_simd_arrays (struct cgraph_node
*node
,
11689 ipa_parm_adjustment_vec adjustments
)
11691 gimple_seq seq
= NULL
;
11692 unsigned i
= 0, j
= 0, k
;
11694 for (tree arg
= DECL_ARGUMENTS (node
->decl
);
11696 arg
= DECL_CHAIN (arg
), i
++, j
++)
11698 if (adjustments
[j
].op
== IPA_PARM_OP_COPY
)
11701 node
->simdclone
->args
[i
].vector_arg
= arg
;
11703 tree array
= node
->simdclone
->args
[i
].simd_array
;
11704 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
)) == node
->simdclone
->simdlen
)
11706 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11707 tree ptr
= build_fold_addr_expr (array
);
11708 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11709 build_int_cst (ptype
, 0));
11710 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11711 gimplify_and_add (t
, &seq
);
11715 unsigned int simdlen
= TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg
));
11716 tree ptype
= build_pointer_type (TREE_TYPE (TREE_TYPE (array
)));
11717 for (k
= 0; k
< node
->simdclone
->simdlen
; k
+= simdlen
)
11719 tree ptr
= build_fold_addr_expr (array
);
11723 arg
= DECL_CHAIN (arg
);
11727 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg
))));
11728 tree t
= build2 (MEM_REF
, TREE_TYPE (arg
), ptr
,
11729 build_int_cst (ptype
, k
* elemsize
));
11730 t
= build2 (MODIFY_EXPR
, TREE_TYPE (t
), t
, arg
);
11731 gimplify_and_add (t
, &seq
);
11738 /* Callback info for ipa_simd_modify_stmt_ops below. */
11740 struct modify_stmt_info
{
11741 ipa_parm_adjustment_vec adjustments
;
11743 /* True if the parent statement was modified by
11744 ipa_simd_modify_stmt_ops. */
11748 /* Callback for walk_gimple_op.
11750 Adjust operands from a given statement as specified in the
11751 adjustments vector in the callback data. */
11754 ipa_simd_modify_stmt_ops (tree
*tp
, int *walk_subtrees
, void *data
)
11756 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
11757 struct modify_stmt_info
*info
= (struct modify_stmt_info
*) wi
->info
;
11758 tree
*orig_tp
= tp
;
11759 if (TREE_CODE (*tp
) == ADDR_EXPR
)
11760 tp
= &TREE_OPERAND (*tp
, 0);
11761 struct ipa_parm_adjustment
*cand
= NULL
;
11762 if (TREE_CODE (*tp
) == PARM_DECL
)
11763 cand
= ipa_get_adjustment_candidate (&tp
, NULL
, info
->adjustments
, true);
11767 *walk_subtrees
= 0;
11770 tree repl
= NULL_TREE
;
11772 repl
= unshare_expr (cand
->new_decl
);
11777 *walk_subtrees
= 0;
11778 bool modified
= info
->modified
;
11779 info
->modified
= false;
11780 walk_tree (tp
, ipa_simd_modify_stmt_ops
, wi
, wi
->pset
);
11781 if (!info
->modified
)
11783 info
->modified
= modified
;
11786 info
->modified
= modified
;
11795 repl
= build_fold_addr_expr (repl
);
11797 if (is_gimple_debug (info
->stmt
))
11799 tree vexpr
= make_node (DEBUG_EXPR_DECL
);
11800 stmt
= gimple_build_debug_source_bind (vexpr
, repl
, NULL
);
11801 DECL_ARTIFICIAL (vexpr
) = 1;
11802 TREE_TYPE (vexpr
) = TREE_TYPE (repl
);
11803 DECL_MODE (vexpr
) = TYPE_MODE (TREE_TYPE (repl
));
11808 stmt
= gimple_build_assign (make_ssa_name (TREE_TYPE (repl
),
11810 repl
= gimple_assign_lhs (stmt
);
11812 gimple_stmt_iterator gsi
= gsi_for_stmt (info
->stmt
);
11813 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11816 else if (!useless_type_conversion_p (TREE_TYPE (*tp
), TREE_TYPE (repl
)))
11818 tree vce
= build1 (VIEW_CONVERT_EXPR
, TREE_TYPE (*tp
), repl
);
11824 info
->modified
= true;
11828 /* Traverse the function body and perform all modifications as
11829 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be
11830 modified such that the replacement/reduction value will now be an
11831 offset into the corresponding simd_array.
11833 This function will replace all function argument uses with their
11834 corresponding simd array elements, and ajust the return values
11838 ipa_simd_modify_function_body (struct cgraph_node
*node
,
11839 ipa_parm_adjustment_vec adjustments
,
11840 tree retval_array
, tree iter
)
11843 unsigned int i
, j
, l
;
11845 /* Re-use the adjustments array, but this time use it to replace
11846 every function argument use to an offset into the corresponding
11848 for (i
= 0, j
= 0; i
< node
->simdclone
->nargs
; ++i
, ++j
)
11850 if (!node
->simdclone
->args
[i
].vector_arg
)
11853 tree basetype
= TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
);
11854 tree vectype
= TREE_TYPE (node
->simdclone
->args
[i
].vector_arg
);
11855 adjustments
[j
].new_decl
11856 = build4 (ARRAY_REF
,
11858 node
->simdclone
->args
[i
].simd_array
,
11860 NULL_TREE
, NULL_TREE
);
11861 if (adjustments
[j
].op
== IPA_PARM_OP_NONE
11862 && TYPE_VECTOR_SUBPARTS (vectype
) < node
->simdclone
->simdlen
)
11863 j
+= node
->simdclone
->simdlen
/ TYPE_VECTOR_SUBPARTS (vectype
) - 1;
11866 l
= adjustments
.length ();
11867 for (i
= 1; i
< num_ssa_names
; i
++)
11869 tree name
= ssa_name (i
);
11871 && SSA_NAME_VAR (name
)
11872 && TREE_CODE (SSA_NAME_VAR (name
)) == PARM_DECL
)
11874 for (j
= 0; j
< l
; j
++)
11875 if (SSA_NAME_VAR (name
) == adjustments
[j
].base
11876 && adjustments
[j
].new_decl
)
11879 if (adjustments
[j
].new_ssa_base
== NULL_TREE
)
11882 = copy_var_decl (adjustments
[j
].base
,
11883 DECL_NAME (adjustments
[j
].base
),
11884 TREE_TYPE (adjustments
[j
].base
));
11885 adjustments
[j
].new_ssa_base
= base_var
;
11888 base_var
= adjustments
[j
].new_ssa_base
;
11889 if (SSA_NAME_IS_DEFAULT_DEF (name
))
11891 bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11892 gimple_stmt_iterator gsi
= gsi_after_labels (bb
);
11893 tree new_decl
= unshare_expr (adjustments
[j
].new_decl
);
11894 set_ssa_default_def (cfun
, adjustments
[j
].base
, NULL_TREE
);
11895 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
11896 SSA_NAME_IS_DEFAULT_DEF (name
) = 0;
11897 gimple stmt
= gimple_build_assign (name
, new_decl
);
11898 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
11901 SET_SSA_NAME_VAR_OR_IDENTIFIER (name
, base_var
);
11906 struct modify_stmt_info info
;
11907 info
.adjustments
= adjustments
;
11909 FOR_EACH_BB_FN (bb
, DECL_STRUCT_FUNCTION (node
->decl
))
11911 gimple_stmt_iterator gsi
;
11913 gsi
= gsi_start_bb (bb
);
11914 while (!gsi_end_p (gsi
))
11916 gimple stmt
= gsi_stmt (gsi
);
11918 struct walk_stmt_info wi
;
11920 memset (&wi
, 0, sizeof (wi
));
11921 info
.modified
= false;
11923 walk_gimple_op (stmt
, ipa_simd_modify_stmt_ops
, &wi
);
11925 if (gimple_code (stmt
) == GIMPLE_RETURN
)
11927 tree retval
= gimple_return_retval (stmt
);
11930 gsi_remove (&gsi
, true);
11934 /* Replace `return foo' with `retval_array[iter] = foo'. */
11935 tree ref
= build4 (ARRAY_REF
, TREE_TYPE (retval
),
11936 retval_array
, iter
, NULL
, NULL
);
11937 stmt
= gimple_build_assign (ref
, retval
);
11938 gsi_replace (&gsi
, stmt
, true);
11939 info
.modified
= true;
11944 update_stmt (stmt
);
11945 if (maybe_clean_eh_stmt (stmt
))
11946 gimple_purge_dead_eh_edges (gimple_bb (stmt
));
11953 /* Adjust the argument types in NODE to their appropriate vector
11957 simd_clone_adjust (struct cgraph_node
*node
)
11959 push_cfun (DECL_STRUCT_FUNCTION (node
->decl
));
11961 targetm
.simd_clone
.adjust (node
);
11963 tree retval
= simd_clone_adjust_return_type (node
);
11964 ipa_parm_adjustment_vec adjustments
11965 = simd_clone_adjust_argument_types (node
);
11967 push_gimplify_context ();
11969 gimple_seq seq
= simd_clone_init_simd_arrays (node
, adjustments
);
11971 /* Adjust all uses of vector arguments accordingly. Adjust all
11972 return values accordingly. */
11973 tree iter
= create_tmp_var (unsigned_type_node
, "iter");
11974 tree iter1
= make_ssa_name (iter
, NULL
);
11975 tree iter2
= make_ssa_name (iter
, NULL
);
11976 ipa_simd_modify_function_body (node
, adjustments
, retval
, iter1
);
11978 /* Initialize the iteration variable. */
11979 basic_block entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
11980 basic_block body_bb
= split_block_after_labels (entry_bb
)->dest
;
11981 gimple_stmt_iterator gsi
= gsi_after_labels (entry_bb
);
11982 /* Insert the SIMD array and iv initialization at function
11984 gsi_insert_seq_before (&gsi
, seq
, GSI_NEW_STMT
);
11986 pop_gimplify_context (NULL
);
11988 /* Create a new BB right before the original exit BB, to hold the
11989 iteration increment and the condition/branch. */
11990 basic_block orig_exit
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), 0)->src
;
11991 basic_block incr_bb
= create_empty_bb (orig_exit
);
11992 add_bb_to_loop (incr_bb
, body_bb
->loop_father
);
11993 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty
11994 flag. Set it now to be a FALLTHRU_EDGE. */
11995 gcc_assert (EDGE_COUNT (orig_exit
->succs
) == 1);
11996 EDGE_SUCC (orig_exit
, 0)->flags
|= EDGE_FALLTHRU
;
11997 for (unsigned i
= 0;
11998 i
< EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun
)->preds
); ++i
)
12000 edge e
= EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun
), i
);
12001 redirect_edge_succ (e
, incr_bb
);
12003 edge e
= make_edge (incr_bb
, EXIT_BLOCK_PTR_FOR_FN (cfun
), 0);
12004 e
->probability
= REG_BR_PROB_BASE
;
12005 gsi
= gsi_last_bb (incr_bb
);
12006 gimple g
= gimple_build_assign_with_ops (PLUS_EXPR
, iter2
, iter1
,
12007 build_int_cst (unsigned_type_node
,
12009 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12011 /* Mostly annotate the loop for the vectorizer (the rest is done below). */
12012 struct loop
*loop
= alloc_loop ();
12013 cfun
->has_force_vectorize_loops
= true;
12014 loop
->safelen
= node
->simdclone
->simdlen
;
12015 loop
->force_vectorize
= true;
12016 loop
->header
= body_bb
;
12018 /* Branch around the body if the mask applies. */
12019 if (node
->simdclone
->inbranch
)
12021 gimple_stmt_iterator gsi
= gsi_last_bb (loop
->header
);
12023 = node
->simdclone
->args
[node
->simdclone
->nargs
- 1].simd_array
;
12024 tree mask
= make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array
)), NULL
);
12025 tree aref
= build4 (ARRAY_REF
,
12026 TREE_TYPE (TREE_TYPE (mask_array
)),
12029 g
= gimple_build_assign (mask
, aref
);
12030 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12031 int bitsize
= GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref
)));
12032 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref
)))
12034 aref
= build1 (VIEW_CONVERT_EXPR
,
12035 build_nonstandard_integer_type (bitsize
, 0), mask
);
12036 mask
= make_ssa_name (TREE_TYPE (aref
), NULL
);
12037 g
= gimple_build_assign (mask
, aref
);
12038 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12041 g
= gimple_build_cond (EQ_EXPR
, mask
, build_zero_cst (TREE_TYPE (mask
)),
12043 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12044 make_edge (loop
->header
, incr_bb
, EDGE_TRUE_VALUE
);
12045 FALLTHRU_EDGE (loop
->header
)->flags
= EDGE_FALSE_VALUE
;
12048 /* Generate the condition. */
12049 g
= gimple_build_cond (LT_EXPR
,
12051 build_int_cst (unsigned_type_node
,
12052 node
->simdclone
->simdlen
),
12054 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12055 e
= split_block (incr_bb
, gsi_stmt (gsi
));
12056 basic_block latch_bb
= e
->dest
;
12057 basic_block new_exit_bb
;
12058 new_exit_bb
= split_block (latch_bb
, NULL
)->dest
;
12059 loop
->latch
= latch_bb
;
12061 redirect_edge_succ (FALLTHRU_EDGE (latch_bb
), body_bb
);
12063 make_edge (incr_bb
, new_exit_bb
, EDGE_FALSE_VALUE
);
12064 /* The successor of incr_bb is already pointing to latch_bb; just
12066 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
12067 FALLTHRU_EDGE (incr_bb
)->flags
= EDGE_TRUE_VALUE
;
12069 gimple phi
= create_phi_node (iter1
, body_bb
);
12070 edge preheader_edge
= find_edge (entry_bb
, body_bb
);
12071 edge latch_edge
= single_succ_edge (latch_bb
);
12072 add_phi_arg (phi
, build_zero_cst (unsigned_type_node
), preheader_edge
,
12074 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
12076 /* Generate the new return. */
12077 gsi
= gsi_last_bb (new_exit_bb
);
12079 && TREE_CODE (retval
) == VIEW_CONVERT_EXPR
12080 && TREE_CODE (TREE_OPERAND (retval
, 0)) == RESULT_DECL
)
12081 retval
= TREE_OPERAND (retval
, 0);
12084 retval
= build1 (VIEW_CONVERT_EXPR
,
12085 TREE_TYPE (TREE_TYPE (node
->decl
)),
12087 retval
= force_gimple_operand_gsi (&gsi
, retval
, true, NULL
,
12088 false, GSI_CONTINUE_LINKING
);
12090 g
= gimple_build_return (retval
);
12091 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
12093 /* Handle aligned clauses by replacing default defs of the aligned
12094 uniform args with __builtin_assume_aligned (arg_N(D), alignment)
12095 lhs. Handle linear by adding PHIs. */
12096 for (unsigned i
= 0; i
< node
->simdclone
->nargs
; i
++)
12097 if (node
->simdclone
->args
[i
].alignment
12098 && node
->simdclone
->args
[i
].arg_type
== SIMD_CLONE_ARG_TYPE_UNIFORM
12099 && (node
->simdclone
->args
[i
].alignment
12100 & (node
->simdclone
->args
[i
].alignment
- 1)) == 0
12101 && TREE_CODE (TREE_TYPE (node
->simdclone
->args
[i
].orig_arg
))
12104 unsigned int alignment
= node
->simdclone
->args
[i
].alignment
;
12105 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
12106 tree def
= ssa_default_def (cfun
, orig_arg
);
12107 if (def
&& !has_zero_uses (def
))
12109 tree fn
= builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED
);
12110 gimple_seq seq
= NULL
;
12111 bool need_cvt
= false;
12113 = gimple_build_call (fn
, 2, def
, size_int (alignment
));
12115 if (!useless_type_conversion_p (TREE_TYPE (orig_arg
),
12118 tree t
= make_ssa_name (need_cvt
? ptr_type_node
: orig_arg
, NULL
);
12119 gimple_call_set_lhs (g
, t
);
12120 gimple_seq_add_stmt_without_update (&seq
, g
);
12123 t
= make_ssa_name (orig_arg
, NULL
);
12124 g
= gimple_build_assign_with_ops (NOP_EXPR
, t
,
12125 gimple_call_lhs (g
),
12127 gimple_seq_add_stmt_without_update (&seq
, g
);
12129 gsi_insert_seq_on_edge_immediate
12130 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun
)), seq
);
12132 entry_bb
= single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun
));
12133 int freq
= compute_call_stmt_bb_frequency (current_function_decl
,
12135 node
->create_edge (cgraph_node::get_create (fn
),
12136 call
, entry_bb
->count
, freq
);
12138 imm_use_iterator iter
;
12139 use_operand_p use_p
;
12141 tree repl
= gimple_get_lhs (g
);
12142 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
12143 if (is_gimple_debug (use_stmt
) || use_stmt
== call
)
12146 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
12147 SET_USE (use_p
, repl
);
12150 else if (node
->simdclone
->args
[i
].arg_type
12151 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP
)
12153 tree orig_arg
= node
->simdclone
->args
[i
].orig_arg
;
12154 tree def
= ssa_default_def (cfun
, orig_arg
);
12155 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
12156 || POINTER_TYPE_P (TREE_TYPE (orig_arg
)));
12157 if (def
&& !has_zero_uses (def
))
12159 iter1
= make_ssa_name (orig_arg
, NULL
);
12160 iter2
= make_ssa_name (orig_arg
, NULL
);
12161 phi
= create_phi_node (iter1
, body_bb
);
12162 add_phi_arg (phi
, def
, preheader_edge
, UNKNOWN_LOCATION
);
12163 add_phi_arg (phi
, iter2
, latch_edge
, UNKNOWN_LOCATION
);
12164 enum tree_code code
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
12165 ? PLUS_EXPR
: POINTER_PLUS_EXPR
;
12166 tree addtype
= INTEGRAL_TYPE_P (TREE_TYPE (orig_arg
))
12167 ? TREE_TYPE (orig_arg
) : sizetype
;
12169 = build_int_cst (addtype
, node
->simdclone
->args
[i
].linear_step
);
12170 g
= gimple_build_assign_with_ops (code
, iter2
, iter1
, addcst
);
12171 gsi
= gsi_last_bb (incr_bb
);
12172 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
12174 imm_use_iterator iter
;
12175 use_operand_p use_p
;
12177 FOR_EACH_IMM_USE_STMT (use_stmt
, iter
, def
)
12178 if (use_stmt
== phi
)
12181 FOR_EACH_IMM_USE_ON_STMT (use_p
, iter
)
12182 SET_USE (use_p
, iter1
);
12186 calculate_dominance_info (CDI_DOMINATORS
);
12187 add_loop (loop
, loop
->header
->loop_father
);
12188 update_ssa (TODO_update_ssa
);
12193 /* If the function in NODE is tagged as an elemental SIMD function,
12194 create the appropriate SIMD clones. */
12197 expand_simd_clones (struct cgraph_node
*node
)
12199 tree attr
= lookup_attribute ("omp declare simd",
12200 DECL_ATTRIBUTES (node
->decl
));
12201 if (attr
== NULL_TREE
12202 || node
->global
.inlined_to
12203 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node
->decl
)))
12207 #pragma omp declare simd
12209 in C, there we don't know the argument types at all. */
12210 if (!node
->definition
12211 && TYPE_ARG_TYPES (TREE_TYPE (node
->decl
)) == NULL_TREE
)
12216 /* Start with parsing the "omp declare simd" attribute(s). */
12217 bool inbranch_clause_specified
;
12218 struct cgraph_simd_clone
*clone_info
12219 = simd_clone_clauses_extract (node
, TREE_VALUE (attr
),
12220 &inbranch_clause_specified
);
12221 if (clone_info
== NULL
)
12224 int orig_simdlen
= clone_info
->simdlen
;
12225 tree base_type
= simd_clone_compute_base_data_type (node
, clone_info
);
12226 /* The target can return 0 (no simd clones should be created),
12227 1 (just one ISA of simd clones should be created) or higher
12228 count of ISA variants. In that case, clone_info is initialized
12229 for the first ISA variant. */
12231 = targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone_info
,
12236 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED,
12237 also create one inbranch and one !inbranch clone of it. */
12238 for (int i
= 0; i
< count
* 2; i
++)
12240 struct cgraph_simd_clone
*clone
= clone_info
;
12241 if (inbranch_clause_specified
&& (i
& 1) != 0)
12246 clone
= simd_clone_struct_alloc (clone_info
->nargs
12248 simd_clone_struct_copy (clone
, clone_info
);
12249 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen
12250 and simd_clone_adjust_argument_types did to the first
12252 clone
->nargs
-= clone_info
->inbranch
;
12253 clone
->simdlen
= orig_simdlen
;
12254 /* And call the target hook again to get the right ISA. */
12255 targetm
.simd_clone
.compute_vecsize_and_simdlen (node
, clone
,
12259 clone
->inbranch
= 1;
12262 /* simd_clone_mangle might fail if such a clone has been created
12264 tree id
= simd_clone_mangle (node
, clone
);
12265 if (id
== NULL_TREE
)
12268 /* Only when we are sure we want to create the clone actually
12269 clone the function (or definitions) or create another
12270 extern FUNCTION_DECL (for prototypes without definitions). */
12271 struct cgraph_node
*n
= simd_clone_create (node
);
12275 n
->simdclone
= clone
;
12276 clone
->origin
= node
;
12277 clone
->next_clone
= NULL
;
12278 if (node
->simd_clones
== NULL
)
12280 clone
->prev_clone
= n
;
12281 node
->simd_clones
= n
;
12285 clone
->prev_clone
= node
->simd_clones
->simdclone
->prev_clone
;
12286 clone
->prev_clone
->simdclone
->next_clone
= n
;
12287 node
->simd_clones
->simdclone
->prev_clone
= n
;
12289 symtab
->change_decl_assembler_name (n
->decl
, id
);
12290 /* And finally adjust the return type, parameters and for
12291 definitions also function body. */
12292 if (node
->definition
)
12293 simd_clone_adjust (n
);
12296 simd_clone_adjust_return_type (n
);
12297 simd_clone_adjust_argument_types (n
);
12301 while ((attr
= lookup_attribute ("omp declare simd", TREE_CHAIN (attr
))));
12304 /* Entry point for IPA simd clone creation pass. */
12306 static unsigned int
12307 ipa_omp_simd_clone (void)
12309 struct cgraph_node
*node
;
12310 FOR_EACH_FUNCTION (node
)
12311 expand_simd_clones (node
);
12317 const pass_data pass_data_omp_simd_clone
=
12319 SIMPLE_IPA_PASS
, /* type */
12320 "simdclone", /* name */
12321 OPTGROUP_NONE
, /* optinfo_flags */
12322 TV_NONE
, /* tv_id */
12323 ( PROP_ssa
| PROP_cfg
), /* properties_required */
12324 0, /* properties_provided */
12325 0, /* properties_destroyed */
12326 0, /* todo_flags_start */
12327 0, /* todo_flags_finish */
12330 class pass_omp_simd_clone
: public simple_ipa_opt_pass
12333 pass_omp_simd_clone(gcc::context
*ctxt
)
12334 : simple_ipa_opt_pass(pass_data_omp_simd_clone
, ctxt
)
12337 /* opt_pass methods: */
12338 virtual bool gate (function
*);
12339 virtual unsigned int execute (function
*) { return ipa_omp_simd_clone (); }
12343 pass_omp_simd_clone::gate (function
*)
12345 return ((flag_openmp
|| flag_openmp_simd
12347 || (in_lto_p
&& !flag_wpa
))
12348 && (targetm
.simd_clone
.compute_vecsize_and_simdlen
!= NULL
));
12351 } // anon namespace
12353 simple_ipa_opt_pass
*
12354 make_pass_omp_simd_clone (gcc::context
*ctxt
)
12356 return new pass_omp_simd_clone (ctxt
);
12359 #include "gt-omp-low.h"