1 /* Bits of OpenMP and OpenACC handling that is specific to device offloading
2 and a lowering pass for OpenACC device directives.
4 Copyright (C) 2005-2018 Free Software Foundation, Inc.
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
24 #include "coretypes.h"
29 #include "tree-pass.h"
32 #include "pretty-print.h"
33 #include "diagnostic-core.h"
34 #include "fold-const.h"
35 #include "internal-fn.h"
36 #include "langhooks.h"
38 #include "gimple-iterator.h"
39 #include "gimplify-me.h"
40 #include "gimple-walk.h"
42 #include "tree-into-ssa.h"
43 #include "tree-nested.h"
44 #include "stor-layout.h"
45 #include "common/common-target.h"
46 #include "omp-general.h"
47 #include "omp-offload.h"
48 #include "lto-section-names.h"
49 #include "gomp-constants.h"
50 #include "gimple-pretty-print.h"
52 #include "stringpool.h"
56 /* Describe the OpenACC looping structure of a function. The entire
57 function is held in a 'NULL' loop. */
61 oacc_loop
*parent
; /* Containing loop. */
63 oacc_loop
*child
; /* First inner loop. */
65 oacc_loop
*sibling
; /* Next loop within same parent. */
67 location_t loc
; /* Location of the loop start. */
69 gcall
*marker
; /* Initial head marker. */
71 gcall
*heads
[GOMP_DIM_MAX
]; /* Head marker functions. */
72 gcall
*tails
[GOMP_DIM_MAX
]; /* Tail marker functions. */
74 tree routine
; /* Pseudo-loop enclosing a routine. */
76 unsigned mask
; /* Partitioning mask. */
77 unsigned e_mask
; /* Partitioning of element loops (when tiling). */
78 unsigned inner
; /* Partitioning of inner loops. */
79 unsigned flags
; /* Partitioning flags. */
80 vec
<gcall
*> ifns
; /* Contained loop abstraction functions. */
81 tree chunk_size
; /* Chunk size. */
82 gcall
*head_end
; /* Final marker of head sequence. */
85 /* Holds offload tables with decls. */
86 vec
<tree
, va_gc
> *offload_funcs
, *offload_vars
;
88 /* Return level at which oacc routine may spawn a partitioned loop, or
89 -1 if it is not a routine (i.e. is an offload fn). */
92 oacc_fn_attrib_level (tree attr
)
94 tree pos
= TREE_VALUE (attr
);
96 if (!TREE_PURPOSE (pos
))
100 for (ix
= 0; ix
!= GOMP_DIM_MAX
;
101 ix
++, pos
= TREE_CHAIN (pos
))
102 if (!integer_zerop (TREE_PURPOSE (pos
)))
108 /* Helper function for omp_finish_file routine. Takes decls from V_DECLS and
109 adds their addresses and sizes to constructor-vector V_CTOR. */
112 add_decls_addresses_to_decl_constructor (vec
<tree
, va_gc
> *v_decls
,
113 vec
<constructor_elt
, va_gc
> *v_ctor
)
115 unsigned len
= vec_safe_length (v_decls
);
116 for (unsigned i
= 0; i
< len
; i
++)
118 tree it
= (*v_decls
)[i
];
119 bool is_var
= VAR_P (it
);
122 #ifdef ACCEL_COMPILER
123 && DECL_HAS_VALUE_EXPR_P (it
)
125 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (it
));
127 tree size
= NULL_TREE
;
129 size
= fold_convert (const_ptr_type_node
, DECL_SIZE_UNIT (it
));
133 addr
= build_fold_addr_expr (it
);
136 #ifdef ACCEL_COMPILER
137 /* For "omp declare target link" vars add address of the pointer to
138 the target table, instead of address of the var. */
139 tree value_expr
= DECL_VALUE_EXPR (it
);
140 tree link_ptr_decl
= TREE_OPERAND (value_expr
, 0);
141 varpool_node::finalize_decl (link_ptr_decl
);
142 addr
= build_fold_addr_expr (link_ptr_decl
);
144 addr
= build_fold_addr_expr (it
);
147 /* Most significant bit of the size marks "omp declare target link"
148 vars in host and target tables. */
149 unsigned HOST_WIDE_INT isize
= tree_to_uhwi (size
);
150 isize
|= 1ULL << (int_size_in_bytes (const_ptr_type_node
)
151 * BITS_PER_UNIT
- 1);
152 size
= wide_int_to_tree (const_ptr_type_node
, isize
);
155 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, addr
);
157 CONSTRUCTOR_APPEND_ELT (v_ctor
, NULL_TREE
, size
);
161 /* Create new symbols containing (address, size) pairs for global variables,
162 marked with "omp declare target" attribute, as well as addresses for the
163 functions, which are outlined offloading regions. */
165 omp_finish_file (void)
167 unsigned num_funcs
= vec_safe_length (offload_funcs
);
168 unsigned num_vars
= vec_safe_length (offload_vars
);
170 if (num_funcs
== 0 && num_vars
== 0)
173 if (targetm_common
.have_named_sections
)
175 vec
<constructor_elt
, va_gc
> *v_f
, *v_v
;
176 vec_alloc (v_f
, num_funcs
);
177 vec_alloc (v_v
, num_vars
* 2);
179 add_decls_addresses_to_decl_constructor (offload_funcs
, v_f
);
180 add_decls_addresses_to_decl_constructor (offload_vars
, v_v
);
182 tree vars_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
184 tree funcs_decl_type
= build_array_type_nelts (pointer_sized_int_node
,
186 SET_TYPE_ALIGN (vars_decl_type
, TYPE_ALIGN (pointer_sized_int_node
));
187 SET_TYPE_ALIGN (funcs_decl_type
, TYPE_ALIGN (pointer_sized_int_node
));
188 tree ctor_v
= build_constructor (vars_decl_type
, v_v
);
189 tree ctor_f
= build_constructor (funcs_decl_type
, v_f
);
190 TREE_CONSTANT (ctor_v
) = TREE_CONSTANT (ctor_f
) = 1;
191 TREE_STATIC (ctor_v
) = TREE_STATIC (ctor_f
) = 1;
192 tree funcs_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
193 get_identifier (".offload_func_table"),
195 tree vars_decl
= build_decl (UNKNOWN_LOCATION
, VAR_DECL
,
196 get_identifier (".offload_var_table"),
198 TREE_STATIC (funcs_decl
) = TREE_STATIC (vars_decl
) = 1;
199 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node),
200 otherwise a joint table in a binary will contain padding between
201 tables from multiple object files. */
202 DECL_USER_ALIGN (funcs_decl
) = DECL_USER_ALIGN (vars_decl
) = 1;
203 SET_DECL_ALIGN (funcs_decl
, TYPE_ALIGN (funcs_decl_type
));
204 SET_DECL_ALIGN (vars_decl
, TYPE_ALIGN (vars_decl_type
));
205 DECL_INITIAL (funcs_decl
) = ctor_f
;
206 DECL_INITIAL (vars_decl
) = ctor_v
;
207 set_decl_section_name (funcs_decl
, OFFLOAD_FUNC_TABLE_SECTION_NAME
);
208 set_decl_section_name (vars_decl
, OFFLOAD_VAR_TABLE_SECTION_NAME
);
210 varpool_node::finalize_decl (vars_decl
);
211 varpool_node::finalize_decl (funcs_decl
);
215 for (unsigned i
= 0; i
< num_funcs
; i
++)
217 tree it
= (*offload_funcs
)[i
];
218 targetm
.record_offload_symbol (it
);
220 for (unsigned i
= 0; i
< num_vars
; i
++)
222 tree it
= (*offload_vars
)[i
];
223 targetm
.record_offload_symbol (it
);
228 /* Call dim_pos (POS == true) or dim_size (POS == false) builtins for
229 axis DIM. Return a tmp var holding the result. */
232 oacc_dim_call (bool pos
, int dim
, gimple_seq
*seq
)
234 tree arg
= build_int_cst (unsigned_type_node
, dim
);
235 tree size
= create_tmp_var (integer_type_node
);
236 enum internal_fn fn
= pos
? IFN_GOACC_DIM_POS
: IFN_GOACC_DIM_SIZE
;
237 gimple
*call
= gimple_build_call_internal (fn
, 1, arg
);
239 gimple_call_set_lhs (call
, size
);
240 gimple_seq_add_stmt (seq
, call
);
245 /* Find the number of threads (POS = false), or thread number (POS =
246 true) for an OpenACC region partitioned as MASK. Setup code
247 required for the calculation is added to SEQ. */
250 oacc_thread_numbers (bool pos
, int mask
, gimple_seq
*seq
)
252 tree res
= pos
? NULL_TREE
: build_int_cst (unsigned_type_node
, 1);
255 /* Start at gang level, and examine relevant dimension indices. */
256 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
257 if (GOMP_DIM_MASK (ix
) & mask
)
261 /* We had an outer index, so scale that by the size of
263 tree n
= oacc_dim_call (false, ix
, seq
);
264 res
= fold_build2 (MULT_EXPR
, integer_type_node
, res
, n
);
268 /* Determine index in this dimension. */
269 tree id
= oacc_dim_call (true, ix
, seq
);
271 res
= fold_build2 (PLUS_EXPR
, integer_type_node
, res
, id
);
277 if (res
== NULL_TREE
)
278 res
= integer_zero_node
;
283 /* Transform IFN_GOACC_LOOP calls to actual code. See
284 expand_oacc_for for where these are generated. At the vector
285 level, we stride loops, such that each member of a warp will
286 operate on adjacent iterations. At the worker and gang level,
287 each gang/warp executes a set of contiguous iterations. Chunking
288 can override this such that each iteration engine executes a
289 contiguous chunk, and then moves on to stride to the next chunk. */
292 oacc_xform_loop (gcall
*call
)
294 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
295 enum ifn_goacc_loop_kind code
296 = (enum ifn_goacc_loop_kind
) TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
297 tree dir
= gimple_call_arg (call
, 1);
298 tree range
= gimple_call_arg (call
, 2);
299 tree step
= gimple_call_arg (call
, 3);
300 tree chunk_size
= NULL_TREE
;
301 unsigned mask
= (unsigned) TREE_INT_CST_LOW (gimple_call_arg (call
, 5));
302 tree lhs
= gimple_call_lhs (call
);
303 tree type
= TREE_TYPE (lhs
);
304 tree diff_type
= TREE_TYPE (range
);
306 gimple_seq seq
= NULL
;
307 bool chunking
= false, striding
= true;
308 unsigned outer_mask
= mask
& (~mask
+ 1); // Outermost partitioning
309 unsigned inner_mask
= mask
& ~outer_mask
; // Inner partitioning (if any)
311 #ifdef ACCEL_COMPILER
312 chunk_size
= gimple_call_arg (call
, 4);
313 if (integer_minus_onep (chunk_size
) /* Force static allocation. */
314 || integer_zerop (chunk_size
)) /* Default (also static). */
316 /* If we're at the gang level, we want each to execute a
317 contiguous run of iterations. Otherwise we want each element
319 striding
= !(outer_mask
& GOMP_DIM_MASK (GOMP_DIM_GANG
));
324 /* Chunk of size 1 is striding. */
325 striding
= integer_onep (chunk_size
);
326 chunking
= !striding
;
330 /* striding=true, chunking=true
332 striding=true, chunking=false
334 striding=false,chunking=true
335 -> chunks=ceil (range/(chunksize*threads*step))
336 striding=false,chunking=false
337 -> chunk_size=ceil(range/(threads*step)),chunks=1 */
338 push_gimplify_context (true);
342 default: gcc_unreachable ();
344 case IFN_GOACC_LOOP_CHUNKS
:
346 r
= build_int_cst (type
, 1);
350 = (range - dir) / (chunks * step * num_threads) + dir */
351 tree per
= oacc_thread_numbers (false, mask
, &seq
);
352 per
= fold_convert (type
, per
);
353 chunk_size
= fold_convert (type
, chunk_size
);
354 per
= fold_build2 (MULT_EXPR
, type
, per
, chunk_size
);
355 per
= fold_build2 (MULT_EXPR
, type
, per
, step
);
356 r
= build2 (MINUS_EXPR
, type
, range
, dir
);
357 r
= build2 (PLUS_EXPR
, type
, r
, per
);
358 r
= build2 (TRUNC_DIV_EXPR
, type
, r
, per
);
362 case IFN_GOACC_LOOP_STEP
:
364 /* If striding, step by the entire compute volume, otherwise
365 step by the inner volume. */
366 unsigned volume
= striding
? mask
: inner_mask
;
368 r
= oacc_thread_numbers (false, volume
, &seq
);
369 r
= build2 (MULT_EXPR
, type
, fold_convert (type
, r
), step
);
373 case IFN_GOACC_LOOP_OFFSET
:
374 /* Enable vectorization on non-SIMT targets. */
376 && outer_mask
== GOMP_DIM_MASK (GOMP_DIM_VECTOR
)
377 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize
379 && (flag_tree_loop_vectorize
380 || !global_options_set
.x_flag_tree_loop_vectorize
))
382 basic_block bb
= gsi_bb (gsi
);
383 struct loop
*parent
= bb
->loop_father
;
384 struct loop
*body
= parent
->inner
;
386 parent
->force_vectorize
= true;
387 parent
->safelen
= INT_MAX
;
389 /* "Chunking loops" may have inner loops. */
392 body
->force_vectorize
= true;
393 body
->safelen
= INT_MAX
;
396 cfun
->has_force_vectorize_loops
= true;
400 r
= oacc_thread_numbers (true, mask
, &seq
);
401 r
= fold_convert (diff_type
, r
);
405 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
406 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
407 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
408 inner_size
, outer_size
);
410 volume
= fold_convert (diff_type
, volume
);
412 chunk_size
= fold_convert (diff_type
, chunk_size
);
415 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
417 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
418 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
419 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
422 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
423 fold_convert (diff_type
, inner_size
));
424 r
= oacc_thread_numbers (true, outer_mask
, &seq
);
425 r
= fold_convert (diff_type
, r
);
426 r
= build2 (MULT_EXPR
, diff_type
, r
, span
);
428 tree inner
= oacc_thread_numbers (true, inner_mask
, &seq
);
429 inner
= fold_convert (diff_type
, inner
);
430 r
= fold_build2 (PLUS_EXPR
, diff_type
, r
, inner
);
434 tree chunk
= fold_convert (diff_type
, gimple_call_arg (call
, 6));
436 = fold_build2 (MULT_EXPR
, diff_type
, volume
, chunk_size
);
437 per
= build2 (MULT_EXPR
, diff_type
, per
, chunk
);
439 r
= build2 (PLUS_EXPR
, diff_type
, r
, per
);
442 r
= fold_build2 (MULT_EXPR
, diff_type
, r
, step
);
443 if (type
!= diff_type
)
444 r
= fold_convert (type
, r
);
447 case IFN_GOACC_LOOP_BOUND
:
452 tree inner_size
= oacc_thread_numbers (false, inner_mask
, &seq
);
453 tree outer_size
= oacc_thread_numbers (false, outer_mask
, &seq
);
454 tree volume
= fold_build2 (MULT_EXPR
, TREE_TYPE (inner_size
),
455 inner_size
, outer_size
);
457 volume
= fold_convert (diff_type
, volume
);
459 chunk_size
= fold_convert (diff_type
, chunk_size
);
462 tree per
= fold_build2 (MULT_EXPR
, diff_type
, volume
, step
);
464 chunk_size
= build2 (MINUS_EXPR
, diff_type
, range
, dir
);
465 chunk_size
= build2 (PLUS_EXPR
, diff_type
, chunk_size
, per
);
466 chunk_size
= build2 (TRUNC_DIV_EXPR
, diff_type
, chunk_size
, per
);
469 tree span
= build2 (MULT_EXPR
, diff_type
, chunk_size
,
470 fold_convert (diff_type
, inner_size
));
472 r
= fold_build2 (MULT_EXPR
, diff_type
, span
, step
);
474 tree offset
= gimple_call_arg (call
, 6);
475 r
= build2 (PLUS_EXPR
, diff_type
, r
,
476 fold_convert (diff_type
, offset
));
477 r
= build2 (integer_onep (dir
) ? MIN_EXPR
: MAX_EXPR
,
478 diff_type
, r
, range
);
480 if (diff_type
!= type
)
481 r
= fold_convert (type
, r
);
485 gimplify_assign (lhs
, r
, &seq
);
487 pop_gimplify_context (NULL
);
489 gsi_replace_with_seq (&gsi
, seq
, true);
492 /* Transform a GOACC_TILE call. Determines the element loop span for
493 the specified loop of the nest. This is 1 if we're not tiling.
495 GOACC_TILE (collapse_count, loop_no, tile_arg, gwv_tile, gwv_element); */
498 oacc_xform_tile (gcall
*call
)
500 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
501 unsigned collapse
= tree_to_uhwi (gimple_call_arg (call
, 0));
502 /* Inner loops have higher loop_nos. */
503 unsigned loop_no
= tree_to_uhwi (gimple_call_arg (call
, 1));
504 tree tile_size
= gimple_call_arg (call
, 2);
505 unsigned e_mask
= tree_to_uhwi (gimple_call_arg (call
, 4));
506 tree lhs
= gimple_call_lhs (call
);
507 tree type
= TREE_TYPE (lhs
);
508 gimple_seq seq
= NULL
;
509 tree span
= build_int_cst (type
, 1);
512 & ~(GOMP_DIM_MASK (GOMP_DIM_VECTOR
)
513 | GOMP_DIM_MASK (GOMP_DIM_WORKER
))));
514 push_gimplify_context (!seen_error ());
516 #ifndef ACCEL_COMPILER
517 /* Partitioning disabled on host compilers. */
521 /* Not paritioning. */
522 span
= integer_one_node
;
523 else if (!integer_zerop (tile_size
))
524 /* User explicitly specified size. */
528 /* Pick a size based on the paritioning of the element loop and
529 the number of loop nests. */
530 tree first_size
= NULL_TREE
;
531 tree second_size
= NULL_TREE
;
533 if (e_mask
& GOMP_DIM_MASK (GOMP_DIM_VECTOR
))
534 first_size
= oacc_dim_call (false, GOMP_DIM_VECTOR
, &seq
);
535 if (e_mask
& GOMP_DIM_MASK (GOMP_DIM_WORKER
))
536 second_size
= oacc_dim_call (false, GOMP_DIM_WORKER
, &seq
);
540 first_size
= second_size
;
541 second_size
= NULL_TREE
;
544 if (loop_no
+ 1 == collapse
)
547 if (!loop_no
&& second_size
)
548 span
= fold_build2 (MULT_EXPR
, TREE_TYPE (span
),
551 else if (loop_no
+ 2 == collapse
)
557 /* There's no obvious element size for this loop. Options
558 are 1, first_size or some non-unity constant (32 is my
559 favourite). We should gather some statistics. */
563 span
= fold_convert (type
, span
);
564 gimplify_assign (lhs
, span
, &seq
);
566 pop_gimplify_context (NULL
);
568 gsi_replace_with_seq (&gsi
, seq
, true);
571 /* Default partitioned and minimum partitioned dimensions. */
573 static int oacc_default_dims
[GOMP_DIM_MAX
];
574 static int oacc_min_dims
[GOMP_DIM_MAX
];
576 /* Parse the default dimension parameter. This is a set of
577 :-separated optional compute dimensions. Each specified dimension
578 is a positive integer. When device type support is added, it is
579 planned to be a comma separated list of such compute dimensions,
580 with all but the first prefixed by the colon-terminated device
584 oacc_parse_default_dims (const char *dims
)
588 for (ix
= GOMP_DIM_MAX
; ix
--;)
590 oacc_default_dims
[ix
] = -1;
591 oacc_min_dims
[ix
] = 1;
594 #ifndef ACCEL_COMPILER
595 /* Cannot be overridden on the host. */
600 const char *pos
= dims
;
602 for (ix
= 0; *pos
&& ix
!= GOMP_DIM_MAX
; ix
++)
617 val
= strtol (pos
, CONST_CAST (char **, &eptr
), 10);
618 if (errno
|| val
<= 0 || (int) val
!= val
)
621 oacc_default_dims
[ix
] = (int) val
;
627 error_at (UNKNOWN_LOCATION
,
628 "-fopenacc-dim operand is malformed at '%s'", pos
);
632 /* Allow the backend to validate the dimensions. */
633 targetm
.goacc
.validate_dims (NULL_TREE
, oacc_default_dims
, -1);
634 targetm
.goacc
.validate_dims (NULL_TREE
, oacc_min_dims
, -2);
637 /* Validate and update the dimensions for offloaded FN. ATTRS is the
638 raw attribute. DIMS is an array of dimensions, which is filled in.
639 LEVEL is the partitioning level of a routine, or -1 for an offload
640 region itself. USED is the mask of partitioned execution in the
644 oacc_validate_dims (tree fn
, tree attrs
, int *dims
, int level
, unsigned used
)
646 tree purpose
[GOMP_DIM_MAX
];
648 tree pos
= TREE_VALUE (attrs
);
650 /* Make sure the attribute creator attached the dimension
654 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
656 purpose
[ix
] = TREE_PURPOSE (pos
);
657 tree val
= TREE_VALUE (pos
);
658 dims
[ix
] = val
? TREE_INT_CST_LOW (val
) : -1;
659 pos
= TREE_CHAIN (pos
);
662 bool changed
= targetm
.goacc
.validate_dims (fn
, dims
, level
);
664 /* Default anything left to 1 or a partitioned default. */
665 for (ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
668 /* The OpenACC spec says 'If the [num_gangs] clause is not
669 specified, an implementation-defined default will be used;
670 the default may depend on the code within the construct.'
671 (2.5.6). Thus an implementation is free to choose
672 non-unity default for a parallel region that doesn't have
673 any gang-partitioned loops. However, it appears that there
674 is a sufficient body of user code that expects non-gang
675 partitioned regions to not execute in gang-redundant mode.
676 So we (a) don't warn about the non-portability and (b) pick
677 the minimum permissible dimension size when there is no
678 partitioned execution. Otherwise we pick the global
679 default for the dimension, which the user can control. The
680 same wording and logic applies to num_workers and
681 vector_length, however the worker- or vector- single
682 execution doesn't have the same impact as gang-redundant
683 execution. (If the minimum gang-level partioning is not 1,
684 the target is probably too confusing.) */
685 dims
[ix
] = (used
& GOMP_DIM_MASK (ix
)
686 ? oacc_default_dims
[ix
] : oacc_min_dims
[ix
]);
692 /* Replace the attribute with new values. */
694 for (ix
= GOMP_DIM_MAX
; ix
--;)
695 pos
= tree_cons (purpose
[ix
],
696 build_int_cst (integer_type_node
, dims
[ix
]), pos
);
697 oacc_replace_fn_attrib (fn
, pos
);
701 /* Create an empty OpenACC loop structure at LOC. */
704 new_oacc_loop_raw (oacc_loop
*parent
, location_t loc
)
706 oacc_loop
*loop
= XCNEW (oacc_loop
);
708 loop
->parent
= parent
;
712 loop
->sibling
= parent
->child
;
713 parent
->child
= loop
;
720 /* Create an outermost, dummy OpenACC loop for offloaded function
724 new_oacc_loop_outer (tree decl
)
726 return new_oacc_loop_raw (NULL
, DECL_SOURCE_LOCATION (decl
));
729 /* Start a new OpenACC loop structure beginning at head marker HEAD.
730 Link into PARENT loop. Return the new loop. */
733 new_oacc_loop (oacc_loop
*parent
, gcall
*marker
)
735 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (marker
));
737 loop
->marker
= marker
;
739 /* TODO: This is where device_type flattening would occur for the loop
742 loop
->flags
= TREE_INT_CST_LOW (gimple_call_arg (marker
, 3));
744 tree chunk_size
= integer_zero_node
;
745 if (loop
->flags
& OLF_GANG_STATIC
)
746 chunk_size
= gimple_call_arg (marker
, 4);
747 loop
->chunk_size
= chunk_size
;
752 /* Create a dummy loop encompassing a call to a openACC routine.
753 Extract the routine's partitioning requirements. */
756 new_oacc_loop_routine (oacc_loop
*parent
, gcall
*call
, tree decl
, tree attrs
)
758 oacc_loop
*loop
= new_oacc_loop_raw (parent
, gimple_location (call
));
759 int level
= oacc_fn_attrib_level (attrs
);
761 gcc_assert (level
>= 0);
764 loop
->routine
= decl
;
765 loop
->mask
= ((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1)
766 ^ (GOMP_DIM_MASK (level
) - 1));
769 /* Finish off the current OpenACC loop ending at tail marker TAIL.
770 Return the parent loop. */
773 finish_oacc_loop (oacc_loop
*loop
)
775 /* If the loop has been collapsed, don't partition it. */
776 if (loop
->ifns
.is_empty ())
777 loop
->mask
= loop
->flags
= 0;
781 /* Free all OpenACC loop structures within LOOP (inclusive). */
784 free_oacc_loop (oacc_loop
*loop
)
787 free_oacc_loop (loop
->sibling
);
789 free_oacc_loop (loop
->child
);
791 loop
->ifns
.release ();
795 /* Dump out the OpenACC loop head or tail beginning at FROM. */
798 dump_oacc_loop_part (FILE *file
, gcall
*from
, int depth
,
799 const char *title
, int level
)
801 enum ifn_unique_kind kind
802 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
804 fprintf (file
, "%*s%s-%d:\n", depth
* 2, "", title
, level
);
805 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
807 gimple
*stmt
= gsi_stmt (gsi
);
809 if (gimple_call_internal_p (stmt
, IFN_UNIQUE
))
811 enum ifn_unique_kind k
812 = ((enum ifn_unique_kind
) TREE_INT_CST_LOW
813 (gimple_call_arg (stmt
, 0)));
815 if (k
== kind
&& stmt
!= from
)
818 print_gimple_stmt (file
, stmt
, depth
* 2 + 2);
821 while (gsi_end_p (gsi
))
822 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
826 /* Dump OpenACC loop LOOP, its children, and its siblings. */
829 dump_oacc_loop (FILE *file
, oacc_loop
*loop
, int depth
)
833 fprintf (file
, "%*sLoop %x(%x) %s:%u\n", depth
* 2, "",
834 loop
->flags
, loop
->mask
,
835 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
));
838 print_gimple_stmt (file
, loop
->marker
, depth
* 2);
841 fprintf (file
, "%*sRoutine %s:%u:%s\n",
842 depth
* 2, "", DECL_SOURCE_FILE (loop
->routine
),
843 DECL_SOURCE_LINE (loop
->routine
),
844 IDENTIFIER_POINTER (DECL_NAME (loop
->routine
)));
846 for (ix
= GOMP_DIM_GANG
; ix
!= GOMP_DIM_MAX
; ix
++)
848 dump_oacc_loop_part (file
, loop
->heads
[ix
], depth
, "Head", ix
);
849 for (ix
= GOMP_DIM_MAX
; ix
--;)
851 dump_oacc_loop_part (file
, loop
->tails
[ix
], depth
, "Tail", ix
);
854 dump_oacc_loop (file
, loop
->child
, depth
+ 1);
856 dump_oacc_loop (file
, loop
->sibling
, depth
);
859 void debug_oacc_loop (oacc_loop
*);
861 /* Dump loops to stderr. */
864 debug_oacc_loop (oacc_loop
*loop
)
866 dump_oacc_loop (stderr
, loop
, 0);
869 /* Provide diagnostics on OpenACC loop LOOP, its children, and its
873 inform_oacc_loop (const oacc_loop
*loop
)
876 = loop
->mask
& GOMP_DIM_MASK (GOMP_DIM_GANG
) ? " gang" : "";
878 = loop
->mask
& GOMP_DIM_MASK (GOMP_DIM_WORKER
) ? " worker" : "";
880 = loop
->mask
& GOMP_DIM_MASK (GOMP_DIM_VECTOR
) ? " vector" : "";
881 const char *seq
= loop
->mask
== 0 ? " seq" : "";
882 const dump_user_location_t loc
883 = dump_user_location_t::from_location_t (loop
->loc
);
884 dump_printf_loc (MSG_OPTIMIZED_LOCATIONS
, loc
,
885 "assigned OpenACC%s%s%s%s loop parallelism\n", gang
, worker
,
889 inform_oacc_loop (loop
->child
);
891 inform_oacc_loop (loop
->sibling
);
894 /* DFS walk of basic blocks BB onwards, creating OpenACC loop
895 structures as we go. By construction these loops are properly
899 oacc_loop_discover_walk (oacc_loop
*loop
, basic_block bb
)
904 if (bb
->flags
& BB_VISITED
)
908 bb
->flags
|= BB_VISITED
;
910 /* Scan for loop markers. */
911 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);
914 gimple
*stmt
= gsi_stmt (gsi
);
916 if (!is_gimple_call (stmt
))
919 gcall
*call
= as_a
<gcall
*> (stmt
);
921 /* If this is a routine, make a dummy loop for it. */
922 if (tree decl
= gimple_call_fndecl (call
))
923 if (tree attrs
= oacc_get_fn_attrib (decl
))
925 gcc_assert (!marker
);
926 new_oacc_loop_routine (loop
, call
, decl
, attrs
);
929 if (!gimple_call_internal_p (call
))
932 switch (gimple_call_internal_fn (call
))
939 /* Record the abstraction function, so we can manipulate it
941 loop
->ifns
.safe_push (call
);
945 enum ifn_unique_kind kind
946 = (enum ifn_unique_kind
) (TREE_INT_CST_LOW
947 (gimple_call_arg (call
, 0)));
948 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
949 || kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
951 if (gimple_call_num_args (call
) == 2)
953 gcc_assert (marker
&& !remaining
);
955 if (kind
== IFN_UNIQUE_OACC_TAIL_MARK
)
956 loop
= finish_oacc_loop (loop
);
958 loop
->head_end
= call
;
962 int count
= TREE_INT_CST_LOW (gimple_call_arg (call
, 2));
966 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
967 loop
= new_oacc_loop (loop
, call
);
970 gcc_assert (count
== remaining
);
974 if (kind
== IFN_UNIQUE_OACC_HEAD_MARK
)
975 loop
->heads
[marker
] = call
;
977 loop
->tails
[remaining
] = call
;
984 if (remaining
|| marker
)
986 bb
= single_succ (bb
);
987 gcc_assert (single_pred_p (bb
) && !(bb
->flags
& BB_VISITED
));
991 /* Walk successor blocks. */
995 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
996 oacc_loop_discover_walk (loop
, e
->dest
);
999 /* LOOP is the first sibling. Reverse the order in place and return
1000 the new first sibling. Recurse to child loops. */
1003 oacc_loop_sibling_nreverse (oacc_loop
*loop
)
1005 oacc_loop
*last
= NULL
;
1009 loop
->child
= oacc_loop_sibling_nreverse (loop
->child
);
1011 oacc_loop
*next
= loop
->sibling
;
1012 loop
->sibling
= last
;
1021 /* Discover the OpenACC loops marked up by HEAD and TAIL markers for
1022 the current function. */
1025 oacc_loop_discovery ()
1027 /* Clear basic block flags, in particular BB_VISITED which we're going to use
1028 in the following. */
1031 oacc_loop
*top
= new_oacc_loop_outer (current_function_decl
);
1032 oacc_loop_discover_walk (top
, ENTRY_BLOCK_PTR_FOR_FN (cfun
));
1034 /* The siblings were constructed in reverse order, reverse them so
1035 that diagnostics come out in an unsurprising order. */
1036 top
= oacc_loop_sibling_nreverse (top
);
1041 /* Transform the abstract internal function markers starting at FROM
1042 to be for partitioning level LEVEL. Stop when we meet another HEAD
1046 oacc_loop_xform_head_tail (gcall
*from
, int level
)
1048 enum ifn_unique_kind kind
1049 = (enum ifn_unique_kind
) TREE_INT_CST_LOW (gimple_call_arg (from
, 0));
1050 tree replacement
= build_int_cst (unsigned_type_node
, level
);
1052 for (gimple_stmt_iterator gsi
= gsi_for_stmt (from
);;)
1054 gimple
*stmt
= gsi_stmt (gsi
);
1056 if (gimple_call_internal_p (stmt
, IFN_UNIQUE
))
1058 enum ifn_unique_kind k
1059 = ((enum ifn_unique_kind
)
1060 TREE_INT_CST_LOW (gimple_call_arg (stmt
, 0)));
1062 if (k
== IFN_UNIQUE_OACC_FORK
|| k
== IFN_UNIQUE_OACC_JOIN
)
1063 *gimple_call_arg_ptr (stmt
, 2) = replacement
;
1064 else if (k
== kind
&& stmt
!= from
)
1067 else if (gimple_call_internal_p (stmt
, IFN_GOACC_REDUCTION
))
1068 *gimple_call_arg_ptr (stmt
, 3) = replacement
;
1071 while (gsi_end_p (gsi
))
1072 gsi
= gsi_start_bb (single_succ (gsi_bb (gsi
)));
1076 /* Process the discovered OpenACC loops, setting the correct
1077 partitioning level etc. */
1080 oacc_loop_process (oacc_loop
*loop
)
1083 oacc_loop_process (loop
->child
);
1085 if (loop
->mask
&& !loop
->routine
)
1088 tree mask_arg
= build_int_cst (unsigned_type_node
, loop
->mask
);
1089 tree e_mask_arg
= build_int_cst (unsigned_type_node
, loop
->e_mask
);
1090 tree chunk_arg
= loop
->chunk_size
;
1093 for (ix
= 0; loop
->ifns
.iterate (ix
, &call
); ix
++)
1094 switch (gimple_call_internal_fn (call
))
1096 case IFN_GOACC_LOOP
:
1098 bool is_e
= gimple_call_arg (call
, 5) == integer_minus_one_node
;
1099 gimple_call_set_arg (call
, 5, is_e
? e_mask_arg
: mask_arg
);
1101 gimple_call_set_arg (call
, 4, chunk_arg
);
1105 case IFN_GOACC_TILE
:
1106 gimple_call_set_arg (call
, 3, mask_arg
);
1107 gimple_call_set_arg (call
, 4, e_mask_arg
);
1114 unsigned dim
= GOMP_DIM_GANG
;
1115 unsigned mask
= loop
->mask
| loop
->e_mask
;
1116 for (ix
= 0; ix
!= GOMP_DIM_MAX
&& mask
; ix
++)
1118 while (!(GOMP_DIM_MASK (dim
) & mask
))
1121 oacc_loop_xform_head_tail (loop
->heads
[ix
], dim
);
1122 oacc_loop_xform_head_tail (loop
->tails
[ix
], dim
);
1124 mask
^= GOMP_DIM_MASK (dim
);
1129 oacc_loop_process (loop
->sibling
);
1132 /* Walk the OpenACC loop heirarchy checking and assigning the
1133 programmer-specified partitionings. OUTER_MASK is the partitioning
1134 this loop is contained within. Return mask of partitioning
1135 encountered. If any auto loops are discovered, set GOMP_DIM_MAX
1139 oacc_loop_fixed_partitions (oacc_loop
*loop
, unsigned outer_mask
)
1141 unsigned this_mask
= loop
->mask
;
1142 unsigned mask_all
= 0;
1145 #ifdef ACCEL_COMPILER
1146 /* When device_type is supported, we want the device compiler to be
1147 noisy, if the loop parameters are device_type-specific. */
1153 bool auto_par
= (loop
->flags
& OLF_AUTO
) != 0;
1154 bool seq_par
= (loop
->flags
& OLF_SEQ
) != 0;
1155 bool tiling
= (loop
->flags
& OLF_TILE
) != 0;
1157 this_mask
= ((loop
->flags
>> OLF_DIM_BASE
)
1158 & (GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1));
1160 /* Apply auto partitioning if this is a non-partitioned regular
1161 loop, or (no more than) single axis tiled loop. */
1163 = !seq_par
&& this_mask
== (tiling
? this_mask
& -this_mask
: 0);
1165 if ((this_mask
!= 0) + auto_par
+ seq_par
> 1)
1168 error_at (loop
->loc
,
1170 ? G_("%<seq%> overrides other OpenACC loop specifiers")
1171 : G_("%<auto%> conflicts with other OpenACC loop "
1174 loop
->flags
&= ~OLF_AUTO
;
1178 &= ~((GOMP_DIM_MASK (GOMP_DIM_MAX
) - 1) << OLF_DIM_BASE
);
1183 if (maybe_auto
&& (loop
->flags
& OLF_INDEPENDENT
))
1185 loop
->flags
|= OLF_AUTO
;
1186 mask_all
|= GOMP_DIM_MASK (GOMP_DIM_MAX
);
1190 if (this_mask
& outer_mask
)
1192 const oacc_loop
*outer
;
1193 for (outer
= loop
->parent
; outer
; outer
= outer
->parent
)
1194 if ((outer
->mask
| outer
->e_mask
) & this_mask
)
1201 error_at (loop
->loc
,
1203 ? G_("routine call uses same OpenACC parallelism"
1204 " as containing loop")
1205 : G_("inner loop uses same OpenACC parallelism"
1206 " as containing loop"));
1207 inform (outer
->loc
, "containing loop here");
1210 error_at (loop
->loc
,
1212 ? G_("routine call uses OpenACC parallelism disallowed"
1213 " by containing routine")
1214 : G_("loop uses OpenACC parallelism disallowed"
1215 " by containing routine"));
1218 inform (DECL_SOURCE_LOCATION (loop
->routine
),
1219 "routine %qD declared here", loop
->routine
);
1221 this_mask
&= ~outer_mask
;
1225 unsigned outermost
= least_bit_hwi (this_mask
);
1227 if (outermost
&& outermost
<= outer_mask
)
1231 error_at (loop
->loc
,
1232 "incorrectly nested OpenACC loop parallelism");
1234 const oacc_loop
*outer
;
1235 for (outer
= loop
->parent
;
1236 outer
->flags
&& outer
->flags
< outermost
;
1237 outer
= outer
->parent
)
1239 inform (outer
->loc
, "containing loop here");
1242 this_mask
&= ~outermost
;
1246 mask_all
|= this_mask
;
1248 if (loop
->flags
& OLF_TILE
)
1250 /* When tiling, vector goes to the element loop, and failing
1251 that we put worker there. The std doesn't contemplate
1252 specifying all three. We choose to put worker and vector on
1253 the element loops in that case. */
1254 unsigned this_e_mask
= this_mask
& GOMP_DIM_MASK (GOMP_DIM_VECTOR
);
1255 if (!this_e_mask
|| this_mask
& GOMP_DIM_MASK (GOMP_DIM_GANG
))
1256 this_e_mask
|= this_mask
& GOMP_DIM_MASK (GOMP_DIM_WORKER
);
1258 loop
->e_mask
= this_e_mask
;
1259 this_mask
^= this_e_mask
;
1262 loop
->mask
= this_mask
;
1265 fprintf (dump_file
, "Loop %s:%d user specified %d & %d\n",
1266 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
),
1267 loop
->mask
, loop
->e_mask
);
1271 unsigned tmp_mask
= outer_mask
| this_mask
| loop
->e_mask
;
1272 loop
->inner
= oacc_loop_fixed_partitions (loop
->child
, tmp_mask
);
1273 mask_all
|= loop
->inner
;
1277 mask_all
|= oacc_loop_fixed_partitions (loop
->sibling
, outer_mask
);
1282 /* Walk the OpenACC loop heirarchy to assign auto-partitioned loops.
1283 OUTER_MASK is the partitioning this loop is contained within.
1284 OUTER_ASSIGN is true if an outer loop is being auto-partitioned.
1285 Return the cumulative partitioning used by this loop, siblings and
1289 oacc_loop_auto_partitions (oacc_loop
*loop
, unsigned outer_mask
,
1292 bool assign
= (loop
->flags
& OLF_AUTO
) && (loop
->flags
& OLF_INDEPENDENT
);
1294 bool tiling
= loop
->flags
& OLF_TILE
;
1296 #ifdef ACCEL_COMPILER
1297 /* When device_type is supported, we want the device compiler to be
1298 noisy, if the loop parameters are device_type-specific. */
1302 if (assign
&& (!outer_assign
|| loop
->inner
))
1304 /* Allocate outermost and non-innermost loops at the outermost
1305 non-innermost available level. */
1306 unsigned this_mask
= GOMP_DIM_MASK (GOMP_DIM_GANG
);
1308 /* Find the first outermost available partition. */
1309 while (this_mask
<= outer_mask
)
1312 /* Grab two axes if tiling, and we've not assigned anything */
1313 if (tiling
&& !(loop
->mask
| loop
->e_mask
))
1314 this_mask
|= this_mask
<< 1;
1316 /* Prohibit the innermost partitioning at the moment. */
1317 this_mask
&= GOMP_DIM_MASK (GOMP_DIM_MAX
- 1) - 1;
1319 /* Don't use any dimension explicitly claimed by an inner loop. */
1320 this_mask
&= ~loop
->inner
;
1322 if (tiling
&& !loop
->e_mask
)
1324 /* If we got two axes, allocate the inner one to the element
1326 loop
->e_mask
= this_mask
& (this_mask
<< 1);
1327 this_mask
^= loop
->e_mask
;
1330 loop
->mask
|= this_mask
;
1335 unsigned tmp_mask
= outer_mask
| loop
->mask
| loop
->e_mask
;
1336 loop
->inner
= oacc_loop_auto_partitions (loop
->child
, tmp_mask
,
1337 outer_assign
| assign
);
1340 if (assign
&& (!loop
->mask
|| (tiling
&& !loop
->e_mask
) || !outer_assign
))
1342 /* Allocate the loop at the innermost available level. Note
1343 that we do this even if we already assigned this loop the
1344 outermost available level above. That way we'll partition
1345 this along 2 axes, if they are available. */
1346 unsigned this_mask
= 0;
1348 /* Determine the outermost partitioning used within this loop. */
1349 this_mask
= loop
->inner
| GOMP_DIM_MASK (GOMP_DIM_MAX
);
1350 this_mask
= least_bit_hwi (this_mask
);
1352 /* Pick the partitioning just inside that one. */
1355 /* And avoid picking one use by an outer loop. */
1356 this_mask
&= ~outer_mask
;
1358 /* If tiling and we failed completely above, grab the next one
1359 too. Making sure it doesn't hit an outer loop. */
1362 this_mask
&= ~(loop
->e_mask
| loop
->mask
);
1363 unsigned tile_mask
= ((this_mask
>> 1)
1364 & ~(outer_mask
| loop
->e_mask
| loop
->mask
));
1366 if (tile_mask
|| loop
->mask
)
1368 loop
->e_mask
|= this_mask
;
1369 this_mask
= tile_mask
;
1371 if (!loop
->e_mask
&& noisy
)
1372 warning_at (loop
->loc
, 0,
1373 "insufficient partitioning available"
1374 " to parallelize element loop");
1377 loop
->mask
|= this_mask
;
1378 if (!loop
->mask
&& noisy
)
1379 warning_at (loop
->loc
, 0,
1381 ? G_("insufficient partitioning available"
1382 " to parallelize tile loop")
1383 : G_("insufficient partitioning available"
1384 " to parallelize loop"));
1387 if (assign
&& dump_file
)
1388 fprintf (dump_file
, "Auto loop %s:%d assigned %d & %d\n",
1389 LOCATION_FILE (loop
->loc
), LOCATION_LINE (loop
->loc
),
1390 loop
->mask
, loop
->e_mask
);
1392 unsigned inner_mask
= 0;
1395 inner_mask
|= oacc_loop_auto_partitions (loop
->sibling
,
1396 outer_mask
, outer_assign
);
1398 inner_mask
|= loop
->inner
| loop
->mask
| loop
->e_mask
;
1403 /* Walk the OpenACC loop heirarchy to check and assign partitioning
1404 axes. Return mask of partitioning. */
1407 oacc_loop_partition (oacc_loop
*loop
, unsigned outer_mask
)
1409 unsigned mask_all
= oacc_loop_fixed_partitions (loop
, outer_mask
);
1411 if (mask_all
& GOMP_DIM_MASK (GOMP_DIM_MAX
))
1413 mask_all
^= GOMP_DIM_MASK (GOMP_DIM_MAX
);
1414 mask_all
|= oacc_loop_auto_partitions (loop
, outer_mask
, false);
1419 /* Default fork/join early expander. Delete the function calls if
1420 there is no RTL expander. */
1423 default_goacc_fork_join (gcall
*ARG_UNUSED (call
),
1424 const int *ARG_UNUSED (dims
), bool is_fork
)
1427 return targetm
.have_oacc_fork ();
1429 return targetm
.have_oacc_join ();
1432 /* Default goacc.reduction early expander.
1434 LHS-opt = IFN_REDUCTION (KIND, RES_PTR, VAR, LEVEL, OP, OFFSET)
1435 If RES_PTR is not integer-zerop:
1436 SETUP - emit 'LHS = *RES_PTR', LHS = NULL
1437 TEARDOWN - emit '*RES_PTR = VAR'
1442 default_goacc_reduction (gcall
*call
)
1444 unsigned code
= (unsigned)TREE_INT_CST_LOW (gimple_call_arg (call
, 0));
1445 gimple_stmt_iterator gsi
= gsi_for_stmt (call
);
1446 tree lhs
= gimple_call_lhs (call
);
1447 tree var
= gimple_call_arg (call
, 2);
1448 gimple_seq seq
= NULL
;
1450 if (code
== IFN_GOACC_REDUCTION_SETUP
1451 || code
== IFN_GOACC_REDUCTION_TEARDOWN
)
1453 /* Setup and Teardown need to copy from/to the receiver object,
1455 tree ref_to_res
= gimple_call_arg (call
, 1);
1457 if (!integer_zerop (ref_to_res
))
1459 tree dst
= build_simple_mem_ref (ref_to_res
);
1462 if (code
== IFN_GOACC_REDUCTION_SETUP
)
1468 gimple_seq_add_stmt (&seq
, gimple_build_assign (dst
, src
));
1472 /* Copy VAR to LHS, if there is an LHS. */
1474 gimple_seq_add_stmt (&seq
, gimple_build_assign (lhs
, var
));
1476 gsi_replace_with_seq (&gsi
, seq
, true);
1479 /* Main entry point for oacc transformations which run on the device
1480 compiler after LTO, so we know what the target device is at this
1481 point (including the host fallback). */
1484 execute_oacc_device_lower ()
1486 tree attrs
= oacc_get_fn_attrib (current_function_decl
);
1489 /* Not an offloaded function. */
1492 /* Parse the default dim argument exactly once. */
1493 if ((const void *)flag_openacc_dims
!= &flag_openacc_dims
)
1495 oacc_parse_default_dims (flag_openacc_dims
);
1496 flag_openacc_dims
= (char *)&flag_openacc_dims
;
1499 bool is_oacc_kernels
1500 = (lookup_attribute ("oacc kernels",
1501 DECL_ATTRIBUTES (current_function_decl
)) != NULL
);
1502 bool is_oacc_kernels_parallelized
1503 = (lookup_attribute ("oacc kernels parallelized",
1504 DECL_ATTRIBUTES (current_function_decl
)) != NULL
);
1506 /* Unparallelized OpenACC kernels constructs must get launched as 1 x 1 x 1
1507 kernels, so remove the parallelism dimensions function attributes
1508 potentially set earlier on. */
1509 if (is_oacc_kernels
&& !is_oacc_kernels_parallelized
)
1511 oacc_set_fn_attrib (current_function_decl
, NULL
, NULL
);
1512 attrs
= oacc_get_fn_attrib (current_function_decl
);
1515 /* Discover, partition and process the loops. */
1516 oacc_loop
*loops
= oacc_loop_discovery ();
1517 int fn_level
= oacc_fn_attrib_level (attrs
);
1522 fprintf (dump_file
, "Function is OpenACC routine level %d\n",
1524 else if (is_oacc_kernels
)
1525 fprintf (dump_file
, "Function is %s OpenACC kernels offload\n",
1526 (is_oacc_kernels_parallelized
1527 ? "parallelized" : "unparallelized"));
1529 fprintf (dump_file
, "Function is OpenACC parallel offload\n");
1532 unsigned outer_mask
= fn_level
>= 0 ? GOMP_DIM_MASK (fn_level
) - 1 : 0;
1533 unsigned used_mask
= oacc_loop_partition (loops
, outer_mask
);
1534 /* OpenACC kernels constructs are special: they currently don't use the
1535 generic oacc_loop infrastructure and attribute/dimension processing. */
1536 if (is_oacc_kernels
&& is_oacc_kernels_parallelized
)
1538 /* Parallelized OpenACC kernels constructs use gang parallelism. See
1539 also tree-parloops.c:create_parallel_loop. */
1540 used_mask
|= GOMP_DIM_MASK (GOMP_DIM_GANG
);
1543 int dims
[GOMP_DIM_MAX
];
1544 oacc_validate_dims (current_function_decl
, attrs
, dims
, fn_level
, used_mask
);
1548 const char *comma
= "Compute dimensions [";
1549 for (int ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++, comma
= ", ")
1550 fprintf (dump_file
, "%s%d", comma
, dims
[ix
]);
1551 fprintf (dump_file
, "]\n");
1554 oacc_loop_process (loops
);
1557 fprintf (dump_file
, "OpenACC loops\n");
1558 dump_oacc_loop (dump_file
, loops
, 0);
1559 fprintf (dump_file
, "\n");
1561 if (dump_enabled_p ())
1563 oacc_loop
*l
= loops
;
1564 /* OpenACC kernels constructs are special: they currently don't use the
1565 generic oacc_loop infrastructure. */
1566 if (is_oacc_kernels
)
1568 /* Create a fake oacc_loop for diagnostic purposes. */
1569 l
= new_oacc_loop_raw (NULL
,
1570 DECL_SOURCE_LOCATION (current_function_decl
));
1571 l
->mask
= used_mask
;
1575 /* Skip the outermost, dummy OpenACC loop */
1579 inform_oacc_loop (l
);
1580 if (is_oacc_kernels
)
1584 /* Offloaded targets may introduce new basic blocks, which require
1585 dominance information to update SSA. */
1586 calculate_dominance_info (CDI_DOMINATORS
);
1588 /* Now lower internal loop functions to target-specific code
1591 FOR_ALL_BB_FN (bb
, cfun
)
1592 for (gimple_stmt_iterator gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
);)
1594 gimple
*stmt
= gsi_stmt (gsi
);
1595 if (!is_gimple_call (stmt
))
1601 gcall
*call
= as_a
<gcall
*> (stmt
);
1602 if (!gimple_call_internal_p (call
))
1608 /* Rewind to allow rescan. */
1610 bool rescan
= false, remove
= false;
1611 enum internal_fn ifn_code
= gimple_call_internal_fn (call
);
1617 case IFN_GOACC_TILE
:
1618 oacc_xform_tile (call
);
1622 case IFN_GOACC_LOOP
:
1623 oacc_xform_loop (call
);
1627 case IFN_GOACC_REDUCTION
:
1628 /* Mark the function for SSA renaming. */
1629 mark_virtual_operands_for_renaming (cfun
);
1631 /* If the level is -1, this ended up being an unused
1632 axis. Handle as a default. */
1633 if (integer_minus_onep (gimple_call_arg (call
, 3)))
1634 default_goacc_reduction (call
);
1636 targetm
.goacc
.reduction (call
);
1642 enum ifn_unique_kind kind
1643 = ((enum ifn_unique_kind
)
1644 TREE_INT_CST_LOW (gimple_call_arg (call
, 0)));
1651 case IFN_UNIQUE_OACC_FORK
:
1652 case IFN_UNIQUE_OACC_JOIN
:
1653 if (integer_minus_onep (gimple_call_arg (call
, 2)))
1655 else if (!targetm
.goacc
.fork_join
1656 (call
, dims
, kind
== IFN_UNIQUE_OACC_FORK
))
1660 case IFN_UNIQUE_OACC_HEAD_MARK
:
1661 case IFN_UNIQUE_OACC_TAIL_MARK
:
1669 if (gsi_end_p (gsi
))
1670 /* We rewound past the beginning of the BB. */
1671 gsi
= gsi_start_bb (bb
);
1673 /* Undo the rewind. */
1678 if (gimple_vdef (call
))
1679 replace_uses_by (gimple_vdef (call
), gimple_vuse (call
));
1680 if (gimple_call_lhs (call
))
1682 /* Propagate the data dependency var. */
1683 gimple
*ass
= gimple_build_assign (gimple_call_lhs (call
),
1684 gimple_call_arg (call
, 1));
1685 gsi_replace (&gsi
, ass
, false);
1688 gsi_remove (&gsi
, true);
1691 /* If not rescanning, advance over the call. */
1695 free_oacc_loop (loops
);
1700 /* Default launch dimension validator. Force everything to 1. A
1701 backend that wants to provide larger dimensions must override this
1705 default_goacc_validate_dims (tree
ARG_UNUSED (decl
), int *dims
,
1706 int ARG_UNUSED (fn_level
))
1708 bool changed
= false;
1710 for (unsigned ix
= 0; ix
!= GOMP_DIM_MAX
; ix
++)
1722 /* Default dimension bound is unknown on accelerator and 1 on host. */
1725 default_goacc_dim_limit (int ARG_UNUSED (axis
))
1727 #ifdef ACCEL_COMPILER
1736 const pass_data pass_data_oacc_device_lower
=
1738 GIMPLE_PASS
, /* type */
1739 "oaccdevlow", /* name */
1740 OPTGROUP_OMP
, /* optinfo_flags */
1741 TV_NONE
, /* tv_id */
1742 PROP_cfg
, /* properties_required */
1743 0 /* Possibly PROP_gimple_eomp. */, /* properties_provided */
1744 0, /* properties_destroyed */
1745 0, /* todo_flags_start */
1746 TODO_update_ssa
| TODO_cleanup_cfg
, /* todo_flags_finish */
1749 class pass_oacc_device_lower
: public gimple_opt_pass
1752 pass_oacc_device_lower (gcc::context
*ctxt
)
1753 : gimple_opt_pass (pass_data_oacc_device_lower
, ctxt
)
1756 /* opt_pass methods: */
1757 virtual bool gate (function
*) { return flag_openacc
; };
1759 virtual unsigned int execute (function
*)
1761 return execute_oacc_device_lower ();
1764 }; // class pass_oacc_device_lower
1769 make_pass_oacc_device_lower (gcc::context
*ctxt
)
1771 return new pass_oacc_device_lower (ctxt
);
1775 /* Rewrite GOMP_SIMT_ENTER_ALLOC call given by GSI and remove the preceding
1776 GOMP_SIMT_ENTER call identifying the privatized variables, which are
1777 turned to structure fields and receive a DECL_VALUE_EXPR accordingly.
1778 Set *REGIMPLIFY to true, except if no privatized variables were seen. */
1781 ompdevlow_adjust_simt_enter (gimple_stmt_iterator
*gsi
, bool *regimplify
)
1783 gimple
*alloc_stmt
= gsi_stmt (*gsi
);
1784 tree simtrec
= gimple_call_lhs (alloc_stmt
);
1785 tree simduid
= gimple_call_arg (alloc_stmt
, 0);
1786 gimple
*enter_stmt
= SSA_NAME_DEF_STMT (simduid
);
1787 gcc_assert (gimple_call_internal_p (enter_stmt
, IFN_GOMP_SIMT_ENTER
));
1788 tree rectype
= lang_hooks
.types
.make_type (RECORD_TYPE
);
1789 TYPE_ARTIFICIAL (rectype
) = TYPE_NAMELESS (rectype
) = 1;
1790 TREE_ADDRESSABLE (rectype
) = 1;
1791 TREE_TYPE (simtrec
) = build_pointer_type (rectype
);
1792 for (unsigned i
= 1; i
< gimple_call_num_args (enter_stmt
); i
++)
1794 tree
*argp
= gimple_call_arg_ptr (enter_stmt
, i
);
1795 if (*argp
== null_pointer_node
)
1797 gcc_assert (TREE_CODE (*argp
) == ADDR_EXPR
1798 && VAR_P (TREE_OPERAND (*argp
, 0)));
1799 tree var
= TREE_OPERAND (*argp
, 0);
1801 tree field
= build_decl (DECL_SOURCE_LOCATION (var
), FIELD_DECL
,
1802 DECL_NAME (var
), TREE_TYPE (var
));
1803 SET_DECL_ALIGN (field
, DECL_ALIGN (var
));
1804 DECL_USER_ALIGN (field
) = DECL_USER_ALIGN (var
);
1805 TREE_THIS_VOLATILE (field
) = TREE_THIS_VOLATILE (var
);
1807 insert_field_into_struct (rectype
, field
);
1809 tree t
= build_simple_mem_ref (simtrec
);
1810 t
= build3 (COMPONENT_REF
, TREE_TYPE (var
), t
, field
, NULL
);
1811 TREE_THIS_VOLATILE (t
) = TREE_THIS_VOLATILE (var
);
1812 SET_DECL_VALUE_EXPR (var
, t
);
1813 DECL_HAS_VALUE_EXPR_P (var
) = 1;
1816 layout_type (rectype
);
1817 tree size
= TYPE_SIZE_UNIT (rectype
);
1818 tree align
= build_int_cst (TREE_TYPE (size
), TYPE_ALIGN_UNIT (rectype
));
1821 = gimple_build_call_internal (IFN_GOMP_SIMT_ENTER_ALLOC
, 2, size
, align
);
1822 gimple_call_set_lhs (alloc_stmt
, simtrec
);
1823 gsi_replace (gsi
, alloc_stmt
, false);
1824 gimple_stmt_iterator enter_gsi
= gsi_for_stmt (enter_stmt
);
1825 enter_stmt
= gimple_build_assign (simduid
, gimple_call_arg (enter_stmt
, 0));
1826 gsi_replace (&enter_gsi
, enter_stmt
, false);
1830 if (single_imm_use (simtrec
, &use
, &exit_stmt
))
1832 gcc_assert (gimple_call_internal_p (exit_stmt
, IFN_GOMP_SIMT_EXIT
));
1833 gimple_stmt_iterator exit_gsi
= gsi_for_stmt (exit_stmt
);
1834 tree clobber
= build_constructor (rectype
, NULL
);
1835 TREE_THIS_VOLATILE (clobber
) = 1;
1836 exit_stmt
= gimple_build_assign (build_simple_mem_ref (simtrec
), clobber
);
1837 gsi_insert_before (&exit_gsi
, exit_stmt
, GSI_SAME_STMT
);
1840 gcc_checking_assert (has_zero_uses (simtrec
));
1843 /* Callback for walk_gimple_stmt used to scan for SIMT-privatized variables. */
1846 find_simtpriv_var_op (tree
*tp
, int *walk_subtrees
, void *)
1851 && DECL_HAS_VALUE_EXPR_P (t
)
1852 && lookup_attribute ("omp simt private", DECL_ATTRIBUTES (t
)))
1860 /* Cleanup uses of SIMT placeholder internal functions: on non-SIMT targets,
1861 VF is 1 and LANE is 0; on SIMT targets, VF is folded to a constant, and
1862 LANE is kept to be expanded to RTL later on. Also cleanup all other SIMT
1863 internal functions on non-SIMT targets, and likewise some SIMD internal
1864 functions on SIMT targets. */
1867 execute_omp_device_lower ()
1869 int vf
= targetm
.simt
.vf
? targetm
.simt
.vf () : 1;
1870 bool regimplify
= false;
1872 gimple_stmt_iterator gsi
;
1873 FOR_EACH_BB_FN (bb
, cfun
)
1874 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
1876 gimple
*stmt
= gsi_stmt (gsi
);
1877 if (!is_gimple_call (stmt
) || !gimple_call_internal_p (stmt
))
1879 tree lhs
= gimple_call_lhs (stmt
), rhs
= NULL_TREE
;
1880 tree type
= lhs
? TREE_TYPE (lhs
) : integer_type_node
;
1881 switch (gimple_call_internal_fn (stmt
))
1883 case IFN_GOMP_USE_SIMT
:
1884 rhs
= vf
== 1 ? integer_zero_node
: integer_one_node
;
1886 case IFN_GOMP_SIMT_ENTER
:
1887 rhs
= vf
== 1 ? gimple_call_arg (stmt
, 0) : NULL_TREE
;
1888 goto simtreg_enter_exit
;
1889 case IFN_GOMP_SIMT_ENTER_ALLOC
:
1891 ompdevlow_adjust_simt_enter (&gsi
, ®implify
);
1892 rhs
= vf
== 1 ? null_pointer_node
: NULL_TREE
;
1893 goto simtreg_enter_exit
;
1894 case IFN_GOMP_SIMT_EXIT
:
1898 unlink_stmt_vdef (stmt
);
1900 case IFN_GOMP_SIMT_LANE
:
1901 case IFN_GOMP_SIMT_LAST_LANE
:
1902 rhs
= vf
== 1 ? build_zero_cst (type
) : NULL_TREE
;
1904 case IFN_GOMP_SIMT_VF
:
1905 rhs
= build_int_cst (type
, vf
);
1907 case IFN_GOMP_SIMT_ORDERED_PRED
:
1908 rhs
= vf
== 1 ? integer_zero_node
: NULL_TREE
;
1910 unlink_stmt_vdef (stmt
);
1912 case IFN_GOMP_SIMT_VOTE_ANY
:
1913 case IFN_GOMP_SIMT_XCHG_BFLY
:
1914 case IFN_GOMP_SIMT_XCHG_IDX
:
1915 rhs
= vf
== 1 ? gimple_call_arg (stmt
, 0) : NULL_TREE
;
1917 case IFN_GOMP_SIMD_LANE
:
1918 case IFN_GOMP_SIMD_LAST_LANE
:
1919 rhs
= vf
!= 1 ? build_zero_cst (type
) : NULL_TREE
;
1921 case IFN_GOMP_SIMD_VF
:
1922 rhs
= vf
!= 1 ? build_one_cst (type
) : NULL_TREE
;
1929 stmt
= lhs
? gimple_build_assign (lhs
, rhs
) : gimple_build_nop ();
1930 gsi_replace (&gsi
, stmt
, false);
1933 FOR_EACH_BB_REVERSE_FN (bb
, cfun
)
1934 for (gsi
= gsi_last_bb (bb
); !gsi_end_p (gsi
); gsi_prev (&gsi
))
1935 if (walk_gimple_stmt (&gsi
, NULL
, find_simtpriv_var_op
, NULL
))
1937 if (gimple_clobber_p (gsi_stmt (gsi
)))
1938 gsi_remove (&gsi
, true);
1940 gimple_regimplify_operands (gsi_stmt (gsi
), &gsi
);
1943 cfun
->has_force_vectorize_loops
= false;
1949 const pass_data pass_data_omp_device_lower
=
1951 GIMPLE_PASS
, /* type */
1952 "ompdevlow", /* name */
1953 OPTGROUP_OMP
, /* optinfo_flags */
1954 TV_NONE
, /* tv_id */
1955 PROP_cfg
, /* properties_required */
1956 PROP_gimple_lomp_dev
, /* properties_provided */
1957 0, /* properties_destroyed */
1958 0, /* todo_flags_start */
1959 TODO_update_ssa
, /* todo_flags_finish */
1962 class pass_omp_device_lower
: public gimple_opt_pass
1965 pass_omp_device_lower (gcc::context
*ctxt
)
1966 : gimple_opt_pass (pass_data_omp_device_lower
, ctxt
)
1969 /* opt_pass methods: */
1970 virtual bool gate (function
*fun
)
1972 return !(fun
->curr_properties
& PROP_gimple_lomp_dev
);
1974 virtual unsigned int execute (function
*)
1976 return execute_omp_device_lower ();
1979 }; // class pass_expand_omp_ssa
1984 make_pass_omp_device_lower (gcc::context
*ctxt
)
1986 return new pass_omp_device_lower (ctxt
);
1989 /* "omp declare target link" handling pass. */
1993 const pass_data pass_data_omp_target_link
=
1995 GIMPLE_PASS
, /* type */
1996 "omptargetlink", /* name */
1997 OPTGROUP_OMP
, /* optinfo_flags */
1998 TV_NONE
, /* tv_id */
1999 PROP_ssa
, /* properties_required */
2000 0, /* properties_provided */
2001 0, /* properties_destroyed */
2002 0, /* todo_flags_start */
2003 TODO_update_ssa
, /* todo_flags_finish */
2006 class pass_omp_target_link
: public gimple_opt_pass
2009 pass_omp_target_link (gcc::context
*ctxt
)
2010 : gimple_opt_pass (pass_data_omp_target_link
, ctxt
)
2013 /* opt_pass methods: */
2014 virtual bool gate (function
*fun
)
2016 #ifdef ACCEL_COMPILER
2017 return offloading_function_p (fun
->decl
);
2024 virtual unsigned execute (function
*);
2027 /* Callback for walk_gimple_stmt used to scan for link var operands. */
2030 find_link_var_op (tree
*tp
, int *walk_subtrees
, void *)
2035 && DECL_HAS_VALUE_EXPR_P (t
)
2036 && is_global_var (t
)
2037 && lookup_attribute ("omp declare target link", DECL_ATTRIBUTES (t
)))
2047 pass_omp_target_link::execute (function
*fun
)
2050 FOR_EACH_BB_FN (bb
, fun
)
2052 gimple_stmt_iterator gsi
;
2053 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
2054 if (walk_gimple_stmt (&gsi
, NULL
, find_link_var_op
, NULL
))
2055 gimple_regimplify_operands (gsi_stmt (gsi
), &gsi
);
2064 make_pass_omp_target_link (gcc::context
*ctxt
)
2066 return new pass_omp_target_link (ctxt
);