Contributed by Diego Novillo <dnovillo@redhat.com>
- Copyright (C) 2005-2018 Free Software Foundation, Inc.
+ Copyright (C) 2005-2021 Free Software Foundation, Inc.
This file is part of GCC.
#include "splay-tree.h"
#include "omp-general.h"
#include "omp-low.h"
-#include "omp-grid.h"
#include "gimple-low.h"
+#include "alloc-pool.h"
#include "symbol-summary.h"
#include "tree-nested.h"
#include "context.h"
#include "gomp-constants.h"
#include "gimple-pretty-print.h"
-#include "hsa-common.h"
#include "stringpool.h"
#include "attribs.h"
otherwise. */
gimple *simt_stmt;
+ /* For task reductions registered in this context, a vector containing
+ the length of the private copies block (if constant, otherwise NULL)
+ and then offsets (if constant, otherwise NULL) for each entry. */
+ vec<tree> task_reductions;
+
+ /* A hash map from the reduction clauses to the registered array
+ elts. */
+ hash_map<tree, unsigned> *task_reduction_map;
+
+ /* And a hash map from the lastprivate(conditional:) variables to their
+ corresponding tracking loop iteration variables. */
+ hash_map<tree, tree> *lastprivate_conditional_map;
+
+ /* And a hash map from the allocate variables to their corresponding
+ allocators. */
+ hash_map<tree, tree> *allocate_map;
+
+ /* A tree_list of the reduction clauses in this context. This is
+ only used for checking the consistency of OpenACC reduction
+ clauses in scan_omp_for and is not guaranteed to contain a valid
+ value outside of this function. */
+ tree local_reduction_clauses;
+
+ /* A tree_list of the reduction clauses in outer contexts. This is
+ only used for checking the consistency of OpenACC reduction
+ clauses in scan_omp_for and is not guaranteed to contain a valid
+ value outside of this function. */
+ tree outer_reduction_clauses;
+
/* Nesting depth of this context. Used to beautify error messages re
invalid gotos. The outermost ctx is depth 1, with depth 0 being
reserved for the main body of the function. */
/* True if this construct can be cancelled. */
bool cancellable;
+
+ /* True if lower_omp_1 should look up lastprivate conditional in parent
+ context. */
+ bool combined_into_simd_safelen1;
+
+ /* True if there is nested scan context with inclusive clause. */
+ bool scan_inclusive;
+
+ /* True if there is nested scan context with exclusive clause. */
+ bool scan_exclusive;
+
+ /* True in the second simd loop of for simd with inscan reductions. */
+ bool for_simd_scan_phase;
+
+ /* True if there is order(concurrent) clause on the construct. */
+ bool order_concurrent;
+
+ /* True if there is bind clause on the construct (i.e. a loop construct). */
+ bool loop_p;
};
static splay_tree all_contexts;
static int taskreg_nesting_level;
static int target_nesting_level;
static bitmap task_shared_vars;
+static bitmap global_nonaddressable_vars;
static vec<omp_context *> taskreg_contexts;
static void scan_omp (gimple_seq *, omp_context *);
*handled_ops_p = false; \
break;
-/* Return true if CTX corresponds to an oacc parallel region. */
+/* Return whether CTX represents an OpenACC 'parallel' or 'serial' construct.
+ (This doesn't include OpenACC 'kernels' decomposed parts.) */
static bool
-is_oacc_parallel (omp_context *ctx)
+is_oacc_parallel_or_serial (omp_context *ctx)
{
enum gimple_code outer_type = gimple_code (ctx->stmt);
return ((outer_type == GIMPLE_OMP_TARGET)
- && (gimple_omp_target_kind (ctx->stmt)
- == GF_OMP_TARGET_KIND_OACC_PARALLEL));
+ && ((gimple_omp_target_kind (ctx->stmt)
+ == GF_OMP_TARGET_KIND_OACC_PARALLEL)
+ || (gimple_omp_target_kind (ctx->stmt)
+ == GF_OMP_TARGET_KIND_OACC_SERIAL)));
}
-/* Return true if CTX corresponds to an oacc kernels region. */
+/* Return whether CTX represents an OpenACC 'kernels' construct.
+ (This doesn't include OpenACC 'kernels' decomposed parts.) */
static bool
is_oacc_kernels (omp_context *ctx)
== GF_OMP_TARGET_KIND_OACC_KERNELS));
}
+/* Return whether CTX represents an OpenACC 'kernels' decomposed part. */
+
+static bool
+is_oacc_kernels_decomposed_part (omp_context *ctx)
+{
+ enum gimple_code outer_type = gimple_code (ctx->stmt);
+ return ((outer_type == GIMPLE_OMP_TARGET)
+ && ((gimple_omp_target_kind (ctx->stmt)
+ == GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED)
+ || (gimple_omp_target_kind (ctx->stmt)
+ == GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE)
+ || (gimple_omp_target_kind (ctx->stmt)
+ == GF_OMP_TARGET_KIND_OACC_DATA_KERNELS)));
+}
+
+/* Return true if STMT corresponds to an OpenMP target region. */
+static bool
+is_omp_target (gimple *stmt)
+{
+ if (gimple_code (stmt) == GIMPLE_OMP_TARGET)
+ {
+ int kind = gimple_omp_target_kind (stmt);
+ return (kind == GF_OMP_TARGET_KIND_REGION
+ || kind == GF_OMP_TARGET_KIND_DATA
+ || kind == GF_OMP_TARGET_KIND_ENTER_DATA
+ || kind == GF_OMP_TARGET_KIND_EXIT_DATA);
+ }
+ return false;
+}
+
/* If DECL is the artificial dummy VAR_DECL created for non-static
data member privatization, return the underlying "this" parameter,
otherwise return NULL. */
}
-/* Return true if CTX is for an omp parallel or omp task. */
+/* Return true if CTX is for a host omp teams. */
+
+static inline bool
+is_host_teams_ctx (omp_context *ctx)
+{
+ return gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
+ && gimple_omp_teams_host (as_a <gomp_teams *> (ctx->stmt));
+}
+
+/* Return true if CTX is for an omp parallel or omp task or host omp teams
+ (the last one is strictly not a task region in OpenMP speak, but we
+ need to treat it similarly). */
static inline bool
is_taskreg_ctx (omp_context *ctx)
{
- return is_parallel_ctx (ctx) || is_task_ctx (ctx);
+ return is_parallel_ctx (ctx) || is_task_ctx (ctx) || is_host_teams_ctx (ctx);
}
/* Return true if EXPR is variable sized. */
be passing an address in this case? Should we simply assert
this to be false, or should we have a cleanup pass that removes
these from the list of mappings? */
- if (TREE_STATIC (decl) || DECL_EXTERNAL (decl))
+ if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, shared_ctx)))
return true;
/* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell
/* Do not use copy-in/copy-out for variables that have their
address taken. */
- if (TREE_ADDRESSABLE (decl))
+ if (is_global_var (decl))
+ {
+ /* For file scope vars, track whether we've seen them as
+ non-addressable initially and in that case, keep the same
+ answer for the duration of the pass, even when they are made
+ addressable later on e.g. through reduction expansion. Global
+ variables which weren't addressable before the pass will not
+ have their privatized copies address taken. See PR91216. */
+ if (!TREE_ADDRESSABLE (decl))
+ {
+ if (!global_nonaddressable_vars)
+ global_nonaddressable_vars = BITMAP_ALLOC (NULL);
+ bitmap_set_bit (global_nonaddressable_vars, DECL_UID (decl));
+ }
+ else if (!global_nonaddressable_vars
+ || !bitmap_bit_p (global_nonaddressable_vars,
+ DECL_UID (decl)))
+ return true;
+ }
+ else if (TREE_ADDRESSABLE (decl))
return true;
/* lower_send_shared_vars only uses copy-in, but not copy-out
omp_context *up;
for (up = shared_ctx->outer; up; up = up->outer)
- if (is_taskreg_ctx (up) && maybe_lookup_decl (decl, up))
+ if ((is_taskreg_ctx (up)
+ || (gimple_code (up->stmt) == GIMPLE_OMP_TARGET
+ && is_gimple_omp_offloaded (up->stmt)))
+ && maybe_lookup_decl (decl, up))
break;
if (up)
{
tree c;
- for (c = gimple_omp_taskreg_clauses (up->stmt);
- c; c = OMP_CLAUSE_CHAIN (c))
- if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
- && OMP_CLAUSE_DECL (c) == decl)
- break;
+ if (gimple_code (up->stmt) == GIMPLE_OMP_TARGET)
+ {
+ for (c = gimple_omp_target_clauses (up->stmt);
+ c; c = OMP_CLAUSE_CHAIN (c))
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
+ && OMP_CLAUSE_DECL (c) == decl)
+ break;
+ }
+ else
+ for (c = gimple_omp_taskreg_clauses (up->stmt);
+ c; c = OMP_CLAUSE_CHAIN (c))
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED
+ && OMP_CLAUSE_DECL (c) == decl)
+ break;
if (c)
goto maybe_mark_addressable_and_ret;
it's address. But we don't need to take address of privatizations
from that var. */
if (TREE_ADDRESSABLE (var)
- && task_shared_vars
- && bitmap_bit_p (task_shared_vars, DECL_UID (var)))
+ && ((task_shared_vars
+ && bitmap_bit_p (task_shared_vars, DECL_UID (var)))
+ || (global_nonaddressable_vars
+ && bitmap_bit_p (global_nonaddressable_vars, DECL_UID (var)))))
TREE_ADDRESSABLE (copy) = 0;
ctx->block_vars = copy;
enum omp_clause_code code = OMP_CLAUSE_ERROR)
{
tree x;
+ omp_context *outer = ctx->outer;
+ while (outer && gimple_code (outer->stmt) == GIMPLE_OMP_TASKGROUP)
+ outer = outer->outer;
if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
x = var;
x = build_receiver_ref (var, by_ref, ctx);
}
else if ((gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
- && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
+ && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
+ || ctx->loop_p
|| (code == OMP_CLAUSE_PRIVATE
&& (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
|| gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS
Similarly for OMP_CLAUSE_PRIVATE with outer ref, that can refer
to private vars in all worksharing constructs. */
x = NULL_TREE;
- if (ctx->outer && is_taskreg_ctx (ctx))
- x = lookup_decl (var, ctx->outer);
- else if (ctx->outer)
+ if (outer && is_taskreg_ctx (outer))
+ x = lookup_decl (var, outer);
+ else if (outer)
x = maybe_lookup_decl_in_outer_ctx (var, ctx);
if (x == NULL_TREE)
x = var;
}
else if (code == OMP_CLAUSE_LASTPRIVATE && is_taskloop_ctx (ctx))
{
- gcc_assert (ctx->outer);
+ gcc_assert (outer);
splay_tree_node n
- = splay_tree_lookup (ctx->outer->field_map,
+ = splay_tree_lookup (outer->field_map,
(splay_tree_key) &DECL_UID (var));
if (n == NULL)
{
- if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx->outer)))
+ if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, outer)))
x = var;
else
- x = lookup_decl (var, ctx->outer);
+ x = lookup_decl (var, outer);
}
else
{
tree field = (tree) n->value;
/* If the receiver record type was remapped in the child function,
remap the field into the new record type. */
- x = maybe_lookup_field (field, ctx->outer);
+ x = maybe_lookup_field (field, outer);
if (x != NULL)
field = x;
- x = build_simple_mem_ref (ctx->outer->receiver_decl);
+ x = build_simple_mem_ref (outer->receiver_decl);
x = omp_build_component_ref (x, field);
- if (use_pointer_for_field (var, ctx->outer))
+ if (use_pointer_for_field (var, outer))
x = build_simple_mem_ref (x);
}
}
- else if (ctx->outer)
- {
- omp_context *outer = ctx->outer;
- if (gimple_code (outer->stmt) == GIMPLE_OMP_GRID_BODY)
- {
- outer = outer->outer;
- gcc_assert (outer
- && gimple_code (outer->stmt) != GIMPLE_OMP_GRID_BODY);
- }
- x = lookup_decl (var, outer);
- }
+ else if (outer)
+ x = lookup_decl (var, outer);
else if (omp_is_reference (var))
/* This can happen with orphaned constructs. If var is reference, it is
possible it is shared and as such valid. */
BASE_POINTERS_RESTRICT, declare the field with restrict. */
static void
-install_var_field (tree var, bool by_ref, int mask, omp_context *ctx,
- bool base_pointers_restrict = false)
+install_var_field (tree var, bool by_ref, int mask, omp_context *ctx)
{
tree field, type, sfield = NULL_TREE;
splay_tree_key key = (splay_tree_key) var;
+ if ((mask & 16) != 0)
+ {
+ key = (splay_tree_key) &DECL_NAME (var);
+ gcc_checking_assert (key != (splay_tree_key) var);
+ }
if ((mask & 8) != 0)
{
key = (splay_tree_key) &DECL_UID (var);
|| !is_gimple_omp_oacc (ctx->stmt));
type = TREE_TYPE (var);
+ if ((mask & 16) != 0)
+ type = lang_hooks.decls.omp_array_data (var, true);
+
/* Prevent redeclaring the var in the split-off function with a restrict
pointer type. Note that we only clear type itself, restrict qualifiers in
the pointed-to type will be ignored by points-to analysis. */
type = build_pointer_type (build_pointer_type (type));
}
else if (by_ref)
- {
- type = build_pointer_type (type);
- if (base_pointers_restrict)
- type = build_qualified_type (type, TYPE_QUAL_RESTRICT);
- }
- else if ((mask & 3) == 1 && omp_is_reference (var))
+ type = build_pointer_type (type);
+ else if ((mask & (32 | 3)) == 1 && omp_is_reference (var))
type = TREE_TYPE (type);
field = build_decl (DECL_SOURCE_LOCATION (var),
side effect of making dwarf2out ignore this member, so for helpful
debugging we clear it later in delete_omp_context. */
DECL_ABSTRACT_ORIGIN (field) = var;
- if (type == TREE_TYPE (var))
+ if ((mask & 16) == 0 && type == TREE_TYPE (var))
{
SET_DECL_ALIGN (field, DECL_ALIGN (var));
DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var);
ctx->cb.copy_decl = omp_copy_decl;
ctx->cb.eh_lp_nr = 0;
ctx->cb.transform_call_graph_edges = CB_CGE_MOVE;
+ ctx->cb.adjust_array_error_bounds = true;
+ ctx->cb.dont_remap_vla_if_no_change = true;
ctx->depth = 1;
}
if (is_task_ctx (ctx))
finalize_task_copyfn (as_a <gomp_task *> (ctx->stmt));
+ if (ctx->task_reduction_map)
+ {
+ ctx->task_reductions.release ();
+ delete ctx->task_reduction_map;
+ }
+
+ delete ctx->lastprivate_conditional_map;
+ delete ctx->allocate_map;
+
XDELETE (ctx);
}
}
/* Instantiate decls as necessary in CTX to satisfy the data sharing
- specified by CLAUSES. If BASE_POINTERS_RESTRICT, install var field with
- restrict. */
+ specified by CLAUSES. */
static void
-scan_sharing_clauses (tree clauses, omp_context *ctx,
- bool base_pointers_restrict = false)
+scan_sharing_clauses (tree clauses, omp_context *ctx)
{
tree c, decl;
bool scan_array_reductions = false;
+ for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_ALLOCATE
+ && (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c) == NULL_TREE
+ /* omp_default_mem_alloc is 1 */
+ || !integer_onep (OMP_CLAUSE_ALLOCATE_ALLOCATOR (c))))
+ {
+ if (ctx->allocate_map == NULL)
+ ctx->allocate_map = new hash_map<tree, tree>;
+ ctx->allocate_map->put (OMP_CLAUSE_DECL (c),
+ OMP_CLAUSE_ALLOCATE_ALLOCATOR (c)
+ ? OMP_CLAUSE_ALLOCATE_ALLOCATOR (c)
+ : integer_zero_node);
+ }
+
for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
{
bool by_ref;
case OMP_CLAUSE_SHARED:
decl = OMP_CLAUSE_DECL (c);
- /* Ignore shared directives in teams construct. */
- if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
+ if (ctx->allocate_map && ctx->allocate_map->get (decl))
+ ctx->allocate_map->remove (decl);
+ /* Ignore shared directives in teams construct inside of
+ target construct. */
+ if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
+ && !is_host_teams_ctx (ctx))
{
/* Global variables don't need to be copied,
the receiver side will use them directly. */
goto do_private;
case OMP_CLAUSE_REDUCTION:
+ /* Collect 'reduction' clauses on OpenACC compute construct. */
+ if (is_gimple_omp_oacc (ctx->stmt)
+ && is_gimple_omp_offloaded (ctx->stmt))
+ {
+ /* No 'reduction' clauses on OpenACC 'kernels'. */
+ gcc_checking_assert (!is_oacc_kernels (ctx));
+ /* Likewise, on OpenACC 'kernels' decomposed parts. */
+ gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
+
+ ctx->local_reduction_clauses
+ = tree_cons (NULL, c, ctx->local_reduction_clauses);
+ }
+ /* FALLTHRU */
+
+ case OMP_CLAUSE_IN_REDUCTION:
decl = OMP_CLAUSE_DECL (c);
- if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
- && TREE_CODE (decl) == MEM_REF)
+ if (ctx->allocate_map
+ && ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
+ && (OMP_CLAUSE_REDUCTION_INSCAN (c)
+ || OMP_CLAUSE_REDUCTION_TASK (c)))
+ || is_task_ctx (ctx)))
+ {
+ /* For now. */
+ if (ctx->allocate_map->get (decl))
+ ctx->allocate_map->remove (decl);
+ }
+ if (TREE_CODE (decl) == MEM_REF)
{
tree t = TREE_OPERAND (decl, 0);
if (TREE_CODE (t) == POINTER_PLUS_EXPR)
t = TREE_OPERAND (t, 0);
install_var_local (t, ctx);
if (is_taskreg_ctx (ctx)
- && !is_global_var (maybe_lookup_decl_in_outer_ctx (t, ctx))
- && !is_variable_sized (t))
+ && (!is_global_var (maybe_lookup_decl_in_outer_ctx (t, ctx))
+ || (is_task_ctx (ctx)
+ && (TREE_CODE (TREE_TYPE (t)) == POINTER_TYPE
+ || (TREE_CODE (TREE_TYPE (t)) == REFERENCE_TYPE
+ && (TREE_CODE (TREE_TYPE (TREE_TYPE (t)))
+ == POINTER_TYPE)))))
+ && !is_variable_sized (t)
+ && (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
+ || (!OMP_CLAUSE_REDUCTION_TASK (c)
+ && !is_task_ctx (ctx))))
+ {
+ by_ref = use_pointer_for_field (t, NULL);
+ if (is_task_ctx (ctx)
+ && TREE_CODE (TREE_TYPE (t)) == REFERENCE_TYPE
+ && TREE_CODE (TREE_TYPE (TREE_TYPE (t))) == POINTER_TYPE)
+ {
+ install_var_field (t, false, 1, ctx);
+ install_var_field (t, by_ref, 2, ctx);
+ }
+ else
+ install_var_field (t, by_ref, 3, ctx);
+ }
+ break;
+ }
+ if (is_task_ctx (ctx)
+ || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
+ && OMP_CLAUSE_REDUCTION_TASK (c)
+ && is_parallel_ctx (ctx)))
+ {
+ /* Global variables don't need to be copied,
+ the receiver side will use them directly. */
+ if (!is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
{
- by_ref = use_pointer_for_field (t, ctx);
- install_var_field (t, by_ref, 3, ctx);
+ by_ref = use_pointer_for_field (decl, ctx);
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION)
+ install_var_field (decl, by_ref, 3, ctx);
}
+ install_var_local (decl, ctx);
+ break;
+ }
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
+ && OMP_CLAUSE_REDUCTION_TASK (c))
+ {
+ install_var_local (decl, ctx);
break;
}
goto do_private;
if (is_variable_sized (decl))
{
if (is_task_ctx (ctx))
- install_var_field (decl, false, 1, ctx);
+ {
+ if (ctx->allocate_map
+ && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE)
+ {
+ /* For now. */
+ if (ctx->allocate_map->get (decl))
+ ctx->allocate_map->remove (decl);
+ }
+ install_var_field (decl, false, 1, ctx);
+ }
break;
}
else if (is_taskreg_ctx (ctx))
if (is_task_ctx (ctx)
&& (global || by_ref || omp_is_reference (decl)))
{
- install_var_field (decl, false, 1, ctx);
+ if (ctx->allocate_map
+ && ctx->allocate_map->get (decl))
+ install_var_field (decl, by_ref, 32 | 1, ctx);
+ else
+ install_var_field (decl, false, 1, ctx);
if (!global)
install_var_field (decl, by_ref, 2, ctx);
}
break;
case OMP_CLAUSE_USE_DEVICE_PTR:
+ case OMP_CLAUSE_USE_DEVICE_ADDR:
decl = OMP_CLAUSE_DECL (c);
- if (TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
- install_var_field (decl, true, 3, ctx);
+
+ /* Fortran array descriptors. */
+ if (lang_hooks.decls.omp_array_data (decl, true))
+ install_var_field (decl, false, 19, ctx);
+ else if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
+ && !omp_is_reference (decl)
+ && !omp_is_allocatable_or_ptr (decl))
+ || TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
+ install_var_field (decl, true, 11, ctx);
else
- install_var_field (decl, false, 3, ctx);
+ install_var_field (decl, false, 11, ctx);
if (DECL_SIZE (decl)
&& TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST)
{
goto do_private;
case OMP_CLAUSE__LOOPTEMP_:
+ case OMP_CLAUSE__REDUCTEMP_:
gcc_assert (is_taskreg_ctx (ctx));
decl = OMP_CLAUSE_DECL (c);
install_var_field (decl, false, 3, ctx);
case OMP_CLAUSE_NUM_GANGS:
case OMP_CLAUSE_NUM_WORKERS:
case OMP_CLAUSE_VECTOR_LENGTH:
+ case OMP_CLAUSE_DETACH:
if (ctx->outer)
scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer);
break;
/* Global variables with "omp declare target" attribute
don't need to be copied, the receiver side will use them
directly. However, global variables with "omp declare target link"
- attribute need to be copied. */
+ attribute need to be copied. Or when ALWAYS modifier is used. */
if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
&& DECL_P (decl)
&& ((OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_FIRSTPRIVATE_POINTER
&& (OMP_CLAUSE_MAP_KIND (c)
- != GOMP_MAP_FIRSTPRIVATE_REFERENCE))
+ != GOMP_MAP_FIRSTPRIVATE_REFERENCE)
+ && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ATTACH
+ && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_DETACH)
|| TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
+ && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_TO
+ && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_FROM
+ && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_ALWAYS_TOFROM
+ && OMP_CLAUSE_MAP_KIND (c) != GOMP_MAP_TO_PSET
&& is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
&& varpool_node::get_create (decl)->offloadable
&& !lookup_attribute ("omp declare target link",
&& !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c))
break;
}
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
+ && DECL_P (decl)
+ && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
+ || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
+ && is_omp_target (ctx->stmt))
+ {
+ /* If this is an offloaded region, an attach operation should
+ only exist when the pointer variable is mapped in a prior
+ clause. */
+ if (is_gimple_omp_offloaded (ctx->stmt))
+ gcc_assert
+ (maybe_lookup_decl (decl, ctx)
+ || (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
+ && lookup_attribute ("omp declare target",
+ DECL_ATTRIBUTES (decl))));
+
+ /* By itself, attach/detach is generated as part of pointer
+ variable mapping and should not create new variables in the
+ offloaded region, however sender refs for it must be created
+ for its address to be passed to the runtime. */
+ tree field
+ = build_decl (OMP_CLAUSE_LOCATION (c),
+ FIELD_DECL, NULL_TREE, ptr_type_node);
+ SET_DECL_ALIGN (field, TYPE_ALIGN (ptr_type_node));
+ insert_field_into_struct (ctx->record_type, field);
+ /* To not clash with a map of the pointer variable itself,
+ attach/detach maps have their field looked up by the *clause*
+ tree expression, not the decl. */
+ gcc_assert (!splay_tree_lookup (ctx->field_map,
+ (splay_tree_key) c));
+ splay_tree_insert (ctx->field_map, (splay_tree_key) c,
+ (splay_tree_value) field);
+ break;
+ }
if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
&& (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_FIRSTPRIVATE_POINTER
|| (OMP_CLAUSE_MAP_KIND (c)
&& TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE)
install_var_field (decl, true, 7, ctx);
else
- install_var_field (decl, true, 3, ctx,
- base_pointers_restrict);
+ install_var_field (decl, true, 3, ctx);
if (is_gimple_omp_offloaded (ctx->stmt)
&& !OMP_CLAUSE_MAP_IN_REDUCTION (c))
install_var_local (decl, ctx);
}
break;
- case OMP_CLAUSE__GRIDDIM_:
- if (ctx->outer)
- {
- scan_omp_op (&OMP_CLAUSE__GRIDDIM__SIZE (c), ctx->outer);
- scan_omp_op (&OMP_CLAUSE__GRIDDIM__GROUP (c), ctx->outer);
- }
+ case OMP_CLAUSE_ORDER:
+ ctx->order_concurrent = true;
+ break;
+
+ case OMP_CLAUSE_BIND:
+ ctx->loop_p = true;
break;
case OMP_CLAUSE_NOWAIT:
case OMP_CLAUSE_TILE:
case OMP_CLAUSE__SIMT_:
case OMP_CLAUSE_DEFAULT:
+ case OMP_CLAUSE_NONTEMPORAL:
+ case OMP_CLAUSE_IF_PRESENT:
+ case OMP_CLAUSE_FINALIZE:
+ case OMP_CLAUSE_TASK_REDUCTION:
+ case OMP_CLAUSE_ALLOCATE:
break;
case OMP_CLAUSE_ALIGNED:
install_var_local (decl, ctx);
break;
+ case OMP_CLAUSE__CONDTEMP_:
+ decl = OMP_CLAUSE_DECL (c);
+ if (is_parallel_ctx (ctx))
+ {
+ install_var_field (decl, false, 3, ctx);
+ install_var_local (decl, ctx);
+ }
+ else if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
+ && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
+ && !OMP_CLAUSE__CONDTEMP__ITER (c))
+ install_var_local (decl, ctx);
+ break;
+
case OMP_CLAUSE__CACHE_:
default:
gcc_unreachable ();
break;
case OMP_CLAUSE_REDUCTION:
+ case OMP_CLAUSE_IN_REDUCTION:
decl = OMP_CLAUSE_DECL (c);
if (TREE_CODE (decl) != MEM_REF)
{
scan_array_reductions = true;
break;
+ case OMP_CLAUSE_TASK_REDUCTION:
+ if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
+ scan_array_reductions = true;
+ break;
+
case OMP_CLAUSE_SHARED:
- /* Ignore shared directives in teams construct. */
- if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
+ /* Ignore shared directives in teams construct inside of
+ target construct. */
+ if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
+ && !is_host_teams_ctx (ctx))
break;
decl = OMP_CLAUSE_DECL (c);
if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)))
&& is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))
&& varpool_node::get_create (decl)->offloadable)
break;
+ if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
+ || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
+ && is_omp_target (ctx->stmt)
+ && !is_gimple_omp_offloaded (ctx->stmt))
+ break;
if (DECL_P (decl))
{
if ((OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER
case OMP_CLAUSE_SIMDLEN:
case OMP_CLAUSE_ALIGNED:
case OMP_CLAUSE_DEPEND:
+ case OMP_CLAUSE_DETACH:
+ case OMP_CLAUSE_ALLOCATE:
case OMP_CLAUSE__LOOPTEMP_:
+ case OMP_CLAUSE__REDUCTEMP_:
case OMP_CLAUSE_TO:
case OMP_CLAUSE_FROM:
case OMP_CLAUSE_PRIORITY:
case OMP_CLAUSE_SIMD:
case OMP_CLAUSE_NOGROUP:
case OMP_CLAUSE_DEFAULTMAP:
+ case OMP_CLAUSE_ORDER:
+ case OMP_CLAUSE_BIND:
case OMP_CLAUSE_USE_DEVICE_PTR:
+ case OMP_CLAUSE_USE_DEVICE_ADDR:
+ case OMP_CLAUSE_NONTEMPORAL:
case OMP_CLAUSE_ASYNC:
case OMP_CLAUSE_WAIT:
case OMP_CLAUSE_NUM_GANGS:
case OMP_CLAUSE_AUTO:
case OMP_CLAUSE_SEQ:
case OMP_CLAUSE_TILE:
- case OMP_CLAUSE__GRIDDIM_:
case OMP_CLAUSE__SIMT_:
+ case OMP_CLAUSE_IF_PRESENT:
+ case OMP_CLAUSE_FINALIZE:
+ case OMP_CLAUSE__CONDTEMP_:
break;
case OMP_CLAUSE__CACHE_:
if (scan_array_reductions)
{
for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
- if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
+ if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
+ || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION
+ || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
&& OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
{
scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), ctx);
static tree
create_omp_child_function_name (bool task_copy)
{
- return clone_function_name (current_function_decl,
- task_copy ? "_omp_cpyfn" : "_omp_fn");
+ return clone_function_name_numbered (current_function_decl,
+ task_copy ? "_omp_cpyfn" : "_omp_fn");
}
/* Return true if CTX may belong to offloaded code: either if current function
return NULL;
}
-/* Add _LOOPTEMP_ clauses on OpenMP parallel or task. */
+/* Add _LOOPTEMP_/_REDUCTEMP_ clauses on OpenMP parallel or task. */
static void
add_taskreg_looptemp_clauses (enum gf_mask msk, gimple *stmt,
GIMPLE_OMP_FOR, add one more temporaries for the total number
of iterations (product of count1 ... countN-1). */
if (omp_find_clause (gimple_omp_for_clauses (for_stmt),
- OMP_CLAUSE_LASTPRIVATE))
- count++;
- else if (msk == GF_OMP_FOR_KIND_FOR
- && omp_find_clause (gimple_omp_parallel_clauses (stmt),
- OMP_CLAUSE_LASTPRIVATE))
- count++;
+ OMP_CLAUSE_LASTPRIVATE)
+ || (msk == GF_OMP_FOR_KIND_FOR
+ && omp_find_clause (gimple_omp_parallel_clauses (stmt),
+ OMP_CLAUSE_LASTPRIVATE)))
+ {
+ tree temp = create_tmp_var (type);
+ tree c = build_omp_clause (UNKNOWN_LOCATION,
+ OMP_CLAUSE__LOOPTEMP_);
+ insert_decl_map (&outer_ctx->cb, temp, temp);
+ OMP_CLAUSE_DECL (c) = temp;
+ OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
+ gimple_omp_taskreg_set_clauses (stmt, c);
+ }
+ if (fd.non_rect
+ && fd.last_nonrect == fd.first_nonrect + 1)
+ if (tree v = gimple_omp_for_index (for_stmt, fd.last_nonrect))
+ if (!TYPE_UNSIGNED (TREE_TYPE (v)))
+ {
+ v = gimple_omp_for_index (for_stmt, fd.first_nonrect);
+ tree type2 = TREE_TYPE (v);
+ count++;
+ for (i = 0; i < 3; i++)
+ {
+ tree temp = create_tmp_var (type2);
+ tree c = build_omp_clause (UNKNOWN_LOCATION,
+ OMP_CLAUSE__LOOPTEMP_);
+ insert_decl_map (&outer_ctx->cb, temp, temp);
+ OMP_CLAUSE_DECL (c) = temp;
+ OMP_CLAUSE_CHAIN (c) = gimple_omp_taskreg_clauses (stmt);
+ gimple_omp_taskreg_set_clauses (stmt, c);
+ }
+ }
}
for (i = 0; i < count; i++)
{
gimple_omp_taskreg_set_clauses (stmt, c);
}
}
+ if (msk == GF_OMP_FOR_KIND_TASKLOOP
+ && omp_find_clause (gimple_omp_task_clauses (stmt),
+ OMP_CLAUSE_REDUCTION))
+ {
+ tree type = build_pointer_type (pointer_sized_int_node);
+ tree temp = create_tmp_var (type);
+ tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
+ insert_decl_map (&outer_ctx->cb, temp, temp);
+ OMP_CLAUSE_DECL (c) = temp;
+ OMP_CLAUSE_CHAIN (c) = gimple_omp_task_clauses (stmt);
+ gimple_omp_task_set_clauses (stmt, c);
+ }
}
/* Scan an OpenMP parallel directive. */
if (gimple_omp_parallel_combined_p (stmt))
add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_FOR, stmt, outer_ctx);
+ for (tree c = omp_find_clause (gimple_omp_parallel_clauses (stmt),
+ OMP_CLAUSE_REDUCTION);
+ c; c = omp_find_clause (OMP_CLAUSE_CHAIN (c), OMP_CLAUSE_REDUCTION))
+ if (OMP_CLAUSE_REDUCTION_TASK (c))
+ {
+ tree type = build_pointer_type (pointer_sized_int_node);
+ tree temp = create_tmp_var (type);
+ tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
+ if (outer_ctx)
+ insert_decl_map (&outer_ctx->cb, temp, temp);
+ OMP_CLAUSE_DECL (c) = temp;
+ OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (stmt);
+ gimple_omp_parallel_set_clauses (stmt, c);
+ break;
+ }
+ else if (OMP_CLAUSE_CHAIN (c) == NULL_TREE)
+ break;
ctx = new_omp_context (stmt, outer_ctx);
taskreg_contexts.safe_push (ctx);
DECL_NAMELESS (name) = 1;
TYPE_NAME (ctx->record_type) = name;
TYPE_ARTIFICIAL (ctx->record_type) = 1;
- if (!gimple_omp_parallel_grid_phony (stmt))
- {
- create_omp_child_function (ctx, false);
- gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn);
- }
+ create_omp_child_function (ctx, false);
+ gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn);
scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx);
scan_omp (gimple_omp_body_ptr (stmt), ctx);
/* Ignore task directives with empty bodies, unless they have depend
clause. */
if (optimize > 0
+ && gimple_omp_body (stmt)
&& empty_body_p (gimple_omp_body (stmt))
&& !omp_find_clause (gimple_omp_task_clauses (stmt), OMP_CLAUSE_DEPEND))
{
add_taskreg_looptemp_clauses (GF_OMP_FOR_KIND_TASKLOOP, stmt, outer_ctx);
ctx = new_omp_context (stmt, outer_ctx);
+
+ if (gimple_omp_task_taskwait_p (stmt))
+ {
+ scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx);
+ return;
+ }
+
taskreg_contexts.safe_push (ctx);
if (taskreg_nesting_level > 1)
ctx->is_nested = true;
return;
/* If any task_shared_vars were needed, verify all
- OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK}
+ OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK,TEAMS}
statements if use_pointer_for_field hasn't changed
because of that. If it did, update field types now. */
if (task_shared_vars)
}
if (gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL)
+ {
+ tree clauses = gimple_omp_parallel_clauses (ctx->stmt);
+ tree c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
+ if (c)
+ {
+ /* Move the _reductemp_ clause first. GOMP_parallel_reductions
+ expects to find it at the start of data. */
+ tree f = lookup_field (OMP_CLAUSE_DECL (c), ctx);
+ tree *p = &TYPE_FIELDS (ctx->record_type);
+ while (*p)
+ if (*p == f)
+ {
+ *p = DECL_CHAIN (*p);
+ break;
+ }
+ else
+ p = &DECL_CHAIN (*p);
+ DECL_CHAIN (f) = TYPE_FIELDS (ctx->record_type);
+ TYPE_FIELDS (ctx->record_type) = f;
+ }
+ layout_type (ctx->record_type);
+ fixup_child_record_type (ctx);
+ }
+ else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
{
layout_type (ctx->record_type);
fixup_child_record_type (ctx);
{
location_t loc = gimple_location (ctx->stmt);
tree *p, vla_fields = NULL_TREE, *q = &vla_fields;
+ tree detach_clause
+ = omp_find_clause (gimple_omp_task_clauses (ctx->stmt),
+ OMP_CLAUSE_DETACH);
/* Move VLA fields to the end. */
p = &TYPE_FIELDS (ctx->record_type);
while (*p)
/* Move fields corresponding to first and second _looptemp_
clause first. There are filled by GOMP_taskloop
and thus need to be in specific positions. */
- tree c1 = gimple_omp_task_clauses (ctx->stmt);
- c1 = omp_find_clause (c1, OMP_CLAUSE__LOOPTEMP_);
+ tree clauses = gimple_omp_task_clauses (ctx->stmt);
+ tree c1 = omp_find_clause (clauses, OMP_CLAUSE__LOOPTEMP_);
tree c2 = omp_find_clause (OMP_CLAUSE_CHAIN (c1),
OMP_CLAUSE__LOOPTEMP_);
+ tree c3 = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
tree f1 = lookup_field (OMP_CLAUSE_DECL (c1), ctx);
tree f2 = lookup_field (OMP_CLAUSE_DECL (c2), ctx);
+ tree f3 = c3 ? lookup_field (OMP_CLAUSE_DECL (c3), ctx) : NULL_TREE;
p = &TYPE_FIELDS (ctx->record_type);
while (*p)
- if (*p == f1 || *p == f2)
+ if (*p == f1 || *p == f2 || *p == f3)
*p = DECL_CHAIN (*p);
else
p = &DECL_CHAIN (*p);
DECL_CHAIN (f1) = f2;
- DECL_CHAIN (f2) = TYPE_FIELDS (ctx->record_type);
+ if (c3)
+ {
+ DECL_CHAIN (f2) = f3;
+ DECL_CHAIN (f3) = TYPE_FIELDS (ctx->record_type);
+ }
+ else
+ DECL_CHAIN (f2) = TYPE_FIELDS (ctx->record_type);
TYPE_FIELDS (ctx->record_type) = f1;
if (ctx->srecord_type)
{
f1 = lookup_sfield (OMP_CLAUSE_DECL (c1), ctx);
f2 = lookup_sfield (OMP_CLAUSE_DECL (c2), ctx);
+ if (c3)
+ f3 = lookup_sfield (OMP_CLAUSE_DECL (c3), ctx);
p = &TYPE_FIELDS (ctx->srecord_type);
while (*p)
- if (*p == f1 || *p == f2)
+ if (*p == f1 || *p == f2 || *p == f3)
*p = DECL_CHAIN (*p);
else
p = &DECL_CHAIN (*p);
DECL_CHAIN (f1) = f2;
DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type);
+ if (c3)
+ {
+ DECL_CHAIN (f2) = f3;
+ DECL_CHAIN (f3) = TYPE_FIELDS (ctx->srecord_type);
+ }
+ else
+ DECL_CHAIN (f2) = TYPE_FIELDS (ctx->srecord_type);
TYPE_FIELDS (ctx->srecord_type) = f1;
}
}
+ if (detach_clause)
+ {
+ tree c, field;
+
+ /* Look for a firstprivate clause with the detach event handle. */
+ for (c = gimple_omp_taskreg_clauses (ctx->stmt);
+ c; c = OMP_CLAUSE_CHAIN (c))
+ {
+ if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FIRSTPRIVATE)
+ continue;
+ if (maybe_lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c), ctx)
+ == OMP_CLAUSE_DECL (detach_clause))
+ break;
+ }
+
+ gcc_assert (c);
+ field = lookup_field (OMP_CLAUSE_DECL (c), ctx);
+
+ /* Move field corresponding to the detach clause first.
+ This is filled by GOMP_task and needs to be in a
+ specific position. */
+ p = &TYPE_FIELDS (ctx->record_type);
+ while (*p)
+ if (*p == field)
+ *p = DECL_CHAIN (*p);
+ else
+ p = &DECL_CHAIN (*p);
+ DECL_CHAIN (field) = TYPE_FIELDS (ctx->record_type);
+ TYPE_FIELDS (ctx->record_type) = field;
+ if (ctx->srecord_type)
+ {
+ field = lookup_sfield (OMP_CLAUSE_DECL (detach_clause), ctx);
+ p = &TYPE_FIELDS (ctx->srecord_type);
+ while (*p)
+ if (*p == field)
+ *p = DECL_CHAIN (*p);
+ else
+ p = &DECL_CHAIN (*p);
+ DECL_CHAIN (field) = TYPE_FIELDS (ctx->srecord_type);
+ TYPE_FIELDS (ctx->srecord_type) = field;
+ }
+ }
layout_type (ctx->record_type);
fixup_child_record_type (ctx);
if (ctx->srecord_type)
return ctx;
}
-/* Return true if ctx is part of an oacc kernels region. */
+/* Return whether CTX's parent compute construct is an OpenACC 'kernels'
+ construct.
+ (This doesn't include OpenACC 'kernels' decomposed parts.) */
static bool
ctx_in_oacc_kernels_region (omp_context *ctx)
return false;
}
-/* Check the parallelism clauses inside a kernels regions.
+/* Check the parallelism clauses inside a OpenACC 'kernels' region.
+ (This doesn't include OpenACC 'kernels' decomposed parts.)
Until kernels handling moves to use the same loop indirection
scheme as parallel, we need to do this checking early. */
{
omp_context *tgt = enclosing_target_ctx (outer_ctx);
- if (!tgt || is_oacc_parallel (tgt))
+ if (!(tgt && is_oacc_kernels (tgt)))
for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
{
- char const *check = NULL;
-
+ tree c_op0;
switch (OMP_CLAUSE_CODE (c))
{
case OMP_CLAUSE_GANG:
- check = "gang";
+ c_op0 = OMP_CLAUSE_GANG_EXPR (c);
break;
case OMP_CLAUSE_WORKER:
- check = "worker";
+ c_op0 = OMP_CLAUSE_WORKER_EXPR (c);
break;
case OMP_CLAUSE_VECTOR:
- check = "vector";
+ c_op0 = OMP_CLAUSE_VECTOR_EXPR (c);
break;
default:
- break;
+ continue;
}
- if (check && OMP_CLAUSE_OPERAND (c, 0))
- error_at (gimple_location (stmt),
- "argument not permitted on %qs clause in"
- " OpenACC %<parallel%>", check);
+ if (c_op0)
+ {
+ /* By construction, this is impossible for OpenACC 'kernels'
+ decomposed parts. */
+ gcc_assert (!(tgt && is_oacc_kernels_decomposed_part (tgt)));
+
+ error_at (OMP_CLAUSE_LOCATION (c),
+ "argument not permitted on %qs clause",
+ omp_clause_code_name[OMP_CLAUSE_CODE (c)]);
+ if (tgt)
+ inform (gimple_location (tgt->stmt),
+ "enclosing parent compute construct");
+ else if (oacc_get_fn_attrib (current_function_decl))
+ inform (DECL_SOURCE_LOCATION (current_function_decl),
+ "enclosing routine");
+ else
+ gcc_unreachable ();
+ }
}
+ if (tgt && is_oacc_kernels (tgt))
+ check_oacc_kernel_gwv (stmt, ctx);
+
+ /* Collect all variables named in reductions on this loop. Ensure
+ that, if this loop has a reduction on some variable v, and there is
+ a reduction on v somewhere in an outer context, then there is a
+ reduction on v on all intervening loops as well. */
+ tree local_reduction_clauses = NULL;
+ for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
+ {
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
+ local_reduction_clauses
+ = tree_cons (NULL, c, local_reduction_clauses);
+ }
+ if (ctx->outer_reduction_clauses == NULL && ctx->outer != NULL)
+ ctx->outer_reduction_clauses
+ = chainon (unshare_expr (ctx->outer->local_reduction_clauses),
+ ctx->outer->outer_reduction_clauses);
+ tree outer_reduction_clauses = ctx->outer_reduction_clauses;
+ tree local_iter = local_reduction_clauses;
+ for (; local_iter; local_iter = TREE_CHAIN (local_iter))
+ {
+ tree local_clause = TREE_VALUE (local_iter);
+ tree local_var = OMP_CLAUSE_DECL (local_clause);
+ tree_code local_op = OMP_CLAUSE_REDUCTION_CODE (local_clause);
+ bool have_outer_reduction = false;
+ tree ctx_iter = outer_reduction_clauses;
+ for (; ctx_iter; ctx_iter = TREE_CHAIN (ctx_iter))
+ {
+ tree outer_clause = TREE_VALUE (ctx_iter);
+ tree outer_var = OMP_CLAUSE_DECL (outer_clause);
+ tree_code outer_op = OMP_CLAUSE_REDUCTION_CODE (outer_clause);
+ if (outer_var == local_var && outer_op != local_op)
+ {
+ warning_at (OMP_CLAUSE_LOCATION (local_clause), 0,
+ "conflicting reduction operations for %qE",
+ local_var);
+ inform (OMP_CLAUSE_LOCATION (outer_clause),
+ "location of the previous reduction for %qE",
+ outer_var);
+ }
+ if (outer_var == local_var)
+ {
+ have_outer_reduction = true;
+ break;
+ }
+ }
+ if (have_outer_reduction)
+ {
+ /* There is a reduction on outer_var both on this loop and on
+ some enclosing loop. Walk up the context tree until such a
+ loop with a reduction on outer_var is found, and complain
+ about all intervening loops that do not have such a
+ reduction. */
+ struct omp_context *curr_loop = ctx->outer;
+ bool found = false;
+ while (curr_loop != NULL)
+ {
+ tree curr_iter = curr_loop->local_reduction_clauses;
+ for (; curr_iter; curr_iter = TREE_CHAIN (curr_iter))
+ {
+ tree curr_clause = TREE_VALUE (curr_iter);
+ tree curr_var = OMP_CLAUSE_DECL (curr_clause);
+ if (curr_var == local_var)
+ {
+ found = true;
+ break;
+ }
+ }
+ if (!found)
+ warning_at (gimple_location (curr_loop->stmt), 0,
+ "nested loop in reduction needs "
+ "reduction clause for %qE",
+ local_var);
+ else
+ break;
+ curr_loop = curr_loop->outer;
+ }
+ }
+ }
+ ctx->local_reduction_clauses = local_reduction_clauses;
+ ctx->outer_reduction_clauses
+ = chainon (unshare_expr (ctx->local_reduction_clauses),
+ ctx->outer_reduction_clauses);
+
if (tgt && is_oacc_kernels (tgt))
{
- /* Strip out reductions, as they are not handled yet. */
+ /* Strip out reductions, as they are not handled yet. */
tree *prev_ptr = &clauses;
while (tree probe = *prev_ptr)
{
tree *next_ptr = &OMP_CLAUSE_CHAIN (probe);
-
+
if (OMP_CLAUSE_CODE (probe) == OMP_CLAUSE_REDUCTION)
*prev_ptr = *next_ptr;
else
}
gimple_omp_for_set_clauses (stmt, clauses);
- check_oacc_kernel_gwv (stmt, ctx);
}
}
scan_omp_for (stmt, outer_ctx)->simt_stmt = new_stmt;
}
+static tree omp_find_scan (gimple_stmt_iterator *, bool *,
+ struct walk_stmt_info *);
+static omp_context *maybe_lookup_ctx (gimple *);
+
+/* Duplicate #pragma omp simd, one for the scan input phase loop and one
+ for scan phase loop. */
+
+static void
+scan_omp_simd_scan (gimple_stmt_iterator *gsi, gomp_for *stmt,
+ omp_context *outer_ctx)
+{
+ /* The only change between inclusive and exclusive scan will be
+ within the first simd loop, so just use inclusive in the
+ worksharing loop. */
+ outer_ctx->scan_inclusive = true;
+ tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_INCLUSIVE);
+ OMP_CLAUSE_DECL (c) = integer_zero_node;
+
+ gomp_scan *input_stmt = gimple_build_omp_scan (NULL, NULL_TREE);
+ gomp_scan *scan_stmt = gimple_build_omp_scan (NULL, c);
+ gsi_replace (gsi, input_stmt, false);
+ gimple_seq input_body = NULL;
+ gimple_seq_add_stmt (&input_body, stmt);
+ gsi_insert_after (gsi, scan_stmt, GSI_NEW_STMT);
+
+ gimple_stmt_iterator input1_gsi = gsi_none ();
+ struct walk_stmt_info wi;
+ memset (&wi, 0, sizeof (wi));
+ wi.val_only = true;
+ wi.info = (void *) &input1_gsi;
+ walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), omp_find_scan, NULL, &wi);
+ gcc_assert (!gsi_end_p (input1_gsi));
+
+ gimple *input_stmt1 = gsi_stmt (input1_gsi);
+ gsi_next (&input1_gsi);
+ gimple *scan_stmt1 = gsi_stmt (input1_gsi);
+ gcc_assert (scan_stmt1 && gimple_code (scan_stmt1) == GIMPLE_OMP_SCAN);
+ c = gimple_omp_scan_clauses (as_a <gomp_scan *> (scan_stmt1));
+ if (c && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_EXCLUSIVE)
+ std::swap (input_stmt1, scan_stmt1);
+
+ gimple_seq input_body1 = gimple_omp_body (input_stmt1);
+ gimple_omp_set_body (input_stmt1, NULL);
+
+ gimple_seq scan_body = copy_gimple_seq_and_replace_locals (stmt);
+ gomp_for *new_stmt = as_a <gomp_for *> (scan_body);
+
+ gimple_omp_set_body (input_stmt1, input_body1);
+ gimple_omp_set_body (scan_stmt1, NULL);
+
+ gimple_stmt_iterator input2_gsi = gsi_none ();
+ memset (&wi, 0, sizeof (wi));
+ wi.val_only = true;
+ wi.info = (void *) &input2_gsi;
+ walk_gimple_seq_mod (gimple_omp_body_ptr (new_stmt), omp_find_scan,
+ NULL, &wi);
+ gcc_assert (!gsi_end_p (input2_gsi));
+
+ gimple *input_stmt2 = gsi_stmt (input2_gsi);
+ gsi_next (&input2_gsi);
+ gimple *scan_stmt2 = gsi_stmt (input2_gsi);
+ gcc_assert (scan_stmt2 && gimple_code (scan_stmt2) == GIMPLE_OMP_SCAN);
+ if (c && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_EXCLUSIVE)
+ std::swap (input_stmt2, scan_stmt2);
+
+ gimple_omp_set_body (input_stmt2, NULL);
+
+ gimple_omp_set_body (input_stmt, input_body);
+ gimple_omp_set_body (scan_stmt, scan_body);
+
+ omp_context *ctx = new_omp_context (input_stmt, outer_ctx);
+ scan_omp (gimple_omp_body_ptr (input_stmt), ctx);
+
+ ctx = new_omp_context (scan_stmt, outer_ctx);
+ scan_omp (gimple_omp_body_ptr (scan_stmt), ctx);
+
+ maybe_lookup_ctx (new_stmt)->for_simd_scan_phase = true;
+}
+
/* Scan an OpenMP sections directive. */
static void
layout_type (ctx->record_type);
}
-/* Return true if the CLAUSES of an omp target guarantee that the base pointers
- used in the corresponding offloaded function are restrict. */
+/* Scan a GIMPLE_OMP_TARGET. */
-static bool
-omp_target_base_pointers_restrict_p (tree clauses)
+static void
+scan_omp_target (gomp_target *stmt, omp_context *outer_ctx)
{
- /* The analysis relies on the GOMP_MAP_FORCE_* mapping kinds, which are only
- used by OpenACC. */
- if (flag_openacc == 0)
- return false;
+ omp_context *ctx;
+ tree name;
+ bool offloaded = is_gimple_omp_offloaded (stmt);
+ tree clauses = gimple_omp_target_clauses (stmt);
- /* I. Basic example:
+ ctx = new_omp_context (stmt, outer_ctx);
+ ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
+ ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
+ name = create_tmp_var_name (".omp_data_t");
+ name = build_decl (gimple_location (stmt),
+ TYPE_DECL, name, ctx->record_type);
+ DECL_ARTIFICIAL (name) = 1;
+ DECL_NAMELESS (name) = 1;
+ TYPE_NAME (ctx->record_type) = name;
+ TYPE_ARTIFICIAL (ctx->record_type) = 1;
- void foo (void)
- {
- unsigned int a[2], b[2];
-
- #pragma acc kernels \
- copyout (a) \
- copyout (b)
- {
- a[0] = 0;
- b[0] = 1;
- }
- }
-
- After gimplification, we have:
-
- #pragma omp target oacc_kernels \
- map(force_from:a [len: 8]) \
- map(force_from:b [len: 8])
- {
- a[0] = 0;
- b[0] = 1;
- }
-
- Because both mappings have the force prefix, we know that they will be
- allocated when calling the corresponding offloaded function, which means we
- can mark the base pointers for a and b in the offloaded function as
- restrict. */
-
- tree c;
- for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
- {
- if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP)
- return false;
-
- switch (OMP_CLAUSE_MAP_KIND (c))
- {
- case GOMP_MAP_FORCE_ALLOC:
- case GOMP_MAP_FORCE_TO:
- case GOMP_MAP_FORCE_FROM:
- case GOMP_MAP_FORCE_TOFROM:
- break;
- default:
- return false;
- }
- }
-
- return true;
-}
-
-/* Scan a GIMPLE_OMP_TARGET. */
-
-static void
-scan_omp_target (gomp_target *stmt, omp_context *outer_ctx)
-{
- omp_context *ctx;
- tree name;
- bool offloaded = is_gimple_omp_offloaded (stmt);
- tree clauses = gimple_omp_target_clauses (stmt);
-
- ctx = new_omp_context (stmt, outer_ctx);
- ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
- ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
- name = create_tmp_var_name (".omp_data_t");
- name = build_decl (gimple_location (stmt),
- TYPE_DECL, name, ctx->record_type);
- DECL_ARTIFICIAL (name) = 1;
- DECL_NAMELESS (name) = 1;
- TYPE_NAME (ctx->record_type) = name;
- TYPE_ARTIFICIAL (ctx->record_type) = 1;
-
- bool base_pointers_restrict = false;
if (offloaded)
{
create_omp_child_function (ctx, false);
gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn);
-
- base_pointers_restrict = omp_target_base_pointers_restrict_p (clauses);
- if (base_pointers_restrict
- && dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file,
- "Base pointers in offloaded function are restrict\n");
}
- scan_sharing_clauses (clauses, ctx, base_pointers_restrict);
+ scan_sharing_clauses (clauses, ctx);
scan_omp (gimple_omp_body_ptr (stmt), ctx);
if (TYPE_FIELDS (ctx->record_type) == NULL)
scan_omp_teams (gomp_teams *stmt, omp_context *outer_ctx)
{
omp_context *ctx = new_omp_context (stmt, outer_ctx);
+
+ if (!gimple_omp_teams_host (stmt))
+ {
+ scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
+ scan_omp (gimple_omp_body_ptr (stmt), ctx);
+ return;
+ }
+ taskreg_contexts.safe_push (ctx);
+ gcc_assert (taskreg_nesting_level == 1);
+ ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0);
+ ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE);
+ tree name = create_tmp_var_name (".omp_data_s");
+ name = build_decl (gimple_location (stmt),
+ TYPE_DECL, name, ctx->record_type);
+ DECL_ARTIFICIAL (name) = 1;
+ DECL_NAMELESS (name) = 1;
+ TYPE_NAME (ctx->record_type) = name;
+ TYPE_ARTIFICIAL (ctx->record_type) = 1;
+ create_omp_child_function (ctx, false);
+ gimple_omp_teams_set_child_fn (stmt, ctx->cb.dst_fn);
+
scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx);
scan_omp (gimple_omp_body_ptr (stmt), ctx);
+
+ if (TYPE_FIELDS (ctx->record_type) == NULL)
+ ctx->record_type = ctx->receiver_decl = NULL;
}
/* Check nesting restrictions. */
{
tree c;
- if (ctx && gimple_code (ctx->stmt) == GIMPLE_OMP_GRID_BODY)
- /* GRID_BODY is an artificial construct, nesting rules will be checked in
- the original copy of its contents. */
- return true;
-
/* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin)
inside an OpenACC CTX. */
if (!(is_gimple_omp (stmt)
if (ctx != NULL)
{
+ if (gimple_code (ctx->stmt) == GIMPLE_OMP_SCAN
+ && ctx->outer
+ && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
+ ctx = ctx->outer;
if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
- && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
+ && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
+ && !ctx->loop_p)
{
c = NULL_TREE;
+ if (ctx->order_concurrent
+ && (gimple_code (stmt) == GIMPLE_OMP_ORDERED
+ || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
+ || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE))
+ {
+ error_at (gimple_location (stmt),
+ "OpenMP constructs other than %<parallel%>, %<loop%>"
+ " or %<simd%> may not be nested inside a region with"
+ " the %<order(concurrent)%> clause");
+ return false;
+ }
if (gimple_code (stmt) == GIMPLE_OMP_ORDERED)
{
c = gimple_omp_ordered_clauses (as_a <gomp_ordered *> (stmt));
{
error_at (gimple_location (stmt),
"%<ordered simd threads%> must be closely "
- "nested inside of %<for simd%> region");
+ "nested inside of %<%s simd%> region",
+ lang_GNU_Fortran () ? "do" : "for");
return false;
}
return true;
}
}
+ else if (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD
+ || gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE
+ || gimple_code (stmt) == GIMPLE_OMP_SCAN)
+ return true;
+ else if (gimple_code (stmt) == GIMPLE_OMP_FOR
+ && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
+ return true;
error_at (gimple_location (stmt),
- "OpenMP constructs other than %<#pragma omp ordered simd%>"
- " may not be nested inside %<simd%> region");
+ "OpenMP constructs other than "
+ "%<ordered simd%>, %<simd%>, %<loop%> or %<atomic%> may "
+ "not be nested inside %<simd%> region");
return false;
}
else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
{
if ((gimple_code (stmt) != GIMPLE_OMP_FOR
- || ((gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_DISTRIBUTE)
- && (gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_GRID_LOOP)))
+ || (gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_DISTRIBUTE
+ && omp_find_clause (gimple_omp_for_clauses (stmt),
+ OMP_CLAUSE_BIND) == NULL_TREE))
&& gimple_code (stmt) != GIMPLE_OMP_PARALLEL)
{
error_at (gimple_location (stmt),
- "only %<distribute%> or %<parallel%> regions are "
- "allowed to be strictly nested inside %<teams%> "
- "region");
+ "only %<distribute%>, %<parallel%> or %<loop%> "
+ "regions are allowed to be strictly nested inside "
+ "%<teams%> region");
return false;
}
}
+ else if (ctx->order_concurrent
+ && gimple_code (stmt) != GIMPLE_OMP_PARALLEL
+ && (gimple_code (stmt) != GIMPLE_OMP_FOR
+ || gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_SIMD)
+ && gimple_code (stmt) != GIMPLE_OMP_SCAN)
+ {
+ if (ctx->loop_p)
+ error_at (gimple_location (stmt),
+ "OpenMP constructs other than %<parallel%>, %<loop%> or "
+ "%<simd%> may not be nested inside a %<loop%> region");
+ else
+ error_at (gimple_location (stmt),
+ "OpenMP constructs other than %<parallel%>, %<loop%> or "
+ "%<simd%> may not be nested inside a region with "
+ "the %<order(concurrent)%> clause");
+ return false;
+ }
}
switch (gimple_code (stmt))
{
case GIMPLE_OMP_FOR:
- if (gimple_omp_for_kind (stmt) & GF_OMP_FOR_SIMD)
+ if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_SIMD)
return true;
if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE)
{
/* We split taskloop into task and nested taskloop in it. */
if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_TASKLOOP)
return true;
+ /* For now, hope this will change and loop bind(parallel) will not
+ be allowed in lots of contexts. */
+ if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR
+ && omp_find_clause (gimple_omp_for_clauses (stmt), OMP_CLAUSE_BIND))
+ return true;
if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_OACC_LOOP)
{
bool ok = false;
{
case GF_OMP_TARGET_KIND_OACC_PARALLEL:
case GF_OMP_TARGET_KIND_OACC_KERNELS:
+ case GF_OMP_TARGET_KIND_OACC_SERIAL:
+ case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
+ case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
ok = true;
break;
const char *construct
= (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
== BUILT_IN_GOMP_CANCEL)
- ? "#pragma omp cancel"
- : "#pragma omp cancellation point";
+ ? "cancel"
+ : "cancellation point";
if (ctx == NULL)
{
error_at (gimple_location (stmt), "orphaned %qs construct",
{
case 1:
if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL)
- bad = "#pragma omp parallel";
+ bad = "parallel";
else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
== BUILT_IN_GOMP_CANCEL
&& !integer_zerop (gimple_call_arg (stmt, 1)))
case 2:
if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
|| gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR)
- bad = "#pragma omp for";
+ bad = "for";
else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
== BUILT_IN_GOMP_CANCEL
&& !integer_zerop (gimple_call_arg (stmt, 1)))
if (omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
OMP_CLAUSE_NOWAIT))
warning_at (gimple_location (stmt), 0,
- "%<#pragma omp cancel for%> inside "
+ "%<cancel for%> inside "
"%<nowait%> for construct");
if (omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
OMP_CLAUSE_ORDERED))
warning_at (gimple_location (stmt), 0,
- "%<#pragma omp cancel for%> inside "
+ "%<cancel for%> inside "
"%<ordered%> for construct");
}
kind = "for";
case 4:
if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS
&& gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION)
- bad = "#pragma omp sections";
+ bad = "sections";
else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt))
== BUILT_IN_GOMP_CANCEL
&& !integer_zerop (gimple_call_arg (stmt, 1)))
(ctx->stmt),
OMP_CLAUSE_NOWAIT))
warning_at (gimple_location (stmt), 0,
- "%<#pragma omp cancel sections%> inside "
+ "%<cancel sections%> inside "
"%<nowait%> sections construct");
}
else
(ctx->outer->stmt),
OMP_CLAUSE_NOWAIT))
warning_at (gimple_location (stmt), 0,
- "%<#pragma omp cancel sections%> inside "
+ "%<cancel sections%> inside "
"%<nowait%> sections construct");
}
}
kind = "sections";
break;
case 8:
- if (gimple_code (ctx->stmt) != GIMPLE_OMP_TASK)
- bad = "#pragma omp task";
+ if (!is_task_ctx (ctx)
+ && (!is_taskloop_ctx (ctx)
+ || ctx->outer == NULL
+ || !is_task_ctx (ctx->outer)))
+ bad = "task";
else
{
for (omp_context *octx = ctx->outer;
"nested inside of %<taskgroup%> region",
construct);
return false;
+ case GIMPLE_OMP_TASK:
+ if (gimple_omp_task_taskloop_p (octx->stmt)
+ && octx->outer
+ && is_taskloop_ctx (octx->outer))
+ {
+ tree clauses
+ = gimple_omp_for_clauses (octx->outer->stmt);
+ if (!omp_find_clause (clauses, OMP_CLAUSE_NOGROUP))
+ break;
+ }
+ continue;
default:
continue;
}
return true;
error_at (gimple_location (stmt),
"barrier region may not be closely nested inside "
- "of work-sharing, %<critical%>, %<ordered%>, "
- "%<master%>, explicit %<task%> or %<taskloop%> "
- "region");
+ "of work-sharing, %<loop%>, %<critical%>, "
+ "%<ordered%>, %<master%>, explicit %<task%> or "
+ "%<taskloop%> region");
return false;
}
error_at (gimple_location (stmt),
"work-sharing region may not be closely nested inside "
- "of work-sharing, %<critical%>, %<ordered%>, "
+ "of work-sharing, %<loop%>, %<critical%>, %<ordered%>, "
"%<master%>, explicit %<task%> or %<taskloop%> region");
return false;
case GIMPLE_OMP_PARALLEL:
case GIMPLE_OMP_TASK:
error_at (gimple_location (stmt),
"%<master%> region may not be closely nested inside "
- "of work-sharing, explicit %<task%> or %<taskloop%> "
- "region");
+ "of work-sharing, %<loop%>, explicit %<task%> or "
+ "%<taskloop%> region");
return false;
case GIMPLE_OMP_PARALLEL:
case GIMPLE_OMP_TEAMS:
case GIMPLE_OMP_FOR:
if (gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_TASKLOOP)
goto ordered_in_taskloop;
- if (omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
- OMP_CLAUSE_ORDERED) == NULL)
+ tree o;
+ o = omp_find_clause (gimple_omp_for_clauses (ctx->stmt),
+ OMP_CLAUSE_ORDERED);
+ if (o == NULL)
{
error_at (gimple_location (stmt),
"%<ordered%> region must be closely nested inside "
"a loop region with an %<ordered%> clause");
return false;
}
+ if (OMP_CLAUSE_ORDERED_EXPR (o) != NULL_TREE
+ && omp_find_clause (c, OMP_CLAUSE_DEPEND) == NULL_TREE)
+ {
+ error_at (gimple_location (stmt),
+ "%<ordered%> region without %<depend%> clause may "
+ "not be closely nested inside a loop region with "
+ "an %<ordered%> clause with a parameter");
+ return false;
+ }
return true;
case GIMPLE_OMP_TARGET:
if (gimple_omp_target_kind (ctx->stmt)
}
break;
case GIMPLE_OMP_TEAMS:
- if (ctx == NULL
- || gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
- || gimple_omp_target_kind (ctx->stmt) != GF_OMP_TARGET_KIND_REGION)
+ if (ctx == NULL)
+ break;
+ else if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET
+ || (gimple_omp_target_kind (ctx->stmt)
+ != GF_OMP_TARGET_KIND_REGION))
{
+ /* Teams construct can appear either strictly nested inside of
+ target construct with no intervening stmts, or can be encountered
+ only by initial task (so must not appear inside any OpenMP
+ construct. */
error_at (gimple_location (stmt),
- "%<teams%> construct not closely nested inside of "
- "%<target%> construct");
+ "%<teams%> construct must be closely nested inside of "
+ "%<target%> construct or not nested in any OpenMP "
+ "construct");
return false;
}
break;
stmt_name = "target exit data"; break;
case GF_OMP_TARGET_KIND_OACC_PARALLEL: stmt_name = "parallel"; break;
case GF_OMP_TARGET_KIND_OACC_KERNELS: stmt_name = "kernels"; break;
+ case GF_OMP_TARGET_KIND_OACC_SERIAL: stmt_name = "serial"; break;
case GF_OMP_TARGET_KIND_OACC_DATA: stmt_name = "data"; break;
case GF_OMP_TARGET_KIND_OACC_UPDATE: stmt_name = "update"; break;
case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
stmt_name = "enter/exit data"; break;
+ case GF_OMP_TARGET_KIND_OACC_DECLARE: stmt_name = "declare"; break;
case GF_OMP_TARGET_KIND_OACC_HOST_DATA: stmt_name = "host_data";
break;
+ case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
+ case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
+ case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
+ /* OpenACC 'kernels' decomposed parts. */
+ stmt_name = "kernels"; break;
default: gcc_unreachable ();
}
switch (gimple_omp_target_kind (ctx->stmt))
ctx_stmt_name = "parallel"; break;
case GF_OMP_TARGET_KIND_OACC_KERNELS:
ctx_stmt_name = "kernels"; break;
+ case GF_OMP_TARGET_KIND_OACC_SERIAL:
+ ctx_stmt_name = "serial"; break;
case GF_OMP_TARGET_KIND_OACC_DATA: ctx_stmt_name = "data"; break;
case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
ctx_stmt_name = "host_data"; break;
+ case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
+ case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
+ case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
+ /* OpenACC 'kernels' decomposed parts. */
+ ctx_stmt_name = "kernels"; break;
default: gcc_unreachable ();
}
static bool
setjmp_or_longjmp_p (const_tree fndecl)
{
- if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL
- && (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_SETJMP
- || DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LONGJMP))
+ if (fndecl_built_in_p (fndecl, BUILT_IN_SETJMP)
+ || fndecl_built_in_p (fndecl, BUILT_IN_LONGJMP))
return true;
tree declname = DECL_NAME (fndecl);
- if (!declname)
+ if (!declname
+ || (DECL_CONTEXT (fndecl) != NULL_TREE
+ && TREE_CODE (DECL_CONTEXT (fndecl)) != TRANSLATION_UNIT_DECL)
+ || !TREE_PUBLIC (fndecl))
return false;
+
const char *name = IDENTIFIER_POINTER (declname);
return !strcmp (name, "setjmp") || !strcmp (name, "longjmp");
}
+/* Return true if FNDECL is an omp_* runtime API call. */
+
+static bool
+omp_runtime_api_call (const_tree fndecl)
+{
+ tree declname = DECL_NAME (fndecl);
+ if (!declname
+ || (DECL_CONTEXT (fndecl) != NULL_TREE
+ && TREE_CODE (DECL_CONTEXT (fndecl)) != TRANSLATION_UNIT_DECL)
+ || !TREE_PUBLIC (fndecl))
+ return false;
+
+ const char *name = IDENTIFIER_POINTER (declname);
+ if (strncmp (name, "omp_", 4) != 0)
+ return false;
+
+ static const char *omp_runtime_apis[] =
+ {
+ /* This array has 3 sections. First omp_* calls that don't
+ have any suffixes. */
+ "target_alloc",
+ "target_associate_ptr",
+ "target_disassociate_ptr",
+ "target_free",
+ "target_is_present",
+ "target_memcpy",
+ "target_memcpy_rect",
+ NULL,
+ /* Now omp_* calls that are available as omp_* and omp_*_. */
+ "capture_affinity",
+ "destroy_lock",
+ "destroy_nest_lock",
+ "display_affinity",
+ "get_active_level",
+ "get_affinity_format",
+ "get_cancellation",
+ "get_default_device",
+ "get_dynamic",
+ "get_initial_device",
+ "get_level",
+ "get_max_active_levels",
+ "get_max_task_priority",
+ "get_max_threads",
+ "get_nested",
+ "get_num_devices",
+ "get_num_places",
+ "get_num_procs",
+ "get_num_teams",
+ "get_num_threads",
+ "get_partition_num_places",
+ "get_place_num",
+ "get_proc_bind",
+ "get_team_num",
+ "get_thread_limit",
+ "get_thread_num",
+ "get_wtick",
+ "get_wtime",
+ "in_final",
+ "in_parallel",
+ "init_lock",
+ "init_nest_lock",
+ "is_initial_device",
+ "pause_resource",
+ "pause_resource_all",
+ "set_affinity_format",
+ "set_lock",
+ "set_nest_lock",
+ "test_lock",
+ "test_nest_lock",
+ "unset_lock",
+ "unset_nest_lock",
+ NULL,
+ /* And finally calls available as omp_*, omp_*_ and omp_*_8_. */
+ "get_ancestor_thread_num",
+ "get_partition_place_nums",
+ "get_place_num_procs",
+ "get_place_proc_ids",
+ "get_schedule",
+ "get_team_size",
+ "set_default_device",
+ "set_dynamic",
+ "set_max_active_levels",
+ "set_nested",
+ "set_num_threads",
+ "set_schedule"
+ };
+
+ int mode = 0;
+ for (unsigned i = 0; i < ARRAY_SIZE (omp_runtime_apis); i++)
+ {
+ if (omp_runtime_apis[i] == NULL)
+ {
+ mode++;
+ continue;
+ }
+ size_t len = strlen (omp_runtime_apis[i]);
+ if (strncmp (name + 4, omp_runtime_apis[i], len) == 0
+ && (name[4 + len] == '\0'
+ || (mode > 0
+ && name[4 + len] == '_'
+ && (name[4 + len + 1] == '\0'
+ || (mode > 1
+ && strcmp (name + 4 + len + 1, "8_") == 0)))))
+ return true;
+ }
+ return false;
+}
/* Helper function for scan_omp.
tree fndecl = gimple_call_fndecl (stmt);
if (fndecl)
{
- if (setjmp_or_longjmp_p (fndecl)
- && ctx
+ if (ctx
&& gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
- && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
+ && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD
+ && setjmp_or_longjmp_p (fndecl)
+ && !ctx->loop_p)
{
remove = true;
error_at (gimple_location (stmt),
- "setjmp/longjmp inside simd construct");
+ "setjmp/longjmp inside %<simd%> construct");
}
else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
switch (DECL_FUNCTION_CODE (fndecl))
default:
break;
}
+ else if (ctx)
+ {
+ omp_context *octx = ctx;
+ if (gimple_code (ctx->stmt) == GIMPLE_OMP_SCAN && ctx->outer)
+ octx = ctx->outer;
+ if (octx->order_concurrent && omp_runtime_api_call (fndecl))
+ {
+ remove = true;
+ error_at (gimple_location (stmt),
+ "OpenMP runtime API call %qD in a region with "
+ "%<order(concurrent)%> clause", fndecl);
+ }
+ }
}
}
if (remove)
break;
case GIMPLE_OMP_FOR:
- if (((gimple_omp_for_kind (as_a <gomp_for *> (stmt))
- & GF_OMP_FOR_KIND_MASK) == GF_OMP_FOR_KIND_SIMD)
+ if ((gimple_omp_for_kind (as_a <gomp_for *> (stmt))
+ == GF_OMP_FOR_KIND_SIMD)
+ && gimple_omp_for_combined_into_p (stmt)
+ && gimple_code (ctx->stmt) != GIMPLE_OMP_SCAN)
+ {
+ tree clauses = gimple_omp_for_clauses (as_a <gomp_for *> (stmt));
+ tree c = omp_find_clause (clauses, OMP_CLAUSE_REDUCTION);
+ if (c && OMP_CLAUSE_REDUCTION_INSCAN (c) && !seen_error ())
+ {
+ scan_omp_simd_scan (gsi, as_a <gomp_for *> (stmt), ctx);
+ break;
+ }
+ }
+ if ((gimple_omp_for_kind (as_a <gomp_for *> (stmt))
+ == GF_OMP_FOR_KIND_SIMD)
&& omp_maybe_offloaded_ctx (ctx)
- && omp_max_simt_vf ())
+ && omp_max_simt_vf ()
+ && gimple_omp_for_collapse (stmt) == 1)
scan_omp_simd (gsi, as_a <gomp_for *> (stmt), ctx);
else
scan_omp_for (as_a <gomp_for *> (stmt), ctx);
scan_omp_single (as_a <gomp_single *> (stmt), ctx);
break;
+ case GIMPLE_OMP_SCAN:
+ if (tree clauses = gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt)))
+ {
+ if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_INCLUSIVE)
+ ctx->scan_inclusive = true;
+ else if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_EXCLUSIVE)
+ ctx->scan_exclusive = true;
+ }
+ /* FALLTHRU */
case GIMPLE_OMP_SECTION:
case GIMPLE_OMP_MASTER:
- case GIMPLE_OMP_TASKGROUP:
case GIMPLE_OMP_ORDERED:
case GIMPLE_OMP_CRITICAL:
- case GIMPLE_OMP_GRID_BODY:
ctx = new_omp_context (stmt, ctx);
scan_omp (gimple_omp_body_ptr (stmt), ctx);
break;
+ case GIMPLE_OMP_TASKGROUP:
+ ctx = new_omp_context (stmt, ctx);
+ scan_sharing_clauses (gimple_omp_taskgroup_clauses (stmt), ctx);
+ scan_omp (gimple_omp_body_ptr (stmt), ctx);
+ break;
+
case GIMPLE_OMP_TARGET:
- scan_omp_target (as_a <gomp_target *> (stmt), ctx);
+ if (is_gimple_omp_offloaded (stmt))
+ {
+ taskreg_nesting_level++;
+ scan_omp_target (as_a <gomp_target *> (stmt), ctx);
+ taskreg_nesting_level--;
+ }
+ else
+ scan_omp_target (as_a <gomp_target *> (stmt), ctx);
break;
case GIMPLE_OMP_TEAMS:
- scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
+ if (gimple_omp_teams_host (as_a <gomp_teams *> (stmt)))
+ {
+ taskreg_nesting_level++;
+ scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
+ taskreg_nesting_level--;
+ }
+ else
+ scan_omp_teams (as_a <gomp_teams *> (stmt), ctx);
break;
case GIMPLE_BIND:
/* Otherwise return implementation defined alignment. */
unsigned int al = 1;
opt_scalar_mode mode_iter;
- auto_vector_sizes sizes;
- targetm.vectorize.autovectorize_vector_sizes (&sizes);
- poly_uint64 vs = 0;
- for (unsigned int i = 0; i < sizes.length (); ++i)
- vs = ordered_max (vs, sizes[i]);
+ auto_vector_modes modes;
+ targetm.vectorize.autovectorize_vector_modes (&modes, true);
static enum mode_class classes[]
= { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT };
for (int i = 0; i < 4; i += 2)
machine_mode vmode = targetm.vectorize.preferred_simd_mode (mode);
if (GET_MODE_CLASS (vmode) != classes[i + 1])
continue;
- while (maybe_ne (vs, 0U)
- && known_lt (GET_MODE_SIZE (vmode), vs)
- && GET_MODE_2XWIDER_MODE (vmode).exists ())
- vmode = GET_MODE_2XWIDER_MODE (vmode).require ();
+ machine_mode alt_vmode;
+ for (unsigned int j = 0; j < modes.length (); ++j)
+ if (related_vector_mode (modes[j], mode).exists (&alt_vmode)
+ && known_ge (GET_MODE_SIZE (alt_vmode), GET_MODE_SIZE (vmode)))
+ vmode = alt_vmode;
tree type = lang_hooks.types.type_for_mode (mode, 1);
if (type == NULL_TREE || TYPE_MODE (type) != mode)
continue;
- poly_uint64 nelts = exact_div (GET_MODE_SIZE (vmode),
- GET_MODE_SIZE (mode));
- type = build_vector_type (type, nelts);
+ type = build_vector_type_for_mode (type, vmode);
if (TYPE_MODE (type) != vmode)
continue;
if (TYPE_ALIGN_UNIT (type) > al)
/* This structure is part of the interface between lower_rec_simd_input_clauses
and lower_rec_input_clauses. */
-struct omplow_simd_context {
+class omplow_simd_context {
+public:
omplow_simd_context () { memset (this, 0, sizeof (*this)); }
tree idx;
tree lane;
+ tree lastlane;
vec<tree, va_heap> simt_eargs;
gimple_seq simt_dlist;
poly_uint64_pod max_vf;
static bool
lower_rec_simd_input_clauses (tree new_var, omp_context *ctx,
- omplow_simd_context *sctx, tree &ivar, tree &lvar)
+ omplow_simd_context *sctx, tree &ivar,
+ tree &lvar, tree *rvar = NULL,
+ tree *rvar2 = NULL)
{
if (known_eq (sctx->max_vf, 0U))
{
DECL_ATTRIBUTES (ivar) = tree_cons (get_identifier ("omp simt private"),
NULL, DECL_ATTRIBUTES (ivar));
sctx->simt_eargs.safe_push (build1 (ADDR_EXPR, ptype, ivar));
- tree clobber = build_constructor (type, NULL);
- TREE_THIS_VOLATILE (clobber) = 1;
+ tree clobber = build_clobber (type);
gimple *g = gimple_build_assign (ivar, clobber);
gimple_seq_add_stmt (&sctx->simt_dlist, g);
}
= tree_cons (get_identifier ("omp simd array"), NULL,
DECL_ATTRIBUTES (avar));
gimple_add_tmp_var (avar);
- ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, sctx->idx,
+ tree iavar = avar;
+ if (rvar && !ctx->for_simd_scan_phase)
+ {
+ /* For inscan reductions, create another array temporary,
+ which will hold the reduced value. */
+ iavar = create_tmp_var_raw (atype);
+ if (TREE_ADDRESSABLE (new_var))
+ TREE_ADDRESSABLE (iavar) = 1;
+ DECL_ATTRIBUTES (iavar)
+ = tree_cons (get_identifier ("omp simd array"), NULL,
+ tree_cons (get_identifier ("omp simd inscan"), NULL,
+ DECL_ATTRIBUTES (iavar)));
+ gimple_add_tmp_var (iavar);
+ ctx->cb.decl_map->put (avar, iavar);
+ if (sctx->lastlane == NULL_TREE)
+ sctx->lastlane = create_tmp_var (unsigned_type_node);
+ *rvar = build4 (ARRAY_REF, TREE_TYPE (new_var), iavar,
+ sctx->lastlane, NULL_TREE, NULL_TREE);
+ TREE_THIS_NOTRAP (*rvar) = 1;
+
+ if (ctx->scan_exclusive)
+ {
+ /* And for exclusive scan yet another one, which will
+ hold the value during the scan phase. */
+ tree savar = create_tmp_var_raw (atype);
+ if (TREE_ADDRESSABLE (new_var))
+ TREE_ADDRESSABLE (savar) = 1;
+ DECL_ATTRIBUTES (savar)
+ = tree_cons (get_identifier ("omp simd array"), NULL,
+ tree_cons (get_identifier ("omp simd inscan "
+ "exclusive"), NULL,
+ DECL_ATTRIBUTES (savar)));
+ gimple_add_tmp_var (savar);
+ ctx->cb.decl_map->put (iavar, savar);
+ *rvar2 = build4 (ARRAY_REF, TREE_TYPE (new_var), savar,
+ sctx->idx, NULL_TREE, NULL_TREE);
+ TREE_THIS_NOTRAP (*rvar2) = 1;
+ }
+ }
+ ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), iavar, sctx->idx,
NULL_TREE, NULL_TREE);
lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, sctx->lane,
NULL_TREE, NULL_TREE);
+ TREE_THIS_NOTRAP (ivar) = 1;
+ TREE_THIS_NOTRAP (lvar) = 1;
}
if (DECL_P (new_var))
{
}
}
+/* Helper function for lower_rec_input_clauses. Emit into ilist sequence
+ code to emit (type) (tskred_temp[idx]). */
+
+static tree
+task_reduction_read (gimple_seq *ilist, tree tskred_temp, tree type,
+ unsigned idx)
+{
+ unsigned HOST_WIDE_INT sz
+ = tree_to_uhwi (TYPE_SIZE_UNIT (pointer_sized_int_node));
+ tree r = build2 (MEM_REF, pointer_sized_int_node,
+ tskred_temp, build_int_cst (TREE_TYPE (tskred_temp),
+ idx * sz));
+ tree v = create_tmp_var (pointer_sized_int_node);
+ gimple *g = gimple_build_assign (v, r);
+ gimple_seq_add_stmt (ilist, g);
+ if (!useless_type_conversion_p (type, pointer_sized_int_node))
+ {
+ v = create_tmp_var (type);
+ g = gimple_build_assign (v, NOP_EXPR, gimple_assign_lhs (g));
+ gimple_seq_add_stmt (ilist, g);
+ }
+ return v;
+}
+
+/* Lower early initialization of privatized variable NEW_VAR
+ if it needs an allocator (has allocate clause). */
+
+static bool
+lower_private_allocate (tree var, tree new_var, tree &allocator,
+ tree &allocate_ptr, gimple_seq *ilist,
+ omp_context *ctx, bool is_ref, tree size)
+{
+ if (allocator)
+ return false;
+ gcc_assert (allocate_ptr == NULL_TREE);
+ if (ctx->allocate_map
+ && (DECL_P (new_var) || (TYPE_P (new_var) && size)))
+ if (tree *allocatorp = ctx->allocate_map->get (var))
+ allocator = *allocatorp;
+ if (allocator == NULL_TREE)
+ return false;
+ if (!is_ref && omp_is_reference (var))
+ {
+ allocator = NULL_TREE;
+ return false;
+ }
+
+ if (TREE_CODE (allocator) != INTEGER_CST)
+ allocator = build_outer_var_ref (allocator, ctx);
+ allocator = fold_convert (pointer_sized_int_node, allocator);
+ if (TREE_CODE (allocator) != INTEGER_CST)
+ {
+ tree var = create_tmp_var (TREE_TYPE (allocator));
+ gimplify_assign (var, allocator, ilist);
+ allocator = var;
+ }
+
+ tree ptr_type, align, sz = size;
+ if (TYPE_P (new_var))
+ {
+ ptr_type = build_pointer_type (new_var);
+ align = build_int_cst (size_type_node, TYPE_ALIGN_UNIT (new_var));
+ }
+ else if (is_ref)
+ {
+ ptr_type = build_pointer_type (TREE_TYPE (TREE_TYPE (new_var)));
+ align = build_int_cst (size_type_node,
+ TYPE_ALIGN_UNIT (TREE_TYPE (ptr_type)));
+ }
+ else
+ {
+ ptr_type = build_pointer_type (TREE_TYPE (new_var));
+ align = build_int_cst (size_type_node, DECL_ALIGN_UNIT (new_var));
+ if (sz == NULL_TREE)
+ sz = fold_convert (size_type_node, DECL_SIZE_UNIT (new_var));
+ }
+ if (TREE_CODE (sz) != INTEGER_CST)
+ {
+ tree szvar = create_tmp_var (size_type_node);
+ gimplify_assign (szvar, sz, ilist);
+ sz = szvar;
+ }
+ allocate_ptr = create_tmp_var (ptr_type);
+ tree a = builtin_decl_explicit (BUILT_IN_GOMP_ALLOC);
+ gimple *g = gimple_build_call (a, 3, align, sz, allocator);
+ gimple_call_set_lhs (g, allocate_ptr);
+ gimple_seq_add_stmt (ilist, g);
+ if (!is_ref)
+ {
+ tree x = build_simple_mem_ref (allocate_ptr);
+ TREE_THIS_NOTRAP (x) = 1;
+ SET_DECL_VALUE_EXPR (new_var, x);
+ DECL_HAS_VALUE_EXPR_P (new_var) = 1;
+ }
+ return true;
+}
+
/* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN,
from the receiver (aka child) side and initializers for REFERENCE_TYPE
private variables. Initialization statements go in ILIST, while calls
lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist,
omp_context *ctx, struct omp_for_data *fd)
{
- tree c, dtor, copyin_seq, x, ptr;
+ tree c, copyin_seq, x, ptr;
bool copyin_by_ref = false;
bool lastprivate_firstprivate = false;
bool reduction_omp_orig_ref = false;
int pass;
bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
- && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD);
+ && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD);
omplow_simd_context sctx = omplow_simd_context ();
tree simt_lane = NULL_TREE, simtrec = NULL_TREE;
tree ivar = NULL_TREE, lvar = NULL_TREE, uid = NULL_TREE;
- gimple_seq llist[3] = { };
+ gimple_seq llist[4] = { };
+ tree nonconst_simd_if = NULL_TREE;
copyin_seq = NULL;
sctx.is_simt = is_simd && omp_find_clause (clauses, OMP_CLAUSE__SIMT_);
case OMP_CLAUSE_LASTPRIVATE:
if (is_variable_sized (OMP_CLAUSE_DECL (c)))
sctx.max_vf = 1;
+ else if (omp_is_reference (OMP_CLAUSE_DECL (c)))
+ {
+ tree rtype = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
+ if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype)))
+ sctx.max_vf = 1;
+ }
break;
case OMP_CLAUSE_REDUCTION:
+ case OMP_CLAUSE_IN_REDUCTION:
if (TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF
|| is_variable_sized (OMP_CLAUSE_DECL (c)))
sctx.max_vf = 1;
+ else if (omp_is_reference (OMP_CLAUSE_DECL (c)))
+ {
+ tree rtype = TREE_TYPE (TREE_TYPE (OMP_CLAUSE_DECL (c)));
+ if (!TREE_CONSTANT (TYPE_SIZE_UNIT (rtype)))
+ sctx.max_vf = 1;
+ }
+ break;
+ case OMP_CLAUSE_IF:
+ if (integer_zerop (OMP_CLAUSE_IF_EXPR (c)))
+ sctx.max_vf = 1;
+ else if (TREE_CODE (OMP_CLAUSE_IF_EXPR (c)) != INTEGER_CST)
+ nonconst_simd_if = OMP_CLAUSE_IF_EXPR (c);
+ break;
+ case OMP_CLAUSE_SIMDLEN:
+ if (integer_onep (OMP_CLAUSE_SIMDLEN_EXPR (c)))
+ sctx.max_vf = 1;
+ break;
+ case OMP_CLAUSE__CONDTEMP_:
+ /* FIXME: lastprivate(conditional:) not handled for SIMT yet. */
+ if (sctx.is_simt)
+ sctx.max_vf = 1;
break;
default:
continue;
if (sctx.is_simt && maybe_ne (sctx.max_vf, 1U))
sctx.simt_eargs.safe_push (NULL_TREE);
+ unsigned task_reduction_cnt = 0;
+ unsigned task_reduction_cntorig = 0;
+ unsigned task_reduction_cnt_full = 0;
+ unsigned task_reduction_cntorig_full = 0;
+ unsigned task_reduction_other_cnt = 0;
+ tree tskred_atype = NULL_TREE, tskred_avar = NULL_TREE;
+ tree tskred_base = NULL_TREE, tskred_temp = NULL_TREE;
/* Do all the fixed sized types in the first pass, and the variable sized
types in the second pass. This makes sure that the scalar arguments to
the variable sized types are processed before we use them in the
- variable sized operations. */
- for (pass = 0; pass < 2; ++pass)
- {
+ variable sized operations. For task reductions we use 4 passes, in the
+ first two we ignore them, in the third one gather arguments for
+ GOMP_task_reduction_remap call and in the last pass actually handle
+ the task reductions. */
+ for (pass = 0; pass < ((task_reduction_cnt || task_reduction_other_cnt)
+ ? 4 : 2); ++pass)
+ {
+ if (pass == 2 && task_reduction_cnt)
+ {
+ tskred_atype
+ = build_array_type_nelts (ptr_type_node, task_reduction_cnt
+ + task_reduction_cntorig);
+ tskred_avar = create_tmp_var_raw (tskred_atype);
+ gimple_add_tmp_var (tskred_avar);
+ TREE_ADDRESSABLE (tskred_avar) = 1;
+ task_reduction_cnt_full = task_reduction_cnt;
+ task_reduction_cntorig_full = task_reduction_cntorig;
+ }
+ else if (pass == 3 && task_reduction_cnt)
+ {
+ x = builtin_decl_explicit (BUILT_IN_GOMP_TASK_REDUCTION_REMAP);
+ gimple *g
+ = gimple_build_call (x, 3, size_int (task_reduction_cnt),
+ size_int (task_reduction_cntorig),
+ build_fold_addr_expr (tskred_avar));
+ gimple_seq_add_stmt (ilist, g);
+ }
+ if (pass == 3 && task_reduction_other_cnt)
+ {
+ /* For reduction clauses, build
+ tskred_base = (void *) tskred_temp[2]
+ + omp_get_thread_num () * tskred_temp[1]
+ or if tskred_temp[1] is known to be constant, that constant
+ directly. This is the start of the private reduction copy block
+ for the current thread. */
+ tree v = create_tmp_var (integer_type_node);
+ x = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
+ gimple *g = gimple_build_call (x, 0);
+ gimple_call_set_lhs (g, v);
+ gimple_seq_add_stmt (ilist, g);
+ c = omp_find_clause (clauses, OMP_CLAUSE__REDUCTEMP_);
+ tskred_temp = OMP_CLAUSE_DECL (c);
+ if (is_taskreg_ctx (ctx))
+ tskred_temp = lookup_decl (tskred_temp, ctx);
+ tree v2 = create_tmp_var (sizetype);
+ g = gimple_build_assign (v2, NOP_EXPR, v);
+ gimple_seq_add_stmt (ilist, g);
+ if (ctx->task_reductions[0])
+ v = fold_convert (sizetype, ctx->task_reductions[0]);
+ else
+ v = task_reduction_read (ilist, tskred_temp, sizetype, 1);
+ tree v3 = create_tmp_var (sizetype);
+ g = gimple_build_assign (v3, MULT_EXPR, v2, v);
+ gimple_seq_add_stmt (ilist, g);
+ v = task_reduction_read (ilist, tskred_temp, ptr_type_node, 2);
+ tskred_base = create_tmp_var (ptr_type_node);
+ g = gimple_build_assign (tskred_base, POINTER_PLUS_EXPR, v, v3);
+ gimple_seq_add_stmt (ilist, g);
+ }
+ task_reduction_cnt = 0;
+ task_reduction_cntorig = 0;
+ task_reduction_other_cnt = 0;
for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
{
enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c);
tree var, new_var;
bool by_ref;
location_t clause_loc = OMP_CLAUSE_LOCATION (c);
+ bool task_reduction_p = false;
+ bool task_reduction_needs_orig_p = false;
+ tree cond = NULL_TREE;
+ tree allocator, allocate_ptr;
switch (c_kind)
{
continue;
break;
case OMP_CLAUSE_SHARED:
- /* Ignore shared directives in teams construct. */
- if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
+ /* Ignore shared directives in teams construct inside
+ of target construct. */
+ if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
+ && !is_host_teams_ctx (ctx))
continue;
if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL)
{
lastprivate_firstprivate = true;
break;
case OMP_CLAUSE_REDUCTION:
- if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
+ case OMP_CLAUSE_IN_REDUCTION:
+ if (is_task_ctx (ctx) || OMP_CLAUSE_REDUCTION_TASK (c))
+ {
+ task_reduction_p = true;
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
+ {
+ task_reduction_other_cnt++;
+ if (pass == 2)
+ continue;
+ }
+ else
+ task_reduction_cnt++;
+ if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
+ {
+ var = OMP_CLAUSE_DECL (c);
+ /* If var is a global variable that isn't privatized
+ in outer contexts, we don't need to look up the
+ original address, it is always the address of the
+ global variable itself. */
+ if (!DECL_P (var)
+ || omp_is_reference (var)
+ || !is_global_var
+ (maybe_lookup_decl_in_outer_ctx (var, ctx)))
+ {
+ task_reduction_needs_orig_p = true;
+ if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
+ task_reduction_cntorig++;
+ }
+ }
+ }
+ else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
reduction_omp_orig_ref = true;
break;
+ case OMP_CLAUSE__REDUCTEMP_:
+ if (!is_taskreg_ctx (ctx))
+ continue;
+ /* FALLTHRU */
case OMP_CLAUSE__LOOPTEMP_:
- /* Handle _looptemp_ clauses only on parallel/task. */
+ /* Handle _looptemp_/_reductemp_ clauses only on
+ parallel/task. */
if (fd)
continue;
break;
lastprivate_firstprivate = true;
break;
case OMP_CLAUSE_ALIGNED:
- if (pass == 0)
+ if (pass != 1)
continue;
var = OMP_CLAUSE_DECL (c);
if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE
DECL_HAS_VALUE_EXPR_P (new_var) = 1;
}
continue;
+ case OMP_CLAUSE__CONDTEMP_:
+ if (is_parallel_ctx (ctx)
+ || (is_simd && !OMP_CLAUSE__CONDTEMP__ITER (c)))
+ break;
+ continue;
default:
continue;
}
+ if (task_reduction_p != (pass >= 2))
+ continue;
+
+ allocator = NULL_TREE;
+ allocate_ptr = NULL_TREE;
new_var = var = OMP_CLAUSE_DECL (c);
- if (c_kind == OMP_CLAUSE_REDUCTION && TREE_CODE (var) == MEM_REF)
+ if ((c_kind == OMP_CLAUSE_REDUCTION
+ || c_kind == OMP_CLAUSE_IN_REDUCTION)
+ && TREE_CODE (var) == MEM_REF)
{
var = TREE_OPERAND (var, 0);
if (TREE_CODE (var) == POINTER_PLUS_EXPR)
continue;
}
/* C/C++ array section reductions. */
- else if (c_kind == OMP_CLAUSE_REDUCTION
+ else if ((c_kind == OMP_CLAUSE_REDUCTION
+ || c_kind == OMP_CLAUSE_IN_REDUCTION)
&& var != OMP_CLAUSE_DECL (c))
{
if (pass == 0)
tree bias = TREE_OPERAND (OMP_CLAUSE_DECL (c), 1);
tree orig_var = TREE_OPERAND (OMP_CLAUSE_DECL (c), 0);
+
if (TREE_CODE (orig_var) == POINTER_PLUS_EXPR)
{
tree b = TREE_OPERAND (orig_var, 1);
}
orig_var = TREE_OPERAND (orig_var, 0);
}
+ if (pass == 2)
+ {
+ tree out = maybe_lookup_decl_in_outer_ctx (var, ctx);
+ if (is_global_var (out)
+ && TREE_CODE (TREE_TYPE (out)) != POINTER_TYPE
+ && (TREE_CODE (TREE_TYPE (out)) != REFERENCE_TYPE
+ || (TREE_CODE (TREE_TYPE (TREE_TYPE (out)))
+ != POINTER_TYPE)))
+ x = var;
+ else
+ {
+ bool by_ref = use_pointer_for_field (var, NULL);
+ x = build_receiver_ref (var, by_ref, ctx);
+ if (TREE_CODE (TREE_TYPE (var)) == REFERENCE_TYPE
+ && (TREE_CODE (TREE_TYPE (TREE_TYPE (var)))
+ == POINTER_TYPE))
+ x = build_fold_addr_expr (x);
+ }
+ if (TREE_CODE (orig_var) == INDIRECT_REF)
+ x = build_simple_mem_ref (x);
+ else if (TREE_CODE (orig_var) == ADDR_EXPR)
+ {
+ if (var == TREE_OPERAND (orig_var, 0))
+ x = build_fold_addr_expr (x);
+ }
+ bias = fold_convert (sizetype, bias);
+ x = fold_convert (ptr_type_node, x);
+ x = fold_build2_loc (clause_loc, POINTER_PLUS_EXPR,
+ TREE_TYPE (x), x, bias);
+ unsigned cnt = task_reduction_cnt - 1;
+ if (!task_reduction_needs_orig_p)
+ cnt += (task_reduction_cntorig_full
+ - task_reduction_cntorig);
+ else
+ cnt = task_reduction_cntorig - 1;
+ tree r = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
+ size_int (cnt), NULL_TREE, NULL_TREE);
+ gimplify_assign (r, x, ilist);
+ continue;
+ }
+
if (TREE_CODE (orig_var) == INDIRECT_REF
|| TREE_CODE (orig_var) == ADDR_EXPR)
orig_var = TREE_OPERAND (orig_var, 0);
tree type = TREE_TYPE (d);
gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
+ tree sz = v;
const char *name = get_name (orig_var);
- if (TREE_CONSTANT (v))
- {
- x = create_tmp_var_raw (type, name);
- gimple_add_tmp_var (x);
- TREE_ADDRESSABLE (x) = 1;
- x = build_fold_addr_expr_loc (clause_loc, x);
- }
- else
+ if (pass != 3 && !TREE_CONSTANT (v))
{
- tree atmp
- = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
tree t = maybe_lookup_decl (v, ctx);
if (t)
v = t;
t = fold_build2_loc (clause_loc, PLUS_EXPR,
TREE_TYPE (v), v,
build_int_cst (TREE_TYPE (v), 1));
- t = fold_build2_loc (clause_loc, MULT_EXPR,
- TREE_TYPE (v), t,
- TYPE_SIZE_UNIT (TREE_TYPE (type)));
+ sz = fold_build2_loc (clause_loc, MULT_EXPR,
+ TREE_TYPE (v), t,
+ TYPE_SIZE_UNIT (TREE_TYPE (type)));
+ }
+ if (pass == 3)
+ {
+ tree xv = create_tmp_var (ptr_type_node);
+ if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
+ {
+ unsigned cnt = task_reduction_cnt - 1;
+ if (!task_reduction_needs_orig_p)
+ cnt += (task_reduction_cntorig_full
+ - task_reduction_cntorig);
+ else
+ cnt = task_reduction_cntorig - 1;
+ x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
+ size_int (cnt), NULL_TREE, NULL_TREE);
+
+ gimple *g = gimple_build_assign (xv, x);
+ gimple_seq_add_stmt (ilist, g);
+ }
+ else
+ {
+ unsigned int idx = *ctx->task_reduction_map->get (c);
+ tree off;
+ if (ctx->task_reductions[1 + idx])
+ off = fold_convert (sizetype,
+ ctx->task_reductions[1 + idx]);
+ else
+ off = task_reduction_read (ilist, tskred_temp, sizetype,
+ 7 + 3 * idx + 1);
+ gimple *g = gimple_build_assign (xv, POINTER_PLUS_EXPR,
+ tskred_base, off);
+ gimple_seq_add_stmt (ilist, g);
+ }
+ x = fold_convert (build_pointer_type (boolean_type_node),
+ xv);
+ if (TREE_CONSTANT (v))
+ x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (x), x,
+ TYPE_SIZE_UNIT (type));
+ else
+ {
+ tree t = maybe_lookup_decl (v, ctx);
+ if (t)
+ v = t;
+ else
+ v = maybe_lookup_decl_in_outer_ctx (v, ctx);
+ gimplify_expr (&v, ilist, NULL, is_gimple_val,
+ fb_rvalue);
+ t = fold_build2_loc (clause_loc, PLUS_EXPR,
+ TREE_TYPE (v), v,
+ build_int_cst (TREE_TYPE (v), 1));
+ t = fold_build2_loc (clause_loc, MULT_EXPR,
+ TREE_TYPE (v), t,
+ TYPE_SIZE_UNIT (TREE_TYPE (type)));
+ x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (x), x, t);
+ }
+ cond = create_tmp_var (TREE_TYPE (x));
+ gimplify_assign (cond, x, ilist);
+ x = xv;
+ }
+ else if (lower_private_allocate (var, type, allocator,
+ allocate_ptr, ilist, ctx,
+ true,
+ TREE_CONSTANT (v)
+ ? TYPE_SIZE_UNIT (type)
+ : sz))
+ x = allocate_ptr;
+ else if (TREE_CONSTANT (v))
+ {
+ x = create_tmp_var_raw (type, name);
+ gimple_add_tmp_var (x);
+ TREE_ADDRESSABLE (x) = 1;
+ x = build_fold_addr_expr_loc (clause_loc, x);
+ }
+ else
+ {
+ tree atmp
+ = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
tree al = size_int (TYPE_ALIGN (TREE_TYPE (type)));
- x = build_call_expr_loc (clause_loc, atmp, 2, t, al);
+ x = build_call_expr_loc (clause_loc, atmp, 2, sz, al);
}
tree ptype = build_pointer_type (TREE_TYPE (type));
tree new_orig_var = lookup_decl (orig_var, ctx);
tree t = build_fold_indirect_ref (new_var);
DECL_IGNORED_P (new_var) = 0;
- TREE_THIS_NOTRAP (t);
+ TREE_THIS_NOTRAP (t) = 1;
SET_DECL_VALUE_EXPR (new_orig_var, t);
DECL_HAS_VALUE_EXPR_P (new_orig_var) = 1;
}
x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x);
gimplify_assign (new_var, x, ilist);
}
- tree y1 = create_tmp_var (ptype, NULL);
- gimplify_assign (y1, y, ilist);
- tree i2 = NULL_TREE, y2 = NULL_TREE;
+ /* GOMP_taskgroup_reduction_register memsets the whole
+ array to zero. If the initializer is zero, we don't
+ need to initialize it again, just mark it as ever
+ used unconditionally, i.e. cond = true. */
+ if (cond
+ && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) == NULL_TREE
+ && initializer_zerop (omp_reduction_init (c,
+ TREE_TYPE (type))))
+ {
+ gimple *g = gimple_build_assign (build_simple_mem_ref (cond),
+ boolean_true_node);
+ gimple_seq_add_stmt (ilist, g);
+ continue;
+ }
+ tree end = create_artificial_label (UNKNOWN_LOCATION);
+ if (cond)
+ {
+ gimple *g;
+ if (!is_parallel_ctx (ctx))
+ {
+ tree condv = create_tmp_var (boolean_type_node);
+ g = gimple_build_assign (condv,
+ build_simple_mem_ref (cond));
+ gimple_seq_add_stmt (ilist, g);
+ tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
+ g = gimple_build_cond (NE_EXPR, condv,
+ boolean_false_node, end, lab1);
+ gimple_seq_add_stmt (ilist, g);
+ gimple_seq_add_stmt (ilist, gimple_build_label (lab1));
+ }
+ g = gimple_build_assign (build_simple_mem_ref (cond),
+ boolean_true_node);
+ gimple_seq_add_stmt (ilist, g);
+ }
+
+ tree y1 = create_tmp_var (ptype);
+ gimplify_assign (y1, y, ilist);
+ tree i2 = NULL_TREE, y2 = NULL_TREE;
tree body2 = NULL_TREE, end2 = NULL_TREE;
tree y3 = NULL_TREE, y4 = NULL_TREE;
- if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) || is_simd)
+ if (task_reduction_needs_orig_p)
{
- y2 = create_tmp_var (ptype, NULL);
- gimplify_assign (y2, y, ilist);
- tree ref = build_outer_var_ref (var, ctx);
- /* For ref build_outer_var_ref already performs this. */
- if (TREE_CODE (d) == INDIRECT_REF)
- gcc_assert (omp_is_reference (var));
- else if (TREE_CODE (d) == ADDR_EXPR)
- ref = build_fold_addr_expr (ref);
- else if (omp_is_reference (var))
- ref = build_fold_addr_expr (ref);
- ref = fold_convert_loc (clause_loc, ptype, ref);
- if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
- && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
+ y3 = create_tmp_var (ptype);
+ tree ref;
+ if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
+ ref = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
+ size_int (task_reduction_cnt_full
+ + task_reduction_cntorig - 1),
+ NULL_TREE, NULL_TREE);
+ else
{
- y3 = create_tmp_var (ptype, NULL);
- gimplify_assign (y3, unshare_expr (ref), ilist);
+ unsigned int idx = *ctx->task_reduction_map->get (c);
+ ref = task_reduction_read (ilist, tskred_temp, ptype,
+ 7 + 3 * idx);
}
- if (is_simd)
+ gimplify_assign (y3, ref, ilist);
+ }
+ else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) || is_simd)
+ {
+ if (pass != 3)
+ {
+ y2 = create_tmp_var (ptype);
+ gimplify_assign (y2, y, ilist);
+ }
+ if (is_simd || OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
{
- y4 = create_tmp_var (ptype, NULL);
- gimplify_assign (y4, ref, dlist);
+ tree ref = build_outer_var_ref (var, ctx);
+ /* For ref build_outer_var_ref already performs this. */
+ if (TREE_CODE (d) == INDIRECT_REF)
+ gcc_assert (omp_is_reference (var));
+ else if (TREE_CODE (d) == ADDR_EXPR)
+ ref = build_fold_addr_expr (ref);
+ else if (omp_is_reference (var))
+ ref = build_fold_addr_expr (ref);
+ ref = fold_convert_loc (clause_loc, ptype, ref);
+ if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
+ && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
+ {
+ y3 = create_tmp_var (ptype);
+ gimplify_assign (y3, unshare_expr (ref), ilist);
+ }
+ if (is_simd)
+ {
+ y4 = create_tmp_var (ptype);
+ gimplify_assign (y4, ref, dlist);
+ }
}
}
- tree i = create_tmp_var (TREE_TYPE (v), NULL);
+ tree i = create_tmp_var (TREE_TYPE (v));
gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), ilist);
tree body = create_artificial_label (UNKNOWN_LOCATION);
- tree end = create_artificial_label (UNKNOWN_LOCATION);
gimple_seq_add_stmt (ilist, gimple_build_label (body));
if (y2)
{
- i2 = create_tmp_var (TREE_TYPE (v), NULL);
+ i2 = create_tmp_var (TREE_TYPE (v));
gimplify_assign (i2, build_int_cst (TREE_TYPE (v), 0), dlist);
body2 = create_artificial_label (UNKNOWN_LOCATION);
end2 = create_artificial_label (UNKNOWN_LOCATION);
}
DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 0;
- x = lang_hooks.decls.omp_clause_dtor
- (c, build_simple_mem_ref (y2));
- if (x)
+ if (y2)
{
- gimple_seq tseq = NULL;
- dtor = x;
- gimplify_stmt (&dtor, &tseq);
- gimple_seq_add_seq (dlist, tseq);
+ x = lang_hooks.decls.omp_clause_dtor
+ (c, build_simple_mem_ref (y2));
+ if (x)
+ gimplify_and_add (x, dlist);
}
}
else
gimple_seq_add_stmt (dlist, g);
gimple_seq_add_stmt (dlist, gimple_build_label (end2));
}
+ if (allocator)
+ {
+ tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
+ g = gimple_build_call (f, 2, allocate_ptr, allocator);
+ gimple_seq_add_stmt (dlist, g);
+ }
+ continue;
+ }
+ else if (pass == 2)
+ {
+ if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx)))
+ x = var;
+ else
+ {
+ bool by_ref = use_pointer_for_field (var, ctx);
+ x = build_receiver_ref (var, by_ref, ctx);
+ }
+ if (!omp_is_reference (var))
+ x = build_fold_addr_expr (x);
+ x = fold_convert (ptr_type_node, x);
+ unsigned cnt = task_reduction_cnt - 1;
+ if (!task_reduction_needs_orig_p)
+ cnt += task_reduction_cntorig_full - task_reduction_cntorig;
+ else
+ cnt = task_reduction_cntorig - 1;
+ tree r = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
+ size_int (cnt), NULL_TREE, NULL_TREE);
+ gimplify_assign (r, x, ilist);
continue;
}
+ else if (pass == 3)
+ {
+ tree type = TREE_TYPE (new_var);
+ if (!omp_is_reference (var))
+ type = build_pointer_type (type);
+ if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
+ {
+ unsigned cnt = task_reduction_cnt - 1;
+ if (!task_reduction_needs_orig_p)
+ cnt += (task_reduction_cntorig_full
+ - task_reduction_cntorig);
+ else
+ cnt = task_reduction_cntorig - 1;
+ x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
+ size_int (cnt), NULL_TREE, NULL_TREE);
+ }
+ else
+ {
+ unsigned int idx = *ctx->task_reduction_map->get (c);
+ tree off;
+ if (ctx->task_reductions[1 + idx])
+ off = fold_convert (sizetype,
+ ctx->task_reductions[1 + idx]);
+ else
+ off = task_reduction_read (ilist, tskred_temp, sizetype,
+ 7 + 3 * idx + 1);
+ x = fold_build2 (POINTER_PLUS_EXPR, ptr_type_node,
+ tskred_base, off);
+ }
+ x = fold_convert (type, x);
+ tree t;
+ if (omp_is_reference (var))
+ {
+ gimplify_assign (new_var, x, ilist);
+ t = new_var;
+ new_var = build_simple_mem_ref (new_var);
+ }
+ else
+ {
+ t = create_tmp_var (type);
+ gimplify_assign (t, x, ilist);
+ SET_DECL_VALUE_EXPR (new_var, build_simple_mem_ref (t));
+ DECL_HAS_VALUE_EXPR_P (new_var) = 1;
+ }
+ t = fold_convert (build_pointer_type (boolean_type_node), t);
+ t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t,
+ TYPE_SIZE_UNIT (TREE_TYPE (type)));
+ cond = create_tmp_var (TREE_TYPE (t));
+ gimplify_assign (cond, t, ilist);
+ }
else if (is_variable_sized (var))
{
/* For variable sized types, we need to allocate the
if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx))
{
- gcall *stmt;
- tree tmp, atmp;
+ tree tmp;
ptr = DECL_VALUE_EXPR (new_var);
gcc_assert (TREE_CODE (ptr) == INDIRECT_REF);
gcc_assert (DECL_P (ptr));
x = TYPE_SIZE_UNIT (TREE_TYPE (new_var));
- /* void *tmp = __builtin_alloca */
- atmp = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
- stmt = gimple_build_call (atmp, 2, x,
- size_int (DECL_ALIGN (var)));
- tmp = create_tmp_var_raw (ptr_type_node);
- gimple_add_tmp_var (tmp);
- gimple_call_set_lhs (stmt, tmp);
-
- gimple_seq_add_stmt (ilist, stmt);
+ if (lower_private_allocate (var, new_var, allocator,
+ allocate_ptr, ilist, ctx,
+ false, x))
+ tmp = allocate_ptr;
+ else
+ {
+ /* void *tmp = __builtin_alloca */
+ tree atmp
+ = builtin_decl_explicit (BUILT_IN_ALLOCA_WITH_ALIGN);
+ gcall *stmt
+ = gimple_build_call (atmp, 2, x,
+ size_int (DECL_ALIGN (var)));
+ cfun->calls_alloca = 1;
+ tmp = create_tmp_var_raw (ptr_type_node);
+ gimple_add_tmp_var (tmp);
+ gimple_call_set_lhs (stmt, tmp);
+
+ gimple_seq_add_stmt (ilist, stmt);
+ }
x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp);
gimplify_assign (ptr, x, ilist);
}
}
- else if (omp_is_reference (var))
+ else if (omp_is_reference (var)
+ && (c_kind != OMP_CLAUSE_FIRSTPRIVATE
+ || !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c)))
{
/* For references that are being privatized for Fortran,
allocate new backing storage for the new pointer
if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx))
{
x = build_receiver_ref (var, false, ctx);
- x = build_fold_addr_expr_loc (clause_loc, x);
+ if (ctx->allocate_map)
+ if (tree *allocatep = ctx->allocate_map->get (var))
+ {
+ allocator = *allocatep;
+ if (TREE_CODE (allocator) != INTEGER_CST)
+ allocator = build_outer_var_ref (allocator, ctx);
+ allocator = fold_convert (pointer_sized_int_node,
+ allocator);
+ allocate_ptr = unshare_expr (x);
+ }
+ if (allocator == NULL_TREE)
+ x = build_fold_addr_expr_loc (clause_loc, x);
}
+ else if (lower_private_allocate (var, new_var, allocator,
+ allocate_ptr,
+ ilist, ctx, true, x))
+ x = allocate_ptr;
else if (TREE_CONSTANT (x))
{
/* For reduction in SIMD loop, defer adding the
initialization of the reference, because if we decide
to use SIMD array for it, the initilization could cause
- expansion ICE. */
- if (c_kind == OMP_CLAUSE_REDUCTION && is_simd)
+ expansion ICE. Ditto for other privatization clauses. */
+ if (is_simd)
x = NULL_TREE;
else
{
new_var = build_simple_mem_ref_loc (clause_loc, new_var);
}
- else if (c_kind == OMP_CLAUSE_REDUCTION
+ else if ((c_kind == OMP_CLAUSE_REDUCTION
+ || c_kind == OMP_CLAUSE_IN_REDUCTION)
&& OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
{
if (pass == 0)
switch (OMP_CLAUSE_CODE (c))
{
case OMP_CLAUSE_SHARED:
- /* Ignore shared directives in teams construct. */
- if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS)
+ /* Ignore shared directives in teams construct inside
+ target construct. */
+ if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS
+ && !is_host_teams_ctx (ctx))
continue;
/* Shared global vars are just accessed directly. */
if (is_global_var (new_var))
TREE_NO_WARNING (var) = 1;
break;
+ case OMP_CLAUSE__CONDTEMP_:
+ if (is_parallel_ctx (ctx))
+ {
+ x = build_receiver_ref (var, false, ctx);
+ SET_DECL_VALUE_EXPR (new_var, x);
+ DECL_HAS_VALUE_EXPR_P (new_var) = 1;
+ }
+ else if (is_simd && !OMP_CLAUSE__CONDTEMP__ITER (c))
+ {
+ x = build_zero_cst (TREE_TYPE (var));
+ goto do_private;
+ }
+ break;
+
case OMP_CLAUSE_LASTPRIVATE:
if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c))
break;
x = NULL;
do_private:
tree nx;
- nx = lang_hooks.decls.omp_clause_default_ctor
- (c, unshare_expr (new_var), x);
+ bool copy_ctor;
+ copy_ctor = false;
+ lower_private_allocate (var, new_var, allocator, allocate_ptr,
+ ilist, ctx, false, NULL_TREE);
+ nx = unshare_expr (new_var);
+ if (is_simd
+ && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
+ && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c))
+ copy_ctor = true;
+ if (copy_ctor)
+ nx = lang_hooks.decls.omp_clause_copy_ctor (c, nx, x);
+ else
+ nx = lang_hooks.decls.omp_clause_default_ctor (c, nx, x);
if (is_simd)
{
tree y = lang_hooks.decls.omp_clause_dtor (c, new_var);
if ((TREE_ADDRESSABLE (new_var) || nx || y
- || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE)
+ || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
+ && (gimple_omp_for_collapse (ctx->stmt) != 1
+ || (gimple_omp_for_index (ctx->stmt, 0)
+ != new_var)))
+ || OMP_CLAUSE_CODE (c) == OMP_CLAUSE__CONDTEMP_
+ || omp_is_reference (var))
&& lower_rec_simd_input_clauses (new_var, ctx, &sctx,
ivar, lvar))
{
+ if (omp_is_reference (var))
+ {
+ gcc_assert (TREE_CODE (new_var) == MEM_REF);
+ tree new_vard = TREE_OPERAND (new_var, 0);
+ gcc_assert (DECL_P (new_vard));
+ SET_DECL_VALUE_EXPR (new_vard,
+ build_fold_addr_expr (lvar));
+ DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
+ }
+
if (nx)
- x = lang_hooks.decls.omp_clause_default_ctor
- (c, unshare_expr (ivar), x);
+ {
+ tree iv = unshare_expr (ivar);
+ if (copy_ctor)
+ x = lang_hooks.decls.omp_clause_copy_ctor (c, iv,
+ x);
+ else
+ x = lang_hooks.decls.omp_clause_default_ctor (c,
+ iv,
+ x);
+ }
+ else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE__CONDTEMP_)
+ {
+ x = build2 (MODIFY_EXPR, TREE_TYPE (ivar),
+ unshare_expr (ivar), x);
+ nx = x;
+ }
if (nx && x)
gimplify_and_add (x, &llist[0]);
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
+ && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
+ {
+ tree v = new_var;
+ if (!DECL_P (v))
+ {
+ gcc_assert (TREE_CODE (v) == MEM_REF);
+ v = TREE_OPERAND (v, 0);
+ gcc_assert (DECL_P (v));
+ }
+ v = *ctx->lastprivate_conditional_map->get (v);
+ tree t = create_tmp_var (TREE_TYPE (v));
+ tree z = build_zero_cst (TREE_TYPE (v));
+ tree orig_v
+ = build_outer_var_ref (var, ctx,
+ OMP_CLAUSE_LASTPRIVATE);
+ gimple_seq_add_stmt (dlist,
+ gimple_build_assign (t, z));
+ gcc_assert (DECL_HAS_VALUE_EXPR_P (v));
+ tree civar = DECL_VALUE_EXPR (v);
+ gcc_assert (TREE_CODE (civar) == ARRAY_REF);
+ civar = unshare_expr (civar);
+ TREE_OPERAND (civar, 1) = sctx.idx;
+ x = build2 (MODIFY_EXPR, TREE_TYPE (t), t,
+ unshare_expr (civar));
+ x = build2 (COMPOUND_EXPR, TREE_TYPE (orig_v), x,
+ build2 (MODIFY_EXPR, TREE_TYPE (orig_v),
+ orig_v, unshare_expr (ivar)));
+ tree cond = build2 (LT_EXPR, boolean_type_node, t,
+ civar);
+ x = build3 (COND_EXPR, void_type_node, cond, x,
+ void_node);
+ gimple_seq tseq = NULL;
+ gimplify_and_add (x, &tseq);
+ if (ctx->outer)
+ lower_omp (&tseq, ctx->outer);
+ gimple_seq_add_seq (&llist[1], tseq);
+ }
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
+ && ctx->for_simd_scan_phase)
+ {
+ x = unshare_expr (ivar);
+ tree orig_v
+ = build_outer_var_ref (var, ctx,
+ OMP_CLAUSE_LASTPRIVATE);
+ x = lang_hooks.decls.omp_clause_assign_op (c, x,
+ orig_v);
+ gimplify_and_add (x, &llist[0]);
+ }
if (y)
{
y = lang_hooks.decls.omp_clause_dtor (c, ivar);
if (y)
- {
- gimple_seq tseq = NULL;
-
- dtor = y;
- gimplify_stmt (&dtor, &tseq);
- gimple_seq_add_seq (&llist[1], tseq);
- }
+ gimplify_and_add (y, &llist[1]);
}
break;
}
+ if (omp_is_reference (var))
+ {
+ gcc_assert (TREE_CODE (new_var) == MEM_REF);
+ tree new_vard = TREE_OPERAND (new_var, 0);
+ gcc_assert (DECL_P (new_vard));
+ tree type = TREE_TYPE (TREE_TYPE (new_vard));
+ x = TYPE_SIZE_UNIT (type);
+ if (TREE_CONSTANT (x))
+ {
+ x = create_tmp_var_raw (type, get_name (var));
+ gimple_add_tmp_var (x);
+ TREE_ADDRESSABLE (x) = 1;
+ x = build_fold_addr_expr_loc (clause_loc, x);
+ x = fold_convert_loc (clause_loc,
+ TREE_TYPE (new_vard), x);
+ gimplify_assign (new_vard, x, ilist);
+ }
+ }
}
if (nx)
gimplify_and_add (nx, ilist);
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
+ && is_simd
+ && ctx->for_simd_scan_phase)
+ {
+ tree orig_v = build_outer_var_ref (var, ctx,
+ OMP_CLAUSE_LASTPRIVATE);
+ x = lang_hooks.decls.omp_clause_assign_op (c, new_var,
+ orig_v);
+ gimplify_and_add (x, ilist);
+ }
/* FALLTHRU */
do_dtor:
x = lang_hooks.decls.omp_clause_dtor (c, new_var);
if (x)
+ gimplify_and_add (x, dlist);
+ if (allocator)
{
- gimple_seq tseq = NULL;
-
- dtor = x;
- gimplify_stmt (&dtor, &tseq);
- gimple_seq_add_seq (dlist, tseq);
+ if (!is_gimple_val (allocator))
+ {
+ tree avar = create_tmp_var (TREE_TYPE (allocator));
+ gimplify_assign (avar, allocator, dlist);
+ allocator = avar;
+ }
+ if (!is_gimple_val (allocate_ptr))
+ {
+ tree apvar = create_tmp_var (TREE_TYPE (allocate_ptr));
+ gimplify_assign (apvar, allocate_ptr, dlist);
+ allocate_ptr = apvar;
+ }
+ tree f = builtin_decl_explicit (BUILT_IN_GOMP_FREE);
+ gimple *g
+ = gimple_build_call (f, 2, allocate_ptr, allocator);
+ gimple_seq_add_stmt (dlist, g);
}
break;
case OMP_CLAUSE_FIRSTPRIVATE:
if (is_task_ctx (ctx))
{
- if (omp_is_reference (var) || is_variable_sized (var))
+ if ((omp_is_reference (var)
+ && !OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c))
+ || is_variable_sized (var))
goto do_dtor;
else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var,
ctx))
|| use_pointer_for_field (var, NULL))
{
x = build_receiver_ref (var, false, ctx);
+ if (ctx->allocate_map)
+ if (tree *allocatep = ctx->allocate_map->get (var))
+ {
+ allocator = *allocatep;
+ if (TREE_CODE (allocator) != INTEGER_CST)
+ allocator = build_outer_var_ref (allocator, ctx);
+ allocator = fold_convert (pointer_sized_int_node,
+ allocator);
+ allocate_ptr = unshare_expr (x);
+ x = build_simple_mem_ref (x);
+ TREE_THIS_NOTRAP (x) = 1;
+ }
SET_DECL_VALUE_EXPR (new_var, x);
DECL_HAS_VALUE_EXPR_P (new_var) = 1;
goto do_dtor;
}
}
+ if (OMP_CLAUSE_FIRSTPRIVATE_NO_REFERENCE (c)
+ && omp_is_reference (var))
+ {
+ x = build_outer_var_ref (var, ctx);
+ gcc_assert (TREE_CODE (x) == MEM_REF
+ && integer_zerop (TREE_OPERAND (x, 1)));
+ x = TREE_OPERAND (x, 0);
+ x = lang_hooks.decls.omp_clause_copy_ctor
+ (c, unshare_expr (new_var), x);
+ gimplify_and_add (x, ilist);
+ goto do_dtor;
+ }
do_firstprivate:
+ lower_private_allocate (var, new_var, allocator, allocate_ptr,
+ ilist, ctx, false, NULL_TREE);
x = build_outer_var_ref (var, ctx);
if (is_simd)
{
if (OMP_CLAUSE_LINEAR_ARRAY (c))
{
+ if (omp_is_reference (var))
+ {
+ gcc_assert (TREE_CODE (new_var) == MEM_REF);
+ tree new_vard = TREE_OPERAND (new_var, 0);
+ gcc_assert (DECL_P (new_vard));
+ tree type = TREE_TYPE (TREE_TYPE (new_vard));
+ nx = TYPE_SIZE_UNIT (type);
+ if (TREE_CONSTANT (nx))
+ {
+ nx = create_tmp_var_raw (type,
+ get_name (var));
+ gimple_add_tmp_var (nx);
+ TREE_ADDRESSABLE (nx) = 1;
+ nx = build_fold_addr_expr_loc (clause_loc,
+ nx);
+ nx = fold_convert_loc (clause_loc,
+ TREE_TYPE (new_vard),
+ nx);
+ gimplify_assign (new_vard, nx, ilist);
+ }
+ }
+
x = lang_hooks.decls.omp_clause_linear_ctor
(c, new_var, x, t);
gimplify_and_add (x, ilist);
}
if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR
- || TREE_ADDRESSABLE (new_var))
+ || TREE_ADDRESSABLE (new_var)
+ || omp_is_reference (var))
&& lower_rec_simd_input_clauses (new_var, ctx, &sctx,
ivar, lvar))
{
+ if (omp_is_reference (var))
+ {
+ gcc_assert (TREE_CODE (new_var) == MEM_REF);
+ tree new_vard = TREE_OPERAND (new_var, 0);
+ gcc_assert (DECL_P (new_vard));
+ SET_DECL_VALUE_EXPR (new_vard,
+ build_fold_addr_expr (lvar));
+ DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
+ }
if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR)
{
tree iv = create_tmp_var (TREE_TYPE (new_var));
gimplify_and_add (x, &llist[0]);
x = lang_hooks.decls.omp_clause_dtor (c, ivar);
if (x)
+ gimplify_and_add (x, &llist[1]);
+ break;
+ }
+ if (omp_is_reference (var))
+ {
+ gcc_assert (TREE_CODE (new_var) == MEM_REF);
+ tree new_vard = TREE_OPERAND (new_var, 0);
+ gcc_assert (DECL_P (new_vard));
+ tree type = TREE_TYPE (TREE_TYPE (new_vard));
+ nx = TYPE_SIZE_UNIT (type);
+ if (TREE_CONSTANT (nx))
{
- gimple_seq tseq = NULL;
-
- dtor = x;
- gimplify_stmt (&dtor, &tseq);
- gimple_seq_add_seq (&llist[1], tseq);
+ nx = create_tmp_var_raw (type, get_name (var));
+ gimple_add_tmp_var (nx);
+ TREE_ADDRESSABLE (nx) = 1;
+ nx = build_fold_addr_expr_loc (clause_loc, nx);
+ nx = fold_convert_loc (clause_loc,
+ TREE_TYPE (new_vard), nx);
+ gimplify_assign (new_vard, nx, ilist);
}
- break;
}
}
x = lang_hooks.decls.omp_clause_copy_ctor
goto do_dtor;
case OMP_CLAUSE__LOOPTEMP_:
+ case OMP_CLAUSE__REDUCTEMP_:
gcc_assert (is_taskreg_ctx (ctx));
x = build_outer_var_ref (var, ctx);
x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x);
break;
case OMP_CLAUSE_REDUCTION:
+ case OMP_CLAUSE_IN_REDUCTION:
/* OpenACC reductions are initialized using the
GOACC_REDUCTION internal function. */
if (is_gimple_omp_oacc (ctx->stmt))
{
tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
gimple *tseq;
- x = build_outer_var_ref (var, ctx);
-
- if (omp_is_reference (var)
- && !useless_type_conversion_p (TREE_TYPE (placeholder),
- TREE_TYPE (x)))
- x = build_fold_addr_expr_loc (clause_loc, x);
+ tree ptype = TREE_TYPE (placeholder);
+ if (cond)
+ {
+ x = error_mark_node;
+ if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)
+ && !task_reduction_needs_orig_p)
+ x = var;
+ else if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c))
+ {
+ tree pptype = build_pointer_type (ptype);
+ if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
+ x = build4 (ARRAY_REF, ptr_type_node, tskred_avar,
+ size_int (task_reduction_cnt_full
+ + task_reduction_cntorig - 1),
+ NULL_TREE, NULL_TREE);
+ else
+ {
+ unsigned int idx
+ = *ctx->task_reduction_map->get (c);
+ x = task_reduction_read (ilist, tskred_temp,
+ pptype, 7 + 3 * idx);
+ }
+ x = fold_convert (pptype, x);
+ x = build_simple_mem_ref (x);
+ }
+ }
+ else
+ {
+ lower_private_allocate (var, new_var, allocator,
+ allocate_ptr, ilist, ctx, false,
+ NULL_TREE);
+ x = build_outer_var_ref (var, ctx);
+
+ if (omp_is_reference (var)
+ && !useless_type_conversion_p (ptype, TREE_TYPE (x)))
+ x = build_fold_addr_expr_loc (clause_loc, x);
+ }
SET_DECL_VALUE_EXPR (placeholder, x);
DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
tree new_vard = new_var;
new_vard = TREE_OPERAND (new_var, 0);
gcc_assert (DECL_P (new_vard));
}
+ tree rvar = NULL_TREE, *rvarp = NULL, rvar2 = NULL_TREE;
+ if (is_simd
+ && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
+ && OMP_CLAUSE_REDUCTION_INSCAN (c))
+ rvarp = &rvar;
if (is_simd
&& lower_rec_simd_input_clauses (new_var, ctx, &sctx,
- ivar, lvar))
+ ivar, lvar, rvarp,
+ &rvar2))
{
if (new_vard == new_var)
{
x = lang_hooks.decls.omp_clause_default_ctor
(c, unshare_expr (ivar),
build_outer_var_ref (var, ctx));
+ if (rvarp && ctx->for_simd_scan_phase)
+ {
+ if (x)
+ gimplify_and_add (x, &llist[0]);
+ x = lang_hooks.decls.omp_clause_dtor (c, ivar);
+ if (x)
+ gimplify_and_add (x, &llist[1]);
+ break;
+ }
+ else if (rvarp)
+ {
+ if (x)
+ {
+ gimplify_and_add (x, &llist[0]);
+
+ tree ivar2 = unshare_expr (lvar);
+ TREE_OPERAND (ivar2, 1) = sctx.idx;
+ x = lang_hooks.decls.omp_clause_default_ctor
+ (c, ivar2, build_outer_var_ref (var, ctx));
+ gimplify_and_add (x, &llist[0]);
+
+ if (rvar2)
+ {
+ x = lang_hooks.decls.omp_clause_default_ctor
+ (c, unshare_expr (rvar2),
+ build_outer_var_ref (var, ctx));
+ gimplify_and_add (x, &llist[0]);
+ }
+
+ /* For types that need construction, add another
+ private var which will be default constructed
+ and optionally initialized with
+ OMP_CLAUSE_REDUCTION_GIMPLE_INIT, as in the
+ loop we want to assign this value instead of
+ constructing and destructing it in each
+ iteration. */
+ tree nv = create_tmp_var_raw (TREE_TYPE (ivar));
+ gimple_add_tmp_var (nv);
+ ctx->cb.decl_map->put (TREE_OPERAND (rvar2
+ ? rvar2
+ : ivar, 0),
+ nv);
+ x = lang_hooks.decls.omp_clause_default_ctor
+ (c, nv, build_outer_var_ref (var, ctx));
+ gimplify_and_add (x, ilist);
+
+ if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
+ {
+ tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
+ x = DECL_VALUE_EXPR (new_vard);
+ tree vexpr = nv;
+ if (new_vard != new_var)
+ vexpr = build_fold_addr_expr (nv);
+ SET_DECL_VALUE_EXPR (new_vard, vexpr);
+ lower_omp (&tseq, ctx);
+ SET_DECL_VALUE_EXPR (new_vard, x);
+ gimple_seq_add_seq (ilist, tseq);
+ OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
+ }
+
+ x = lang_hooks.decls.omp_clause_dtor (c, nv);
+ if (x)
+ gimplify_and_add (x, dlist);
+ }
+
+ tree ref = build_outer_var_ref (var, ctx);
+ x = unshare_expr (ivar);
+ x = lang_hooks.decls.omp_clause_assign_op (c, x,
+ ref);
+ gimplify_and_add (x, &llist[0]);
+
+ ref = build_outer_var_ref (var, ctx);
+ x = lang_hooks.decls.omp_clause_assign_op (c, ref,
+ rvar);
+ gimplify_and_add (x, &llist[3]);
+
+ DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
+ if (new_vard == new_var)
+ SET_DECL_VALUE_EXPR (new_var, lvar);
+ else
+ SET_DECL_VALUE_EXPR (new_vard,
+ build_fold_addr_expr (lvar));
+
+ x = lang_hooks.decls.omp_clause_dtor (c, ivar);
+ if (x)
+ gimplify_and_add (x, &llist[1]);
+
+ tree ivar2 = unshare_expr (lvar);
+ TREE_OPERAND (ivar2, 1) = sctx.idx;
+ x = lang_hooks.decls.omp_clause_dtor (c, ivar2);
+ if (x)
+ gimplify_and_add (x, &llist[1]);
+
+ if (rvar2)
+ {
+ x = lang_hooks.decls.omp_clause_dtor (c, rvar2);
+ if (x)
+ gimplify_and_add (x, &llist[1]);
+ }
+ break;
+ }
if (x)
gimplify_and_add (x, &llist[0]);
if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
build_fold_addr_expr (lvar));
x = lang_hooks.decls.omp_clause_dtor (c, ivar);
if (x)
- {
- tseq = NULL;
- dtor = x;
- gimplify_stmt (&dtor, &tseq);
- gimple_seq_add_seq (&llist[1], tseq);
- }
+ gimplify_and_add (x, &llist[1]);
break;
}
/* If this is a reference to constant size reduction var
initialization now. */
else if (omp_is_reference (var) && is_simd)
handle_simd_reference (clause_loc, new_vard, ilist);
+
+ tree lab2 = NULL_TREE;
+ if (cond)
+ {
+ gimple *g;
+ if (!is_parallel_ctx (ctx))
+ {
+ tree condv = create_tmp_var (boolean_type_node);
+ tree m = build_simple_mem_ref (cond);
+ g = gimple_build_assign (condv, m);
+ gimple_seq_add_stmt (ilist, g);
+ tree lab1
+ = create_artificial_label (UNKNOWN_LOCATION);
+ lab2 = create_artificial_label (UNKNOWN_LOCATION);
+ g = gimple_build_cond (NE_EXPR, condv,
+ boolean_false_node,
+ lab2, lab1);
+ gimple_seq_add_stmt (ilist, g);
+ gimple_seq_add_stmt (ilist,
+ gimple_build_label (lab1));
+ }
+ g = gimple_build_assign (build_simple_mem_ref (cond),
+ boolean_true_node);
+ gimple_seq_add_stmt (ilist, g);
+ }
x = lang_hooks.decls.omp_clause_default_ctor
(c, unshare_expr (new_var),
- build_outer_var_ref (var, ctx));
+ cond ? NULL_TREE
+ : build_outer_var_ref (var, ctx));
if (x)
gimplify_and_add (x, ilist);
+
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
+ && OMP_CLAUSE_REDUCTION_INSCAN (c))
+ {
+ if (ctx->for_simd_scan_phase)
+ goto do_dtor;
+ if (x || (!is_simd
+ && OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)))
+ {
+ tree nv = create_tmp_var_raw (TREE_TYPE (new_var));
+ gimple_add_tmp_var (nv);
+ ctx->cb.decl_map->put (new_vard, nv);
+ x = lang_hooks.decls.omp_clause_default_ctor
+ (c, nv, build_outer_var_ref (var, ctx));
+ if (x)
+ gimplify_and_add (x, ilist);
+ if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
+ {
+ tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
+ tree vexpr = nv;
+ if (new_vard != new_var)
+ vexpr = build_fold_addr_expr (nv);
+ SET_DECL_VALUE_EXPR (new_vard, vexpr);
+ DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
+ lower_omp (&tseq, ctx);
+ SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
+ DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
+ gimple_seq_add_seq (ilist, tseq);
+ }
+ OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
+ if (is_simd && ctx->scan_exclusive)
+ {
+ tree nv2
+ = create_tmp_var_raw (TREE_TYPE (new_var));
+ gimple_add_tmp_var (nv2);
+ ctx->cb.decl_map->put (nv, nv2);
+ x = lang_hooks.decls.omp_clause_default_ctor
+ (c, nv2, build_outer_var_ref (var, ctx));
+ gimplify_and_add (x, ilist);
+ x = lang_hooks.decls.omp_clause_dtor (c, nv2);
+ if (x)
+ gimplify_and_add (x, dlist);
+ }
+ x = lang_hooks.decls.omp_clause_dtor (c, nv);
+ if (x)
+ gimplify_and_add (x, dlist);
+ }
+ else if (is_simd
+ && ctx->scan_exclusive
+ && TREE_ADDRESSABLE (TREE_TYPE (new_var)))
+ {
+ tree nv2 = create_tmp_var_raw (TREE_TYPE (new_var));
+ gimple_add_tmp_var (nv2);
+ ctx->cb.decl_map->put (new_vard, nv2);
+ x = lang_hooks.decls.omp_clause_dtor (c, nv2);
+ if (x)
+ gimplify_and_add (x, dlist);
+ }
+ DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
+ goto do_dtor;
+ }
+
if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
{
tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
}
DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
+ if (cond)
+ {
+ if (lab2)
+ gimple_seq_add_stmt (ilist, gimple_build_label (lab2));
+ break;
+ }
goto do_dtor;
}
else
gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE);
enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
+ if (cond)
+ {
+ gimple *g;
+ tree lab2 = NULL_TREE;
+ /* GOMP_taskgroup_reduction_register memsets the whole
+ array to zero. If the initializer is zero, we don't
+ need to initialize it again, just mark it as ever
+ used unconditionally, i.e. cond = true. */
+ if (initializer_zerop (x))
+ {
+ g = gimple_build_assign (build_simple_mem_ref (cond),
+ boolean_true_node);
+ gimple_seq_add_stmt (ilist, g);
+ break;
+ }
+
+ /* Otherwise, emit
+ if (!cond) { cond = true; new_var = x; } */
+ if (!is_parallel_ctx (ctx))
+ {
+ tree condv = create_tmp_var (boolean_type_node);
+ tree m = build_simple_mem_ref (cond);
+ g = gimple_build_assign (condv, m);
+ gimple_seq_add_stmt (ilist, g);
+ tree lab1
+ = create_artificial_label (UNKNOWN_LOCATION);
+ lab2 = create_artificial_label (UNKNOWN_LOCATION);
+ g = gimple_build_cond (NE_EXPR, condv,
+ boolean_false_node,
+ lab2, lab1);
+ gimple_seq_add_stmt (ilist, g);
+ gimple_seq_add_stmt (ilist,
+ gimple_build_label (lab1));
+ }
+ g = gimple_build_assign (build_simple_mem_ref (cond),
+ boolean_true_node);
+ gimple_seq_add_stmt (ilist, g);
+ gimplify_assign (new_var, x, ilist);
+ if (lab2)
+ gimple_seq_add_stmt (ilist, gimple_build_label (lab2));
+ break;
+ }
+
/* reduction(-:var) sums up the partial results, so it
acts identically to reduction(+:var). */
if (code == MINUS_EXPR)
new_vard = TREE_OPERAND (new_var, 0);
gcc_assert (DECL_P (new_vard));
}
+ tree rvar = NULL_TREE, *rvarp = NULL, rvar2 = NULL_TREE;
+ if (is_simd
+ && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
+ && OMP_CLAUSE_REDUCTION_INSCAN (c))
+ rvarp = &rvar;
if (is_simd
&& lower_rec_simd_input_clauses (new_var, ctx, &sctx,
- ivar, lvar))
+ ivar, lvar, rvarp,
+ &rvar2))
{
+ if (new_vard != new_var)
+ {
+ SET_DECL_VALUE_EXPR (new_vard,
+ build_fold_addr_expr (lvar));
+ DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
+ }
+
tree ref = build_outer_var_ref (var, ctx);
+ if (rvarp)
+ {
+ if (ctx->for_simd_scan_phase)
+ break;
+ gimplify_assign (ivar, ref, &llist[0]);
+ ref = build_outer_var_ref (var, ctx);
+ gimplify_assign (ref, rvar, &llist[3]);
+ break;
+ }
+
gimplify_assign (unshare_expr (ivar), x, &llist[0]);
if (sctx.is_simt)
ref = build_outer_var_ref (var, ctx);
gimplify_assign (ref, x, &llist[1]);
- if (new_vard != new_var)
- {
- SET_DECL_VALUE_EXPR (new_vard,
- build_fold_addr_expr (lvar));
- DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
- }
}
else
{
+ lower_private_allocate (var, new_var, allocator,
+ allocate_ptr, ilist, ctx,
+ false, NULL_TREE);
if (omp_is_reference (var) && is_simd)
handle_simd_reference (clause_loc, new_vard, ilist);
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
+ && OMP_CLAUSE_REDUCTION_INSCAN (c))
+ break;
gimplify_assign (new_var, x, ilist);
if (is_simd)
{
ref = build_outer_var_ref (var, ctx);
gimplify_assign (ref, x, dlist);
}
+ if (allocator)
+ goto do_dtor;
}
}
break;
}
}
}
+ if (tskred_avar)
+ {
+ tree clobber = build_clobber (TREE_TYPE (tskred_avar));
+ gimple_seq_add_stmt (ilist, gimple_build_assign (tskred_avar, clobber));
+ }
if (known_eq (sctx.max_vf, 1U))
- sctx.is_simt = false;
+ {
+ sctx.is_simt = false;
+ if (ctx->lastprivate_conditional_map)
+ {
+ if (gimple_omp_for_combined_into_p (ctx->stmt))
+ {
+ /* Signal to lower_omp_1 that it should use parent context. */
+ ctx->combined_into_simd_safelen1 = true;
+ for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c))
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
+ && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
+ {
+ tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
+ omp_context *outer = ctx->outer;
+ if (gimple_code (outer->stmt) == GIMPLE_OMP_SCAN)
+ outer = outer->outer;
+ tree *v = ctx->lastprivate_conditional_map->get (o);
+ tree po = lookup_decl (OMP_CLAUSE_DECL (c), outer);
+ tree *pv = outer->lastprivate_conditional_map->get (po);
+ *v = *pv;
+ }
+ }
+ else
+ {
+ /* When not vectorized, treat lastprivate(conditional:) like
+ normal lastprivate, as there will be just one simd lane
+ writing the privatized variable. */
+ delete ctx->lastprivate_conditional_map;
+ ctx->lastprivate_conditional_map = NULL;
+ }
+ }
+ }
+
+ if (nonconst_simd_if)
+ {
+ if (sctx.lane == NULL_TREE)
+ {
+ sctx.idx = create_tmp_var (unsigned_type_node);
+ sctx.lane = create_tmp_var (unsigned_type_node);
+ }
+ /* FIXME: For now. */
+ sctx.is_simt = false;
+ }
if (sctx.lane || sctx.is_simt)
{
}
if (sctx.lane)
{
- gimple *g
- = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 1, uid);
+ gimple *g = gimple_build_call_internal (IFN_GOMP_SIMD_LANE,
+ 2 + (nonconst_simd_if != NULL),
+ uid, integer_zero_node,
+ nonconst_simd_if);
gimple_call_set_lhs (g, sctx.lane);
gimple_stmt_iterator gsi = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt));
gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT);
g = gimple_build_assign (sctx.lane, INTEGER_CST,
build_int_cst (unsigned_type_node, 0));
gimple_seq_add_stmt (ilist, g);
+ if (sctx.lastlane)
+ {
+ g = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE,
+ 2, uid, sctx.lane);
+ gimple_call_set_lhs (g, sctx.lastlane);
+ gimple_seq_add_stmt (dlist, g);
+ gimple_seq_add_seq (dlist, llist[3]);
+ }
/* Emit reductions across SIMT lanes in log_2(simt_vf) steps. */
if (llist[2])
{
lastprivate clauses we need to ensure the lastprivate copying
happens after firstprivate copying in all threads. And similarly
for UDRs if initializer expression refers to omp_orig. */
- if (copyin_by_ref || lastprivate_firstprivate || reduction_omp_orig_ref)
+ if (copyin_by_ref || lastprivate_firstprivate
+ || (reduction_omp_orig_ref
+ && !ctx->scan_inclusive
+ && !ctx->scan_exclusive))
{
/* Don't add any barrier for #pragma omp simd or
#pragma omp distribute. */
- if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
- || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR)
+ if (!is_task_ctx (ctx)
+ && (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR
+ || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR))
gimple_seq_add_stmt (ilist, omp_build_barrier (NULL_TREE));
}
}
}
-
-/* Generate code to implement the LASTPRIVATE clauses. This is used for
- both parallel and workshare constructs. PREDICATE may be NULL if it's
- always true. */
+/* Create temporary variables for lastprivate(conditional:) implementation
+ in context CTX with CLAUSES. */
static void
-lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *stmt_list,
- omp_context *ctx)
+lower_lastprivate_conditional_clauses (tree *clauses, omp_context *ctx)
{
- tree x, c, label = NULL, orig_clauses = clauses;
- bool par_clauses = false;
- tree simduid = NULL, lastlane = NULL, simtcond = NULL, simtlast = NULL;
-
- /* Early exit if there are no lastprivate or linear clauses. */
+ tree iter_type = NULL_TREE;
+ tree cond_ptr = NULL_TREE;
+ tree iter_var = NULL_TREE;
+ bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
+ && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD);
+ tree next = *clauses;
+ for (tree c = *clauses; c; c = OMP_CLAUSE_CHAIN (c))
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
+ && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c))
+ {
+ if (is_simd)
+ {
+ tree cc = omp_find_clause (next, OMP_CLAUSE__CONDTEMP_);
+ gcc_assert (cc);
+ if (iter_type == NULL_TREE)
+ {
+ iter_type = TREE_TYPE (OMP_CLAUSE_DECL (cc));
+ iter_var = create_tmp_var_raw (iter_type);
+ DECL_CONTEXT (iter_var) = current_function_decl;
+ DECL_SEEN_IN_BIND_EXPR_P (iter_var) = 1;
+ DECL_CHAIN (iter_var) = ctx->block_vars;
+ ctx->block_vars = iter_var;
+ tree c3
+ = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_);
+ OMP_CLAUSE__CONDTEMP__ITER (c3) = 1;
+ OMP_CLAUSE_DECL (c3) = iter_var;
+ OMP_CLAUSE_CHAIN (c3) = *clauses;
+ *clauses = c3;
+ ctx->lastprivate_conditional_map = new hash_map<tree, tree>;
+ }
+ next = OMP_CLAUSE_CHAIN (cc);
+ tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
+ tree v = lookup_decl (OMP_CLAUSE_DECL (cc), ctx);
+ ctx->lastprivate_conditional_map->put (o, v);
+ continue;
+ }
+ if (iter_type == NULL)
+ {
+ if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR)
+ {
+ struct omp_for_data fd;
+ omp_extract_for_data (as_a <gomp_for *> (ctx->stmt), &fd,
+ NULL);
+ iter_type = unsigned_type_for (fd.iter_type);
+ }
+ else if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS)
+ iter_type = unsigned_type_node;
+ tree c2 = omp_find_clause (*clauses, OMP_CLAUSE__CONDTEMP_);
+ if (c2)
+ {
+ cond_ptr
+ = lookup_decl_in_outer_ctx (OMP_CLAUSE_DECL (c2), ctx);
+ OMP_CLAUSE_DECL (c2) = cond_ptr;
+ }
+ else
+ {
+ cond_ptr = create_tmp_var_raw (build_pointer_type (iter_type));
+ DECL_CONTEXT (cond_ptr) = current_function_decl;
+ DECL_SEEN_IN_BIND_EXPR_P (cond_ptr) = 1;
+ DECL_CHAIN (cond_ptr) = ctx->block_vars;
+ ctx->block_vars = cond_ptr;
+ c2 = build_omp_clause (UNKNOWN_LOCATION,
+ OMP_CLAUSE__CONDTEMP_);
+ OMP_CLAUSE_DECL (c2) = cond_ptr;
+ OMP_CLAUSE_CHAIN (c2) = *clauses;
+ *clauses = c2;
+ }
+ iter_var = create_tmp_var_raw (iter_type);
+ DECL_CONTEXT (iter_var) = current_function_decl;
+ DECL_SEEN_IN_BIND_EXPR_P (iter_var) = 1;
+ DECL_CHAIN (iter_var) = ctx->block_vars;
+ ctx->block_vars = iter_var;
+ tree c3
+ = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__CONDTEMP_);
+ OMP_CLAUSE__CONDTEMP__ITER (c3) = 1;
+ OMP_CLAUSE_DECL (c3) = iter_var;
+ OMP_CLAUSE_CHAIN (c3) = OMP_CLAUSE_CHAIN (c2);
+ OMP_CLAUSE_CHAIN (c2) = c3;
+ ctx->lastprivate_conditional_map = new hash_map<tree, tree>;
+ }
+ tree v = create_tmp_var_raw (iter_type);
+ DECL_CONTEXT (v) = current_function_decl;
+ DECL_SEEN_IN_BIND_EXPR_P (v) = 1;
+ DECL_CHAIN (v) = ctx->block_vars;
+ ctx->block_vars = v;
+ tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
+ ctx->lastprivate_conditional_map->put (o, v);
+ }
+}
+
+
+/* Generate code to implement the LASTPRIVATE clauses. This is used for
+ both parallel and workshare constructs. PREDICATE may be NULL if it's
+ always true. BODY_P is the sequence to insert early initialization
+ if needed, STMT_LIST is where the non-conditional lastprivate handling
+ goes into and CSTMT_LIST is a sequence that needs to be run in a critical
+ section. */
+
+static void
+lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *body_p,
+ gimple_seq *stmt_list, gimple_seq *cstmt_list,
+ omp_context *ctx)
+{
+ tree x, c, label = NULL, orig_clauses = clauses;
+ bool par_clauses = false;
+ tree simduid = NULL, lastlane = NULL, simtcond = NULL, simtlast = NULL;
+ unsigned HOST_WIDE_INT conditional_off = 0;
+ gimple_seq post_stmt_list = NULL;
+
+ /* Early exit if there are no lastprivate or linear clauses. */
for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses))
if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE
|| (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR
bool maybe_simt = false;
if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
- && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
+ && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
{
maybe_simt = omp_find_clause (orig_clauses, OMP_CLAUSE__SIMT_);
simduid = omp_find_clause (orig_clauses, OMP_CLAUSE__SIMDUID_);
gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true));
}
+ tree cond_ptr = NULL_TREE;
for (c = clauses; c ;)
{
tree var, new_var;
location_t clause_loc = OMP_CLAUSE_LOCATION (c);
+ gimple_seq *this_stmt_list = stmt_list;
+ tree lab2 = NULL_TREE;
+
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
+ && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)
+ && ctx->lastprivate_conditional_map
+ && !ctx->combined_into_simd_safelen1)
+ {
+ gcc_assert (body_p);
+ if (simduid)
+ goto next;
+ if (cond_ptr == NULL_TREE)
+ {
+ cond_ptr = omp_find_clause (orig_clauses, OMP_CLAUSE__CONDTEMP_);
+ cond_ptr = OMP_CLAUSE_DECL (cond_ptr);
+ }
+ tree type = TREE_TYPE (TREE_TYPE (cond_ptr));
+ tree o = lookup_decl (OMP_CLAUSE_DECL (c), ctx);
+ tree v = *ctx->lastprivate_conditional_map->get (o);
+ gimplify_assign (v, build_zero_cst (type), body_p);
+ this_stmt_list = cstmt_list;
+ tree mem;
+ if (POINTER_TYPE_P (TREE_TYPE (cond_ptr)))
+ {
+ mem = build2 (MEM_REF, type, cond_ptr,
+ build_int_cst (TREE_TYPE (cond_ptr),
+ conditional_off));
+ conditional_off += tree_to_uhwi (TYPE_SIZE_UNIT (type));
+ }
+ else
+ mem = build4 (ARRAY_REF, type, cond_ptr,
+ size_int (conditional_off++), NULL_TREE, NULL_TREE);
+ tree mem2 = copy_node (mem);
+ gimple_seq seq = NULL;
+ mem = force_gimple_operand (mem, &seq, true, NULL_TREE);
+ gimple_seq_add_seq (this_stmt_list, seq);
+ tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
+ lab2 = create_artificial_label (UNKNOWN_LOCATION);
+ gimple *g = gimple_build_cond (GT_EXPR, v, mem, lab1, lab2);
+ gimple_seq_add_stmt (this_stmt_list, g);
+ gimple_seq_add_stmt (this_stmt_list, gimple_build_label (lab1));
+ gimplify_assign (mem2, v, this_stmt_list);
+ }
+ else if (predicate
+ && ctx->combined_into_simd_safelen1
+ && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
+ && OMP_CLAUSE_LASTPRIVATE_CONDITIONAL (c)
+ && ctx->lastprivate_conditional_map)
+ this_stmt_list = &post_stmt_list;
if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
|| (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR
2, simduid,
TREE_OPERAND (val, 1));
gimple_call_set_lhs (g, lastlane);
- gimple_seq_add_stmt (stmt_list, g);
+ gimple_seq_add_stmt (this_stmt_list, g);
}
new_var = build4 (ARRAY_REF, TREE_TYPE (val),
TREE_OPERAND (val, 0), lastlane,
NULL_TREE, NULL_TREE);
+ TREE_THIS_NOTRAP (new_var) = 1;
}
}
else if (maybe_simt)
gcall *g = gimple_build_call_internal
(IFN_GOMP_SIMT_LAST_LANE, 1, simtcond);
gimple_call_set_lhs (g, simtlast);
- gimple_seq_add_stmt (stmt_list, g);
+ gimple_seq_add_stmt (this_stmt_list, g);
}
x = build_call_expr_internal_loc
(UNKNOWN_LOCATION, IFN_GOMP_SIMT_XCHG_IDX,
TREE_TYPE (val), 2, val, simtlast);
new_var = unshare_expr (new_var);
- gimplify_assign (new_var, x, stmt_list);
+ gimplify_assign (new_var, x, this_stmt_list);
new_var = unshare_expr (new_var);
}
&& OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c))
{
lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx);
- gimple_seq_add_seq (stmt_list,
+ gimple_seq_add_seq (this_stmt_list,
OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c));
OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL;
}
&& OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c))
{
lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx);
- gimple_seq_add_seq (stmt_list,
+ gimple_seq_add_seq (this_stmt_list,
OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c));
OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) = NULL;
}
x = NULL_TREE;
if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE
- && OMP_CLAUSE_LASTPRIVATE_TASKLOOP_IV (c))
+ && OMP_CLAUSE_LASTPRIVATE_LOOP_IV (c)
+ && is_taskloop_ctx (ctx))
{
- gcc_checking_assert (is_taskloop_ctx (ctx));
tree ovar = maybe_lookup_decl_in_outer_ctx (var,
ctx->outer->outer);
if (is_global_var (ovar))
if (omp_is_reference (var))
new_var = build_simple_mem_ref_loc (clause_loc, new_var);
x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var);
- gimplify_and_add (x, stmt_list);
+ gimplify_and_add (x, this_stmt_list);
+
+ if (lab2)
+ gimple_seq_add_stmt (this_stmt_list, gimple_build_label (lab2));
}
+
+ next:
c = OMP_CLAUSE_CHAIN (c);
if (c == NULL && !par_clauses)
{
if (label)
gimple_seq_add_stmt (stmt_list, gimple_build_label (label));
+ gimple_seq_add_seq (stmt_list, post_stmt_list);
}
/* Lower the OpenACC reductions of CLAUSES for compute axis LEVEL
for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
{
+ /* No 'reduction' clauses on OpenACC 'kernels'. */
+ gcc_checking_assert (!is_oacc_kernels (ctx));
+ /* Likewise, on OpenACC 'kernels' decomposed parts. */
+ gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
+
tree orig = OMP_CLAUSE_DECL (c);
tree var = maybe_lookup_decl (orig, ctx);
tree ref_to_res = NULL_TREE;
break;
case GIMPLE_OMP_TARGET:
- if (gimple_omp_target_kind (probe->stmt)
- != GF_OMP_TARGET_KIND_OACC_PARALLEL)
+ /* No 'reduction' clauses inside OpenACC 'kernels'
+ regions. */
+ gcc_checking_assert (!is_oacc_kernels (probe));
+
+ if (!is_gimple_omp_offloaded (probe->stmt))
goto do_lookup;
cls = gimple_omp_target_clauses (probe->stmt);
gimple_seq_add_seq (join_seq, after_join);
}
-/* Generate code to implement the REDUCTION clauses. */
+/* Generate code to implement the REDUCTION clauses, append it
+ to STMT_SEQP. CLIST if non-NULL is a pointer to a sequence
+ that should be emitted also inside of the critical section,
+ in that case clear *CLIST afterwards, otherwise leave it as is
+ and let the caller emit it itself. */
static void
-lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp, omp_context *ctx)
+lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp,
+ gimple_seq *clist, omp_context *ctx)
{
gimple_seq sub_seq = NULL;
gimple *stmt;
/* SIMD reductions are handled in lower_rec_input_clauses. */
if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR
- && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD)
+ && gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_SIMD)
+ return;
+
+ /* inscan reductions are handled elsewhere. */
+ if (ctx->scan_inclusive || ctx->scan_exclusive)
return;
/* First see if there is exactly one reduction clause. Use OMP_ATOMIC
update in that case, otherwise use a lock. */
for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c))
- if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION)
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
+ && !OMP_CLAUSE_REDUCTION_TASK (c))
{
if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)
|| TREE_CODE (OMP_CLAUSE_DECL (c)) == MEM_REF)
enum tree_code code;
location_t clause_loc = OMP_CLAUSE_LOCATION (c);
- if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION)
+ if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
+ || OMP_CLAUSE_REDUCTION_TASK (c))
continue;
enum omp_clause_code ccode = OMP_CLAUSE_REDUCTION;
ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr);
x = fold_build2_loc (clause_loc, code, TREE_TYPE (ref), ref, new_var);
x = build2 (OMP_ATOMIC, void_type_node, addr, x);
+ OMP_ATOMIC_MEMORY_ORDER (x) = OMP_MEMORY_ORDER_RELAXED;
gimplify_and_add (x, stmt_seqp);
return;
}
tree d = OMP_CLAUSE_DECL (c);
tree type = TREE_TYPE (d);
tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
- tree i = create_tmp_var (TREE_TYPE (v), NULL);
+ tree i = create_tmp_var (TREE_TYPE (v));
tree ptype = build_pointer_type (TREE_TYPE (type));
tree bias = TREE_OPERAND (d, 1);
d = TREE_OPERAND (d, 0);
}
new_var = fold_convert_loc (clause_loc, ptype, new_var);
ref = fold_convert_loc (clause_loc, ptype, ref);
- tree m = create_tmp_var (ptype, NULL);
+ tree m = create_tmp_var (ptype);
gimplify_assign (m, new_var, stmt_seqp);
new_var = m;
- m = create_tmp_var (ptype, NULL);
+ m = create_tmp_var (ptype);
gimplify_assign (m, ref, stmt_seqp);
ref = m;
gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), stmt_seqp);
gimple_seq_add_seq (stmt_seqp, sub_seq);
+ if (clist)
+ {
+ gimple_seq_add_seq (stmt_seqp, *clist);
+ *clist = NULL;
+ }
+
stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END),
0);
gimple_seq_add_stmt (stmt_seqp, stmt);
case OMP_CLAUSE_FIRSTPRIVATE:
case OMP_CLAUSE_COPYIN:
case OMP_CLAUSE_LASTPRIVATE:
+ case OMP_CLAUSE_IN_REDUCTION:
+ case OMP_CLAUSE__REDUCTEMP_:
+ break;
case OMP_CLAUSE_REDUCTION:
+ if (is_task_ctx (ctx) || OMP_CLAUSE_REDUCTION_TASK (c))
+ continue;
break;
case OMP_CLAUSE_SHARED:
if (OMP_CLAUSE_SHARED_FIRSTPRIVATE (c))
}
val = OMP_CLAUSE_DECL (c);
- if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
+ if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
+ || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_IN_REDUCTION)
&& TREE_CODE (val) == MEM_REF)
{
val = TREE_OPERAND (val, 0);
var = lookup_decl_in_outer_ctx (val, ctx_for_o);
if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN
- && is_global_var (var))
+ && is_global_var (var)
+ && (val == OMP_CLAUSE_DECL (c)
+ || !is_task_ctx (ctx)
+ || (TREE_CODE (TREE_TYPE (val)) != POINTER_TYPE
+ && (TREE_CODE (TREE_TYPE (val)) != REFERENCE_TYPE
+ || (TREE_CODE (TREE_TYPE (TREE_TYPE (val)))
+ != POINTER_TYPE)))))
continue;
t = omp_member_access_dummy_var (var);
continue;
}
- if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
+ if (((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION
+ && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IN_REDUCTION)
|| val == OMP_CLAUSE_DECL (c))
&& is_variable_sized (val))
continue;
case OMP_CLAUSE_PRIVATE:
case OMP_CLAUSE_COPYIN:
case OMP_CLAUSE__LOOPTEMP_:
+ case OMP_CLAUSE__REDUCTEMP_:
do_in = true;
break;
break;
case OMP_CLAUSE_REDUCTION:
+ case OMP_CLAUSE_IN_REDUCTION:
do_in = true;
if (val == OMP_CLAUSE_DECL (c))
- do_out = !(by_ref || omp_is_reference (val));
+ {
+ if (is_task_ctx (ctx))
+ by_ref = use_pointer_for_field (val, ctx);
+ else
+ do_out = !(by_ref || omp_is_reference (val));
+ }
else
by_ref = TREE_CODE (TREE_TYPE (val)) == ARRAY_TYPE;
break;
continue;
nvar = maybe_lookup_decl (ovar, ctx);
- if (!nvar || !DECL_HAS_VALUE_EXPR_P (nvar))
+ if (!nvar
+ || !DECL_HAS_VALUE_EXPR_P (nvar)
+ || (ctx->allocate_map
+ && ctx->allocate_map->get (ovar)))
continue;
/* If CTX is a nested parallel directive. Find the immediately
if (use_pointer_for_field (ovar, ctx))
{
x = build_sender_ref (ovar, ctx);
- var = build_fold_addr_expr (var);
+ if (TREE_CODE (TREE_TYPE (f)) == ARRAY_TYPE
+ && TREE_TYPE (f) == TREE_TYPE (ovar))
+ {
+ gcc_assert (is_parallel_ctx (ctx)
+ && DECL_ARTIFICIAL (ovar));
+ /* _condtemp_ clause. */
+ var = build_constructor (TREE_TYPE (x), NULL);
+ }
+ else
+ var = build_fold_addr_expr (var);
gimplify_assign (x, var, ilist);
}
else
tag |= OLF_GANG_STATIC;
}
- /* In a parallel region, loops are implicitly INDEPENDENT. */
omp_context *tgt = enclosing_target_ctx (ctx);
- if (!tgt || is_oacc_parallel (tgt))
+ if (!tgt || is_oacc_parallel_or_serial (tgt))
+ ;
+ else if (is_oacc_kernels (tgt))
+ /* Not using this loops handling inside OpenACC 'kernels' regions. */
+ gcc_unreachable ();
+ else if (is_oacc_kernels_decomposed_part (tgt))
+ ;
+ else
+ gcc_unreachable ();
+
+ /* In a parallel region, loops are implicitly INDEPENDENT. */
+ if (!tgt || is_oacc_parallel_or_serial (tgt))
tag |= OLF_INDEPENDENT;
+ /* Loops inside OpenACC 'kernels' decomposed parts' regions are expected to
+ have an explicit 'seq' or 'independent' clause, and no 'auto' clause. */
+ if (tgt && is_oacc_kernels_decomposed_part (tgt))
+ {
+ gcc_assert (tag & (OLF_SEQ | OLF_INDEPENDENT));
+ gcc_assert (!(tag & OLF_AUTO));
+ }
+
if (tag & OLF_TILE)
/* Tiling could use all 3 levels. */
levels = 3;
cancellation in the implicit barrier. */
static void
-maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple_seq *body)
+maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple *omp_return,
+ gimple_seq *body)
{
- gimple *omp_return = gimple_seq_last_stmt (*body);
gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN);
if (gimple_omp_return_nowait_p (omp_return))
return;
- if (ctx->outer
- && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_PARALLEL
- && ctx->outer->cancellable)
- {
- tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_CANCEL);
- tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl));
- tree lhs = create_tmp_var (c_bool_type);
- gimple_omp_return_set_lhs (omp_return, lhs);
- tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
- gimple *g = gimple_build_cond (NE_EXPR, lhs,
- fold_convert (c_bool_type,
- boolean_false_node),
- ctx->outer->cancel_label, fallthru_label);
- gimple_seq_add_stmt (body, g);
- gimple_seq_add_stmt (body, gimple_build_label (fallthru_label));
+ for (omp_context *outer = ctx->outer; outer; outer = outer->outer)
+ if (gimple_code (outer->stmt) == GIMPLE_OMP_PARALLEL
+ && outer->cancellable)
+ {
+ tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_CANCEL);
+ tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl));
+ tree lhs = create_tmp_var (c_bool_type);
+ gimple_omp_return_set_lhs (omp_return, lhs);
+ tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION);
+ gimple *g = gimple_build_cond (NE_EXPR, lhs,
+ fold_convert (c_bool_type,
+ boolean_false_node),
+ outer->cancel_label, fallthru_label);
+ gimple_seq_add_stmt (body, g);
+ gimple_seq_add_stmt (body, gimple_build_label (fallthru_label));
+ }
+ else if (gimple_code (outer->stmt) != GIMPLE_OMP_TASKGROUP)
+ return;
+}
+
+/* Find the first task_reduction or reduction clause or return NULL
+ if there are none. */
+
+static inline tree
+omp_task_reductions_find_first (tree clauses, enum tree_code code,
+ enum omp_clause_code ccode)
+{
+ while (1)
+ {
+ clauses = omp_find_clause (clauses, ccode);
+ if (clauses == NULL_TREE)
+ return NULL_TREE;
+ if (ccode != OMP_CLAUSE_REDUCTION
+ || code == OMP_TASKLOOP
+ || OMP_CLAUSE_REDUCTION_TASK (clauses))
+ return clauses;
+ clauses = OMP_CLAUSE_CHAIN (clauses);
}
}
+static void lower_omp_task_reductions (omp_context *, enum tree_code, tree,
+ gimple_seq *, gimple_seq *);
+
/* Lower the OpenMP sections directive in the current statement in GSI_P.
CTX is the enclosing OMP context for the current statement. */
gomp_sections *stmt;
gimple *t;
gbind *new_stmt, *bind;
- gimple_seq ilist, dlist, olist, new_body;
+ gimple_seq ilist, dlist, olist, tred_dlist = NULL, clist = NULL, new_body;
stmt = as_a <gomp_sections *> (gsi_stmt (*gsi_p));
dlist = NULL;
ilist = NULL;
+
+ tree rclauses
+ = omp_task_reductions_find_first (gimple_omp_sections_clauses (stmt),
+ OMP_SECTIONS, OMP_CLAUSE_REDUCTION);
+ tree rtmp = NULL_TREE;
+ if (rclauses)
+ {
+ tree type = build_pointer_type (pointer_sized_int_node);
+ tree temp = create_tmp_var (type);
+ tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
+ OMP_CLAUSE_DECL (c) = temp;
+ OMP_CLAUSE_CHAIN (c) = gimple_omp_sections_clauses (stmt);
+ gimple_omp_sections_set_clauses (stmt, c);
+ lower_omp_task_reductions (ctx, OMP_SECTIONS,
+ gimple_omp_sections_clauses (stmt),
+ &ilist, &tred_dlist);
+ rclauses = c;
+ rtmp = make_ssa_name (type);
+ gimple_seq_add_stmt (&ilist, gimple_build_assign (rtmp, temp));
+ }
+
+ tree *clauses_ptr = gimple_omp_sections_clauses_ptr (stmt);
+ lower_lastprivate_conditional_clauses (clauses_ptr, ctx);
+
lower_rec_input_clauses (gimple_omp_sections_clauses (stmt),
&ilist, &dlist, ctx, NULL);
+ control = create_tmp_var (unsigned_type_node, ".section");
+ gimple_omp_sections_set_control (stmt, control);
+
new_body = gimple_omp_body (stmt);
gimple_omp_set_body (stmt, NULL);
tgsi = gsi_start (new_body);
{
gimple_seq l = NULL;
lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL,
- &l, ctx);
+ &ilist, &l, &clist, ctx);
gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING);
gimple_omp_section_set_last (sec_start);
}
bind = gimple_build_bind (NULL, new_body, block);
olist = NULL;
- lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist, ctx);
+ lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist,
+ &clist, ctx);
+ if (clist)
+ {
+ tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
+ gcall *g = gimple_build_call (fndecl, 0);
+ gimple_seq_add_stmt (&olist, g);
+ gimple_seq_add_seq (&olist, clist);
+ fndecl = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
+ g = gimple_build_call (fndecl, 0);
+ gimple_seq_add_stmt (&olist, g);
+ }
block = make_node (BLOCK);
new_stmt = gimple_build_bind (NULL, NULL, block);
gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ());
gimple_seq_add_stmt (&new_body, bind);
- control = create_tmp_var (unsigned_type_node, ".section");
t = gimple_build_omp_continue (control, control);
- gimple_omp_sections_set_control (stmt, control);
gimple_seq_add_stmt (&new_body, t);
gimple_seq_add_seq (&new_body, olist);
OMP_CLAUSE_NOWAIT) != NULL_TREE;
t = gimple_build_omp_return (nowait);
gimple_seq_add_stmt (&new_body, t);
- maybe_add_implicit_barrier_cancel (ctx, &new_body);
+ gimple_seq_add_seq (&new_body, tred_dlist);
+ maybe_add_implicit_barrier_cancel (ctx, t, &new_body);
+
+ if (rclauses)
+ OMP_CLAUSE_DECL (rclauses) = rtmp;
gimple_bind_set_body (new_stmt, new_body);
}
OMP_CLAUSE_NOWAIT) != NULL_TREE;
gimple *g = gimple_build_omp_return (nowait);
gimple_seq_add_stmt (&bind_body_tail, g);
- maybe_add_implicit_barrier_cancel (ctx, &bind_body_tail);
+ maybe_add_implicit_barrier_cancel (ctx, g, &bind_body_tail);
if (ctx->record_type)
{
gimple_stmt_iterator gsi = gsi_start (bind_body_tail);
- tree clobber = build_constructor (ctx->record_type, NULL);
- TREE_THIS_VOLATILE (clobber) = 1;
+ tree clobber = build_clobber (ctx->record_type);
gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl,
clobber), GSI_SAME_STMT);
}
BLOCK_VARS (block) = ctx->block_vars;
}
+/* Helper function for lower_omp_task_reductions. For a specific PASS
+ find out the current clause it should be processed, or return false
+ if all have been processed already. */
+
+static inline bool
+omp_task_reduction_iterate (int pass, enum tree_code code,
+ enum omp_clause_code ccode, tree *c, tree *decl,
+ tree *type, tree *next)
+{
+ for (; *c; *c = omp_find_clause (OMP_CLAUSE_CHAIN (*c), ccode))
+ {
+ if (ccode == OMP_CLAUSE_REDUCTION
+ && code != OMP_TASKLOOP
+ && !OMP_CLAUSE_REDUCTION_TASK (*c))
+ continue;
+ *decl = OMP_CLAUSE_DECL (*c);
+ *type = TREE_TYPE (*decl);
+ if (TREE_CODE (*decl) == MEM_REF)
+ {
+ if (pass != 1)
+ continue;
+ }
+ else
+ {
+ if (omp_is_reference (*decl))
+ *type = TREE_TYPE (*type);
+ if (pass != (!TREE_CONSTANT (TYPE_SIZE_UNIT (*type))))
+ continue;
+ }
+ *next = omp_find_clause (OMP_CLAUSE_CHAIN (*c), ccode);
+ return true;
+ }
+ *decl = NULL_TREE;
+ *type = NULL_TREE;
+ *next = NULL_TREE;
+ return false;
+}
+
+/* Lower task_reduction and reduction clauses (the latter unless CODE is
+ OMP_TASKGROUP only with task modifier). Register mapping of those in
+ START sequence and reducing them and unregister them in the END sequence. */
+
+static void
+lower_omp_task_reductions (omp_context *ctx, enum tree_code code, tree clauses,
+ gimple_seq *start, gimple_seq *end)
+{
+ enum omp_clause_code ccode
+ = (code == OMP_TASKGROUP
+ ? OMP_CLAUSE_TASK_REDUCTION : OMP_CLAUSE_REDUCTION);
+ tree cancellable = NULL_TREE;
+ clauses = omp_task_reductions_find_first (clauses, code, ccode);
+ if (clauses == NULL_TREE)
+ return;
+ if (code == OMP_FOR || code == OMP_SECTIONS)
+ {
+ for (omp_context *outer = ctx->outer; outer; outer = outer->outer)
+ if (gimple_code (outer->stmt) == GIMPLE_OMP_PARALLEL
+ && outer->cancellable)
+ {
+ cancellable = error_mark_node;
+ break;
+ }
+ else if (gimple_code (outer->stmt) != GIMPLE_OMP_TASKGROUP)
+ break;
+ }
+ tree record_type = lang_hooks.types.make_type (RECORD_TYPE);
+ tree *last = &TYPE_FIELDS (record_type);
+ unsigned cnt = 0;
+ if (cancellable)
+ {
+ tree field = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
+ ptr_type_node);
+ tree ifield = build_decl (UNKNOWN_LOCATION, FIELD_DECL, NULL_TREE,
+ integer_type_node);
+ *last = field;
+ DECL_CHAIN (field) = ifield;
+ last = &DECL_CHAIN (ifield);
+ DECL_CONTEXT (field) = record_type;
+ if (TYPE_ALIGN (record_type) < DECL_ALIGN (field))
+ SET_TYPE_ALIGN (record_type, DECL_ALIGN (field));
+ DECL_CONTEXT (ifield) = record_type;
+ if (TYPE_ALIGN (record_type) < DECL_ALIGN (ifield))
+ SET_TYPE_ALIGN (record_type, DECL_ALIGN (ifield));
+ }
+ for (int pass = 0; pass < 2; pass++)
+ {
+ tree decl, type, next;
+ for (tree c = clauses;
+ omp_task_reduction_iterate (pass, code, ccode,
+ &c, &decl, &type, &next); c = next)
+ {
+ ++cnt;
+ tree new_type = type;
+ if (ctx->outer)
+ new_type = remap_type (type, &ctx->outer->cb);
+ tree field
+ = build_decl (OMP_CLAUSE_LOCATION (c), FIELD_DECL,
+ DECL_P (decl) ? DECL_NAME (decl) : NULL_TREE,
+ new_type);
+ if (DECL_P (decl) && type == TREE_TYPE (decl))
+ {
+ SET_DECL_ALIGN (field, DECL_ALIGN (decl));
+ DECL_USER_ALIGN (field) = DECL_USER_ALIGN (decl);
+ TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (decl);
+ }
+ else
+ SET_DECL_ALIGN (field, TYPE_ALIGN (type));
+ DECL_CONTEXT (field) = record_type;
+ if (TYPE_ALIGN (record_type) < DECL_ALIGN (field))
+ SET_TYPE_ALIGN (record_type, DECL_ALIGN (field));
+ *last = field;
+ last = &DECL_CHAIN (field);
+ tree bfield
+ = build_decl (OMP_CLAUSE_LOCATION (c), FIELD_DECL, NULL_TREE,
+ boolean_type_node);
+ DECL_CONTEXT (bfield) = record_type;
+ if (TYPE_ALIGN (record_type) < DECL_ALIGN (bfield))
+ SET_TYPE_ALIGN (record_type, DECL_ALIGN (bfield));
+ *last = bfield;
+ last = &DECL_CHAIN (bfield);
+ }
+ }
+ *last = NULL_TREE;
+ layout_type (record_type);
+
+ /* Build up an array which registers with the runtime all the reductions
+ and deregisters them at the end. Format documented in libgomp/task.c. */
+ tree atype = build_array_type_nelts (pointer_sized_int_node, 7 + cnt * 3);
+ tree avar = create_tmp_var_raw (atype);
+ gimple_add_tmp_var (avar);
+ TREE_ADDRESSABLE (avar) = 1;
+ tree r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_zero_node,
+ NULL_TREE, NULL_TREE);
+ tree t = build_int_cst (pointer_sized_int_node, cnt);
+ gimple_seq_add_stmt (start, gimple_build_assign (r, t));
+ gimple_seq seq = NULL;
+ tree sz = fold_convert (pointer_sized_int_node,
+ TYPE_SIZE_UNIT (record_type));
+ int cachesz = 64;
+ sz = fold_build2 (PLUS_EXPR, pointer_sized_int_node, sz,
+ build_int_cst (pointer_sized_int_node, cachesz - 1));
+ sz = fold_build2 (BIT_AND_EXPR, pointer_sized_int_node, sz,
+ build_int_cst (pointer_sized_int_node, ~(cachesz - 1)));
+ ctx->task_reductions.create (1 + cnt);
+ ctx->task_reduction_map = new hash_map<tree, unsigned>;
+ ctx->task_reductions.quick_push (TREE_CODE (sz) == INTEGER_CST
+ ? sz : NULL_TREE);
+ sz = force_gimple_operand (sz, &seq, true, NULL_TREE);
+ gimple_seq_add_seq (start, seq);
+ r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_one_node,
+ NULL_TREE, NULL_TREE);
+ gimple_seq_add_stmt (start, gimple_build_assign (r, sz));
+ r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (2),
+ NULL_TREE, NULL_TREE);
+ t = build_int_cst (pointer_sized_int_node,
+ MAX (TYPE_ALIGN_UNIT (record_type), (unsigned) cachesz));
+ gimple_seq_add_stmt (start, gimple_build_assign (r, t));
+ r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (3),
+ NULL_TREE, NULL_TREE);
+ t = build_int_cst (pointer_sized_int_node, -1);
+ gimple_seq_add_stmt (start, gimple_build_assign (r, t));
+ r = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (4),
+ NULL_TREE, NULL_TREE);
+ t = build_int_cst (pointer_sized_int_node, 0);
+ gimple_seq_add_stmt (start, gimple_build_assign (r, t));
+
+ /* In end, build a loop that iterates from 0 to < omp_get_num_threads ()
+ and for each task reduction checks a bool right after the private variable
+ within that thread's chunk; if the bool is clear, it hasn't been
+ initialized and thus isn't going to be reduced nor destructed, otherwise
+ reduce and destruct it. */
+ tree idx = create_tmp_var (size_type_node);
+ gimple_seq_add_stmt (end, gimple_build_assign (idx, size_zero_node));
+ tree num_thr_sz = create_tmp_var (size_type_node);
+ tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
+ tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
+ tree lab3 = NULL_TREE;
+ gimple *g;
+ if (code == OMP_FOR || code == OMP_SECTIONS)
+ {
+ /* For worksharing constructs, only perform it in the master thread,
+ with the exception of cancelled implicit barriers - then only handle
+ the current thread. */
+ tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
+ t = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
+ tree thr_num = create_tmp_var (integer_type_node);
+ g = gimple_build_call (t, 0);
+ gimple_call_set_lhs (g, thr_num);
+ gimple_seq_add_stmt (end, g);
+ if (cancellable)
+ {
+ tree c;
+ tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
+ tree lab6 = create_artificial_label (UNKNOWN_LOCATION);
+ lab3 = create_artificial_label (UNKNOWN_LOCATION);
+ if (code == OMP_FOR)
+ c = gimple_omp_for_clauses (ctx->stmt);
+ else /* if (code == OMP_SECTIONS) */
+ c = gimple_omp_sections_clauses (ctx->stmt);
+ c = OMP_CLAUSE_DECL (omp_find_clause (c, OMP_CLAUSE__REDUCTEMP_));
+ cancellable = c;
+ g = gimple_build_cond (NE_EXPR, c, build_zero_cst (TREE_TYPE (c)),
+ lab5, lab6);
+ gimple_seq_add_stmt (end, g);
+ gimple_seq_add_stmt (end, gimple_build_label (lab5));
+ g = gimple_build_assign (idx, NOP_EXPR, thr_num);
+ gimple_seq_add_stmt (end, g);
+ g = gimple_build_assign (num_thr_sz, PLUS_EXPR, idx,
+ build_one_cst (TREE_TYPE (idx)));
+ gimple_seq_add_stmt (end, g);
+ gimple_seq_add_stmt (end, gimple_build_goto (lab3));
+ gimple_seq_add_stmt (end, gimple_build_label (lab6));
+ }
+ g = gimple_build_cond (NE_EXPR, thr_num, integer_zero_node, lab2, lab4);
+ gimple_seq_add_stmt (end, g);
+ gimple_seq_add_stmt (end, gimple_build_label (lab4));
+ }
+ if (code != OMP_PARALLEL)
+ {
+ t = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
+ tree num_thr = create_tmp_var (integer_type_node);
+ g = gimple_build_call (t, 0);
+ gimple_call_set_lhs (g, num_thr);
+ gimple_seq_add_stmt (end, g);
+ g = gimple_build_assign (num_thr_sz, NOP_EXPR, num_thr);
+ gimple_seq_add_stmt (end, g);
+ if (cancellable)
+ gimple_seq_add_stmt (end, gimple_build_label (lab3));
+ }
+ else
+ {
+ tree c = omp_find_clause (gimple_omp_parallel_clauses (ctx->stmt),
+ OMP_CLAUSE__REDUCTEMP_);
+ t = fold_convert (pointer_sized_int_node, OMP_CLAUSE_DECL (c));
+ t = fold_convert (size_type_node, t);
+ gimplify_assign (num_thr_sz, t, end);
+ }
+ t = build4 (ARRAY_REF, pointer_sized_int_node, avar, size_int (2),
+ NULL_TREE, NULL_TREE);
+ tree data = create_tmp_var (pointer_sized_int_node);
+ gimple_seq_add_stmt (end, gimple_build_assign (data, t));
+ gimple_seq_add_stmt (end, gimple_build_label (lab1));
+ tree ptr;
+ if (TREE_CODE (TYPE_SIZE_UNIT (record_type)) == INTEGER_CST)
+ ptr = create_tmp_var (build_pointer_type (record_type));
+ else
+ ptr = create_tmp_var (ptr_type_node);
+ gimple_seq_add_stmt (end, gimple_build_assign (ptr, NOP_EXPR, data));
+
+ tree field = TYPE_FIELDS (record_type);
+ cnt = 0;
+ if (cancellable)
+ field = DECL_CHAIN (DECL_CHAIN (field));
+ for (int pass = 0; pass < 2; pass++)
+ {
+ tree decl, type, next;
+ for (tree c = clauses;
+ omp_task_reduction_iterate (pass, code, ccode,
+ &c, &decl, &type, &next); c = next)
+ {
+ tree var = decl, ref;
+ if (TREE_CODE (decl) == MEM_REF)
+ {
+ var = TREE_OPERAND (var, 0);
+ if (TREE_CODE (var) == POINTER_PLUS_EXPR)
+ var = TREE_OPERAND (var, 0);
+ tree v = var;
+ if (TREE_CODE (var) == ADDR_EXPR)
+ var = TREE_OPERAND (var, 0);
+ else if (TREE_CODE (var) == INDIRECT_REF)
+ var = TREE_OPERAND (var, 0);
+ tree orig_var = var;
+ if (is_variable_sized (var))
+ {
+ gcc_assert (DECL_HAS_VALUE_EXPR_P (var));
+ var = DECL_VALUE_EXPR (var);
+ gcc_assert (TREE_CODE (var) == INDIRECT_REF);
+ var = TREE_OPERAND (var, 0);
+ gcc_assert (DECL_P (var));
+ }
+ t = ref = maybe_lookup_decl_in_outer_ctx (var, ctx);
+ if (orig_var != var)
+ gcc_assert (TREE_CODE (v) == ADDR_EXPR);
+ else if (TREE_CODE (v) == ADDR_EXPR)
+ t = build_fold_addr_expr (t);
+ else if (TREE_CODE (v) == INDIRECT_REF)
+ t = build_fold_indirect_ref (t);
+ if (TREE_CODE (TREE_OPERAND (decl, 0)) == POINTER_PLUS_EXPR)
+ {
+ tree b = TREE_OPERAND (TREE_OPERAND (decl, 0), 1);
+ b = maybe_lookup_decl_in_outer_ctx (b, ctx);
+ t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t, b);
+ }
+ if (!integer_zerop (TREE_OPERAND (decl, 1)))
+ t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (t), t,
+ fold_convert (size_type_node,
+ TREE_OPERAND (decl, 1)));
+ }
+ else
+ {
+ t = ref = maybe_lookup_decl_in_outer_ctx (var, ctx);
+ if (!omp_is_reference (decl))
+ t = build_fold_addr_expr (t);
+ }
+ t = fold_convert (pointer_sized_int_node, t);
+ seq = NULL;
+ t = force_gimple_operand (t, &seq, true, NULL_TREE);
+ gimple_seq_add_seq (start, seq);
+ r = build4 (ARRAY_REF, pointer_sized_int_node, avar,
+ size_int (7 + cnt * 3), NULL_TREE, NULL_TREE);
+ gimple_seq_add_stmt (start, gimple_build_assign (r, t));
+ t = unshare_expr (byte_position (field));
+ t = fold_convert (pointer_sized_int_node, t);
+ ctx->task_reduction_map->put (c, cnt);
+ ctx->task_reductions.quick_push (TREE_CODE (t) == INTEGER_CST
+ ? t : NULL_TREE);
+ seq = NULL;
+ t = force_gimple_operand (t, &seq, true, NULL_TREE);
+ gimple_seq_add_seq (start, seq);
+ r = build4 (ARRAY_REF, pointer_sized_int_node, avar,
+ size_int (7 + cnt * 3 + 1), NULL_TREE, NULL_TREE);
+ gimple_seq_add_stmt (start, gimple_build_assign (r, t));
+
+ tree bfield = DECL_CHAIN (field);
+ tree cond;
+ if (code == OMP_PARALLEL || code == OMP_FOR || code == OMP_SECTIONS)
+ /* In parallel or worksharing all threads unconditionally
+ initialize all their task reduction private variables. */
+ cond = boolean_true_node;
+ else if (TREE_TYPE (ptr) == ptr_type_node)
+ {
+ cond = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr,
+ unshare_expr (byte_position (bfield)));
+ seq = NULL;
+ cond = force_gimple_operand (cond, &seq, true, NULL_TREE);
+ gimple_seq_add_seq (end, seq);
+ tree pbool = build_pointer_type (TREE_TYPE (bfield));
+ cond = build2 (MEM_REF, TREE_TYPE (bfield), cond,
+ build_int_cst (pbool, 0));
+ }
+ else
+ cond = build3 (COMPONENT_REF, TREE_TYPE (bfield),
+ build_simple_mem_ref (ptr), bfield, NULL_TREE);
+ tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
+ tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
+ tree condv = create_tmp_var (boolean_type_node);
+ gimple_seq_add_stmt (end, gimple_build_assign (condv, cond));
+ g = gimple_build_cond (NE_EXPR, condv, boolean_false_node,
+ lab3, lab4);
+ gimple_seq_add_stmt (end, g);
+ gimple_seq_add_stmt (end, gimple_build_label (lab3));
+ if (cancellable && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) == NULL_TREE)
+ {
+ /* If this reduction doesn't need destruction and parallel
+ has been cancelled, there is nothing to do for this
+ reduction, so jump around the merge operation. */
+ tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
+ g = gimple_build_cond (NE_EXPR, cancellable,
+ build_zero_cst (TREE_TYPE (cancellable)),
+ lab4, lab5);
+ gimple_seq_add_stmt (end, g);
+ gimple_seq_add_stmt (end, gimple_build_label (lab5));
+ }
+
+ tree new_var;
+ if (TREE_TYPE (ptr) == ptr_type_node)
+ {
+ new_var = build2 (POINTER_PLUS_EXPR, ptr_type_node, ptr,
+ unshare_expr (byte_position (field)));
+ seq = NULL;
+ new_var = force_gimple_operand (new_var, &seq, true, NULL_TREE);
+ gimple_seq_add_seq (end, seq);
+ tree pbool = build_pointer_type (TREE_TYPE (field));
+ new_var = build2 (MEM_REF, TREE_TYPE (field), new_var,
+ build_int_cst (pbool, 0));
+ }
+ else
+ new_var = build3 (COMPONENT_REF, TREE_TYPE (field),
+ build_simple_mem_ref (ptr), field, NULL_TREE);
+
+ enum tree_code rcode = OMP_CLAUSE_REDUCTION_CODE (c);
+ if (TREE_CODE (decl) != MEM_REF && omp_is_reference (decl))
+ ref = build_simple_mem_ref (ref);
+ /* reduction(-:var) sums up the partial results, so it acts
+ identically to reduction(+:var). */
+ if (rcode == MINUS_EXPR)
+ rcode = PLUS_EXPR;
+ if (TREE_CODE (decl) == MEM_REF)
+ {
+ tree type = TREE_TYPE (new_var);
+ tree v = TYPE_MAX_VALUE (TYPE_DOMAIN (type));
+ tree i = create_tmp_var (TREE_TYPE (v));
+ tree ptype = build_pointer_type (TREE_TYPE (type));
+ if (DECL_P (v))
+ {
+ v = maybe_lookup_decl_in_outer_ctx (v, ctx);
+ tree vv = create_tmp_var (TREE_TYPE (v));
+ gimplify_assign (vv, v, start);
+ v = vv;
+ }
+ ref = build4 (ARRAY_REF, pointer_sized_int_node, avar,
+ size_int (7 + cnt * 3), NULL_TREE, NULL_TREE);
+ new_var = build_fold_addr_expr (new_var);
+ new_var = fold_convert (ptype, new_var);
+ ref = fold_convert (ptype, ref);
+ tree m = create_tmp_var (ptype);
+ gimplify_assign (m, new_var, end);
+ new_var = m;
+ m = create_tmp_var (ptype);
+ gimplify_assign (m, ref, end);
+ ref = m;
+ gimplify_assign (i, build_int_cst (TREE_TYPE (v), 0), end);
+ tree body = create_artificial_label (UNKNOWN_LOCATION);
+ tree endl = create_artificial_label (UNKNOWN_LOCATION);
+ gimple_seq_add_stmt (end, gimple_build_label (body));
+ tree priv = build_simple_mem_ref (new_var);
+ tree out = build_simple_mem_ref (ref);
+ if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
+ {
+ tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
+ tree decl_placeholder
+ = OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c);
+ tree lab6 = NULL_TREE;
+ if (cancellable)
+ {
+ /* If this reduction needs destruction and parallel
+ has been cancelled, jump around the merge operation
+ to the destruction. */
+ tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
+ lab6 = create_artificial_label (UNKNOWN_LOCATION);
+ tree zero = build_zero_cst (TREE_TYPE (cancellable));
+ g = gimple_build_cond (NE_EXPR, cancellable, zero,
+ lab6, lab5);
+ gimple_seq_add_stmt (end, g);
+ gimple_seq_add_stmt (end, gimple_build_label (lab5));
+ }
+ SET_DECL_VALUE_EXPR (placeholder, out);
+ DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
+ SET_DECL_VALUE_EXPR (decl_placeholder, priv);
+ DECL_HAS_VALUE_EXPR_P (decl_placeholder) = 1;
+ lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
+ gimple_seq_add_seq (end,
+ OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
+ OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
+ {
+ OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
+ OMP_CLAUSE_REDUCTION_DECL_PLACEHOLDER (c) = NULL;
+ }
+ if (cancellable)
+ gimple_seq_add_stmt (end, gimple_build_label (lab6));
+ tree x = lang_hooks.decls.omp_clause_dtor (c, priv);
+ if (x)
+ {
+ gimple_seq tseq = NULL;
+ gimplify_stmt (&x, &tseq);
+ gimple_seq_add_seq (end, tseq);
+ }
+ }
+ else
+ {
+ tree x = build2 (rcode, TREE_TYPE (out), out, priv);
+ out = unshare_expr (out);
+ gimplify_assign (out, x, end);
+ }
+ gimple *g
+ = gimple_build_assign (new_var, POINTER_PLUS_EXPR, new_var,
+ TYPE_SIZE_UNIT (TREE_TYPE (type)));
+ gimple_seq_add_stmt (end, g);
+ g = gimple_build_assign (ref, POINTER_PLUS_EXPR, ref,
+ TYPE_SIZE_UNIT (TREE_TYPE (type)));
+ gimple_seq_add_stmt (end, g);
+ g = gimple_build_assign (i, PLUS_EXPR, i,
+ build_int_cst (TREE_TYPE (i), 1));
+ gimple_seq_add_stmt (end, g);
+ g = gimple_build_cond (LE_EXPR, i, v, body, endl);
+ gimple_seq_add_stmt (end, g);
+ gimple_seq_add_stmt (end, gimple_build_label (endl));
+ }
+ else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
+ {
+ tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
+ tree oldv = NULL_TREE;
+ tree lab6 = NULL_TREE;
+ if (cancellable)
+ {
+ /* If this reduction needs destruction and parallel
+ has been cancelled, jump around the merge operation
+ to the destruction. */
+ tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
+ lab6 = create_artificial_label (UNKNOWN_LOCATION);
+ tree zero = build_zero_cst (TREE_TYPE (cancellable));
+ g = gimple_build_cond (NE_EXPR, cancellable, zero,
+ lab6, lab5);
+ gimple_seq_add_stmt (end, g);
+ gimple_seq_add_stmt (end, gimple_build_label (lab5));
+ }
+ if (omp_is_reference (decl)
+ && !useless_type_conversion_p (TREE_TYPE (placeholder),
+ TREE_TYPE (ref)))
+ ref = build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c), ref);
+ ref = build_fold_addr_expr_loc (OMP_CLAUSE_LOCATION (c), ref);
+ tree refv = create_tmp_var (TREE_TYPE (ref));
+ gimplify_assign (refv, ref, end);
+ ref = build_simple_mem_ref_loc (OMP_CLAUSE_LOCATION (c), refv);
+ SET_DECL_VALUE_EXPR (placeholder, ref);
+ DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
+ tree d = maybe_lookup_decl (decl, ctx);
+ gcc_assert (d);
+ if (DECL_HAS_VALUE_EXPR_P (d))
+ oldv = DECL_VALUE_EXPR (d);
+ if (omp_is_reference (var))
+ {
+ tree v = fold_convert (TREE_TYPE (d),
+ build_fold_addr_expr (new_var));
+ SET_DECL_VALUE_EXPR (d, v);
+ }
+ else
+ SET_DECL_VALUE_EXPR (d, new_var);
+ DECL_HAS_VALUE_EXPR_P (d) = 1;
+ lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx);
+ if (oldv)
+ SET_DECL_VALUE_EXPR (d, oldv);
+ else
+ {
+ SET_DECL_VALUE_EXPR (d, NULL_TREE);
+ DECL_HAS_VALUE_EXPR_P (d) = 0;
+ }
+ gimple_seq_add_seq (end, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c));
+ OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TASK_REDUCTION)
+ OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL;
+ if (cancellable)
+ gimple_seq_add_stmt (end, gimple_build_label (lab6));
+ tree x = lang_hooks.decls.omp_clause_dtor (c, new_var);
+ if (x)
+ {
+ gimple_seq tseq = NULL;
+ gimplify_stmt (&x, &tseq);
+ gimple_seq_add_seq (end, tseq);
+ }
+ }
+ else
+ {
+ tree x = build2 (rcode, TREE_TYPE (ref), ref, new_var);
+ ref = unshare_expr (ref);
+ gimplify_assign (ref, x, end);
+ }
+ gimple_seq_add_stmt (end, gimple_build_label (lab4));
+ ++cnt;
+ field = DECL_CHAIN (bfield);
+ }
+ }
+
+ if (code == OMP_TASKGROUP)
+ {
+ t = builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_REDUCTION_REGISTER);
+ g = gimple_build_call (t, 1, build_fold_addr_expr (avar));
+ gimple_seq_add_stmt (start, g);
+ }
+ else
+ {
+ tree c;
+ if (code == OMP_FOR)
+ c = gimple_omp_for_clauses (ctx->stmt);
+ else if (code == OMP_SECTIONS)
+ c = gimple_omp_sections_clauses (ctx->stmt);
+ else
+ c = gimple_omp_taskreg_clauses (ctx->stmt);
+ c = omp_find_clause (c, OMP_CLAUSE__REDUCTEMP_);
+ t = fold_convert (TREE_TYPE (OMP_CLAUSE_DECL (c)),
+ build_fold_addr_expr (avar));
+ gimplify_assign (OMP_CLAUSE_DECL (c), t, start);
+ }
+
+ gimple_seq_add_stmt (end, gimple_build_assign (data, PLUS_EXPR, data, sz));
+ gimple_seq_add_stmt (end, gimple_build_assign (idx, PLUS_EXPR, idx,
+ size_one_node));
+ g = gimple_build_cond (NE_EXPR, idx, num_thr_sz, lab1, lab2);
+ gimple_seq_add_stmt (end, g);
+ gimple_seq_add_stmt (end, gimple_build_label (lab2));
+ if (code == OMP_FOR || code == OMP_SECTIONS)
+ {
+ enum built_in_function bfn
+ = BUILT_IN_GOMP_WORKSHARE_TASK_REDUCTION_UNREGISTER;
+ t = builtin_decl_explicit (bfn);
+ tree c_bool_type = TREE_VALUE (TYPE_ARG_TYPES (TREE_TYPE (t)));
+ tree arg;
+ if (cancellable)
+ {
+ arg = create_tmp_var (c_bool_type);
+ gimple_seq_add_stmt (end, gimple_build_assign (arg, NOP_EXPR,
+ cancellable));
+ }
+ else
+ arg = build_int_cst (c_bool_type, 0);
+ g = gimple_build_call (t, 1, arg);
+ }
+ else
+ {
+ t = builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_REDUCTION_UNREGISTER);
+ g = gimple_build_call (t, 1, build_fold_addr_expr (avar));
+ }
+ gimple_seq_add_stmt (end, g);
+ t = build_constructor (atype, NULL);
+ TREE_THIS_VOLATILE (t) = 1;
+ gimple_seq_add_stmt (end, gimple_build_assign (avar, t));
+}
/* Expand code for an OpenMP taskgroup directive. */
gimple *stmt = gsi_stmt (*gsi_p);
gcall *x;
gbind *bind;
+ gimple_seq dseq = NULL;
tree block = make_node (BLOCK);
bind = gimple_build_bind (NULL, NULL, block);
gsi_replace (gsi_p, bind, true);
gimple_bind_add_stmt (bind, stmt);
+ push_gimplify_context ();
+
x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START),
0);
gimple_bind_add_stmt (bind, x);
+ lower_omp_task_reductions (ctx, OMP_TASKGROUP,
+ gimple_omp_taskgroup_clauses (stmt),
+ gimple_bind_body_ptr (bind), &dseq);
+
lower_omp (gimple_omp_body_ptr (stmt), ctx);
gimple_bind_add_seq (bind, gimple_omp_body (stmt));
gimple_omp_set_body (stmt, NULL);
gimple_bind_add_stmt (bind, gimple_build_omp_return (true));
+ gimple_bind_add_seq (bind, dseq);
+
+ pop_gimplify_context (bind);
gimple_bind_append_vars (bind, ctx->block_vars);
BLOCK_VARS (block) = ctx->block_vars;
}
+/* Expand code for an OpenMP scan directive and the structured block
+ before the scan directive. */
+
+static void
+lower_omp_scan (gimple_stmt_iterator *gsi_p, omp_context *ctx)
+{
+ gimple *stmt = gsi_stmt (*gsi_p);
+ bool has_clauses
+ = gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt)) != NULL;
+ tree lane = NULL_TREE;
+ gimple_seq before = NULL;
+ omp_context *octx = ctx->outer;
+ gcc_assert (octx);
+ if (octx->scan_exclusive && !has_clauses)
+ {
+ gimple_stmt_iterator gsi2 = *gsi_p;
+ gsi_next (&gsi2);
+ gimple *stmt2 = gsi_stmt (gsi2);
+ /* For exclusive scan, swap GIMPLE_OMP_SCAN without clauses
+ with following GIMPLE_OMP_SCAN with clauses, so that input_phase,
+ the one with exclusive clause(s), comes first. */
+ if (stmt2
+ && gimple_code (stmt2) == GIMPLE_OMP_SCAN
+ && gimple_omp_scan_clauses (as_a <gomp_scan *> (stmt2)) != NULL)
+ {
+ gsi_remove (gsi_p, false);
+ gsi_insert_after (gsi_p, stmt, GSI_SAME_STMT);
+ ctx = maybe_lookup_ctx (stmt2);
+ gcc_assert (ctx);
+ lower_omp_scan (gsi_p, ctx);
+ return;
+ }
+ }
+
+ bool input_phase = has_clauses ^ octx->scan_inclusive;
+ bool is_simd = (gimple_code (octx->stmt) == GIMPLE_OMP_FOR
+ && gimple_omp_for_kind (octx->stmt) == GF_OMP_FOR_KIND_SIMD);
+ bool is_for = (gimple_code (octx->stmt) == GIMPLE_OMP_FOR
+ && gimple_omp_for_kind (octx->stmt) == GF_OMP_FOR_KIND_FOR
+ && !gimple_omp_for_combined_p (octx->stmt));
+ bool is_for_simd = is_simd && gimple_omp_for_combined_into_p (octx->stmt);
+ if (is_for_simd && octx->for_simd_scan_phase)
+ is_simd = false;
+ if (is_simd)
+ if (tree c = omp_find_clause (gimple_omp_for_clauses (octx->stmt),
+ OMP_CLAUSE__SIMDUID_))
+ {
+ tree uid = OMP_CLAUSE__SIMDUID__DECL (c);
+ lane = create_tmp_var (unsigned_type_node);
+ tree t = build_int_cst (integer_type_node,
+ input_phase ? 1
+ : octx->scan_inclusive ? 2 : 3);
+ gimple *g
+ = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 2, uid, t);
+ gimple_call_set_lhs (g, lane);
+ gimple_seq_add_stmt (&before, g);
+ }
+
+ if (is_simd || is_for)
+ {
+ for (tree c = gimple_omp_for_clauses (octx->stmt);
+ c; c = OMP_CLAUSE_CHAIN (c))
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
+ && OMP_CLAUSE_REDUCTION_INSCAN (c))
+ {
+ location_t clause_loc = OMP_CLAUSE_LOCATION (c);
+ tree var = OMP_CLAUSE_DECL (c);
+ tree new_var = lookup_decl (var, octx);
+ tree val = new_var;
+ tree var2 = NULL_TREE;
+ tree var3 = NULL_TREE;
+ tree var4 = NULL_TREE;
+ tree lane0 = NULL_TREE;
+ tree new_vard = new_var;
+ if (omp_is_reference (var))
+ {
+ new_var = build_simple_mem_ref_loc (clause_loc, new_var);
+ val = new_var;
+ }
+ if (DECL_HAS_VALUE_EXPR_P (new_vard))
+ {
+ val = DECL_VALUE_EXPR (new_vard);
+ if (new_vard != new_var)
+ {
+ gcc_assert (TREE_CODE (val) == ADDR_EXPR);
+ val = TREE_OPERAND (val, 0);
+ }
+ if (TREE_CODE (val) == ARRAY_REF
+ && VAR_P (TREE_OPERAND (val, 0)))
+ {
+ tree v = TREE_OPERAND (val, 0);
+ if (lookup_attribute ("omp simd array",
+ DECL_ATTRIBUTES (v)))
+ {
+ val = unshare_expr (val);
+ lane0 = TREE_OPERAND (val, 1);
+ TREE_OPERAND (val, 1) = lane;
+ var2 = lookup_decl (v, octx);
+ if (octx->scan_exclusive)
+ var4 = lookup_decl (var2, octx);
+ if (input_phase
+ && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
+ var3 = maybe_lookup_decl (var4 ? var4 : var2, octx);
+ if (!input_phase)
+ {
+ var2 = build4 (ARRAY_REF, TREE_TYPE (val),
+ var2, lane, NULL_TREE, NULL_TREE);
+ TREE_THIS_NOTRAP (var2) = 1;
+ if (octx->scan_exclusive)
+ {
+ var4 = build4 (ARRAY_REF, TREE_TYPE (val),
+ var4, lane, NULL_TREE,
+ NULL_TREE);
+ TREE_THIS_NOTRAP (var4) = 1;
+ }
+ }
+ else
+ var2 = val;
+ }
+ }
+ gcc_assert (var2);
+ }
+ else
+ {
+ var2 = build_outer_var_ref (var, octx);
+ if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
+ {
+ var3 = maybe_lookup_decl (new_vard, octx);
+ if (var3 == new_vard || var3 == NULL_TREE)
+ var3 = NULL_TREE;
+ else if (is_simd && octx->scan_exclusive && !input_phase)
+ {
+ var4 = maybe_lookup_decl (var3, octx);
+ if (var4 == var3 || var4 == NULL_TREE)
+ {
+ if (TREE_ADDRESSABLE (TREE_TYPE (new_var)))
+ {
+ var4 = var3;
+ var3 = NULL_TREE;
+ }
+ else
+ var4 = NULL_TREE;
+ }
+ }
+ }
+ if (is_simd
+ && octx->scan_exclusive
+ && !input_phase
+ && var4 == NULL_TREE)
+ var4 = create_tmp_var (TREE_TYPE (val));
+ }
+ if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
+ {
+ tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
+ if (input_phase)
+ {
+ if (var3)
+ {
+ /* If we've added a separate identity element
+ variable, copy it over into val. */
+ tree x = lang_hooks.decls.omp_clause_assign_op (c, val,
+ var3);
+ gimplify_and_add (x, &before);
+ }
+ else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
+ {
+ /* Otherwise, assign to it the identity element. */
+ gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
+ if (is_for)
+ tseq = copy_gimple_seq_and_replace_locals (tseq);
+ tree ref = build_outer_var_ref (var, octx);
+ tree x = (DECL_HAS_VALUE_EXPR_P (new_vard)
+ ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
+ if (x)
+ {
+ if (new_vard != new_var)
+ val = build_fold_addr_expr_loc (clause_loc, val);
+ SET_DECL_VALUE_EXPR (new_vard, val);
+ }
+ SET_DECL_VALUE_EXPR (placeholder, ref);
+ DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
+ lower_omp (&tseq, octx);
+ if (x)
+ SET_DECL_VALUE_EXPR (new_vard, x);
+ SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
+ DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
+ gimple_seq_add_seq (&before, tseq);
+ if (is_simd)
+ OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
+ }
+ }
+ else if (is_simd)
+ {
+ tree x;
+ if (octx->scan_exclusive)
+ {
+ tree v4 = unshare_expr (var4);
+ tree v2 = unshare_expr (var2);
+ x = lang_hooks.decls.omp_clause_assign_op (c, v4, v2);
+ gimplify_and_add (x, &before);
+ }
+ gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
+ x = (DECL_HAS_VALUE_EXPR_P (new_vard)
+ ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
+ tree vexpr = val;
+ if (x && new_vard != new_var)
+ vexpr = build_fold_addr_expr_loc (clause_loc, val);
+ if (x)
+ SET_DECL_VALUE_EXPR (new_vard, vexpr);
+ SET_DECL_VALUE_EXPR (placeholder, var2);
+ DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
+ lower_omp (&tseq, octx);
+ gimple_seq_add_seq (&before, tseq);
+ OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
+ if (x)
+ SET_DECL_VALUE_EXPR (new_vard, x);
+ SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
+ DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
+ if (octx->scan_inclusive)
+ {
+ x = lang_hooks.decls.omp_clause_assign_op (c, val,
+ var2);
+ gimplify_and_add (x, &before);
+ }
+ else if (lane0 == NULL_TREE)
+ {
+ x = lang_hooks.decls.omp_clause_assign_op (c, val,
+ var4);
+ gimplify_and_add (x, &before);
+ }
+ }
+ }
+ else
+ {
+ if (input_phase)
+ {
+ /* input phase. Set val to initializer before
+ the body. */
+ tree x = omp_reduction_init (c, TREE_TYPE (new_var));
+ gimplify_assign (val, x, &before);
+ }
+ else if (is_simd)
+ {
+ /* scan phase. */
+ enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
+ if (code == MINUS_EXPR)
+ code = PLUS_EXPR;
+
+ tree x = build2 (code, TREE_TYPE (var2),
+ unshare_expr (var2), unshare_expr (val));
+ if (octx->scan_inclusive)
+ {
+ gimplify_assign (unshare_expr (var2), x, &before);
+ gimplify_assign (val, var2, &before);
+ }
+ else
+ {
+ gimplify_assign (unshare_expr (var4),
+ unshare_expr (var2), &before);
+ gimplify_assign (var2, x, &before);
+ if (lane0 == NULL_TREE)
+ gimplify_assign (val, var4, &before);
+ }
+ }
+ }
+ if (octx->scan_exclusive && !input_phase && lane0)
+ {
+ tree vexpr = unshare_expr (var4);
+ TREE_OPERAND (vexpr, 1) = lane0;
+ if (new_vard != new_var)
+ vexpr = build_fold_addr_expr_loc (clause_loc, vexpr);
+ SET_DECL_VALUE_EXPR (new_vard, vexpr);
+ }
+ }
+ }
+ if (is_simd && !is_for_simd)
+ {
+ gsi_insert_seq_after (gsi_p, gimple_omp_body (stmt), GSI_SAME_STMT);
+ gsi_insert_seq_after (gsi_p, before, GSI_SAME_STMT);
+ gsi_replace (gsi_p, gimple_build_nop (), true);
+ return;
+ }
+ lower_omp (gimple_omp_body_ptr (stmt), octx);
+ if (before)
+ {
+ gimple_stmt_iterator gsi = gsi_start_1 (gimple_omp_body_ptr (stmt));
+ gsi_insert_seq_before (&gsi, before, GSI_SAME_STMT);
+ }
+}
+
+
/* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple
substitution of a couple of function calls. But in the NAMED case,
requires that languages coordinate a symbol name. It is therefore
for a lastprivate clause. Given a loop control predicate of (V
cond N2), we gate the clause on (!(V cond N2)). The lowered form
is appended to *DLIST, iterator initialization is appended to
- *BODY_P. */
+ *BODY_P. *CLIST is for lastprivate(conditional:) code that needs
+ to be emitted in a critical section. */
static void
lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p,
- gimple_seq *dlist, struct omp_context *ctx)
+ gimple_seq *dlist, gimple_seq *clist,
+ struct omp_context *ctx)
{
tree clauses, cond, vinit;
enum tree_code cond_code;
cond_code = EQ_EXPR;
}
- if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_GRID_LOOP
- || gimple_omp_for_grid_phony (fd->for_stmt))
- cond = omp_grid_lastprivate_predicate (fd);
- else
+ tree n2 = fd->loop.n2;
+ if (fd->collapse > 1
+ && TREE_CODE (n2) != INTEGER_CST
+ && gimple_omp_for_combined_into_p (fd->for_stmt))
{
- tree n2 = fd->loop.n2;
- if (fd->collapse > 1
- && TREE_CODE (n2) != INTEGER_CST
- && gimple_omp_for_combined_into_p (fd->for_stmt))
+ struct omp_context *taskreg_ctx = NULL;
+ if (gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
{
- struct omp_context *taskreg_ctx = NULL;
- if (gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR)
+ gomp_for *gfor = as_a <gomp_for *> (ctx->outer->stmt);
+ if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_FOR
+ || gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_DISTRIBUTE)
{
- gomp_for *gfor = as_a <gomp_for *> (ctx->outer->stmt);
- if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_FOR
- || gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_DISTRIBUTE)
+ if (gimple_omp_for_combined_into_p (gfor))
{
- if (gimple_omp_for_combined_into_p (gfor))
- {
- gcc_assert (ctx->outer->outer
- && is_parallel_ctx (ctx->outer->outer));
- taskreg_ctx = ctx->outer->outer;
- }
- else
- {
- struct omp_for_data outer_fd;
- omp_extract_for_data (gfor, &outer_fd, NULL);
- n2 = fold_convert (TREE_TYPE (n2), outer_fd.loop.n2);
- }
+ gcc_assert (ctx->outer->outer
+ && is_parallel_ctx (ctx->outer->outer));
+ taskreg_ctx = ctx->outer->outer;
}
- else if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_TASKLOOP)
- taskreg_ctx = ctx->outer->outer;
- }
- else if (is_taskreg_ctx (ctx->outer))
- taskreg_ctx = ctx->outer;
- if (taskreg_ctx)
- {
- int i;
- tree taskreg_clauses
- = gimple_omp_taskreg_clauses (taskreg_ctx->stmt);
- tree innerc = omp_find_clause (taskreg_clauses,
- OMP_CLAUSE__LOOPTEMP_);
- gcc_assert (innerc);
- for (i = 0; i < fd->collapse; i++)
+ else
{
- innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
- OMP_CLAUSE__LOOPTEMP_);
- gcc_assert (innerc);
+ struct omp_for_data outer_fd;
+ omp_extract_for_data (gfor, &outer_fd, NULL);
+ n2 = fold_convert (TREE_TYPE (n2), outer_fd.loop.n2);
}
+ }
+ else if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_TASKLOOP)
+ taskreg_ctx = ctx->outer->outer;
+ }
+ else if (is_taskreg_ctx (ctx->outer))
+ taskreg_ctx = ctx->outer;
+ if (taskreg_ctx)
+ {
+ int i;
+ tree taskreg_clauses
+ = gimple_omp_taskreg_clauses (taskreg_ctx->stmt);
+ tree innerc = omp_find_clause (taskreg_clauses,
+ OMP_CLAUSE__LOOPTEMP_);
+ gcc_assert (innerc);
+ int count = fd->collapse;
+ if (fd->non_rect
+ && fd->last_nonrect == fd->first_nonrect + 1)
+ if (tree v = gimple_omp_for_index (fd->for_stmt, fd->last_nonrect))
+ if (!TYPE_UNSIGNED (TREE_TYPE (v)))
+ count += 4;
+ for (i = 0; i < count; i++)
+ {
innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
OMP_CLAUSE__LOOPTEMP_);
- if (innerc)
- n2 = fold_convert (TREE_TYPE (n2),
- lookup_decl (OMP_CLAUSE_DECL (innerc),
- taskreg_ctx));
+ gcc_assert (innerc);
}
+ innerc = omp_find_clause (OMP_CLAUSE_CHAIN (innerc),
+ OMP_CLAUSE__LOOPTEMP_);
+ if (innerc)
+ n2 = fold_convert (TREE_TYPE (n2),
+ lookup_decl (OMP_CLAUSE_DECL (innerc),
+ taskreg_ctx));
}
- cond = build2 (cond_code, boolean_type_node, fd->loop.v, n2);
}
+ cond = build2 (cond_code, boolean_type_node, fd->loop.v, n2);
+
+ clauses = gimple_omp_for_clauses (fd->for_stmt);
+ stmts = NULL;
+ lower_lastprivate_clauses (clauses, cond, body_p, &stmts, clist, ctx);
+ if (!gimple_seq_empty_p (stmts))
+ {
+ gimple_seq_add_seq (&stmts, *dlist);
+ *dlist = stmts;
+
+ /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
+ vinit = fd->loop.n1;
+ if (cond_code == EQ_EXPR
+ && tree_fits_shwi_p (fd->loop.n2)
+ && ! integer_zerop (fd->loop.n2))
+ vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0);
+ else
+ vinit = unshare_expr (vinit);
+
+ /* Initialize the iterator variable, so that threads that don't execute
+ any iterations don't execute the lastprivate clauses by accident. */
+ gimplify_assign (fd->loop.v, vinit, body_p);
+ }
+}
+
+/* Callback for walk_gimple_seq. Find #pragma omp scan statement. */
+
+static tree
+omp_find_scan (gimple_stmt_iterator *gsi_p, bool *handled_ops_p,
+ struct walk_stmt_info *wi)
+{
+ gimple *stmt = gsi_stmt (*gsi_p);
+
+ *handled_ops_p = true;
+ switch (gimple_code (stmt))
+ {
+ WALK_SUBSTMTS;
+
+ case GIMPLE_OMP_FOR:
+ if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_SIMD
+ && gimple_omp_for_combined_into_p (stmt))
+ *handled_ops_p = false;
+ break;
+
+ case GIMPLE_OMP_SCAN:
+ *(gimple_stmt_iterator *) (wi->info) = *gsi_p;
+ return integer_zero_node;
+ default:
+ break;
+ }
+ return NULL;
+}
+
+/* Helper function for lower_omp_for, add transformations for a worksharing
+ loop with scan directives inside of it.
+ For worksharing loop not combined with simd, transform:
+ #pragma omp for reduction(inscan,+:r) private(i)
+ for (i = 0; i < n; i = i + 1)
+ {
+ {
+ update (r);
+ }
+ #pragma omp scan inclusive(r)
+ {
+ use (r);
+ }
+ }
+
+ into two worksharing loops + code to merge results:
+
+ num_threads = omp_get_num_threads ();
+ thread_num = omp_get_thread_num ();
+ if (thread_num == 0) goto <D.2099>; else goto <D.2100>;
+ <D.2099>:
+ var2 = r;
+ goto <D.2101>;
+ <D.2100>:
+ // For UDRs this is UDR init, or if ctors are needed, copy from
+ // var3 that has been constructed to contain the neutral element.
+ var2 = 0;
+ <D.2101>:
+ ivar = 0;
+ // The _scantemp_ clauses will arrange for rpriva to be initialized to
+ // a shared array with num_threads elements and rprivb to a local array
+ // number of elements equal to the number of (contiguous) iterations the
+ // current thread will perform. controlb and controlp variables are
+ // temporaries to handle deallocation of rprivb at the end of second
+ // GOMP_FOR.
+ #pragma omp for _scantemp_(rpriva) _scantemp_(rprivb) _scantemp_(controlb) \
+ _scantemp_(controlp) reduction(inscan,+:r) private(i) nowait
+ for (i = 0; i < n; i = i + 1)
+ {
+ {
+ // For UDRs this is UDR init or copy from var3.
+ r = 0;
+ // This is the input phase from user code.
+ update (r);
+ }
+ {
+ // For UDRs this is UDR merge.
+ var2 = var2 + r;
+ // Rather than handing it over to the user, save to local thread's
+ // array.
+ rprivb[ivar] = var2;
+ // For exclusive scan, the above two statements are swapped.
+ ivar = ivar + 1;
+ }
+ }
+ // And remember the final value from this thread's into the shared
+ // rpriva array.
+ rpriva[(sizetype) thread_num] = var2;
+ // If more than one thread, compute using Work-Efficient prefix sum
+ // the inclusive parallel scan of the rpriva array.
+ if (num_threads > 1) goto <D.2102>; else goto <D.2103>;
+ <D.2102>:
+ GOMP_barrier ();
+ down = 0;
+ k = 1;
+ num_threadsu = (unsigned int) num_threads;
+ thread_numup1 = (unsigned int) thread_num + 1;
+ <D.2108>:
+ twok = k << 1;
+ if (twok > num_threadsu) goto <D.2110>; else goto <D.2111>;
+ <D.2110>:
+ down = 4294967295;
+ k = k >> 1;
+ if (k == num_threadsu) goto <D.2112>; else goto <D.2111>;
+ <D.2112>:
+ k = k >> 1;
+ <D.2111>:
+ twok = k << 1;
+ cplx = .MUL_OVERFLOW (thread_nump1, twok);
+ mul = REALPART_EXPR <cplx>;
+ ovf = IMAGPART_EXPR <cplx>;
+ if (ovf == 0) goto <D.2116>; else goto <D.2117>;
+ <D.2116>:
+ andv = k & down;
+ andvm1 = andv + 4294967295;
+ l = mul + andvm1;
+ if (l < num_threadsu) goto <D.2120>; else goto <D.2117>;
+ <D.2120>:
+ // For UDRs this is UDR merge, performed using var2 variable as temporary,
+ // i.e. var2 = rpriva[l - k]; UDR merge (var2, rpriva[l]); rpriva[l] = var2;
+ rpriva[l] = rpriva[l - k] + rpriva[l];
+ <D.2117>:
+ if (down == 0) goto <D.2121>; else goto <D.2122>;
+ <D.2121>:
+ k = k << 1;
+ goto <D.2123>;
+ <D.2122>:
+ k = k >> 1;
+ <D.2123>:
+ GOMP_barrier ();
+ if (k != 0) goto <D.2108>; else goto <D.2103>;
+ <D.2103>:
+ if (thread_num == 0) goto <D.2124>; else goto <D.2125>;
+ <D.2124>:
+ // For UDRs this is UDR init or copy from var3.
+ var2 = 0;
+ goto <D.2126>;
+ <D.2125>:
+ var2 = rpriva[thread_num - 1];
+ <D.2126>:
+ ivar = 0;
+ #pragma omp for _scantemp_(controlb) _scantemp_(controlp) \
+ reduction(inscan,+:r) private(i)
+ for (i = 0; i < n; i = i + 1)
+ {
+ {
+ // For UDRs, this is r = var2; UDR merge (r, rprivb[ivar]);
+ r = var2 + rprivb[ivar];
+ }
+ {
+ // This is the scan phase from user code.
+ use (r);
+ // Plus a bump of the iterator.
+ ivar = ivar + 1;
+ }
+ } */
+
+static void
+lower_omp_for_scan (gimple_seq *body_p, gimple_seq *dlist, gomp_for *stmt,
+ struct omp_for_data *fd, omp_context *ctx)
+{
+ bool is_for_simd = gimple_omp_for_combined_p (stmt);
+ gcc_assert (ctx->scan_inclusive || ctx->scan_exclusive);
+
+ gimple_seq body = gimple_omp_body (stmt);
+ gimple_stmt_iterator input1_gsi = gsi_none ();
+ struct walk_stmt_info wi;
+ memset (&wi, 0, sizeof (wi));
+ wi.val_only = true;
+ wi.info = (void *) &input1_gsi;
+ walk_gimple_seq_mod (&body, omp_find_scan, NULL, &wi);
+ gcc_assert (!gsi_end_p (input1_gsi));
+
+ gimple *input_stmt1 = gsi_stmt (input1_gsi);
+ gimple_stmt_iterator gsi = input1_gsi;
+ gsi_next (&gsi);
+ gimple_stmt_iterator scan1_gsi = gsi;
+ gimple *scan_stmt1 = gsi_stmt (gsi);
+ gcc_assert (scan_stmt1 && gimple_code (scan_stmt1) == GIMPLE_OMP_SCAN);
+
+ gimple_seq input_body = gimple_omp_body (input_stmt1);
+ gimple_seq scan_body = gimple_omp_body (scan_stmt1);
+ gimple_omp_set_body (input_stmt1, NULL);
+ gimple_omp_set_body (scan_stmt1, NULL);
+ gimple_omp_set_body (stmt, NULL);
+
+ gomp_for *new_stmt = as_a <gomp_for *> (gimple_copy (stmt));
+ gimple_seq new_body = copy_gimple_seq_and_replace_locals (body);
+ gimple_omp_set_body (stmt, body);
+ gimple_omp_set_body (input_stmt1, input_body);
+
+ gimple_stmt_iterator input2_gsi = gsi_none ();
+ memset (&wi, 0, sizeof (wi));
+ wi.val_only = true;
+ wi.info = (void *) &input2_gsi;
+ walk_gimple_seq_mod (&new_body, omp_find_scan, NULL, &wi);
+ gcc_assert (!gsi_end_p (input2_gsi));
+
+ gimple *input_stmt2 = gsi_stmt (input2_gsi);
+ gsi = input2_gsi;
+ gsi_next (&gsi);
+ gimple_stmt_iterator scan2_gsi = gsi;
+ gimple *scan_stmt2 = gsi_stmt (gsi);
+ gcc_assert (scan_stmt2 && gimple_code (scan_stmt2) == GIMPLE_OMP_SCAN);
+ gimple_omp_set_body (scan_stmt2, scan_body);
+
+ gimple_stmt_iterator input3_gsi = gsi_none ();
+ gimple_stmt_iterator scan3_gsi = gsi_none ();
+ gimple_stmt_iterator input4_gsi = gsi_none ();
+ gimple_stmt_iterator scan4_gsi = gsi_none ();
+ gimple *input_stmt3 = NULL, *scan_stmt3 = NULL;
+ gimple *input_stmt4 = NULL, *scan_stmt4 = NULL;
+ omp_context *input_simd_ctx = NULL, *scan_simd_ctx = NULL;
+ if (is_for_simd)
+ {
+ memset (&wi, 0, sizeof (wi));
+ wi.val_only = true;
+ wi.info = (void *) &input3_gsi;
+ walk_gimple_seq_mod (&input_body, omp_find_scan, NULL, &wi);
+ gcc_assert (!gsi_end_p (input3_gsi));
+
+ input_stmt3 = gsi_stmt (input3_gsi);
+ gsi = input3_gsi;
+ gsi_next (&gsi);
+ scan3_gsi = gsi;
+ scan_stmt3 = gsi_stmt (gsi);
+ gcc_assert (scan_stmt3 && gimple_code (scan_stmt3) == GIMPLE_OMP_SCAN);
+
+ memset (&wi, 0, sizeof (wi));
+ wi.val_only = true;
+ wi.info = (void *) &input4_gsi;
+ walk_gimple_seq_mod (&scan_body, omp_find_scan, NULL, &wi);
+ gcc_assert (!gsi_end_p (input4_gsi));
+
+ input_stmt4 = gsi_stmt (input4_gsi);
+ gsi = input4_gsi;
+ gsi_next (&gsi);
+ scan4_gsi = gsi;
+ scan_stmt4 = gsi_stmt (gsi);
+ gcc_assert (scan_stmt4 && gimple_code (scan_stmt4) == GIMPLE_OMP_SCAN);
+
+ input_simd_ctx = maybe_lookup_ctx (input_stmt3)->outer;
+ scan_simd_ctx = maybe_lookup_ctx (input_stmt4)->outer;
+ }
+
+ tree num_threads = create_tmp_var (integer_type_node);
+ tree thread_num = create_tmp_var (integer_type_node);
+ tree nthreads_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS);
+ tree threadnum_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM);
+ gimple *g = gimple_build_call (nthreads_decl, 0);
+ gimple_call_set_lhs (g, num_threads);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_call (threadnum_decl, 0);
+ gimple_call_set_lhs (g, thread_num);
+ gimple_seq_add_stmt (body_p, g);
+
+ tree ivar = create_tmp_var (sizetype);
+ tree new_clauses1 = NULL_TREE, new_clauses2 = NULL_TREE;
+ tree *cp1 = &new_clauses1, *cp2 = &new_clauses2;
+ tree k = create_tmp_var (unsigned_type_node);
+ tree l = create_tmp_var (unsigned_type_node);
+
+ gimple_seq clist = NULL, mdlist = NULL;
+ gimple_seq thr01_list = NULL, thrn1_list = NULL;
+ gimple_seq thr02_list = NULL, thrn2_list = NULL;
+ gimple_seq scan1_list = NULL, input2_list = NULL;
+ gimple_seq last_list = NULL, reduc_list = NULL;
+ for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION
+ && OMP_CLAUSE_REDUCTION_INSCAN (c))
+ {
+ location_t clause_loc = OMP_CLAUSE_LOCATION (c);
+ tree var = OMP_CLAUSE_DECL (c);
+ tree new_var = lookup_decl (var, ctx);
+ tree var3 = NULL_TREE;
+ tree new_vard = new_var;
+ if (omp_is_reference (var))
+ new_var = build_simple_mem_ref_loc (clause_loc, new_var);
+ if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
+ {
+ var3 = maybe_lookup_decl (new_vard, ctx);
+ if (var3 == new_vard)
+ var3 = NULL_TREE;
+ }
+
+ tree ptype = build_pointer_type (TREE_TYPE (new_var));
+ tree rpriva = create_tmp_var (ptype);
+ tree nc = build_omp_clause (clause_loc, OMP_CLAUSE__SCANTEMP_);
+ OMP_CLAUSE_DECL (nc) = rpriva;
+ *cp1 = nc;
+ cp1 = &OMP_CLAUSE_CHAIN (nc);
+
+ tree rprivb = create_tmp_var (ptype);
+ nc = build_omp_clause (clause_loc, OMP_CLAUSE__SCANTEMP_);
+ OMP_CLAUSE_DECL (nc) = rprivb;
+ OMP_CLAUSE__SCANTEMP__ALLOC (nc) = 1;
+ *cp1 = nc;
+ cp1 = &OMP_CLAUSE_CHAIN (nc);
+
+ tree var2 = create_tmp_var_raw (TREE_TYPE (new_var));
+ if (new_vard != new_var)
+ TREE_ADDRESSABLE (var2) = 1;
+ gimple_add_tmp_var (var2);
+
+ tree x = fold_convert_loc (clause_loc, sizetype, thread_num);
+ x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
+ TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
+ x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
+ tree rpriva_ref = build_simple_mem_ref_loc (clause_loc, x);
+
+ x = fold_build2_loc (clause_loc, PLUS_EXPR, integer_type_node,
+ thread_num, integer_minus_one_node);
+ x = fold_convert_loc (clause_loc, sizetype, x);
+ x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
+ TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
+ x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
+ tree rprivam1_ref = build_simple_mem_ref_loc (clause_loc, x);
+
+ x = fold_convert_loc (clause_loc, sizetype, l);
+ x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
+ TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
+ x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
+ tree rprival_ref = build_simple_mem_ref_loc (clause_loc, x);
+
+ x = fold_build2_loc (clause_loc, MINUS_EXPR, unsigned_type_node, l, k);
+ x = fold_convert_loc (clause_loc, sizetype, x);
+ x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, x,
+ TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
+ x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rpriva), rpriva, x);
+ tree rprivalmk_ref = build_simple_mem_ref_loc (clause_loc, x);
+
+ x = fold_build2_loc (clause_loc, MULT_EXPR, sizetype, ivar,
+ TYPE_SIZE_UNIT (TREE_TYPE (ptype)));
+ x = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (rprivb), rprivb, x);
+ tree rprivb_ref = build_simple_mem_ref_loc (clause_loc, x);
+
+ tree var4 = is_for_simd ? new_var : var2;
+ tree var5 = NULL_TREE, var6 = NULL_TREE;
+ if (is_for_simd)
+ {
+ var5 = lookup_decl (var, input_simd_ctx);
+ var6 = lookup_decl (var, scan_simd_ctx);
+ if (new_vard != new_var)
+ {
+ var5 = build_simple_mem_ref_loc (clause_loc, var5);
+ var6 = build_simple_mem_ref_loc (clause_loc, var6);
+ }
+ }
+ if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c))
+ {
+ tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c);
+ tree val = var2;
+
+ x = lang_hooks.decls.omp_clause_default_ctor
+ (c, var2, build_outer_var_ref (var, ctx));
+ if (x)
+ gimplify_and_add (x, &clist);
+
+ x = build_outer_var_ref (var, ctx);
+ x = lang_hooks.decls.omp_clause_assign_op (c, unshare_expr (var4),
+ x);
+ gimplify_and_add (x, &thr01_list);
+
+ tree y = (DECL_HAS_VALUE_EXPR_P (new_vard)
+ ? DECL_VALUE_EXPR (new_vard) : NULL_TREE);
+ if (var3)
+ {
+ x = unshare_expr (var4);
+ x = lang_hooks.decls.omp_clause_assign_op (c, x, var3);
+ gimplify_and_add (x, &thrn1_list);
+ x = unshare_expr (var4);
+ x = lang_hooks.decls.omp_clause_assign_op (c, x, var3);
+ gimplify_and_add (x, &thr02_list);
+ }
+ else if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c))
+ {
+ /* Otherwise, assign to it the identity element. */
+ gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
+ tseq = copy_gimple_seq_and_replace_locals (tseq);
+ if (!is_for_simd)
+ {
+ if (new_vard != new_var)
+ val = build_fold_addr_expr_loc (clause_loc, val);
+ SET_DECL_VALUE_EXPR (new_vard, val);
+ DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
+ }
+ SET_DECL_VALUE_EXPR (placeholder, error_mark_node);
+ DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
+ lower_omp (&tseq, ctx);
+ gimple_seq_add_seq (&thrn1_list, tseq);
+ tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c);
+ lower_omp (&tseq, ctx);
+ gimple_seq_add_seq (&thr02_list, tseq);
+ SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
+ DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
+ OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL;
+ if (y)
+ SET_DECL_VALUE_EXPR (new_vard, y);
+ else
+ {
+ DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
+ SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
+ }
+ }
+
+ x = unshare_expr (var4);
+ x = lang_hooks.decls.omp_clause_assign_op (c, x, rprivam1_ref);
+ gimplify_and_add (x, &thrn2_list);
+
+ if (is_for_simd)
+ {
+ x = unshare_expr (rprivb_ref);
+ x = lang_hooks.decls.omp_clause_assign_op (c, x, var5);
+ gimplify_and_add (x, &scan1_list);
+ }
+ else
+ {
+ if (ctx->scan_exclusive)
+ {
+ x = unshare_expr (rprivb_ref);
+ x = lang_hooks.decls.omp_clause_assign_op (c, x, var2);
+ gimplify_and_add (x, &scan1_list);
+ }
+
+ gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
+ tseq = copy_gimple_seq_and_replace_locals (tseq);
+ SET_DECL_VALUE_EXPR (placeholder, var2);
+ DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
+ lower_omp (&tseq, ctx);
+ gimple_seq_add_seq (&scan1_list, tseq);
+
+ if (ctx->scan_inclusive)
+ {
+ x = unshare_expr (rprivb_ref);
+ x = lang_hooks.decls.omp_clause_assign_op (c, x, var2);
+ gimplify_and_add (x, &scan1_list);
+ }
+ }
+
+ x = unshare_expr (rpriva_ref);
+ x = lang_hooks.decls.omp_clause_assign_op (c, x,
+ unshare_expr (var4));
+ gimplify_and_add (x, &mdlist);
+
+ x = unshare_expr (is_for_simd ? var6 : new_var);
+ x = lang_hooks.decls.omp_clause_assign_op (c, x, var4);
+ gimplify_and_add (x, &input2_list);
+
+ val = rprivb_ref;
+ if (new_vard != new_var)
+ val = build_fold_addr_expr_loc (clause_loc, val);
+
+ gimple_seq tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
+ tseq = copy_gimple_seq_and_replace_locals (tseq);
+ SET_DECL_VALUE_EXPR (new_vard, val);
+ DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
+ if (is_for_simd)
+ {
+ SET_DECL_VALUE_EXPR (placeholder, var6);
+ DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
+ }
+ else
+ DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
+ lower_omp (&tseq, ctx);
+ if (y)
+ SET_DECL_VALUE_EXPR (new_vard, y);
+ else
+ {
+ DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
+ SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
+ }
+ if (!is_for_simd)
+ {
+ SET_DECL_VALUE_EXPR (placeholder, new_var);
+ DECL_HAS_VALUE_EXPR_P (placeholder) = 1;
+ lower_omp (&tseq, ctx);
+ }
+ gimple_seq_add_seq (&input2_list, tseq);
+
+ x = build_outer_var_ref (var, ctx);
+ x = lang_hooks.decls.omp_clause_assign_op (c, x, rpriva_ref);
+ gimplify_and_add (x, &last_list);
+
+ x = lang_hooks.decls.omp_clause_assign_op (c, var2, rprivalmk_ref);
+ gimplify_and_add (x, &reduc_list);
+ tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c);
+ tseq = copy_gimple_seq_and_replace_locals (tseq);
+ val = rprival_ref;
+ if (new_vard != new_var)
+ val = build_fold_addr_expr_loc (clause_loc, val);
+ SET_DECL_VALUE_EXPR (new_vard, val);
+ DECL_HAS_VALUE_EXPR_P (new_vard) = 1;
+ SET_DECL_VALUE_EXPR (placeholder, var2);
+ lower_omp (&tseq, ctx);
+ OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL;
+ SET_DECL_VALUE_EXPR (placeholder, NULL_TREE);
+ DECL_HAS_VALUE_EXPR_P (placeholder) = 0;
+ if (y)
+ SET_DECL_VALUE_EXPR (new_vard, y);
+ else
+ {
+ DECL_HAS_VALUE_EXPR_P (new_vard) = 0;
+ SET_DECL_VALUE_EXPR (new_vard, NULL_TREE);
+ }
+ gimple_seq_add_seq (&reduc_list, tseq);
+ x = lang_hooks.decls.omp_clause_assign_op (c, rprival_ref, var2);
+ gimplify_and_add (x, &reduc_list);
+
+ x = lang_hooks.decls.omp_clause_dtor (c, var2);
+ if (x)
+ gimplify_and_add (x, dlist);
+ }
+ else
+ {
+ x = build_outer_var_ref (var, ctx);
+ gimplify_assign (unshare_expr (var4), x, &thr01_list);
+
+ x = omp_reduction_init (c, TREE_TYPE (new_var));
+ gimplify_assign (unshare_expr (var4), unshare_expr (x),
+ &thrn1_list);
+ gimplify_assign (unshare_expr (var4), x, &thr02_list);
+
+ gimplify_assign (unshare_expr (var4), rprivam1_ref, &thrn2_list);
+
+ enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c);
+ if (code == MINUS_EXPR)
+ code = PLUS_EXPR;
+
+ if (is_for_simd)
+ gimplify_assign (unshare_expr (rprivb_ref), var5, &scan1_list);
+ else
+ {
+ if (ctx->scan_exclusive)
+ gimplify_assign (unshare_expr (rprivb_ref), var2,
+ &scan1_list);
+ x = build2 (code, TREE_TYPE (new_var), var2, new_var);
+ gimplify_assign (var2, x, &scan1_list);
+ if (ctx->scan_inclusive)
+ gimplify_assign (unshare_expr (rprivb_ref), var2,
+ &scan1_list);
+ }
+
+ gimplify_assign (unshare_expr (rpriva_ref), unshare_expr (var4),
+ &mdlist);
+
+ x = build2 (code, TREE_TYPE (new_var), var4, rprivb_ref);
+ gimplify_assign (is_for_simd ? var6 : new_var, x, &input2_list);
+
+ gimplify_assign (build_outer_var_ref (var, ctx), rpriva_ref,
+ &last_list);
+
+ x = build2 (code, TREE_TYPE (new_var), rprivalmk_ref,
+ unshare_expr (rprival_ref));
+ gimplify_assign (rprival_ref, x, &reduc_list);
+ }
+ }
+
+ g = gimple_build_assign (ivar, PLUS_EXPR, ivar, size_one_node);
+ gimple_seq_add_stmt (&scan1_list, g);
+ g = gimple_build_assign (ivar, PLUS_EXPR, ivar, size_one_node);
+ gimple_seq_add_stmt (gimple_omp_body_ptr (is_for_simd
+ ? scan_stmt4 : scan_stmt2), g);
+
+ tree controlb = create_tmp_var (boolean_type_node);
+ tree controlp = create_tmp_var (ptr_type_node);
+ tree nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
+ OMP_CLAUSE_DECL (nc) = controlb;
+ OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
+ *cp1 = nc;
+ cp1 = &OMP_CLAUSE_CHAIN (nc);
+ nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
+ OMP_CLAUSE_DECL (nc) = controlp;
+ OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
+ *cp1 = nc;
+ cp1 = &OMP_CLAUSE_CHAIN (nc);
+ nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
+ OMP_CLAUSE_DECL (nc) = controlb;
+ OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
+ *cp2 = nc;
+ cp2 = &OMP_CLAUSE_CHAIN (nc);
+ nc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SCANTEMP_);
+ OMP_CLAUSE_DECL (nc) = controlp;
+ OMP_CLAUSE__SCANTEMP__CONTROL (nc) = 1;
+ *cp2 = nc;
+ cp2 = &OMP_CLAUSE_CHAIN (nc);
+
+ *cp1 = gimple_omp_for_clauses (stmt);
+ gimple_omp_for_set_clauses (stmt, new_clauses1);
+ *cp2 = gimple_omp_for_clauses (new_stmt);
+ gimple_omp_for_set_clauses (new_stmt, new_clauses2);
+
+ if (is_for_simd)
+ {
+ gimple_seq_add_seq (gimple_omp_body_ptr (scan_stmt3), scan1_list);
+ gimple_seq_add_seq (gimple_omp_body_ptr (input_stmt4), input2_list);
+
+ gsi_insert_seq_after (&input3_gsi, gimple_omp_body (input_stmt3),
+ GSI_SAME_STMT);
+ gsi_remove (&input3_gsi, true);
+ gsi_insert_seq_after (&scan3_gsi, gimple_omp_body (scan_stmt3),
+ GSI_SAME_STMT);
+ gsi_remove (&scan3_gsi, true);
+ gsi_insert_seq_after (&input4_gsi, gimple_omp_body (input_stmt4),
+ GSI_SAME_STMT);
+ gsi_remove (&input4_gsi, true);
+ gsi_insert_seq_after (&scan4_gsi, gimple_omp_body (scan_stmt4),
+ GSI_SAME_STMT);
+ gsi_remove (&scan4_gsi, true);
+ }
+ else
+ {
+ gimple_omp_set_body (scan_stmt1, scan1_list);
+ gimple_omp_set_body (input_stmt2, input2_list);
+ }
+
+ gsi_insert_seq_after (&input1_gsi, gimple_omp_body (input_stmt1),
+ GSI_SAME_STMT);
+ gsi_remove (&input1_gsi, true);
+ gsi_insert_seq_after (&scan1_gsi, gimple_omp_body (scan_stmt1),
+ GSI_SAME_STMT);
+ gsi_remove (&scan1_gsi, true);
+ gsi_insert_seq_after (&input2_gsi, gimple_omp_body (input_stmt2),
+ GSI_SAME_STMT);
+ gsi_remove (&input2_gsi, true);
+ gsi_insert_seq_after (&scan2_gsi, gimple_omp_body (scan_stmt2),
+ GSI_SAME_STMT);
+ gsi_remove (&scan2_gsi, true);
+
+ gimple_seq_add_seq (body_p, clist);
+
+ tree lab1 = create_artificial_label (UNKNOWN_LOCATION);
+ tree lab2 = create_artificial_label (UNKNOWN_LOCATION);
+ tree lab3 = create_artificial_label (UNKNOWN_LOCATION);
+ g = gimple_build_cond (EQ_EXPR, thread_num, integer_zero_node, lab1, lab2);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_label (lab1);
+ gimple_seq_add_stmt (body_p, g);
+ gimple_seq_add_seq (body_p, thr01_list);
+ g = gimple_build_goto (lab3);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_label (lab2);
+ gimple_seq_add_stmt (body_p, g);
+ gimple_seq_add_seq (body_p, thrn1_list);
+ g = gimple_build_label (lab3);
+ gimple_seq_add_stmt (body_p, g);
- clauses = gimple_omp_for_clauses (fd->for_stmt);
- stmts = NULL;
- lower_lastprivate_clauses (clauses, cond, &stmts, ctx);
- if (!gimple_seq_empty_p (stmts))
- {
- gimple_seq_add_seq (&stmts, *dlist);
- *dlist = stmts;
+ g = gimple_build_assign (ivar, size_zero_node);
+ gimple_seq_add_stmt (body_p, g);
- /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */
- vinit = fd->loop.n1;
- if (cond_code == EQ_EXPR
- && tree_fits_shwi_p (fd->loop.n2)
- && ! integer_zerop (fd->loop.n2))
- vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0);
- else
- vinit = unshare_expr (vinit);
+ gimple_seq_add_stmt (body_p, stmt);
+ gimple_seq_add_seq (body_p, body);
+ gimple_seq_add_stmt (body_p, gimple_build_omp_continue (fd->loop.v,
+ fd->loop.v));
- /* Initialize the iterator variable, so that threads that don't execute
- any iterations don't execute the lastprivate clauses by accident. */
- gimplify_assign (fd->loop.v, vinit, body_p);
- }
-}
+ g = gimple_build_omp_return (true);
+ gimple_seq_add_stmt (body_p, g);
+ gimple_seq_add_seq (body_p, mdlist);
+
+ lab1 = create_artificial_label (UNKNOWN_LOCATION);
+ lab2 = create_artificial_label (UNKNOWN_LOCATION);
+ g = gimple_build_cond (GT_EXPR, num_threads, integer_one_node, lab1, lab2);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_label (lab1);
+ gimple_seq_add_stmt (body_p, g);
+
+ g = omp_build_barrier (NULL);
+ gimple_seq_add_stmt (body_p, g);
+
+ tree down = create_tmp_var (unsigned_type_node);
+ g = gimple_build_assign (down, build_zero_cst (unsigned_type_node));
+ gimple_seq_add_stmt (body_p, g);
+
+ g = gimple_build_assign (k, build_one_cst (unsigned_type_node));
+ gimple_seq_add_stmt (body_p, g);
+
+ tree num_threadsu = create_tmp_var (unsigned_type_node);
+ g = gimple_build_assign (num_threadsu, NOP_EXPR, num_threads);
+ gimple_seq_add_stmt (body_p, g);
+ tree thread_numu = create_tmp_var (unsigned_type_node);
+ g = gimple_build_assign (thread_numu, NOP_EXPR, thread_num);
+ gimple_seq_add_stmt (body_p, g);
+
+ tree thread_nump1 = create_tmp_var (unsigned_type_node);
+ g = gimple_build_assign (thread_nump1, PLUS_EXPR, thread_numu,
+ build_int_cst (unsigned_type_node, 1));
+ gimple_seq_add_stmt (body_p, g);
+
+ lab3 = create_artificial_label (UNKNOWN_LOCATION);
+ g = gimple_build_label (lab3);
+ gimple_seq_add_stmt (body_p, g);
+
+ tree twok = create_tmp_var (unsigned_type_node);
+ g = gimple_build_assign (twok, LSHIFT_EXPR, k, integer_one_node);
+ gimple_seq_add_stmt (body_p, g);
+
+ tree lab4 = create_artificial_label (UNKNOWN_LOCATION);
+ tree lab5 = create_artificial_label (UNKNOWN_LOCATION);
+ tree lab6 = create_artificial_label (UNKNOWN_LOCATION);
+ g = gimple_build_cond (GT_EXPR, twok, num_threadsu, lab4, lab5);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_label (lab4);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_assign (down, build_all_ones_cst (unsigned_type_node));
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
+ gimple_seq_add_stmt (body_p, g);
+
+ g = gimple_build_cond (EQ_EXPR, k, num_threadsu, lab6, lab5);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_label (lab6);
+ gimple_seq_add_stmt (body_p, g);
+
+ g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
+ gimple_seq_add_stmt (body_p, g);
+
+ g = gimple_build_label (lab5);
+ gimple_seq_add_stmt (body_p, g);
+
+ g = gimple_build_assign (twok, LSHIFT_EXPR, k, integer_one_node);
+ gimple_seq_add_stmt (body_p, g);
+
+ tree cplx = create_tmp_var (build_complex_type (unsigned_type_node, false));
+ g = gimple_build_call_internal (IFN_MUL_OVERFLOW, 2, thread_nump1, twok);
+ gimple_call_set_lhs (g, cplx);
+ gimple_seq_add_stmt (body_p, g);
+ tree mul = create_tmp_var (unsigned_type_node);
+ g = gimple_build_assign (mul, REALPART_EXPR,
+ build1 (REALPART_EXPR, unsigned_type_node, cplx));
+ gimple_seq_add_stmt (body_p, g);
+ tree ovf = create_tmp_var (unsigned_type_node);
+ g = gimple_build_assign (ovf, IMAGPART_EXPR,
+ build1 (IMAGPART_EXPR, unsigned_type_node, cplx));
+ gimple_seq_add_stmt (body_p, g);
+
+ tree lab7 = create_artificial_label (UNKNOWN_LOCATION);
+ tree lab8 = create_artificial_label (UNKNOWN_LOCATION);
+ g = gimple_build_cond (EQ_EXPR, ovf, build_zero_cst (unsigned_type_node),
+ lab7, lab8);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_label (lab7);
+ gimple_seq_add_stmt (body_p, g);
+
+ tree andv = create_tmp_var (unsigned_type_node);
+ g = gimple_build_assign (andv, BIT_AND_EXPR, k, down);
+ gimple_seq_add_stmt (body_p, g);
+ tree andvm1 = create_tmp_var (unsigned_type_node);
+ g = gimple_build_assign (andvm1, PLUS_EXPR, andv,
+ build_minus_one_cst (unsigned_type_node));
+ gimple_seq_add_stmt (body_p, g);
+
+ g = gimple_build_assign (l, PLUS_EXPR, mul, andvm1);
+ gimple_seq_add_stmt (body_p, g);
+
+ tree lab9 = create_artificial_label (UNKNOWN_LOCATION);
+ g = gimple_build_cond (LT_EXPR, l, num_threadsu, lab9, lab8);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_label (lab9);
+ gimple_seq_add_stmt (body_p, g);
+ gimple_seq_add_seq (body_p, reduc_list);
+ g = gimple_build_label (lab8);
+ gimple_seq_add_stmt (body_p, g);
+
+ tree lab10 = create_artificial_label (UNKNOWN_LOCATION);
+ tree lab11 = create_artificial_label (UNKNOWN_LOCATION);
+ tree lab12 = create_artificial_label (UNKNOWN_LOCATION);
+ g = gimple_build_cond (EQ_EXPR, down, build_zero_cst (unsigned_type_node),
+ lab10, lab11);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_label (lab10);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_assign (k, LSHIFT_EXPR, k, integer_one_node);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_goto (lab12);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_label (lab11);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_assign (k, RSHIFT_EXPR, k, integer_one_node);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_label (lab12);
+ gimple_seq_add_stmt (body_p, g);
+
+ g = omp_build_barrier (NULL);
+ gimple_seq_add_stmt (body_p, g);
+
+ g = gimple_build_cond (NE_EXPR, k, build_zero_cst (unsigned_type_node),
+ lab3, lab2);
+ gimple_seq_add_stmt (body_p, g);
+
+ g = gimple_build_label (lab2);
+ gimple_seq_add_stmt (body_p, g);
+
+ lab1 = create_artificial_label (UNKNOWN_LOCATION);
+ lab2 = create_artificial_label (UNKNOWN_LOCATION);
+ lab3 = create_artificial_label (UNKNOWN_LOCATION);
+ g = gimple_build_cond (EQ_EXPR, thread_num, integer_zero_node, lab1, lab2);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_label (lab1);
+ gimple_seq_add_stmt (body_p, g);
+ gimple_seq_add_seq (body_p, thr02_list);
+ g = gimple_build_goto (lab3);
+ gimple_seq_add_stmt (body_p, g);
+ g = gimple_build_label (lab2);
+ gimple_seq_add_stmt (body_p, g);
+ gimple_seq_add_seq (body_p, thrn2_list);
+ g = gimple_build_label (lab3);
+ gimple_seq_add_stmt (body_p, g);
+
+ g = gimple_build_assign (ivar, size_zero_node);
+ gimple_seq_add_stmt (body_p, g);
+ gimple_seq_add_stmt (body_p, new_stmt);
+ gimple_seq_add_seq (body_p, new_body);
+
+ gimple_seq new_dlist = NULL;
+ lab1 = create_artificial_label (UNKNOWN_LOCATION);
+ lab2 = create_artificial_label (UNKNOWN_LOCATION);
+ tree num_threadsm1 = create_tmp_var (integer_type_node);
+ g = gimple_build_assign (num_threadsm1, PLUS_EXPR, num_threads,
+ integer_minus_one_node);
+ gimple_seq_add_stmt (&new_dlist, g);
+ g = gimple_build_cond (EQ_EXPR, thread_num, num_threadsm1, lab1, lab2);
+ gimple_seq_add_stmt (&new_dlist, g);
+ g = gimple_build_label (lab1);
+ gimple_seq_add_stmt (&new_dlist, g);
+ gimple_seq_add_seq (&new_dlist, last_list);
+ g = gimple_build_label (lab2);
+ gimple_seq_add_stmt (&new_dlist, g);
+ gimple_seq_add_seq (&new_dlist, *dlist);
+ *dlist = new_dlist;
+}
/* Lower code for an OMP loop directive. */
struct omp_for_data fd, *fdp = NULL;
gomp_for *stmt = as_a <gomp_for *> (gsi_stmt (*gsi_p));
gbind *new_stmt;
- gimple_seq omp_for_body, body, dlist;
+ gimple_seq omp_for_body, body, dlist, tred_ilist = NULL, tred_dlist = NULL;
+ gimple_seq cnt_list = NULL, clist = NULL;
gimple_seq oacc_head = NULL, oacc_tail = NULL;
size_t i;
if (fd.collapse > 1
&& TREE_CODE (fd.loop.n2) != INTEGER_CST)
count += fd.collapse - 1;
+ size_t count2 = 0;
+ tree type2 = NULL_TREE;
bool taskreg_for
= (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR
|| gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_TASKLOOP);
tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt);
tree simtc = NULL;
tree clauses = *pc;
+ if (fd.collapse > 1
+ && fd.non_rect
+ && fd.last_nonrect == fd.first_nonrect + 1
+ && TREE_CODE (fd.loop.n2) != INTEGER_CST)
+ if (tree v = gimple_omp_for_index (stmt, fd.last_nonrect))
+ if (!TYPE_UNSIGNED (TREE_TYPE (v)))
+ {
+ v = gimple_omp_for_index (stmt, fd.first_nonrect);
+ type2 = TREE_TYPE (v);
+ count++;
+ count2 = 3;
+ }
if (taskreg_for)
outerc
= omp_find_clause (gimple_omp_taskreg_clauses (ctx->outer->stmt),
if (ctx->simt_stmt)
simtc = omp_find_clause (gimple_omp_for_clauses (ctx->simt_stmt),
OMP_CLAUSE__LOOPTEMP_);
- for (i = 0; i < count; i++)
+ for (i = 0; i < count + count2; i++)
{
tree temp;
if (taskreg_for)
if (ctx->simt_stmt)
temp = OMP_CLAUSE_DECL (simtc);
else
- temp = create_tmp_var (type);
+ temp = create_tmp_var (i >= count ? type2 : type);
insert_decl_map (&ctx->outer->cb, temp, temp);
}
*pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_);
/* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */
dlist = NULL;
body = NULL;
+ tree rclauses
+ = omp_task_reductions_find_first (gimple_omp_for_clauses (stmt), OMP_FOR,
+ OMP_CLAUSE_REDUCTION);
+ tree rtmp = NULL_TREE;
+ if (rclauses)
+ {
+ tree type = build_pointer_type (pointer_sized_int_node);
+ tree temp = create_tmp_var (type);
+ tree c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__REDUCTEMP_);
+ OMP_CLAUSE_DECL (c) = temp;
+ OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (stmt);
+ gimple_omp_for_set_clauses (stmt, c);
+ lower_omp_task_reductions (ctx, OMP_FOR,
+ gimple_omp_for_clauses (stmt),
+ &tred_ilist, &tred_dlist);
+ rclauses = c;
+ rtmp = make_ssa_name (type);
+ gimple_seq_add_stmt (&body, gimple_build_assign (rtmp, temp));
+ }
+
+ lower_lastprivate_conditional_clauses (gimple_omp_for_clauses_ptr (stmt),
+ ctx);
+
lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx,
fdp);
- gimple_seq_add_seq (&body, gimple_omp_for_pre_body (stmt));
+ gimple_seq_add_seq (rclauses ? &tred_ilist : &body,
+ gimple_omp_for_pre_body (stmt));
lower_omp (gimple_omp_body_ptr (stmt), ctx);
for (i = 0; i < gimple_omp_for_collapse (stmt); i++)
{
rhs_p = gimple_omp_for_initial_ptr (stmt, i);
- if (!is_gimple_min_invariant (*rhs_p))
- *rhs_p = get_formal_tmp_var (*rhs_p, &body);
+ if (TREE_CODE (*rhs_p) == TREE_VEC)
+ {
+ if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 1)))
+ TREE_VEC_ELT (*rhs_p, 1)
+ = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 1), &cnt_list);
+ if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 2)))
+ TREE_VEC_ELT (*rhs_p, 2)
+ = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 2), &cnt_list);
+ }
+ else if (!is_gimple_min_invariant (*rhs_p))
+ *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
else if (TREE_CODE (*rhs_p) == ADDR_EXPR)
recompute_tree_invariant_for_addr_expr (*rhs_p);
rhs_p = gimple_omp_for_final_ptr (stmt, i);
- if (!is_gimple_min_invariant (*rhs_p))
- *rhs_p = get_formal_tmp_var (*rhs_p, &body);
+ if (TREE_CODE (*rhs_p) == TREE_VEC)
+ {
+ if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 1)))
+ TREE_VEC_ELT (*rhs_p, 1)
+ = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 1), &cnt_list);
+ if (!is_gimple_min_invariant (TREE_VEC_ELT (*rhs_p, 2)))
+ TREE_VEC_ELT (*rhs_p, 2)
+ = get_formal_tmp_var (TREE_VEC_ELT (*rhs_p, 2), &cnt_list);
+ }
+ else if (!is_gimple_min_invariant (*rhs_p))
+ *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
else if (TREE_CODE (*rhs_p) == ADDR_EXPR)
recompute_tree_invariant_for_addr_expr (*rhs_p);
rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1);
if (!is_gimple_min_invariant (*rhs_p))
- *rhs_p = get_formal_tmp_var (*rhs_p, &body);
+ *rhs_p = get_formal_tmp_var (*rhs_p, &cnt_list);
}
+ if (rclauses)
+ gimple_seq_add_seq (&tred_ilist, cnt_list);
+ else
+ gimple_seq_add_seq (&body, cnt_list);
/* Once lowered, extract the bounds and clauses. */
omp_extract_for_data (stmt, &fd, NULL);
if (oacc_head)
gimple_seq_add_seq (&body, oacc_head);
- lower_omp_for_lastprivate (&fd, &body, &dlist, ctx);
+ lower_omp_for_lastprivate (&fd, &body, &dlist, &clist, ctx);
if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
for (tree c = gimple_omp_for_clauses (stmt); c; c = OMP_CLAUSE_CHAIN (c))
ctx);
}
- bool phony_loop = (gimple_omp_for_kind (stmt) != GF_OMP_FOR_KIND_GRID_LOOP
- && gimple_omp_for_grid_phony (stmt));
- if (!phony_loop)
- gimple_seq_add_stmt (&body, stmt);
- gimple_seq_add_seq (&body, gimple_omp_body (stmt));
+ if ((ctx->scan_inclusive || ctx->scan_exclusive)
+ && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR)
+ lower_omp_for_scan (&body, &dlist, stmt, &fd, ctx);
+ else
+ {
+ gimple_seq_add_stmt (&body, stmt);
+ gimple_seq_add_seq (&body, gimple_omp_body (stmt));
+ }
- if (!phony_loop)
- gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v,
- fd.loop.v));
+ gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v,
+ fd.loop.v));
/* After the loop, add exit clauses. */
- lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, ctx);
+ lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, &clist, ctx);
+
+ if (clist)
+ {
+ tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START);
+ gcall *g = gimple_build_call (fndecl, 0);
+ gimple_seq_add_stmt (&body, g);
+ gimple_seq_add_seq (&body, clist);
+ fndecl = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END);
+ g = gimple_build_call (fndecl, 0);
+ gimple_seq_add_stmt (&body, g);
+ }
if (ctx->cancellable)
gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label));
gimple_seq_add_seq (&body, dlist);
- body = maybe_catch_exception (body);
-
- if (!phony_loop)
+ if (rclauses)
{
- /* Region exit marker goes at the end of the loop body. */
- gimple_seq_add_stmt (&body, gimple_build_omp_return (fd.have_nowait));
- maybe_add_implicit_barrier_cancel (ctx, &body);
+ gimple_seq_add_seq (&tred_ilist, body);
+ body = tred_ilist;
}
+ body = maybe_catch_exception (body);
+
+ /* Region exit marker goes at the end of the loop body. */
+ gimple *g = gimple_build_omp_return (fd.have_nowait);
+ gimple_seq_add_stmt (&body, g);
+
+ gimple_seq_add_seq (&body, tred_dlist);
+
+ maybe_add_implicit_barrier_cancel (ctx, g, &body);
+
+ if (rclauses)
+ OMP_CLAUSE_DECL (rclauses) = rtmp;
+
/* Add OpenACC joining and reduction markers just after the loop. */
if (oacc_tail)
gimple_seq_add_seq (&body, oacc_tail);
splay_tree_node n;
struct omp_taskcopy_context tcctx;
location_t loc = gimple_location (task_stmt);
+ size_t looptempno = 0;
child_fn = gimple_omp_task_copy_fn (task_stmt);
child_cfun = DECL_STRUCT_FUNCTION (child_fn);
t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
append_to_statement_list (t, &list);
break;
+ case OMP_CLAUSE_REDUCTION:
+ case OMP_CLAUSE_IN_REDUCTION:
+ decl = OMP_CLAUSE_DECL (c);
+ if (TREE_CODE (decl) == MEM_REF)
+ {
+ decl = TREE_OPERAND (decl, 0);
+ if (TREE_CODE (decl) == POINTER_PLUS_EXPR)
+ decl = TREE_OPERAND (decl, 0);
+ if (TREE_CODE (decl) == INDIRECT_REF
+ || TREE_CODE (decl) == ADDR_EXPR)
+ decl = TREE_OPERAND (decl, 0);
+ }
+ key = (splay_tree_key) decl;
+ n = splay_tree_lookup (ctx->field_map, key);
+ if (n == NULL)
+ break;
+ f = (tree) n->value;
+ if (tcctx.cb.decl_map)
+ f = *tcctx.cb.decl_map->get (f);
+ n = splay_tree_lookup (ctx->sfield_map, key);
+ sf = (tree) n->value;
+ if (tcctx.cb.decl_map)
+ sf = *tcctx.cb.decl_map->get (sf);
+ src = build_simple_mem_ref_loc (loc, sarg);
+ src = omp_build_component_ref (src, sf);
+ if (decl != OMP_CLAUSE_DECL (c)
+ && TREE_CODE (TREE_TYPE (decl)) == REFERENCE_TYPE
+ && TREE_CODE (TREE_TYPE (TREE_TYPE (decl))) == POINTER_TYPE)
+ src = build_simple_mem_ref_loc (loc, src);
+ dst = build_simple_mem_ref_loc (loc, arg);
+ dst = omp_build_component_ref (dst, f);
+ t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
+ append_to_statement_list (t, &list);
+ break;
+ case OMP_CLAUSE__LOOPTEMP_:
+ /* Fields for first two _looptemp_ clauses are initialized by
+ GOMP_taskloop*, the rest are handled like firstprivate. */
+ if (looptempno < 2)
+ {
+ looptempno++;
+ break;
+ }
+ /* FALLTHRU */
+ case OMP_CLAUSE__REDUCTEMP_:
case OMP_CLAUSE_FIRSTPRIVATE:
decl = OMP_CLAUSE_DECL (c);
if (is_variable_sized (decl))
src = decl;
dst = build_simple_mem_ref_loc (loc, arg);
dst = omp_build_component_ref (dst, f);
- t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
+ if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FIRSTPRIVATE)
+ t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src);
+ else
+ {
+ if (ctx->allocate_map)
+ if (tree *allocatorp = ctx->allocate_map->get (decl))
+ {
+ tree allocator = *allocatorp;
+ if (TREE_CODE (allocator) != INTEGER_CST)
+ {
+ n = splay_tree_lookup (ctx->sfield_map,
+ (splay_tree_key) allocator);
+ allocator = (tree) n->value;
+ if (tcctx.cb.decl_map)
+ allocator = *tcctx.cb.decl_map->get (allocator);
+ tree a = build_simple_mem_ref_loc (loc, sarg);
+ allocator = omp_build_component_ref (a, allocator);
+ }
+ allocator = fold_convert (pointer_sized_int_node, allocator);
+ tree a = builtin_decl_explicit (BUILT_IN_GOMP_ALLOC);
+ tree align = build_int_cst (size_type_node,
+ DECL_ALIGN_UNIT (decl));
+ tree sz = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (dst)));
+ tree ptr = build_call_expr_loc (loc, a, 3, align, sz,
+ allocator);
+ ptr = fold_convert (TREE_TYPE (dst), ptr);
+ t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, ptr);
+ append_to_statement_list (t, &list);
+ dst = build_simple_mem_ref_loc (loc, dst);
+ }
+ t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src);
+ }
append_to_statement_list (t, &list);
break;
case OMP_CLAUSE_PRIVATE:
{
tree c, clauses;
gimple *g;
- size_t n_in = 0, n_out = 0, idx = 2, i;
+ size_t cnt[4] = { 0, 0, 0, 0 }, idx = 2, i;
clauses = omp_find_clause (*pclauses, OMP_CLAUSE_DEPEND);
gcc_assert (clauses);
if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND)
switch (OMP_CLAUSE_DEPEND_KIND (c))
{
+ case OMP_CLAUSE_DEPEND_LAST:
+ /* Lowering already done at gimplification. */
+ return;
case OMP_CLAUSE_DEPEND_IN:
- n_in++;
+ cnt[2]++;
break;
case OMP_CLAUSE_DEPEND_OUT:
case OMP_CLAUSE_DEPEND_INOUT:
- n_out++;
+ cnt[0]++;
+ break;
+ case OMP_CLAUSE_DEPEND_MUTEXINOUTSET:
+ cnt[1]++;
+ break;
+ case OMP_CLAUSE_DEPEND_DEPOBJ:
+ cnt[3]++;
break;
case OMP_CLAUSE_DEPEND_SOURCE:
case OMP_CLAUSE_DEPEND_SINK:
default:
gcc_unreachable ();
}
- tree type = build_array_type_nelts (ptr_type_node, n_in + n_out + 2);
+ if (cnt[1] || cnt[3])
+ idx = 5;
+ size_t total = cnt[0] + cnt[1] + cnt[2] + cnt[3];
+ tree type = build_array_type_nelts (ptr_type_node, total + idx);
tree array = create_tmp_var (type);
TREE_ADDRESSABLE (array) = 1;
tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE,
NULL_TREE);
- g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_in + n_out));
- gimple_seq_add_stmt (iseq, g);
- r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE,
- NULL_TREE);
- g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_out));
+ if (idx == 5)
+ {
+ g = gimple_build_assign (r, build_int_cst (ptr_type_node, 0));
+ gimple_seq_add_stmt (iseq, g);
+ r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE,
+ NULL_TREE);
+ }
+ g = gimple_build_assign (r, build_int_cst (ptr_type_node, total));
gimple_seq_add_stmt (iseq, g);
- for (i = 0; i < 2; i++)
+ for (i = 0; i < (idx == 5 ? 3 : 1); i++)
+ {
+ r = build4 (ARRAY_REF, ptr_type_node, array,
+ size_int (i + 1 + (idx == 5)), NULL_TREE, NULL_TREE);
+ g = gimple_build_assign (r, build_int_cst (ptr_type_node, cnt[i]));
+ gimple_seq_add_stmt (iseq, g);
+ }
+ for (i = 0; i < 4; i++)
{
- if ((i ? n_in : n_out) == 0)
+ if (cnt[i] == 0)
continue;
for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c))
- if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND
- && ((OMP_CLAUSE_DEPEND_KIND (c) != OMP_CLAUSE_DEPEND_IN) ^ i))
+ if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_DEPEND)
+ continue;
+ else
{
+ switch (OMP_CLAUSE_DEPEND_KIND (c))
+ {
+ case OMP_CLAUSE_DEPEND_IN:
+ if (i != 2)
+ continue;
+ break;
+ case OMP_CLAUSE_DEPEND_OUT:
+ case OMP_CLAUSE_DEPEND_INOUT:
+ if (i != 0)
+ continue;
+ break;
+ case OMP_CLAUSE_DEPEND_MUTEXINOUTSET:
+ if (i != 1)
+ continue;
+ break;
+ case OMP_CLAUSE_DEPEND_DEPOBJ:
+ if (i != 3)
+ continue;
+ break;
+ default:
+ gcc_unreachable ();
+ }
tree t = OMP_CLAUSE_DECL (c);
t = fold_convert (ptr_type_node, t);
gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue);
}
}
c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND);
+ OMP_CLAUSE_DEPEND_KIND (c) = OMP_CLAUSE_DEPEND_LAST;
OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array);
OMP_CLAUSE_CHAIN (c) = *pclauses;
*pclauses = c;
- tree clobber = build_constructor (type, NULL);
- TREE_THIS_VOLATILE (clobber) = 1;
+ tree clobber = build_clobber (type);
g = gimple_build_assign (array, clobber);
gimple_seq_add_stmt (oseq, g);
}
tree child_fn, t;
gimple *stmt = gsi_stmt (*gsi_p);
gbind *par_bind, *bind, *dep_bind = NULL;
- gimple_seq par_body, olist, ilist, par_olist, par_rlist, par_ilist, new_body;
+ gimple_seq par_body;
location_t loc = gimple_location (stmt);
clauses = gimple_omp_taskreg_clauses (stmt);
- par_bind
- = as_a <gbind *> (gimple_seq_first_stmt (gimple_omp_body (stmt)));
- par_body = gimple_bind_body (par_bind);
+ if (gimple_code (stmt) == GIMPLE_OMP_TASK
+ && gimple_omp_task_taskwait_p (stmt))
+ {
+ par_bind = NULL;
+ par_body = NULL;
+ }
+ else
+ {
+ par_bind
+ = as_a <gbind *> (gimple_seq_first_stmt (gimple_omp_body (stmt)));
+ par_body = gimple_bind_body (par_bind);
+ }
child_fn = ctx->cb.dst_fn;
if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL
&& !gimple_omp_parallel_combined_p (stmt))
&dep_ilist, &dep_olist);
}
+ if (gimple_code (stmt) == GIMPLE_OMP_TASK
+ && gimple_omp_task_taskwait_p (stmt))
+ {
+ if (dep_bind)
+ {
+ gsi_replace (gsi_p, dep_bind, true);
+ gimple_bind_add_seq (dep_bind, dep_ilist);
+ gimple_bind_add_stmt (dep_bind, stmt);
+ gimple_bind_add_seq (dep_bind, dep_olist);
+ pop_gimplify_context (dep_bind);
+ }
+ return;
+ }
+
if (ctx->srecord_type)
create_task_copyfn (as_a <gomp_task *> (stmt), ctx);
+ gimple_seq tskred_ilist = NULL;
+ gimple_seq tskred_olist = NULL;
+ if ((is_task_ctx (ctx)
+ && gimple_omp_task_taskloop_p (ctx->stmt)
+ && omp_find_clause (gimple_omp_task_clauses (ctx->stmt),
+ OMP_CLAUSE_REDUCTION))
+ || (is_parallel_ctx (ctx)
+ && omp_find_clause (gimple_omp_parallel_clauses (stmt),
+ OMP_CLAUSE__REDUCTEMP_)))
+ {
+ if (dep_bind == NULL)
+ {
+ push_gimplify_context ();
+ dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
+ }
+ lower_omp_task_reductions (ctx, is_task_ctx (ctx) ? OMP_TASKLOOP
+ : OMP_PARALLEL,
+ gimple_omp_taskreg_clauses (ctx->stmt),
+ &tskred_ilist, &tskred_olist);
+ }
+
push_gimplify_context ();
- par_olist = NULL;
- par_ilist = NULL;
- par_rlist = NULL;
- bool phony_construct = gimple_code (stmt) == GIMPLE_OMP_PARALLEL
- && gimple_omp_parallel_grid_phony (as_a <gomp_parallel *> (stmt));
- if (phony_construct && ctx->record_type)
- {
- gcc_checking_assert (!ctx->receiver_decl);
- ctx->receiver_decl = create_tmp_var
- (build_reference_type (ctx->record_type), ".omp_rec");
- }
+ gimple_seq par_olist = NULL;
+ gimple_seq par_ilist = NULL;
+ gimple_seq par_rlist = NULL;
lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL);
lower_omp (&par_body, ctx);
- if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL)
- lower_reduction_clauses (clauses, &par_rlist, ctx);
+ if (gimple_code (stmt) != GIMPLE_OMP_TASK)
+ lower_reduction_clauses (clauses, &par_rlist, NULL, ctx);
/* Declare all the variables created by mapping and the variables
declared in the scope of the parallel body. */
gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl);
}
- olist = NULL;
- ilist = NULL;
+ gimple_seq olist = NULL;
+ gimple_seq ilist = NULL;
lower_send_clauses (clauses, &ilist, &olist, ctx);
lower_send_shared_vars (&ilist, &olist, ctx);
if (ctx->record_type)
{
- tree clobber = build_constructor (TREE_TYPE (ctx->sender_decl), NULL);
- TREE_THIS_VOLATILE (clobber) = 1;
+ tree clobber = build_clobber (TREE_TYPE (ctx->sender_decl));
gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
clobber));
}
/* Once all the expansions are done, sequence all the different
fragments inside gimple_omp_body. */
- new_body = NULL;
+ gimple_seq new_body = NULL;
if (ctx->record_type)
{
gimple_seq_add_stmt (&new_body,
gimple_build_omp_continue (integer_zero_node,
integer_zero_node));
- if (!phony_construct)
- {
- gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
- gimple_omp_set_body (stmt, new_body);
- }
+ gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
+ gimple_omp_set_body (stmt, new_body);
- bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
+ if (dep_bind && gimple_bind_block (par_bind) == NULL_TREE)
+ bind = gimple_build_bind (NULL, NULL, make_node (BLOCK));
+ else
+ bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind));
gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true);
gimple_bind_add_seq (bind, ilist);
- if (!phony_construct)
- gimple_bind_add_stmt (bind, stmt);
- else
- gimple_bind_add_seq (bind, new_body);
+ gimple_bind_add_stmt (bind, stmt);
gimple_bind_add_seq (bind, olist);
pop_gimplify_context (NULL);
if (dep_bind)
{
gimple_bind_add_seq (dep_bind, dep_ilist);
+ gimple_bind_add_seq (dep_bind, tskred_ilist);
gimple_bind_add_stmt (dep_bind, bind);
+ gimple_bind_add_seq (dep_bind, tskred_olist);
gimple_bind_add_seq (dep_bind, dep_olist);
pop_gimplify_context (dep_bind);
}
case GF_OMP_TARGET_KIND_EXIT_DATA:
case GF_OMP_TARGET_KIND_OACC_PARALLEL:
case GF_OMP_TARGET_KIND_OACC_KERNELS:
+ case GF_OMP_TARGET_KIND_OACC_SERIAL:
case GF_OMP_TARGET_KIND_OACC_UPDATE:
case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA:
case GF_OMP_TARGET_KIND_OACC_DECLARE:
+ case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_PARALLELIZED:
+ case GF_OMP_TARGET_KIND_OACC_PARALLEL_KERNELS_GANG_SINGLE:
data_region = false;
break;
case GF_OMP_TARGET_KIND_DATA:
case GF_OMP_TARGET_KIND_OACC_DATA:
case GF_OMP_TARGET_KIND_OACC_HOST_DATA:
+ case GF_OMP_TARGET_KIND_OACC_DATA_KERNELS:
data_region = true;
break;
default:
case GOMP_MAP_FIRSTPRIVATE_REFERENCE:
case GOMP_MAP_STRUCT:
case GOMP_MAP_ALWAYS_POINTER:
+ case GOMP_MAP_ATTACH:
+ case GOMP_MAP_DETACH:
break;
+ case GOMP_MAP_IF_PRESENT:
case GOMP_MAP_FORCE_ALLOC:
case GOMP_MAP_FORCE_TO:
case GOMP_MAP_FORCE_FROM:
case GOMP_MAP_FORCE_DEVICEPTR:
case GOMP_MAP_DEVICE_RESIDENT:
case GOMP_MAP_LINK:
+ case GOMP_MAP_FORCE_DETACH:
gcc_assert (is_gimple_omp_oacc (stmt));
break;
default:
continue;
}
+ if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
+ && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
+ || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
+ && is_omp_target (stmt))
+ {
+ gcc_assert (maybe_lookup_field (c, ctx));
+ map_cnt++;
+ continue;
+ }
+
if (!maybe_lookup_field (var, ctx))
continue;
- /* Don't remap oacc parallel reduction variables, because the
+ /* Don't remap compute constructs' reduction variables, because the
intermediate result must be local to each gang. */
if (offloaded && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
&& OMP_CLAUSE_MAP_IN_REDUCTION (c)))
{
gcc_assert (is_gimple_omp_oacc (ctx->stmt));
if (omp_is_reference (new_var)
- && TREE_CODE (TREE_TYPE (new_var)) != POINTER_TYPE)
+ && (TREE_CODE (TREE_TYPE (new_var)) != POINTER_TYPE
+ || DECL_BY_REFERENCE (var)))
{
/* Create a local object to hold the instance
value. */
break;
case OMP_CLAUSE_FIRSTPRIVATE:
- if (is_oacc_parallel (ctx))
- goto oacc_firstprivate;
+ gcc_checking_assert (offloaded);
+ if (is_gimple_omp_oacc (ctx->stmt))
+ {
+ /* No 'firstprivate' clauses on OpenACC 'kernels'. */
+ gcc_checking_assert (!is_oacc_kernels (ctx));
+ /* Likewise, on OpenACC 'kernels' decomposed parts. */
+ gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
+
+ goto oacc_firstprivate;
+ }
map_cnt++;
var = OMP_CLAUSE_DECL (c);
if (!omp_is_reference (var)
break;
case OMP_CLAUSE_PRIVATE:
+ gcc_checking_assert (offloaded);
if (is_gimple_omp_oacc (ctx->stmt))
- break;
+ {
+ /* No 'private' clauses on OpenACC 'kernels'. */
+ gcc_checking_assert (!is_oacc_kernels (ctx));
+ /* Likewise, on OpenACC 'kernels' decomposed parts. */
+ gcc_checking_assert (!is_oacc_kernels_decomposed_part (ctx));
+
+ break;
+ }
var = OMP_CLAUSE_DECL (c);
if (is_variable_sized (var))
{
break;
case OMP_CLAUSE_USE_DEVICE_PTR:
+ case OMP_CLAUSE_USE_DEVICE_ADDR:
case OMP_CLAUSE_IS_DEVICE_PTR:
var = OMP_CLAUSE_DECL (c);
map_cnt++;
SET_DECL_VALUE_EXPR (new_var, x);
DECL_HAS_VALUE_EXPR_P (new_var) = 1;
}
- else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
+ else if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
+ && !omp_is_reference (var)
+ && !omp_is_allocatable_or_ptr (var)
+ && !lang_hooks.decls.omp_array_data (var, true))
+ || TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
{
tree new_var = lookup_decl (var, ctx);
tree type = build_pointer_type (TREE_TYPE (var));
gcc_assert (DECL_P (ovar2));
ovar = ovar2;
}
- if (!maybe_lookup_field (ovar, ctx))
+ if (!maybe_lookup_field (ovar, ctx)
+ && !(OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
+ && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
+ || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)))
continue;
}
talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar));
if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign)
talign = DECL_ALIGN_UNIT (ovar);
- if (nc)
+
+ if (nc
+ && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP
+ && (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_ATTACH
+ || OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_DETACH)
+ && is_omp_target (stmt))
+ {
+ var = lookup_decl_in_outer_ctx (ovar, ctx);
+ x = build_sender_ref (c, ctx);
+ gimplify_assign (x, build_fold_addr_expr (var), &ilist);
+ }
+ else if (nc)
{
var = lookup_decl_in_outer_ctx (ovar, ctx);
x = build_sender_ref (ovar, ctx);
}
else
{
- var = build_fold_addr_expr (var);
+ /* While MAP is handled explicitly by the FE,
+ for 'target update', only the identified is passed. */
+ if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FROM
+ || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_TO)
+ && (omp_is_allocatable_or_ptr (var)
+ && omp_check_optional_argument (var, false)))
+ var = build_fold_indirect_ref (var);
+ else if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_FROM
+ && OMP_CLAUSE_CODE (c) != OMP_CLAUSE_TO)
+ || (!omp_is_allocatable_or_ptr (var)
+ && !omp_check_optional_argument (var, false)))
+ var = build_fold_addr_expr (var);
gimplify_assign (x, var, &ilist);
}
}
{
gcc_checking_assert (is_gimple_omp_oacc (ctx->stmt));
s = TREE_TYPE (ovar);
- if (TREE_CODE (s) == REFERENCE_TYPE)
+ if (TREE_CODE (s) == REFERENCE_TYPE
+ || omp_check_optional_argument (ovar, false))
s = TREE_TYPE (s);
s = TYPE_SIZE_UNIT (s);
}
switch (tkind)
{
case GOMP_MAP_ALLOC:
+ case GOMP_MAP_IF_PRESENT:
case GOMP_MAP_TO:
case GOMP_MAP_FROM:
case GOMP_MAP_TOFROM:
break;
case OMP_CLAUSE_FIRSTPRIVATE:
- if (is_oacc_parallel (ctx))
+ if (is_gimple_omp_oacc (ctx->stmt))
goto oacc_firstprivate_map;
ovar = OMP_CLAUSE_DECL (c);
if (omp_is_reference (ovar))
break;
case OMP_CLAUSE_USE_DEVICE_PTR:
+ case OMP_CLAUSE_USE_DEVICE_ADDR:
case OMP_CLAUSE_IS_DEVICE_PTR:
ovar = OMP_CLAUSE_DECL (c);
var = lookup_decl_in_outer_ctx (ovar, ctx);
- x = build_sender_ref (ovar, ctx);
- if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_PTR)
- tkind = GOMP_MAP_USE_DEVICE_PTR;
+
+ if (lang_hooks.decls.omp_array_data (ovar, true))
+ {
+ tkind = (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR
+ ? GOMP_MAP_USE_DEVICE_PTR : GOMP_MAP_FIRSTPRIVATE_INT);
+ x = build_sender_ref ((splay_tree_key) &DECL_NAME (ovar), ctx);
+ }
+ else if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR)
+ {
+ tkind = GOMP_MAP_USE_DEVICE_PTR;
+ x = build_sender_ref ((splay_tree_key) &DECL_UID (ovar), ctx);
+ }
else
- tkind = GOMP_MAP_FIRSTPRIVATE_INT;
+ {
+ tkind = GOMP_MAP_FIRSTPRIVATE_INT;
+ x = build_sender_ref (ovar, ctx);
+ }
+
+ if (is_gimple_omp_oacc (ctx->stmt))
+ {
+ gcc_assert (tkind == GOMP_MAP_USE_DEVICE_PTR);
+
+ if (OMP_CLAUSE_USE_DEVICE_PTR_IF_PRESENT (c))
+ tkind = GOMP_MAP_USE_DEVICE_PTR_IF_PRESENT;
+ }
+
type = TREE_TYPE (ovar);
- if (TREE_CODE (type) == ARRAY_TYPE)
+ if (lang_hooks.decls.omp_array_data (ovar, true))
+ var = lang_hooks.decls.omp_array_data (ovar, false);
+ else if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
+ && !omp_is_reference (ovar)
+ && !omp_is_allocatable_or_ptr (ovar))
+ || TREE_CODE (type) == ARRAY_TYPE)
var = build_fold_addr_expr (var);
else
{
- if (omp_is_reference (ovar))
+ if (omp_is_reference (ovar)
+ || omp_check_optional_argument (ovar, false)
+ || omp_is_allocatable_or_ptr (ovar))
{
type = TREE_TYPE (type);
- if (TREE_CODE (type) != ARRAY_TYPE)
+ if (POINTER_TYPE_P (type)
+ && TREE_CODE (type) != ARRAY_TYPE
+ && ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_USE_DEVICE_ADDR
+ && !omp_is_allocatable_or_ptr (ovar))
+ || (omp_is_reference (ovar)
+ && omp_is_allocatable_or_ptr (ovar))))
var = build_simple_mem_ref (var);
var = fold_convert (TREE_TYPE (x), var);
}
}
- gimplify_assign (x, var, &ilist);
+ tree present;
+ present = omp_check_optional_argument (ovar, true);
+ if (present)
+ {
+ tree null_label = create_artificial_label (UNKNOWN_LOCATION);
+ tree notnull_label = create_artificial_label (UNKNOWN_LOCATION);
+ tree opt_arg_label = create_artificial_label (UNKNOWN_LOCATION);
+ tree new_x = unshare_expr (x);
+ gimplify_expr (&present, &ilist, NULL, is_gimple_val,
+ fb_rvalue);
+ gcond *cond = gimple_build_cond_from_tree (present,
+ notnull_label,
+ null_label);
+ gimple_seq_add_stmt (&ilist, cond);
+ gimple_seq_add_stmt (&ilist, gimple_build_label (null_label));
+ gimplify_assign (new_x, null_pointer_node, &ilist);
+ gimple_seq_add_stmt (&ilist, gimple_build_goto (opt_arg_label));
+ gimple_seq_add_stmt (&ilist,
+ gimple_build_label (notnull_label));
+ gimplify_assign (x, var, &ilist);
+ gimple_seq_add_stmt (&ilist,
+ gimple_build_label (opt_arg_label));
+ }
+ else
+ gimplify_assign (x, var, &ilist);
s = size_int (0);
purpose = size_int (map_idx++);
CONSTRUCTOR_APPEND_ELT (vsize, purpose, s);
&initlist, true, NULL_TREE);
gimple_seq_add_seq (&ilist, initlist);
- tree clobber = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, i)),
- NULL);
- TREE_THIS_VOLATILE (clobber) = 1;
+ tree clobber = build_clobber (TREE_TYPE (TREE_VEC_ELT (t, i)));
gimple_seq_add_stmt (&olist,
gimple_build_assign (TREE_VEC_ELT (t, i),
clobber));
}
- tree clobber = build_constructor (ctx->record_type, NULL);
- TREE_THIS_VOLATILE (clobber) = 1;
+ tree clobber = build_clobber (ctx->record_type);
gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl,
clobber));
}
}
break;
case OMP_CLAUSE_USE_DEVICE_PTR:
+ case OMP_CLAUSE_USE_DEVICE_ADDR:
case OMP_CLAUSE_IS_DEVICE_PTR:
+ tree new_var;
+ gimple_seq assign_body;
+ bool is_array_data;
+ bool do_optional_check;
+ assign_body = NULL;
+ do_optional_check = false;
var = OMP_CLAUSE_DECL (c);
- if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_PTR)
- x = build_sender_ref (var, ctx);
+ is_array_data = lang_hooks.decls.omp_array_data (var, true) != NULL;
+
+ if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_IS_DEVICE_PTR)
+ x = build_sender_ref (is_array_data
+ ? (splay_tree_key) &DECL_NAME (var)
+ : (splay_tree_key) &DECL_UID (var), ctx);
else
x = build_receiver_ref (var, false, ctx);
- if (is_variable_sized (var))
+
+ if (is_array_data)
+ {
+ bool is_ref = omp_is_reference (var);
+ do_optional_check = true;
+ /* First, we copy the descriptor data from the host; then
+ we update its data to point to the target address. */
+ new_var = lookup_decl (var, ctx);
+ new_var = DECL_VALUE_EXPR (new_var);
+ tree v = new_var;
+
+ if (is_ref)
+ {
+ var = build_fold_indirect_ref (var);
+ gimplify_expr (&var, &assign_body, NULL, is_gimple_val,
+ fb_rvalue);
+ v = create_tmp_var_raw (TREE_TYPE (var), get_name (var));
+ gimple_add_tmp_var (v);
+ TREE_ADDRESSABLE (v) = 1;
+ gimple_seq_add_stmt (&assign_body,
+ gimple_build_assign (v, var));
+ tree rhs = build_fold_addr_expr (v);
+ gimple_seq_add_stmt (&assign_body,
+ gimple_build_assign (new_var, rhs));
+ }
+ else
+ gimple_seq_add_stmt (&assign_body,
+ gimple_build_assign (new_var, var));
+
+ tree v2 = lang_hooks.decls.omp_array_data (unshare_expr (v), false);
+ gcc_assert (v2);
+ gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
+ gimple_seq_add_stmt (&assign_body,
+ gimple_build_assign (v2, x));
+ }
+ else if (is_variable_sized (var))
{
tree pvar = DECL_VALUE_EXPR (var);
gcc_assert (TREE_CODE (pvar) == INDIRECT_REF);
pvar = TREE_OPERAND (pvar, 0);
gcc_assert (DECL_P (pvar));
- tree new_var = lookup_decl (pvar, ctx);
- gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
- gimple_seq_add_stmt (&new_body,
+ new_var = lookup_decl (pvar, ctx);
+ gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
+ gimple_seq_add_stmt (&assign_body,
gimple_build_assign (new_var, x));
}
- else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
+ else if ((OMP_CLAUSE_CODE (c) == OMP_CLAUSE_USE_DEVICE_ADDR
+ && !omp_is_reference (var)
+ && !omp_is_allocatable_or_ptr (var))
+ || TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE)
{
- tree new_var = lookup_decl (var, ctx);
+ new_var = lookup_decl (var, ctx);
new_var = DECL_VALUE_EXPR (new_var);
gcc_assert (TREE_CODE (new_var) == MEM_REF);
new_var = TREE_OPERAND (new_var, 0);
gcc_assert (DECL_P (new_var));
- gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
- gimple_seq_add_stmt (&new_body,
+ gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
+ gimple_seq_add_stmt (&assign_body,
gimple_build_assign (new_var, x));
}
else
{
tree type = TREE_TYPE (var);
- tree new_var = lookup_decl (var, ctx);
+ new_var = lookup_decl (var, ctx);
if (omp_is_reference (var))
{
type = TREE_TYPE (type);
- if (TREE_CODE (type) != ARRAY_TYPE)
+ if (POINTER_TYPE_P (type)
+ && TREE_CODE (type) != ARRAY_TYPE
+ && (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_USE_DEVICE_ADDR
+ || (omp_is_reference (var)
+ && omp_is_allocatable_or_ptr (var))))
{
tree v = create_tmp_var_raw (type, get_name (var));
gimple_add_tmp_var (v);
TREE_ADDRESSABLE (v) = 1;
x = fold_convert (type, x);
- gimplify_expr (&x, &new_body, NULL, is_gimple_val,
+ gimplify_expr (&x, &assign_body, NULL, is_gimple_val,
fb_rvalue);
- gimple_seq_add_stmt (&new_body,
+ gimple_seq_add_stmt (&assign_body,
gimple_build_assign (v, x));
x = build_fold_addr_expr (v);
+ do_optional_check = true;
}
}
new_var = DECL_VALUE_EXPR (new_var);
x = fold_convert (TREE_TYPE (new_var), x);
- gimplify_expr (&x, &new_body, NULL, is_gimple_val, fb_rvalue);
- gimple_seq_add_stmt (&new_body,
+ gimplify_expr (&x, &assign_body, NULL, is_gimple_val, fb_rvalue);
+ gimple_seq_add_stmt (&assign_body,
gimple_build_assign (new_var, x));
}
+ tree present;
+ present = (do_optional_check
+ ? omp_check_optional_argument (OMP_CLAUSE_DECL (c), true)
+ : NULL_TREE);
+ if (present)
+ {
+ tree null_label = create_artificial_label (UNKNOWN_LOCATION);
+ tree notnull_label = create_artificial_label (UNKNOWN_LOCATION);
+ tree opt_arg_label = create_artificial_label (UNKNOWN_LOCATION);
+ glabel *null_glabel = gimple_build_label (null_label);
+ glabel *notnull_glabel = gimple_build_label (notnull_label);
+ ggoto *opt_arg_ggoto = gimple_build_goto (opt_arg_label);
+ gimplify_expr (&x, &new_body, NULL, is_gimple_val,
+ fb_rvalue);
+ gimplify_expr (&present, &new_body, NULL, is_gimple_val,
+ fb_rvalue);
+ gcond *cond = gimple_build_cond_from_tree (present,
+ notnull_label,
+ null_label);
+ gimple_seq_add_stmt (&new_body, cond);
+ gimple_seq_add_stmt (&new_body, null_glabel);
+ gimplify_assign (new_var, null_pointer_node, &new_body);
+ gimple_seq_add_stmt (&new_body, opt_arg_ggoto);
+ gimple_seq_add_stmt (&new_body, notnull_glabel);
+ gimple_seq_add_seq (&new_body, assign_body);
+ gimple_seq_add_stmt (&new_body,
+ gimple_build_label (opt_arg_label));
+ }
+ else
+ gimple_seq_add_seq (&new_body, assign_body);
break;
}
/* Handle GOMP_MAP_FIRSTPRIVATE_{POINTER,REFERENCE} in second pass,
gimple_seq fork_seq = NULL;
gimple_seq join_seq = NULL;
- if (is_oacc_parallel (ctx))
+ if (offloaded && is_gimple_omp_oacc (ctx->stmt))
{
/* If there are reductions on the offloaded region itself, treat
them as a dummy GANG loop. */
gimple_seq_add_seq (&new_body, join_seq);
if (offloaded)
- new_body = maybe_catch_exception (new_body);
-
- gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
+ {
+ new_body = maybe_catch_exception (new_body);
+ gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false));
+ }
gimple_omp_set_body (stmt, new_body);
}
lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt),
&bind_body, &dlist, ctx, NULL);
lower_omp (gimple_omp_body_ptr (teams_stmt), ctx);
- lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist, ctx);
- if (!gimple_omp_teams_grid_phony (teams_stmt))
- {
- gimple_seq_add_stmt (&bind_body, teams_stmt);
- location_t loc = gimple_location (teams_stmt);
- tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS);
- gimple *call = gimple_build_call (decl, 2, num_teams, thread_limit);
- gimple_set_location (call, loc);
- gimple_seq_add_stmt (&bind_body, call);
- }
+ lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist,
+ NULL, ctx);
+ gimple_seq_add_stmt (&bind_body, teams_stmt);
+
+ location_t loc = gimple_location (teams_stmt);
+ tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS);
+ gimple *call = gimple_build_call (decl, 2, num_teams, thread_limit);
+ gimple_set_location (call, loc);
+ gimple_seq_add_stmt (&bind_body, call);
gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt));
gimple_omp_set_body (teams_stmt, NULL);
gimple_seq_add_seq (&bind_body, olist);
gimple_seq_add_seq (&bind_body, dlist);
- if (!gimple_omp_teams_grid_phony (teams_stmt))
- gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true));
+ gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true));
gimple_bind_set_body (bind, bind_body);
pop_gimplify_context (bind);
TREE_USED (block) = 1;
}
-/* Expand code within an artificial GIMPLE_OMP_GRID_BODY OMP construct. */
-
-static void
-lower_omp_grid_body (gimple_stmt_iterator *gsi_p, omp_context *ctx)
-{
- gimple *stmt = gsi_stmt (*gsi_p);
- lower_omp (gimple_omp_body_ptr (stmt), ctx);
- gimple_seq_add_stmt (gimple_omp_body_ptr (stmt),
- gimple_build_omp_return (false));
-}
-
-
/* Callback for lower_omp_1. Return non-NULL if *tp needs to be
regimplified. If DATA is non-NULL, lower_omp_1 is outside
of OMP context, but with task_shared_vars set. */
gcc_assert (ctx);
lower_omp_ordered (gsi_p, ctx);
break;
+ case GIMPLE_OMP_SCAN:
+ ctx = maybe_lookup_ctx (stmt);
+ gcc_assert (ctx);
+ lower_omp_scan (gsi_p, ctx);
+ break;
case GIMPLE_OMP_CRITICAL:
ctx = maybe_lookup_ctx (stmt);
gcc_assert (ctx);
case GIMPLE_OMP_TEAMS:
ctx = maybe_lookup_ctx (stmt);
gcc_assert (ctx);
- lower_omp_teams (gsi_p, ctx);
- break;
- case GIMPLE_OMP_GRID_BODY:
- ctx = maybe_lookup_ctx (stmt);
- gcc_assert (ctx);
- lower_omp_grid_body (gsi_p, ctx);
+ if (gimple_omp_teams_host (as_a <gomp_teams *> (stmt)))
+ lower_omp_taskreg (gsi_p, ctx);
+ else
+ lower_omp_teams (gsi_p, ctx);
break;
case GIMPLE_CALL:
tree fndecl;
call_stmt = as_a <gcall *> (stmt);
fndecl = gimple_call_fndecl (call_stmt);
if (fndecl
- && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
+ && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL))
switch (DECL_FUNCTION_CODE (fndecl))
{
case BUILT_IN_GOMP_BARRIER:
default:
break;
}
+ goto regimplify;
+
+ case GIMPLE_ASSIGN:
+ for (omp_context *up = ctx; up; up = up->outer)
+ {
+ if (gimple_code (up->stmt) == GIMPLE_OMP_ORDERED
+ || gimple_code (up->stmt) == GIMPLE_OMP_CRITICAL
+ || gimple_code (up->stmt) == GIMPLE_OMP_TASKGROUP
+ || gimple_code (up->stmt) == GIMPLE_OMP_SECTION
+ || gimple_code (up->stmt) == GIMPLE_OMP_SCAN
+ || (gimple_code (up->stmt) == GIMPLE_OMP_TARGET
+ && (gimple_omp_target_kind (up->stmt)
+ == GF_OMP_TARGET_KIND_DATA)))
+ continue;
+ else if (!up->lastprivate_conditional_map)
+ break;
+ tree lhs = get_base_address (gimple_assign_lhs (stmt));
+ if (TREE_CODE (lhs) == MEM_REF
+ && DECL_P (TREE_OPERAND (lhs, 0))
+ && TREE_CODE (TREE_TYPE (TREE_OPERAND (lhs,
+ 0))) == REFERENCE_TYPE)
+ lhs = TREE_OPERAND (lhs, 0);
+ if (DECL_P (lhs))
+ if (tree *v = up->lastprivate_conditional_map->get (lhs))
+ {
+ tree clauses;
+ if (up->combined_into_simd_safelen1)
+ {
+ up = up->outer;
+ if (gimple_code (up->stmt) == GIMPLE_OMP_SCAN)
+ up = up->outer;
+ }
+ if (gimple_code (up->stmt) == GIMPLE_OMP_FOR)
+ clauses = gimple_omp_for_clauses (up->stmt);
+ else
+ clauses = gimple_omp_sections_clauses (up->stmt);
+ tree c = omp_find_clause (clauses, OMP_CLAUSE__CONDTEMP_);
+ if (!OMP_CLAUSE__CONDTEMP__ITER (c))
+ c = omp_find_clause (OMP_CLAUSE_CHAIN (c),
+ OMP_CLAUSE__CONDTEMP_);
+ gcc_assert (OMP_CLAUSE__CONDTEMP__ITER (c));
+ gimple *g = gimple_build_assign (*v, OMP_CLAUSE_DECL (c));
+ gsi_insert_after (gsi_p, g, GSI_SAME_STMT);
+ }
+ }
/* FALLTHRU */
+
default:
+ regimplify:
if ((ctx || task_shared_vars)
&& walk_gimple_op (stmt, lower_omp_regimplify_p,
ctx ? NULL : &wi))
body = gimple_body (current_function_decl);
- if (hsa_gen_requested_p ())
- omp_grid_gridify_all_targets (&body);
-
scan_omp (&body, NULL);
gcc_assert (taskreg_nesting_level == 0);
FOR_EACH_VEC_ELT (taskreg_contexts, i, ctx)
all_contexts = NULL;
}
BITMAP_FREE (task_shared_vars);
+ BITMAP_FREE (global_nonaddressable_vars);
/* If current function is a method, remove artificial dummy VAR_DECL created
for non-static data member privatization, they aren't needed for
case GIMPLE_OMP_SECTION:
case GIMPLE_OMP_MASTER:
case GIMPLE_OMP_ORDERED:
+ case GIMPLE_OMP_SCAN:
case GIMPLE_OMP_CRITICAL:
case GIMPLE_OMP_TARGET:
case GIMPLE_OMP_TEAMS:
case GIMPLE_OMP_SECTION:
case GIMPLE_OMP_MASTER:
case GIMPLE_OMP_ORDERED:
+ case GIMPLE_OMP_SCAN:
case GIMPLE_OMP_CRITICAL:
case GIMPLE_OMP_TARGET:
case GIMPLE_OMP_TEAMS:
projects / gcc.git / blobdiff