-/* Copyright (C) 2005-2015 Free Software Foundation, Inc.
+/* Copyright (C) 2005-2020 Free Software Foundation, Inc.
Contributed by Richard Henderson <rth@redhat.com>.
This file is part of the GNU Offloading and Multi Processing Library
Either way we get correct results.
However, there is an implicit flush on entry to an ordered region,
so we do need to have a barrier here. If we were taking a lock
- this could be MEMMODEL_RELEASE since the acquire would be coverd
+ this could be MEMMODEL_RELEASE since the acquire would be covered
by the lock. */
__atomic_thread_fence (MEMMODEL_ACQ_REL);
However, the current implementation has a flaw in that it does not allow
the next thread into the ORDERED section immediately after the current
- thread exits the ORDERED section in its last iteration. The existance
+ thread exits the ORDERED section in its last iteration. The existence
of this function allows the implementation to change. */
void
#define MAX_COLLAPSED_BITS (__SIZEOF_LONG__ * __CHAR_BIT__)
void
-gomp_doacross_init (unsigned ncounts, long *counts, long chunk_size)
+gomp_doacross_init (unsigned ncounts, long *counts, long chunk_size,
+ size_t extra)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
struct gomp_doacross_work_share *doacross;
if (team == NULL || team->nthreads == 1)
- return;
+ {
+ empty:
+ if (!extra)
+ ws->doacross = NULL;
+ else
+ {
+ doacross = gomp_malloc_cleared (sizeof (*doacross) + extra);
+ doacross->extra = (void *) (doacross + 1);
+ ws->doacross = doacross;
+ }
+ return;
+ }
for (i = 0; i < ncounts; i++)
{
/* If any count is 0, GOMP_doacross_{post,wait} can't be called. */
if (counts[i] == 0)
- return;
+ goto empty;
if (num_bits <= MAX_COLLAPSED_BITS)
{
if (ws->sched == GFS_STATIC)
num_ents = team->nthreads;
+ else if (ws->sched == GFS_GUIDED)
+ num_ents = counts[0];
else
num_ents = (counts[0] - 1) / chunk_size + 1;
if (num_bits <= MAX_COLLAPSED_BITS)
elt_sz = (elt_sz + 63) & ~63UL;
doacross = gomp_malloc (sizeof (*doacross) + 63 + num_ents * elt_sz
- + shift_sz);
+ + shift_sz + extra);
doacross->chunk_size = chunk_size;
doacross->elt_sz = elt_sz;
doacross->ncounts = ncounts;
doacross->array = (unsigned char *)
((((uintptr_t) (doacross + 1)) + 63 + shift_sz)
& ~(uintptr_t) 63);
+ if (extra)
+ {
+ doacross->extra = doacross->array + num_ents * elt_sz;
+ memset (doacross->extra, '\0', extra);
+ }
+ else
+ doacross->extra = NULL;
if (num_bits <= MAX_COLLAPSED_BITS)
{
unsigned int shift_count = 0;
unsigned long ent;
unsigned int i;
- if (__builtin_expect (doacross == NULL, 0))
+ if (__builtin_expect (doacross == NULL, 0)
+ || __builtin_expect (doacross->array == NULL, 0))
{
__sync_synchronize ();
return;
if (__builtin_expect (ws->sched == GFS_STATIC, 1))
ent = thr->ts.team_id;
+ else if (ws->sched == GFS_GUIDED)
+ ent = counts[0];
else
ent = counts[0] / doacross->chunk_size;
unsigned long *array = (unsigned long *) (doacross->array
unsigned long ent;
unsigned int i;
- if (__builtin_expect (doacross == NULL, 0))
+ if (__builtin_expect (doacross == NULL, 0)
+ || __builtin_expect (doacross->array == NULL, 0))
{
__sync_synchronize ();
return;
else
ent = first / ws->chunk_size % thr->ts.team->nthreads;
}
+ else if (ws->sched == GFS_GUIDED)
+ ent = first;
else
ent = first / doacross->chunk_size;
unsigned long *array = (unsigned long *) (doacross->array
typedef unsigned long long gomp_ull;
void
-gomp_doacross_ull_init (unsigned ncounts, gomp_ull *counts, gomp_ull chunk_size)
+gomp_doacross_ull_init (unsigned ncounts, gomp_ull *counts,
+ gomp_ull chunk_size, size_t extra)
{
struct gomp_thread *thr = gomp_thread ();
struct gomp_team *team = thr->ts.team;
struct gomp_doacross_work_share *doacross;
if (team == NULL || team->nthreads == 1)
- return;
+ {
+ empty:
+ if (!extra)
+ ws->doacross = NULL;
+ else
+ {
+ doacross = gomp_malloc_cleared (sizeof (*doacross) + extra);
+ doacross->extra = (void *) (doacross + 1);
+ ws->doacross = doacross;
+ }
+ return;
+ }
for (i = 0; i < ncounts; i++)
{
/* If any count is 0, GOMP_doacross_{post,wait} can't be called. */
if (counts[i] == 0)
- return;
+ goto empty;
if (num_bits <= MAX_COLLAPSED_BITS)
{
if (ws->sched == GFS_STATIC)
num_ents = team->nthreads;
+ else if (ws->sched == GFS_GUIDED)
+ num_ents = counts[0];
else
num_ents = (counts[0] - 1) / chunk_size + 1;
if (num_bits <= MAX_COLLAPSED_BITS)
doacross->array = (unsigned char *)
((((uintptr_t) (doacross + 1)) + 63 + shift_sz)
& ~(uintptr_t) 63);
+ if (extra)
+ {
+ doacross->extra = doacross->array + num_ents * elt_sz;
+ memset (doacross->extra, '\0', extra);
+ }
+ else
+ doacross->extra = NULL;
if (num_bits <= MAX_COLLAPSED_BITS)
{
unsigned int shift_count = 0;
unsigned long ent;
unsigned int i;
- if (__builtin_expect (doacross == NULL, 0))
+ if (__builtin_expect (doacross == NULL, 0)
+ || __builtin_expect (doacross->array == NULL, 0))
{
__sync_synchronize ();
return;
if (__builtin_expect (ws->sched == GFS_STATIC, 1))
ent = thr->ts.team_id;
+ else if (ws->sched == GFS_GUIDED)
+ ent = counts[0];
else
ent = counts[0] / doacross->chunk_size_ull;
unsigned long ent;
unsigned int i;
- if (__builtin_expect (doacross == NULL, 0))
+ if (__builtin_expect (doacross == NULL, 0)
+ || __builtin_expect (doacross->array == NULL, 0))
{
__sync_synchronize ();
return;
else
ent = first / ws->chunk_size_ull % thr->ts.team->nthreads;
}
+ else if (ws->sched == GFS_GUIDED)
+ ent = first;
else
ent = first / doacross->chunk_size_ull;