c++: Respect current_constraint_diagnosis_depth in diagnose_compound_requirement

[gcc.git] / libgomp / libgomp.h

diff --git a/libgomp/libgomp.h b/libgomp/libgomp.h
index 322a43520ee79201554624d15dccfa2d073ccf50..f5415bb156c6eb24511adbfa293710620043a3ab 100644 (file)
--- a/libgomp/libgomp.h
+++ b/libgomp/libgomp.h
@@ -1,7 +1,8 @@
-/* Copyright (C) 2005-2013 Free Software Foundation, Inc.
+/* Copyright (C) 2005-2020 Free Software Foundation, Inc.

    Contributed by Richard Henderson <rth@redhat.com>.
 
    Contributed by Richard Henderson <rth@redhat.com>.
 

-   This file is part of the GNU OpenMP Library (libgomp).
+   This file is part of the GNU Offloading and Multi Processing Library
+   (libgomp).

 
    Libgomp is free software; you can redistribute it and/or modify it
    under the terms of the GNU General Public License as published by
 
    Libgomp is free software; you can redistribute it and/or modify it
    under the terms of the GNU General Public License as published by


@@ -23,9 +24,10 @@
    <http://www.gnu.org/licenses/>.  */
 
 /* This file contains data types and function declarations that are not
    <http://www.gnu.org/licenses/>.  */
 
 /* This file contains data types and function declarations that are not

-   part of the official OpenMP user interface.  There are declarations
-   in here that are part of the GNU OpenMP ABI, in that the compiler is
-   required to know about them and use them.
+   part of the official OpenACC or OpenMP user interfaces.  There are
+   declarations in here that are part of the GNU Offloading and Multi
+   Processing ABI, in that the compiler is required to know about them
+   and use them.

 
    The convention is that the all caps prefix "GOMP" is used group items
    that are part of the external ABI, and the lower case prefix "gomp"
 
    The convention is that the all caps prefix "GOMP" is used group items
    that are part of the external ABI, and the lower case prefix "gomp"


@@ -34,11 +36,38 @@
 #ifndef LIBGOMP_H 
 #define LIBGOMP_H 1
 
 #ifndef LIBGOMP_H 
 #define LIBGOMP_H 1
 

+#ifndef _LIBGOMP_CHECKING_
+/* Define to 1 to perform internal sanity checks.  */
+#define _LIBGOMP_CHECKING_ 0
+#endif
+

 #include "config.h"
 #include "config.h"

-#include "gstdint.h"
+#include <stdint.h>
+#include "libgomp-plugin.h"
+#include "gomp-constants.h"

 
 

+#ifdef HAVE_PTHREAD_H

 #include <pthread.h>
 #include <pthread.h>

+#endif

 #include <stdbool.h>
 #include <stdbool.h>

+#include <stdlib.h>
+#include <stdarg.h>
+
+/* Needed for memset in priority_queue.c.  */
+#if _LIBGOMP_CHECKING_
+# ifdef STRING_WITH_STRINGS
+#  include <string.h>
+#  include <strings.h>
+# else
+#  ifdef HAVE_STRING_H
+#   include <string.h>
+#  else
+#   ifdef HAVE_STRINGS_H
+#    include <strings.h>
+#   endif
+#  endif
+# endif
+#endif

 
 #ifdef HAVE_ATTRIBUTE_VISIBILITY
 # pragma GCC visibility push(hidden)
 
 #ifdef HAVE_ATTRIBUTE_VISIBILITY
 # pragma GCC visibility push(hidden)


@@ -55,9 +84,123 @@ enum memmodel
   MEMMODEL_SEQ_CST = 5
 };
 
   MEMMODEL_SEQ_CST = 5
 };
 

+/* alloc.c */
+
+#if defined(HAVE_ALIGNED_ALLOC) \
+    || defined(HAVE__ALIGNED_MALLOC) \
+    || defined(HAVE_POSIX_MEMALIGN) \
+    || defined(HAVE_MEMALIGN)
+/* Defined if gomp_aligned_alloc doesn't use fallback version
+   and free can be used instead of gomp_aligned_free.  */
+#define GOMP_HAVE_EFFICIENT_ALIGNED_ALLOC 1
+#endif
+
+extern void *gomp_malloc (size_t) __attribute__((malloc));
+extern void *gomp_malloc_cleared (size_t) __attribute__((malloc));
+extern void *gomp_realloc (void *, size_t);
+extern void *gomp_aligned_alloc (size_t, size_t)
+  __attribute__((malloc, alloc_size (2)));
+extern void gomp_aligned_free (void *);
+
+/* Avoid conflicting prototypes of alloca() in system headers by using
+   GCC's builtin alloca().  */
+#define gomp_alloca(x)  __builtin_alloca(x)
+
+/* Optimized allocators for team-specific data that will die with the team.  */
+
+#ifdef __AMDGCN__
+/* The arena is initialized in config/gcn/team.c.  */
+#define TEAM_ARENA_SIZE  64*1024  /* Must match the value in plugin-gcn.c.  */
+#define TEAM_ARENA_START 16  /* LDS offset of free pointer.  */
+#define TEAM_ARENA_FREE  24  /* LDS offset of free pointer.  */
+#define TEAM_ARENA_END   32  /* LDS offset of end pointer.  */
+
+static inline void * __attribute__((malloc))
+team_malloc (size_t size)
+{
+  /* 4-byte align the size.  */
+  size = (size + 3) & ~3;
+
+  /* Allocate directly from the arena.
+     The compiler does not support DS atomics, yet. */
+  void *result;
+  asm ("ds_add_rtn_u64 %0, %1, %2\n\ts_waitcnt 0"
+       : "=v"(result) : "v"(TEAM_ARENA_FREE), "v"(size), "e"(1L) : "memory");
+
+  /* Handle OOM.  */
+  if (result + size > *(void * __lds *)TEAM_ARENA_END)
+    {
+      /* While this is experimental, let's make sure we know when OOM
+        happens.  */
+      const char msg[] = "GCN team arena exhausted\n";
+      write (2, msg, sizeof(msg)-1);
+
+      /* Fall back to using the heap (slowly).  */
+      result = gomp_malloc (size);
+    }
+  return result;
+}
+
+static inline void * __attribute__((malloc))
+team_malloc_cleared (size_t size)
+{
+  char *result = team_malloc (size);
+
+  /* Clear the allocated memory.  */
+  __builtin_memset (result, 0, size);
+
+  return result;
+}
+
+static inline void
+team_free (void *ptr)
+{
+  /* The whole arena is freed when the kernel exits.
+     However, if we fell back to using heap then we should free it.
+     It would be better if this function could be a no-op, but at least
+     LDS loads are cheap.  */
+  if (ptr < *(void * __lds *)TEAM_ARENA_START
+      || ptr >= *(void * __lds *)TEAM_ARENA_END)
+    free (ptr);
+}
+#else
+#define team_malloc(...) gomp_malloc (__VA_ARGS__)
+#define team_malloc_cleared(...) gomp_malloc_cleared (__VA_ARGS__)
+#define team_free(...) free (__VA_ARGS__)
+#endif
+
+/* error.c */
+
+extern void gomp_vdebug (int, const char *, va_list);
+extern void gomp_debug (int, const char *, ...)
+       __attribute__ ((format (printf, 2, 3)));
+#define gomp_vdebug(KIND, FMT, VALIST) \
+  do { \
+    if (__builtin_expect (gomp_debug_var, 0)) \
+      (gomp_vdebug) ((KIND), (FMT), (VALIST)); \
+  } while (0)
+#define gomp_debug(KIND, ...) \
+  do { \
+    if (__builtin_expect (gomp_debug_var, 0)) \
+      (gomp_debug) ((KIND), __VA_ARGS__); \
+  } while (0)
+extern void gomp_verror (const char *, va_list);
+extern void gomp_error (const char *, ...)
+       __attribute__ ((format (printf, 1, 2)));
+extern void gomp_vfatal (const char *, va_list)
+       __attribute__ ((noreturn));
+extern void gomp_fatal (const char *, ...)
+       __attribute__ ((noreturn, format (printf, 1, 2)));
+
+struct gomp_task;
+struct gomp_taskgroup;
+struct htab;
+
+#include "priority_queue.h"

 #include "sem.h"
 #include "mutex.h"
 #include "bar.h"
 #include "sem.h"
 #include "mutex.h"
 #include "bar.h"

+#include "simple-bar.h"

 #include "ptrlock.h"
 
 
 #include "ptrlock.h"
 
 


@@ -70,7 +213,48 @@ enum gomp_schedule_type
   GFS_STATIC,
   GFS_DYNAMIC,
   GFS_GUIDED,
   GFS_STATIC,
   GFS_DYNAMIC,
   GFS_GUIDED,

-  GFS_AUTO
+  GFS_AUTO,
+  GFS_MONOTONIC = 0x80000000U
+};
+
+struct gomp_doacross_work_share
+{
+  union {
+    /* chunk_size copy, as ws->chunk_size is multiplied by incr for
+       GFS_DYNAMIC.  */
+    long chunk_size;
+    /* Likewise, but for ull implementation.  */
+    unsigned long long chunk_size_ull;
+    /* For schedule(static,0) this is the number
+       of iterations assigned to the last thread, i.e. number of
+       iterations / number of threads.  */
+    long q;
+    /* Likewise, but for ull implementation.  */
+    unsigned long long q_ull;
+  };
+  /* Size of each array entry (padded to cache line size).  */
+  unsigned long elt_sz;
+  /* Number of dimensions in sink vectors.  */
+  unsigned int ncounts;
+  /* True if the iterations can be flattened.  */
+  bool flattened;
+  /* Actual array (of elt_sz sized units), aligned to cache line size.
+     This is indexed by team_id for GFS_STATIC and outermost iteration
+     / chunk_size for other schedules.  */
+  unsigned char *array;
+  /* These two are only used for schedule(static,0).  */
+  /* This one is number of iterations % number of threads.  */
+  long t;
+  union {
+    /* And this one is cached t * (q + 1).  */
+    long boundary;
+    /* Likewise, but for the ull implementation.  */
+    unsigned long long boundary_ull;
+  };
+  /* Pointer to extra memory if needed for lastprivate(conditional).  */
+  void *extra;
+  /* Array of shift counts for each dimension if they can be flattened.  */
+  unsigned int shift_counts[];

 };
 
 struct gomp_work_share
 };
 
 struct gomp_work_share


@@ -104,13 +288,18 @@ struct gomp_work_share
     };
   };
 
     };
   };
 

-  /* This is a circular queue that details which threads will be allowed
-     into the ordered region and in which order.  When a thread allocates
-     iterations on which it is going to work, it also registers itself at
-     the end of the array.  When a thread reaches the ordered region, it
-     checks to see if it is the one at the head of the queue.  If not, it
-     blocks on its RELEASE semaphore.  */
-  unsigned *ordered_team_ids;
+  union {
+    /* This is a circular queue that details which threads will be allowed
+       into the ordered region and in which order.  When a thread allocates
+       iterations on which it is going to work, it also registers itself at
+       the end of the array.  When a thread reaches the ordered region, it
+       checks to see if it is the one at the head of the queue.  If not, it
+       blocks on its RELEASE semaphore.  */
+    unsigned *ordered_team_ids;
+
+    /* This is a pointer to DOACROSS work share data.  */
+    struct gomp_doacross_work_share *doacross;
+  };

 
   /* This is the number of threads that have registered themselves in
      the circular queue ordered_team_ids.  */
 
   /* This is the number of threads that have registered themselves in
      the circular queue ordered_team_ids.  */


@@ -165,6 +354,9 @@ struct gomp_work_share
     struct gomp_work_share *next_free;
   };
 
     struct gomp_work_share *next_free;
   };
 

+  /* Task reductions for this work-sharing construct.  */
+  uintptr_t *task_reductions;
+

   /* If only few threads are in the team, ordered_team_ids can point
      to this array which fills the padding at the end of this struct.  */
   unsigned inline_ordered_team_ids[0];
   /* If only few threads are in the team, ordered_team_ids can point
      to this array which fills the padding at the end of this struct.  */
   unsigned inline_ordered_team_ids[0];


@@ -201,6 +393,10 @@ struct gomp_team_state
   /* Active nesting level.  Only active parallel regions are counted.  */
   unsigned active_level;
 
   /* Active nesting level.  Only active parallel regions are counted.  */
   unsigned active_level;
 

+  /* Place-partition-var, offset and length into gomp_places_list array.  */
+  unsigned place_partition_off;
+  unsigned place_partition_len;
+

 #ifdef HAVE_SYNC_BUILTINS
   /* Number of single stmts encountered.  */
   unsigned long single_count;
 #ifdef HAVE_SYNC_BUILTINS
   /* Number of single stmts encountered.  */
   unsigned long single_count;


@@ -214,57 +410,187 @@ struct gomp_team_state
   unsigned long static_trip;
 };
 
   unsigned long static_trip;
 };
 

-/* These are the OpenMP 3.0 Internal Control Variables described in
+struct target_mem_desc;
+
+/* These are the OpenMP 4.0 Internal Control Variables described in

    section 2.3.1.  Those described as having one copy per task are
    stored within the structure; those described as having one copy
    for the whole program are (naturally) global variables.  */
    section 2.3.1.  Those described as having one copy per task are
    stored within the structure; those described as having one copy
    for the whole program are (naturally) global variables.  */

-
+   

 struct gomp_task_icv
 {
   unsigned long nthreads_var;
   enum gomp_schedule_type run_sched_var;
 struct gomp_task_icv
 {
   unsigned long nthreads_var;
   enum gomp_schedule_type run_sched_var;

-  int run_sched_modifier;
+  int run_sched_chunk_size;
+  int default_device_var;
+  unsigned int thread_limit_var;

   bool dyn_var;
   bool nest_var;
   bool dyn_var;
   bool nest_var;

+  char bind_var;
+  /* Internal ICV.  */
+  struct target_mem_desc *target_data;

 };
 
 extern struct gomp_task_icv gomp_global_icv;
 };
 
 extern struct gomp_task_icv gomp_global_icv;

-extern unsigned long gomp_thread_limit_var;
-extern unsigned long gomp_remaining_threads_count;

 #ifndef HAVE_SYNC_BUILTINS
 #ifndef HAVE_SYNC_BUILTINS

-extern gomp_mutex_t gomp_remaining_threads_lock;
+extern gomp_mutex_t gomp_managed_threads_lock;

 #endif
 extern unsigned long gomp_max_active_levels_var;
 #endif
 extern unsigned long gomp_max_active_levels_var;

+extern bool gomp_cancel_var;
+extern int gomp_max_task_priority_var;

 extern unsigned long long gomp_spin_count_var, gomp_throttled_spin_count_var;
 extern unsigned long gomp_available_cpus, gomp_managed_threads;
 extern unsigned long *gomp_nthreads_var_list, gomp_nthreads_var_list_len;
 extern unsigned long long gomp_spin_count_var, gomp_throttled_spin_count_var;
 extern unsigned long gomp_available_cpus, gomp_managed_threads;
 extern unsigned long *gomp_nthreads_var_list, gomp_nthreads_var_list_len;

+extern char *gomp_bind_var_list;
+extern unsigned long gomp_bind_var_list_len;
+extern void **gomp_places_list;
+extern unsigned long gomp_places_list_len;
+extern unsigned int gomp_num_teams_var;
+extern int gomp_debug_var;
+extern bool gomp_display_affinity_var;
+extern char *gomp_affinity_format_var;
+extern size_t gomp_affinity_format_len;
+extern int goacc_device_num;
+extern char *goacc_device_type;
+extern int goacc_default_dims[GOMP_DIM_MAX];

 
 enum gomp_task_kind
 {
 
 enum gomp_task_kind
 {

+  /* Implicit task.  */

   GOMP_TASK_IMPLICIT,
   GOMP_TASK_IMPLICIT,

-  GOMP_TASK_IFFALSE,
+  /* Undeferred task.  */
+  GOMP_TASK_UNDEFERRED,
+  /* Task created by GOMP_task and waiting to be run.  */

   GOMP_TASK_WAITING,
   GOMP_TASK_WAITING,

-  GOMP_TASK_TIED
+  /* Task currently executing or scheduled and about to execute.  */
+  GOMP_TASK_TIED,
+  /* Used for target tasks that have vars mapped and async run started,
+     but not yet completed.  Once that completes, they will be readded
+     into the queues as GOMP_TASK_WAITING in order to perform the var
+     unmapping.  */
+  GOMP_TASK_ASYNC_RUNNING
+};
+
+struct gomp_task_depend_entry
+{
+  /* Address of dependency.  */
+  void *addr;
+  struct gomp_task_depend_entry *next;
+  struct gomp_task_depend_entry *prev;
+  /* Task that provides the dependency in ADDR.  */
+  struct gomp_task *task;
+  /* Depend entry is of type "IN".  */
+  bool is_in;
+  bool redundant;
+  bool redundant_out;
+};
+
+struct gomp_dependers_vec
+{
+  size_t n_elem;
+  size_t allocated;
+  struct gomp_task *elem[];
+};
+
+/* Used when in GOMP_taskwait or in gomp_task_maybe_wait_for_dependencies.  */
+
+struct gomp_taskwait
+{
+  bool in_taskwait;
+  bool in_depend_wait;
+  /* Number of tasks we are waiting for.  */
+  size_t n_depend;
+  gomp_sem_t taskwait_sem;

 };
 
 /* This structure describes a "task" to be run by a thread.  */
 
 struct gomp_task
 {
 };
 
 /* This structure describes a "task" to be run by a thread.  */
 
 struct gomp_task
 {

+  /* Parent of this task.  */

   struct gomp_task *parent;
   struct gomp_task *parent;

-  struct gomp_task *children;
-  struct gomp_task *next_child;
-  struct gomp_task *prev_child;
-  struct gomp_task *next_queue;
-  struct gomp_task *prev_queue;
+  /* Children of this task.  */
+  struct priority_queue children_queue;
+  /* Taskgroup this task belongs in.  */
+  struct gomp_taskgroup *taskgroup;
+  /* Tasks that depend on this task.  */
+  struct gomp_dependers_vec *dependers;
+  struct htab *depend_hash;
+  struct gomp_taskwait *taskwait;
+  /* Number of items in DEPEND.  */
+  size_t depend_count;
+  /* Number of tasks this task depends on.  Once this counter reaches
+     0, we have no unsatisfied dependencies, and this task can be put
+     into the various queues to be scheduled.  */
+  size_t num_dependees;
+
+  /* Priority of this task.  */
+  int priority;
+  /* The priority node for this task in each of the different queues.
+     We put this here to avoid allocating space for each priority
+     node.  Then we play offsetof() games to convert between pnode[]
+     entries and the gomp_task in which they reside.  */
+  struct priority_node pnode[3];
+

   struct gomp_task_icv icv;
   void (*fn) (void *);
   void *fn_data;
   enum gomp_task_kind kind;
   struct gomp_task_icv icv;
   void (*fn) (void *);
   void *fn_data;
   enum gomp_task_kind kind;

-  bool in_taskwait;

   bool in_tied_task;
   bool final_task;
   bool in_tied_task;
   bool final_task;

-  gomp_sem_t taskwait_sem;
+  bool copy_ctors_done;
+  /* Set for undeferred tasks with unsatisfied dependencies which
+     block further execution of their parent until the dependencies
+     are satisfied.  */
+  bool parent_depends_on;
+  /* Dependencies provided and/or needed for this task.  DEPEND_COUNT
+     is the number of items available.  */
+  struct gomp_task_depend_entry depend[];
+};
+
+/* This structure describes a single #pragma omp taskgroup.  */
+
+struct gomp_taskgroup
+{
+  struct gomp_taskgroup *prev;
+  /* Queue of tasks that belong in this taskgroup.  */
+  struct priority_queue taskgroup_queue;
+  uintptr_t *reductions;
+  bool in_taskgroup_wait;
+  bool cancelled;
+  bool workshare;
+  gomp_sem_t taskgroup_sem;
+  size_t num_children;
+};
+
+/* Various state of OpenMP async offloading tasks.  */
+enum gomp_target_task_state
+{
+  GOMP_TARGET_TASK_DATA,
+  GOMP_TARGET_TASK_BEFORE_MAP,
+  GOMP_TARGET_TASK_FALLBACK,
+  GOMP_TARGET_TASK_READY_TO_RUN,
+  GOMP_TARGET_TASK_RUNNING,
+  GOMP_TARGET_TASK_FINISHED
+};
+
+/* This structure describes a target task.  */
+
+struct gomp_target_task
+{
+  struct gomp_device_descr *devicep;
+  void (*fn) (void *);
+  size_t mapnum;
+  size_t *sizes;
+  unsigned short *kinds;
+  unsigned int flags;
+  enum gomp_target_task_state state;
+  struct target_mem_desc *tgt;
+  struct gomp_task *task;
+  struct gomp_team *team;
+  /* Device-specific target arguments.  */
+  void **args;
+  void *hostaddrs[];

 };
 
 /* This structure describes a "team" of threads.  These are the threads
 };
 
 /* This structure describes a "team" of threads.  These are the threads


@@ -293,6 +619,12 @@ struct gomp_team
      of the threads in the team.  */
   gomp_sem_t **ordered_release;
 
      of the threads in the team.  */
   gomp_sem_t **ordered_release;
 

+  /* List of work shares on which gomp_fini_work_share hasn't been
+     called yet.  If the team hasn't been cancelled, this should be
+     equal to each thr->ts.work_share, but otherwise it can be a possibly
+     long list of workshares.  */
+  struct gomp_work_share *work_shares_to_free;
+

   /* List of gomp_work_share structs chained through next_free fields.
      This is populated and taken off only by the first thread in the
      team encountering a new work sharing construct, in a critical
   /* List of gomp_work_share structs chained through next_free fields.
      This is populated and taken off only by the first thread in the
      team encountering a new work sharing construct, in a critical


@@ -323,9 +655,22 @@ struct gomp_team
   struct gomp_work_share work_shares[8];
 
   gomp_mutex_t task_lock;
   struct gomp_work_share work_shares[8];
 
   gomp_mutex_t task_lock;

-  struct gomp_task *task_queue;
-  int task_count;
-  int task_running_count;
+  /* Scheduled tasks.  */
+  struct priority_queue task_queue;
+  /* Number of all GOMP_TASK_{WAITING,TIED} tasks in the team.  */
+  unsigned int task_count;
+  /* Number of GOMP_TASK_WAITING tasks currently waiting to be scheduled.  */
+  unsigned int task_queued_count;
+  /* Number of GOMP_TASK_{WAITING,TIED} tasks currently running
+     directly in gomp_barrier_handle_tasks; tasks spawned
+     from e.g. GOMP_taskwait or GOMP_taskgroup_end don't count, even when
+     that is called from a task run from gomp_barrier_handle_tasks.
+     task_running_count should be always <= team->nthreads,
+     and if current task isn't in_tied_task, then it will be
+     even < team->nthreads.  */
+  unsigned int task_running_count;
+  int work_share_cancelled;
+  int team_cancelled;

 
   /* This array contains structures for implicit tasks.  */
   struct gomp_task implicit_task[];
 
   /* This array contains structures for implicit tasks.  */
   struct gomp_task implicit_task[];


@@ -350,8 +695,25 @@ struct gomp_thread
   /* This semaphore is used for ordered loops.  */
   gomp_sem_t release;
 
   /* This semaphore is used for ordered loops.  */
   gomp_sem_t release;
 

-  /* user pthread thread pool */
+  /* Place this thread is bound to plus one, or zero if not bound
+     to any place.  */
+  unsigned int place;
+
+  /* User pthread thread pool */

   struct gomp_thread_pool *thread_pool;
   struct gomp_thread_pool *thread_pool;

+
+#if defined(LIBGOMP_USE_PTHREADS) \
+    && (!defined(HAVE_TLS) \
+       || !defined(__GLIBC__) \
+       || !defined(USING_INITIAL_EXEC_TLS))
+  /* pthread_t of the thread containing this gomp_thread.
+     On Linux when using initial-exec TLS,
+     (typeof (pthread_t)) gomp_thread () - pthread_self ()
+     is constant in all threads, so we can optimize and not
+     store it.  */
+#define GOMP_NEEDS_THREAD_HANDLE 1
+  pthread_t handle;
+#endif

 };
 
 
 };
 
 


@@ -362,15 +724,56 @@ struct gomp_thread_pool
   struct gomp_thread **threads;
   unsigned threads_size;
   unsigned threads_used;
   struct gomp_thread **threads;
   unsigned threads_size;
   unsigned threads_used;

+  /* The last team is used for non-nested teams to delay their destruction to
+     make sure all the threads in the team move on to the pool's barrier before
+     the team's barrier is destroyed.  */

   struct gomp_team *last_team;
   struct gomp_team *last_team;

+  /* Number of threads running in this contention group.  */
+  unsigned long threads_busy;

 
 

-  /* This barrier holds and releases threads waiting in threads.  */
-  gomp_barrier_t threads_dock;
+  /* This barrier holds and releases threads waiting in thread pools.  */
+  gomp_simple_barrier_t threads_dock;
+};
+
+enum gomp_cancel_kind
+{
+  GOMP_CANCEL_PARALLEL = 1,
+  GOMP_CANCEL_LOOP = 2,
+  GOMP_CANCEL_FOR = GOMP_CANCEL_LOOP,
+  GOMP_CANCEL_DO = GOMP_CANCEL_LOOP,
+  GOMP_CANCEL_SECTIONS = 4,
+  GOMP_CANCEL_TASKGROUP = 8

 };
 
 /* ... and here is that TLS data.  */
 
 };
 
 /* ... and here is that TLS data.  */
 

-#ifdef HAVE_TLS
+#if defined __nvptx__
+extern struct gomp_thread *nvptx_thrs __attribute__((shared));
+static inline struct gomp_thread *gomp_thread (void)
+{
+  int tid;
+  asm ("mov.u32 %0, %%tid.y;" : "=r" (tid));
+  return nvptx_thrs + tid;
+}
+#elif defined __AMDGCN__
+static inline struct gomp_thread *gcn_thrs (void)
+{
+  /* The value is at the bottom of LDS.  */
+  struct gomp_thread * __lds *thrs = (struct gomp_thread * __lds *)4;
+  return *thrs;
+}
+static inline void set_gcn_thrs (struct gomp_thread *val)
+{
+  /* The value is at the bottom of LDS.  */
+  struct gomp_thread * __lds *thrs = (struct gomp_thread * __lds *)4;
+  *thrs = val;
+}
+static inline struct gomp_thread *gomp_thread (void)
+{
+  int tid = __builtin_gcn_dim_pos(1);
+  return gcn_thrs () + tid;
+}
+#elif defined HAVE_TLS || defined USE_EMUTLS

 extern __thread struct gomp_thread gomp_tls_data;
 static inline struct gomp_thread *gomp_thread (void)
 {
 extern __thread struct gomp_thread gomp_tls_data;
 static inline struct gomp_thread *gomp_thread (void)
 {


@@ -399,37 +802,51 @@ static inline struct gomp_task_icv *gomp_icv (bool write)
     return &gomp_global_icv;
 }
 
     return &gomp_global_icv;
 }
 

+#ifdef LIBGOMP_USE_PTHREADS

 /* The attributes to be used during thread creation.  */
 extern pthread_attr_t gomp_thread_attr;
 
 /* The attributes to be used during thread creation.  */
 extern pthread_attr_t gomp_thread_attr;
 

-/* Other variables.  */
-
-extern unsigned short *gomp_cpu_affinity;
-extern size_t gomp_cpu_affinity_len;
+extern pthread_key_t gomp_thread_destructor;
+#endif

 
 /* Function prototypes.  */
 
 /* affinity.c */
 
 extern void gomp_init_affinity (void);
 
 /* Function prototypes.  */
 
 /* affinity.c */
 
 extern void gomp_init_affinity (void);

-extern void gomp_init_thread_affinity (pthread_attr_t *);
-
-/* alloc.c */
-
-extern void *gomp_malloc (size_t) __attribute__((malloc));
-extern void *gomp_malloc_cleared (size_t) __attribute__((malloc));
-extern void *gomp_realloc (void *, size_t);
-
-/* Avoid conflicting prototypes of alloca() in system headers by using
-   GCC's builtin alloca().  */
-#define gomp_alloca(x)  __builtin_alloca(x)
-
-/* error.c */
-
-extern void gomp_error (const char *, ...)
-       __attribute__((format (printf, 1, 2)));
-extern void gomp_fatal (const char *, ...)
-       __attribute__((noreturn, format (printf, 1, 2)));
+#ifdef LIBGOMP_USE_PTHREADS
+extern void gomp_init_thread_affinity (pthread_attr_t *, unsigned int);
+#endif
+extern void **gomp_affinity_alloc (unsigned long, bool);
+extern void gomp_affinity_init_place (void *);
+extern bool gomp_affinity_add_cpus (void *, unsigned long, unsigned long,
+                                   long, bool);
+extern bool gomp_affinity_remove_cpu (void *, unsigned long);
+extern bool gomp_affinity_copy_place (void *, void *, long);
+extern bool gomp_affinity_same_place (void *, void *);
+extern bool gomp_affinity_finalize_place_list (bool);
+extern bool gomp_affinity_init_level (int, unsigned long, bool);
+extern void gomp_affinity_print_place (void *);
+extern void gomp_get_place_proc_ids_8 (int, int64_t *);
+extern void gomp_display_affinity_place (char *, size_t, size_t *, int);
+
+/* affinity-fmt.c */
+
+extern bool gomp_print_string (const char *str, size_t len);
+extern void gomp_set_affinity_format (const char *, size_t);
+extern void gomp_display_string (char *, size_t, size_t *, const char *,
+                                size_t);
+#ifdef LIBGOMP_USE_PTHREADS
+typedef pthread_t gomp_thread_handle;
+#else
+typedef struct {} gomp_thread_handle;
+#endif
+extern size_t gomp_display_affinity (char *, size_t, const char *,
+                                    gomp_thread_handle,
+                                    struct gomp_team_state *, unsigned int);
+extern void gomp_display_affinity_thread (gomp_thread_handle,
+                                         struct gomp_team_state *,
+                                         unsigned int) __attribute__((cold));

 
 /* iter.c */
 
 
 /* iter.c */
 


@@ -466,6 +883,9 @@ extern void gomp_ordered_next (void);
 extern void gomp_ordered_static_init (void);
 extern void gomp_ordered_static_next (void);
 extern void gomp_ordered_sync (void);
 extern void gomp_ordered_static_init (void);
 extern void gomp_ordered_static_next (void);
 extern void gomp_ordered_sync (void);

+extern void gomp_doacross_init (unsigned, long *, long, size_t);
+extern void gomp_doacross_ull_init (unsigned, unsigned long long *,
+                                   unsigned long long, size_t);

 
 /* parallel.c */
 
 
 /* parallel.c */
 


@@ -482,26 +902,303 @@ extern void gomp_init_task (struct gomp_task *, struct gomp_task *,
                            struct gomp_task_icv *);
 extern void gomp_end_task (void);
 extern void gomp_barrier_handle_tasks (gomp_barrier_state_t);
                            struct gomp_task_icv *);
 extern void gomp_end_task (void);
 extern void gomp_barrier_handle_tasks (gomp_barrier_state_t);

+extern void gomp_task_maybe_wait_for_dependencies (void **);
+extern bool gomp_create_target_task (struct gomp_device_descr *,
+                                    void (*) (void *), size_t, void **,
+                                    size_t *, unsigned short *, unsigned int,
+                                    void **, void **,
+                                    enum gomp_target_task_state);
+extern struct gomp_taskgroup *gomp_parallel_reduction_register (uintptr_t *,
+                                                               unsigned);
+extern void gomp_workshare_taskgroup_start (void);
+extern void gomp_workshare_task_reduction_register (uintptr_t *, uintptr_t *);

 
 static void inline
 gomp_finish_task (struct gomp_task *task)
 {
 
 static void inline
 gomp_finish_task (struct gomp_task *task)
 {

-  gomp_sem_destroy (&task->taskwait_sem);
+  if (__builtin_expect (task->depend_hash != NULL, 0))
+    free (task->depend_hash);

 }
 
 /* team.c */
 
 extern struct gomp_team *gomp_new_team (unsigned);
 extern void gomp_team_start (void (*) (void *), void *, unsigned,
 }
 
 /* team.c */
 
 extern struct gomp_team *gomp_new_team (unsigned);
 extern void gomp_team_start (void (*) (void *), void *, unsigned,

-                            struct gomp_team *);
+                            unsigned, struct gomp_team *,
+                            struct gomp_taskgroup *);

 extern void gomp_team_end (void);
 extern void gomp_team_end (void);

+extern void gomp_free_thread (void *);
+extern int gomp_pause_host (void);
+
+/* target.c */
+
+extern void gomp_init_targets_once (void);
+extern int gomp_get_num_devices (void);
+extern bool gomp_target_task_fn (void *);
+
+/* Splay tree definitions.  */
+typedef struct splay_tree_node_s *splay_tree_node;
+typedef struct splay_tree_s *splay_tree;
+typedef struct splay_tree_key_s *splay_tree_key;
+
+struct target_var_desc {
+  /* Splay key.  */
+  splay_tree_key key;
+  /* True if data should be copied from device to host at the end.  */
+  bool copy_from;
+  /* True if data always should be copied from device to host at the end.  */
+  bool always_copy_from;
+  /* True if variable should be detached at end of region.  */
+  bool do_detach;
+  /* Relative offset against key host_start.  */
+  uintptr_t offset;
+  /* Actual length.  */
+  uintptr_t length;
+};
+
+struct target_mem_desc {
+  /* Reference count.  */
+  uintptr_t refcount;
+  /* All the splay nodes allocated together.  */
+  splay_tree_node array;
+  /* Start of the target region.  */
+  uintptr_t tgt_start;
+  /* End of the targer region.  */
+  uintptr_t tgt_end;
+  /* Handle to free.  */
+  void *to_free;
+  /* Previous target_mem_desc.  */
+  struct target_mem_desc *prev;
+  /* Number of items in following list.  */
+  size_t list_count;
+
+  /* Corresponding target device descriptor.  */
+  struct gomp_device_descr *device_descr;
+
+  /* List of target items to remove (or decrease refcount)
+     at the end of region.  */
+  struct target_var_desc list[];
+};
+
+/* Special value for refcount - infinity.  */
+#define REFCOUNT_INFINITY (~(uintptr_t) 0)
+/* Special value for refcount - tgt_offset contains target address of the
+   artificial pointer to "omp declare target link" object.  */
+#define REFCOUNT_LINK (~(uintptr_t) 1)
+
+/* Special offset values.  */
+#define OFFSET_INLINED (~(uintptr_t) 0)
+#define OFFSET_POINTER (~(uintptr_t) 1)
+#define OFFSET_STRUCT (~(uintptr_t) 2)
+
+/* Auxiliary structure for infrequently-used or API-specific data.  */
+
+struct splay_tree_aux {
+  /* Pointer to the original mapping of "omp declare target link" object.  */
+  splay_tree_key link_key;
+  /* For a block with attached pointers, the attachment counters for each.
+     Only used for OpenACC.  */
+  uintptr_t *attach_count;
+};
+
+struct splay_tree_key_s {
+  /* Address of the host object.  */
+  uintptr_t host_start;
+  /* Address immediately after the host object.  */
+  uintptr_t host_end;
+  /* Descriptor of the target memory.  */
+  struct target_mem_desc *tgt;
+  /* Offset from tgt->tgt_start to the start of the target object.  */
+  uintptr_t tgt_offset;
+  /* Reference count.  */
+  uintptr_t refcount;
+  /* Reference counts beyond those that represent genuine references in the
+     linked splay tree key/target memory structures, e.g. for multiple OpenACC
+     "present increment" operations (via "acc enter data") referring to the same
+     host-memory block.  */
+  uintptr_t virtual_refcount;
+  struct splay_tree_aux *aux;
+};
+
+/* The comparison function.  */
+
+static inline int
+splay_compare (splay_tree_key x, splay_tree_key y)
+{
+  if (x->host_start == x->host_end
+      && y->host_start == y->host_end)
+    return 0;
+  if (x->host_end <= y->host_start)
+    return -1;
+  if (x->host_start >= y->host_end)
+    return 1;
+  return 0;
+}
+
+#include "splay-tree.h"
+
+typedef struct acc_dispatch_t
+{
+  /* Execute.  */
+  __typeof (GOMP_OFFLOAD_openacc_exec) *exec_func;
+
+  /* Create/destroy TLS data.  */
+  __typeof (GOMP_OFFLOAD_openacc_create_thread_data) *create_thread_data_func;
+  __typeof (GOMP_OFFLOAD_openacc_destroy_thread_data)
+    *destroy_thread_data_func;
+  
+  struct {
+    /* Once created and put into the "active" list, asyncqueues are then never
+       destructed and removed from the "active" list, other than if the TODO
+       device is shut down.  */
+    gomp_mutex_t lock;
+    int nasyncqueue;
+    struct goacc_asyncqueue **asyncqueue;
+    struct goacc_asyncqueue_list *active;
+
+    __typeof (GOMP_OFFLOAD_openacc_async_construct) *construct_func;
+    __typeof (GOMP_OFFLOAD_openacc_async_destruct) *destruct_func;
+    __typeof (GOMP_OFFLOAD_openacc_async_test) *test_func;
+    __typeof (GOMP_OFFLOAD_openacc_async_synchronize) *synchronize_func;
+    __typeof (GOMP_OFFLOAD_openacc_async_serialize) *serialize_func;
+    __typeof (GOMP_OFFLOAD_openacc_async_queue_callback) *queue_callback_func;
+
+    __typeof (GOMP_OFFLOAD_openacc_async_exec) *exec_func;
+    __typeof (GOMP_OFFLOAD_openacc_async_dev2host) *dev2host_func;
+    __typeof (GOMP_OFFLOAD_openacc_async_host2dev) *host2dev_func;
+  } async;
+
+  __typeof (GOMP_OFFLOAD_openacc_get_property) *get_property_func;
+
+  /* NVIDIA target specific routines.  */
+  struct {
+    __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_device)
+      *get_current_device_func;
+    __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_context)
+      *get_current_context_func;
+    __typeof (GOMP_OFFLOAD_openacc_cuda_get_stream) *get_stream_func;
+    __typeof (GOMP_OFFLOAD_openacc_cuda_set_stream) *set_stream_func;
+  } cuda;
+} acc_dispatch_t;
+
+/* Various state of the accelerator device.  */
+enum gomp_device_state
+{
+  GOMP_DEVICE_UNINITIALIZED,
+  GOMP_DEVICE_INITIALIZED,
+  GOMP_DEVICE_FINALIZED
+};
+
+/* This structure describes accelerator device.
+   It contains name of the corresponding libgomp plugin, function handlers for
+   interaction with the device, ID-number of the device, and information about
+   mapped memory.  */
+struct gomp_device_descr
+{
+  /* Immutable data, which is only set during initialization, and which is not
+     guarded by the lock.  */
+
+  /* The name of the device.  */
+  const char *name;
+
+  /* Capabilities of device (supports OpenACC, OpenMP).  */
+  unsigned int capabilities;
+
+  /* This is the ID number of device among devices of the same type.  */
+  int target_id;
+
+  /* This is the TYPE of device.  */
+  enum offload_target_type type;
+
+  /* Function handlers.  */
+  __typeof (GOMP_OFFLOAD_get_name) *get_name_func;
+  __typeof (GOMP_OFFLOAD_get_caps) *get_caps_func;
+  __typeof (GOMP_OFFLOAD_get_type) *get_type_func;
+  __typeof (GOMP_OFFLOAD_get_num_devices) *get_num_devices_func;
+  __typeof (GOMP_OFFLOAD_init_device) *init_device_func;
+  __typeof (GOMP_OFFLOAD_fini_device) *fini_device_func;
+  __typeof (GOMP_OFFLOAD_version) *version_func;
+  __typeof (GOMP_OFFLOAD_load_image) *load_image_func;
+  __typeof (GOMP_OFFLOAD_unload_image) *unload_image_func;
+  __typeof (GOMP_OFFLOAD_alloc) *alloc_func;
+  __typeof (GOMP_OFFLOAD_free) *free_func;
+  __typeof (GOMP_OFFLOAD_dev2host) *dev2host_func;
+  __typeof (GOMP_OFFLOAD_host2dev) *host2dev_func;
+  __typeof (GOMP_OFFLOAD_dev2dev) *dev2dev_func;
+  __typeof (GOMP_OFFLOAD_can_run) *can_run_func;
+  __typeof (GOMP_OFFLOAD_run) *run_func;
+  __typeof (GOMP_OFFLOAD_async_run) *async_run_func;
+
+  /* Splay tree containing information about mapped memory regions.  */
+  struct splay_tree_s mem_map;
+
+  /* Mutex for the mutable data.  */
+  gomp_mutex_t lock;
+
+  /* Current state of the device.  OpenACC allows to move from INITIALIZED state
+     back to UNINITIALIZED state.  OpenMP allows only to move from INITIALIZED
+     to FINALIZED state (at program shutdown).  */
+  enum gomp_device_state state;
+
+  /* OpenACC-specific data and functions.  */
+  /* This is mutable because of its mutable target_data member.  */
+  acc_dispatch_t openacc;
+};
+
+/* Kind of the pragma, for which gomp_map_vars () is called.  */
+enum gomp_map_vars_kind
+{
+  GOMP_MAP_VARS_OPENACC,
+  GOMP_MAP_VARS_OPENACC_ENTER_DATA,
+  GOMP_MAP_VARS_TARGET,
+  GOMP_MAP_VARS_DATA,
+  GOMP_MAP_VARS_ENTER_DATA
+};
+
+extern void gomp_acc_declare_allocate (bool, size_t, void **, size_t *,
+                                      unsigned short *);
+struct gomp_coalesce_buf;
+extern void gomp_copy_host2dev (struct gomp_device_descr *,
+                               struct goacc_asyncqueue *, void *, const void *,
+                               size_t, struct gomp_coalesce_buf *);
+extern void gomp_copy_dev2host (struct gomp_device_descr *,
+                               struct goacc_asyncqueue *, void *, const void *,
+                               size_t);
+extern uintptr_t gomp_map_val (struct target_mem_desc *, void **, size_t);
+extern void gomp_attach_pointer (struct gomp_device_descr *,
+                                struct goacc_asyncqueue *, splay_tree,
+                                splay_tree_key, uintptr_t, size_t,
+                                struct gomp_coalesce_buf *);
+extern void gomp_detach_pointer (struct gomp_device_descr *,
+                                struct goacc_asyncqueue *, splay_tree_key,
+                                uintptr_t, bool, struct gomp_coalesce_buf *);
+
+extern struct target_mem_desc *gomp_map_vars (struct gomp_device_descr *,
+                                             size_t, void **, void **,
+                                             size_t *, void *, bool,
+                                             enum gomp_map_vars_kind);
+extern struct target_mem_desc *gomp_map_vars_async (struct gomp_device_descr *,
+                                                   struct goacc_asyncqueue *,
+                                                   size_t, void **, void **,
+                                                   size_t *, void *, bool,
+                                                   enum gomp_map_vars_kind);
+extern void gomp_unmap_vars (struct target_mem_desc *, bool);
+extern void gomp_unmap_vars_async (struct target_mem_desc *, bool,
+                                  struct goacc_asyncqueue *);
+extern void gomp_init_device (struct gomp_device_descr *);
+extern bool gomp_fini_device (struct gomp_device_descr *);
+extern void gomp_unload_device (struct gomp_device_descr *);
+extern bool gomp_remove_var (struct gomp_device_descr *, splay_tree_key);
+extern void gomp_remove_var_async (struct gomp_device_descr *, splay_tree_key,
+                                  struct goacc_asyncqueue *);

 
 /* work.c */
 
 
 /* work.c */
 

-extern void gomp_init_work_share (struct gomp_work_share *, bool, unsigned);
+extern void gomp_init_work_share (struct gomp_work_share *, size_t, unsigned);

 extern void gomp_fini_work_share (struct gomp_work_share *);
 extern void gomp_fini_work_share (struct gomp_work_share *);

-extern bool gomp_work_share_start (bool);
+extern bool gomp_work_share_start (size_t);

 extern void gomp_work_share_end (void);
 extern void gomp_work_share_end (void);

+extern bool gomp_work_share_end_cancel (void);

 extern void gomp_work_share_end_nowait (void);
 
 static inline void
 extern void gomp_work_share_end_nowait (void);
 
 static inline void


@@ -555,8 +1252,6 @@ extern void gomp_set_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
 extern void gomp_unset_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
 extern int gomp_test_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
 
 extern void gomp_unset_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
 extern int gomp_test_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
 

-# define strong_alias(fn, al) \
-  extern __typeof (fn) al __attribute__ ((alias (#fn)));

 # define omp_lock_symver(fn) \
   __asm (".symver g" #fn "_30, " #fn "@@OMP_3.0"); \
   __asm (".symver g" #fn "_25, " #fn "@OMP_1.0");
 # define omp_lock_symver(fn) \
   __asm (".symver g" #fn "_30, " #fn "@@OMP_3.0"); \
   __asm (".symver g" #fn "_25, " #fn "@OMP_1.0");


@@ -579,12 +1274,101 @@ extern int gomp_test_nest_lock_25 (omp_nest_lock_25_t *) __GOMP_NOTHROW;
 # define attribute_hidden
 #endif
 
 # define attribute_hidden
 #endif
 

+#if __GNUC__ >= 9
+#  define HAVE_ATTRIBUTE_COPY
+#endif
+
+#ifdef HAVE_ATTRIBUTE_COPY
+# define attribute_copy(arg) __attribute__ ((copy (arg)))
+#else
+# define attribute_copy(arg)
+#endif
+

 #ifdef HAVE_ATTRIBUTE_ALIAS
 #ifdef HAVE_ATTRIBUTE_ALIAS

+# define strong_alias(fn, al) \
+  extern __typeof (fn) al __attribute__ ((alias (#fn))) attribute_copy (fn);
+
+# define ialias_ulp    ialias_str1(__USER_LABEL_PREFIX__)
+# define ialias_str1(x)        ialias_str2(x)
+# define ialias_str2(x)        #x

 # define ialias(fn) \
   extern __typeof (fn) gomp_ialias_##fn \
 # define ialias(fn) \
   extern __typeof (fn) gomp_ialias_##fn \

-    __attribute__ ((alias (#fn))) attribute_hidden;
+    __attribute__ ((alias (#fn))) attribute_hidden attribute_copy (fn);
+# define ialias_redirect(fn) \
+  extern __typeof (fn) fn __asm__ (ialias_ulp "gomp_ialias_" #fn) attribute_hidden;
+# define ialias_call(fn) gomp_ialias_ ## fn

 #else
 # define ialias(fn)
 #else
 # define ialias(fn)

+# define ialias_redirect(fn)
+# define ialias_call(fn) fn
+#endif
+
+/* Helper function for priority_node_to_task() and
+   task_to_priority_node().
+
+   Return the offset from a task to its priority_node entry.  The
+   priority_node entry is has a type of TYPE.  */
+
+static inline size_t
+priority_queue_offset (enum priority_queue_type type)
+{
+  return offsetof (struct gomp_task, pnode[(int) type]);
+}
+
+/* Return the task associated with a priority NODE of type TYPE.  */
+
+static inline struct gomp_task *
+priority_node_to_task (enum priority_queue_type type,
+                      struct priority_node *node)
+{
+  return (struct gomp_task *) ((char *) node - priority_queue_offset (type));
+}
+
+/* Return the priority node of type TYPE for a given TASK.  */
+
+static inline struct priority_node *
+task_to_priority_node (enum priority_queue_type type,
+                      struct gomp_task *task)
+{
+  return (struct priority_node *) ((char *) task
+                                  + priority_queue_offset (type));
+}
+
+#ifdef LIBGOMP_USE_PTHREADS
+static inline gomp_thread_handle
+gomp_thread_self (void)
+{
+  return pthread_self ();
+}
+
+static inline gomp_thread_handle
+gomp_thread_to_pthread_t (struct gomp_thread *thr)
+{
+  struct gomp_thread *this_thr = gomp_thread ();
+  if (thr == this_thr)
+    return pthread_self ();
+#ifdef GOMP_NEEDS_THREAD_HANDLE
+  return thr->handle;
+#else
+  /* On Linux with initial-exec TLS, the pthread_t of the thread containing
+     thr can be computed from thr, this_thr and pthread_self (),
+     as the distance between this_thr and pthread_self () is constant.  */
+  return pthread_self () + ((uintptr_t) thr - (uintptr_t) this_thr);
+#endif
+}
+#else
+static inline gomp_thread_handle
+gomp_thread_self (void)
+{
+  return (gomp_thread_handle) {};
+}
+
+static inline gomp_thread_handle
+gomp_thread_to_pthread_t (struct gomp_thread *thr)
+{
+  (void) thr;
+  return gomp_thread_self ();
+}

 #endif
 
 #endif /* LIBGOMP_H */
 #endif
 
 #endif /* LIBGOMP_H */