1 /* Copyright (C) 2005-2020 Free Software Foundation, Inc.
2 Contributed by Richard Henderson <rth@redhat.com>.
4 This file is part of the GNU Offloading and Multi Processing Library
7 Libgomp is free software; you can redistribute it and/or modify it
8 under the terms of the GNU General Public License as published by
9 the Free Software Foundation; either version 3, or (at your option)
12 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
13 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
14 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
17 Under Section 7 of GPL version 3, you are granted additional
18 permissions described in the GCC Runtime Library Exception, version
19 3.1, as published by the Free Software Foundation.
21 You should have received a copy of the GNU General Public License and
22 a copy of the GCC Runtime Library Exception along with this program;
23 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
24 <http://www.gnu.org/licenses/>. */
26 /* This file contains data types and function declarations that are not
27 part of the official OpenACC or OpenMP user interfaces. There are
28 declarations in here that are part of the GNU Offloading and Multi
29 Processing ABI, in that the compiler is required to know about them
32 The convention is that the all caps prefix "GOMP" is used group items
33 that are part of the external ABI, and the lower case prefix "gomp"
34 is used group items that are completely private to the library. */
39 #ifndef _LIBGOMP_CHECKING_
40 /* Define to 1 to perform internal sanity checks. */
41 #define _LIBGOMP_CHECKING_ 0
46 #include "libgomp-plugin.h"
47 #include "gomp-constants.h"
56 /* Needed for memset in priority_queue.c. */
57 #if _LIBGOMP_CHECKING_
58 # ifdef STRING_WITH_STRINGS
65 # ifdef HAVE_STRINGS_H
72 #ifdef HAVE_ATTRIBUTE_VISIBILITY
73 # pragma GCC visibility push(hidden)
76 /* If we were a C++ library, we'd get this from <std/atomic>. */
89 #if defined(HAVE_ALIGNED_ALLOC) \
90 || defined(HAVE__ALIGNED_MALLOC) \
91 || defined(HAVE_POSIX_MEMALIGN) \
92 || defined(HAVE_MEMALIGN)
93 /* Defined if gomp_aligned_alloc doesn't use fallback version
94 and free can be used instead of gomp_aligned_free. */
95 #define GOMP_HAVE_EFFICIENT_ALIGNED_ALLOC 1
98 extern void *gomp_malloc (size_t) __attribute__((malloc
));
99 extern void *gomp_malloc_cleared (size_t) __attribute__((malloc
));
100 extern void *gomp_realloc (void *, size_t);
101 extern void *gomp_aligned_alloc (size_t, size_t)
102 __attribute__((malloc
, alloc_size (2)));
103 extern void gomp_aligned_free (void *);
105 /* Avoid conflicting prototypes of alloca() in system headers by using
106 GCC's builtin alloca(). */
107 #define gomp_alloca(x) __builtin_alloca(x)
109 /* Optimized allocators for team-specific data that will die with the team. */
112 /* The arena is initialized in config/gcn/team.c. */
113 #define TEAM_ARENA_SIZE 64*1024 /* Must match the value in plugin-gcn.c. */
114 #define TEAM_ARENA_START 16 /* LDS offset of free pointer. */
115 #define TEAM_ARENA_FREE 24 /* LDS offset of free pointer. */
116 #define TEAM_ARENA_END 32 /* LDS offset of end pointer. */
118 static inline void * __attribute__((malloc
))
119 team_malloc (size_t size
)
121 /* 4-byte align the size. */
122 size
= (size
+ 3) & ~3;
124 /* Allocate directly from the arena.
125 The compiler does not support DS atomics, yet. */
127 asm ("ds_add_rtn_u64 %0, %1, %2\n\ts_waitcnt 0"
128 : "=v"(result
) : "v"(TEAM_ARENA_FREE
), "v"(size
), "e"(1L) : "memory");
131 if (result
+ size
> *(void * __lds
*)TEAM_ARENA_END
)
133 /* While this is experimental, let's make sure we know when OOM
135 const char msg
[] = "GCN team arena exhausted\n";
136 write (2, msg
, sizeof(msg
)-1);
138 /* Fall back to using the heap (slowly). */
139 result
= gomp_malloc (size
);
144 static inline void * __attribute__((malloc
))
145 team_malloc_cleared (size_t size
)
147 char *result
= team_malloc (size
);
149 /* Clear the allocated memory. */
150 __builtin_memset (result
, 0, size
);
156 team_free (void *ptr
)
158 /* The whole arena is freed when the kernel exits.
159 However, if we fell back to using heap then we should free it.
160 It would be better if this function could be a no-op, but at least
161 LDS loads are cheap. */
162 if (ptr
< *(void * __lds
*)TEAM_ARENA_START
163 || ptr
>= *(void * __lds
*)TEAM_ARENA_END
)
167 #define team_malloc(...) gomp_malloc (__VA_ARGS__)
168 #define team_malloc_cleared(...) gomp_malloc_cleared (__VA_ARGS__)
169 #define team_free(...) free (__VA_ARGS__)
174 extern void gomp_vdebug (int, const char *, va_list);
175 extern void gomp_debug (int, const char *, ...)
176 __attribute__ ((format (printf
, 2, 3)));
177 #define gomp_vdebug(KIND, FMT, VALIST) \
179 if (__builtin_expect (gomp_debug_var, 0)) \
180 (gomp_vdebug) ((KIND), (FMT), (VALIST)); \
182 #define gomp_debug(KIND, ...) \
184 if (__builtin_expect (gomp_debug_var, 0)) \
185 (gomp_debug) ((KIND), __VA_ARGS__); \
187 extern void gomp_verror (const char *, va_list);
188 extern void gomp_error (const char *, ...)
189 __attribute__ ((format (printf
, 1, 2)));
190 extern void gomp_vfatal (const char *, va_list)
191 __attribute__ ((noreturn
));
192 extern void gomp_fatal (const char *, ...)
193 __attribute__ ((noreturn
, format (printf
, 1, 2)));
196 struct gomp_taskgroup
;
199 #include "priority_queue.h"
203 #include "simple-bar.h"
207 /* This structure contains the data to control one work-sharing construct,
208 either a LOOP (FOR/DO) or a SECTIONS. */
210 enum gomp_schedule_type
217 GFS_MONOTONIC
= 0x80000000U
220 struct gomp_doacross_work_share
223 /* chunk_size copy, as ws->chunk_size is multiplied by incr for
226 /* Likewise, but for ull implementation. */
227 unsigned long long chunk_size_ull
;
228 /* For schedule(static,0) this is the number
229 of iterations assigned to the last thread, i.e. number of
230 iterations / number of threads. */
232 /* Likewise, but for ull implementation. */
233 unsigned long long q_ull
;
235 /* Size of each array entry (padded to cache line size). */
236 unsigned long elt_sz
;
237 /* Number of dimensions in sink vectors. */
238 unsigned int ncounts
;
239 /* True if the iterations can be flattened. */
241 /* Actual array (of elt_sz sized units), aligned to cache line size.
242 This is indexed by team_id for GFS_STATIC and outermost iteration
243 / chunk_size for other schedules. */
244 unsigned char *array
;
245 /* These two are only used for schedule(static,0). */
246 /* This one is number of iterations % number of threads. */
249 /* And this one is cached t * (q + 1). */
251 /* Likewise, but for the ull implementation. */
252 unsigned long long boundary_ull
;
254 /* Pointer to extra memory if needed for lastprivate(conditional). */
256 /* Array of shift counts for each dimension if they can be flattened. */
257 unsigned int shift_counts
[];
260 struct gomp_work_share
262 /* This member records the SCHEDULE clause to be used for this construct.
263 The user specification of "runtime" will already have been resolved.
264 If this is a SECTIONS construct, this value will always be DYNAMIC. */
265 enum gomp_schedule_type sched
;
271 /* This is the chunk_size argument to the SCHEDULE clause. */
274 /* This is the iteration end point. If this is a SECTIONS construct,
275 this is the number of contained sections. */
278 /* This is the iteration step. If this is a SECTIONS construct, this
284 /* The same as above, but for the unsigned long long loop variants. */
285 unsigned long long chunk_size_ull
;
286 unsigned long long end_ull
;
287 unsigned long long incr_ull
;
292 /* This is a circular queue that details which threads will be allowed
293 into the ordered region and in which order. When a thread allocates
294 iterations on which it is going to work, it also registers itself at
295 the end of the array. When a thread reaches the ordered region, it
296 checks to see if it is the one at the head of the queue. If not, it
297 blocks on its RELEASE semaphore. */
298 unsigned *ordered_team_ids
;
300 /* This is a pointer to DOACROSS work share data. */
301 struct gomp_doacross_work_share
*doacross
;
304 /* This is the number of threads that have registered themselves in
305 the circular queue ordered_team_ids. */
306 unsigned ordered_num_used
;
308 /* This is the team_id of the currently acknowledged owner of the ordered
309 section, or -1u if the ordered section has not been acknowledged by
310 any thread. This is distinguished from the thread that is *allowed*
311 to take the section next. */
312 unsigned ordered_owner
;
314 /* This is the index into the circular queue ordered_team_ids of the
315 current thread that's allowed into the ordered reason. */
316 unsigned ordered_cur
;
318 /* This is a chain of allocated gomp_work_share blocks, valid only
319 in the first gomp_work_share struct in the block. */
320 struct gomp_work_share
*next_alloc
;
322 /* The above fields are written once during workshare initialization,
323 or related to ordered worksharing. Make sure the following fields
324 are in a different cache line. */
326 /* This lock protects the update of the following members. */
327 gomp_mutex_t lock
__attribute__((aligned (64)));
329 /* This is the count of the number of threads that have exited the work
330 share construct. If the construct was marked nowait, they have moved on
331 to other work; otherwise they're blocked on a barrier. The last member
332 of the team to exit the work share construct must deallocate it. */
333 unsigned threads_completed
;
336 /* This is the next iteration value to be allocated. In the case of
337 GFS_STATIC loops, this the iteration start point and never changes. */
340 /* The same, but with unsigned long long type. */
341 unsigned long long next_ull
;
343 /* This is the returned data structure for SINGLE COPYPRIVATE. */
348 /* Link to gomp_work_share struct for next work sharing construct
349 encountered after this one. */
350 gomp_ptrlock_t next_ws
;
352 /* gomp_work_share structs are chained in the free work share cache
354 struct gomp_work_share
*next_free
;
357 /* Task reductions for this work-sharing construct. */
358 uintptr_t *task_reductions
;
360 /* If only few threads are in the team, ordered_team_ids can point
361 to this array which fills the padding at the end of this struct. */
362 unsigned inline_ordered_team_ids
[0];
365 /* This structure contains all of the thread-local data associated with
366 a thread team. This is the data that must be saved when a thread
367 encounters a nested PARALLEL construct. */
369 struct gomp_team_state
371 /* This is the team of which the thread is currently a member. */
372 struct gomp_team
*team
;
374 /* This is the work share construct which this thread is currently
375 processing. Recall that with NOWAIT, not all threads may be
376 processing the same construct. */
377 struct gomp_work_share
*work_share
;
379 /* This is the previous work share construct or NULL if there wasn't any.
380 When all threads are done with the current work sharing construct,
381 the previous one can be freed. The current one can't, as its
382 next_ws field is used. */
383 struct gomp_work_share
*last_work_share
;
385 /* This is the ID of this thread within the team. This value is
386 guaranteed to be between 0 and N-1, where N is the number of
387 threads in the team. */
393 /* Active nesting level. Only active parallel regions are counted. */
394 unsigned active_level
;
396 /* Place-partition-var, offset and length into gomp_places_list array. */
397 unsigned place_partition_off
;
398 unsigned place_partition_len
;
400 /* Def-allocator-var ICV. */
401 uintptr_t def_allocator
;
403 #ifdef HAVE_SYNC_BUILTINS
404 /* Number of single stmts encountered. */
405 unsigned long single_count
;
408 /* For GFS_RUNTIME loops that resolved to GFS_STATIC, this is the
409 trip number through the loop. So first time a particular loop
410 is encountered this number is 0, the second time through the loop
411 is 1, etc. This is unused when the compiler knows in advance that
412 the loop is statically scheduled. */
413 unsigned long static_trip
;
416 struct target_mem_desc
;
418 /* These are the OpenMP 4.0 Internal Control Variables described in
419 section 2.3.1. Those described as having one copy per task are
420 stored within the structure; those described as having one copy
421 for the whole program are (naturally) global variables. */
425 unsigned long nthreads_var
;
426 enum gomp_schedule_type run_sched_var
;
427 int run_sched_chunk_size
;
428 int default_device_var
;
429 unsigned int thread_limit_var
;
434 struct target_mem_desc
*target_data
;
437 extern struct gomp_task_icv gomp_global_icv
;
438 #ifndef HAVE_SYNC_BUILTINS
439 extern gomp_mutex_t gomp_managed_threads_lock
;
441 extern unsigned long gomp_max_active_levels_var
;
442 extern bool gomp_cancel_var
;
443 extern int gomp_max_task_priority_var
;
444 extern unsigned long long gomp_spin_count_var
, gomp_throttled_spin_count_var
;
445 extern unsigned long gomp_available_cpus
, gomp_managed_threads
;
446 extern unsigned long *gomp_nthreads_var_list
, gomp_nthreads_var_list_len
;
447 extern char *gomp_bind_var_list
;
448 extern unsigned long gomp_bind_var_list_len
;
449 extern void **gomp_places_list
;
450 extern unsigned long gomp_places_list_len
;
451 extern unsigned int gomp_num_teams_var
;
452 extern int gomp_debug_var
;
453 extern bool gomp_display_affinity_var
;
454 extern char *gomp_affinity_format_var
;
455 extern size_t gomp_affinity_format_len
;
456 extern uintptr_t gomp_def_allocator
;
457 extern int goacc_device_num
;
458 extern char *goacc_device_type
;
459 extern int goacc_default_dims
[GOMP_DIM_MAX
];
465 /* Undeferred task. */
466 GOMP_TASK_UNDEFERRED
,
467 /* Task created by GOMP_task and waiting to be run. */
469 /* Task currently executing or scheduled and about to execute. */
471 /* Used for target tasks that have vars mapped and async run started,
472 but not yet completed. Once that completes, they will be readded
473 into the queues as GOMP_TASK_WAITING in order to perform the var
475 GOMP_TASK_ASYNC_RUNNING
478 struct gomp_task_depend_entry
480 /* Address of dependency. */
482 struct gomp_task_depend_entry
*next
;
483 struct gomp_task_depend_entry
*prev
;
484 /* Task that provides the dependency in ADDR. */
485 struct gomp_task
*task
;
486 /* Depend entry is of type "IN". */
492 struct gomp_dependers_vec
496 struct gomp_task
*elem
[];
499 /* Used when in GOMP_taskwait or in gomp_task_maybe_wait_for_dependencies. */
505 /* Number of tasks we are waiting for. */
507 gomp_sem_t taskwait_sem
;
510 /* This structure describes a "task" to be run by a thread. */
514 /* Parent of this task. */
515 struct gomp_task
*parent
;
516 /* Children of this task. */
517 struct priority_queue children_queue
;
518 /* Taskgroup this task belongs in. */
519 struct gomp_taskgroup
*taskgroup
;
520 /* Tasks that depend on this task. */
521 struct gomp_dependers_vec
*dependers
;
522 struct htab
*depend_hash
;
523 struct gomp_taskwait
*taskwait
;
524 /* Number of items in DEPEND. */
526 /* Number of tasks this task depends on. Once this counter reaches
527 0, we have no unsatisfied dependencies, and this task can be put
528 into the various queues to be scheduled. */
529 size_t num_dependees
;
531 /* Priority of this task. */
533 /* The priority node for this task in each of the different queues.
534 We put this here to avoid allocating space for each priority
535 node. Then we play offsetof() games to convert between pnode[]
536 entries and the gomp_task in which they reside. */
537 struct priority_node pnode
[3];
539 struct gomp_task_icv icv
;
542 enum gomp_task_kind kind
;
545 bool copy_ctors_done
;
546 /* Set for undeferred tasks with unsatisfied dependencies which
547 block further execution of their parent until the dependencies
549 bool parent_depends_on
;
550 /* Dependencies provided and/or needed for this task. DEPEND_COUNT
551 is the number of items available. */
552 struct gomp_task_depend_entry depend
[];
555 /* This structure describes a single #pragma omp taskgroup. */
557 struct gomp_taskgroup
559 struct gomp_taskgroup
*prev
;
560 /* Queue of tasks that belong in this taskgroup. */
561 struct priority_queue taskgroup_queue
;
562 uintptr_t *reductions
;
563 bool in_taskgroup_wait
;
566 gomp_sem_t taskgroup_sem
;
570 /* Various state of OpenMP async offloading tasks. */
571 enum gomp_target_task_state
573 GOMP_TARGET_TASK_DATA
,
574 GOMP_TARGET_TASK_BEFORE_MAP
,
575 GOMP_TARGET_TASK_FALLBACK
,
576 GOMP_TARGET_TASK_READY_TO_RUN
,
577 GOMP_TARGET_TASK_RUNNING
,
578 GOMP_TARGET_TASK_FINISHED
581 /* This structure describes a target task. */
583 struct gomp_target_task
585 struct gomp_device_descr
*devicep
;
589 unsigned short *kinds
;
591 enum gomp_target_task_state state
;
592 struct target_mem_desc
*tgt
;
593 struct gomp_task
*task
;
594 struct gomp_team
*team
;
595 /* Device-specific target arguments. */
600 /* This structure describes a "team" of threads. These are the threads
601 that are spawned by a PARALLEL constructs, as well as the work sharing
602 constructs that the team encounters. */
606 /* This is the number of threads in the current team. */
609 /* This is number of gomp_work_share structs that have been allocated
610 as a block last time. */
611 unsigned work_share_chunk
;
613 /* This is the saved team state that applied to a master thread before
614 the current thread was created. */
615 struct gomp_team_state prev_ts
;
617 /* This semaphore should be used by the master thread instead of its
618 "native" semaphore in the thread structure. Required for nested
619 parallels, as the master is a member of two teams. */
620 gomp_sem_t master_release
;
622 /* This points to an array with pointers to the release semaphore
623 of the threads in the team. */
624 gomp_sem_t
**ordered_release
;
626 /* List of work shares on which gomp_fini_work_share hasn't been
627 called yet. If the team hasn't been cancelled, this should be
628 equal to each thr->ts.work_share, but otherwise it can be a possibly
629 long list of workshares. */
630 struct gomp_work_share
*work_shares_to_free
;
632 /* List of gomp_work_share structs chained through next_free fields.
633 This is populated and taken off only by the first thread in the
634 team encountering a new work sharing construct, in a critical
636 struct gomp_work_share
*work_share_list_alloc
;
638 /* List of gomp_work_share structs freed by free_work_share. New
639 entries are atomically added to the start of the list, and
640 alloc_work_share can safely only move all but the first entry
641 to work_share_list alloc, as free_work_share can happen concurrently
642 with alloc_work_share. */
643 struct gomp_work_share
*work_share_list_free
;
645 #ifdef HAVE_SYNC_BUILTINS
646 /* Number of simple single regions encountered by threads in this
648 unsigned long single_count
;
650 /* Mutex protecting addition of workshares to work_share_list_free. */
651 gomp_mutex_t work_share_list_free_lock
;
654 /* This barrier is used for most synchronization of the team. */
655 gomp_barrier_t barrier
;
657 /* Initial work shares, to avoid allocating any gomp_work_share
658 structs in the common case. */
659 struct gomp_work_share work_shares
[8];
661 gomp_mutex_t task_lock
;
662 /* Scheduled tasks. */
663 struct priority_queue task_queue
;
664 /* Number of all GOMP_TASK_{WAITING,TIED} tasks in the team. */
665 unsigned int task_count
;
666 /* Number of GOMP_TASK_WAITING tasks currently waiting to be scheduled. */
667 unsigned int task_queued_count
;
668 /* Number of GOMP_TASK_{WAITING,TIED} tasks currently running
669 directly in gomp_barrier_handle_tasks; tasks spawned
670 from e.g. GOMP_taskwait or GOMP_taskgroup_end don't count, even when
671 that is called from a task run from gomp_barrier_handle_tasks.
672 task_running_count should be always <= team->nthreads,
673 and if current task isn't in_tied_task, then it will be
674 even < team->nthreads. */
675 unsigned int task_running_count
;
676 int work_share_cancelled
;
679 /* This array contains structures for implicit tasks. */
680 struct gomp_task implicit_task
[];
683 /* This structure contains all data that is private to libgomp and is
684 allocated per thread. */
688 /* This is the function that the thread should run upon launch. */
689 void (*fn
) (void *data
);
692 /* This is the current team state for this thread. The ts.team member
693 is NULL only if the thread is idle. */
694 struct gomp_team_state ts
;
696 /* This is the task that the thread is currently executing. */
697 struct gomp_task
*task
;
699 /* This semaphore is used for ordered loops. */
702 /* Place this thread is bound to plus one, or zero if not bound
706 /* User pthread thread pool */
707 struct gomp_thread_pool
*thread_pool
;
709 #if defined(LIBGOMP_USE_PTHREADS) \
710 && (!defined(HAVE_TLS) \
711 || !defined(__GLIBC__) \
712 || !defined(USING_INITIAL_EXEC_TLS))
713 /* pthread_t of the thread containing this gomp_thread.
714 On Linux when using initial-exec TLS,
715 (typeof (pthread_t)) gomp_thread () - pthread_self ()
716 is constant in all threads, so we can optimize and not
718 #define GOMP_NEEDS_THREAD_HANDLE 1
724 struct gomp_thread_pool
726 /* This array manages threads spawned from the top level, which will
727 return to the idle loop once the current PARALLEL construct ends. */
728 struct gomp_thread
**threads
;
729 unsigned threads_size
;
730 unsigned threads_used
;
731 /* The last team is used for non-nested teams to delay their destruction to
732 make sure all the threads in the team move on to the pool's barrier before
733 the team's barrier is destroyed. */
734 struct gomp_team
*last_team
;
735 /* Number of threads running in this contention group. */
736 unsigned long threads_busy
;
738 /* This barrier holds and releases threads waiting in thread pools. */
739 gomp_simple_barrier_t threads_dock
;
742 enum gomp_cancel_kind
744 GOMP_CANCEL_PARALLEL
= 1,
745 GOMP_CANCEL_LOOP
= 2,
746 GOMP_CANCEL_FOR
= GOMP_CANCEL_LOOP
,
747 GOMP_CANCEL_DO
= GOMP_CANCEL_LOOP
,
748 GOMP_CANCEL_SECTIONS
= 4,
749 GOMP_CANCEL_TASKGROUP
= 8
752 /* ... and here is that TLS data. */
754 #if defined __nvptx__
755 extern struct gomp_thread
*nvptx_thrs
__attribute__((shared
));
756 static inline struct gomp_thread
*gomp_thread (void)
759 asm ("mov.u32 %0, %%tid.y;" : "=r" (tid
));
760 return nvptx_thrs
+ tid
;
762 #elif defined __AMDGCN__
763 static inline struct gomp_thread
*gcn_thrs (void)
765 /* The value is at the bottom of LDS. */
766 struct gomp_thread
* __lds
*thrs
= (struct gomp_thread
* __lds
*)4;
769 static inline void set_gcn_thrs (struct gomp_thread
*val
)
771 /* The value is at the bottom of LDS. */
772 struct gomp_thread
* __lds
*thrs
= (struct gomp_thread
* __lds
*)4;
775 static inline struct gomp_thread
*gomp_thread (void)
777 int tid
= __builtin_gcn_dim_pos(1);
778 return gcn_thrs () + tid
;
780 #elif defined HAVE_TLS || defined USE_EMUTLS
781 extern __thread
struct gomp_thread gomp_tls_data
;
782 static inline struct gomp_thread
*gomp_thread (void)
784 return &gomp_tls_data
;
787 extern pthread_key_t gomp_tls_key
;
788 static inline struct gomp_thread
*gomp_thread (void)
790 return pthread_getspecific (gomp_tls_key
);
794 extern struct gomp_task_icv
*gomp_new_icv (void);
796 /* Here's how to access the current copy of the ICVs. */
798 static inline struct gomp_task_icv
*gomp_icv (bool write
)
800 struct gomp_task
*task
= gomp_thread ()->task
;
804 return gomp_new_icv ();
806 return &gomp_global_icv
;
809 #ifdef LIBGOMP_USE_PTHREADS
810 /* The attributes to be used during thread creation. */
811 extern pthread_attr_t gomp_thread_attr
;
813 extern pthread_key_t gomp_thread_destructor
;
816 /* Function prototypes. */
820 extern void gomp_init_affinity (void);
821 #ifdef LIBGOMP_USE_PTHREADS
822 extern void gomp_init_thread_affinity (pthread_attr_t
*, unsigned int);
824 extern void **gomp_affinity_alloc (unsigned long, bool);
825 extern void gomp_affinity_init_place (void *);
826 extern bool gomp_affinity_add_cpus (void *, unsigned long, unsigned long,
828 extern bool gomp_affinity_remove_cpu (void *, unsigned long);
829 extern bool gomp_affinity_copy_place (void *, void *, long);
830 extern bool gomp_affinity_same_place (void *, void *);
831 extern bool gomp_affinity_finalize_place_list (bool);
832 extern bool gomp_affinity_init_level (int, unsigned long, bool);
833 extern void gomp_affinity_print_place (void *);
834 extern void gomp_get_place_proc_ids_8 (int, int64_t *);
835 extern void gomp_display_affinity_place (char *, size_t, size_t *, int);
839 extern bool gomp_print_string (const char *str
, size_t len
);
840 extern void gomp_set_affinity_format (const char *, size_t);
841 extern void gomp_display_string (char *, size_t, size_t *, const char *,
843 #ifdef LIBGOMP_USE_PTHREADS
844 typedef pthread_t gomp_thread_handle
;
846 typedef struct {} gomp_thread_handle
;
848 extern size_t gomp_display_affinity (char *, size_t, const char *,
850 struct gomp_team_state
*, unsigned int);
851 extern void gomp_display_affinity_thread (gomp_thread_handle
,
852 struct gomp_team_state
*,
853 unsigned int) __attribute__((cold
));
857 extern int gomp_iter_static_next (long *, long *);
858 extern bool gomp_iter_dynamic_next_locked (long *, long *);
859 extern bool gomp_iter_guided_next_locked (long *, long *);
861 #ifdef HAVE_SYNC_BUILTINS
862 extern bool gomp_iter_dynamic_next (long *, long *);
863 extern bool gomp_iter_guided_next (long *, long *);
868 extern int gomp_iter_ull_static_next (unsigned long long *,
869 unsigned long long *);
870 extern bool gomp_iter_ull_dynamic_next_locked (unsigned long long *,
871 unsigned long long *);
872 extern bool gomp_iter_ull_guided_next_locked (unsigned long long *,
873 unsigned long long *);
875 #if defined HAVE_SYNC_BUILTINS && defined __LP64__
876 extern bool gomp_iter_ull_dynamic_next (unsigned long long *,
877 unsigned long long *);
878 extern bool gomp_iter_ull_guided_next (unsigned long long *,
879 unsigned long long *);
884 extern void gomp_ordered_first (void);
885 extern void gomp_ordered_last (void);
886 extern void gomp_ordered_next (void);
887 extern void gomp_ordered_static_init (void);
888 extern void gomp_ordered_static_next (void);
889 extern void gomp_ordered_sync (void);
890 extern void gomp_doacross_init (unsigned, long *, long, size_t);
891 extern void gomp_doacross_ull_init (unsigned, unsigned long long *,
892 unsigned long long, size_t);
896 extern unsigned gomp_resolve_num_threads (unsigned, unsigned);
898 /* proc.c (in config/) */
900 extern void gomp_init_num_threads (void);
901 extern unsigned gomp_dynamic_max_threads (void);
905 extern void gomp_init_task (struct gomp_task
*, struct gomp_task
*,
906 struct gomp_task_icv
*);
907 extern void gomp_end_task (void);
908 extern void gomp_barrier_handle_tasks (gomp_barrier_state_t
);
909 extern void gomp_task_maybe_wait_for_dependencies (void **);
910 extern bool gomp_create_target_task (struct gomp_device_descr
*,
911 void (*) (void *), size_t, void **,
912 size_t *, unsigned short *, unsigned int,
914 enum gomp_target_task_state
);
915 extern struct gomp_taskgroup
*gomp_parallel_reduction_register (uintptr_t *,
917 extern void gomp_workshare_taskgroup_start (void);
918 extern void gomp_workshare_task_reduction_register (uintptr_t *, uintptr_t *);
921 gomp_finish_task (struct gomp_task
*task
)
923 if (__builtin_expect (task
->depend_hash
!= NULL
, 0))
924 free (task
->depend_hash
);
929 extern struct gomp_team
*gomp_new_team (unsigned);
930 extern void gomp_team_start (void (*) (void *), void *, unsigned,
931 unsigned, struct gomp_team
*,
932 struct gomp_taskgroup
*);
933 extern void gomp_team_end (void);
934 extern void gomp_free_thread (void *);
935 extern int gomp_pause_host (void);
939 extern void gomp_init_targets_once (void);
940 extern int gomp_get_num_devices (void);
941 extern bool gomp_target_task_fn (void *);
943 /* Splay tree definitions. */
944 typedef struct splay_tree_node_s
*splay_tree_node
;
945 typedef struct splay_tree_s
*splay_tree
;
946 typedef struct splay_tree_key_s
*splay_tree_key
;
948 struct target_var_desc
{
951 /* True if data should be copied from device to host at the end. */
953 /* True if data always should be copied from device to host at the end. */
954 bool always_copy_from
;
955 /* True if this is for OpenACC 'attach'. */
957 /* Relative offset against key host_start. */
963 struct target_mem_desc
{
964 /* Reference count. */
966 /* All the splay nodes allocated together. */
967 splay_tree_node array
;
968 /* Start of the target region. */
970 /* End of the targer region. */
972 /* Handle to free. */
974 /* Previous target_mem_desc. */
975 struct target_mem_desc
*prev
;
976 /* Number of items in following list. */
979 /* Corresponding target device descriptor. */
980 struct gomp_device_descr
*device_descr
;
982 /* List of target items to remove (or decrease refcount)
983 at the end of region. */
984 struct target_var_desc list
[];
987 /* Special value for refcount - infinity. */
988 #define REFCOUNT_INFINITY (~(uintptr_t) 0)
989 /* Special value for refcount - tgt_offset contains target address of the
990 artificial pointer to "omp declare target link" object. */
991 #define REFCOUNT_LINK (~(uintptr_t) 1)
993 /* Special offset values. */
994 #define OFFSET_INLINED (~(uintptr_t) 0)
995 #define OFFSET_POINTER (~(uintptr_t) 1)
996 #define OFFSET_STRUCT (~(uintptr_t) 2)
998 /* Auxiliary structure for infrequently-used or API-specific data. */
1000 struct splay_tree_aux
{
1001 /* Pointer to the original mapping of "omp declare target link" object. */
1002 splay_tree_key link_key
;
1003 /* For a block with attached pointers, the attachment counters for each.
1004 Only used for OpenACC. */
1005 uintptr_t *attach_count
;
1008 struct splay_tree_key_s
{
1009 /* Address of the host object. */
1010 uintptr_t host_start
;
1011 /* Address immediately after the host object. */
1013 /* Descriptor of the target memory. */
1014 struct target_mem_desc
*tgt
;
1015 /* Offset from tgt->tgt_start to the start of the target object. */
1016 uintptr_t tgt_offset
;
1017 /* Reference count. */
1019 /* Dynamic reference count. */
1020 uintptr_t dynamic_refcount
;
1021 struct splay_tree_aux
*aux
;
1024 /* The comparison function. */
1027 splay_compare (splay_tree_key x
, splay_tree_key y
)
1029 if (x
->host_start
== x
->host_end
1030 && y
->host_start
== y
->host_end
)
1032 if (x
->host_end
<= y
->host_start
)
1034 if (x
->host_start
>= y
->host_end
)
1039 #include "splay-tree.h"
1041 typedef struct acc_dispatch_t
1044 __typeof (GOMP_OFFLOAD_openacc_exec
) *exec_func
;
1046 /* Create/destroy TLS data. */
1047 __typeof (GOMP_OFFLOAD_openacc_create_thread_data
) *create_thread_data_func
;
1048 __typeof (GOMP_OFFLOAD_openacc_destroy_thread_data
)
1049 *destroy_thread_data_func
;
1052 /* Once created and put into the "active" list, asyncqueues are then never
1053 destructed and removed from the "active" list, other than if the TODO
1054 device is shut down. */
1057 struct goacc_asyncqueue
**asyncqueue
;
1058 struct goacc_asyncqueue_list
*active
;
1060 __typeof (GOMP_OFFLOAD_openacc_async_construct
) *construct_func
;
1061 __typeof (GOMP_OFFLOAD_openacc_async_destruct
) *destruct_func
;
1062 __typeof (GOMP_OFFLOAD_openacc_async_test
) *test_func
;
1063 __typeof (GOMP_OFFLOAD_openacc_async_synchronize
) *synchronize_func
;
1064 __typeof (GOMP_OFFLOAD_openacc_async_serialize
) *serialize_func
;
1065 __typeof (GOMP_OFFLOAD_openacc_async_queue_callback
) *queue_callback_func
;
1067 __typeof (GOMP_OFFLOAD_openacc_async_exec
) *exec_func
;
1068 __typeof (GOMP_OFFLOAD_openacc_async_dev2host
) *dev2host_func
;
1069 __typeof (GOMP_OFFLOAD_openacc_async_host2dev
) *host2dev_func
;
1072 __typeof (GOMP_OFFLOAD_openacc_get_property
) *get_property_func
;
1074 /* NVIDIA target specific routines. */
1076 __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_device
)
1077 *get_current_device_func
;
1078 __typeof (GOMP_OFFLOAD_openacc_cuda_get_current_context
)
1079 *get_current_context_func
;
1080 __typeof (GOMP_OFFLOAD_openacc_cuda_get_stream
) *get_stream_func
;
1081 __typeof (GOMP_OFFLOAD_openacc_cuda_set_stream
) *set_stream_func
;
1085 /* Various state of the accelerator device. */
1086 enum gomp_device_state
1088 GOMP_DEVICE_UNINITIALIZED
,
1089 GOMP_DEVICE_INITIALIZED
,
1090 GOMP_DEVICE_FINALIZED
1093 /* This structure describes accelerator device.
1094 It contains name of the corresponding libgomp plugin, function handlers for
1095 interaction with the device, ID-number of the device, and information about
1097 struct gomp_device_descr
1099 /* Immutable data, which is only set during initialization, and which is not
1100 guarded by the lock. */
1102 /* The name of the device. */
1105 /* Capabilities of device (supports OpenACC, OpenMP). */
1106 unsigned int capabilities
;
1108 /* This is the ID number of device among devices of the same type. */
1111 /* This is the TYPE of device. */
1112 enum offload_target_type type
;
1114 /* Function handlers. */
1115 __typeof (GOMP_OFFLOAD_get_name
) *get_name_func
;
1116 __typeof (GOMP_OFFLOAD_get_caps
) *get_caps_func
;
1117 __typeof (GOMP_OFFLOAD_get_type
) *get_type_func
;
1118 __typeof (GOMP_OFFLOAD_get_num_devices
) *get_num_devices_func
;
1119 __typeof (GOMP_OFFLOAD_init_device
) *init_device_func
;
1120 __typeof (GOMP_OFFLOAD_fini_device
) *fini_device_func
;
1121 __typeof (GOMP_OFFLOAD_version
) *version_func
;
1122 __typeof (GOMP_OFFLOAD_load_image
) *load_image_func
;
1123 __typeof (GOMP_OFFLOAD_unload_image
) *unload_image_func
;
1124 __typeof (GOMP_OFFLOAD_alloc
) *alloc_func
;
1125 __typeof (GOMP_OFFLOAD_free
) *free_func
;
1126 __typeof (GOMP_OFFLOAD_dev2host
) *dev2host_func
;
1127 __typeof (GOMP_OFFLOAD_host2dev
) *host2dev_func
;
1128 __typeof (GOMP_OFFLOAD_dev2dev
) *dev2dev_func
;
1129 __typeof (GOMP_OFFLOAD_can_run
) *can_run_func
;
1130 __typeof (GOMP_OFFLOAD_run
) *run_func
;
1131 __typeof (GOMP_OFFLOAD_async_run
) *async_run_func
;
1133 /* Splay tree containing information about mapped memory regions. */
1134 struct splay_tree_s mem_map
;
1136 /* Mutex for the mutable data. */
1139 /* Current state of the device. OpenACC allows to move from INITIALIZED state
1140 back to UNINITIALIZED state. OpenMP allows only to move from INITIALIZED
1141 to FINALIZED state (at program shutdown). */
1142 enum gomp_device_state state
;
1144 /* OpenACC-specific data and functions. */
1145 /* This is mutable because of its mutable target_data member. */
1146 acc_dispatch_t openacc
;
1149 /* Kind of the pragma, for which gomp_map_vars () is called. */
1150 enum gomp_map_vars_kind
1152 GOMP_MAP_VARS_OPENACC
,
1153 GOMP_MAP_VARS_TARGET
,
1155 GOMP_MAP_VARS_ENTER_DATA
1158 extern void gomp_acc_declare_allocate (bool, size_t, void **, size_t *,
1160 struct gomp_coalesce_buf
;
1161 extern void gomp_copy_host2dev (struct gomp_device_descr
*,
1162 struct goacc_asyncqueue
*, void *, const void *,
1163 size_t, struct gomp_coalesce_buf
*);
1164 extern void gomp_copy_dev2host (struct gomp_device_descr
*,
1165 struct goacc_asyncqueue
*, void *, const void *,
1167 extern uintptr_t gomp_map_val (struct target_mem_desc
*, void **, size_t);
1168 extern void gomp_attach_pointer (struct gomp_device_descr
*,
1169 struct goacc_asyncqueue
*, splay_tree
,
1170 splay_tree_key
, uintptr_t, size_t,
1171 struct gomp_coalesce_buf
*);
1172 extern void gomp_detach_pointer (struct gomp_device_descr
*,
1173 struct goacc_asyncqueue
*, splay_tree_key
,
1174 uintptr_t, bool, struct gomp_coalesce_buf
*);
1176 extern struct target_mem_desc
*gomp_map_vars (struct gomp_device_descr
*,
1177 size_t, void **, void **,
1178 size_t *, void *, bool,
1179 enum gomp_map_vars_kind
);
1180 extern struct target_mem_desc
*gomp_map_vars_async (struct gomp_device_descr
*,
1181 struct goacc_asyncqueue
*,
1182 size_t, void **, void **,
1183 size_t *, void *, bool,
1184 enum gomp_map_vars_kind
);
1185 extern void gomp_unmap_vars (struct target_mem_desc
*, bool);
1186 extern void gomp_unmap_vars_async (struct target_mem_desc
*, bool,
1187 struct goacc_asyncqueue
*);
1188 extern void gomp_init_device (struct gomp_device_descr
*);
1189 extern bool gomp_fini_device (struct gomp_device_descr
*);
1190 extern void gomp_unload_device (struct gomp_device_descr
*);
1191 extern bool gomp_remove_var (struct gomp_device_descr
*, splay_tree_key
);
1192 extern void gomp_remove_var_async (struct gomp_device_descr
*, splay_tree_key
,
1193 struct goacc_asyncqueue
*);
1197 extern void gomp_init_work_share (struct gomp_work_share
*, size_t, unsigned);
1198 extern void gomp_fini_work_share (struct gomp_work_share
*);
1199 extern bool gomp_work_share_start (size_t);
1200 extern void gomp_work_share_end (void);
1201 extern bool gomp_work_share_end_cancel (void);
1202 extern void gomp_work_share_end_nowait (void);
1205 gomp_work_share_init_done (void)
1207 struct gomp_thread
*thr
= gomp_thread ();
1208 if (__builtin_expect (thr
->ts
.last_work_share
!= NULL
, 1))
1209 gomp_ptrlock_set (&thr
->ts
.last_work_share
->next_ws
, thr
->ts
.work_share
);
1212 #ifdef HAVE_ATTRIBUTE_VISIBILITY
1213 # pragma GCC visibility pop
1216 /* Now that we're back to default visibility, include the globals. */
1217 #include "libgomp_g.h"
1219 /* Include omp.h by parts. */
1220 #include "omp-lock.h"
1221 #define _LIBGOMP_OMP_LOCK_DEFINED 1
1224 #if !defined (HAVE_ATTRIBUTE_VISIBILITY) \
1225 || !defined (HAVE_ATTRIBUTE_ALIAS) \
1226 || !defined (HAVE_AS_SYMVER_DIRECTIVE) \
1228 || !defined (HAVE_SYMVER_SYMBOL_RENAMING_RUNTIME_SUPPORT)
1229 # undef LIBGOMP_GNU_SYMBOL_VERSIONING
1232 #ifdef LIBGOMP_GNU_SYMBOL_VERSIONING
1233 extern void gomp_init_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1234 extern void gomp_destroy_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1235 extern void gomp_set_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1236 extern void gomp_unset_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1237 extern int gomp_test_lock_30 (omp_lock_t
*) __GOMP_NOTHROW
;
1238 extern void gomp_init_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1239 extern void gomp_destroy_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1240 extern void gomp_set_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1241 extern void gomp_unset_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1242 extern int gomp_test_nest_lock_30 (omp_nest_lock_t
*) __GOMP_NOTHROW
;
1244 extern void gomp_init_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1245 extern void gomp_destroy_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1246 extern void gomp_set_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1247 extern void gomp_unset_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1248 extern int gomp_test_lock_25 (omp_lock_25_t
*) __GOMP_NOTHROW
;
1249 extern void gomp_init_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1250 extern void gomp_destroy_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1251 extern void gomp_set_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1252 extern void gomp_unset_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1253 extern int gomp_test_nest_lock_25 (omp_nest_lock_25_t
*) __GOMP_NOTHROW
;
1255 # define omp_lock_symver(fn) \
1256 __asm (".symver g" #fn "_30, " #fn "@@OMP_3.0"); \
1257 __asm (".symver g" #fn "_25, " #fn "@OMP_1.0");
1259 # define gomp_init_lock_30 omp_init_lock
1260 # define gomp_destroy_lock_30 omp_destroy_lock
1261 # define gomp_set_lock_30 omp_set_lock
1262 # define gomp_unset_lock_30 omp_unset_lock
1263 # define gomp_test_lock_30 omp_test_lock
1264 # define gomp_init_nest_lock_30 omp_init_nest_lock
1265 # define gomp_destroy_nest_lock_30 omp_destroy_nest_lock
1266 # define gomp_set_nest_lock_30 omp_set_nest_lock
1267 # define gomp_unset_nest_lock_30 omp_unset_nest_lock
1268 # define gomp_test_nest_lock_30 omp_test_nest_lock
1271 #ifdef HAVE_ATTRIBUTE_VISIBILITY
1272 # define attribute_hidden __attribute__ ((visibility ("hidden")))
1274 # define attribute_hidden
1278 # define HAVE_ATTRIBUTE_COPY
1281 #ifdef HAVE_ATTRIBUTE_COPY
1282 # define attribute_copy(arg) __attribute__ ((copy (arg)))
1284 # define attribute_copy(arg)
1287 #ifdef HAVE_ATTRIBUTE_ALIAS
1288 # define strong_alias(fn, al) \
1289 extern __typeof (fn) al __attribute__ ((alias (#fn))) attribute_copy (fn);
1291 # define ialias_ulp ialias_str1(__USER_LABEL_PREFIX__)
1292 # define ialias_str1(x) ialias_str2(x)
1293 # define ialias_str2(x) #x
1294 # define ialias(fn) \
1295 extern __typeof (fn) gomp_ialias_##fn \
1296 __attribute__ ((alias (#fn))) attribute_hidden attribute_copy (fn);
1297 # define ialias_redirect(fn) \
1298 extern __typeof (fn) fn __asm__ (ialias_ulp "gomp_ialias_" #fn) attribute_hidden;
1299 # define ialias_call(fn) gomp_ialias_ ## fn
1302 # define ialias_redirect(fn)
1303 # define ialias_call(fn) fn
1306 /* Helper function for priority_node_to_task() and
1307 task_to_priority_node().
1309 Return the offset from a task to its priority_node entry. The
1310 priority_node entry is has a type of TYPE. */
1312 static inline size_t
1313 priority_queue_offset (enum priority_queue_type type
)
1315 return offsetof (struct gomp_task
, pnode
[(int) type
]);
1318 /* Return the task associated with a priority NODE of type TYPE. */
1320 static inline struct gomp_task
*
1321 priority_node_to_task (enum priority_queue_type type
,
1322 struct priority_node
*node
)
1324 return (struct gomp_task
*) ((char *) node
- priority_queue_offset (type
));
1327 /* Return the priority node of type TYPE for a given TASK. */
1329 static inline struct priority_node
*
1330 task_to_priority_node (enum priority_queue_type type
,
1331 struct gomp_task
*task
)
1333 return (struct priority_node
*) ((char *) task
1334 + priority_queue_offset (type
));
1337 #ifdef LIBGOMP_USE_PTHREADS
1338 static inline gomp_thread_handle
1339 gomp_thread_self (void)
1341 return pthread_self ();
1344 static inline gomp_thread_handle
1345 gomp_thread_to_pthread_t (struct gomp_thread
*thr
)
1347 struct gomp_thread
*this_thr
= gomp_thread ();
1348 if (thr
== this_thr
)
1349 return pthread_self ();
1350 #ifdef GOMP_NEEDS_THREAD_HANDLE
1353 /* On Linux with initial-exec TLS, the pthread_t of the thread containing
1354 thr can be computed from thr, this_thr and pthread_self (),
1355 as the distance between this_thr and pthread_self () is constant. */
1356 return pthread_self () + ((uintptr_t) thr
- (uintptr_t) this_thr
);
1360 static inline gomp_thread_handle
1361 gomp_thread_self (void)
1363 return (gomp_thread_handle
) {};
1366 static inline gomp_thread_handle
1367 gomp_thread_to_pthread_t (struct gomp_thread
*thr
)
1370 return gomp_thread_self ();
1374 #endif /* LIBGOMP_H */