1 /* OpenACC Runtime initialization routines
3 Copyright (C) 2013-2019 Free Software Foundation, Inc.
5 Contributed by Mentor Embedded.
7 This file is part of the GNU Offloading and Multi Processing Library
10 Libgomp is free software; you can redistribute it and/or modify it
11 under the terms of the GNU General Public License as published by
12 the Free Software Foundation; either version 3, or (at your option)
15 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
20 Under Section 7 of GPL version 3, you are granted additional
21 permissions described in the GCC Runtime Library Exception, version
22 3.1, as published by the Free Software Foundation.
24 You should have received a copy of the GNU General Public License and
25 a copy of the GCC Runtime Library Exception along with this program;
26 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
27 <http://www.gnu.org/licenses/>. */
31 #include "gomp-constants.h"
36 /* Return block containing [H->S), or NULL if not contained. The device lock
37 for DEV must be locked on entry, and remains locked on exit. */
40 lookup_host (struct gomp_device_descr
*dev
, void *h
, size_t s
)
42 struct splay_tree_key_s node
;
45 node
.host_start
= (uintptr_t) h
;
46 node
.host_end
= (uintptr_t) h
+ s
;
48 key
= splay_tree_lookup (&dev
->mem_map
, &node
);
53 /* Helper for lookup_dev. Iterate over splay tree. */
56 lookup_dev_1 (splay_tree_node node
, uintptr_t d
, size_t s
)
58 splay_tree_key key
= &node
->key
;
59 if (d
>= key
->tgt
->tgt_start
&& d
+ s
<= key
->tgt
->tgt_end
)
64 key
= lookup_dev_1 (node
->left
, d
, s
);
65 if (!key
&& node
->right
)
66 key
= lookup_dev_1 (node
->right
, d
, s
);
71 /* Return block containing [D->S), or NULL if not contained.
73 This iterates over the splay tree. This is not expected to be a common
76 The device lock associated with MEM_MAP must be locked on entry, and remains
80 lookup_dev (splay_tree mem_map
, void *d
, size_t s
)
82 if (!mem_map
|| !mem_map
->root
)
85 return lookup_dev_1 (mem_map
->root
, (uintptr_t) d
, s
);
89 /* OpenACC is silent on how memory exhaustion is indicated. We return
98 goacc_lazy_initialize ();
100 struct goacc_thread
*thr
= goacc_thread ();
104 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
107 acc_prof_info prof_info
;
108 acc_api_info api_info
;
109 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
111 void *res
= thr
->dev
->alloc_func (thr
->dev
->target_id
, s
);
115 thr
->prof_info
= NULL
;
116 thr
->api_info
= NULL
;
130 struct goacc_thread
*thr
= goacc_thread ();
132 assert (thr
&& thr
->dev
);
134 struct gomp_device_descr
*acc_dev
= thr
->dev
;
136 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
139 acc_prof_info prof_info
;
140 acc_api_info api_info
;
141 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
143 gomp_mutex_lock (&acc_dev
->lock
);
145 /* We don't have to call lazy open here, as the ptr value must have
146 been returned by acc_malloc. It's not permitted to pass NULL in
147 (unless you got that null from acc_malloc). */
148 if ((k
= lookup_dev (&acc_dev
->mem_map
, d
, 1)))
150 void *offset
= d
- k
->tgt
->tgt_start
+ k
->tgt_offset
;
151 void *h
= k
->host_start
+ offset
;
152 size_t h_size
= k
->host_end
- k
->host_start
;
153 gomp_mutex_unlock (&acc_dev
->lock
);
154 /* PR92503 "[OpenACC] Behavior of 'acc_free' if the memory space is still
155 used in a mapping". */
156 gomp_fatal ("refusing to free device memory space at %p that is still"
157 " mapped at [%p,+%d]",
161 gomp_mutex_unlock (&acc_dev
->lock
);
163 if (!acc_dev
->free_func (acc_dev
->target_id
, d
))
164 gomp_fatal ("error in freeing device memory in %s", __FUNCTION__
);
168 thr
->prof_info
= NULL
;
169 thr
->api_info
= NULL
;
174 memcpy_tofrom_device (bool from
, void *d
, void *h
, size_t s
, int async
,
175 const char *libfnname
)
177 /* No need to call lazy open here, as the device pointer must have
178 been obtained from a routine that did that. */
179 struct goacc_thread
*thr
= goacc_thread ();
181 assert (thr
&& thr
->dev
);
183 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
192 acc_prof_info prof_info
;
193 acc_api_info api_info
;
194 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
197 prof_info
.async
= async
;
198 prof_info
.async_queue
= prof_info
.async
;
201 goacc_aq aq
= get_goacc_asyncqueue (async
);
203 gomp_copy_dev2host (thr
->dev
, aq
, h
, d
, s
);
205 gomp_copy_host2dev (thr
->dev
, aq
, d
, h
, s
, /* TODO: cbuf? */ NULL
);
209 thr
->prof_info
= NULL
;
210 thr
->api_info
= NULL
;
215 acc_memcpy_to_device (void *d
, void *h
, size_t s
)
217 memcpy_tofrom_device (false, d
, h
, s
, acc_async_sync
, __FUNCTION__
);
221 acc_memcpy_to_device_async (void *d
, void *h
, size_t s
, int async
)
223 memcpy_tofrom_device (false, d
, h
, s
, async
, __FUNCTION__
);
227 acc_memcpy_from_device (void *h
, void *d
, size_t s
)
229 memcpy_tofrom_device (true, d
, h
, s
, acc_async_sync
, __FUNCTION__
);
233 acc_memcpy_from_device_async (void *h
, void *d
, size_t s
, int async
)
235 memcpy_tofrom_device (true, d
, h
, s
, async
, __FUNCTION__
);
238 /* Return the device pointer that corresponds to host data H. Or NULL
242 acc_deviceptr (void *h
)
248 goacc_lazy_initialize ();
250 struct goacc_thread
*thr
= goacc_thread ();
251 struct gomp_device_descr
*dev
= thr
->dev
;
253 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
256 /* In the following, no OpenACC Profiling Interface events can possibly be
259 gomp_mutex_lock (&dev
->lock
);
261 n
= lookup_host (dev
, h
, 1);
265 gomp_mutex_unlock (&dev
->lock
);
269 offset
= h
- n
->host_start
;
271 d
= n
->tgt
->tgt_start
+ n
->tgt_offset
+ offset
;
273 gomp_mutex_unlock (&dev
->lock
);
278 /* Return the host pointer that corresponds to device data D. Or NULL
282 acc_hostptr (void *d
)
288 goacc_lazy_initialize ();
290 struct goacc_thread
*thr
= goacc_thread ();
291 struct gomp_device_descr
*acc_dev
= thr
->dev
;
293 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
296 /* In the following, no OpenACC Profiling Interface events can possibly be
299 gomp_mutex_lock (&acc_dev
->lock
);
301 n
= lookup_dev (&acc_dev
->mem_map
, d
, 1);
305 gomp_mutex_unlock (&acc_dev
->lock
);
309 offset
= d
- n
->tgt
->tgt_start
+ n
->tgt_offset
;
311 h
= n
->host_start
+ offset
;
313 gomp_mutex_unlock (&acc_dev
->lock
);
318 /* Return 1 if host data [H,+S] is present on the device. */
321 acc_is_present (void *h
, size_t s
)
328 goacc_lazy_initialize ();
330 struct goacc_thread
*thr
= goacc_thread ();
331 struct gomp_device_descr
*acc_dev
= thr
->dev
;
333 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
336 /* In the following, no OpenACC Profiling Interface events can possibly be
339 gomp_mutex_lock (&acc_dev
->lock
);
341 n
= lookup_host (acc_dev
, h
, s
);
343 if (n
&& ((uintptr_t)h
< n
->host_start
344 || (uintptr_t)h
+ s
> n
->host_end
345 || s
> n
->host_end
- n
->host_start
))
348 gomp_mutex_unlock (&acc_dev
->lock
);
353 /* Create a mapping for host [H,+S] -> device [D,+S] */
356 acc_map_data (void *h
, void *d
, size_t s
)
358 struct target_mem_desc
*tgt
= NULL
;
363 unsigned short kinds
= GOMP_MAP_ALLOC
;
365 goacc_lazy_initialize ();
367 struct goacc_thread
*thr
= goacc_thread ();
368 struct gomp_device_descr
*acc_dev
= thr
->dev
;
370 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
373 gomp_fatal ("cannot map data on shared-memory system");
377 struct goacc_thread
*thr
= goacc_thread ();
380 gomp_fatal ("[%p,+%d]->[%p,+%d] is a bad map",
381 (void *)h
, (int)s
, (void *)d
, (int)s
);
383 acc_prof_info prof_info
;
384 acc_api_info api_info
;
385 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
387 gomp_mutex_lock (&acc_dev
->lock
);
389 if (lookup_host (acc_dev
, h
, s
))
391 gomp_mutex_unlock (&acc_dev
->lock
);
392 gomp_fatal ("host address [%p, +%d] is already mapped", (void *)h
,
396 if (lookup_dev (&thr
->dev
->mem_map
, d
, s
))
398 gomp_mutex_unlock (&acc_dev
->lock
);
399 gomp_fatal ("device address [%p, +%d] is already mapped", (void *)d
,
403 gomp_mutex_unlock (&acc_dev
->lock
);
405 tgt
= gomp_map_vars (acc_dev
, mapnum
, &hostaddrs
, &devaddrs
, &sizes
,
406 &kinds
, true, GOMP_MAP_VARS_ENTER_DATA
);
408 splay_tree_key n
= tgt
->list
[0].key
;
409 assert (n
->refcount
== 1);
410 assert (n
->dynamic_refcount
== 0);
411 /* Special reference counting behavior. */
412 n
->refcount
= REFCOUNT_INFINITY
;
416 thr
->prof_info
= NULL
;
417 thr
->api_info
= NULL
;
423 acc_unmap_data (void *h
)
425 struct goacc_thread
*thr
= goacc_thread ();
426 struct gomp_device_descr
*acc_dev
= thr
->dev
;
428 /* No need to call lazy open, as the address must have been mapped. */
430 /* This is a no-op on shared-memory targets. */
431 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
434 acc_prof_info prof_info
;
435 acc_api_info api_info
;
436 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
440 gomp_mutex_lock (&acc_dev
->lock
);
442 splay_tree_key n
= lookup_host (acc_dev
, h
, 1);
443 struct target_mem_desc
*t
;
447 gomp_mutex_unlock (&acc_dev
->lock
);
448 gomp_fatal ("%p is not a mapped block", (void *)h
);
451 host_size
= n
->host_end
- n
->host_start
;
453 if (n
->host_start
!= (uintptr_t) h
)
455 gomp_mutex_unlock (&acc_dev
->lock
);
456 gomp_fatal ("[%p,%d] surrounds %p",
457 (void *) n
->host_start
, (int) host_size
, (void *) h
);
459 /* TODO This currently doesn't catch 'REFCOUNT_INFINITY' usage different from
460 'acc_map_data'. Maybe 'dynamic_refcount' can be used for disambiguating
461 the different 'REFCOUNT_INFINITY' cases, or simply separate
462 'REFCOUNT_INFINITY' values per different usage ('REFCOUNT_ACC_MAP_DATA'
464 else if (n
->refcount
!= REFCOUNT_INFINITY
)
466 gomp_mutex_unlock (&acc_dev
->lock
);
467 gomp_fatal ("refusing to unmap block [%p,+%d] that has not been mapped"
468 " by 'acc_map_data'",
469 (void *) h
, (int) host_size
);
474 if (t
->refcount
== 1)
476 /* This is the last reference, so pull the descriptor off the
477 chain. This prevents 'gomp_unmap_tgt' via 'gomp_remove_var' from
478 freeing the device memory. */
483 bool is_tgt_unmapped
= gomp_remove_var (acc_dev
, n
);
484 assert (is_tgt_unmapped
);
486 gomp_mutex_unlock (&acc_dev
->lock
);
490 thr
->prof_info
= NULL
;
491 thr
->api_info
= NULL
;
496 /* Enter a dynamic mapping.
498 Return the device pointer. */
501 goacc_enter_data (void *h
, size_t s
, unsigned short kind
, int async
)
507 gomp_fatal ("[%p,+%d] is a bad range", (void *)h
, (int)s
);
509 goacc_lazy_initialize ();
511 struct goacc_thread
*thr
= goacc_thread ();
512 struct gomp_device_descr
*acc_dev
= thr
->dev
;
514 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
517 acc_prof_info prof_info
;
518 acc_api_info api_info
;
519 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
522 prof_info
.async
= async
;
523 prof_info
.async_queue
= prof_info
.async
;
526 gomp_mutex_lock (&acc_dev
->lock
);
528 n
= lookup_host (acc_dev
, h
, s
);
532 d
= (void *) (n
->tgt
->tgt_start
+ n
->tgt_offset
+ h
- n
->host_start
);
534 if ((h
+ s
) > (void *)n
->host_end
)
536 gomp_mutex_unlock (&acc_dev
->lock
);
537 gomp_fatal ("[%p,+%d] not mapped", (void *)h
, (int)s
);
540 assert (n
->refcount
!= REFCOUNT_LINK
);
541 if (n
->refcount
!= REFCOUNT_INFINITY
)
543 n
->dynamic_refcount
++;
545 gomp_mutex_unlock (&acc_dev
->lock
);
549 struct target_mem_desc
*tgt
;
553 gomp_mutex_unlock (&acc_dev
->lock
);
555 goacc_aq aq
= get_goacc_asyncqueue (async
);
557 tgt
= gomp_map_vars_async (acc_dev
, aq
, mapnum
, &hostaddrs
, NULL
, &s
,
558 &kind
, true, GOMP_MAP_VARS_ENTER_DATA
);
560 n
= tgt
->list
[0].key
;
561 assert (n
->refcount
== 1);
562 assert (n
->dynamic_refcount
== 0);
563 n
->dynamic_refcount
++;
570 thr
->prof_info
= NULL
;
571 thr
->api_info
= NULL
;
578 acc_create (void *h
, size_t s
)
580 return goacc_enter_data (h
, s
, GOMP_MAP_ALLOC
, acc_async_sync
);
584 acc_create_async (void *h
, size_t s
, int async
)
586 goacc_enter_data (h
, s
, GOMP_MAP_ALLOC
, async
);
589 /* acc_present_or_create used to be what acc_create is now. */
590 /* acc_pcreate is acc_present_or_create by a different name. */
591 #ifdef HAVE_ATTRIBUTE_ALIAS
592 strong_alias (acc_create
, acc_present_or_create
)
593 strong_alias (acc_create
, acc_pcreate
)
596 acc_present_or_create (void *h
, size_t s
)
598 return acc_create (h
, s
);
602 acc_pcreate (void *h
, size_t s
)
604 return acc_create (h
, s
);
609 acc_copyin (void *h
, size_t s
)
611 return goacc_enter_data (h
, s
, GOMP_MAP_TO
, acc_async_sync
);
615 acc_copyin_async (void *h
, size_t s
, int async
)
617 goacc_enter_data (h
, s
, GOMP_MAP_TO
, async
);
620 /* acc_present_or_copyin used to be what acc_copyin is now. */
621 /* acc_pcopyin is acc_present_or_copyin by a different name. */
622 #ifdef HAVE_ATTRIBUTE_ALIAS
623 strong_alias (acc_copyin
, acc_present_or_copyin
)
624 strong_alias (acc_copyin
, acc_pcopyin
)
627 acc_present_or_copyin (void *h
, size_t s
)
629 return acc_copyin (h
, s
);
633 acc_pcopyin (void *h
, size_t s
)
635 return acc_copyin (h
, s
);
639 #define FLAG_COPYOUT (1 << 0)
640 #define FLAG_FINALIZE (1 << 1)
643 delete_copyout (unsigned f
, void *h
, size_t s
, int async
, const char *libfnname
)
645 /* No need to call lazy open, as the data must already have been
648 struct goacc_thread
*thr
= goacc_thread ();
649 struct gomp_device_descr
*acc_dev
= thr
->dev
;
651 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
654 acc_prof_info prof_info
;
655 acc_api_info api_info
;
656 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
659 prof_info
.async
= async
;
660 prof_info
.async_queue
= prof_info
.async
;
663 gomp_mutex_lock (&acc_dev
->lock
);
665 splay_tree_key n
= lookup_host (acc_dev
, h
, s
);
667 /* PR92726, RP92970, PR92984: no-op. */
670 if ((uintptr_t) h
< n
->host_start
|| (uintptr_t) h
+ s
> n
->host_end
)
672 size_t host_size
= n
->host_end
- n
->host_start
;
673 gomp_mutex_unlock (&acc_dev
->lock
);
674 gomp_fatal ("[%p,+%d] outside mapped block [%p,+%d]",
675 (void *) h
, (int) s
, (void *) n
->host_start
, (int) host_size
);
678 assert (n
->refcount
!= REFCOUNT_LINK
);
679 if (n
->refcount
!= REFCOUNT_INFINITY
680 && n
->refcount
< n
->dynamic_refcount
)
682 gomp_mutex_unlock (&acc_dev
->lock
);
683 gomp_fatal ("Dynamic reference counting assert fail\n");
686 if (f
& FLAG_FINALIZE
)
688 if (n
->refcount
!= REFCOUNT_INFINITY
)
689 n
->refcount
-= n
->dynamic_refcount
;
690 n
->dynamic_refcount
= 0;
692 else if (n
->dynamic_refcount
)
694 if (n
->refcount
!= REFCOUNT_INFINITY
)
696 n
->dynamic_refcount
--;
699 if (n
->refcount
== 0)
701 goacc_aq aq
= get_goacc_asyncqueue (async
);
703 if (f
& FLAG_COPYOUT
)
705 void *d
= (void *) (n
->tgt
->tgt_start
+ n
->tgt_offset
706 + (uintptr_t) h
- n
->host_start
);
707 gomp_copy_dev2host (acc_dev
, aq
, h
, d
, s
);
711 /* TODO We can't do the 'is_tgt_unmapped' checking -- see the
712 'gomp_unref_tgt' comment in
713 <http://mid.mail-archive.com/878snl36eu.fsf@euler.schwinge.homeip.net>;
715 gomp_remove_var_async (acc_dev
, n
, aq
);
718 bool is_tgt_unmapped
= gomp_remove_var (acc_dev
, n
);
719 assert (is_tgt_unmapped
);
724 gomp_mutex_unlock (&acc_dev
->lock
);
728 thr
->prof_info
= NULL
;
729 thr
->api_info
= NULL
;
734 acc_delete (void *h
, size_t s
)
736 delete_copyout (0, h
, s
, acc_async_sync
, __FUNCTION__
);
740 acc_delete_async (void *h
, size_t s
, int async
)
742 delete_copyout (0, h
, s
, async
, __FUNCTION__
);
746 acc_delete_finalize (void *h
, size_t s
)
748 delete_copyout (FLAG_FINALIZE
, h
, s
, acc_async_sync
, __FUNCTION__
);
752 acc_delete_finalize_async (void *h
, size_t s
, int async
)
754 delete_copyout (FLAG_FINALIZE
, h
, s
, async
, __FUNCTION__
);
758 acc_copyout (void *h
, size_t s
)
760 delete_copyout (FLAG_COPYOUT
, h
, s
, acc_async_sync
, __FUNCTION__
);
764 acc_copyout_async (void *h
, size_t s
, int async
)
766 delete_copyout (FLAG_COPYOUT
, h
, s
, async
, __FUNCTION__
);
770 acc_copyout_finalize (void *h
, size_t s
)
772 delete_copyout (FLAG_COPYOUT
| FLAG_FINALIZE
, h
, s
, acc_async_sync
,
777 acc_copyout_finalize_async (void *h
, size_t s
, int async
)
779 delete_copyout (FLAG_COPYOUT
| FLAG_FINALIZE
, h
, s
, async
, __FUNCTION__
);
783 update_dev_host (int is_dev
, void *h
, size_t s
, int async
)
788 goacc_lazy_initialize ();
790 struct goacc_thread
*thr
= goacc_thread ();
791 struct gomp_device_descr
*acc_dev
= thr
->dev
;
793 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
796 /* Fortran optional arguments that are non-present result in a
797 NULL host address here. This can safely be ignored as it is
798 not possible to 'update' a non-present optional argument. */
802 acc_prof_info prof_info
;
803 acc_api_info api_info
;
804 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
807 prof_info
.async
= async
;
808 prof_info
.async_queue
= prof_info
.async
;
811 gomp_mutex_lock (&acc_dev
->lock
);
813 n
= lookup_host (acc_dev
, h
, s
);
817 gomp_mutex_unlock (&acc_dev
->lock
);
818 gomp_fatal ("[%p,%d] is not mapped", h
, (int)s
);
821 d
= (void *) (n
->tgt
->tgt_start
+ n
->tgt_offset
822 + (uintptr_t) h
- n
->host_start
);
824 goacc_aq aq
= get_goacc_asyncqueue (async
);
827 gomp_copy_host2dev (acc_dev
, aq
, d
, h
, s
, /* TODO: cbuf? */ NULL
);
829 gomp_copy_dev2host (acc_dev
, aq
, h
, d
, s
);
831 gomp_mutex_unlock (&acc_dev
->lock
);
835 thr
->prof_info
= NULL
;
836 thr
->api_info
= NULL
;
841 acc_update_device (void *h
, size_t s
)
843 update_dev_host (1, h
, s
, acc_async_sync
);
847 acc_update_device_async (void *h
, size_t s
, int async
)
849 update_dev_host (1, h
, s
, async
);
853 acc_update_self (void *h
, size_t s
)
855 update_dev_host (0, h
, s
, acc_async_sync
);
859 acc_update_self_async (void *h
, size_t s
, int async
)
861 update_dev_host (0, h
, s
, async
);
865 /* OpenACC 'enter data', 'exit data': 'GOACC_enter_exit_data' and its helper
868 /* Special handling for 'GOMP_MAP_POINTER', 'GOMP_MAP_TO_PSET'.
870 Only the first mapping is considered in reference counting; the following
871 ones implicitly follow suit. Similarly, 'copyout' ('force_copyfrom') is
872 done only for the first mapping. */
875 goacc_insert_pointer (size_t mapnum
, void **hostaddrs
, size_t *sizes
,
876 void *kinds
, int async
)
878 struct target_mem_desc
*tgt
;
879 struct goacc_thread
*thr
= goacc_thread ();
880 struct gomp_device_descr
*acc_dev
= thr
->dev
;
882 if (*hostaddrs
== NULL
)
885 if (acc_is_present (*hostaddrs
, *sizes
))
888 gomp_mutex_lock (&acc_dev
->lock
);
889 n
= lookup_host (acc_dev
, *hostaddrs
, *sizes
);
890 assert (n
->refcount
!= REFCOUNT_INFINITY
891 && n
->refcount
!= REFCOUNT_LINK
);
892 gomp_mutex_unlock (&acc_dev
->lock
);
895 for (size_t i
= 0; i
< tgt
->list_count
; i
++)
896 if (tgt
->list
[i
].key
== n
)
898 for (size_t j
= 0; j
< mapnum
; j
++)
899 if (i
+ j
< tgt
->list_count
&& tgt
->list
[i
+ j
].key
)
901 tgt
->list
[i
+ j
].key
->refcount
++;
902 tgt
->list
[i
+ j
].key
->dynamic_refcount
++;
906 /* Should not reach here. */
907 gomp_fatal ("Dynamic refcount incrementing failed for pointer/pset");
910 gomp_debug (0, " %s: prepare mappings\n", __FUNCTION__
);
911 goacc_aq aq
= get_goacc_asyncqueue (async
);
912 tgt
= gomp_map_vars_async (acc_dev
, aq
, mapnum
, hostaddrs
,
913 NULL
, sizes
, kinds
, true, GOMP_MAP_VARS_ENTER_DATA
);
915 splay_tree_key n
= tgt
->list
[0].key
;
916 assert (n
->refcount
== 1);
917 assert (n
->dynamic_refcount
== 0);
918 n
->dynamic_refcount
++;
919 gomp_debug (0, " %s: mappings prepared\n", __FUNCTION__
);
923 goacc_remove_pointer (void *h
, size_t s
, bool force_copyfrom
, int async
,
926 struct goacc_thread
*thr
= goacc_thread ();
927 struct gomp_device_descr
*acc_dev
= thr
->dev
;
929 struct target_mem_desc
*t
;
931 if (!acc_is_present (h
, s
))
934 gomp_mutex_lock (&acc_dev
->lock
);
936 n
= lookup_host (acc_dev
, h
, 1);
940 gomp_mutex_unlock (&acc_dev
->lock
);
941 gomp_fatal ("%p is not a mapped block", (void *)h
);
944 gomp_debug (0, " %s: restore mappings\n", __FUNCTION__
);
948 assert (n
->refcount
!= REFCOUNT_INFINITY
949 && n
->refcount
!= REFCOUNT_LINK
);
950 if (n
->refcount
< n
->dynamic_refcount
)
952 gomp_mutex_unlock (&acc_dev
->lock
);
953 gomp_fatal ("Dynamic reference counting assert fail\n");
958 n
->refcount
-= n
->dynamic_refcount
;
959 n
->dynamic_refcount
= 0;
961 else if (n
->dynamic_refcount
)
964 n
->dynamic_refcount
--;
967 if (n
->refcount
== 0)
969 goacc_aq aq
= get_goacc_asyncqueue (async
);
973 void *d
= (void *) (t
->tgt_start
+ n
->tgt_offset
974 + (uintptr_t) h
- n
->host_start
);
976 gomp_copy_dev2host (acc_dev
, aq
, h
, d
, s
);
981 /* TODO The way the following code is currently implemented, we need
982 the 'is_tgt_unmapped' return value from 'gomp_remove_var', so
983 can't use 'gomp_remove_var_async' here -- see the 'gomp_unref_tgt'
985 <http://mid.mail-archive.com/878snl36eu.fsf@euler.schwinge.homeip.net>;
986 PR92881 -- so have to synchronize here. */
987 if (!acc_dev
->openacc
.async
.synchronize_func (aq
))
989 gomp_mutex_unlock (&acc_dev
->lock
);
990 gomp_fatal ("synchronize failed");
993 bool is_tgt_unmapped
= false;
994 for (size_t i
= 0; i
< t
->list_count
; i
++)
996 is_tgt_unmapped
= gomp_remove_var (acc_dev
, t
->list
[i
].key
);
1000 assert (is_tgt_unmapped
);
1003 gomp_mutex_unlock (&acc_dev
->lock
);
1005 gomp_debug (0, " %s: mappings restored\n", __FUNCTION__
);
1008 /* Return the number of mappings associated with 'GOMP_MAP_TO_PSET' or
1009 'GOMP_MAP_POINTER'. */
1012 find_pointer (int pos
, size_t mapnum
, unsigned short *kinds
)
1014 if (pos
+ 1 >= mapnum
)
1017 unsigned char kind
= kinds
[pos
+1] & 0xff;
1019 if (kind
== GOMP_MAP_TO_PSET
)
1021 else if (kind
== GOMP_MAP_POINTER
)
1028 GOACC_enter_exit_data (int flags_m
, size_t mapnum
, void **hostaddrs
,
1029 size_t *sizes
, unsigned short *kinds
, int async
,
1032 int flags
= GOACC_FLAGS_UNMARSHAL (flags_m
);
1034 struct goacc_thread
*thr
;
1035 struct gomp_device_descr
*acc_dev
;
1036 bool data_enter
= false;
1039 goacc_lazy_initialize ();
1041 thr
= goacc_thread ();
1044 /* Determine if this is an "acc enter data". */
1045 for (i
= 0; i
< mapnum
; ++i
)
1047 unsigned char kind
= kinds
[i
] & 0xff;
1049 if (kind
== GOMP_MAP_POINTER
|| kind
== GOMP_MAP_TO_PSET
)
1052 if (kind
== GOMP_MAP_FORCE_ALLOC
1053 || kind
== GOMP_MAP_FORCE_PRESENT
1054 || kind
== GOMP_MAP_FORCE_TO
1055 || kind
== GOMP_MAP_TO
1056 || kind
== GOMP_MAP_ALLOC
)
1062 if (kind
== GOMP_MAP_RELEASE
1063 || kind
== GOMP_MAP_DELETE
1064 || kind
== GOMP_MAP_FROM
1065 || kind
== GOMP_MAP_FORCE_FROM
)
1068 gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
1072 bool profiling_p
= GOACC_PROFILING_DISPATCH_P (true);
1074 acc_prof_info prof_info
;
1077 thr
->prof_info
= &prof_info
;
1079 prof_info
.event_type
1080 = data_enter
? acc_ev_enter_data_start
: acc_ev_exit_data_start
;
1081 prof_info
.valid_bytes
= _ACC_PROF_INFO_VALID_BYTES
;
1082 prof_info
.version
= _ACC_PROF_INFO_VERSION
;
1083 prof_info
.device_type
= acc_device_type (acc_dev
->type
);
1084 prof_info
.device_number
= acc_dev
->target_id
;
1085 prof_info
.thread_id
= -1;
1086 prof_info
.async
= async
;
1087 prof_info
.async_queue
= prof_info
.async
;
1088 prof_info
.src_file
= NULL
;
1089 prof_info
.func_name
= NULL
;
1090 prof_info
.line_no
= -1;
1091 prof_info
.end_line_no
= -1;
1092 prof_info
.func_line_no
= -1;
1093 prof_info
.func_end_line_no
= -1;
1095 acc_event_info enter_exit_data_event_info
;
1098 enter_exit_data_event_info
.other_event
.event_type
1099 = prof_info
.event_type
;
1100 enter_exit_data_event_info
.other_event
.valid_bytes
1101 = _ACC_OTHER_EVENT_INFO_VALID_BYTES
;
1102 enter_exit_data_event_info
.other_event
.parent_construct
1103 = data_enter
? acc_construct_enter_data
: acc_construct_exit_data
;
1104 enter_exit_data_event_info
.other_event
.implicit
= 0;
1105 enter_exit_data_event_info
.other_event
.tool_info
= NULL
;
1107 acc_api_info api_info
;
1110 thr
->api_info
= &api_info
;
1112 api_info
.device_api
= acc_device_api_none
;
1113 api_info
.valid_bytes
= _ACC_API_INFO_VALID_BYTES
;
1114 api_info
.device_type
= prof_info
.device_type
;
1115 api_info
.vendor
= -1;
1116 api_info
.device_handle
= NULL
;
1117 api_info
.context_handle
= NULL
;
1118 api_info
.async_handle
= NULL
;
1122 goacc_profiling_dispatch (&prof_info
, &enter_exit_data_event_info
,
1125 if ((acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
1126 || (flags
& GOACC_FLAG_HOST_FALLBACK
))
1128 prof_info
.device_type
= acc_device_host
;
1129 api_info
.device_type
= prof_info
.device_type
;
1138 va_start (ap
, num_waits
);
1139 goacc_wait (async
, num_waits
, &ap
);
1143 /* In c, non-pointers and arrays are represented by a single data clause.
1144 Dynamically allocated arrays and subarrays are represented by a data
1145 clause followed by an internal GOMP_MAP_POINTER.
1147 In fortran, scalars and not allocated arrays are represented by a
1148 single data clause. Allocated arrays and subarrays have three mappings:
1149 1) the original data clause, 2) a PSET 3) a pointer to the array data.
1154 for (i
= 0; i
< mapnum
; i
++)
1156 unsigned char kind
= kinds
[i
] & 0xff;
1158 /* Scan for pointers and PSETs. */
1159 int pointer
= find_pointer (i
, mapnum
, kinds
);
1165 case GOMP_MAP_ALLOC
:
1166 case GOMP_MAP_FORCE_ALLOC
:
1167 acc_create_async (hostaddrs
[i
], sizes
[i
], async
);
1170 case GOMP_MAP_FORCE_TO
:
1171 acc_copyin_async (hostaddrs
[i
], sizes
[i
], async
);
1174 gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
1181 goacc_insert_pointer (pointer
, &hostaddrs
[i
], &sizes
[i
], &kinds
[i
],
1183 /* Increment 'i' by two because OpenACC requires fortran
1184 arrays to be contiguous, so each PSET is associated with
1185 one of MAP_FORCE_ALLOC/MAP_FORCE_PRESET/MAP_FORCE_TO, and
1192 for (i
= 0; i
< mapnum
; ++i
)
1194 unsigned char kind
= kinds
[i
] & 0xff;
1196 bool finalize
= (kind
== GOMP_MAP_DELETE
1197 || kind
== GOMP_MAP_FORCE_FROM
);
1199 int pointer
= find_pointer (i
, mapnum
, kinds
);
1205 case GOMP_MAP_RELEASE
:
1206 case GOMP_MAP_DELETE
:
1208 acc_delete_finalize_async (hostaddrs
[i
], sizes
[i
], async
);
1210 acc_delete_async (hostaddrs
[i
], sizes
[i
], async
);
1213 case GOMP_MAP_FORCE_FROM
:
1215 acc_copyout_finalize_async (hostaddrs
[i
], sizes
[i
], async
);
1217 acc_copyout_async (hostaddrs
[i
], sizes
[i
], async
);
1220 gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
1227 bool copyfrom
= (kind
== GOMP_MAP_FORCE_FROM
1228 || kind
== GOMP_MAP_FROM
);
1229 goacc_remove_pointer (hostaddrs
[i
], sizes
[i
], copyfrom
, async
,
1231 /* See the above comment. */
1239 prof_info
.event_type
1240 = data_enter
? acc_ev_enter_data_end
: acc_ev_exit_data_end
;
1241 enter_exit_data_event_info
.other_event
.event_type
= prof_info
.event_type
;
1242 goacc_profiling_dispatch (&prof_info
, &enter_exit_data_event_info
,
1245 thr
->prof_info
= NULL
;
1246 thr
->api_info
= NULL
;