1 /* OpenACC Runtime initialization routines
3 Copyright (C) 2013-2019 Free Software Foundation, Inc.
5 Contributed by Mentor Embedded.
7 This file is part of the GNU Offloading and Multi Processing Library
10 Libgomp is free software; you can redistribute it and/or modify it
11 under the terms of the GNU General Public License as published by
12 the Free Software Foundation; either version 3, or (at your option)
15 Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
16 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for
20 Under Section 7 of GPL version 3, you are granted additional
21 permissions described in the GCC Runtime Library Exception, version
22 3.1, as published by the Free Software Foundation.
24 You should have received a copy of the GNU General Public License and
25 a copy of the GCC Runtime Library Exception along with this program;
26 see the files COPYING3 and COPYING.RUNTIME respectively. If not, see
27 <http://www.gnu.org/licenses/>. */
31 #include "gomp-constants.h"
36 /* Return block containing [H->S), or NULL if not contained. The device lock
37 for DEV must be locked on entry, and remains locked on exit. */
40 lookup_host (struct gomp_device_descr
*dev
, void *h
, size_t s
)
42 struct splay_tree_key_s node
;
45 node
.host_start
= (uintptr_t) h
;
46 node
.host_end
= (uintptr_t) h
+ s
;
48 key
= splay_tree_lookup (&dev
->mem_map
, &node
);
53 /* Helper for lookup_dev. Iterate over splay tree. */
56 lookup_dev_1 (splay_tree_node node
, uintptr_t d
, size_t s
)
58 splay_tree_key key
= &node
->key
;
59 if (d
>= key
->tgt
->tgt_start
&& d
+ s
<= key
->tgt
->tgt_end
)
64 key
= lookup_dev_1 (node
->left
, d
, s
);
65 if (!key
&& node
->right
)
66 key
= lookup_dev_1 (node
->right
, d
, s
);
71 /* Return block containing [D->S), or NULL if not contained.
73 This iterates over the splay tree. This is not expected to be a common
76 The device lock associated with MEM_MAP must be locked on entry, and remains
80 lookup_dev (splay_tree mem_map
, void *d
, size_t s
)
82 if (!mem_map
|| !mem_map
->root
)
85 return lookup_dev_1 (mem_map
->root
, (uintptr_t) d
, s
);
89 /* OpenACC is silent on how memory exhaustion is indicated. We return
98 goacc_lazy_initialize ();
100 struct goacc_thread
*thr
= goacc_thread ();
104 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
107 acc_prof_info prof_info
;
108 acc_api_info api_info
;
109 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
111 void *res
= thr
->dev
->alloc_func (thr
->dev
->target_id
, s
);
115 thr
->prof_info
= NULL
;
116 thr
->api_info
= NULL
;
130 struct goacc_thread
*thr
= goacc_thread ();
132 assert (thr
&& thr
->dev
);
134 struct gomp_device_descr
*acc_dev
= thr
->dev
;
136 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
139 acc_prof_info prof_info
;
140 acc_api_info api_info
;
141 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
143 gomp_mutex_lock (&acc_dev
->lock
);
145 /* We don't have to call lazy open here, as the ptr value must have
146 been returned by acc_malloc. It's not permitted to pass NULL in
147 (unless you got that null from acc_malloc). */
148 if ((k
= lookup_dev (&acc_dev
->mem_map
, d
, 1)))
150 void *offset
= d
- k
->tgt
->tgt_start
+ k
->tgt_offset
;
151 void *h
= k
->host_start
+ offset
;
152 size_t h_size
= k
->host_end
- k
->host_start
;
153 gomp_mutex_unlock (&acc_dev
->lock
);
154 /* PR92503 "[OpenACC] Behavior of 'acc_free' if the memory space is still
155 used in a mapping". */
156 gomp_fatal ("refusing to free device memory space at %p that is still"
157 " mapped at [%p,+%d]",
161 gomp_mutex_unlock (&acc_dev
->lock
);
163 if (!acc_dev
->free_func (acc_dev
->target_id
, d
))
164 gomp_fatal ("error in freeing device memory in %s", __FUNCTION__
);
168 thr
->prof_info
= NULL
;
169 thr
->api_info
= NULL
;
174 memcpy_tofrom_device (bool from
, void *d
, void *h
, size_t s
, int async
,
175 const char *libfnname
)
177 /* No need to call lazy open here, as the device pointer must have
178 been obtained from a routine that did that. */
179 struct goacc_thread
*thr
= goacc_thread ();
181 assert (thr
&& thr
->dev
);
183 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
192 acc_prof_info prof_info
;
193 acc_api_info api_info
;
194 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
197 prof_info
.async
= async
;
198 prof_info
.async_queue
= prof_info
.async
;
201 goacc_aq aq
= get_goacc_asyncqueue (async
);
203 gomp_copy_dev2host (thr
->dev
, aq
, h
, d
, s
);
205 gomp_copy_host2dev (thr
->dev
, aq
, d
, h
, s
, /* TODO: cbuf? */ NULL
);
209 thr
->prof_info
= NULL
;
210 thr
->api_info
= NULL
;
215 acc_memcpy_to_device (void *d
, void *h
, size_t s
)
217 memcpy_tofrom_device (false, d
, h
, s
, acc_async_sync
, __FUNCTION__
);
221 acc_memcpy_to_device_async (void *d
, void *h
, size_t s
, int async
)
223 memcpy_tofrom_device (false, d
, h
, s
, async
, __FUNCTION__
);
227 acc_memcpy_from_device (void *h
, void *d
, size_t s
)
229 memcpy_tofrom_device (true, d
, h
, s
, acc_async_sync
, __FUNCTION__
);
233 acc_memcpy_from_device_async (void *h
, void *d
, size_t s
, int async
)
235 memcpy_tofrom_device (true, d
, h
, s
, async
, __FUNCTION__
);
238 /* Return the device pointer that corresponds to host data H. Or NULL
242 acc_deviceptr (void *h
)
248 goacc_lazy_initialize ();
250 struct goacc_thread
*thr
= goacc_thread ();
251 struct gomp_device_descr
*dev
= thr
->dev
;
253 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
256 /* In the following, no OpenACC Profiling Interface events can possibly be
259 gomp_mutex_lock (&dev
->lock
);
261 n
= lookup_host (dev
, h
, 1);
265 gomp_mutex_unlock (&dev
->lock
);
269 offset
= h
- n
->host_start
;
271 d
= n
->tgt
->tgt_start
+ n
->tgt_offset
+ offset
;
273 gomp_mutex_unlock (&dev
->lock
);
278 /* Return the host pointer that corresponds to device data D. Or NULL
282 acc_hostptr (void *d
)
288 goacc_lazy_initialize ();
290 struct goacc_thread
*thr
= goacc_thread ();
291 struct gomp_device_descr
*acc_dev
= thr
->dev
;
293 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
296 /* In the following, no OpenACC Profiling Interface events can possibly be
299 gomp_mutex_lock (&acc_dev
->lock
);
301 n
= lookup_dev (&acc_dev
->mem_map
, d
, 1);
305 gomp_mutex_unlock (&acc_dev
->lock
);
309 offset
= d
- n
->tgt
->tgt_start
+ n
->tgt_offset
;
311 h
= n
->host_start
+ offset
;
313 gomp_mutex_unlock (&acc_dev
->lock
);
318 /* Return 1 if host data [H,+S] is present on the device. */
321 acc_is_present (void *h
, size_t s
)
328 goacc_lazy_initialize ();
330 struct goacc_thread
*thr
= goacc_thread ();
331 struct gomp_device_descr
*acc_dev
= thr
->dev
;
333 if (thr
->dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
336 /* In the following, no OpenACC Profiling Interface events can possibly be
339 gomp_mutex_lock (&acc_dev
->lock
);
341 n
= lookup_host (acc_dev
, h
, s
);
343 if (n
&& ((uintptr_t)h
< n
->host_start
344 || (uintptr_t)h
+ s
> n
->host_end
345 || s
> n
->host_end
- n
->host_start
))
348 gomp_mutex_unlock (&acc_dev
->lock
);
353 /* Create a mapping for host [H,+S] -> device [D,+S] */
356 acc_map_data (void *h
, void *d
, size_t s
)
358 struct target_mem_desc
*tgt
= NULL
;
363 unsigned short kinds
= GOMP_MAP_ALLOC
;
365 goacc_lazy_initialize ();
367 struct goacc_thread
*thr
= goacc_thread ();
368 struct gomp_device_descr
*acc_dev
= thr
->dev
;
370 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
373 gomp_fatal ("cannot map data on shared-memory system");
377 struct goacc_thread
*thr
= goacc_thread ();
380 gomp_fatal ("[%p,+%d]->[%p,+%d] is a bad map",
381 (void *)h
, (int)s
, (void *)d
, (int)s
);
383 acc_prof_info prof_info
;
384 acc_api_info api_info
;
385 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
387 gomp_mutex_lock (&acc_dev
->lock
);
389 if (lookup_host (acc_dev
, h
, s
))
391 gomp_mutex_unlock (&acc_dev
->lock
);
392 gomp_fatal ("host address [%p, +%d] is already mapped", (void *)h
,
396 if (lookup_dev (&thr
->dev
->mem_map
, d
, s
))
398 gomp_mutex_unlock (&acc_dev
->lock
);
399 gomp_fatal ("device address [%p, +%d] is already mapped", (void *)d
,
403 gomp_mutex_unlock (&acc_dev
->lock
);
405 tgt
= gomp_map_vars (acc_dev
, mapnum
, &hostaddrs
, &devaddrs
, &sizes
,
406 &kinds
, true, GOMP_MAP_VARS_ENTER_DATA
);
408 splay_tree_key n
= tgt
->list
[0].key
;
409 assert (n
->refcount
== 1);
410 assert (n
->virtual_refcount
== 0);
411 /* Special reference counting behavior. */
412 n
->refcount
= REFCOUNT_INFINITY
;
416 thr
->prof_info
= NULL
;
417 thr
->api_info
= NULL
;
423 acc_unmap_data (void *h
)
425 struct goacc_thread
*thr
= goacc_thread ();
426 struct gomp_device_descr
*acc_dev
= thr
->dev
;
428 /* No need to call lazy open, as the address must have been mapped. */
430 /* This is a no-op on shared-memory targets. */
431 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
434 acc_prof_info prof_info
;
435 acc_api_info api_info
;
436 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
438 gomp_mutex_lock (&acc_dev
->lock
);
440 splay_tree_key n
= lookup_host (acc_dev
, h
, 1);
444 gomp_mutex_unlock (&acc_dev
->lock
);
445 gomp_fatal ("%p is not a mapped block", (void *)h
);
448 size_t host_size
= n
->host_end
- n
->host_start
;
450 if (n
->host_start
!= (uintptr_t) h
)
452 gomp_mutex_unlock (&acc_dev
->lock
);
453 gomp_fatal ("[%p,%d] surrounds %p",
454 (void *) n
->host_start
, (int) host_size
, (void *) h
);
456 /* TODO This currently doesn't catch 'REFCOUNT_INFINITY' usage different from
457 'acc_map_data'. Maybe 'virtual_refcount' can be used for disambiguating
458 the different 'REFCOUNT_INFINITY' cases, or simply separate
459 'REFCOUNT_INFINITY' values per different usage ('REFCOUNT_ACC_MAP_DATA'
461 else if (n
->refcount
!= REFCOUNT_INFINITY
)
463 gomp_mutex_unlock (&acc_dev
->lock
);
464 gomp_fatal ("refusing to unmap block [%p,+%d] that has not been mapped"
465 " by 'acc_map_data'",
466 (void *) h
, (int) host_size
);
469 splay_tree_remove (&acc_dev
->mem_map
, n
);
471 struct target_mem_desc
*tgt
= n
->tgt
;
473 if (tgt
->refcount
== REFCOUNT_INFINITY
)
475 gomp_mutex_unlock (&acc_dev
->lock
);
476 gomp_fatal ("cannot unmap target block");
478 else if (tgt
->refcount
> 1)
486 gomp_mutex_unlock (&acc_dev
->lock
);
490 thr
->prof_info
= NULL
;
491 thr
->api_info
= NULL
;
496 /* Enter dynamic mapping for a single datum. Return the device pointer. */
499 goacc_enter_datum (void **hostaddrs
, size_t *sizes
, void *kinds
, int async
)
504 if (!hostaddrs
[0] || !sizes
[0])
505 gomp_fatal ("[%p,+%d] is a bad range", hostaddrs
[0], (int) sizes
[0]);
507 goacc_lazy_initialize ();
509 struct goacc_thread
*thr
= goacc_thread ();
510 struct gomp_device_descr
*acc_dev
= thr
->dev
;
512 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
515 acc_prof_info prof_info
;
516 acc_api_info api_info
;
517 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
520 prof_info
.async
= async
;
521 prof_info
.async_queue
= prof_info
.async
;
524 gomp_mutex_lock (&acc_dev
->lock
);
526 n
= lookup_host (acc_dev
, hostaddrs
[0], sizes
[0]);
529 void *h
= hostaddrs
[0];
533 d
= (void *) (n
->tgt
->tgt_start
+ n
->tgt_offset
+ h
- n
->host_start
);
535 if ((h
+ s
) > (void *)n
->host_end
)
537 gomp_mutex_unlock (&acc_dev
->lock
);
538 gomp_fatal ("[%p,+%d] not mapped", (void *)h
, (int)s
);
541 assert (n
->refcount
!= REFCOUNT_LINK
);
542 if (n
->refcount
!= REFCOUNT_INFINITY
)
545 n
->virtual_refcount
++;
548 gomp_mutex_unlock (&acc_dev
->lock
);
552 const size_t mapnum
= 1;
554 gomp_mutex_unlock (&acc_dev
->lock
);
556 goacc_aq aq
= get_goacc_asyncqueue (async
);
558 gomp_map_vars_async (acc_dev
, aq
, mapnum
, hostaddrs
, NULL
, sizes
, kinds
,
559 true, GOMP_MAP_VARS_OPENACC_ENTER_DATA
);
561 gomp_mutex_lock (&acc_dev
->lock
);
562 n
= lookup_host (acc_dev
, hostaddrs
[0], sizes
[0]);
564 assert (n
->tgt_offset
== 0);
565 assert ((uintptr_t) hostaddrs
[0] == n
->host_start
);
566 d
= (void *) n
->tgt
->tgt_start
;
567 gomp_mutex_unlock (&acc_dev
->lock
);
572 thr
->prof_info
= NULL
;
573 thr
->api_info
= NULL
;
580 acc_create (void *h
, size_t s
)
582 unsigned short kinds
[1] = { GOMP_MAP_ALLOC
};
583 return goacc_enter_datum (&h
, &s
, &kinds
, acc_async_sync
);
587 acc_create_async (void *h
, size_t s
, int async
)
589 unsigned short kinds
[1] = { GOMP_MAP_ALLOC
};
590 goacc_enter_datum (&h
, &s
, &kinds
, async
);
593 /* acc_present_or_create used to be what acc_create is now. */
594 /* acc_pcreate is acc_present_or_create by a different name. */
595 #ifdef HAVE_ATTRIBUTE_ALIAS
596 strong_alias (acc_create
, acc_present_or_create
)
597 strong_alias (acc_create
, acc_pcreate
)
600 acc_present_or_create (void *h
, size_t s
)
602 return acc_create (h
, s
);
606 acc_pcreate (void *h
, size_t s
)
608 return acc_create (h
, s
);
613 acc_copyin (void *h
, size_t s
)
615 unsigned short kinds
[1] = { GOMP_MAP_TO
};
616 return goacc_enter_datum (&h
, &s
, &kinds
, acc_async_sync
);
620 acc_copyin_async (void *h
, size_t s
, int async
)
622 unsigned short kinds
[1] = { GOMP_MAP_TO
};
623 goacc_enter_datum (&h
, &s
, &kinds
, async
);
626 /* acc_present_or_copyin used to be what acc_copyin is now. */
627 /* acc_pcopyin is acc_present_or_copyin by a different name. */
628 #ifdef HAVE_ATTRIBUTE_ALIAS
629 strong_alias (acc_copyin
, acc_present_or_copyin
)
630 strong_alias (acc_copyin
, acc_pcopyin
)
633 acc_present_or_copyin (void *h
, size_t s
)
635 return acc_copyin (h
, s
);
639 acc_pcopyin (void *h
, size_t s
)
641 return acc_copyin (h
, s
);
646 /* Exit a dynamic mapping for a single variable. */
649 goacc_exit_datum (void *h
, size_t s
, unsigned short kind
, int async
)
651 /* No need to call lazy open, as the data must already have been
656 struct goacc_thread
*thr
= goacc_thread ();
657 struct gomp_device_descr
*acc_dev
= thr
->dev
;
659 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
662 acc_prof_info prof_info
;
663 acc_api_info api_info
;
664 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
667 prof_info
.async
= async
;
668 prof_info
.async_queue
= prof_info
.async
;
671 gomp_mutex_lock (&acc_dev
->lock
);
673 splay_tree_key n
= lookup_host (acc_dev
, h
, s
);
675 /* PR92726, RP92970, PR92984: no-op. */
678 if ((uintptr_t) h
< n
->host_start
|| (uintptr_t) h
+ s
> n
->host_end
)
680 size_t host_size
= n
->host_end
- n
->host_start
;
681 gomp_mutex_unlock (&acc_dev
->lock
);
682 gomp_fatal ("[%p,+%d] outside mapped block [%p,+%d]",
683 (void *) h
, (int) s
, (void *) n
->host_start
, (int) host_size
);
686 bool finalize
= (kind
== GOMP_MAP_DELETE
687 || kind
== GOMP_MAP_FORCE_FROM
);
690 if (n
->refcount
!= REFCOUNT_INFINITY
)
691 n
->refcount
-= n
->virtual_refcount
;
692 n
->virtual_refcount
= 0;
695 if (n
->virtual_refcount
> 0)
697 if (n
->refcount
!= REFCOUNT_INFINITY
)
699 n
->virtual_refcount
--;
701 else if (n
->refcount
> 0 && n
->refcount
!= REFCOUNT_INFINITY
)
704 if (n
->refcount
== 0)
706 goacc_aq aq
= get_goacc_asyncqueue (async
);
708 bool copyout
= (kind
== GOMP_MAP_FROM
709 || kind
== GOMP_MAP_FORCE_FROM
);
712 void *d
= (void *) (n
->tgt
->tgt_start
+ n
->tgt_offset
713 + (uintptr_t) h
- n
->host_start
);
714 gomp_copy_dev2host (acc_dev
, aq
, h
, d
, s
);
718 /* TODO We can't do the 'is_tgt_unmapped' checking -- see the
719 'gomp_unref_tgt' comment in
720 <http://mid.mail-archive.com/878snl36eu.fsf@euler.schwinge.homeip.net>;
722 gomp_remove_var_async (acc_dev
, n
, aq
);
725 bool is_tgt_unmapped
= gomp_remove_var (acc_dev
, n
);
726 assert (is_tgt_unmapped
);
731 gomp_mutex_unlock (&acc_dev
->lock
);
735 thr
->prof_info
= NULL
;
736 thr
->api_info
= NULL
;
741 acc_delete (void *h
, size_t s
)
743 goacc_exit_datum (h
, s
, GOMP_MAP_RELEASE
, acc_async_sync
);
747 acc_delete_async (void *h
, size_t s
, int async
)
749 goacc_exit_datum (h
, s
, GOMP_MAP_RELEASE
, async
);
753 acc_delete_finalize (void *h
, size_t s
)
755 goacc_exit_datum (h
, s
, GOMP_MAP_DELETE
, acc_async_sync
);
759 acc_delete_finalize_async (void *h
, size_t s
, int async
)
761 goacc_exit_datum (h
, s
, GOMP_MAP_DELETE
, async
);
765 acc_copyout (void *h
, size_t s
)
767 goacc_exit_datum (h
, s
, GOMP_MAP_FROM
, acc_async_sync
);
771 acc_copyout_async (void *h
, size_t s
, int async
)
773 goacc_exit_datum (h
, s
, GOMP_MAP_FROM
, async
);
777 acc_copyout_finalize (void *h
, size_t s
)
779 goacc_exit_datum (h
, s
, GOMP_MAP_FORCE_FROM
, acc_async_sync
);
783 acc_copyout_finalize_async (void *h
, size_t s
, int async
)
785 goacc_exit_datum (h
, s
, GOMP_MAP_FORCE_FROM
, async
);
789 update_dev_host (int is_dev
, void *h
, size_t s
, int async
)
794 goacc_lazy_initialize ();
796 struct goacc_thread
*thr
= goacc_thread ();
797 struct gomp_device_descr
*acc_dev
= thr
->dev
;
799 if (acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
802 /* Fortran optional arguments that are non-present result in a
803 NULL host address here. This can safely be ignored as it is
804 not possible to 'update' a non-present optional argument. */
808 acc_prof_info prof_info
;
809 acc_api_info api_info
;
810 bool profiling_p
= GOACC_PROFILING_SETUP_P (thr
, &prof_info
, &api_info
);
813 prof_info
.async
= async
;
814 prof_info
.async_queue
= prof_info
.async
;
817 gomp_mutex_lock (&acc_dev
->lock
);
819 n
= lookup_host (acc_dev
, h
, s
);
823 gomp_mutex_unlock (&acc_dev
->lock
);
824 gomp_fatal ("[%p,%d] is not mapped", h
, (int)s
);
827 d
= (void *) (n
->tgt
->tgt_start
+ n
->tgt_offset
828 + (uintptr_t) h
- n
->host_start
);
830 goacc_aq aq
= get_goacc_asyncqueue (async
);
833 gomp_copy_host2dev (acc_dev
, aq
, d
, h
, s
, /* TODO: cbuf? */ NULL
);
835 gomp_copy_dev2host (acc_dev
, aq
, h
, d
, s
);
837 gomp_mutex_unlock (&acc_dev
->lock
);
841 thr
->prof_info
= NULL
;
842 thr
->api_info
= NULL
;
847 acc_update_device (void *h
, size_t s
)
849 update_dev_host (1, h
, s
, acc_async_sync
);
853 acc_update_device_async (void *h
, size_t s
, int async
)
855 update_dev_host (1, h
, s
, async
);
859 acc_update_self (void *h
, size_t s
)
861 update_dev_host (0, h
, s
, acc_async_sync
);
865 acc_update_self_async (void *h
, size_t s
, int async
)
867 update_dev_host (0, h
, s
, async
);
870 /* Some types of (pointer) variables use several consecutive mappings, which
871 must be treated as a group for enter/exit data directives. This function
872 returns the last mapping in such a group (inclusive), or POS for singleton
876 find_group_last (int pos
, size_t mapnum
, unsigned short *kinds
)
878 unsigned char kind0
= kinds
[pos
] & 0xff;
879 int first_pos
= pos
, last_pos
= pos
;
881 if (kind0
== GOMP_MAP_TO_PSET
)
883 while (pos
+ 1 < mapnum
&& (kinds
[pos
+ 1] & 0xff) == GOMP_MAP_POINTER
)
885 /* We expect at least one GOMP_MAP_POINTER after a GOMP_MAP_TO_PSET. */
886 assert (last_pos
> first_pos
);
890 /* GOMP_MAP_ALWAYS_POINTER can only appear directly after some other
893 && (kinds
[pos
+ 1] & 0xff) == GOMP_MAP_ALWAYS_POINTER
)
896 /* We can have one or several GOMP_MAP_POINTER mappings after a to/from
898 while (pos
+ 1 < mapnum
&& (kinds
[pos
+ 1] & 0xff) == GOMP_MAP_POINTER
)
905 /* Map variables for OpenACC "enter data". We can't just call
906 gomp_map_vars_async once, because individual mapped variables might have
907 "exit data" called for them at different times. */
910 goacc_enter_data_internal (struct gomp_device_descr
*acc_dev
, size_t mapnum
,
911 void **hostaddrs
, size_t *sizes
,
912 unsigned short *kinds
, goacc_aq aq
)
914 for (size_t i
= 0; i
< mapnum
; i
++)
916 int group_last
= find_group_last (i
, mapnum
, kinds
);
918 gomp_map_vars_async (acc_dev
, aq
,
919 (group_last
- i
) + 1,
921 &sizes
[i
], &kinds
[i
], true,
922 GOMP_MAP_VARS_OPENACC_ENTER_DATA
);
928 /* Unmap variables for OpenACC "exit data". */
931 goacc_exit_data_internal (struct gomp_device_descr
*acc_dev
, size_t mapnum
,
932 void **hostaddrs
, size_t *sizes
,
933 unsigned short *kinds
, goacc_aq aq
)
935 gomp_mutex_lock (&acc_dev
->lock
);
937 for (size_t i
= 0; i
< mapnum
; ++i
)
939 unsigned char kind
= kinds
[i
] & 0xff;
940 bool copyfrom
= false;
941 bool finalize
= false;
943 if (kind
== GOMP_MAP_FORCE_FROM
944 || kind
== GOMP_MAP_DELETE
)
950 case GOMP_MAP_FORCE_FROM
:
951 case GOMP_MAP_ALWAYS_FROM
:
955 case GOMP_MAP_TO_PSET
:
956 case GOMP_MAP_POINTER
:
957 case GOMP_MAP_DELETE
:
958 case GOMP_MAP_RELEASE
:
960 struct splay_tree_key_s cur_node
;
962 if (kind
== GOMP_MAP_POINTER
)
963 size
= sizeof (void *);
966 cur_node
.host_start
= (uintptr_t) hostaddrs
[i
];
967 cur_node
.host_end
= cur_node
.host_start
+ size
;
969 = splay_tree_lookup (&acc_dev
->mem_map
, &cur_node
);
976 if (n
->refcount
!= REFCOUNT_INFINITY
)
977 n
->refcount
-= n
->virtual_refcount
;
978 n
->virtual_refcount
= 0;
981 if (n
->virtual_refcount
> 0)
983 if (n
->refcount
!= REFCOUNT_INFINITY
)
985 n
->virtual_refcount
--;
987 else if (n
->refcount
> 0 && n
->refcount
!= REFCOUNT_INFINITY
)
991 && (kind
!= GOMP_MAP_FROM
|| n
->refcount
== 0))
992 gomp_copy_dev2host (acc_dev
, aq
, (void *) cur_node
.host_start
,
993 (void *) (n
->tgt
->tgt_start
+ n
->tgt_offset
994 + cur_node
.host_start
996 cur_node
.host_end
- cur_node
.host_start
);
998 if (n
->refcount
== 0)
999 gomp_remove_var_async (acc_dev
, n
, aq
);
1003 gomp_fatal (">>>> goacc_exit_data_internal UNHANDLED kind 0x%.2x",
1008 gomp_mutex_unlock (&acc_dev
->lock
);
1012 GOACC_enter_exit_data (int flags_m
, size_t mapnum
, void **hostaddrs
,
1013 size_t *sizes
, unsigned short *kinds
, int async
,
1016 int flags
= GOACC_FLAGS_UNMARSHAL (flags_m
);
1018 struct goacc_thread
*thr
;
1019 struct gomp_device_descr
*acc_dev
;
1020 bool data_enter
= false;
1023 goacc_lazy_initialize ();
1025 thr
= goacc_thread ();
1028 /* Determine if this is an "acc enter data". */
1029 for (i
= 0; i
< mapnum
; ++i
)
1031 unsigned char kind
= kinds
[i
] & 0xff;
1033 if (kind
== GOMP_MAP_POINTER
|| kind
== GOMP_MAP_TO_PSET
)
1036 if (kind
== GOMP_MAP_FORCE_ALLOC
1037 || kind
== GOMP_MAP_FORCE_PRESENT
1038 || kind
== GOMP_MAP_FORCE_TO
1039 || kind
== GOMP_MAP_TO
1040 || kind
== GOMP_MAP_ALLOC
)
1046 if (kind
== GOMP_MAP_RELEASE
1047 || kind
== GOMP_MAP_DELETE
1048 || kind
== GOMP_MAP_FROM
1049 || kind
== GOMP_MAP_FORCE_FROM
)
1052 gomp_fatal (">>>> GOACC_enter_exit_data UNHANDLED kind 0x%.2x",
1056 bool profiling_p
= GOACC_PROFILING_DISPATCH_P (true);
1058 acc_prof_info prof_info
;
1061 thr
->prof_info
= &prof_info
;
1063 prof_info
.event_type
1064 = data_enter
? acc_ev_enter_data_start
: acc_ev_exit_data_start
;
1065 prof_info
.valid_bytes
= _ACC_PROF_INFO_VALID_BYTES
;
1066 prof_info
.version
= _ACC_PROF_INFO_VERSION
;
1067 prof_info
.device_type
= acc_device_type (acc_dev
->type
);
1068 prof_info
.device_number
= acc_dev
->target_id
;
1069 prof_info
.thread_id
= -1;
1070 prof_info
.async
= async
;
1071 prof_info
.async_queue
= prof_info
.async
;
1072 prof_info
.src_file
= NULL
;
1073 prof_info
.func_name
= NULL
;
1074 prof_info
.line_no
= -1;
1075 prof_info
.end_line_no
= -1;
1076 prof_info
.func_line_no
= -1;
1077 prof_info
.func_end_line_no
= -1;
1079 acc_event_info enter_exit_data_event_info
;
1082 enter_exit_data_event_info
.other_event
.event_type
1083 = prof_info
.event_type
;
1084 enter_exit_data_event_info
.other_event
.valid_bytes
1085 = _ACC_OTHER_EVENT_INFO_VALID_BYTES
;
1086 enter_exit_data_event_info
.other_event
.parent_construct
1087 = data_enter
? acc_construct_enter_data
: acc_construct_exit_data
;
1088 enter_exit_data_event_info
.other_event
.implicit
= 0;
1089 enter_exit_data_event_info
.other_event
.tool_info
= NULL
;
1091 acc_api_info api_info
;
1094 thr
->api_info
= &api_info
;
1096 api_info
.device_api
= acc_device_api_none
;
1097 api_info
.valid_bytes
= _ACC_API_INFO_VALID_BYTES
;
1098 api_info
.device_type
= prof_info
.device_type
;
1099 api_info
.vendor
= -1;
1100 api_info
.device_handle
= NULL
;
1101 api_info
.context_handle
= NULL
;
1102 api_info
.async_handle
= NULL
;
1106 goacc_profiling_dispatch (&prof_info
, &enter_exit_data_event_info
,
1109 if ((acc_dev
->capabilities
& GOMP_OFFLOAD_CAP_SHARED_MEM
)
1110 || (flags
& GOACC_FLAG_HOST_FALLBACK
))
1112 prof_info
.device_type
= acc_device_host
;
1113 api_info
.device_type
= prof_info
.device_type
;
1122 va_start (ap
, num_waits
);
1123 goacc_wait (async
, num_waits
, &ap
);
1127 goacc_aq aq
= get_goacc_asyncqueue (async
);
1130 goacc_enter_data_internal (acc_dev
, mapnum
, hostaddrs
, sizes
, kinds
, aq
);
1132 goacc_exit_data_internal (acc_dev
, mapnum
, hostaddrs
, sizes
, kinds
, aq
);
1137 prof_info
.event_type
1138 = data_enter
? acc_ev_enter_data_end
: acc_ev_exit_data_end
;
1139 enter_exit_data_event_info
.other_event
.event_type
= prof_info
.event_type
;
1140 goacc_profiling_dispatch (&prof_info
, &enter_exit_data_event_info
,
1143 thr
->prof_info
= NULL
;
1144 thr
->api_info
= NULL
;