libgomp/oacc-init.c

   1 /* OpenACC Runtime initialization routines
   2
   3    Copyright (C) 2013-2015 Free Software Foundation, Inc.
   4
   5    Contributed by Mentor Embedded.
   6
   7    This file is part of the GNU Offloading and Multi Processing Library
   8    (libgomp).
   9
  10    Libgomp is free software; you can redistribute it and/or modify it
  11    under the terms of the GNU General Public License as published by
  12    the Free Software Foundation; either version 3, or (at your option)
  13    any later version.
  14
  15    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  16    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  17    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  18    more details.
  19
  20    Under Section 7 of GPL version 3, you are granted additional
  21    permissions described in the GCC Runtime Library Exception, version
  22    3.1, as published by the Free Software Foundation.
  23
  24    You should have received a copy of the GNU General Public License and
  25    a copy of the GCC Runtime Library Exception along with this program;
  26    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  27    <http://www.gnu.org/licenses/>.  */
  28
  29 #include "libgomp.h"
  30 #include "oacc-int.h"
  31 #include "openacc.h"
  32 #include <assert.h>
  33 #include <stdlib.h>
  34 #include <strings.h>
  35 #include <stdbool.h>
  36 #include <string.h>
  37
  38 static gomp_mutex_t acc_device_lock;
  39
  40 /* The dispatch table for the current accelerator device.  This is global, so
  41    you can only have one type of device open at any given time in a program.
  42    This is the "base" device in that several devices that use the same
  43    dispatch table may be active concurrently: this one (the "zeroth") is used
  44    for overall initialisation/shutdown, and other instances -- not necessarily
  45    including this one -- may be opened and closed once the base device has
  46    been initialized.  */
  47 struct gomp_device_descr *base_dev;
  48
  49 #if defined HAVE_TLS || defined USE_EMUTLS
  50 __thread struct goacc_thread *goacc_tls_data;
  51 #else
  52 pthread_key_t goacc_tls_key;
  53 #endif
  54 static pthread_key_t goacc_cleanup_key;
  55
  56 /* Current dispatcher, and how it was initialized */
  57 static acc_device_t init_key = _ACC_device_hwm;
  58
  59 static struct goacc_thread *goacc_threads;
  60 static gomp_mutex_t goacc_thread_lock;
  61
  62 /* An array of dispatchers for device types, indexed by the type.  This array
  63    only references "base" devices, and other instances of the same type are
  64    found by simply indexing from each such device (which are stored linearly,
  65    grouped by device in target.c:devices).  */
  66 static struct gomp_device_descr *dispatchers[_ACC_device_hwm] = { 0 };
  67
  68 attribute_hidden void
  69 goacc_register (struct gomp_device_descr *disp)
  70 {
  71   /* Only register the 0th device here.  */
  72   if (disp->target_id != 0)
  73     return;
  74
  75   gomp_mutex_lock (&acc_device_lock);
  76
  77   assert (acc_device_type (disp->type) != acc_device_none
  78           && acc_device_type (disp->type) != acc_device_default
  79           && acc_device_type (disp->type) != acc_device_not_host);
  80   assert (!dispatchers[disp->type]);
  81   dispatchers[disp->type] = disp;
  82
  83   gomp_mutex_unlock (&acc_device_lock);
  84 }
  85
  86 /* OpenACC names some things a little differently.  */
  87
  88 static const char *
  89 get_openacc_name (const char *name)
  90 {
  91   if (strcmp (name, "nvptx") == 0)
  92     return "nvidia";
  93   else
  94     return name;
  95 }
  96
  97 static struct gomp_device_descr *
  98 resolve_device (acc_device_t d)
  99 {
 100   acc_device_t d_arg = d;
 101
 102   switch (d)
 103     {
 104     case acc_device_default:
 105       {
 106         if (goacc_device_type)
 107           {
 108             /* Lookup the named device.  */
 109             while (++d != _ACC_device_hwm)
 110               if (dispatchers[d]
 111                   && !strcasecmp (goacc_device_type,
 112                                   get_openacc_name (dispatchers[d]->name))
 113                   && dispatchers[d]->get_num_devices_func () > 0)
 114                 goto found;
 115
 116             gomp_fatal ("device type %s not supported", goacc_device_type);
 117           }
 118
 119         /* No default device specified, so start scanning for any non-host
 120            device that is available.  */
 121         d = acc_device_not_host;
 122       }
 123       /* FALLTHROUGH */
 124
 125     case acc_device_not_host:
 126       /* Find the first available device after acc_device_not_host.  */
 127       while (++d != _ACC_device_hwm)
 128         if (dispatchers[d] && dispatchers[d]->get_num_devices_func () > 0)
 129           goto found;
 130       if (d_arg == acc_device_default)
 131         {
 132           d = acc_device_host;
 133           goto found;
 134         }
 135       gomp_fatal ("no device found");
 136       break;
 137
 138     case acc_device_host:
 139       break;
 140
 141     default:
 142       if (d > _ACC_device_hwm)
 143         gomp_fatal ("device %u out of range", (unsigned)d);
 144       break;
 145     }
 146  found:
 147
 148   assert (d != acc_device_none
 149           && d != acc_device_default
 150           && d != acc_device_not_host);
 151
 152   return dispatchers[d];
 153 }
 154
 155 /* This is called when plugins have been initialized, and serves to call
 156    (indirectly) the target's device_init hook.  Calling multiple times without
 157    an intervening acc_shutdown_1 call is an error.  */
 158
 159 static struct gomp_device_descr *
 160 acc_init_1 (acc_device_t d)
 161 {
 162   struct gomp_device_descr *acc_dev;
 163
 164   acc_dev = resolve_device (d);
 165
 166   if (!acc_dev || acc_dev->get_num_devices_func () <= 0)
 167     gomp_fatal ("device %u not supported", (unsigned)d);
 168
 169   if (acc_dev->is_initialized)
 170     gomp_fatal ("device already active");
 171
 172   /* We need to remember what we were intialized as, to check shutdown etc.  */
 173   init_key = d;
 174
 175   gomp_init_device (acc_dev);
 176
 177   return acc_dev;
 178 }
 179
 180 static struct goacc_thread *
 181 goacc_new_thread (void)
 182 {
 183   struct goacc_thread *thr = gomp_malloc (sizeof (struct gomp_thread));
 184
 185 #if defined HAVE_TLS || defined USE_EMUTLS
 186   goacc_tls_data = thr;
 187 #else
 188   pthread_setspecific (goacc_tls_key, thr);
 189 #endif
 190
 191   pthread_setspecific (goacc_cleanup_key, thr);
 192
 193   gomp_mutex_lock (&goacc_thread_lock);
 194   thr->next = goacc_threads;
 195   goacc_threads = thr;
 196   gomp_mutex_unlock (&goacc_thread_lock);
 197
 198   return thr;
 199 }
 200
 201 static void
 202 goacc_destroy_thread (void *data)
 203 {
 204   struct goacc_thread *thr = data, *walk, *prev;
 205
 206   gomp_mutex_lock (&goacc_thread_lock);
 207
 208   if (thr)
 209     {
 210       if (base_dev && thr->target_tls)
 211         {
 212           base_dev->openacc.destroy_thread_data_func (thr->target_tls);
 213           thr->target_tls = NULL;
 214         }
 215
 216       assert (!thr->mapped_data);
 217
 218       /* Remove from thread list.  */
 219       for (prev = NULL, walk = goacc_threads; walk;
 220            prev = walk, walk = walk->next)
 221         if (walk == thr)
 222           {
 223             if (prev == NULL)
 224               goacc_threads = walk->next;
 225             else
 226               prev->next = walk->next;
 227
 228             free (thr);
 229
 230             break;
 231           }
 232
 233       assert (walk);
 234     }
 235
 236   gomp_mutex_unlock (&goacc_thread_lock);
 237 }
 238
 239 /* Open the ORD'th device of the currently-active type (base_dev must be
 240    initialised before calling).  If ORD is < 0, open the default-numbered
 241    device (set by the ACC_DEVICE_NUM environment variable or a call to
 242    acc_set_device_num), or leave any currently-opened device as is.  "Opening"
 243    consists of calling the device's open_device_func hook, and setting up
 244    thread-local data (maybe allocating, then initializing with information
 245    pertaining to the newly-opened or previously-opened device).  */
 246
 247 static void
 248 lazy_open (int ord)
 249 {
 250   struct goacc_thread *thr = goacc_thread ();
 251   struct gomp_device_descr *acc_dev;
 252
 253   if (thr && thr->dev)
 254     {
 255       assert (ord < 0 || ord == thr->dev->target_id);
 256       return;
 257     }
 258
 259   assert (base_dev);
 260
 261   if (ord < 0)
 262     ord = goacc_device_num;
 263
 264   /* The OpenACC 2.0 spec leaves the runtime's behaviour when an out-of-range
 265      device is requested as implementation-defined (4.2 ACC_DEVICE_NUM).
 266      We choose to raise an error in such a case.  */
 267   if (ord >= base_dev->get_num_devices_func ())
 268     gomp_fatal ("device %u does not exist", ord);
 269
 270   if (!thr)
 271     thr = goacc_new_thread ();
 272
 273   acc_dev = thr->dev = &base_dev[ord];
 274
 275   assert (acc_dev->target_id == ord);
 276
 277   thr->saved_bound_dev = NULL;
 278   thr->mapped_data = NULL;
 279
 280   if (!acc_dev->openacc.target_data)
 281     acc_dev->openacc.target_data = acc_dev->openacc.open_device_func (ord);
 282
 283   thr->target_tls
 284     = acc_dev->openacc.create_thread_data_func (acc_dev->openacc.target_data);
 285
 286   acc_dev->openacc.async_set_async_func (acc_async_sync);
 287 }
 288
 289 /* OpenACC 2.0a (3.2.12, 3.2.13) doesn't specify whether the serialization of
 290    init/shutdown is per-process or per-thread.  We choose per-process.  */
 291
 292 void
 293 acc_init (acc_device_t d)
 294 {
 295   if (!base_dev)
 296     gomp_init_targets_once ();
 297
 298   gomp_mutex_lock (&acc_device_lock);
 299
 300   base_dev = acc_init_1 (d);
 301
 302   lazy_open (-1);
 303
 304   gomp_mutex_unlock (&acc_device_lock);
 305 }
 306
 307 ialias (acc_init)
 308
 309 static void
 310 acc_shutdown_1 (acc_device_t d)
 311 {
 312   struct goacc_thread *walk;
 313
 314   /* We don't check whether d matches the actual device found, because
 315      OpenACC 2.0 (3.2.12) says the parameters to the init and this
 316      call must match (for the shutdown call anyway, it's silent on
 317      others).  */
 318
 319   if (!base_dev)
 320     gomp_fatal ("no device initialized");
 321   if (d != init_key)
 322     gomp_fatal ("device %u(%u) is initialized",
 323                 (unsigned) init_key, (unsigned) base_dev->type);
 324
 325   gomp_mutex_lock (&goacc_thread_lock);
 326
 327   /* Free target-specific TLS data and close all devices.  */
 328   for (walk = goacc_threads; walk != NULL; walk = walk->next)
 329     {
 330       if (walk->target_tls)
 331         base_dev->openacc.destroy_thread_data_func (walk->target_tls);
 332
 333       walk->target_tls = NULL;
 334
 335       /* This would mean the user is shutting down OpenACC in the middle of an
 336          "acc data" pragma.  Likely not intentional.  */
 337       if (walk->mapped_data)
 338         gomp_fatal ("shutdown in 'acc data' region");
 339
 340       if (walk->dev)
 341         {
 342           void *target_data = walk->dev->openacc.target_data;
 343           if (walk->dev->openacc.close_device_func (target_data) < 0)
 344             gomp_fatal ("failed to close device");
 345
 346           walk->dev->openacc.target_data = target_data = NULL;
 347
 348           gomp_mutex_lock (&walk->dev->lock);
 349           gomp_free_memmap (&walk->dev->mem_map);
 350           gomp_mutex_unlock (&walk->dev->lock);
 351
 352           walk->dev = NULL;
 353         }
 354     }
 355
 356   gomp_mutex_unlock (&goacc_thread_lock);
 357
 358   gomp_fini_device (base_dev);
 359
 360   base_dev = NULL;
 361 }
 362
 363 void
 364 acc_shutdown (acc_device_t d)
 365 {
 366   gomp_mutex_lock (&acc_device_lock);
 367
 368   acc_shutdown_1 (d);
 369
 370   gomp_mutex_unlock (&acc_device_lock);
 371 }
 372
 373 ialias (acc_shutdown)
 374
 375 /* This function is called after plugins have been initialized.  It deals with
 376    the "base" device, and is used to prepare the runtime for dealing with a
 377    number of such devices (as implemented by some particular plugin).  If the
 378    argument device type D matches a previous call to the function, return the
 379    current base device, else shut the old device down and re-initialize with
 380    the new device type.  */
 381
 382 static struct gomp_device_descr *
 383 lazy_init (acc_device_t d)
 384 {
 385   if (base_dev)
 386     {
 387       /* Re-initializing the same device, do nothing.  */
 388       if (d == init_key)
 389         return base_dev;
 390
 391       acc_shutdown_1 (init_key);
 392     }
 393
 394   assert (!base_dev);
 395
 396   return acc_init_1 (d);
 397 }
 398
 399 /* Ensure that plugins are loaded, initialize and open the (default-numbered)
 400    device.  */
 401
 402 static void
 403 lazy_init_and_open (acc_device_t d)
 404 {
 405   if (!base_dev)
 406     gomp_init_targets_once ();
 407
 408   gomp_mutex_lock (&acc_device_lock);
 409
 410   base_dev = lazy_init (d);
 411
 412   lazy_open (-1);
 413
 414   gomp_mutex_unlock (&acc_device_lock);
 415 }
 416
 417 int
 418 acc_get_num_devices (acc_device_t d)
 419 {
 420   int n = 0;
 421   const struct gomp_device_descr *acc_dev;
 422
 423   if (d == acc_device_none)
 424     return 0;
 425
 426   if (!base_dev)
 427     gomp_init_targets_once ();
 428
 429   acc_dev = resolve_device (d);
 430   if (!acc_dev)
 431     return 0;
 432
 433   n = acc_dev->get_num_devices_func ();
 434   if (n < 0)
 435     n = 0;
 436
 437   return n;
 438 }
 439
 440 ialias (acc_get_num_devices)
 441
 442 void
 443 acc_set_device_type (acc_device_t d)
 444 {
 445   lazy_init_and_open (d);
 446 }
 447
 448 ialias (acc_set_device_type)
 449
 450 acc_device_t
 451 acc_get_device_type (void)
 452 {
 453   acc_device_t res = acc_device_none;
 454   const struct gomp_device_descr *dev;
 455
 456   if (base_dev)
 457     res = acc_device_type (base_dev->type);
 458   else
 459     {
 460       gomp_init_targets_once ();
 461
 462       dev = resolve_device (acc_device_default);
 463       res = acc_device_type (dev->type);
 464     }
 465
 466   assert (res != acc_device_default
 467           && res != acc_device_not_host);
 468
 469   return res;
 470 }
 471
 472 ialias (acc_get_device_type)
 473
 474 int
 475 acc_get_device_num (acc_device_t d)
 476 {
 477   const struct gomp_device_descr *dev;
 478   int num;
 479
 480   if (d >= _ACC_device_hwm)
 481     gomp_fatal ("device %u out of range", (unsigned)d);
 482
 483   if (!base_dev)
 484     gomp_init_targets_once ();
 485
 486   dev = resolve_device (d);
 487   if (!dev)
 488     gomp_fatal ("no devices of type %u", d);
 489
 490   /* We might not have called lazy_open for this host thread yet, in which case
 491      the get_device_num_func hook will return -1.  */
 492   num = dev->openacc.get_device_num_func ();
 493   if (num < 0)
 494     num = goacc_device_num;
 495
 496   return num;
 497 }
 498
 499 ialias (acc_get_device_num)
 500
 501 void
 502 acc_set_device_num (int n, acc_device_t d)
 503 {
 504   const struct gomp_device_descr *dev;
 505   int num_devices;
 506
 507   if (!base_dev)
 508     gomp_init_targets_once ();
 509
 510   if ((int) d == 0)
 511     {
 512       int i;
 513
 514       /* A device setting of zero sets all device types on the system to use
 515          the Nth instance of that device type.  Only attempt it for initialized
 516          devices though.  */
 517       for (i = acc_device_not_host + 1; i < _ACC_device_hwm; i++)
 518         {
 519           dev = resolve_device (d);
 520           if (dev && dev->is_initialized)
 521             dev->openacc.set_device_num_func (n);
 522         }
 523
 524       /* ...and for future calls to acc_init/acc_set_device_type, etc.  */
 525       goacc_device_num = n;
 526     }
 527   else
 528     {
 529       struct goacc_thread *thr = goacc_thread ();
 530
 531       gomp_mutex_lock (&acc_device_lock);
 532
 533       base_dev = lazy_init (d);
 534
 535       num_devices = base_dev->get_num_devices_func ();
 536
 537       if (n >= num_devices)
 538         gomp_fatal ("device %u out of range", n);
 539
 540       /* If we're changing the device number, de-associate this thread with
 541          the device (but don't close the device, since it may be in use by
 542          other threads).  */
 543       if (thr && thr->dev && n != thr->dev->target_id)
 544         thr->dev = NULL;
 545
 546       lazy_open (n);
 547
 548       gomp_mutex_unlock (&acc_device_lock);
 549     }
 550 }
 551
 552 ialias (acc_set_device_num)
 553
 554 int
 555 acc_on_device (acc_device_t dev)
 556 {
 557   struct goacc_thread *thr = goacc_thread ();
 558
 559   if (thr && thr->dev
 560       && acc_device_type (thr->dev->type) == acc_device_host_nonshm)
 561     return dev == acc_device_host_nonshm || dev == acc_device_not_host;
 562
 563   /* Just rely on the compiler builtin.  */
 564   return __builtin_acc_on_device (dev);
 565 }
 566
 567 ialias (acc_on_device)
 568
 569 attribute_hidden void
 570 goacc_runtime_initialize (void)
 571 {
 572   gomp_mutex_init (&acc_device_lock);
 573
 574 #if !(defined HAVE_TLS || defined USE_EMUTLS)
 575   pthread_key_create (&goacc_tls_key, NULL);
 576 #endif
 577
 578   pthread_key_create (&goacc_cleanup_key, goacc_destroy_thread);
 579
 580   base_dev = NULL;
 581
 582   goacc_threads = NULL;
 583   gomp_mutex_init (&goacc_thread_lock);
 584 }
 585
 586 /* Compiler helper functions */
 587
 588 attribute_hidden void
 589 goacc_save_and_set_bind (acc_device_t d)
 590 {
 591   struct goacc_thread *thr = goacc_thread ();
 592
 593   assert (!thr->saved_bound_dev);
 594
 595   thr->saved_bound_dev = thr->dev;
 596   thr->dev = dispatchers[d];
 597 }
 598
 599 attribute_hidden void
 600 goacc_restore_bind (void)
 601 {
 602   struct goacc_thread *thr = goacc_thread ();
 603
 604   thr->dev = thr->saved_bound_dev;
 605   thr->saved_bound_dev = NULL;
 606 }
 607
 608 /* This is called from any OpenACC support function that may need to implicitly
 609    initialize the libgomp runtime.  On exit all such initialization will have
 610    been done, and both the global ACC_dev and the per-host-thread ACC_memmap
 611    pointers will be valid.  */
 612
 613 attribute_hidden void
 614 goacc_lazy_initialize (void)
 615 {
 616   struct goacc_thread *thr = goacc_thread ();
 617
 618   if (thr && thr->dev)
 619     return;
 620
 621   if (!base_dev)
 622     lazy_init_and_open (acc_device_default);
 623   else
 624     {
 625       gomp_mutex_lock (&acc_device_lock);
 626       lazy_open (-1);
 627       gomp_mutex_unlock (&acc_device_lock);
 628     }
 629 }