libgomp/allocator.c

   1 /* Copyright (C) 2020 Free Software Foundation, Inc.
   2    Contributed by Jakub Jelinek <jakub@redhat.com>.
   3
   4    This file is part of the GNU Offloading and Multi Processing Library
   5    (libgomp).
   6
   7    Libgomp is free software; you can redistribute it and/or modify it
   8    under the terms of the GNU General Public License as published by
   9    the Free Software Foundation; either version 3, or (at your option)
  10    any later version.
  11
  12    Libgomp is distributed in the hope that it will be useful, but WITHOUT ANY
  13    WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  14    FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  15    more details.
  16
  17    Under Section 7 of GPL version 3, you are granted additional
  18    permissions described in the GCC Runtime Library Exception, version
  19    3.1, as published by the Free Software Foundation.
  20
  21    You should have received a copy of the GNU General Public License and
  22    a copy of the GCC Runtime Library Exception along with this program;
  23    see the files COPYING3 and COPYING.RUNTIME respectively.  If not, see
  24    <http://www.gnu.org/licenses/>.  */
  25
  26 /* This file contains wrappers for the system allocation routines.  Most
  27    places in the OpenMP API do not make any provision for failure, so in
  28    general we cannot allow memory allocation to fail.  */
  29
  30 #define _GNU_SOURCE
  31 #include "libgomp.h"
  32 #include <stdlib.h>
  33
  34 #define omp_max_predefined_alloc omp_thread_mem_alloc
  35
  36 struct omp_allocator_data
  37 {
  38   omp_memspace_handle_t memspace;
  39   omp_uintptr_t alignment;
  40   omp_uintptr_t pool_size;
  41   omp_uintptr_t used_pool_size;
  42   omp_allocator_handle_t fb_data;
  43   unsigned int sync_hint : 8;
  44   unsigned int access : 8;
  45   unsigned int fallback : 8;
  46   unsigned int pinned : 1;
  47   unsigned int partition : 7;
  48 #ifndef HAVE_SYNC_BUILTINS
  49   gomp_mutex_t lock;
  50 #endif
  51 };
  52
  53 struct omp_mem_header
  54 {
  55   void *ptr;
  56   size_t size;
  57   omp_allocator_handle_t allocator;
  58   void *pad;
  59 };
  60
  61 omp_allocator_handle_t
  62 omp_init_allocator (omp_memspace_handle_t memspace, int ntraits,
  63                     const omp_alloctrait_t traits[])
  64 {
  65   struct omp_allocator_data data
  66     = { memspace, 1, ~(uintptr_t) 0, 0, 0, omp_atv_contended, omp_atv_all,
  67         omp_atv_default_mem_fb, omp_atv_false, omp_atv_environment };
  68   struct omp_allocator_data *ret;
  69   int i;
  70
  71   if (memspace > omp_low_lat_mem_space)
  72     return omp_null_allocator;
  73   for (i = 0; i < ntraits; i++)
  74     switch (traits[i].key)
  75       {
  76       case omp_atk_sync_hint:
  77         switch (traits[i].value)
  78           {
  79           case omp_atv_default:
  80             data.sync_hint = omp_atv_contended;
  81             break;
  82           case omp_atv_contended:
  83           case omp_atv_uncontended:
  84           case omp_atv_sequential:
  85           case omp_atv_private:
  86             data.sync_hint = traits[i].value;
  87             break;
  88           default:
  89             return omp_null_allocator;
  90           }
  91         break;
  92       case omp_atk_alignment:
  93         if ((traits[i].value & (traits[i].value - 1)) != 0
  94             || !traits[i].value)
  95           return omp_null_allocator;
  96         data.alignment = traits[i].value;
  97         break;
  98       case omp_atk_access:
  99         switch (traits[i].value)
 100           {
 101           case omp_atv_default:
 102             data.access = omp_atv_all;
 103             break;
 104           case omp_atv_all:
 105           case omp_atv_cgroup:
 106           case omp_atv_pteam:
 107           case omp_atv_thread:
 108             data.access = traits[i].value;
 109             break;
 110           default:
 111             return omp_null_allocator;
 112           }
 113         break;
 114       case omp_atk_pool_size:
 115         data.pool_size = traits[i].value;
 116         break;
 117       case omp_atk_fallback:
 118         switch (traits[i].value)
 119           {
 120           case omp_atv_default:
 121             data.fallback = omp_atv_default_mem_fb;
 122             break;
 123           case omp_atv_default_mem_fb:
 124           case omp_atv_null_fb:
 125           case omp_atv_abort_fb:
 126           case omp_atv_allocator_fb:
 127             data.fallback = traits[i].value;
 128             break;
 129           default:
 130             return omp_null_allocator;
 131           }
 132         break;
 133       case omp_atk_fb_data:
 134         data.fb_data = traits[i].value;
 135         break;
 136       case omp_atk_pinned:
 137         switch (traits[i].value)
 138           {
 139           case omp_atv_default:
 140           case omp_atv_false:
 141             data.pinned = omp_atv_false;
 142             break;
 143           case omp_atv_true:
 144             data.pinned = omp_atv_true;
 145             break;
 146           default:
 147             return omp_null_allocator;
 148           }
 149         break;
 150       case omp_atk_partition:
 151         switch (traits[i].value)
 152           {
 153           case omp_atv_default:
 154             data.partition = omp_atv_environment;
 155             break;
 156           case omp_atv_environment:
 157           case omp_atv_nearest:
 158           case omp_atv_blocked:
 159           case omp_atv_interleaved:
 160             data.partition = traits[i].value;
 161             break;
 162           default:
 163             return omp_null_allocator;
 164           }
 165         break;
 166       default:
 167         return omp_null_allocator;
 168       }
 169
 170   if (data.alignment < sizeof (void *))
 171     data.alignment = sizeof (void *);
 172
 173   /* No support for these so far (for hbw will use memkind).  */
 174   if (data.pinned || data.memspace == omp_high_bw_mem_space)
 175     return omp_null_allocator;
 176
 177   ret = gomp_malloc (sizeof (struct omp_allocator_data));
 178   *ret = data;
 179 #ifndef HAVE_SYNC_BUILTINS
 180   gomp_mutex_init (&ret->lock);
 181 #endif
 182   return (omp_allocator_handle_t) ret;
 183 }
 184
 185 void
 186 omp_destroy_allocator (omp_allocator_handle_t allocator)
 187 {
 188   if (allocator != omp_null_allocator)
 189     {
 190 #ifndef HAVE_SYNC_BUILTINS
 191       gomp_mutex_destroy (&((struct omp_allocator_data *) allocator)->lock);
 192 #endif
 193       free ((void *) allocator);
 194     }
 195 }
 196
 197 void *
 198 omp_alloc (size_t size, omp_allocator_handle_t allocator)
 199 {
 200   struct omp_allocator_data *allocator_data;
 201   size_t alignment, new_size;
 202   void *ptr, *ret;
 203
 204 retry:
 205   if (allocator == omp_null_allocator)
 206     {
 207       struct gomp_thread *thr = gomp_thread ();
 208       if (thr->ts.def_allocator == omp_null_allocator)
 209         thr->ts.def_allocator = gomp_def_allocator;
 210       allocator = (omp_allocator_handle_t) thr->ts.def_allocator;
 211     }
 212
 213   if (allocator > omp_max_predefined_alloc)
 214     {
 215       allocator_data = (struct omp_allocator_data *) allocator;
 216       alignment = allocator_data->alignment;
 217     }
 218   else
 219     {
 220       allocator_data = NULL;
 221       alignment = sizeof (void *);
 222     }
 223
 224   new_size = sizeof (struct omp_mem_header);
 225   if (alignment > sizeof (void *))
 226     new_size += alignment - sizeof (void *);
 227   if (__builtin_add_overflow (size, new_size, &new_size))
 228     goto fail;
 229
 230   if (__builtin_expect (allocator_data
 231                         && allocator_data->pool_size < ~(uintptr_t) 0, 0))
 232     {
 233       uintptr_t used_pool_size;
 234       if (new_size > allocator_data->pool_size)
 235         goto fail;
 236 #ifdef HAVE_SYNC_BUILTINS
 237       used_pool_size = __atomic_load_n (&allocator_data->used_pool_size,
 238                                         MEMMODEL_RELAXED);
 239       do
 240         {
 241           uintptr_t new_pool_size;
 242           if (__builtin_add_overflow (used_pool_size, new_size,
 243                                       &new_pool_size)
 244               || new_pool_size > allocator_data->pool_size)
 245             goto fail;
 246           if (__atomic_compare_exchange_n (&allocator_data->used_pool_size,
 247                                            &used_pool_size, new_pool_size,
 248                                            true, MEMMODEL_RELAXED,
 249                                            MEMMODEL_RELAXED))
 250             break;
 251         }
 252       while (1);
 253 #else
 254       gomp_mutex_lock (&allocator_data->lock);
 255       if (__builtin_add_overflow (allocator_data->used_pool_size, new_size,
 256                                   &used_pool_size)
 257           || used_pool_size > allocator_data->pool_size)
 258         {
 259           gomp_mutex_unlock (&allocator_data->lock);
 260           goto fail;
 261         }
 262       allocator_data->used_pool_size = used_pool_size;
 263       gomp_mutex_unlock (&allocator_data->lock);
 264 #endif
 265       ptr = malloc (new_size);
 266       if (ptr == NULL)
 267         {
 268 #ifdef HAVE_SYNC_BUILTINS
 269           __atomic_add_fetch (&allocator_data->used_pool_size, -new_size,
 270                               MEMMODEL_RELAXED);
 271 #else
 272           gomp_mutex_lock (&allocator_data->lock);
 273           allocator_data->used_pool_size -= new_size;
 274           gomp_mutex_unlock (&allocator_data->lock);
 275 #endif
 276           goto fail;
 277         }
 278     }
 279   else
 280     {
 281       ptr = malloc (new_size);
 282       if (ptr == NULL)
 283         goto fail;
 284     }
 285
 286   if (alignment > sizeof (void *))
 287     ret = (void *) (((uintptr_t) ptr
 288                      + sizeof (struct omp_mem_header)
 289                      + alignment - sizeof (void *)) & ~(alignment - 1));
 290   else
 291     ret = (char *) ptr + sizeof (struct omp_mem_header);
 292   ((struct omp_mem_header *) ret)[-1].ptr = ptr;
 293   ((struct omp_mem_header *) ret)[-1].size = new_size;
 294   ((struct omp_mem_header *) ret)[-1].allocator = allocator;
 295   return ret;
 296
 297 fail:
 298   if (allocator_data)
 299     {
 300       switch (allocator_data->fallback)
 301         {
 302         case omp_atv_default_mem_fb:
 303           if (alignment > sizeof (void *)
 304               || (allocator_data
 305                   && allocator_data->pool_size < ~(uintptr_t) 0))
 306             {
 307               allocator = omp_default_mem_alloc;
 308               goto retry;
 309             }
 310           /* Otherwise, we've already performed default mem allocation
 311              and if that failed, it won't succeed again (unless it was
 312              intermitent.  Return NULL then, as that is the fallback.  */
 313           break;
 314         case omp_atv_null_fb:
 315           break;
 316         default:
 317         case omp_atv_abort_fb:
 318           gomp_fatal ("Out of memory allocating %lu bytes",
 319                       (unsigned long) size);
 320         case omp_atv_allocator_fb:
 321           allocator = allocator_data->fb_data;
 322           goto retry;
 323         }
 324     }
 325   return NULL;
 326 }
 327
 328 void
 329 omp_free (void *ptr, omp_allocator_handle_t allocator)
 330 {
 331   struct omp_mem_header *data;
 332
 333   if (ptr == NULL)
 334     return;
 335   (void) allocator;
 336   data = &((struct omp_mem_header *) ptr)[-1];
 337   if (data->allocator > omp_max_predefined_alloc)
 338     {
 339       struct omp_allocator_data *allocator_data
 340         = (struct omp_allocator_data *) (data->allocator);
 341       if (allocator_data->pool_size < ~(uintptr_t) 0)
 342         {
 343 #ifdef HAVE_SYNC_BUILTINS
 344           __atomic_add_fetch (&allocator_data->used_pool_size, -data->size,
 345                               MEMMODEL_RELAXED);
 346 #else
 347           gomp_mutex_lock (&allocator_data->lock);
 348           allocator_data->used_pool_size -= data->new_size;
 349           gomp_mutex_unlock (&allocator_data->lock);
 350 #endif
 351         }
 352     }
 353   free (data->ptr);
 354 }