src/panfrost/encoder/pan_bo.c

   1 /*
   2  * Copyright 2019 Collabora, Ltd.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors (Collabora):
  24  *   Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
  25  */
  26 #include <errno.h>
  27 #include <stdio.h>
  28 #include <fcntl.h>
  29 #include <xf86drm.h>
  30 #include <pthread.h>
  31 #include "drm-uapi/panfrost_drm.h"
  32
  33 #include "pan_bo.h"
  34
  35 #include "os/os_mman.h"
  36
  37 #include "util/u_inlines.h"
  38 #include "util/u_math.h"
  39
  40 /* This file implements a userspace BO cache. Allocating and freeing
  41  * GPU-visible buffers is very expensive, and even the extra kernel roundtrips
  42  * adds more work than we would like at this point. So caching BOs in userspace
  43  * solves both of these problems and does not require kernel updates.
  44  *
  45  * Cached BOs are sorted into a bucket based on rounding their size down to the
  46  * nearest power-of-two. Each bucket contains a linked list of free panfrost_bo
  47  * objects. Putting a BO into the cache is accomplished by adding it to the
  48  * corresponding bucket. Getting a BO from the cache consists of finding the
  49  * appropriate bucket and sorting. A cache eviction is a kernel-level free of a
  50  * BO and removing it from the bucket. We special case evicting all BOs from
  51  * the cache, since that's what helpful in practice and avoids extra logic
  52  * around the linked list.
  53  */
  54
  55 static struct panfrost_bo *
  56 panfrost_bo_alloc(struct panfrost_device *dev, size_t size,
  57                   uint32_t flags)
  58 {
  59         struct drm_panfrost_create_bo create_bo = { .size = size };
  60         struct panfrost_bo *bo;
  61         int ret;
  62
  63         if (dev->kernel_version->version_major > 1 ||
  64             dev->kernel_version->version_minor >= 1) {
  65                 if (flags & PAN_BO_GROWABLE)
  66                         create_bo.flags |= PANFROST_BO_HEAP;
  67                 if (!(flags & PAN_BO_EXECUTE))
  68                         create_bo.flags |= PANFROST_BO_NOEXEC;
  69         }
  70
  71         ret = drmIoctl(dev->fd, DRM_IOCTL_PANFROST_CREATE_BO, &create_bo);
  72         if (ret) {
  73                 fprintf(stderr, "DRM_IOCTL_PANFROST_CREATE_BO failed: %m\n");
  74                 return NULL;
  75         }
  76
  77         bo = rzalloc(dev->memctx, struct panfrost_bo);
  78         assert(bo);
  79         bo->size = create_bo.size;
  80         bo->gpu = create_bo.offset;
  81         bo->gem_handle = create_bo.handle;
  82         bo->flags = flags;
  83         bo->dev = dev;
  84         return bo;
  85 }
  86
  87 static void
  88 panfrost_bo_free(struct panfrost_bo *bo)
  89 {
  90         struct drm_gem_close gem_close = { .handle = bo->gem_handle };
  91         int ret;
  92
  93         if (bo->cpu && os_munmap((void *) (uintptr_t)bo->cpu, bo->size)) {
  94                 perror("munmap");
  95                 abort();
  96         }
  97
  98         ret = drmIoctl(bo->dev->fd, DRM_IOCTL_GEM_CLOSE, &gem_close);
  99         if (ret) {
 100                 fprintf(stderr, "DRM_IOCTL_GEM_CLOSE failed: %m\n");
 101                 assert(0);
 102         }
 103
 104         ralloc_free(bo);
 105 }
 106
 107 /* Returns true if the BO is ready, false otherwise.
 108  * access_type is encoding the type of access one wants to ensure is done.
 109  * Say you want to make sure all writers are done writing, you should pass
 110  * PAN_BO_ACCESS_WRITE.
 111  * If you want to wait for all users, you should pass PAN_BO_ACCESS_RW.
 112  * PAN_BO_ACCESS_READ would work too as waiting for readers implies
 113  * waiting for writers as well, but we want to make things explicit and waiting
 114  * only for readers is impossible.
 115  */
 116 bool
 117 panfrost_bo_wait(struct panfrost_bo *bo, int64_t timeout_ns,
 118                  uint32_t access_type)
 119 {
 120         struct drm_panfrost_wait_bo req = {
 121                 .handle = bo->gem_handle,
 122                 .timeout_ns = timeout_ns,
 123         };
 124         int ret;
 125
 126         assert(access_type == PAN_BO_ACCESS_WRITE ||
 127                access_type == PAN_BO_ACCESS_RW);
 128
 129         /* If the BO has been exported or imported we can't rely on the cached
 130          * state, we need to call the WAIT_BO ioctl.
 131          */
 132         if (!(bo->flags & (PAN_BO_IMPORTED | PAN_BO_EXPORTED))) {
 133                 /* If ->gpu_access is 0, the BO is idle, no need to wait. */
 134                 if (!bo->gpu_access)
 135                         return true;
 136
 137                 /* If the caller only wants to wait for writers and no
 138                  * writes are pending, we don't have to wait.
 139                  */
 140                 if (access_type == PAN_BO_ACCESS_WRITE &&
 141                     !(bo->gpu_access & PAN_BO_ACCESS_WRITE))
 142                         return true;
 143         }
 144
 145         /* The ioctl returns >= 0 value when the BO we are waiting for is ready
 146          * -1 otherwise.
 147          */
 148         ret = drmIoctl(bo->dev->fd, DRM_IOCTL_PANFROST_WAIT_BO, &req);
 149         if (ret != -1) {
 150                 /* Set gpu_access to 0 so that the next call to bo_wait()
 151                  * doesn't have to call the WAIT_BO ioctl.
 152                  */
 153                 bo->gpu_access = 0;
 154                 return true;
 155         }
 156
 157         /* If errno is not ETIMEDOUT or EBUSY that means the handle we passed
 158          * is invalid, which shouldn't happen here.
 159          */
 160         assert(errno == ETIMEDOUT || errno == EBUSY);
 161         return false;
 162 }
 163
 164 /* Helper to calculate the bucket index of a BO */
 165
 166 static unsigned
 167 pan_bucket_index(unsigned size)
 168 {
 169         /* Round down to POT to compute a bucket index */
 170
 171         unsigned bucket_index = util_logbase2(size);
 172
 173         /* Clamp the bucket index; all huge allocations will be
 174          * sorted into the largest bucket */
 175
 176         bucket_index = MIN2(bucket_index, MAX_BO_CACHE_BUCKET);
 177
 178         /* The minimum bucket size must equal the minimum allocation
 179          * size; the maximum we clamped */
 180
 181         assert(bucket_index >= MIN_BO_CACHE_BUCKET);
 182         assert(bucket_index <= MAX_BO_CACHE_BUCKET);
 183
 184         /* Reindex from 0 */
 185         return (bucket_index - MIN_BO_CACHE_BUCKET);
 186 }
 187
 188 static struct list_head *
 189 pan_bucket(struct panfrost_device *dev, unsigned size)
 190 {
 191         return &dev->bo_cache.buckets[pan_bucket_index(size)];
 192 }
 193
 194 /* Tries to fetch a BO of sufficient size with the appropriate flags from the
 195  * BO cache. If it succeeds, it returns that BO and removes the BO from the
 196  * cache. If it fails, it returns NULL signaling the caller to allocate a new
 197  * BO. */
 198
 199 static struct panfrost_bo *
 200 panfrost_bo_cache_fetch(struct panfrost_device *dev,
 201                         size_t size, uint32_t flags, bool dontwait)
 202 {
 203         pthread_mutex_lock(&dev->bo_cache.lock);
 204         struct list_head *bucket = pan_bucket(dev, size);
 205         struct panfrost_bo *bo = NULL;
 206
 207         /* Iterate the bucket looking for something suitable */
 208         list_for_each_entry_safe(struct panfrost_bo, entry, bucket,
 209                                  bucket_link) {
 210                 if (entry->size < size || entry->flags != flags)
 211                         continue;
 212
 213                 if (!panfrost_bo_wait(entry, dontwait ? 0 : INT64_MAX,
 214                                       PAN_BO_ACCESS_RW))
 215                         continue;
 216
 217                 struct drm_panfrost_madvise madv = {
 218                         .handle = entry->gem_handle,
 219                         .madv = PANFROST_MADV_WILLNEED,
 220                 };
 221                 int ret;
 222
 223                 /* This one works, splice it out of the cache */
 224                 list_del(&entry->bucket_link);
 225                 list_del(&entry->lru_link);
 226
 227                 ret = drmIoctl(dev->fd, DRM_IOCTL_PANFROST_MADVISE, &madv);
 228                 if (!ret && !madv.retained) {
 229                         panfrost_bo_free(entry);
 230                         continue;
 231                 }
 232                 /* Let's go! */
 233                 bo = entry;
 234                 break;
 235         }
 236         pthread_mutex_unlock(&dev->bo_cache.lock);
 237
 238         return bo;
 239 }
 240
 241 static void
 242 panfrost_bo_cache_evict_stale_bos(struct panfrost_device *dev)
 243 {
 244         struct timespec time;
 245
 246         clock_gettime(CLOCK_MONOTONIC, &time);
 247         list_for_each_entry_safe(struct panfrost_bo, entry,
 248                                  &dev->bo_cache.lru, lru_link) {
 249                 /* We want all entries that have been used more than 1 sec
 250                  * ago to be dropped, others can be kept.
 251                  * Note the <= 2 check and not <= 1. It's here to account for
 252                  * the fact that we're only testing ->tv_sec, not ->tv_nsec.
 253                  * That means we might keep entries that are between 1 and 2
 254                  * seconds old, but we don't really care, as long as unused BOs
 255                  * are dropped at some point.
 256                  */
 257                 if (time.tv_sec - entry->last_used <= 2)
 258                         break;
 259
 260                 list_del(&entry->bucket_link);
 261                 list_del(&entry->lru_link);
 262                 panfrost_bo_free(entry);
 263         }
 264 }
 265
 266 /* Tries to add a BO to the cache. Returns if it was
 267  * successful */
 268
 269 static bool
 270 panfrost_bo_cache_put(struct panfrost_bo *bo)
 271 {
 272         struct panfrost_device *dev = bo->dev;
 273
 274         if (bo->flags & PAN_BO_DONT_REUSE)
 275                 return false;
 276
 277         pthread_mutex_lock(&dev->bo_cache.lock);
 278         struct list_head *bucket = pan_bucket(dev, MAX2(bo->size, 4096));
 279         struct drm_panfrost_madvise madv;
 280         struct timespec time;
 281
 282         madv.handle = bo->gem_handle;
 283         madv.madv = PANFROST_MADV_DONTNEED;
 284         madv.retained = 0;
 285
 286         drmIoctl(dev->fd, DRM_IOCTL_PANFROST_MADVISE, &madv);
 287
 288         /* Add us to the bucket */
 289         list_addtail(&bo->bucket_link, bucket);
 290
 291         /* Add us to the LRU list and update the last_used field. */
 292         list_addtail(&bo->lru_link, &dev->bo_cache.lru);
 293         clock_gettime(CLOCK_MONOTONIC, &time);
 294         bo->last_used = time.tv_sec;
 295
 296         /* Let's do some cleanup in the BO cache while we hold the
 297          * lock.
 298          */
 299         panfrost_bo_cache_evict_stale_bos(dev);
 300         pthread_mutex_unlock(&dev->bo_cache.lock);
 301
 302         return true;
 303 }
 304
 305 /* Evicts all BOs from the cache. Called during context
 306  * destroy or during low-memory situations (to free up
 307  * memory that may be unused by us just sitting in our
 308  * cache, but still reserved from the perspective of the
 309  * OS) */
 310
 311 void
 312 panfrost_bo_cache_evict_all(
 313                 struct panfrost_device *dev)
 314 {
 315         pthread_mutex_lock(&dev->bo_cache.lock);
 316         for (unsigned i = 0; i < ARRAY_SIZE(dev->bo_cache.buckets); ++i) {
 317                 struct list_head *bucket = &dev->bo_cache.buckets[i];
 318
 319                 list_for_each_entry_safe(struct panfrost_bo, entry, bucket,
 320                                          bucket_link) {
 321                         list_del(&entry->bucket_link);
 322                         list_del(&entry->lru_link);
 323                         panfrost_bo_free(entry);
 324                 }
 325         }
 326         pthread_mutex_unlock(&dev->bo_cache.lock);
 327 }
 328
 329 void
 330 panfrost_bo_mmap(struct panfrost_bo *bo)
 331 {
 332         struct drm_panfrost_mmap_bo mmap_bo = { .handle = bo->gem_handle };
 333         int ret;
 334
 335         if (bo->cpu)
 336                 return;
 337
 338         ret = drmIoctl(bo->dev->fd, DRM_IOCTL_PANFROST_MMAP_BO, &mmap_bo);
 339         if (ret) {
 340                 fprintf(stderr, "DRM_IOCTL_PANFROST_MMAP_BO failed: %m\n");
 341                 assert(0);
 342         }
 343
 344         bo->cpu = os_mmap(NULL, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
 345                           bo->dev->fd, mmap_bo.offset);
 346         if (bo->cpu == MAP_FAILED) {
 347                 fprintf(stderr, "mmap failed: %p %m\n", bo->cpu);
 348                 assert(0);
 349         }
 350 }
 351
 352 struct panfrost_bo *
 353 panfrost_bo_create(struct panfrost_device *dev, size_t size,
 354                    uint32_t flags)
 355 {
 356         struct panfrost_bo *bo;
 357
 358         /* Kernel will fail (confusingly) with EPERM otherwise */
 359         assert(size > 0);
 360
 361         /* To maximize BO cache usage, don't allocate tiny BOs */
 362         size = MAX2(size, 4096);
 363
 364         /* GROWABLE BOs cannot be mmapped */
 365         if (flags & PAN_BO_GROWABLE)
 366                 assert(flags & PAN_BO_INVISIBLE);
 367
 368         /* Before creating a BO, we first want to check the cache but without
 369          * waiting for BO readiness (BOs in the cache can still be referenced
 370          * by jobs that are not finished yet).
 371          * If the cached allocation fails we fall back on fresh BO allocation,
 372          * and if that fails too, we try one more time to allocate from the
 373          * cache, but this time we accept to wait.
 374          */
 375         bo = panfrost_bo_cache_fetch(dev, size, flags, true);
 376         if (!bo)
 377                 bo = panfrost_bo_alloc(dev, size, flags);
 378         if (!bo)
 379                 bo = panfrost_bo_cache_fetch(dev, size, flags, false);
 380
 381         if (!bo)
 382                 fprintf(stderr, "BO creation failed\n");
 383
 384         assert(bo);
 385
 386         /* Only mmap now if we know we need to. For CPU-invisible buffers, we
 387          * never map since we don't care about their contents; they're purely
 388          * for GPU-internal use. But we do trace them anyway. */
 389
 390         if (!(flags & (PAN_BO_INVISIBLE | PAN_BO_DELAY_MMAP)))
 391                 panfrost_bo_mmap(bo);
 392
 393         p_atomic_set(&bo->refcnt, 1);
 394
 395         pthread_mutex_lock(&dev->active_bos_lock);
 396         _mesa_set_add(bo->dev->active_bos, bo);
 397         pthread_mutex_unlock(&dev->active_bos_lock);
 398
 399         return bo;
 400 }
 401
 402 void
 403 panfrost_bo_reference(struct panfrost_bo *bo)
 404 {
 405         if (bo) {
 406                 ASSERTED int count = p_atomic_inc_return(&bo->refcnt);
 407                 assert(count != 1);
 408         }
 409 }
 410
 411 void
 412 panfrost_bo_unreference(struct panfrost_bo *bo)
 413 {
 414         if (!bo)
 415                 return;
 416
 417         /* Don't return to cache if there are still references */
 418         if (p_atomic_dec_return(&bo->refcnt))
 419                 return;
 420
 421         struct panfrost_device *dev = bo->dev;
 422
 423         pthread_mutex_lock(&dev->active_bos_lock);
 424         /* Someone might have imported this BO while we were waiting for the
 425          * lock, let's make sure it's still not referenced before freeing it.
 426          */
 427         if (p_atomic_read(&bo->refcnt) == 0) {
 428                 _mesa_set_remove_key(bo->dev->active_bos, bo);
 429
 430                 /* Rather than freeing the BO now, we'll cache the BO for later
 431                  * allocations if we're allowed to.
 432                  */
 433                 if (!panfrost_bo_cache_put(bo))
 434                         panfrost_bo_free(bo);
 435         }
 436         pthread_mutex_unlock(&dev->active_bos_lock);
 437 }
 438
 439 struct panfrost_bo *
 440 panfrost_bo_import(struct panfrost_device *dev, int fd)
 441 {
 442         struct panfrost_bo *bo, *newbo = rzalloc(dev->memctx, struct panfrost_bo);
 443         struct drm_panfrost_get_bo_offset get_bo_offset = {0,};
 444         struct set_entry *entry;
 445         ASSERTED int ret;
 446         unsigned gem_handle;
 447
 448         newbo->dev = dev;
 449
 450         ret = drmPrimeFDToHandle(dev->fd, fd, &gem_handle);
 451         assert(!ret);
 452
 453         newbo->gem_handle = gem_handle;
 454
 455         pthread_mutex_lock(&dev->active_bos_lock);
 456         entry = _mesa_set_search_or_add(dev->active_bos, newbo);
 457         assert(entry);
 458         bo = (struct panfrost_bo *)entry->key;
 459         if (newbo == bo) {
 460                 get_bo_offset.handle = gem_handle;
 461                 ret = drmIoctl(dev->fd, DRM_IOCTL_PANFROST_GET_BO_OFFSET, &get_bo_offset);
 462                 assert(!ret);
 463
 464                 newbo->gpu = (mali_ptr) get_bo_offset.offset;
 465                 newbo->size = lseek(fd, 0, SEEK_END);
 466                 newbo->flags |= PAN_BO_DONT_REUSE | PAN_BO_IMPORTED;
 467                 assert(newbo->size > 0);
 468                 p_atomic_set(&newbo->refcnt, 1);
 469                 // TODO map and unmap on demand?
 470                 panfrost_bo_mmap(newbo);
 471         } else {
 472                 ralloc_free(newbo);
 473                 /* bo->refcnt == 0 can happen if the BO
 474                  * was being released but panfrost_bo_import() acquired the
 475                  * lock before panfrost_bo_unreference(). In that case, refcnt
 476                  * is 0 and we can't use panfrost_bo_reference() directly, we
 477                  * have to re-initialize the refcnt().
 478                  * Note that panfrost_bo_unreference() checks
 479                  * refcnt value just after acquiring the lock to
 480                  * make sure the object is not freed if panfrost_bo_import()
 481                  * acquired it in the meantime.
 482                  */
 483                 if (p_atomic_read(&bo->refcnt) == 0)
 484                         p_atomic_set(&newbo->refcnt, 1);
 485                 else
 486                         panfrost_bo_reference(bo);
 487                 assert(bo->cpu);
 488         }
 489         pthread_mutex_unlock(&dev->active_bos_lock);
 490
 491         return bo;
 492 }
 493
 494 int
 495 panfrost_bo_export(struct panfrost_bo *bo)
 496 {
 497         struct drm_prime_handle args = {
 498                 .handle = bo->gem_handle,
 499                 .flags = DRM_CLOEXEC,
 500         };
 501
 502         int ret = drmIoctl(bo->dev->fd, DRM_IOCTL_PRIME_HANDLE_TO_FD, &args);
 503         if (ret == -1)
 504                 return -1;
 505
 506         bo->flags |= PAN_BO_DONT_REUSE | PAN_BO_EXPORTED;
 507         return args.fd;
 508 }
 509