src/gallium/drivers/panfrost/pan_bo_cache.c

   1 /*
   2  * Copyright 2019 Collabora, Ltd.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  21  * SOFTWARE.
  22  *
  23  * Authors (Collabora):
  24  *   Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>
  25  */
  26 #include <xf86drm.h>
  27 #include <pthread.h>
  28 #include "drm-uapi/panfrost_drm.h"
  29
  30 #include "pan_screen.h"
  31 #include "util/u_math.h"
  32
  33 /* This file implements a userspace BO cache. Allocating and freeing
  34  * GPU-visible buffers is very expensive, and even the extra kernel roundtrips
  35  * adds more work than we would like at this point. So caching BOs in userspace
  36  * solves both of these problems and does not require kernel updates.
  37  *
  38  * Cached BOs are sorted into a bucket based on rounding their size down to the
  39  * nearest power-of-two. Each bucket contains a linked list of free panfrost_bo
  40  * objects. Putting a BO into the cache is accomplished by adding it to the
  41  * corresponding bucket. Getting a BO from the cache consists of finding the
  42  * appropriate bucket and sorting. A cache eviction is a kernel-level free of a
  43  * BO and removing it from the bucket. We special case evicting all BOs from
  44  * the cache, since that's what helpful in practice and avoids extra logic
  45  * around the linked list.
  46  */
  47
  48 /* Helper to calculate the bucket index of a BO */
  49
  50 static unsigned
  51 pan_bucket_index(unsigned size)
  52 {
  53         /* Round down to POT to compute a bucket index */
  54
  55         unsigned bucket_index = util_logbase2(size);
  56
  57         /* Clamp the bucket index; all huge allocations will be
  58          * sorted into the largest bucket */
  59
  60         bucket_index = MIN2(bucket_index, MAX_BO_CACHE_BUCKET);
  61
  62         /* The minimum bucket size must equal the minimum allocation
  63          * size; the maximum we clamped */
  64
  65         assert(bucket_index >= MIN_BO_CACHE_BUCKET);
  66         assert(bucket_index <= MAX_BO_CACHE_BUCKET);
  67
  68         /* Reindex from 0 */
  69         return (bucket_index - MIN_BO_CACHE_BUCKET);
  70 }
  71
  72 static struct list_head *
  73 pan_bucket(struct panfrost_screen *screen, unsigned size)
  74 {
  75         return &screen->bo_cache[pan_bucket_index(size)];
  76 }
  77
  78 /* Tries to fetch a BO of sufficient size with the appropriate flags from the
  79  * BO cache. If it succeeds, it returns that BO and removes the BO from the
  80  * cache. If it fails, it returns NULL signaling the caller to allocate a new
  81  * BO. */
  82
  83 struct panfrost_bo *
  84 panfrost_bo_cache_fetch(
  85                 struct panfrost_screen *screen,
  86                 size_t size, uint32_t flags)
  87 {
  88         pthread_mutex_lock(&screen->bo_cache_lock);
  89         struct list_head *bucket = pan_bucket(screen, size);
  90         struct panfrost_bo *bo = NULL;
  91
  92         /* Iterate the bucket looking for something suitable */
  93         list_for_each_entry_safe(struct panfrost_bo, entry, bucket, link) {
  94                 if (entry->size >= size &&
  95                     entry->flags == flags) {
  96                         int ret;
  97                         struct drm_panfrost_madvise madv;
  98
  99                         /* This one works, splice it out of the cache */
 100                         list_del(&entry->link);
 101
 102                         madv.handle = entry->gem_handle;
 103                         madv.madv = PANFROST_MADV_WILLNEED;
 104                         madv.retained = 0;
 105
 106                         ret = drmIoctl(screen->fd, DRM_IOCTL_PANFROST_MADVISE, &madv);
 107                         if (!ret && !madv.retained) {
 108                                 panfrost_drm_release_bo(screen, entry, false);
 109                                 continue;
 110                         }
 111                         /* Let's go! */
 112                         bo = entry;
 113                         break;
 114                 }
 115         }
 116         pthread_mutex_unlock(&screen->bo_cache_lock);
 117
 118         return bo;
 119 }
 120
 121 /* Tries to add a BO to the cache. Returns if it was
 122  * successful */
 123
 124 bool
 125 panfrost_bo_cache_put(
 126                 struct panfrost_screen *screen,
 127                 struct panfrost_bo *bo)
 128 {
 129         pthread_mutex_lock(&screen->bo_cache_lock);
 130         struct list_head *bucket = pan_bucket(screen, bo->size);
 131         struct drm_panfrost_madvise madv;
 132
 133         madv.handle = bo->gem_handle;
 134         madv.madv = PANFROST_MADV_DONTNEED;
 135         madv.retained = 0;
 136
 137         drmIoctl(screen->fd, DRM_IOCTL_PANFROST_MADVISE, &madv);
 138
 139         /* Add us to the bucket */
 140         list_addtail(&bo->link, bucket);
 141         pthread_mutex_unlock(&screen->bo_cache_lock);
 142
 143         return true;
 144 }
 145
 146 /* Evicts all BOs from the cache. Called during context
 147  * destroy or during low-memory situations (to free up
 148  * memory that may be unused by us just sitting in our
 149  * cache, but still reserved from the perspective of the
 150  * OS) */
 151
 152 void
 153 panfrost_bo_cache_evict_all(
 154                 struct panfrost_screen *screen)
 155 {
 156         pthread_mutex_lock(&screen->bo_cache_lock);
 157         for (unsigned i = 0; i < ARRAY_SIZE(screen->bo_cache); ++i) {
 158                 struct list_head *bucket = &screen->bo_cache[i];
 159
 160                 list_for_each_entry_safe(struct panfrost_bo, entry, bucket, link) {
 161                         list_del(&entry->link);
 162                         panfrost_drm_release_bo(screen, entry, false);
 163                 }
 164         }
 165         pthread_mutex_unlock(&screen->bo_cache_lock);
 166 }
 167