panfrost: Try to evict unused BOs from the cache

The panfrost BO cache can only grow since all newly allocated BOs are
returned to the cache (unless they've been exported).

With the MADVISE ioctl that's not a big issue because the kernel can
come and reclaim this memory, but MADVISE will only be available on 5.4
kernels. This means an app can currently allocate a lot memory without
ever releasing it, leading to some situations where the OOM-killer kicks
in and kills the app (or even worse, kills another process consuming
more memory than the GL app) to get some of this memory back.

Let's try to limit the amount of BOs we keep in the cache by evicting
entries that have not been used for more than one second (if the app
stopped allocating BOs of this size, it's likely to not allocate
similar BOs in a near future).

This solution is based on the VC4/V3D implementation.

Signed-off-by: Boris Brezillon <boris.brezillon@collabora.com>
Reviewed-by: Alyssa Rosenzweig <alyssa.rosenzweig@collabora.com>

diff --git a/src/gallium/drivers/panfrost/pan_bo.c b/src/gallium/drivers/panfrost/pan_bo.c
index 1fe7e225b8a74890d92c96ddd1d2cf11f6264e5d..1a945daf23727145eda6d81e283fd177014db48d 100644 (file)
--- a/src/gallium/drivers/panfrost/pan_bo.c
+++ b/src/gallium/drivers/panfrost/pan_bo.c
@@ -203,7 +203,8 @@ panfrost_bo_cache_fetch(struct panfrost_screen *screen,
         struct panfrost_bo *bo = NULL;
 
         /* Iterate the bucket looking for something suitable */
-        list_for_each_entry_safe(struct panfrost_bo, entry, bucket, link) {
+        list_for_each_entry_safe(struct panfrost_bo, entry, bucket,
+                                 bucket_link) {
                 if (entry->size < size || entry->flags != flags)
                         continue;
 
@@ -218,7 +219,8 @@ panfrost_bo_cache_fetch(struct panfrost_screen *screen,
                 int ret;
 
                 /* This one works, splice it out of the cache */
-                list_del(&entry->link);
+                list_del(&entry->bucket_link);
+                list_del(&entry->lru_link);
 
                 ret = drmIoctl(screen->fd, DRM_IOCTL_PANFROST_MADVISE, &madv);
                 if (!ret && !madv.retained) {
@@ -234,6 +236,31 @@ panfrost_bo_cache_fetch(struct panfrost_screen *screen,
         return bo;
 }
 
+static void
+panfrost_bo_cache_evict_stale_bos(struct panfrost_screen *screen)
+{
+        struct timespec time;
+
+        clock_gettime(CLOCK_MONOTONIC, &time);
+        list_for_each_entry_safe(struct panfrost_bo, entry,
+                                 &screen->bo_cache.lru, lru_link) {
+                /* We want all entries that have been used more than 1 sec
+                 * ago to be dropped, others can be kept.
+                 * Note the <= 2 check and not <= 1. It's here to account for
+                 * the fact that we're only testing ->tv_sec, not ->tv_nsec.
+                 * That means we might keep entries that are between 1 and 2
+                 * seconds old, but we don't really care, as long as unused BOs
+                 * are dropped at some point.
+                 */
+                if (time.tv_sec - entry->last_used <= 2)
+                        break;
+
+                list_del(&entry->bucket_link);
+                list_del(&entry->lru_link);
+                panfrost_bo_free(entry);
+        }
+}
+
 /* Tries to add a BO to the cache. Returns if it was
  * successful */
 
@@ -248,6 +275,7 @@ panfrost_bo_cache_put(struct panfrost_bo *bo)
         pthread_mutex_lock(&screen->bo_cache.lock);
         struct list_head *bucket = pan_bucket(screen, bo->size);
         struct drm_panfrost_madvise madv;
+        struct timespec time;
 
         madv.handle = bo->gem_handle;
         madv.madv = PANFROST_MADV_DONTNEED;
@@ -256,7 +284,17 @@ panfrost_bo_cache_put(struct panfrost_bo *bo)
         drmIoctl(screen->fd, DRM_IOCTL_PANFROST_MADVISE, &madv);
 
         /* Add us to the bucket */
-        list_addtail(&bo->link, bucket);
+        list_addtail(&bo->bucket_link, bucket);
+
+        /* Add us to the LRU list and update the last_used field. */
+        list_addtail(&bo->lru_link, &screen->bo_cache.lru);
+        clock_gettime(CLOCK_MONOTONIC, &time);
+        bo->last_used = time.tv_sec;
+
+        /* Let's do some cleanup in the BO cache while we hold the
+         * lock.
+         */
+        panfrost_bo_cache_evict_stale_bos(screen);
         pthread_mutex_unlock(&screen->bo_cache.lock);
 
         return true;
@@ -276,8 +314,10 @@ panfrost_bo_cache_evict_all(
         for (unsigned i = 0; i < ARRAY_SIZE(screen->bo_cache.buckets); ++i) {
                 struct list_head *bucket = &screen->bo_cache.buckets[i];
 
-                list_for_each_entry_safe(struct panfrost_bo, entry, bucket, link) {
-                        list_del(&entry->link);
+                list_for_each_entry_safe(struct panfrost_bo, entry, bucket,
+                                         bucket_link) {
+                        list_del(&entry->bucket_link);
+                        list_del(&entry->lru_link);
                         panfrost_bo_free(entry);
                 }
         }

diff --git a/src/gallium/drivers/panfrost/pan_bo.h b/src/gallium/drivers/panfrost/pan_bo.h
index 78e9b7526e01a0c3ebcd3872aba72b86a1a0b7f8..414c356b95c195a0816a2dcbbdf8c3e8e6995357 100644 (file)
--- a/src/gallium/drivers/panfrost/pan_bo.h
+++ b/src/gallium/drivers/panfrost/pan_bo.h
@@ -82,7 +82,15 @@ struct panfrost_screen;
 
 struct panfrost_bo {
         /* Must be first for casting */
-        struct list_head link;
+        struct list_head bucket_link;
+
+        /* Used to link the BO to the BO cache LRU list. */
+        struct list_head lru_link;
+
+        /* Store the time this BO was use last, so the BO cache logic can evict
+         * stale BOs.
+         */
+        time_t last_used;
 
         struct pipe_reference reference;
 

diff --git a/src/gallium/drivers/panfrost/pan_screen.c b/src/gallium/drivers/panfrost/pan_screen.c
index 9e98cf3375c9b569ba670fa7f918a566b7bdbb18..c9c50e4ef18a2e2d57d7a994c13a5dea838e2346 100644 (file)
--- a/src/gallium/drivers/panfrost/pan_screen.c
+++ b/src/gallium/drivers/panfrost/pan_screen.c
@@ -755,6 +755,7 @@ panfrost_create_screen(int fd, struct renderonly *ro)
                                               panfrost_active_bos_cmp);
 
         pthread_mutex_init(&screen->bo_cache.lock, NULL);
+        list_inithead(&screen->bo_cache.lru);
         for (unsigned i = 0; i < ARRAY_SIZE(screen->bo_cache.buckets); ++i)
                 list_inithead(&screen->bo_cache.buckets[i]);
 

diff --git a/src/gallium/drivers/panfrost/pan_screen.h b/src/gallium/drivers/panfrost/pan_screen.h
index dde745fb0ed2f7b4c58f01d19b1c47b4733632b2..c864629a3721bccd0ca129a103ad0fcf99538432 100644 (file)
--- a/src/gallium/drivers/panfrost/pan_screen.h
+++ b/src/gallium/drivers/panfrost/pan_screen.h
@@ -92,6 +92,12 @@ struct panfrost_screen {
         struct {
                 pthread_mutex_t lock;
 
+                /* List containing all cached BOs sorted in LRU (Least
+                 * Recently Used) order. This allows us to quickly evict BOs
+                 * that are more than 1 second old.
+                 */
+                struct list_head lru;
+
                 /* The BO cache is a set of buckets with power-of-two sizes
                  * ranging from 2^12 (4096, the page size) to
                  * 2^(12 + MAX_BO_CACHE_BUCKETS).