X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fdrivers%2Firis%2Firis_bufmgr.c;h=5529d3767dac3145fd4aa27dceb804de9289ba21;hb=4de678cd30fa165f1e10d70c147d81c6d186e15a;hp=6531118d1b0a35eff503220565845ff2af053448;hpb=53878f7a8989879b0f3ca37df9fd1fb37f2525ca;p=mesa.git

diff --git a/src/gallium/drivers/iris/iris_bufmgr.c b/src/gallium/drivers/iris/iris_bufmgr.c
index 6531118d1b0..5529d3767da 100644
--- a/src/gallium/drivers/iris/iris_bufmgr.c
+++ b/src/gallium/drivers/iris/iris_bufmgr.c
@@ -31,10 +31,6 @@
  * - main interface to GEM in the kernel
  */
 
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-
 #include <xf86drm.h>
 #include <util/u_atomic.h>
 #include <fcntl.h>
@@ -49,20 +45,21 @@
 #include <sys/types.h>
 #include <stdbool.h>
 #include <time.h>
+#include <unistd.h>
 
 #include "errno.h"
-#ifndef ETIME
-#define ETIME ETIMEDOUT
-#endif
+#include "common/gen_aux_map.h"
 #include "common/gen_clflush.h"
 #include "dev/gen_debug.h"
 #include "common/gen_gem.h"
 #include "dev/gen_device_info.h"
 #include "main/macros.h"
+#include "os/os_mman.h"
 #include "util/debug.h"
 #include "util/macros.h"
 #include "util/hash_table.h"
 #include "util/list.h"
+#include "util/os_file.h"
 #include "util/u_dynarray.h"
 #include "util/vma.h"
 #include "iris_bufmgr.h"
@@ -91,21 +88,18 @@
 
 #define PAGE_SIZE 4096
 
-#define FILE_DEBUG_FLAG DEBUG_BUFMGR
-
-/**
- * Call ioctl, restarting if it is interupted
- */
-int
-drm_ioctl(int fd, unsigned long request, void *arg)
-{
-    int ret;
+#define WARN_ONCE(cond, fmt...) do {                            \
+   if (unlikely(cond)) {                                        \
+      static bool _warned = false;                              \
+      if (!_warned) {                                           \
+         fprintf(stderr, "WARNING: ");                          \
+         fprintf(stderr, fmt);                                  \
+         _warned = true;                                        \
+      }                                                         \
+   }                                                            \
+} while (0)
 
-    do {
-        ret = ioctl(fd, request, arg);
-    } while (ret == -1 && (errno == EINTR || errno == EAGAIN));
-    return ret;
-}
+#define FILE_DEBUG_FLAG DEBUG_BUFMGR
 
 static inline int
 atomic_add_unless(int *v, int add, int unless)
@@ -140,7 +134,24 @@ struct bo_cache_bucket {
    uint64_t size;
 };
 
+struct bo_export {
+   /** File descriptor associated with a handle export. */
+   int drm_fd;
+
+   /** GEM handle in drm_fd */
+   uint32_t gem_handle;
+
+   struct list_head link;
+};
+
 struct iris_bufmgr {
+   /**
+    * List into the list of bufmgr.
+    */
+   struct list_head link;
+
+   uint32_t refcount;
+
    int fd;
 
    mtx_t lock;
@@ -153,10 +164,25 @@ struct iris_bufmgr {
    struct hash_table *name_table;
    struct hash_table *handle_table;
 
+   /**
+    * List of BOs which we've effectively freed, but are hanging on to
+    * until they're idle before closing and returning the VMA.
+    */
+   struct list_head zombie_list;
+
    struct util_vma_heap vma_allocator[IRIS_MEMZONE_COUNT];
 
    bool has_llc:1;
+   bool has_mmap_offset:1;
    bool bo_reuse:1;
+
+   struct gen_aux_map_context *aux_map_ctx;
+};
+
+static mtx_t global_bufmgr_list_mutex = _MTX_INITIALIZER_NP;
+static struct list_head global_bufmgr_list = {
+   .next = &global_bufmgr_list,
+   .prev = &global_bufmgr_list,
 };
 
 static int bo_set_tiling_internal(struct iris_bo *bo, uint32_t tiling_mode,
@@ -164,27 +190,28 @@ static int bo_set_tiling_internal(struct iris_bo *bo, uint32_t tiling_mode,
 
 static void bo_free(struct iris_bo *bo);
 
-static uint64_t vma_alloc(struct iris_bufmgr *bufmgr,
-                          enum iris_memory_zone memzone,
-                          uint64_t size, uint64_t alignment);
-
-static uint32_t
-key_hash_uint(const void *key)
-{
-   return _mesa_hash_data(key, 4);
-}
-
-static bool
-key_uint_equal(const void *a, const void *b)
-{
-   return *((unsigned *) a) == *((unsigned *) b);
-}
-
 static struct iris_bo *
-hash_find_bo(struct hash_table *ht, unsigned int key)
+find_and_ref_external_bo(struct hash_table *ht, unsigned int key)
 {
    struct hash_entry *entry = _mesa_hash_table_search(ht, &key);
-   return entry ? (struct iris_bo *) entry->data : NULL;
+   struct iris_bo *bo = entry ? entry->data : NULL;
+
+   if (bo) {
+      assert(bo->external);
+      assert(!bo->reusable);
+
+      /* Being non-reusable, the BO cannot be in the cache lists, but it
+       * may be in the zombie list if it had reached zero references, but
+       * we hadn't yet closed it...and then reimported the same BO.  If it
+       * is, then remove it since it's now been resurrected.
+       */
+      if (bo->head.prev || bo->head.next)
+         list_del(&bo->head);
+
+      iris_bo_reference(bo);
+   }
+
+   return bo;
 }
 
 /**
@@ -314,7 +341,7 @@ iris_bo_busy(struct iris_bo *bo)
    struct iris_bufmgr *bufmgr = bo->bufmgr;
    struct drm_i915_gem_busy busy = { .handle = bo->gem_handle };
 
-   int ret = drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_BUSY, &busy);
+   int ret = gen_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_BUSY, &busy);
    if (ret == 0) {
       bo->idle = !busy.busy;
       return busy.busy;
@@ -331,7 +358,7 @@ iris_bo_madvise(struct iris_bo *bo, int state)
       .retained = 1,
    };
 
-   drm_ioctl(bo->bufmgr->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv);
+   gen_ioctl(bo->bufmgr->fd, DRM_IOCTL_I915_GEM_MADVISE, &madv);
 
    return madv.retained;
 }
@@ -340,15 +367,20 @@ static struct iris_bo *
 bo_calloc(void)
 {
    struct iris_bo *bo = calloc(1, sizeof(*bo));
-   if (bo) {
-      bo->hash = _mesa_hash_pointer(bo);
-   }
+   if (!bo)
+      return NULL;
+
+   list_inithead(&bo->exports);
+
+   bo->hash = _mesa_hash_pointer(bo);
+
    return bo;
 }
 
 static struct iris_bo *
 alloc_bo_from_cache(struct iris_bufmgr *bufmgr,
                     struct bo_cache_bucket *bucket,
+                    uint32_t alignment,
                     enum iris_memory_zone memzone,
                     unsigned flags,
                     bool match_zone)
@@ -385,10 +417,25 @@ alloc_bo_from_cache(struct iris_bufmgr *bufmgr,
    if (!bo)
       return NULL;
 
-   /* If the cached BO isn't in the right memory zone, free the old
-    * memory and assign it a new address.
+   if (bo->aux_map_address) {
+      /* This buffer was associated with an aux-buffer range. We make sure
+       * that buffers are not reused from the cache while the buffer is (busy)
+       * being used by an executing batch. Since we are here, the buffer is no
+       * longer being used by a batch and the buffer was deleted (in order to
+       * end up in the cache). Therefore its old aux-buffer range can be
+       * removed from the aux-map.
+       */
+      if (bo->bufmgr->aux_map_ctx)
+         gen_aux_map_unmap_range(bo->bufmgr->aux_map_ctx, bo->gtt_offset,
+                                 bo->size);
+      bo->aux_map_address = 0;
+   }
+
+   /* If the cached BO isn't in the right memory zone, or the alignment
+    * isn't sufficient, free the old memory and assign it a new address.
     */
-   if (memzone != iris_memzone_for_address(bo->gtt_offset)) {
+   if (memzone != iris_memzone_for_address(bo->gtt_offset) ||
+       bo->gtt_offset % alignment != 0) {
       vma_free(bufmgr, bo->gtt_offset, bo->size);
       bo->gtt_offset = 0ull;
    }
@@ -421,7 +468,7 @@ alloc_fresh_bo(struct iris_bufmgr *bufmgr, uint64_t bo_size)
    /* All new BOs we get from the kernel are zeroed, so we don't need to
     * worry about that here.
     */
-   if (drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_CREATE, &create) != 0) {
+   if (gen_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_CREATE, &create) != 0) {
       free(bo);
       return NULL;
    }
@@ -431,9 +478,23 @@ alloc_fresh_bo(struct iris_bufmgr *bufmgr, uint64_t bo_size)
    bo->size = bo_size;
    bo->idle = true;
    bo->tiling_mode = I915_TILING_NONE;
-   bo->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
    bo->stride = 0;
 
+   /* Calling set_domain() will allocate pages for the BO outside of the
+    * struct mutex lock in the kernel, which is more efficient than waiting
+    * to create them during the first execbuf that uses the BO.
+    */
+   struct drm_i915_gem_set_domain sd = {
+      .handle = bo->gem_handle,
+      .read_domains = I915_GEM_DOMAIN_CPU,
+      .write_domain = 0,
+   };
+
+   if (gen_ioctl(bo->bufmgr->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &sd) != 0) {
+      bo_free(bo);
+      return NULL;
+   }
+
    return bo;
 }
 
@@ -441,6 +502,7 @@ static struct iris_bo *
 bo_alloc_internal(struct iris_bufmgr *bufmgr,
                   const char *name,
                   uint64_t size,
+                  uint32_t alignment,
                   enum iris_memory_zone memzone,
                   unsigned flags,
                   uint32_t tiling_mode,
@@ -448,7 +510,6 @@ bo_alloc_internal(struct iris_bufmgr *bufmgr,
 {
    struct iris_bo *bo;
    unsigned int page_size = getpagesize();
-   bool alloc_pages = false;
    struct bo_cache_bucket *bucket = bucket_for_size(bufmgr, size);
 
    /* Round the size up to the bucket size, or if we don't have caching
@@ -462,21 +523,26 @@ bo_alloc_internal(struct iris_bufmgr *bufmgr,
    /* Get a buffer out of the cache if available.  First, we try to find
     * one with a matching memory zone so we can avoid reallocating VMA.
     */
-   bo = alloc_bo_from_cache(bufmgr, bucket, memzone, flags, true);
+   bo = alloc_bo_from_cache(bufmgr, bucket, alignment, memzone, flags, true);
 
    /* If that fails, we try for any cached BO, without matching memzone. */
-   if (!bo)
-      bo = alloc_bo_from_cache(bufmgr, bucket, memzone, flags, false);
+   if (!bo) {
+      bo = alloc_bo_from_cache(bufmgr, bucket, alignment, memzone, flags,
+                               false);
+   }
+
+   mtx_unlock(&bufmgr->lock);
 
    if (!bo) {
-      alloc_pages = true;
       bo = alloc_fresh_bo(bufmgr, bo_size);
       if (!bo)
-         goto err;
+         return NULL;
    }
 
    if (bo->gtt_offset == 0ull) {
-      bo->gtt_offset = vma_alloc(bufmgr, memzone, bo->size, 1);
+      mtx_lock(&bufmgr->lock);
+      bo->gtt_offset = vma_alloc(bufmgr, memzone, bo->size, alignment);
+      mtx_unlock(&bufmgr->lock);
 
       if (bo->gtt_offset == 0ull)
          goto err_free;
@@ -485,23 +551,6 @@ bo_alloc_internal(struct iris_bufmgr *bufmgr,
    if (bo_set_tiling_internal(bo, tiling_mode, stride))
       goto err_free;
 
-   if (alloc_pages) {
-      /* Calling set_domain() will allocate pages for the BO outside of the
-       * struct mutex lock in the kernel, which is more efficient than waiting
-       * to create them during the first execbuf that uses the BO.
-       */
-      struct drm_i915_gem_set_domain sd = {
-         .handle = bo->gem_handle,
-         .read_domains = I915_GEM_DOMAIN_CPU,
-         .write_domain = 0,
-      };
-
-      if (drm_ioctl(bo->bufmgr->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &sd) != 0)
-         goto err_free;
-   }
-
-   mtx_unlock(&bufmgr->lock);
-
    bo->name = name;
    p_atomic_set(&bo->refcount, 1);
    bo->reusable = bucket && bufmgr->bo_reuse;
@@ -520,7 +569,7 @@ bo_alloc_internal(struct iris_bufmgr *bufmgr,
          .handle = bo->gem_handle,
          .caching = 1,
       };
-      if (drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_SET_CACHING, &arg) == 0) {
+      if (gen_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_SET_CACHING, &arg) == 0) {
          bo->cache_coherent = true;
          bo->reusable = false;
       }
@@ -533,8 +582,6 @@ bo_alloc_internal(struct iris_bufmgr *bufmgr,
 
 err_free:
    bo_free(bo);
-err:
-   mtx_unlock(&bufmgr->lock);
    return NULL;
 }
 
@@ -544,16 +591,17 @@ iris_bo_alloc(struct iris_bufmgr *bufmgr,
               uint64_t size,
               enum iris_memory_zone memzone)
 {
-   return bo_alloc_internal(bufmgr, name, size, memzone,
+   return bo_alloc_internal(bufmgr, name, size, 1, memzone,
                             0, I915_TILING_NONE, 0);
 }
 
 struct iris_bo *
 iris_bo_alloc_tiled(struct iris_bufmgr *bufmgr, const char *name,
-                    uint64_t size, enum iris_memory_zone memzone,
+                    uint64_t size, uint32_t alignment,
+                    enum iris_memory_zone memzone,
                     uint32_t tiling_mode, uint32_t pitch, unsigned flags)
 {
-   return bo_alloc_internal(bufmgr, name, size, memzone,
+   return bo_alloc_internal(bufmgr, name, size, alignment, memzone,
                             flags, tiling_mode, pitch);
 }
 
@@ -562,6 +610,7 @@ iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name,
                        void *ptr, size_t size,
                        enum iris_memory_zone memzone)
 {
+   struct drm_gem_close close = { 0, };
    struct iris_bo *bo;
 
    bo = bo_calloc();
@@ -572,7 +621,7 @@ iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name,
       .user_ptr = (uintptr_t)ptr,
       .user_size = size,
    };
-   if (drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_USERPTR, &arg))
+   if (gen_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_USERPTR, &arg))
       goto err_free;
    bo->gem_handle = arg.handle;
 
@@ -581,7 +630,7 @@ iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name,
       .handle = bo->gem_handle,
       .read_domains = I915_GEM_DOMAIN_CPU,
    };
-   if (drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &sd))
+   if (gen_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_SET_DOMAIN, &sd))
       goto err_close;
 
    bo->name = name;
@@ -607,7 +656,8 @@ iris_bo_create_userptr(struct iris_bufmgr *bufmgr, const char *name,
    return bo;
 
 err_close:
-   drm_ioctl(bufmgr->fd, DRM_IOCTL_GEM_CLOSE, &bo->gem_handle);
+   close.handle = bo->gem_handle;
+   gen_ioctl(bufmgr->fd, DRM_IOCTL_GEM_CLOSE, &close);
 err_free:
    free(bo);
    return NULL;
@@ -632,14 +682,12 @@ iris_bo_gem_create_from_name(struct iris_bufmgr *bufmgr,
     * provides a sufficiently fast match.
     */
    mtx_lock(&bufmgr->lock);
-   bo = hash_find_bo(bufmgr->name_table, handle);
-   if (bo) {
-      iris_bo_reference(bo);
+   bo = find_and_ref_external_bo(bufmgr->name_table, handle);
+   if (bo)
       goto out;
-   }
 
    struct drm_gem_open open_arg = { .name = handle };
-   int ret = drm_ioctl(bufmgr->fd, DRM_IOCTL_GEM_OPEN, &open_arg);
+   int ret = gen_ioctl(bufmgr->fd, DRM_IOCTL_GEM_OPEN, &open_arg);
    if (ret != 0) {
       DBG("Couldn't reference %s handle 0x%08x: %s\n",
           name, handle, strerror(errno));
@@ -650,11 +698,9 @@ iris_bo_gem_create_from_name(struct iris_bufmgr *bufmgr,
     * object from the kernel before by looking through the list
     * again for a matching gem_handle
     */
-   bo = hash_find_bo(bufmgr->handle_table, open_arg.handle);
-   if (bo) {
-      iris_bo_reference(bo);
+   bo = find_and_ref_external_bo(bufmgr->handle_table, open_arg.handle);
+   if (bo)
       goto out;
-   }
 
    bo = bo_calloc();
    if (!bo)
@@ -663,7 +709,6 @@ iris_bo_gem_create_from_name(struct iris_bufmgr *bufmgr,
    p_atomic_set(&bo->refcount, 1);
 
    bo->size = open_arg.size;
-   bo->gtt_offset = 0;
    bo->bufmgr = bufmgr;
    bo->gem_handle = open_arg.handle;
    bo->name = name;
@@ -677,12 +722,12 @@ iris_bo_gem_create_from_name(struct iris_bufmgr *bufmgr,
    _mesa_hash_table_insert(bufmgr->name_table, &bo->global_name, bo);
 
    struct drm_i915_gem_get_tiling get_tiling = { .handle = bo->gem_handle };
-   ret = drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_GET_TILING, &get_tiling);
+   ret = gen_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_GET_TILING, &get_tiling);
    if (ret != 0)
       goto err_unref;
 
    bo->tiling_mode = get_tiling.tiling_mode;
-   bo->swizzle_mode = get_tiling.swizzle_mode;
+
    /* XXX stride is unknown */
    DBG("bo_create_from_handle: %d (%s)\n", handle, bo->name);
 
@@ -697,23 +742,10 @@ err_unref:
 }
 
 static void
-bo_free(struct iris_bo *bo)
+bo_close(struct iris_bo *bo)
 {
    struct iris_bufmgr *bufmgr = bo->bufmgr;
 
-   if (bo->map_cpu && !bo->userptr) {
-      VG_NOACCESS(bo->map_cpu, bo->size);
-      munmap(bo->map_cpu, bo->size);
-   }
-   if (bo->map_wc) {
-      VG_NOACCESS(bo->map_wc, bo->size);
-      munmap(bo->map_wc, bo->size);
-   }
-   if (bo->map_gtt) {
-      VG_NOACCESS(bo->map_gtt, bo->size);
-      munmap(bo->map_gtt, bo->size);
-   }
-
    if (bo->external) {
       struct hash_entry *entry;
 
@@ -724,21 +756,65 @@ bo_free(struct iris_bo *bo)
 
       entry = _mesa_hash_table_search(bufmgr->handle_table, &bo->gem_handle);
       _mesa_hash_table_remove(bufmgr->handle_table, entry);
+
+      list_for_each_entry_safe(struct bo_export, export, &bo->exports, link) {
+         struct drm_gem_close close = { .handle = export->gem_handle };
+         gen_ioctl(export->drm_fd, DRM_IOCTL_GEM_CLOSE, &close);
+
+         list_del(&export->link);
+         free(export);
+      }
+   } else {
+      assert(list_is_empty(&bo->exports));
    }
 
    /* Close this object */
    struct drm_gem_close close = { .handle = bo->gem_handle };
-   int ret = drm_ioctl(bufmgr->fd, DRM_IOCTL_GEM_CLOSE, &close);
+   int ret = gen_ioctl(bufmgr->fd, DRM_IOCTL_GEM_CLOSE, &close);
    if (ret != 0) {
       DBG("DRM_IOCTL_GEM_CLOSE %d failed (%s): %s\n",
           bo->gem_handle, bo->name, strerror(errno));
    }
 
+   if (bo->aux_map_address && bo->bufmgr->aux_map_ctx) {
+      gen_aux_map_unmap_range(bo->bufmgr->aux_map_ctx, bo->gtt_offset,
+                              bo->size);
+   }
+
+   /* Return the VMA for reuse */
    vma_free(bo->bufmgr, bo->gtt_offset, bo->size);
 
    free(bo);
 }
 
+static void
+bo_free(struct iris_bo *bo)
+{
+   struct iris_bufmgr *bufmgr = bo->bufmgr;
+
+   if (bo->map_cpu && !bo->userptr) {
+      VG_NOACCESS(bo->map_cpu, bo->size);
+      os_munmap(bo->map_cpu, bo->size);
+   }
+   if (bo->map_wc) {
+      VG_NOACCESS(bo->map_wc, bo->size);
+      os_munmap(bo->map_wc, bo->size);
+   }
+   if (bo->map_gtt) {
+      VG_NOACCESS(bo->map_gtt, bo->size);
+      os_munmap(bo->map_gtt, bo->size);
+   }
+
+   if (bo->idle) {
+      bo_close(bo);
+   } else {
+      /* Defer closing the GEM BO and returning the VMA for reuse until the
+       * BO is idle.  Just move it to the dead list for now.
+       */
+      list_addtail(&bo->head, &bufmgr->zombie_list);
+   }
+}
+
 /** Frees all cached buffers significantly older than @time. */
 static void
 cleanup_bo_cache(struct iris_bufmgr *bufmgr, time_t time)
@@ -761,6 +837,17 @@ cleanup_bo_cache(struct iris_bufmgr *bufmgr, time_t time)
       }
    }
 
+   list_for_each_entry_safe(struct iris_bo, bo, &bufmgr->zombie_list, head) {
+      /* Stop once we reach a busy BO - all others past this point were
+       * freed more recently so are likely also busy.
+       */
+      if (!bo->idle && iris_bo_busy(bo))
+         break;
+
+      list_del(&bo->head);
+      bo_close(bo);
+   }
+
    bufmgr->time = time;
 }
 
@@ -849,11 +936,74 @@ print_flags(unsigned flags)
 }
 
 static void *
-iris_bo_map_cpu(struct pipe_debug_callback *dbg,
-                struct iris_bo *bo, unsigned flags)
+iris_bo_gem_mmap_legacy(struct pipe_debug_callback *dbg,
+                        struct iris_bo *bo, bool wc)
+{
+   struct iris_bufmgr *bufmgr = bo->bufmgr;
+
+   struct drm_i915_gem_mmap mmap_arg = {
+      .handle = bo->gem_handle,
+      .size = bo->size,
+      .flags = wc ? I915_MMAP_WC : 0,
+   };
+
+   int ret = gen_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg);
+   if (ret != 0) {
+      DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
+          __FILE__, __LINE__, bo->gem_handle, bo->name, strerror(errno));
+      return NULL;
+   }
+   void *map = (void *) (uintptr_t) mmap_arg.addr_ptr;
+
+   return map;
+}
+
+static void *
+iris_bo_gem_mmap_offset(struct pipe_debug_callback *dbg, struct iris_bo *bo,
+                        bool wc)
+{
+   struct iris_bufmgr *bufmgr = bo->bufmgr;
+
+   struct drm_i915_gem_mmap_offset mmap_arg = {
+      .handle = bo->gem_handle,
+      .flags = wc ? I915_MMAP_OFFSET_WC : I915_MMAP_OFFSET_WB,
+   };
+
+   /* Get the fake offset back */
+   int ret = gen_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_MMAP_OFFSET, &mmap_arg);
+   if (ret != 0) {
+      DBG("%s:%d: Error preparing buffer %d (%s): %s .\n",
+          __FILE__, __LINE__, bo->gem_handle, bo->name, strerror(errno));
+      return NULL;
+   }
+
+   /* And map it */
+   void *map = mmap(0, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED,
+                    bufmgr->fd, mmap_arg.offset);
+   if (map == MAP_FAILED) {
+      DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
+          __FILE__, __LINE__, bo->gem_handle, bo->name, strerror(errno));
+      return NULL;
+   }
+
+   return map;
+}
+
+static void *
+iris_bo_gem_mmap(struct pipe_debug_callback *dbg, struct iris_bo *bo, bool wc)
 {
    struct iris_bufmgr *bufmgr = bo->bufmgr;
 
+   if (bufmgr->has_mmap_offset)
+      return iris_bo_gem_mmap_offset(dbg, bo, wc);
+   else
+      return iris_bo_gem_mmap_legacy(dbg, bo, wc);
+}
+
+static void *
+iris_bo_map_cpu(struct pipe_debug_callback *dbg,
+                struct iris_bo *bo, unsigned flags)
+{
    /* We disallow CPU maps for writing to non-coherent buffers, as the
     * CPU map can become invalidated when a batch is flushed out, which
     * can happen at unpredictable times.  You should use WC maps instead.
@@ -862,23 +1012,16 @@ iris_bo_map_cpu(struct pipe_debug_callback *dbg,
 
    if (!bo->map_cpu) {
       DBG("iris_bo_map_cpu: %d (%s)\n", bo->gem_handle, bo->name);
-
-      struct drm_i915_gem_mmap mmap_arg = {
-         .handle = bo->gem_handle,
-         .size = bo->size,
-      };
-      int ret = drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg);
-      if (ret != 0) {
-         DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
-             __FILE__, __LINE__, bo->gem_handle, bo->name, strerror(errno));
+      void *map = iris_bo_gem_mmap(dbg, bo, false);
+      if (!map) {
          return NULL;
       }
-      void *map = (void *) (uintptr_t) mmap_arg.addr_ptr;
+
       VG_DEFINED(map, bo->size);
 
       if (p_atomic_cmpxchg(&bo->map_cpu, NULL, map)) {
          VG_NOACCESS(map, bo->size);
-         munmap(map, bo->size);
+         os_munmap(map, bo->size);
       }
    }
    assert(bo->map_cpu);
@@ -918,29 +1061,18 @@ static void *
 iris_bo_map_wc(struct pipe_debug_callback *dbg,
                struct iris_bo *bo, unsigned flags)
 {
-   struct iris_bufmgr *bufmgr = bo->bufmgr;
-
    if (!bo->map_wc) {
       DBG("iris_bo_map_wc: %d (%s)\n", bo->gem_handle, bo->name);
-
-      struct drm_i915_gem_mmap mmap_arg = {
-         .handle = bo->gem_handle,
-         .size = bo->size,
-         .flags = I915_MMAP_WC,
-      };
-      int ret = drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_MMAP, &mmap_arg);
-      if (ret != 0) {
-         DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
-             __FILE__, __LINE__, bo->gem_handle, bo->name, strerror(errno));
+      void *map = iris_bo_gem_mmap(dbg, bo, true);
+      if (!map) {
          return NULL;
       }
 
-      void *map = (void *) (uintptr_t) mmap_arg.addr_ptr;
       VG_DEFINED(map, bo->size);
 
       if (p_atomic_cmpxchg(&bo->map_wc, NULL, map)) {
          VG_NOACCESS(map, bo->size);
-         munmap(map, bo->size);
+         os_munmap(map, bo->size);
       }
    }
    assert(bo->map_wc);
@@ -990,7 +1122,7 @@ iris_bo_map_gtt(struct pipe_debug_callback *dbg,
       struct drm_i915_gem_mmap_gtt mmap_arg = { .handle = bo->gem_handle };
 
       /* Get the fake offset back... */
-      int ret = drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_MMAP_GTT, &mmap_arg);
+      int ret = gen_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_MMAP_GTT, &mmap_arg);
       if (ret != 0) {
          DBG("%s:%d: Error preparing buffer map %d (%s): %s .\n",
              __FILE__, __LINE__, bo->gem_handle, bo->name, strerror(errno));
@@ -998,8 +1130,8 @@ iris_bo_map_gtt(struct pipe_debug_callback *dbg,
       }
 
       /* and mmap it. */
-      void *map = mmap(0, bo->size, PROT_READ | PROT_WRITE,
-                       MAP_SHARED, bufmgr->fd, mmap_arg.offset);
+      void *map = os_mmap(0, bo->size, PROT_READ | PROT_WRITE,
+                          MAP_SHARED, bufmgr->fd, mmap_arg.offset);
       if (map == MAP_FAILED) {
          DBG("%s:%d: Error mapping buffer %d (%s): %s .\n",
              __FILE__, __LINE__, bo->gem_handle, bo->name, strerror(errno));
@@ -1015,7 +1147,7 @@ iris_bo_map_gtt(struct pipe_debug_callback *dbg,
 
       if (p_atomic_cmpxchg(&bo->map_gtt, NULL, map)) {
          VG_NOACCESS(map, bo->size);
-         munmap(map, bo->size);
+         os_munmap(map, bo->size);
       }
    }
    assert(bo->map_gtt);
@@ -1148,7 +1280,7 @@ iris_bo_wait(struct iris_bo *bo, int64_t timeout_ns)
       .bo_handle = bo->gem_handle,
       .timeout_ns = timeout_ns,
    };
-   int ret = drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
+   int ret = gen_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_WAIT, &wait);
    if (ret != 0)
       return -errno;
 
@@ -1157,9 +1289,15 @@ iris_bo_wait(struct iris_bo *bo, int64_t timeout_ns)
    return ret;
 }
 
-void
+static void
 iris_bufmgr_destroy(struct iris_bufmgr *bufmgr)
 {
+   /* Free aux-map buffers */
+   gen_aux_map_finish(bufmgr->aux_map_ctx);
+
+   /* bufmgr will no longer try to free VMA entries in the aux-map */
+   bufmgr->aux_map_ctx = NULL;
+
    mtx_destroy(&bufmgr->lock);
 
    /* Free any cached buffer objects we were going to reuse */
@@ -1173,6 +1311,12 @@ iris_bufmgr_destroy(struct iris_bufmgr *bufmgr)
       }
    }
 
+   /* Close any buffer objects on the dead list. */
+   list_for_each_entry_safe(struct iris_bo, bo, &bufmgr->zombie_list, head) {
+      list_del(&bo->head);
+      bo_close(bo);
+   }
+
    _mesa_hash_table_destroy(bufmgr->name_table, NULL);
    _mesa_hash_table_destroy(bufmgr->handle_table, NULL);
 
@@ -1181,6 +1325,8 @@ iris_bufmgr_destroy(struct iris_bufmgr *bufmgr)
          util_vma_heap_finish(&bufmgr->vma_allocator[z]);
    }
 
+   close(bufmgr->fd);
+
    free(bufmgr);
 }
 
@@ -1212,22 +1358,13 @@ bo_set_tiling_internal(struct iris_bo *bo, uint32_t tiling_mode,
       return -errno;
 
    bo->tiling_mode = set_tiling.tiling_mode;
-   bo->swizzle_mode = set_tiling.swizzle_mode;
    bo->stride = set_tiling.stride;
    return 0;
 }
 
-int
-iris_bo_get_tiling(struct iris_bo *bo, uint32_t *tiling_mode,
-                  uint32_t *swizzle_mode)
-{
-   *tiling_mode = bo->tiling_mode;
-   *swizzle_mode = bo->swizzle_mode;
-   return 0;
-}
-
 struct iris_bo *
-iris_bo_import_dmabuf(struct iris_bufmgr *bufmgr, int prime_fd)
+iris_bo_import_dmabuf(struct iris_bufmgr *bufmgr, int prime_fd,
+                      uint32_t tiling, uint32_t stride)
 {
    uint32_t handle;
    struct iris_bo *bo;
@@ -1246,11 +1383,9 @@ iris_bo_import_dmabuf(struct iris_bufmgr *bufmgr, int prime_fd)
     * for named buffers, we must not create two bo's pointing at the same
     * kernel object
     */
-   bo = hash_find_bo(bufmgr->handle_table, handle);
-   if (bo) {
-      iris_bo_reference(bo);
+   bo = find_and_ref_external_bo(bufmgr->handle_table, handle);
+   if (bo)
       goto out;
-   }
 
    bo = bo_calloc();
    if (!bo)
@@ -1268,23 +1403,26 @@ iris_bo_import_dmabuf(struct iris_bufmgr *bufmgr, int prime_fd)
       bo->size = ret;
 
    bo->bufmgr = bufmgr;
-
-   bo->gem_handle = handle;
-   _mesa_hash_table_insert(bufmgr->handle_table, &bo->gem_handle, bo);
-
    bo->name = "prime";
    bo->reusable = false;
    bo->external = true;
    bo->kflags = EXEC_OBJECT_SUPPORTS_48B_ADDRESS | EXEC_OBJECT_PINNED;
    bo->gtt_offset = vma_alloc(bufmgr, IRIS_MEMZONE_OTHER, bo->size, 1);
+   bo->gem_handle = handle;
+   _mesa_hash_table_insert(bufmgr->handle_table, &bo->gem_handle, bo);
 
    struct drm_i915_gem_get_tiling get_tiling = { .handle = bo->gem_handle };
-   if (drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_GET_TILING, &get_tiling))
+   if (gen_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_GET_TILING, &get_tiling))
       goto err;
 
-   bo->tiling_mode = get_tiling.tiling_mode;
-   bo->swizzle_mode = get_tiling.swizzle_mode;
-   /* XXX stride is unknown */
+   if (get_tiling.tiling_mode == tiling || tiling > I915_TILING_LAST) {
+      bo->tiling_mode = get_tiling.tiling_mode;
+       /* XXX stride is unknown */
+   } else {
+      if (bo_set_tiling_internal(bo, tiling, stride)) {
+         goto err;
+      }
+   }
 
 out:
    mtx_unlock(&bufmgr->lock);
@@ -1301,17 +1439,25 @@ iris_bo_make_external_locked(struct iris_bo *bo)
 {
    if (!bo->external) {
       _mesa_hash_table_insert(bo->bufmgr->handle_table, &bo->gem_handle, bo);
+      /* If a BO is going to be used externally, it could be sent to the
+       * display HW. So make sure our CPU mappings don't assume cache
+       * coherency since display is outside that cache.
+       */
+      bo->cache_coherent = false;
       bo->external = true;
+      bo->reusable = false;
    }
 }
 
-static void
+void
 iris_bo_make_external(struct iris_bo *bo)
 {
    struct iris_bufmgr *bufmgr = bo->bufmgr;
 
-   if (bo->external)
+   if (bo->external) {
+      assert(!bo->reusable);
       return;
+   }
 
    mtx_lock(&bufmgr->lock);
    iris_bo_make_external_locked(bo);
@@ -1329,8 +1475,6 @@ iris_bo_export_dmabuf(struct iris_bo *bo, int *prime_fd)
                           DRM_CLOEXEC, prime_fd) != 0)
       return -errno;
 
-   bo->reusable = false;
-
    return 0;
 }
 
@@ -1350,7 +1494,7 @@ iris_bo_flink(struct iris_bo *bo, uint32_t *name)
    if (!bo->global_name) {
       struct drm_gem_flink flink = { .handle = bo->gem_handle };
 
-      if (drm_ioctl(bufmgr->fd, DRM_IOCTL_GEM_FLINK, &flink))
+      if (gen_ioctl(bufmgr->fd, DRM_IOCTL_GEM_FLINK, &flink))
          return -errno;
 
       mtx_lock(&bufmgr->lock);
@@ -1360,14 +1504,75 @@ iris_bo_flink(struct iris_bo *bo, uint32_t *name)
          _mesa_hash_table_insert(bufmgr->name_table, &bo->global_name, bo);
       }
       mtx_unlock(&bufmgr->lock);
-
-      bo->reusable = false;
    }
 
    *name = bo->global_name;
    return 0;
 }
 
+int
+iris_bo_export_gem_handle_for_device(struct iris_bo *bo, int drm_fd,
+                                     uint32_t *out_handle)
+{
+   /* Only add the new GEM handle to the list of export if it belongs to a
+    * different GEM device. Otherwise we might close the same buffer multiple
+    * times.
+    */
+   struct iris_bufmgr *bufmgr = bo->bufmgr;
+   int ret = os_same_file_description(drm_fd, bufmgr->fd);
+   WARN_ONCE(ret < 0,
+             "Kernel has no file descriptor comparison support: %s\n",
+             strerror(errno));
+   if (ret == 0) {
+      *out_handle = iris_bo_export_gem_handle(bo);
+      return 0;
+   }
+
+   struct bo_export *export = calloc(1, sizeof(*export));
+   if (!export)
+      return -ENOMEM;
+
+   export->drm_fd = drm_fd;
+
+   int dmabuf_fd = -1;
+   int err = iris_bo_export_dmabuf(bo, &dmabuf_fd);
+   if (err) {
+      free(export);
+      return err;
+   }
+
+   mtx_lock(&bufmgr->lock);
+   err = drmPrimeFDToHandle(drm_fd, dmabuf_fd, &export->gem_handle);
+   close(dmabuf_fd);
+   if (err) {
+      mtx_unlock(&bufmgr->lock);
+      free(export);
+      return err;
+   }
+
+   bool found = false;
+   list_for_each_entry(struct bo_export, iter, &bo->exports, link) {
+      if (iter->drm_fd != drm_fd)
+         continue;
+      /* Here we assume that for a given DRM fd, we'll always get back the
+       * same GEM handle for a given buffer.
+       */
+      assert(iter->gem_handle == export->gem_handle);
+      free(export);
+      export = iter;
+      found = true;
+      break;
+   }
+   if (!found)
+      list_addtail(&export->link, &bo->exports);
+
+   mtx_unlock(&bufmgr->lock);
+
+   *out_handle = export->gem_handle;
+
+   return 0;
+}
+
 static void
 add_bucket(struct iris_bufmgr *bufmgr, int size)
 {
@@ -1415,7 +1620,7 @@ uint32_t
 iris_create_hw_context(struct iris_bufmgr *bufmgr)
 {
    struct drm_i915_gem_context_create create = { };
-   int ret = drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE, &create);
+   int ret = gen_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE, &create);
    if (ret != 0) {
       DBG("DRM_IOCTL_I915_GEM_CONTEXT_CREATE failed: %s\n", strerror(errno));
       return 0;
@@ -1470,7 +1675,7 @@ iris_hw_context_set_priority(struct iris_bufmgr *bufmgr,
    int err;
 
    err = 0;
-   if (drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM, &p))
+   if (gen_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_CONTEXT_SETPARAM, &p))
       err = -errno;
 
    return err;
@@ -1495,7 +1700,7 @@ iris_destroy_hw_context(struct iris_bufmgr *bufmgr, uint32_t ctx_id)
    struct drm_i915_gem_context_destroy d = { .ctx_id = ctx_id };
 
    if (ctx_id != 0 &&
-       drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_CONTEXT_DESTROY, &d) != 0) {
+       gen_ioctl(bufmgr->fd, DRM_IOCTL_I915_GEM_CONTEXT_DESTROY, &d) != 0) {
       fprintf(stderr, "DRM_IOCTL_I915_GEM_CONTEXT_DESTROY failed: %s\n",
               strerror(errno));
    }
@@ -1505,7 +1710,7 @@ int
 iris_reg_read(struct iris_bufmgr *bufmgr, uint32_t offset, uint64_t *result)
 {
    struct drm_i915_reg_read reg_read = { .offset = offset };
-   int ret = drm_ioctl(bufmgr->fd, DRM_IOCTL_I915_REG_READ, &reg_read);
+   int ret = gen_ioctl(bufmgr->fd, DRM_IOCTL_I915_REG_READ, &reg_read);
 
    *result = reg_read.val;
    return ret;
@@ -1520,20 +1725,64 @@ iris_gtt_size(int fd)
    struct drm_i915_gem_context_param p = {
       .param = I915_CONTEXT_PARAM_GTT_SIZE,
    };
-   if (!drm_ioctl(fd, DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM, &p))
+   if (!gen_ioctl(fd, DRM_IOCTL_I915_GEM_CONTEXT_GETPARAM, &p))
       return p.value;
 
    return 0;
 }
 
+static struct gen_buffer *
+gen_aux_map_buffer_alloc(void *driver_ctx, uint32_t size)
+{
+   struct gen_buffer *buf = malloc(sizeof(struct gen_buffer));
+   if (!buf)
+      return NULL;
+
+   struct iris_bufmgr *bufmgr = (struct iris_bufmgr *)driver_ctx;
+
+   struct iris_bo *bo =
+      iris_bo_alloc_tiled(bufmgr, "aux-map", size, 64 * 1024,
+                          IRIS_MEMZONE_OTHER, I915_TILING_NONE, 0, 0);
+
+   buf->driver_bo = bo;
+   buf->gpu = bo->gtt_offset;
+   buf->gpu_end = buf->gpu + bo->size;
+   buf->map = iris_bo_map(NULL, bo, MAP_WRITE | MAP_RAW);
+   return buf;
+}
+
+static void
+gen_aux_map_buffer_free(void *driver_ctx, struct gen_buffer *buffer)
+{
+   iris_bo_unreference((struct iris_bo*)buffer->driver_bo);
+   free(buffer);
+}
+
+static struct gen_mapped_pinned_buffer_alloc aux_map_allocator = {
+   .alloc = gen_aux_map_buffer_alloc,
+   .free = gen_aux_map_buffer_free,
+};
+
+static int
+gem_param(int fd, int name)
+{
+   int v = -1; /* No param uses (yet) the sign bit, reserve it for errors */
+
+   struct drm_i915_getparam gp = { .param = name, .value = &v };
+   if (gen_ioctl(fd, DRM_IOCTL_I915_GETPARAM, &gp))
+      return -1;
+
+   return v;
+}
+
 /**
  * Initializes the GEM buffer manager, which uses the kernel to allocate, map,
  * and manage map buffer objections.
  *
  * \param fd File descriptor of the opened DRM device.
  */
-struct iris_bufmgr *
-iris_bufmgr_init(struct gen_device_info *devinfo, int fd)
+static struct iris_bufmgr *
+iris_bufmgr_create(struct gen_device_info *devinfo, int fd, bool bo_reuse)
 {
    uint64_t gtt_size = iris_gtt_size(fd);
    if (gtt_size <= IRIS_MEMZONE_OTHER_START)
@@ -1552,17 +1801,25 @@ iris_bufmgr_init(struct gen_device_info *devinfo, int fd)
     * Don't do this! Ensure that each library/bufmgr has its own device
     * fd so that its namespace does not clash with another.
     */
-   bufmgr->fd = fd;
+   bufmgr->fd = dup(fd);
+
+   p_atomic_set(&bufmgr->refcount, 1);
 
    if (mtx_init(&bufmgr->lock, mtx_plain) != 0) {
+      close(bufmgr->fd);
       free(bufmgr);
       return NULL;
    }
 
+   list_inithead(&bufmgr->zombie_list);
+
    bufmgr->has_llc = devinfo->has_llc;
+   bufmgr->bo_reuse = bo_reuse;
+   bufmgr->has_mmap_offset = gem_param(fd, I915_PARAM_MMAP_GTT_VERSION) >= 4;
 
    STATIC_ASSERT(IRIS_MEMZONE_SHADER_START == 0ull);
    const uint64_t _4GB = 1ull << 32;
+   const uint64_t _2GB = 1ul << 31;
 
    /* The STATE_BASE_ADDRESS size field can only hold 1 page shy of 4GB */
    const uint64_t _4GB_minus_1 = _4GB - PAGE_SIZE;
@@ -1572,9 +1829,16 @@ iris_bufmgr_init(struct gen_device_info *devinfo, int fd)
    util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_SURFACE],
                       IRIS_MEMZONE_SURFACE_START,
                       _4GB_minus_1 - IRIS_MAX_BINDERS * IRIS_BINDER_SIZE);
+   /* TODO: Why does limiting to 2GB help some state items on gen12?
+    *  - CC Viewport Pointer
+    *  - Blend State Pointer
+    *  - Color Calc State Pointer
+    */
+   const uint64_t dynamic_pool_size =
+      (devinfo->gen >= 12 ? _2GB : _4GB_minus_1) - IRIS_BORDER_COLOR_POOL_SIZE;
    util_vma_heap_init(&bufmgr->vma_allocator[IRIS_MEMZONE_DYNAMIC],
                       IRIS_MEMZONE_DYNAMIC_START + IRIS_BORDER_COLOR_POOL_SIZE,
-                      _4GB_minus_1 - IRIS_BORDER_COLOR_POOL_SIZE);
+                      dynamic_pool_size);
 
    /* Leave the last 4GB out of the high vma range, so that no state
     * base address + size can overflow 48 bits.
@@ -1583,15 +1847,85 @@ iris_bufmgr_init(struct gen_device_info *devinfo, int fd)
                       IRIS_MEMZONE_OTHER_START,
                       (gtt_size - _4GB) - IRIS_MEMZONE_OTHER_START);
 
-   // XXX: driconf
-   bufmgr->bo_reuse = env_var_as_boolean("bo_reuse", true);
-
    init_cache_buckets(bufmgr);
 
    bufmgr->name_table =
-      _mesa_hash_table_create(NULL, key_hash_uint, key_uint_equal);
+      _mesa_hash_table_create(NULL, _mesa_hash_uint, _mesa_key_uint_equal);
    bufmgr->handle_table =
-      _mesa_hash_table_create(NULL, key_hash_uint, key_uint_equal);
+      _mesa_hash_table_create(NULL, _mesa_hash_uint, _mesa_key_uint_equal);
+
+   if (devinfo->gen >= 12) {
+      bufmgr->aux_map_ctx = gen_aux_map_init(bufmgr, &aux_map_allocator,
+                                             devinfo);
+      assert(bufmgr->aux_map_ctx);
+   }
+
+   return bufmgr;
+}
 
+static struct iris_bufmgr *
+iris_bufmgr_ref(struct iris_bufmgr *bufmgr)
+{
+   p_atomic_inc(&bufmgr->refcount);
    return bufmgr;
 }
+
+void
+iris_bufmgr_unref(struct iris_bufmgr *bufmgr)
+{
+   mtx_lock(&global_bufmgr_list_mutex);
+   if (p_atomic_dec_zero(&bufmgr->refcount)) {
+      list_del(&bufmgr->link);
+      iris_bufmgr_destroy(bufmgr);
+   }
+   mtx_unlock(&global_bufmgr_list_mutex);
+}
+
+/**
+ * Gets an already existing GEM buffer manager or create a new one.
+ *
+ * \param fd File descriptor of the opened DRM device.
+ */
+struct iris_bufmgr *
+iris_bufmgr_get_for_fd(struct gen_device_info *devinfo, int fd, bool bo_reuse)
+{
+   struct stat st;
+
+   if (fstat(fd, &st))
+      return NULL;
+
+   struct iris_bufmgr *bufmgr = NULL;
+
+   mtx_lock(&global_bufmgr_list_mutex);
+   list_for_each_entry(struct iris_bufmgr, iter_bufmgr, &global_bufmgr_list, link) {
+      struct stat iter_st;
+      if (fstat(iter_bufmgr->fd, &iter_st))
+         continue;
+
+      if (st.st_rdev == iter_st.st_rdev) {
+         assert(iter_bufmgr->bo_reuse == bo_reuse);
+         bufmgr = iris_bufmgr_ref(iter_bufmgr);
+         goto unlock;
+      }
+   }
+
+   bufmgr = iris_bufmgr_create(devinfo, fd, bo_reuse);
+   list_addtail(&bufmgr->link, &global_bufmgr_list);
+
+ unlock:
+   mtx_unlock(&global_bufmgr_list_mutex);
+
+   return bufmgr;
+}
+
+int
+iris_bufmgr_get_fd(struct iris_bufmgr *bufmgr)
+{
+   return bufmgr->fd;
+}
+
+void*
+iris_bufmgr_get_aux_map_context(struct iris_bufmgr *bufmgr)
+{
+   return bufmgr->aux_map_ctx;
+}