src/vulkan/anv_allocator.c

   1 /*
   2  * Copyright © 2015 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice (including the next
  12  * paragraph) shall be included in all copies or substantial portions of the
  13  * Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21  * IN THE SOFTWARE.
  22  */
  23
  24 #define _DEFAULT_SOURCE
  25
  26 #include <stdint.h>
  27 #include <stdlib.h>
  28 #include <unistd.h>
  29 #include <values.h>
  30 #include <assert.h>
  31 #include <linux/futex.h>
  32 #include <linux/memfd.h>
  33 #include <sys/time.h>
  34 #include <sys/mman.h>
  35 #include <sys/syscall.h>
  36
  37 #include "anv_private.h"
  38
  39 #ifdef HAVE_VALGRIND
  40 #define VG_NOACCESS_READ(__ptr) ({                       \
  41    VALGRIND_MAKE_MEM_DEFINED((__ptr), sizeof(*(__ptr))); \
  42    __typeof(*(__ptr)) __val = *(__ptr);                  \
  43    VALGRIND_MAKE_MEM_NOACCESS((__ptr), sizeof(*(__ptr)));\
  44    __val;                                                \
  45 })
  46 #define VG_NOACCESS_WRITE(__ptr, __val) ({                  \
  47    VALGRIND_MAKE_MEM_UNDEFINED((__ptr), sizeof(*(__ptr)));  \
  48    *(__ptr) = (__val);                                      \
  49    VALGRIND_MAKE_MEM_NOACCESS((__ptr), sizeof(*(__ptr)));   \
  50 })
  51 #else
  52 #define VG_NOACCESS_READ(__ptr) (*(__ptr))
  53 #define VG_NOACCESS_WRITE(__ptr, __val) (*(__ptr) = (__val))
  54 #endif
  55
  56 /* Design goals:
  57  *
  58  *  - Lock free (except when resizing underlying bos)
  59  *
  60  *  - Constant time allocation with typically only one atomic
  61  *
  62  *  - Multiple allocation sizes without fragmentation
  63  *
  64  *  - Can grow while keeping addresses and offset of contents stable
  65  *
  66  *  - All allocations within one bo so we can point one of the
  67  *    STATE_BASE_ADDRESS pointers at it.
  68  *
  69  * The overall design is a two-level allocator: top level is a fixed size, big
  70  * block (8k) allocator, which operates out of a bo.  Allocation is done by
  71  * either pulling a block from the free list or growing the used range of the
  72  * bo.  Growing the range may run out of space in the bo which we then need to
  73  * grow.  Growing the bo is tricky in a multi-threaded, lockless environment:
  74  * we need to keep all pointers and contents in the old map valid.  GEM bos in
  75  * general can't grow, but we use a trick: we create a memfd and use ftruncate
  76  * to grow it as necessary.  We mmap the new size and then create a gem bo for
  77  * it using the new gem userptr ioctl.  Without heavy-handed locking around
  78  * our allocation fast-path, there isn't really a way to munmap the old mmap,
  79  * so we just keep it around until garbage collection time.  While the block
  80  * allocator is lockless for normal operations, we block other threads trying
  81  * to allocate while we're growing the map.  It sholdn't happen often, and
  82  * growing is fast anyway.
  83  *
  84  * At the next level we can use various sub-allocators.  The state pool is a
  85  * pool of smaller, fixed size objects, which operates much like the block
  86  * pool.  It uses a free list for freeing objects, but when it runs out of
  87  * space it just allocates a new block from the block pool.  This allocator is
  88  * intended for longer lived state objects such as SURFACE_STATE and most
  89  * other persistent state objects in the API.  We may need to track more info
  90  * with these object and a pointer back to the CPU object (eg VkImage).  In
  91  * those cases we just allocate a slightly bigger object and put the extra
  92  * state after the GPU state object.
  93  *
  94  * The state stream allocator works similar to how the i965 DRI driver streams
  95  * all its state.  Even with Vulkan, we need to emit transient state (whether
  96  * surface state base or dynamic state base), and for that we can just get a
  97  * block and fill it up.  These cases are local to a command buffer and the
  98  * sub-allocator need not be thread safe.  The streaming allocator gets a new
  99  * block when it runs out of space and chains them together so they can be
 100  * easily freed.
 101  */
 102
 103 /* Allocations are always at least 64 byte aligned, so 1 is an invalid value.
 104  * We use it to indicate the free list is empty. */
 105 #define EMPTY 1
 106
 107 struct anv_mmap_cleanup {
 108    void *map;
 109    size_t size;
 110    uint32_t gem_handle;
 111 };
 112
 113 #define ANV_MMAP_CLEANUP_INIT ((struct anv_mmap_cleanup){0})
 114
 115 static inline long
 116 sys_futex(void *addr1, int op, int val1,
 117           struct timespec *timeout, void *addr2, int val3)
 118 {
 119    return syscall(SYS_futex, addr1, op, val1, timeout, addr2, val3);
 120 }
 121
 122 static inline int
 123 futex_wake(uint32_t *addr, int count)
 124 {
 125    return sys_futex(addr, FUTEX_WAKE, count, NULL, NULL, 0);
 126 }
 127
 128 static inline int
 129 futex_wait(uint32_t *addr, int32_t value)
 130 {
 131    return sys_futex(addr, FUTEX_WAIT, value, NULL, NULL, 0);
 132 }
 133
 134 static inline int
 135 memfd_create(const char *name, unsigned int flags)
 136 {
 137    return syscall(SYS_memfd_create, name, flags);
 138 }
 139
 140 static inline uint32_t
 141 ilog2_round_up(uint32_t value)
 142 {
 143    assert(value != 0);
 144    return 32 - __builtin_clz(value - 1);
 145 }
 146
 147 static inline uint32_t
 148 round_to_power_of_two(uint32_t value)
 149 {
 150    return 1 << ilog2_round_up(value);
 151 }
 152
 153 static bool
 154 anv_free_list_pop(union anv_free_list *list, void **map, uint32_t *offset)
 155 {
 156    union anv_free_list current, new, old;
 157
 158    current.u64 = list->u64;
 159    while (current.offset != EMPTY) {
 160       /* We have to add a memory barrier here so that the list head (and
 161        * offset) gets read before we read the map pointer.  This way we
 162        * know that the map pointer is valid for the given offset at the
 163        * point where we read it.
 164        */
 165       __sync_synchronize();
 166
 167       uint32_t *next_ptr = *map + current.offset;
 168       new.offset = VG_NOACCESS_READ(next_ptr);
 169       new.count = current.count + 1;
 170       old.u64 = __sync_val_compare_and_swap(&list->u64, current.u64, new.u64);
 171       if (old.u64 == current.u64) {
 172          *offset = current.offset;
 173          return true;
 174       }
 175       current = old;
 176    }
 177
 178    return false;
 179 }
 180
 181 static void
 182 anv_free_list_push(union anv_free_list *list, void *map, uint32_t offset)
 183 {
 184    union anv_free_list current, old, new;
 185    uint32_t *next_ptr = map + offset;
 186
 187    old = *list;
 188    do {
 189       current = old;
 190       VG_NOACCESS_WRITE(next_ptr, current.offset);
 191       new.offset = offset;
 192       new.count = current.count + 1;
 193       old.u64 = __sync_val_compare_and_swap(&list->u64, current.u64, new.u64);
 194    } while (old.u64 != current.u64);
 195 }
 196
 197 /* All pointers in the ptr_free_list are assumed to be page-aligned.  This
 198  * means that the bottom 12 bits should all be zero.
 199  */
 200 #define PFL_COUNT(x) ((uintptr_t)(x) & 0xfff)
 201 #define PFL_PTR(x) ((void *)((uintptr_t)(x) & ~0xfff))
 202 #define PFL_PACK(ptr, count) ({           \
 203    assert(((uintptr_t)(ptr) & 0xfff) == 0); \
 204    (void *)((uintptr_t)(ptr) | (uintptr_t)((count) & 0xfff)); \
 205 })
 206
 207 static bool
 208 anv_ptr_free_list_pop(void **list, void **elem)
 209 {
 210    void *current = *list;
 211    while (PFL_PTR(current) != NULL) {
 212       void **next_ptr = PFL_PTR(current);
 213       void *new_ptr = VG_NOACCESS_READ(next_ptr);
 214       unsigned new_count = PFL_COUNT(current) + 1;
 215       void *new = PFL_PACK(new_ptr, new_count);
 216       void *old = __sync_val_compare_and_swap(list, current, new);
 217       if (old == current) {
 218          *elem = PFL_PTR(current);
 219          return true;
 220       }
 221       current = old;
 222    }
 223
 224    return false;
 225 }
 226
 227 static void
 228 anv_ptr_free_list_push(void **list, void *elem)
 229 {
 230    void *old, *current;
 231    void **next_ptr = elem;
 232
 233    old = *list;
 234    do {
 235       current = old;
 236       VG_NOACCESS_WRITE(next_ptr, PFL_PTR(current));
 237       unsigned new_count = PFL_COUNT(current) + 1;
 238       void *new = PFL_PACK(elem, new_count);
 239       old = __sync_val_compare_and_swap(list, current, new);
 240    } while (old != current);
 241 }
 242
 243 static uint32_t
 244 anv_block_pool_grow(struct anv_block_pool *pool, uint32_t old_size);
 245
 246 void
 247 anv_block_pool_init(struct anv_block_pool *pool,
 248                     struct anv_device *device, uint32_t block_size)
 249 {
 250    assert(is_power_of_two(block_size));
 251
 252    pool->device = device;
 253    pool->bo.gem_handle = 0;
 254    pool->bo.offset = 0;
 255    pool->block_size = block_size;
 256    pool->free_list = ANV_FREE_LIST_EMPTY;
 257    anv_vector_init(&pool->mmap_cleanups,
 258                    round_to_power_of_two(sizeof(struct anv_mmap_cleanup)), 128);
 259
 260    /* Immediately grow the pool so we'll have a backing bo. */
 261    pool->state.next = 0;
 262    pool->state.end = anv_block_pool_grow(pool, 0);
 263 }
 264
 265 void
 266 anv_block_pool_finish(struct anv_block_pool *pool)
 267 {
 268    struct anv_mmap_cleanup *cleanup;
 269
 270    anv_vector_foreach(cleanup, &pool->mmap_cleanups) {
 271       if (cleanup->map)
 272          munmap(cleanup->map, cleanup->size);
 273       if (cleanup->gem_handle)
 274          anv_gem_close(pool->device, cleanup->gem_handle);
 275    }
 276
 277    anv_vector_finish(&pool->mmap_cleanups);
 278
 279    close(pool->fd);
 280 }
 281
 282 static uint32_t
 283 anv_block_pool_grow(struct anv_block_pool *pool, uint32_t old_size)
 284 {
 285    size_t size;
 286    void *map;
 287    int gem_handle;
 288    struct anv_mmap_cleanup *cleanup;
 289
 290    if (old_size == 0) {
 291       size = 32 * pool->block_size;
 292    } else {
 293       size = old_size * 2;
 294    }
 295
 296    cleanup = anv_vector_add(&pool->mmap_cleanups);
 297    if (!cleanup)
 298       return 0;
 299    *cleanup = ANV_MMAP_CLEANUP_INIT;
 300
 301    if (old_size == 0)
 302       pool->fd = memfd_create("block pool", MFD_CLOEXEC);
 303
 304    if (pool->fd == -1)
 305       return 0;
 306
 307    if (ftruncate(pool->fd, size) == -1)
 308       return 0;
 309
 310    /* First try to see if mremap can grow the map in place. */
 311    map = MAP_FAILED;
 312    if (old_size > 0)
 313       map = mremap(pool->map, old_size, size, 0);
 314    if (map == MAP_FAILED) {
 315       /* Just leak the old map until we destroy the pool.  We can't munmap it
 316        * without races or imposing locking on the block allocate fast path. On
 317        * the whole the leaked maps adds up to less than the size of the
 318        * current map.  MAP_POPULATE seems like the right thing to do, but we
 319        * should try to get some numbers.
 320        */
 321       map = mmap(NULL, size, PROT_READ | PROT_WRITE,
 322                  MAP_SHARED | MAP_POPULATE, pool->fd, 0);
 323       cleanup->map = map;
 324       cleanup->size = size;
 325    }
 326    if (map == MAP_FAILED)
 327       return 0;
 328
 329    gem_handle = anv_gem_userptr(pool->device, map, size);
 330    if (gem_handle == 0)
 331       return 0;
 332    cleanup->gem_handle = gem_handle;
 333
 334    /* Now that we successfull allocated everything, we can write the new
 335     * values back into pool. */
 336    pool->map = map;
 337    pool->bo.gem_handle = gem_handle;
 338    pool->bo.size = size;
 339    pool->bo.map = map;
 340    pool->bo.index = 0;
 341
 342    return size;
 343 }
 344
 345 uint32_t
 346 anv_block_pool_alloc(struct anv_block_pool *pool)
 347 {
 348    uint32_t offset;
 349    struct anv_block_state state, old, new;
 350
 351    /* Try free list first. */
 352    if (anv_free_list_pop(&pool->free_list, &pool->map, &offset)) {
 353       assert(pool->map);
 354       return offset;
 355    }
 356
 357  restart:
 358    state.u64 = __sync_fetch_and_add(&pool->state.u64, pool->block_size);
 359    if (state.next < state.end) {
 360       assert(pool->map);
 361       return state.next;
 362    } else if (state.next == state.end) {
 363       /* We allocated the first block outside the pool, we have to grow it.
 364        * pool->next_block acts a mutex: threads who try to allocate now will
 365        * get block indexes above the current limit and hit futex_wait
 366        * below. */
 367       new.next = state.next + pool->block_size;
 368       new.end = anv_block_pool_grow(pool, state.end);
 369       assert(new.end > 0);
 370       old.u64 = __sync_lock_test_and_set(&pool->state.u64, new.u64);
 371       if (old.next != state.next)
 372          futex_wake(&pool->state.end, INT_MAX);
 373       return state.next;
 374    } else {
 375       futex_wait(&pool->state.end, state.end);
 376       goto restart;
 377    }
 378 }
 379
 380 void
 381 anv_block_pool_free(struct anv_block_pool *pool, uint32_t offset)
 382 {
 383    anv_free_list_push(&pool->free_list, pool->map, offset);
 384 }
 385
 386 static void
 387 anv_fixed_size_state_pool_init(struct anv_fixed_size_state_pool *pool,
 388                                size_t state_size)
 389 {
 390    /* At least a cache line and must divide the block size. */
 391    assert(state_size >= 64 && is_power_of_two(state_size));
 392
 393    pool->state_size = state_size;
 394    pool->free_list = ANV_FREE_LIST_EMPTY;
 395    pool->block.next = 0;
 396    pool->block.end = 0;
 397 }
 398
 399 static uint32_t
 400 anv_fixed_size_state_pool_alloc(struct anv_fixed_size_state_pool *pool,
 401                                 struct anv_block_pool *block_pool)
 402 {
 403    uint32_t offset;
 404    struct anv_block_state block, old, new;
 405
 406    /* Try free list first. */
 407    if (anv_free_list_pop(&pool->free_list, &block_pool->map, &offset))
 408       return offset;
 409
 410    /* If free list was empty (or somebody raced us and took the items) we
 411     * allocate a new item from the end of the block */
 412  restart:
 413    block.u64 = __sync_fetch_and_add(&pool->block.u64, pool->state_size);
 414
 415    if (block.next < block.end) {
 416       return block.next;
 417    } else if (block.next == block.end) {
 418       offset = anv_block_pool_alloc(block_pool);
 419       new.next = offset + pool->state_size;
 420       new.end = offset + block_pool->block_size;
 421       old.u64 = __sync_lock_test_and_set(&pool->block.u64, new.u64);
 422       if (old.next != block.next)
 423          futex_wake(&pool->block.end, INT_MAX);
 424       return offset;
 425    } else {
 426       futex_wait(&pool->block.end, block.end);
 427       goto restart;
 428    }
 429 }
 430
 431 static void
 432 anv_fixed_size_state_pool_free(struct anv_fixed_size_state_pool *pool,
 433                                struct anv_block_pool *block_pool,
 434                                uint32_t offset)
 435 {
 436    anv_free_list_push(&pool->free_list, block_pool->map, offset);
 437 }
 438
 439 void
 440 anv_state_pool_init(struct anv_state_pool *pool,
 441                     struct anv_block_pool *block_pool)
 442 {
 443    pool->block_pool = block_pool;
 444    for (unsigned i = 0; i < ANV_STATE_BUCKETS; i++) {
 445       size_t size = 1 << (ANV_MIN_STATE_SIZE_LOG2 + i);
 446       anv_fixed_size_state_pool_init(&pool->buckets[i], size);
 447    }
 448    VG(VALGRIND_CREATE_MEMPOOL(pool, 0, false));
 449 }
 450
 451 void
 452 anv_state_pool_finish(struct anv_state_pool *pool)
 453 {
 454    VG(VALGRIND_DESTROY_MEMPOOL(pool));
 455 }
 456
 457 struct anv_state
 458 anv_state_pool_alloc(struct anv_state_pool *pool, size_t size, size_t align)
 459 {
 460    unsigned size_log2 = ilog2_round_up(size < align ? align : size);
 461    assert(size_log2 <= ANV_MAX_STATE_SIZE_LOG2);
 462    if (size_log2 < ANV_MIN_STATE_SIZE_LOG2)
 463       size_log2 = ANV_MIN_STATE_SIZE_LOG2;
 464    unsigned bucket = size_log2 - ANV_MIN_STATE_SIZE_LOG2;
 465
 466    struct anv_state state;
 467    state.alloc_size = 1 << size_log2;
 468    state.offset = anv_fixed_size_state_pool_alloc(&pool->buckets[bucket],
 469                                                   pool->block_pool);
 470    state.map = pool->block_pool->map + state.offset;
 471    VG(VALGRIND_MEMPOOL_ALLOC(pool, state.map, size));
 472    return state;
 473 }
 474
 475 void
 476 anv_state_pool_free(struct anv_state_pool *pool, struct anv_state state)
 477 {
 478    assert(is_power_of_two(state.alloc_size));
 479    unsigned size_log2 = ilog2_round_up(state.alloc_size);
 480    assert(size_log2 >= ANV_MIN_STATE_SIZE_LOG2 &&
 481           size_log2 <= ANV_MAX_STATE_SIZE_LOG2);
 482    unsigned bucket = size_log2 - ANV_MIN_STATE_SIZE_LOG2;
 483
 484    VG(VALGRIND_MEMPOOL_FREE(pool, state.map));
 485    anv_fixed_size_state_pool_free(&pool->buckets[bucket],
 486                                   pool->block_pool, state.offset);
 487 }
 488
 489 #define NULL_BLOCK 1
 490 struct stream_block {
 491    uint32_t next;
 492
 493    /* The map for the BO at the time the block was givne to us */
 494    void *current_map;
 495
 496 #ifdef HAVE_VALGRIND
 497    void *_vg_ptr;
 498 #endif
 499 };
 500
 501 /* The state stream allocator is a one-shot, single threaded allocator for
 502  * variable sized blocks.  We use it for allocating dynamic state.
 503  */
 504 void
 505 anv_state_stream_init(struct anv_state_stream *stream,
 506                       struct anv_block_pool *block_pool)
 507 {
 508    stream->block_pool = block_pool;
 509    stream->next = 0;
 510    stream->end = 0;
 511    stream->current_block = NULL_BLOCK;
 512
 513    VG(VALGRIND_CREATE_MEMPOOL(stream, 0, false));
 514 }
 515
 516 void
 517 anv_state_stream_finish(struct anv_state_stream *stream)
 518 {
 519    struct stream_block *sb;
 520    uint32_t block, next_block;
 521
 522    block = stream->current_block;
 523    while (block != NULL_BLOCK) {
 524       sb = stream->block_pool->map + block;
 525       next_block = VG_NOACCESS_READ(&sb->next);
 526       VG(VALGRIND_MEMPOOL_FREE(stream, VG_NOACCESS_READ(&sb->_vg_ptr)));
 527       anv_block_pool_free(stream->block_pool, block);
 528       block = next_block;
 529    }
 530
 531    VG(VALGRIND_DESTROY_MEMPOOL(stream));
 532 }
 533
 534 struct anv_state
 535 anv_state_stream_alloc(struct anv_state_stream *stream,
 536                        uint32_t size, uint32_t alignment)
 537 {
 538    struct stream_block *sb;
 539    struct anv_state state;
 540    uint32_t block;
 541
 542    state.offset = align_u32(stream->next, alignment);
 543    if (state.offset + size > stream->end) {
 544       block = anv_block_pool_alloc(stream->block_pool);
 545       void *current_map = stream->block_pool->map;
 546       sb = current_map + block;
 547       VG_NOACCESS_WRITE(&sb->current_map, current_map);
 548       VG_NOACCESS_WRITE(&sb->next, stream->current_block);
 549       VG(VG_NOACCESS_WRITE(&sb->_vg_ptr, 0));
 550       stream->current_block = block;
 551       stream->next = block + sizeof(*sb);
 552       stream->end = block + stream->block_pool->block_size;
 553       state.offset = align_u32(stream->next, alignment);
 554       assert(state.offset + size <= stream->end);
 555    }
 556
 557    sb = stream->block_pool->map + stream->current_block;
 558    void *current_map = VG_NOACCESS_READ(&sb->current_map);
 559
 560    state.map = current_map + state.offset;
 561    state.alloc_size = size;
 562
 563 #ifdef HAVE_VALGRIND
 564    void *vg_ptr = VG_NOACCESS_READ(&sb->_vg_ptr);
 565    if (vg_ptr == NULL) {
 566       vg_ptr = state.map;
 567       VG_NOACCESS_WRITE(&sb->_vg_ptr, vg_ptr);
 568       VALGRIND_MEMPOOL_ALLOC(stream, vg_ptr, size);
 569    } else {
 570       ptrdiff_t vg_offset = vg_ptr - current_map;
 571       assert(vg_offset >= stream->current_block &&
 572              vg_offset < stream->end);
 573       VALGRIND_MEMPOOL_CHANGE(stream, vg_ptr, vg_ptr,
 574                               (state.offset + size) - vg_offset);
 575    }
 576 #endif
 577
 578    stream->next = state.offset + size;
 579
 580    return state;
 581 }
 582
 583 struct bo_pool_bo_link {
 584    struct bo_pool_bo_link *next;
 585    struct anv_bo bo;
 586 };
 587
 588 void
 589 anv_bo_pool_init(struct anv_bo_pool *pool,
 590                  struct anv_device *device, uint32_t bo_size)
 591 {
 592    pool->device = device;
 593    pool->bo_size = bo_size;
 594    pool->free_list = NULL;
 595
 596    VG(VALGRIND_CREATE_MEMPOOL(pool, 0, false));
 597 }
 598
 599 void
 600 anv_bo_pool_finish(struct anv_bo_pool *pool)
 601 {
 602    struct bo_pool_bo_link *link = PFL_PTR(pool->free_list);
 603    while (link != NULL) {
 604       struct bo_pool_bo_link link_copy = VG_NOACCESS_READ(link);
 605
 606       anv_gem_munmap(link_copy.bo.map, pool->bo_size);
 607       anv_gem_close(pool->device, link_copy.bo.gem_handle);
 608       link = link_copy.next;
 609    }
 610
 611    VG(VALGRIND_DESTROY_MEMPOOL(pool));
 612 }
 613
 614 VkResult
 615 anv_bo_pool_alloc(struct anv_bo_pool *pool, struct anv_bo *bo)
 616 {
 617    VkResult result;
 618
 619    void *next_free_void;
 620    if (anv_ptr_free_list_pop(&pool->free_list, &next_free_void)) {
 621       struct bo_pool_bo_link *next_free = next_free_void;
 622       *bo = VG_NOACCESS_READ(&next_free->bo);
 623       assert(bo->map == next_free);
 624       assert(bo->size == pool->bo_size);
 625
 626       VG(VALGRIND_MEMPOOL_ALLOC(pool, bo->map, pool->bo_size));
 627
 628       return VK_SUCCESS;
 629    }
 630
 631    struct anv_bo new_bo;
 632
 633    result = anv_bo_init_new(&new_bo, pool->device, pool->bo_size);
 634    if (result != VK_SUCCESS)
 635       return result;
 636
 637    assert(new_bo.size == pool->bo_size);
 638
 639    new_bo.map = anv_gem_mmap(pool->device, new_bo.gem_handle, 0, pool->bo_size);
 640    if (new_bo.map == NULL) {
 641       anv_gem_close(pool->device, new_bo.gem_handle);
 642       return vk_error(VK_ERROR_MEMORY_MAP_FAILED);
 643    }
 644
 645    *bo = new_bo;
 646
 647    VG(VALGRIND_MEMPOOL_ALLOC(pool, bo->map, pool->bo_size));
 648
 649    return VK_SUCCESS;
 650 }
 651
 652 void
 653 anv_bo_pool_free(struct anv_bo_pool *pool, const struct anv_bo *bo)
 654 {
 655    struct bo_pool_bo_link *link = bo->map;
 656    link->bo = *bo;
 657
 658    VG(VALGRIND_MEMPOOL_FREE(pool, bo->map));
 659    anv_ptr_free_list_push(&pool->free_list, link);
 660 }