X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fintel%2Fvulkan%2Fanv_allocator.c;h=9007cd00e855ac23cdbee9246ff977d65ba91a97;hb=fcdefa7e479541a92f02b1933f58439e0fd03a1f;hp=6358566ae2e4281e0404b6fb496ecd4054a47a17;hpb=e049dea5b280bebd6480823a4cddd70baf46fea7;p=mesa.git diff --git a/src/intel/vulkan/anv_allocator.c b/src/intel/vulkan/anv_allocator.c index 6358566ae2e..9007cd00e85 100644 --- a/src/intel/vulkan/anv_allocator.c +++ b/src/intel/vulkan/anv_allocator.c @@ -21,20 +21,16 @@ * IN THE SOFTWARE. */ -#include #include #include #include #include -#include -#include -#include #include -#include #include "anv_private.h" -#include "util/hash_table.h" +#include "common/gen_aux_map.h" +#include "util/anon_file.h" #ifdef HAVE_VALGRIND #define VG_NOACCESS_READ(__ptr) ({ \ @@ -53,6 +49,10 @@ #define VG_NOACCESS_WRITE(__ptr, __val) (*(__ptr) = (__val)) #endif +#ifndef MAP_POPULATE +#define MAP_POPULATE 0 +#endif + /* Design goals: * * - Lock free (except when resizing underlying bos) @@ -102,177 +102,299 @@ /* Allocations are always at least 64 byte aligned, so 1 is an invalid value. * We use it to indicate the free list is empty. */ -#define EMPTY 1 +#define EMPTY UINT32_MAX + +#define PAGE_SIZE 4096 struct anv_mmap_cleanup { void *map; size_t size; - uint32_t gem_handle; }; -#define ANV_MMAP_CLEANUP_INIT ((struct anv_mmap_cleanup){0}) - -static inline long -sys_futex(void *addr1, int op, int val1, - struct timespec *timeout, void *addr2, int val3) +static inline uint32_t +ilog2_round_up(uint32_t value) { - return syscall(SYS_futex, addr1, op, val1, timeout, addr2, val3); + assert(value != 0); + return 32 - __builtin_clz(value - 1); } -static inline int -futex_wake(uint32_t *addr, int count) +static inline uint32_t +round_to_power_of_two(uint32_t value) { - return sys_futex(addr, FUTEX_WAKE, count, NULL, NULL, 0); + return 1 << ilog2_round_up(value); } -static inline int -futex_wait(uint32_t *addr, int32_t value) +struct anv_state_table_cleanup { + void *map; + size_t size; +}; + +#define ANV_STATE_TABLE_CLEANUP_INIT ((struct anv_state_table_cleanup){0}) +#define ANV_STATE_ENTRY_SIZE (sizeof(struct anv_free_entry)) + +static VkResult +anv_state_table_expand_range(struct anv_state_table *table, uint32_t size); + +VkResult +anv_state_table_init(struct anv_state_table *table, + struct anv_device *device, + uint32_t initial_entries) { - return sys_futex(addr, FUTEX_WAIT, value, NULL, NULL, 0); + VkResult result; + + table->device = device; + + /* Just make it 2GB up-front. The Linux kernel won't actually back it + * with pages until we either map and fault on one of them or we use + * userptr and send a chunk of it off to the GPU. + */ + table->fd = os_create_anonymous_file(BLOCK_POOL_MEMFD_SIZE, "state table"); + if (table->fd == -1) { + result = vk_error(VK_ERROR_INITIALIZATION_FAILED); + goto fail_fd; + } + + if (!u_vector_init(&table->cleanups, + round_to_power_of_two(sizeof(struct anv_state_table_cleanup)), + 128)) { + result = vk_error(VK_ERROR_INITIALIZATION_FAILED); + goto fail_fd; + } + + table->state.next = 0; + table->state.end = 0; + table->size = 0; + + uint32_t initial_size = initial_entries * ANV_STATE_ENTRY_SIZE; + result = anv_state_table_expand_range(table, initial_size); + if (result != VK_SUCCESS) + goto fail_cleanups; + + return VK_SUCCESS; + + fail_cleanups: + u_vector_finish(&table->cleanups); + fail_fd: + close(table->fd); + + return result; } -static inline int -memfd_create(const char *name, unsigned int flags) +static VkResult +anv_state_table_expand_range(struct anv_state_table *table, uint32_t size) { - return syscall(SYS_memfd_create, name, flags); + void *map; + struct anv_state_table_cleanup *cleanup; + + /* Assert that we only ever grow the pool */ + assert(size >= table->state.end); + + /* Make sure that we don't go outside the bounds of the memfd */ + if (size > BLOCK_POOL_MEMFD_SIZE) + return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); + + cleanup = u_vector_add(&table->cleanups); + if (!cleanup) + return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); + + *cleanup = ANV_STATE_TABLE_CLEANUP_INIT; + + /* Just leak the old map until we destroy the pool. We can't munmap it + * without races or imposing locking on the block allocate fast path. On + * the whole the leaked maps adds up to less than the size of the + * current map. MAP_POPULATE seems like the right thing to do, but we + * should try to get some numbers. + */ + map = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, table->fd, 0); + if (map == MAP_FAILED) { + return vk_errorf(table->device, table->device, + VK_ERROR_OUT_OF_HOST_MEMORY, "mmap failed: %m"); + } + + cleanup->map = map; + cleanup->size = size; + + table->map = map; + table->size = size; + + return VK_SUCCESS; } -static inline uint32_t -ilog2_round_up(uint32_t value) +static VkResult +anv_state_table_grow(struct anv_state_table *table) { - assert(value != 0); - return 32 - __builtin_clz(value - 1); + VkResult result = VK_SUCCESS; + + uint32_t used = align_u32(table->state.next * ANV_STATE_ENTRY_SIZE, + PAGE_SIZE); + uint32_t old_size = table->size; + + /* The block pool is always initialized to a nonzero size and this function + * is always called after initialization. + */ + assert(old_size > 0); + + uint32_t required = MAX2(used, old_size); + if (used * 2 <= required) { + /* If we're in this case then this isn't the firsta allocation and we + * already have enough space on both sides to hold double what we + * have allocated. There's nothing for us to do. + */ + goto done; + } + + uint32_t size = old_size * 2; + while (size < required) + size *= 2; + + assert(size > table->size); + + result = anv_state_table_expand_range(table, size); + + done: + return result; } -static inline uint32_t -round_to_power_of_two(uint32_t value) +void +anv_state_table_finish(struct anv_state_table *table) { - return 1 << ilog2_round_up(value); + struct anv_state_table_cleanup *cleanup; + + u_vector_foreach(cleanup, &table->cleanups) { + if (cleanup->map) + munmap(cleanup->map, cleanup->size); + } + + u_vector_finish(&table->cleanups); + + close(table->fd); } -static bool -anv_free_list_pop(union anv_free_list *list, void **map, int32_t *offset) +VkResult +anv_state_table_add(struct anv_state_table *table, uint32_t *idx, + uint32_t count) { - union anv_free_list current, new, old; - - current.u64 = list->u64; - while (current.offset != EMPTY) { - /* We have to add a memory barrier here so that the list head (and - * offset) gets read before we read the map pointer. This way we - * know that the map pointer is valid for the given offset at the - * point where we read it. - */ - __sync_synchronize(); + struct anv_block_state state, old, new; + VkResult result; - int32_t *next_ptr = *map + current.offset; - new.offset = VG_NOACCESS_READ(next_ptr); - new.count = current.count + 1; - old.u64 = __sync_val_compare_and_swap(&list->u64, current.u64, new.u64); - if (old.u64 == current.u64) { - *offset = current.offset; - return true; + assert(idx); + + while(1) { + state.u64 = __sync_fetch_and_add(&table->state.u64, count); + if (state.next + count <= state.end) { + assert(table->map); + struct anv_free_entry *entry = &table->map[state.next]; + for (int i = 0; i < count; i++) { + entry[i].state.idx = state.next + i; + } + *idx = state.next; + return VK_SUCCESS; + } else if (state.next <= state.end) { + /* We allocated the first block outside the pool so we have to grow + * the pool. pool_state->next acts a mutex: threads who try to + * allocate now will get block indexes above the current limit and + * hit futex_wait below. + */ + new.next = state.next + count; + do { + result = anv_state_table_grow(table); + if (result != VK_SUCCESS) + return result; + new.end = table->size / ANV_STATE_ENTRY_SIZE; + } while (new.end < new.next); + + old.u64 = __sync_lock_test_and_set(&table->state.u64, new.u64); + if (old.next != state.next) + futex_wake(&table->state.end, INT_MAX); + } else { + futex_wait(&table->state.end, state.end, NULL); + continue; } - current = old; } - - return false; } -static void -anv_free_list_push(union anv_free_list *list, void *map, int32_t offset) +void +anv_free_list_push(union anv_free_list *list, + struct anv_state_table *table, + uint32_t first, uint32_t count) { union anv_free_list current, old, new; - int32_t *next_ptr = map + offset; + uint32_t last = first; + + for (uint32_t i = 1; i < count; i++, last++) + table->map[last].next = last + 1; old = *list; do { current = old; - VG_NOACCESS_WRITE(next_ptr, current.offset); - new.offset = offset; + table->map[last].next = current.offset; + new.offset = first; new.count = current.count + 1; old.u64 = __sync_val_compare_and_swap(&list->u64, current.u64, new.u64); } while (old.u64 != current.u64); } -/* All pointers in the ptr_free_list are assumed to be page-aligned. This - * means that the bottom 12 bits should all be zero. - */ -#define PFL_COUNT(x) ((uintptr_t)(x) & 0xfff) -#define PFL_PTR(x) ((void *)((uintptr_t)(x) & ~(uintptr_t)0xfff)) -#define PFL_PACK(ptr, count) ({ \ - (void *)(((uintptr_t)(ptr) & ~(uintptr_t)0xfff) | ((count) & 0xfff)); \ -}) - -static bool -anv_ptr_free_list_pop(void **list, void **elem) +struct anv_state * +anv_free_list_pop(union anv_free_list *list, + struct anv_state_table *table) { - void *current = *list; - while (PFL_PTR(current) != NULL) { - void **next_ptr = PFL_PTR(current); - void *new_ptr = VG_NOACCESS_READ(next_ptr); - unsigned new_count = PFL_COUNT(current) + 1; - void *new = PFL_PACK(new_ptr, new_count); - void *old = __sync_val_compare_and_swap(list, current, new); - if (old == current) { - *elem = PFL_PTR(current); - return true; + union anv_free_list current, new, old; + + current.u64 = list->u64; + while (current.offset != EMPTY) { + __sync_synchronize(); + new.offset = table->map[current.offset].next; + new.count = current.count + 1; + old.u64 = __sync_val_compare_and_swap(&list->u64, current.u64, new.u64); + if (old.u64 == current.u64) { + struct anv_free_entry *entry = &table->map[current.offset]; + return &entry->state; } current = old; } - return false; + return NULL; } -static void -anv_ptr_free_list_push(void **list, void *elem) -{ - void *old, *current; - void **next_ptr = elem; - - /* The pointer-based free list requires that the pointer be - * page-aligned. This is because we use the bottom 12 bits of the - * pointer to store a counter to solve the ABA concurrency problem. - */ - assert(((uintptr_t)elem & 0xfff) == 0); - - old = *list; - do { - current = old; - VG_NOACCESS_WRITE(next_ptr, PFL_PTR(current)); - unsigned new_count = PFL_COUNT(current) + 1; - void *new = PFL_PACK(elem, new_count); - old = __sync_val_compare_and_swap(list, current, new); - } while (old != current); -} - -static uint32_t -anv_block_pool_grow(struct anv_block_pool *pool, struct anv_block_state *state); +static VkResult +anv_block_pool_expand_range(struct anv_block_pool *pool, + uint32_t center_bo_offset, uint32_t size); VkResult anv_block_pool_init(struct anv_block_pool *pool, - struct anv_device *device, uint32_t block_size) + struct anv_device *device, + uint64_t start_address, + uint32_t initial_size) { VkResult result; - assert(util_is_power_of_two(block_size)); - pool->device = device; - anv_bo_init(&pool->bo, 0, 0); - pool->block_size = block_size; - pool->free_list = ANV_FREE_LIST_EMPTY; - pool->back_free_list = ANV_FREE_LIST_EMPTY; - - pool->fd = memfd_create("block pool", MFD_CLOEXEC); - if (pool->fd == -1) - return vk_error(VK_ERROR_INITIALIZATION_FAILED); - - /* Just make it 2GB up-front. The Linux kernel won't actually back it - * with pages until we either map and fault on one of them or we use - * userptr and send a chunk of it off to the GPU. - */ - if (ftruncate(pool->fd, BLOCK_POOL_MEMFD_SIZE) == -1) { - result = vk_error(VK_ERROR_INITIALIZATION_FAILED); - goto fail_fd; + pool->use_softpin = device->physical->use_softpin; + pool->nbos = 0; + pool->size = 0; + pool->center_bo_offset = 0; + pool->start_address = gen_canonical_address(start_address); + pool->map = NULL; + + if (pool->use_softpin) { + pool->bo = NULL; + pool->fd = -1; + } else { + /* Just make it 2GB up-front. The Linux kernel won't actually back it + * with pages until we either map and fault on one of them or we use + * userptr and send a chunk of it off to the GPU. + */ + pool->fd = os_create_anonymous_file(BLOCK_POOL_MEMFD_SIZE, "block pool"); + if (pool->fd == -1) + return vk_error(VK_ERROR_INITIALIZATION_FAILED); + + pool->wrapper_bo = (struct anv_bo) { + .refcount = 1, + .offset = -1, + .is_wrapper = true, + }; + pool->bo = &pool->wrapper_bo; } if (!u_vector_init(&pool->mmap_cleanups, @@ -287,13 +409,22 @@ anv_block_pool_init(struct anv_block_pool *pool, pool->back_state.next = 0; pool->back_state.end = 0; - /* Immediately grow the pool so we'll have a backing bo. */ - pool->state.end = anv_block_pool_grow(pool, &pool->state); + result = anv_block_pool_expand_range(pool, 0, initial_size); + if (result != VK_SUCCESS) + goto fail_mmap_cleanups; + + /* Make the entire pool available in the front of the pool. If back + * allocation needs to use this space, the "ends" will be re-arranged. + */ + pool->state.end = pool->size; return VK_SUCCESS; + fail_mmap_cleanups: + u_vector_finish(&pool->mmap_cleanups); fail_fd: - close(pool->fd); + if (pool->fd >= 0) + close(pool->fd); return result; } @@ -301,21 +432,160 @@ anv_block_pool_init(struct anv_block_pool *pool, void anv_block_pool_finish(struct anv_block_pool *pool) { + anv_block_pool_foreach_bo(bo, pool) { + if (bo->map) + anv_gem_munmap(pool->device, bo->map, bo->size); + anv_gem_close(pool->device, bo->gem_handle); + } + struct anv_mmap_cleanup *cleanup; + u_vector_foreach(cleanup, &pool->mmap_cleanups) + munmap(cleanup->map, cleanup->size); + u_vector_finish(&pool->mmap_cleanups); - u_vector_foreach(cleanup, &pool->mmap_cleanups) { - if (cleanup->map) - munmap(cleanup->map, cleanup->size); - if (cleanup->gem_handle) - anv_gem_close(pool->device, cleanup->gem_handle); + if (pool->fd >= 0) + close(pool->fd); +} + +static VkResult +anv_block_pool_expand_range(struct anv_block_pool *pool, + uint32_t center_bo_offset, uint32_t size) +{ + /* Assert that we only ever grow the pool */ + assert(center_bo_offset >= pool->back_state.end); + assert(size - center_bo_offset >= pool->state.end); + + /* Assert that we don't go outside the bounds of the memfd */ + assert(center_bo_offset <= BLOCK_POOL_MEMFD_CENTER); + assert(pool->use_softpin || + size - center_bo_offset <= + BLOCK_POOL_MEMFD_SIZE - BLOCK_POOL_MEMFD_CENTER); + + /* For state pool BOs we have to be a bit careful about where we place them + * in the GTT. There are two documented workarounds for state base address + * placement : Wa32bitGeneralStateOffset and Wa32bitInstructionBaseOffset + * which state that those two base addresses do not support 48-bit + * addresses and need to be placed in the bottom 32-bit range. + * Unfortunately, this is not quite accurate. + * + * The real problem is that we always set the size of our state pools in + * STATE_BASE_ADDRESS to 0xfffff (the maximum) even though the BO is most + * likely significantly smaller. We do this because we do not no at the + * time we emit STATE_BASE_ADDRESS whether or not we will need to expand + * the pool during command buffer building so we don't actually have a + * valid final size. If the address + size, as seen by STATE_BASE_ADDRESS + * overflows 48 bits, the GPU appears to treat all accesses to the buffer + * as being out of bounds and returns zero. For dynamic state, this + * usually just leads to rendering corruptions, but shaders that are all + * zero hang the GPU immediately. + * + * The easiest solution to do is exactly what the bogus workarounds say to + * do: restrict these buffers to 32-bit addresses. We could also pin the + * BO to some particular location of our choosing, but that's significantly + * more work than just not setting a flag. So, we explicitly DO NOT set + * the EXEC_OBJECT_SUPPORTS_48B_ADDRESS flag and the kernel does all of the + * hard work for us. When using softpin, we're in control and the fixed + * addresses we choose are fine for base addresses. + */ + enum anv_bo_alloc_flags bo_alloc_flags = ANV_BO_ALLOC_CAPTURE; + if (!pool->use_softpin) + bo_alloc_flags |= ANV_BO_ALLOC_32BIT_ADDRESS; + + if (pool->use_softpin) { + uint32_t new_bo_size = size - pool->size; + struct anv_bo *new_bo; + assert(center_bo_offset == 0); + VkResult result = anv_device_alloc_bo(pool->device, new_bo_size, + bo_alloc_flags | + ANV_BO_ALLOC_FIXED_ADDRESS | + ANV_BO_ALLOC_MAPPED | + ANV_BO_ALLOC_SNOOPED, + pool->start_address + pool->size, + &new_bo); + if (result != VK_SUCCESS) + return result; + + pool->bos[pool->nbos++] = new_bo; + + /* This pointer will always point to the first BO in the list */ + pool->bo = pool->bos[0]; + } else { + /* Just leak the old map until we destroy the pool. We can't munmap it + * without races or imposing locking on the block allocate fast path. On + * the whole the leaked maps adds up to less than the size of the + * current map. MAP_POPULATE seems like the right thing to do, but we + * should try to get some numbers. + */ + void *map = mmap(NULL, size, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_POPULATE, pool->fd, + BLOCK_POOL_MEMFD_CENTER - center_bo_offset); + if (map == MAP_FAILED) + return vk_errorf(pool->device, pool->device, + VK_ERROR_MEMORY_MAP_FAILED, "mmap failed: %m"); + + struct anv_bo *new_bo; + VkResult result = anv_device_import_bo_from_host_ptr(pool->device, + map, size, + bo_alloc_flags, + 0 /* client_address */, + &new_bo); + if (result != VK_SUCCESS) { + munmap(map, size); + return result; + } + + struct anv_mmap_cleanup *cleanup = u_vector_add(&pool->mmap_cleanups); + if (!cleanup) { + munmap(map, size); + anv_device_release_bo(pool->device, new_bo); + return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); + } + cleanup->map = map; + cleanup->size = size; + + /* Now that we mapped the new memory, we can write the new + * center_bo_offset back into pool and update pool->map. */ + pool->center_bo_offset = center_bo_offset; + pool->map = map + center_bo_offset; + + pool->bos[pool->nbos++] = new_bo; + pool->wrapper_bo.map = new_bo; } - u_vector_finish(&pool->mmap_cleanups); + assert(pool->nbos < ANV_MAX_BLOCK_POOL_BOS); + pool->size = size; - close(pool->fd); + return VK_SUCCESS; } -#define PAGE_SIZE 4096 +/** Returns current memory map of the block pool. + * + * The returned pointer points to the map for the memory at the specified + * offset. The offset parameter is relative to the "center" of the block pool + * rather than the start of the block pool BO map. + */ +void* +anv_block_pool_map(struct anv_block_pool *pool, int32_t offset, uint32_t size) +{ + if (pool->use_softpin) { + struct anv_bo *bo = NULL; + int32_t bo_offset = 0; + anv_block_pool_foreach_bo(iter_bo, pool) { + if (offset < bo_offset + iter_bo->size) { + bo = iter_bo; + break; + } + bo_offset += iter_bo->size; + } + assert(bo != NULL); + assert(offset >= bo_offset); + assert((offset - bo_offset) + size <= bo->size); + + return bo->map + (offset - bo_offset); + } else { + return pool->map + offset; + } +} /** Grows and re-centers the block pool. * @@ -342,12 +612,10 @@ anv_block_pool_finish(struct anv_block_pool *pool) * the pool and a 4K CPU page. */ static uint32_t -anv_block_pool_grow(struct anv_block_pool *pool, struct anv_block_state *state) +anv_block_pool_grow(struct anv_block_pool *pool, struct anv_block_state *state, + uint32_t contiguous_size) { - size_t size; - void *map; - uint32_t gem_handle; - struct anv_mmap_cleanup *cleanup; + VkResult result = VK_SUCCESS; pthread_mutex_lock(&pool->device->mutex); @@ -369,11 +637,33 @@ anv_block_pool_grow(struct anv_block_pool *pool, struct anv_block_state *state) assert(state == &pool->state || back_used > 0); - size_t old_size = pool->bo.size; + uint32_t old_size = pool->size; + + /* The block pool is always initialized to a nonzero size and this function + * is always called after initialization. + */ + assert(old_size > 0); + + const uint32_t old_back = pool->center_bo_offset; + const uint32_t old_front = old_size - pool->center_bo_offset; - if (old_size != 0 && - back_used * 2 <= pool->center_bo_offset && - front_used * 2 <= (old_size - pool->center_bo_offset)) { + /* The back_used and front_used may actually be smaller than the actual + * requirement because they are based on the next pointers which are + * updated prior to calling this function. + */ + uint32_t back_required = MAX2(back_used, old_back); + uint32_t front_required = MAX2(front_used, old_front); + + if (pool->use_softpin) { + /* With softpin, the pool is made up of a bunch of buffers with separate + * maps. Make sure we have enough contiguous space that we can get a + * properly contiguous map for the next chunk. + */ + assert(old_back == 0); + front_required = MAX2(front_required, old_front + contiguous_size); + } + + if (back_used * 2 <= back_required && front_used * 2 <= front_required) { /* If we're in this case then this isn't the firsta allocation and we * already have enough space on both sides to hold double what we * have allocated. There's nothing for us to do. @@ -381,18 +671,11 @@ anv_block_pool_grow(struct anv_block_pool *pool, struct anv_block_state *state) goto done; } - if (old_size == 0) { - /* This is the first allocation */ - size = MAX2(32 * pool->block_size, PAGE_SIZE); - } else { - size = old_size * 2; - } + uint32_t size = old_size * 2; + while (size < back_required + front_required) + size *= 2; - /* We can't have a block pool bigger than 1GB because we use signed - * 32-bit offsets in the free list and we don't want overflow. We - * should never need a block pool bigger than 1GB anyway. - */ - assert(size <= (1u << 31)); + assert(size > pool->size); /* We compute a new center_bo_offset such that, when we double the size * of the pool, we maintain the ratio of how much is used by each side. @@ -411,167 +694,105 @@ anv_block_pool_grow(struct anv_block_pool *pool, struct anv_block_state *state) */ center_bo_offset = ((uint64_t)size * back_used) / total_used; - /* Align down to a multiple of both the block size and page size */ - uint32_t granularity = MAX2(pool->block_size, PAGE_SIZE); - assert(util_is_power_of_two(granularity)); - center_bo_offset &= ~(granularity - 1); + /* Align down to a multiple of the page size */ + center_bo_offset &= ~(PAGE_SIZE - 1); assert(center_bo_offset >= back_used); /* Make sure we don't shrink the back end of the pool */ - if (center_bo_offset < pool->back_state.end) - center_bo_offset = pool->back_state.end; + if (center_bo_offset < back_required) + center_bo_offset = back_required; /* Make sure that we don't shrink the front end of the pool */ - if (size - center_bo_offset < pool->state.end) - center_bo_offset = size - pool->state.end; + if (size - center_bo_offset < front_required) + center_bo_offset = size - front_required; } - assert(center_bo_offset % pool->block_size == 0); assert(center_bo_offset % PAGE_SIZE == 0); - /* Assert that we only ever grow the pool */ - assert(center_bo_offset >= pool->back_state.end); - assert(size - center_bo_offset >= pool->state.end); - - cleanup = u_vector_add(&pool->mmap_cleanups); - if (!cleanup) - goto fail; - *cleanup = ANV_MMAP_CLEANUP_INIT; - - /* Just leak the old map until we destroy the pool. We can't munmap it - * without races or imposing locking on the block allocate fast path. On - * the whole the leaked maps adds up to less than the size of the - * current map. MAP_POPULATE seems like the right thing to do, but we - * should try to get some numbers. - */ - map = mmap(NULL, size, PROT_READ | PROT_WRITE, - MAP_SHARED | MAP_POPULATE, pool->fd, - BLOCK_POOL_MEMFD_CENTER - center_bo_offset); - cleanup->map = map; - cleanup->size = size; - - if (map == MAP_FAILED) - goto fail; - - gem_handle = anv_gem_userptr(pool->device, map, size); - if (gem_handle == 0) - goto fail; - cleanup->gem_handle = gem_handle; - -#if 0 - /* Regular objects are created I915_CACHING_CACHED on LLC platforms and - * I915_CACHING_NONE on non-LLC platforms. However, userptr objects are - * always created as I915_CACHING_CACHED, which on non-LLC means - * snooped. That can be useful but comes with a bit of overheard. Since - * we're eplicitly clflushing and don't want the overhead we need to turn - * it off. */ - if (!pool->device->info.has_llc) { - anv_gem_set_caching(pool->device, gem_handle, I915_CACHING_NONE); - anv_gem_set_domain(pool->device, gem_handle, - I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT); - } -#endif - - /* Now that we successfull allocated everything, we can write the new - * values back into pool. */ - pool->map = map + center_bo_offset; - pool->center_bo_offset = center_bo_offset; - - /* For block pool BOs we have to be a bit careful about where we place them - * in the GTT. There are two documented workarounds for state base address - * placement : Wa32bitGeneralStateOffset and Wa32bitInstructionBaseOffset - * which state that those two base addresses do not support 48-bit - * addresses and need to be placed in the bottom 32-bit range. - * Unfortunately, this is not quite accurate. - * - * The real problem is that we always set the size of our state pools in - * STATE_BASE_ADDRESS to 0xfffff (the maximum) even though the BO is most - * likely significantly smaller. We do this because we do not no at the - * time we emit STATE_BASE_ADDRESS whether or not we will need to expand - * the pool during command buffer building so we don't actually have a - * valid final size. If the address + size, as seen by STATE_BASE_ADDRESS - * overflows 48 bits, the GPU appears to treat all accesses to the buffer - * as being out of bounds and returns zero. For dynamic state, this - * usually just leads to rendering corruptions, but shaders that are all - * zero hang the GPU immediately. - * - * The easiest solution to do is exactly what the bogus workarounds say to - * do: restrict these buffers to 32-bit addresses. We could also pin the - * BO to some particular location of our choosing, but that's significantly - * more work than just not setting a flag. So, we explicitly DO NOT set - * the EXEC_OBJECT_SUPPORTS_48B_ADDRESS flag and the kernel does all of the - * hard work for us. - */ - anv_bo_init(&pool->bo, gem_handle, size); - pool->bo.map = map; - - if (pool->device->instance->physicalDevice.has_exec_async) - pool->bo.flags |= EXEC_OBJECT_ASYNC; + result = anv_block_pool_expand_range(pool, center_bo_offset, size); done: pthread_mutex_unlock(&pool->device->mutex); - /* Return the appropreate new size. This function never actually - * updates state->next. Instead, we let the caller do that because it - * needs to do so in order to maintain its concurrency model. - */ - if (state == &pool->state) { - return pool->bo.size - pool->center_bo_offset; + if (result == VK_SUCCESS) { + /* Return the appropriate new size. This function never actually + * updates state->next. Instead, we let the caller do that because it + * needs to do so in order to maintain its concurrency model. + */ + if (state == &pool->state) { + return pool->size - pool->center_bo_offset; + } else { + assert(pool->center_bo_offset > 0); + return pool->center_bo_offset; + } } else { - assert(pool->center_bo_offset > 0); - return pool->center_bo_offset; + return 0; } - -fail: - pthread_mutex_unlock(&pool->device->mutex); - - return 0; } static uint32_t anv_block_pool_alloc_new(struct anv_block_pool *pool, - struct anv_block_state *pool_state) + struct anv_block_state *pool_state, + uint32_t block_size, uint32_t *padding) { struct anv_block_state state, old, new; + /* Most allocations won't generate any padding */ + if (padding) + *padding = 0; + while (1) { - state.u64 = __sync_fetch_and_add(&pool_state->u64, pool->block_size); - if (state.next < state.end) { - assert(pool->map); + state.u64 = __sync_fetch_and_add(&pool_state->u64, block_size); + if (state.next + block_size <= state.end) { return state.next; - } else if (state.next == state.end) { - /* We allocated the first block outside the pool, we have to grow it. - * pool_state->next acts a mutex: threads who try to allocate now will - * get block indexes above the current limit and hit futex_wait - * below. */ - new.next = state.next + pool->block_size; - new.end = anv_block_pool_grow(pool, pool_state); - assert(new.end >= new.next && new.end % pool->block_size == 0); + } else if (state.next <= state.end) { + if (pool->use_softpin && state.next < state.end) { + /* We need to grow the block pool, but still have some leftover + * space that can't be used by that particular allocation. So we + * add that as a "padding", and return it. + */ + uint32_t leftover = state.end - state.next; + + /* If there is some leftover space in the pool, the caller must + * deal with it. + */ + assert(leftover == 0 || padding); + if (padding) + *padding = leftover; + state.next += leftover; + } + + /* We allocated the first block outside the pool so we have to grow + * the pool. pool_state->next acts a mutex: threads who try to + * allocate now will get block indexes above the current limit and + * hit futex_wait below. + */ + new.next = state.next + block_size; + do { + new.end = anv_block_pool_grow(pool, pool_state, block_size); + } while (new.end < new.next); + old.u64 = __sync_lock_test_and_set(&pool_state->u64, new.u64); if (old.next != state.next) futex_wake(&pool_state->end, INT_MAX); return state.next; } else { - futex_wait(&pool_state->end, state.end); + futex_wait(&pool_state->end, state.end, NULL); continue; } } } int32_t -anv_block_pool_alloc(struct anv_block_pool *pool) +anv_block_pool_alloc(struct anv_block_pool *pool, + uint32_t block_size, uint32_t *padding) { - int32_t offset; + uint32_t offset; - /* Try free list first. */ - if (anv_free_list_pop(&pool->free_list, &pool->map, &offset)) { - assert(offset >= 0); - assert(pool->map); - return offset; - } + offset = anv_block_pool_alloc_new(pool, &pool->state, block_size, padding); - return anv_block_pool_alloc_new(pool, &pool->state); + return offset; } /* Allocates a block out of the back of the block pool. @@ -584,18 +805,11 @@ anv_block_pool_alloc(struct anv_block_pool *pool) * gymnastics with the block pool's BO when doing relocations. */ int32_t -anv_block_pool_alloc_back(struct anv_block_pool *pool) +anv_block_pool_alloc_back(struct anv_block_pool *pool, + uint32_t block_size) { - int32_t offset; - - /* Try free list first. */ - if (anv_free_list_pop(&pool->back_free_list, &pool->map, &offset)) { - assert(offset < 0); - assert(pool->map); - return offset; - } - - offset = anv_block_pool_alloc_new(pool, &pool->back_state); + int32_t offset = anv_block_pool_alloc_new(pool, &pool->back_state, + block_size, NULL); /* The offset we get out of anv_block_pool_alloc_new() is actually the * number of bytes downwards from the middle to the end of the block. @@ -603,112 +817,294 @@ anv_block_pool_alloc_back(struct anv_block_pool *pool) * start of the block. */ assert(offset >= 0); - return -(offset + pool->block_size); + return -(offset + block_size); } -void -anv_block_pool_free(struct anv_block_pool *pool, int32_t offset) -{ - if (offset < 0) { - anv_free_list_push(&pool->back_free_list, pool->map, offset); - } else { - anv_free_list_push(&pool->free_list, pool->map, offset); +VkResult +anv_state_pool_init(struct anv_state_pool *pool, + struct anv_device *device, + uint64_t base_address, + int32_t start_offset, + uint32_t block_size) +{ + /* We don't want to ever see signed overflow */ + assert(start_offset < INT32_MAX - (int32_t)BLOCK_POOL_MEMFD_SIZE); + + VkResult result = anv_block_pool_init(&pool->block_pool, device, + base_address + start_offset, + block_size * 16); + if (result != VK_SUCCESS) + return result; + + pool->start_offset = start_offset; + + result = anv_state_table_init(&pool->table, device, 64); + if (result != VK_SUCCESS) { + anv_block_pool_finish(&pool->block_pool); + return result; + } + + assert(util_is_power_of_two_or_zero(block_size)); + pool->block_size = block_size; + pool->back_alloc_free_list = ANV_FREE_LIST_EMPTY; + for (unsigned i = 0; i < ANV_STATE_BUCKETS; i++) { + pool->buckets[i].free_list = ANV_FREE_LIST_EMPTY; + pool->buckets[i].block.next = 0; + pool->buckets[i].block.end = 0; } + VG(VALGRIND_CREATE_MEMPOOL(pool, 0, false)); + + return VK_SUCCESS; } -static void -anv_fixed_size_state_pool_init(struct anv_fixed_size_state_pool *pool, - size_t state_size) +void +anv_state_pool_finish(struct anv_state_pool *pool) { - /* At least a cache line and must divide the block size. */ - assert(state_size >= 64 && util_is_power_of_two(state_size)); - - pool->state_size = state_size; - pool->free_list = ANV_FREE_LIST_EMPTY; - pool->block.next = 0; - pool->block.end = 0; + VG(VALGRIND_DESTROY_MEMPOOL(pool)); + anv_state_table_finish(&pool->table); + anv_block_pool_finish(&pool->block_pool); } static uint32_t -anv_fixed_size_state_pool_alloc(struct anv_fixed_size_state_pool *pool, - struct anv_block_pool *block_pool) +anv_fixed_size_state_pool_alloc_new(struct anv_fixed_size_state_pool *pool, + struct anv_block_pool *block_pool, + uint32_t state_size, + uint32_t block_size, + uint32_t *padding) { - int32_t offset; struct anv_block_state block, old, new; + uint32_t offset; - /* Try free list first. */ - if (anv_free_list_pop(&pool->free_list, &block_pool->map, &offset)) { - assert(offset >= 0); - return offset; - } + /* We don't always use anv_block_pool_alloc(), which would set *padding to + * zero for us. So if we have a pointer to padding, we must zero it out + * ourselves here, to make sure we always return some sensible value. + */ + if (padding) + *padding = 0; + + /* If our state is large, we don't need any sub-allocation from a block. + * Instead, we just grab whole (potentially large) blocks. + */ + if (state_size >= block_size) + return anv_block_pool_alloc(block_pool, state_size, padding); - /* If free list was empty (or somebody raced us and took the items) we - * allocate a new item from the end of the block */ restart: - block.u64 = __sync_fetch_and_add(&pool->block.u64, pool->state_size); + block.u64 = __sync_fetch_and_add(&pool->block.u64, state_size); if (block.next < block.end) { return block.next; } else if (block.next == block.end) { - offset = anv_block_pool_alloc(block_pool); - new.next = offset + pool->state_size; - new.end = offset + block_pool->block_size; + offset = anv_block_pool_alloc(block_pool, block_size, padding); + new.next = offset + state_size; + new.end = offset + block_size; old.u64 = __sync_lock_test_and_set(&pool->block.u64, new.u64); if (old.next != block.next) futex_wake(&pool->block.end, INT_MAX); return offset; } else { - futex_wait(&pool->block.end, block.end); + futex_wait(&pool->block.end, block.end, NULL); goto restart; } } -static void -anv_fixed_size_state_pool_free(struct anv_fixed_size_state_pool *pool, - struct anv_block_pool *block_pool, - uint32_t offset) +static uint32_t +anv_state_pool_get_bucket(uint32_t size) { - anv_free_list_push(&pool->free_list, block_pool->map, offset); + unsigned size_log2 = ilog2_round_up(size); + assert(size_log2 <= ANV_MAX_STATE_SIZE_LOG2); + if (size_log2 < ANV_MIN_STATE_SIZE_LOG2) + size_log2 = ANV_MIN_STATE_SIZE_LOG2; + return size_log2 - ANV_MIN_STATE_SIZE_LOG2; } -void -anv_state_pool_init(struct anv_state_pool *pool, - struct anv_block_pool *block_pool) +static uint32_t +anv_state_pool_get_bucket_size(uint32_t bucket) { - pool->block_pool = block_pool; - for (unsigned i = 0; i < ANV_STATE_BUCKETS; i++) { - size_t size = 1 << (ANV_MIN_STATE_SIZE_LOG2 + i); - anv_fixed_size_state_pool_init(&pool->buckets[i], size); + uint32_t size_log2 = bucket + ANV_MIN_STATE_SIZE_LOG2; + return 1 << size_log2; +} + +/** Helper to push a chunk into the state table. + * + * It creates 'count' entries into the state table and update their sizes, + * offsets and maps, also pushing them as "free" states. + */ +static void +anv_state_pool_return_blocks(struct anv_state_pool *pool, + uint32_t chunk_offset, uint32_t count, + uint32_t block_size) +{ + /* Disallow returning 0 chunks */ + assert(count != 0); + + /* Make sure we always return chunks aligned to the block_size */ + assert(chunk_offset % block_size == 0); + + uint32_t st_idx; + UNUSED VkResult result = anv_state_table_add(&pool->table, &st_idx, count); + assert(result == VK_SUCCESS); + for (int i = 0; i < count; i++) { + /* update states that were added back to the state table */ + struct anv_state *state_i = anv_state_table_get(&pool->table, + st_idx + i); + state_i->alloc_size = block_size; + state_i->offset = pool->start_offset + chunk_offset + block_size * i; + state_i->map = anv_block_pool_map(&pool->block_pool, + state_i->offset, + state_i->alloc_size); } - VG(VALGRIND_CREATE_MEMPOOL(pool, 0, false)); + + uint32_t block_bucket = anv_state_pool_get_bucket(block_size); + anv_free_list_push(&pool->buckets[block_bucket].free_list, + &pool->table, st_idx, count); } -void -anv_state_pool_finish(struct anv_state_pool *pool) +/** Returns a chunk of memory back to the state pool. + * + * Do a two-level split. If chunk_size is bigger than divisor + * (pool->block_size), we return as many divisor sized blocks as we can, from + * the end of the chunk. + * + * The remaining is then split into smaller blocks (starting at small_size if + * it is non-zero), with larger blocks always being taken from the end of the + * chunk. + */ +static void +anv_state_pool_return_chunk(struct anv_state_pool *pool, + uint32_t chunk_offset, uint32_t chunk_size, + uint32_t small_size) { - VG(VALGRIND_DESTROY_MEMPOOL(pool)); + uint32_t divisor = pool->block_size; + uint32_t nblocks = chunk_size / divisor; + uint32_t rest = chunk_size - nblocks * divisor; + + if (nblocks > 0) { + /* First return divisor aligned and sized chunks. We start returning + * larger blocks from the end fo the chunk, since they should already be + * aligned to divisor. Also anv_state_pool_return_blocks() only accepts + * aligned chunks. + */ + uint32_t offset = chunk_offset + rest; + anv_state_pool_return_blocks(pool, offset, nblocks, divisor); + } + + chunk_size = rest; + divisor /= 2; + + if (small_size > 0 && small_size < divisor) + divisor = small_size; + + uint32_t min_size = 1 << ANV_MIN_STATE_SIZE_LOG2; + + /* Just as before, return larger divisor aligned blocks from the end of the + * chunk first. + */ + while (chunk_size > 0 && divisor >= min_size) { + nblocks = chunk_size / divisor; + rest = chunk_size - nblocks * divisor; + if (nblocks > 0) { + anv_state_pool_return_blocks(pool, chunk_offset + rest, + nblocks, divisor); + chunk_size = rest; + } + divisor /= 2; + } } static struct anv_state anv_state_pool_alloc_no_vg(struct anv_state_pool *pool, - size_t size, size_t align) + uint32_t size, uint32_t align) { - unsigned size_log2 = ilog2_round_up(size < align ? align : size); - assert(size_log2 <= ANV_MAX_STATE_SIZE_LOG2); - if (size_log2 < ANV_MIN_STATE_SIZE_LOG2) - size_log2 = ANV_MIN_STATE_SIZE_LOG2; - unsigned bucket = size_log2 - ANV_MIN_STATE_SIZE_LOG2; + uint32_t bucket = anv_state_pool_get_bucket(MAX2(size, align)); - struct anv_state state; - state.alloc_size = 1 << size_log2; - state.offset = anv_fixed_size_state_pool_alloc(&pool->buckets[bucket], - pool->block_pool); - state.map = pool->block_pool->map + state.offset; - return state; + struct anv_state *state; + uint32_t alloc_size = anv_state_pool_get_bucket_size(bucket); + int32_t offset; + + /* Try free list first. */ + state = anv_free_list_pop(&pool->buckets[bucket].free_list, + &pool->table); + if (state) { + assert(state->offset >= pool->start_offset); + goto done; + } + + /* Try to grab a chunk from some larger bucket and split it up */ + for (unsigned b = bucket + 1; b < ANV_STATE_BUCKETS; b++) { + state = anv_free_list_pop(&pool->buckets[b].free_list, &pool->table); + if (state) { + unsigned chunk_size = anv_state_pool_get_bucket_size(b); + int32_t chunk_offset = state->offset; + + /* First lets update the state we got to its new size. offset and map + * remain the same. + */ + state->alloc_size = alloc_size; + + /* Now return the unused part of the chunk back to the pool as free + * blocks + * + * There are a couple of options as to what we do with it: + * + * 1) We could fully split the chunk into state.alloc_size sized + * pieces. However, this would mean that allocating a 16B + * state could potentially split a 2MB chunk into 512K smaller + * chunks. This would lead to unnecessary fragmentation. + * + * 2) The classic "buddy allocator" method would have us split the + * chunk in half and return one half. Then we would split the + * remaining half in half and return one half, and repeat as + * needed until we get down to the size we want. However, if + * you are allocating a bunch of the same size state (which is + * the common case), this means that every other allocation has + * to go up a level and every fourth goes up two levels, etc. + * This is not nearly as efficient as it could be if we did a + * little more work up-front. + * + * 3) Split the difference between (1) and (2) by doing a + * two-level split. If it's bigger than some fixed block_size, + * we split it into block_size sized chunks and return all but + * one of them. Then we split what remains into + * state.alloc_size sized chunks and return them. + * + * We choose something close to option (3), which is implemented with + * anv_state_pool_return_chunk(). That is done by returning the + * remaining of the chunk, with alloc_size as a hint of the size that + * we want the smaller chunk split into. + */ + anv_state_pool_return_chunk(pool, chunk_offset + alloc_size, + chunk_size - alloc_size, alloc_size); + goto done; + } + } + + uint32_t padding; + offset = anv_fixed_size_state_pool_alloc_new(&pool->buckets[bucket], + &pool->block_pool, + alloc_size, + pool->block_size, + &padding); + /* Everytime we allocate a new state, add it to the state pool */ + uint32_t idx; + UNUSED VkResult result = anv_state_table_add(&pool->table, &idx, 1); + assert(result == VK_SUCCESS); + + state = anv_state_table_get(&pool->table, idx); + state->offset = pool->start_offset + offset; + state->alloc_size = alloc_size; + state->map = anv_block_pool_map(&pool->block_pool, offset, alloc_size); + + if (padding > 0) { + uint32_t return_offset = offset - padding; + anv_state_pool_return_chunk(pool, return_offset, padding, 0); + } + +done: + return *state; } struct anv_state -anv_state_pool_alloc(struct anv_state_pool *pool, size_t size, size_t align) +anv_state_pool_alloc(struct anv_state_pool *pool, uint32_t size, uint32_t align) { if (size == 0) return ANV_STATE_NULL; @@ -718,17 +1114,52 @@ anv_state_pool_alloc(struct anv_state_pool *pool, size_t size, size_t align) return state; } +struct anv_state +anv_state_pool_alloc_back(struct anv_state_pool *pool) +{ + struct anv_state *state; + uint32_t alloc_size = pool->block_size; + + /* This function is only used with pools where start_offset == 0 */ + assert(pool->start_offset == 0); + + state = anv_free_list_pop(&pool->back_alloc_free_list, &pool->table); + if (state) { + assert(state->offset < pool->start_offset); + goto done; + } + + int32_t offset; + offset = anv_block_pool_alloc_back(&pool->block_pool, + pool->block_size); + uint32_t idx; + UNUSED VkResult result = anv_state_table_add(&pool->table, &idx, 1); + assert(result == VK_SUCCESS); + + state = anv_state_table_get(&pool->table, idx); + state->offset = pool->start_offset + offset; + state->alloc_size = alloc_size; + state->map = anv_block_pool_map(&pool->block_pool, offset, alloc_size); + +done: + VG(VALGRIND_MEMPOOL_ALLOC(pool, state->map, state->alloc_size)); + return *state; +} + static void anv_state_pool_free_no_vg(struct anv_state_pool *pool, struct anv_state state) { - assert(util_is_power_of_two(state.alloc_size)); - unsigned size_log2 = ilog2_round_up(state.alloc_size); - assert(size_log2 >= ANV_MIN_STATE_SIZE_LOG2 && - size_log2 <= ANV_MAX_STATE_SIZE_LOG2); - unsigned bucket = size_log2 - ANV_MIN_STATE_SIZE_LOG2; - - anv_fixed_size_state_pool_free(&pool->buckets[bucket], - pool->block_pool, state.offset); + assert(util_is_power_of_two_or_zero(state.alloc_size)); + unsigned bucket = anv_state_pool_get_bucket(state.alloc_size); + + if (state.offset < pool->start_offset) { + assert(state.alloc_size == pool->block_size); + anv_free_list_push(&pool->back_alloc_free_list, + &pool->table, state.idx, 1); + } else { + anv_free_list_push(&pool->buckets[bucket].free_list, + &pool->table, state.idx, 1); + } } void @@ -741,14 +1172,12 @@ anv_state_pool_free(struct anv_state_pool *pool, struct anv_state state) anv_state_pool_free_no_vg(pool, state); } -#define NULL_BLOCK 1 struct anv_state_stream_block { + struct anv_state block; + /* The next block */ struct anv_state_stream_block *next; - /* The offset into the block pool at which this block starts */ - uint32_t offset; - #ifdef HAVE_VALGRIND /* A pointer to the first user-allocated thing in this block. This is * what valgrind sees as the start of the block. @@ -762,16 +1191,20 @@ struct anv_state_stream_block { */ void anv_state_stream_init(struct anv_state_stream *stream, - struct anv_block_pool *block_pool) + struct anv_state_pool *state_pool, + uint32_t block_size) { - stream->block_pool = block_pool; - stream->block = NULL; + stream->state_pool = state_pool; + stream->block_size = block_size; + + stream->block = ANV_STATE_NULL; - /* Ensure that next + whatever > end. This way the first call to + /* Ensure that next + whatever > block_size. This way the first call to * state_stream_alloc fetches a new block. */ - stream->next = 1; - stream->end = 0; + stream->next = block_size; + + util_dynarray_init(&stream->all_blocks, NULL); VG(VALGRIND_CREATE_MEMPOOL(stream, 0, false)); } @@ -779,16 +1212,12 @@ anv_state_stream_init(struct anv_state_stream *stream, void anv_state_stream_finish(struct anv_state_stream *stream) { - VG(const uint32_t block_size = stream->block_pool->block_size); - - struct anv_state_stream_block *next = stream->block; - while (next != NULL) { - struct anv_state_stream_block sb = VG_NOACCESS_READ(next); - VG(VALGRIND_MEMPOOL_FREE(stream, sb._vg_ptr)); - VG(VALGRIND_MAKE_MEM_UNDEFINED(next, block_size)); - anv_block_pool_free(stream->block_pool, sb.offset); - next = sb.next; + util_dynarray_foreach(&stream->all_blocks, struct anv_state, block) { + VG(VALGRIND_MEMPOOL_FREE(stream, block->map)); + VG(VALGRIND_MAKE_MEM_NOACCESS(block->map, block->alloc_size)); + anv_state_pool_free_no_vg(stream->state_pool, *block); } + util_dynarray_fini(&stream->all_blocks); VG(VALGRIND_DESTROY_MEMPOOL(stream)); } @@ -800,65 +1229,97 @@ anv_state_stream_alloc(struct anv_state_stream *stream, if (size == 0) return ANV_STATE_NULL; - struct anv_state_stream_block *sb = stream->block; - - struct anv_state state; + assert(alignment <= PAGE_SIZE); - state.offset = align_u32(stream->next, alignment); - if (state.offset + size > stream->end) { - uint32_t block = anv_block_pool_alloc(stream->block_pool); - sb = stream->block_pool->map + block; + uint32_t offset = align_u32(stream->next, alignment); + if (offset + size > stream->block.alloc_size) { + uint32_t block_size = stream->block_size; + if (block_size < size) + block_size = round_to_power_of_two(size); - VG(VALGRIND_MAKE_MEM_UNDEFINED(sb, sizeof(*sb))); - sb->next = stream->block; - sb->offset = block; - VG(sb->_vg_ptr = NULL); - VG(VALGRIND_MAKE_MEM_NOACCESS(sb, stream->block_pool->block_size)); + stream->block = anv_state_pool_alloc_no_vg(stream->state_pool, + block_size, PAGE_SIZE); + util_dynarray_append(&stream->all_blocks, + struct anv_state, stream->block); + VG(VALGRIND_MAKE_MEM_NOACCESS(stream->block.map, block_size)); - stream->block = sb; - stream->start = block; - stream->next = block + sizeof(*sb); - stream->end = block + stream->block_pool->block_size; - - state.offset = align_u32(stream->next, alignment); - assert(state.offset + size <= stream->end); + /* Reset back to the start */ + stream->next = offset = 0; + assert(offset + size <= stream->block.alloc_size); } + const bool new_block = stream->next == 0; - assert(state.offset > stream->start); - state.map = (void *)sb + (state.offset - stream->start); + struct anv_state state = stream->block; + state.offset += offset; state.alloc_size = size; + state.map += offset; -#ifdef HAVE_VALGRIND - void *vg_ptr = VG_NOACCESS_READ(&sb->_vg_ptr); - if (vg_ptr == NULL) { - vg_ptr = state.map; - VG_NOACCESS_WRITE(&sb->_vg_ptr, vg_ptr); - VALGRIND_MEMPOOL_ALLOC(stream, vg_ptr, size); + stream->next = offset + size; + + if (new_block) { + assert(state.map == stream->block.map); + VG(VALGRIND_MEMPOOL_ALLOC(stream, state.map, size)); } else { - void *state_end = state.map + state.alloc_size; /* This only updates the mempool. The newly allocated chunk is still * marked as NOACCESS. */ - VALGRIND_MEMPOOL_CHANGE(stream, vg_ptr, vg_ptr, state_end - vg_ptr); + VG(VALGRIND_MEMPOOL_CHANGE(stream, stream->block.map, stream->block.map, + stream->next)); /* Mark the newly allocated chunk as undefined */ - VALGRIND_MAKE_MEM_UNDEFINED(state.map, state.alloc_size); + VG(VALGRIND_MAKE_MEM_UNDEFINED(state.map, state.alloc_size)); } -#endif - - stream->next = state.offset + size; return state; } -struct bo_pool_bo_link { - struct bo_pool_bo_link *next; - struct anv_bo bo; -}; +void +anv_state_reserved_pool_init(struct anv_state_reserved_pool *pool, + struct anv_state_pool *parent, + uint32_t count, uint32_t size, uint32_t alignment) +{ + pool->pool = parent; + pool->reserved_blocks = ANV_FREE_LIST_EMPTY; + pool->count = count; + + for (unsigned i = 0; i < count; i++) { + struct anv_state state = anv_state_pool_alloc(pool->pool, size, alignment); + anv_free_list_push(&pool->reserved_blocks, &pool->pool->table, state.idx, 1); + } +} + +void +anv_state_reserved_pool_finish(struct anv_state_reserved_pool *pool) +{ + struct anv_state *state; + + while ((state = anv_free_list_pop(&pool->reserved_blocks, &pool->pool->table))) { + anv_state_pool_free(pool->pool, *state); + pool->count--; + } + assert(pool->count == 0); +} + +struct anv_state +anv_state_reserved_pool_alloc(struct anv_state_reserved_pool *pool) +{ + return *anv_free_list_pop(&pool->reserved_blocks, &pool->pool->table); +} + +void +anv_state_reserved_pool_free(struct anv_state_reserved_pool *pool, + struct anv_state state) +{ + anv_free_list_push(&pool->reserved_blocks, &pool->pool->table, state.idx, 1); +} void anv_bo_pool_init(struct anv_bo_pool *pool, struct anv_device *device) { pool->device = device; - memset(pool->free_list, 0, sizeof(pool->free_list)); + for (unsigned i = 0; i < ARRAY_SIZE(pool->free_list); i++) { + util_sparse_array_free_list_init(&pool->free_list[i], + &device->bo_cache.bo_map, 0, + offsetof(struct anv_bo, free_index)); + } VG(VALGRIND_CREATE_MEMPOOL(pool, 0, false)); } @@ -867,13 +1328,15 @@ void anv_bo_pool_finish(struct anv_bo_pool *pool) { for (unsigned i = 0; i < ARRAY_SIZE(pool->free_list); i++) { - struct bo_pool_bo_link *link = PFL_PTR(pool->free_list[i]); - while (link != NULL) { - struct bo_pool_bo_link link_copy = VG_NOACCESS_READ(link); - - anv_gem_munmap(link_copy.bo.map, link_copy.bo.size); - anv_gem_close(pool->device, link_copy.bo.gem_handle); - link = link_copy.next; + while (1) { + struct anv_bo *bo = + util_sparse_array_free_list_pop_elem(&pool->free_list[i]); + if (bo == NULL) + break; + + /* anv_device_release_bo is going to "free" it */ + VG(VALGRIND_MALLOCLIKE_BLOCK(bo->map, bo->size, 0, 1)); + anv_device_release_bo(pool->device, bo); } } @@ -881,66 +1344,55 @@ anv_bo_pool_finish(struct anv_bo_pool *pool) } VkResult -anv_bo_pool_alloc(struct anv_bo_pool *pool, struct anv_bo *bo, uint32_t size) +anv_bo_pool_alloc(struct anv_bo_pool *pool, uint32_t size, + struct anv_bo **bo_out) { - VkResult result; - const unsigned size_log2 = size < 4096 ? 12 : ilog2_round_up(size); const unsigned pow2_size = 1 << size_log2; const unsigned bucket = size_log2 - 12; assert(bucket < ARRAY_SIZE(pool->free_list)); - void *next_free_void; - if (anv_ptr_free_list_pop(&pool->free_list[bucket], &next_free_void)) { - struct bo_pool_bo_link *next_free = next_free_void; - *bo = VG_NOACCESS_READ(&next_free->bo); - assert(bo->gem_handle); - assert(bo->map == next_free); - assert(size <= bo->size); - + struct anv_bo *bo = + util_sparse_array_free_list_pop_elem(&pool->free_list[bucket]); + if (bo != NULL) { VG(VALGRIND_MEMPOOL_ALLOC(pool, bo->map, size)); - + *bo_out = bo; return VK_SUCCESS; } - struct anv_bo new_bo; - - result = anv_bo_init_new(&new_bo, pool->device, pow2_size); + VkResult result = anv_device_alloc_bo(pool->device, + pow2_size, + ANV_BO_ALLOC_MAPPED | + ANV_BO_ALLOC_SNOOPED | + ANV_BO_ALLOC_CAPTURE, + 0 /* explicit_address */, + &bo); if (result != VK_SUCCESS) return result; - assert(new_bo.size == pow2_size); - - new_bo.map = anv_gem_mmap(pool->device, new_bo.gem_handle, 0, pow2_size, 0); - if (new_bo.map == MAP_FAILED) { - anv_gem_close(pool->device, new_bo.gem_handle); - return vk_error(VK_ERROR_MEMORY_MAP_FAILED); - } - - *bo = new_bo; - + /* We want it to look like it came from this pool */ + VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0)); VG(VALGRIND_MEMPOOL_ALLOC(pool, bo->map, size)); + *bo_out = bo; + return VK_SUCCESS; } void -anv_bo_pool_free(struct anv_bo_pool *pool, const struct anv_bo *bo_in) +anv_bo_pool_free(struct anv_bo_pool *pool, struct anv_bo *bo) { - /* Make a copy in case the anv_bo happens to be storred in the BO */ - struct anv_bo bo = *bo_in; - - VG(VALGRIND_MEMPOOL_FREE(pool, bo.map)); + VG(VALGRIND_MEMPOOL_FREE(pool, bo->map)); - struct bo_pool_bo_link *link = bo.map; - VG_NOACCESS_WRITE(&link->bo, bo); - - assert(util_is_power_of_two(bo.size)); - const unsigned size_log2 = ilog2_round_up(bo.size); + assert(util_is_power_of_two_or_zero(bo->size)); + const unsigned size_log2 = ilog2_round_up(bo->size); const unsigned bucket = size_log2 - 12; assert(bucket < ARRAY_SIZE(pool->free_list)); - anv_ptr_free_list_push(&pool->free_list[bucket], link); + assert(util_sparse_array_get(&pool->device->bo_cache.bo_map, + bo->gem_handle) == bo); + util_sparse_array_free_list_push(&pool->free_list[bucket], + &bo->gem_handle, 1); } // Scratch pool @@ -956,9 +1408,8 @@ anv_scratch_pool_finish(struct anv_device *device, struct anv_scratch_pool *pool { for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) { for (unsigned i = 0; i < 16; i++) { - struct anv_scratch_bo *bo = &pool->bos[i][s]; - if (bo->exists > 0) - anv_gem_close(device, bo->bo.gem_handle); + if (pool->bos[i][s] != NULL) + anv_device_release_bo(device, pool->bos[i][s]); } } } @@ -973,40 +1424,78 @@ anv_scratch_pool_alloc(struct anv_device *device, struct anv_scratch_pool *pool, unsigned scratch_size_log2 = ffs(per_thread_scratch / 2048); assert(scratch_size_log2 < 16); - struct anv_scratch_bo *bo = &pool->bos[scratch_size_log2][stage]; - - /* We can use "exists" to shortcut and ignore the critical section */ - if (bo->exists) - return &bo->bo; + struct anv_bo *bo = p_atomic_read(&pool->bos[scratch_size_log2][stage]); - pthread_mutex_lock(&device->mutex); + if (bo != NULL) + return bo; - __sync_synchronize(); - if (bo->exists) - return &bo->bo; + const struct gen_device_info *devinfo = &device->info; - const struct anv_physical_device *physical_device = - &device->instance->physicalDevice; - const struct gen_device_info *devinfo = &physical_device->info; + unsigned subslices = MAX2(device->physical->subslice_total, 1); - /* WaCSScratchSize:hsw + /* The documentation for 3DSTATE_PS "Scratch Space Base Pointer" says: + * + * "Scratch Space per slice is computed based on 4 sub-slices. SW + * must allocate scratch space enough so that each slice has 4 + * slices allowed." + * + * According to the other driver team, this applies to compute shaders + * as well. This is not currently documented at all. * - * Haswell's scratch space address calculation appears to be sparse - * rather than tightly packed. The Thread ID has bits indicating which - * subslice, EU within a subslice, and thread within an EU it is. - * There's a maximum of two slices and two subslices, so these can be - * stored with a single bit. Even though there are only 10 EUs per - * subslice, this is stored in 4 bits, so there's an effective maximum - * value of 16 EUs. Similarly, although there are only 7 threads per EU, - * this is stored in a 3 bit number, giving an effective maximum value - * of 8 threads per EU. + * This hack is no longer necessary on Gen11+. * - * This means that we need to use 16 * 8 instead of 10 * 7 for the - * number of threads per subslice. + * For, Gen11+, scratch space allocation is based on the number of threads + * in the base configuration. */ - const unsigned subslices = MAX2(physical_device->subslice_total, 1); - const unsigned scratch_ids_per_subslice = - device->info.is_haswell ? 16 * 8 : devinfo->max_cs_threads; + if (devinfo->gen >= 12) + subslices = devinfo->num_subslices[0]; + else if (devinfo->gen == 11) + subslices = 8; + else if (devinfo->gen >= 9) + subslices = 4 * devinfo->num_slices; + + unsigned scratch_ids_per_subslice; + if (devinfo->gen >= 12) { + /* Same as ICL below, but with 16 EUs. */ + scratch_ids_per_subslice = 16 * 8; + } else if (devinfo->gen == 11) { + /* The MEDIA_VFE_STATE docs say: + * + * "Starting with this configuration, the Maximum Number of + * Threads must be set to (#EU * 8) for GPGPU dispatches. + * + * Although there are only 7 threads per EU in the configuration, + * the FFTID is calculated as if there are 8 threads per EU, + * which in turn requires a larger amount of Scratch Space to be + * allocated by the driver." + */ + scratch_ids_per_subslice = 8 * 8; + } else if (devinfo->is_haswell) { + /* WaCSScratchSize:hsw + * + * Haswell's scratch space address calculation appears to be sparse + * rather than tightly packed. The Thread ID has bits indicating + * which subslice, EU within a subslice, and thread within an EU it + * is. There's a maximum of two slices and two subslices, so these + * can be stored with a single bit. Even though there are only 10 EUs + * per subslice, this is stored in 4 bits, so there's an effective + * maximum value of 16 EUs. Similarly, although there are only 7 + * threads per EU, this is stored in a 3 bit number, giving an + * effective maximum value of 8 threads per EU. + * + * This means that we need to use 16 * 8 instead of 10 * 7 for the + * number of threads per subslice. + */ + scratch_ids_per_subslice = 16 * 8; + } else if (devinfo->is_cherryview) { + /* Cherryview devices have either 6 or 8 EUs per subslice, and each EU + * has 7 threads. The 6 EU devices appear to calculate thread IDs as if + * it had 8 EUs. + */ + scratch_ids_per_subslice = 8 * 7; + } else { + scratch_ids_per_subslice = devinfo->max_cs_threads; + } uint32_t max_threads[] = { [MESA_SHADER_VERTEX] = devinfo->max_vs_threads, @@ -1019,8 +1508,6 @@ anv_scratch_pool_alloc(struct anv_device *device, struct anv_scratch_pool *pool, uint32_t size = per_thread_scratch * max_threads[stage]; - anv_bo_init_new(&bo->bo, device, size); - /* Even though the Scratch base pointers in 3DSTATE_*S are 64 bits, they * are still relative to the general state base address. When we emit * STATE_BASE_ADDRESS, we set general state base address to 0 and the size @@ -1038,34 +1525,31 @@ anv_scratch_pool_alloc(struct anv_device *device, struct anv_scratch_pool *pool, * * so nothing will ever touch the top page. */ - bo->bo.flags &= ~EXEC_OBJECT_SUPPORTS_48B_ADDRESS; - - /* Set the exists last because it may be read by other threads */ - __sync_synchronize(); - bo->exists = true; - - pthread_mutex_unlock(&device->mutex); + VkResult result = anv_device_alloc_bo(device, size, + ANV_BO_ALLOC_32BIT_ADDRESS, + 0 /* explicit_address */, + &bo); + if (result != VK_SUCCESS) + return NULL; /* TODO */ - return &bo->bo; + struct anv_bo *current_bo = + p_atomic_cmpxchg(&pool->bos[scratch_size_log2][stage], NULL, bo); + if (current_bo) { + anv_device_release_bo(device, bo); + return current_bo; + } else { + return bo; + } } -struct anv_cached_bo { - struct anv_bo bo; - - uint32_t refcount; -}; - VkResult anv_bo_cache_init(struct anv_bo_cache *cache) { - cache->bo_map = _mesa_hash_table_create(NULL, _mesa_hash_pointer, - _mesa_key_pointer_equal); - if (!cache->bo_map) - return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); + util_sparse_array_init(&cache->bo_map, sizeof(struct anv_bo), 1024); if (pthread_mutex_init(&cache->mutex, NULL)) { - _mesa_hash_table_destroy(cache->bo_map, NULL); - return vk_errorf(VK_ERROR_OUT_OF_HOST_MEMORY, + util_sparse_array_finish(&cache->bo_map); + return vk_errorf(NULL, NULL, VK_ERROR_OUT_OF_HOST_MEMORY, "pthread_mutex_init failed: %m"); } @@ -1075,160 +1559,409 @@ anv_bo_cache_init(struct anv_bo_cache *cache) void anv_bo_cache_finish(struct anv_bo_cache *cache) { - _mesa_hash_table_destroy(cache->bo_map, NULL); + util_sparse_array_finish(&cache->bo_map); pthread_mutex_destroy(&cache->mutex); } -static struct anv_cached_bo * -anv_bo_cache_lookup_locked(struct anv_bo_cache *cache, uint32_t gem_handle) +#define ANV_BO_CACHE_SUPPORTED_FLAGS \ + (EXEC_OBJECT_WRITE | \ + EXEC_OBJECT_ASYNC | \ + EXEC_OBJECT_SUPPORTS_48B_ADDRESS | \ + EXEC_OBJECT_PINNED | \ + EXEC_OBJECT_CAPTURE) + +static uint32_t +anv_bo_alloc_flags_to_bo_flags(struct anv_device *device, + enum anv_bo_alloc_flags alloc_flags) { - struct hash_entry *entry = - _mesa_hash_table_search(cache->bo_map, - (const void *)(uintptr_t)gem_handle); - if (!entry) - return NULL; + struct anv_physical_device *pdevice = device->physical; - struct anv_cached_bo *bo = (struct anv_cached_bo *)entry->data; - assert(bo->bo.gem_handle == gem_handle); + uint64_t bo_flags = 0; + if (!(alloc_flags & ANV_BO_ALLOC_32BIT_ADDRESS) && + pdevice->supports_48bit_addresses) + bo_flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS; - return bo; -} + if ((alloc_flags & ANV_BO_ALLOC_CAPTURE) && pdevice->has_exec_capture) + bo_flags |= EXEC_OBJECT_CAPTURE; -static struct anv_bo * -anv_bo_cache_lookup(struct anv_bo_cache *cache, uint32_t gem_handle) -{ - pthread_mutex_lock(&cache->mutex); + if (alloc_flags & ANV_BO_ALLOC_IMPLICIT_WRITE) { + assert(alloc_flags & ANV_BO_ALLOC_IMPLICIT_SYNC); + bo_flags |= EXEC_OBJECT_WRITE; + } - struct anv_cached_bo *bo = anv_bo_cache_lookup_locked(cache, gem_handle); + if (!(alloc_flags & ANV_BO_ALLOC_IMPLICIT_SYNC) && pdevice->has_exec_async) + bo_flags |= EXEC_OBJECT_ASYNC; - pthread_mutex_unlock(&cache->mutex); + if (pdevice->use_softpin) + bo_flags |= EXEC_OBJECT_PINNED; + + return bo_flags; +} + +static uint32_t +anv_device_get_bo_align(struct anv_device *device, + enum anv_bo_alloc_flags alloc_flags) +{ + /* Gen12 CCS surface addresses need to be 64K aligned. */ + if (device->info.gen >= 12 && (alloc_flags & ANV_BO_ALLOC_IMPLICIT_CCS)) + return 64 * 1024; - return bo ? &bo->bo : NULL; + return 4096; } VkResult -anv_bo_cache_alloc(struct anv_device *device, - struct anv_bo_cache *cache, - uint64_t size, struct anv_bo **bo_out) +anv_device_alloc_bo(struct anv_device *device, + uint64_t size, + enum anv_bo_alloc_flags alloc_flags, + uint64_t explicit_address, + struct anv_bo **bo_out) { - struct anv_cached_bo *bo = - vk_alloc(&device->alloc, sizeof(struct anv_cached_bo), 8, - VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); - if (!bo) - return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); + if (!device->physical->has_implicit_ccs) + assert(!(alloc_flags & ANV_BO_ALLOC_IMPLICIT_CCS)); - bo->refcount = 1; + const uint32_t bo_flags = + anv_bo_alloc_flags_to_bo_flags(device, alloc_flags); + assert(bo_flags == (bo_flags & ANV_BO_CACHE_SUPPORTED_FLAGS)); /* The kernel is going to give us whole pages anyway */ size = align_u64(size, 4096); - VkResult result = anv_bo_init_new(&bo->bo, device, size); - if (result != VK_SUCCESS) { - vk_free(&device->alloc, bo); - return result; + const uint32_t align = anv_device_get_bo_align(device, alloc_flags); + + uint64_t ccs_size = 0; + if (device->info.has_aux_map && (alloc_flags & ANV_BO_ALLOC_IMPLICIT_CCS)) { + /* Align the size up to the next multiple of 64K so we don't have any + * AUX-TT entries pointing from a 64K page to itself. + */ + size = align_u64(size, 64 * 1024); + + /* See anv_bo::_ccs_size */ + ccs_size = align_u64(DIV_ROUND_UP(size, GEN_AUX_MAP_GEN12_CCS_SCALE), 4096); + } + + uint32_t gem_handle = anv_gem_create(device, size + ccs_size); + if (gem_handle == 0) + return vk_error(VK_ERROR_OUT_OF_DEVICE_MEMORY); + + struct anv_bo new_bo = { + .gem_handle = gem_handle, + .refcount = 1, + .offset = -1, + .size = size, + ._ccs_size = ccs_size, + .flags = bo_flags, + .is_external = (alloc_flags & ANV_BO_ALLOC_EXTERNAL), + .has_client_visible_address = + (alloc_flags & ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS) != 0, + .has_implicit_ccs = ccs_size > 0, + }; + + if (alloc_flags & ANV_BO_ALLOC_MAPPED) { + new_bo.map = anv_gem_mmap(device, new_bo.gem_handle, 0, size, 0); + if (new_bo.map == MAP_FAILED) { + anv_gem_close(device, new_bo.gem_handle); + return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); + } + } + + if (alloc_flags & ANV_BO_ALLOC_SNOOPED) { + assert(alloc_flags & ANV_BO_ALLOC_MAPPED); + /* We don't want to change these defaults if it's going to be shared + * with another process. + */ + assert(!(alloc_flags & ANV_BO_ALLOC_EXTERNAL)); + + /* Regular objects are created I915_CACHING_CACHED on LLC platforms and + * I915_CACHING_NONE on non-LLC platforms. For many internal state + * objects, we'd rather take the snooping overhead than risk forgetting + * a CLFLUSH somewhere. Userptr objects are always created as + * I915_CACHING_CACHED, which on non-LLC means snooped so there's no + * need to do this there. + */ + if (!device->info.has_llc) { + anv_gem_set_caching(device, new_bo.gem_handle, + I915_CACHING_CACHED); + } } - assert(bo->bo.gem_handle); + if (alloc_flags & ANV_BO_ALLOC_FIXED_ADDRESS) { + new_bo.has_fixed_address = true; + new_bo.offset = explicit_address; + } else if (new_bo.flags & EXEC_OBJECT_PINNED) { + new_bo.offset = anv_vma_alloc(device, new_bo.size + new_bo._ccs_size, + align, alloc_flags, explicit_address); + if (new_bo.offset == 0) { + if (new_bo.map) + anv_gem_munmap(device, new_bo.map, size); + anv_gem_close(device, new_bo.gem_handle); + return vk_errorf(device, NULL, VK_ERROR_OUT_OF_DEVICE_MEMORY, + "failed to allocate virtual address for BO"); + } + } else { + assert(!new_bo.has_client_visible_address); + } - pthread_mutex_lock(&cache->mutex); + if (new_bo._ccs_size > 0) { + assert(device->info.has_aux_map); + gen_aux_map_add_mapping(device->aux_map_ctx, + gen_canonical_address(new_bo.offset), + gen_canonical_address(new_bo.offset + new_bo.size), + new_bo.size, 0 /* format_bits */); + } - _mesa_hash_table_insert(cache->bo_map, - (void *)(uintptr_t)bo->bo.gem_handle, bo); + assert(new_bo.gem_handle); - pthread_mutex_unlock(&cache->mutex); + /* If we just got this gem_handle from anv_bo_init_new then we know no one + * else is touching this BO at the moment so we don't need to lock here. + */ + struct anv_bo *bo = anv_device_lookup_bo(device, new_bo.gem_handle); + *bo = new_bo; - *bo_out = &bo->bo; + *bo_out = bo; return VK_SUCCESS; } VkResult -anv_bo_cache_import(struct anv_device *device, - struct anv_bo_cache *cache, - int fd, uint64_t size, struct anv_bo **bo_out) +anv_device_import_bo_from_host_ptr(struct anv_device *device, + void *host_ptr, uint32_t size, + enum anv_bo_alloc_flags alloc_flags, + uint64_t client_address, + struct anv_bo **bo_out) { + assert(!(alloc_flags & (ANV_BO_ALLOC_MAPPED | + ANV_BO_ALLOC_SNOOPED | + ANV_BO_ALLOC_FIXED_ADDRESS))); + + /* We can't do implicit CCS with an aux table on shared memory */ + if (!device->physical->has_implicit_ccs || device->info.has_aux_map) + assert(!(alloc_flags & ANV_BO_ALLOC_IMPLICIT_CCS)); + + struct anv_bo_cache *cache = &device->bo_cache; + const uint32_t bo_flags = + anv_bo_alloc_flags_to_bo_flags(device, alloc_flags); + assert(bo_flags == (bo_flags & ANV_BO_CACHE_SUPPORTED_FLAGS)); + + uint32_t gem_handle = anv_gem_userptr(device, host_ptr, size); + if (!gem_handle) + return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); + pthread_mutex_lock(&cache->mutex); - /* The kernel is going to give us whole pages anyway */ - size = align_u64(size, 4096); + struct anv_bo *bo = anv_device_lookup_bo(device, gem_handle); + if (bo->refcount > 0) { + /* VK_EXT_external_memory_host doesn't require handling importing the + * same pointer twice at the same time, but we don't get in the way. If + * kernel gives us the same gem_handle, only succeed if the flags match. + */ + assert(bo->gem_handle == gem_handle); + if (bo_flags != bo->flags) { + pthread_mutex_unlock(&cache->mutex); + return vk_errorf(device, NULL, VK_ERROR_INVALID_EXTERNAL_HANDLE, + "same host pointer imported two different ways"); + } + + if (bo->has_client_visible_address != + ((alloc_flags & ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS) != 0)) { + pthread_mutex_unlock(&cache->mutex); + return vk_errorf(device, NULL, VK_ERROR_INVALID_EXTERNAL_HANDLE, + "The same BO was imported with and without buffer " + "device address"); + } + + if (client_address && client_address != gen_48b_address(bo->offset)) { + pthread_mutex_unlock(&cache->mutex); + return vk_errorf(device, NULL, VK_ERROR_INVALID_EXTERNAL_HANDLE, + "The same BO was imported at two different " + "addresses"); + } + + __sync_fetch_and_add(&bo->refcount, 1); + } else { + struct anv_bo new_bo = { + .gem_handle = gem_handle, + .refcount = 1, + .offset = -1, + .size = size, + .map = host_ptr, + .flags = bo_flags, + .is_external = true, + .from_host_ptr = true, + .has_client_visible_address = + (alloc_flags & ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS) != 0, + }; + + assert(client_address == gen_48b_address(client_address)); + if (new_bo.flags & EXEC_OBJECT_PINNED) { + assert(new_bo._ccs_size == 0); + new_bo.offset = anv_vma_alloc(device, new_bo.size, + anv_device_get_bo_align(device, + alloc_flags), + alloc_flags, client_address); + if (new_bo.offset == 0) { + anv_gem_close(device, new_bo.gem_handle); + pthread_mutex_unlock(&cache->mutex); + return vk_errorf(device, NULL, VK_ERROR_OUT_OF_DEVICE_MEMORY, + "failed to allocate virtual address for BO"); + } + } else { + assert(!new_bo.has_client_visible_address); + } + + *bo = new_bo; + } + + pthread_mutex_unlock(&cache->mutex); + *bo_out = bo; + + return VK_SUCCESS; +} + +VkResult +anv_device_import_bo(struct anv_device *device, + int fd, + enum anv_bo_alloc_flags alloc_flags, + uint64_t client_address, + struct anv_bo **bo_out) +{ + assert(!(alloc_flags & (ANV_BO_ALLOC_MAPPED | + ANV_BO_ALLOC_SNOOPED | + ANV_BO_ALLOC_FIXED_ADDRESS))); + + /* We can't do implicit CCS with an aux table on shared memory */ + if (!device->physical->has_implicit_ccs || device->info.has_aux_map) + assert(!(alloc_flags & ANV_BO_ALLOC_IMPLICIT_CCS)); + + struct anv_bo_cache *cache = &device->bo_cache; + const uint32_t bo_flags = + anv_bo_alloc_flags_to_bo_flags(device, alloc_flags); + assert(bo_flags == (bo_flags & ANV_BO_CACHE_SUPPORTED_FLAGS)); + + pthread_mutex_lock(&cache->mutex); uint32_t gem_handle = anv_gem_fd_to_handle(device, fd); if (!gem_handle) { pthread_mutex_unlock(&cache->mutex); - return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE_KHX); + return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); } - struct anv_cached_bo *bo = anv_bo_cache_lookup_locked(cache, gem_handle); - if (bo) { - if (bo->bo.size != size) { + struct anv_bo *bo = anv_device_lookup_bo(device, gem_handle); + if (bo->refcount > 0) { + /* We have to be careful how we combine flags so that it makes sense. + * Really, though, if we get to this case and it actually matters, the + * client has imported a BO twice in different ways and they get what + * they have coming. + */ + uint64_t new_flags = 0; + new_flags |= (bo->flags | bo_flags) & EXEC_OBJECT_WRITE; + new_flags |= (bo->flags & bo_flags) & EXEC_OBJECT_ASYNC; + new_flags |= (bo->flags & bo_flags) & EXEC_OBJECT_SUPPORTS_48B_ADDRESS; + new_flags |= (bo->flags | bo_flags) & EXEC_OBJECT_PINNED; + new_flags |= (bo->flags | bo_flags) & EXEC_OBJECT_CAPTURE; + + /* It's theoretically possible for a BO to get imported such that it's + * both pinned and not pinned. The only way this can happen is if it + * gets imported as both a semaphore and a memory object and that would + * be an application error. Just fail out in that case. + */ + if ((bo->flags & EXEC_OBJECT_PINNED) != + (bo_flags & EXEC_OBJECT_PINNED)) { pthread_mutex_unlock(&cache->mutex); - return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE_KHX); + return vk_errorf(device, NULL, VK_ERROR_INVALID_EXTERNAL_HANDLE, + "The same BO was imported two different ways"); } - __sync_fetch_and_add(&bo->refcount, 1); - } else { - /* For security purposes, we reject BO imports where the size does not - * match exactly. This prevents a malicious client from passing a - * buffer to a trusted client, lying about the size, and telling the - * trusted client to try and texture from an image that goes - * out-of-bounds. This sort of thing could lead to GPU hangs or worse - * in the trusted client. The trusted client can protect itself against - * this sort of attack but only if it can trust the buffer size. + + /* It's also theoretically possible that someone could export a BO from + * one heap and import it into another or to import the same BO into two + * different heaps. If this happens, we could potentially end up both + * allowing and disallowing 48-bit addresses. There's not much we can + * do about it if we're pinning so we just throw an error and hope no + * app is actually that stupid. */ - off_t import_size = lseek(fd, 0, SEEK_END); - if (import_size == (off_t)-1 || import_size != size) { - anv_gem_close(device, gem_handle); + if ((new_flags & EXEC_OBJECT_PINNED) && + (bo->flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) != + (bo_flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS)) { pthread_mutex_unlock(&cache->mutex); - return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE_KHX); + return vk_errorf(device, NULL, VK_ERROR_INVALID_EXTERNAL_HANDLE, + "The same BO was imported on two different heaps"); } - bo = vk_alloc(&device->alloc, sizeof(struct anv_cached_bo), 8, - VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); - if (!bo) { - anv_gem_close(device, gem_handle); + if (bo->has_client_visible_address != + ((alloc_flags & ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS) != 0)) { pthread_mutex_unlock(&cache->mutex); - return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); + return vk_errorf(device, NULL, VK_ERROR_INVALID_EXTERNAL_HANDLE, + "The same BO was imported with and without buffer " + "device address"); } - bo->refcount = 1; + if (client_address && client_address != gen_48b_address(bo->offset)) { + pthread_mutex_unlock(&cache->mutex); + return vk_errorf(device, NULL, VK_ERROR_INVALID_EXTERNAL_HANDLE, + "The same BO was imported at two different " + "addresses"); + } - anv_bo_init(&bo->bo, gem_handle, size); + bo->flags = new_flags; - if (device->instance->physicalDevice.supports_48bit_addresses) - bo->bo.flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS; + __sync_fetch_and_add(&bo->refcount, 1); + } else { + off_t size = lseek(fd, 0, SEEK_END); + if (size == (off_t)-1) { + anv_gem_close(device, gem_handle); + pthread_mutex_unlock(&cache->mutex); + return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); + } - if (device->instance->physicalDevice.has_exec_async) - bo->bo.flags |= EXEC_OBJECT_ASYNC; + struct anv_bo new_bo = { + .gem_handle = gem_handle, + .refcount = 1, + .offset = -1, + .size = size, + .flags = bo_flags, + .is_external = true, + .has_client_visible_address = + (alloc_flags & ANV_BO_ALLOC_CLIENT_VISIBLE_ADDRESS) != 0, + }; + + assert(client_address == gen_48b_address(client_address)); + if (new_bo.flags & EXEC_OBJECT_PINNED) { + assert(new_bo._ccs_size == 0); + new_bo.offset = anv_vma_alloc(device, new_bo.size, + anv_device_get_bo_align(device, + alloc_flags), + alloc_flags, client_address); + if (new_bo.offset == 0) { + anv_gem_close(device, new_bo.gem_handle); + pthread_mutex_unlock(&cache->mutex); + return vk_errorf(device, NULL, VK_ERROR_OUT_OF_DEVICE_MEMORY, + "failed to allocate virtual address for BO"); + } + } else { + assert(!new_bo.has_client_visible_address); + } - _mesa_hash_table_insert(cache->bo_map, (void *)(uintptr_t)gem_handle, bo); + *bo = new_bo; } pthread_mutex_unlock(&cache->mutex); - - /* From the Vulkan spec: - * - * "Importing memory from a file descriptor transfers ownership of - * the file descriptor from the application to the Vulkan - * implementation. The application must not perform any operations on - * the file descriptor after a successful import." - * - * If the import fails, we leave the file descriptor open. - */ - close(fd); - - *bo_out = &bo->bo; + *bo_out = bo; return VK_SUCCESS; } VkResult -anv_bo_cache_export(struct anv_device *device, - struct anv_bo_cache *cache, - struct anv_bo *bo_in, int *fd_out) +anv_device_export_bo(struct anv_device *device, + struct anv_bo *bo, int *fd_out) { - assert(anv_bo_cache_lookup(cache, bo_in->gem_handle) == bo_in); - struct anv_cached_bo *bo = (struct anv_cached_bo *)bo_in; + assert(anv_device_lookup_bo(device, bo->gem_handle) == bo); + + /* This BO must have been flagged external in order for us to be able + * to export it. This is done based on external options passed into + * anv_AllocateMemory. + */ + assert(bo->is_external); - int fd = anv_gem_handle_to_fd(device, bo->bo.gem_handle); + int fd = anv_gem_handle_to_fd(device, bo->gem_handle); if (fd < 0) return vk_error(VK_ERROR_TOO_MANY_OBJECTS); @@ -1256,12 +1989,11 @@ atomic_dec_not_one(uint32_t *counter) } void -anv_bo_cache_release(struct anv_device *device, - struct anv_bo_cache *cache, - struct anv_bo *bo_in) +anv_device_release_bo(struct anv_device *device, + struct anv_bo *bo) { - assert(anv_bo_cache_lookup(cache, bo_in->gem_handle) == bo_in); - struct anv_cached_bo *bo = (struct anv_cached_bo *)bo_in; + struct anv_bo_cache *cache = &device->bo_cache; + assert(anv_device_lookup_bo(device, bo->gem_handle) == bo); /* Try to decrement the counter but don't go below one. If this succeeds * then the refcount has been decremented and we are not the last @@ -1282,17 +2014,35 @@ anv_bo_cache_release(struct anv_device *device, pthread_mutex_unlock(&cache->mutex); return; } + assert(bo->refcount == 0); + + if (bo->map && !bo->from_host_ptr) + anv_gem_munmap(device, bo->map, bo->size); + + if (bo->_ccs_size > 0) { + assert(device->physical->has_implicit_ccs); + assert(device->info.has_aux_map); + assert(bo->has_implicit_ccs); + gen_aux_map_unmap_range(device->aux_map_ctx, + gen_canonical_address(bo->offset), + bo->size); + } + + if ((bo->flags & EXEC_OBJECT_PINNED) && !bo->has_fixed_address) + anv_vma_free(device, bo->offset, bo->size + bo->_ccs_size); - struct hash_entry *entry = - _mesa_hash_table_search(cache->bo_map, - (const void *)(uintptr_t)bo->bo.gem_handle); - assert(entry); - _mesa_hash_table_remove(cache->bo_map, entry); + uint32_t gem_handle = bo->gem_handle; - if (bo->bo.map) - anv_gem_munmap(bo->bo.map, bo->bo.size); + /* Memset the BO just in case. The refcount being zero should be enough to + * prevent someone from assuming the data is valid but it's safer to just + * stomp to zero just in case. We explicitly do this *before* we close the + * GEM handle to ensure that if anyone allocates something and gets the + * same GEM handle, the memset has already happen and won't stomp all over + * any data they may write in this BO. + */ + memset(bo, 0, sizeof(*bo)); - anv_gem_close(device, bo->bo.gem_handle); + anv_gem_close(device, gem_handle); /* Don't unlock until we've actually closed the BO. The whole point of * the BO cache is to ensure that we correctly handle races with creating @@ -1300,6 +2050,4 @@ anv_bo_cache_release(struct anv_device *device, * again between mutex unlock and closing the GEM handle. */ pthread_mutex_unlock(&cache->mutex); - - vk_free(&device->alloc, bo); }