From: Kenneth Graunke Date: Tue, 10 Apr 2018 07:39:40 +0000 (-0700) Subject: i965: Add virtual memory allocator infrastructure to brw_bufmgr. X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=01058a552294ed641f2e933c0cc374917fb3bc70;p=mesa.git i965: Add virtual memory allocator infrastructure to brw_bufmgr. This introduces a new fast virtual memory allocator integrated with our BO cache bucketing. For larger objects, it falls back to the simple free-list allocator (util_vma). This puts the allocators in place but doesn't enable softpin yet. v2: (feedback from Chris Wilson) - Check (bo->kflags & EXEC_OBJECT_PINNED) instead of a global flag - Avoid vma_free(0ull) on the err_free path. - Only enable if the kernel says we have full PPGTT support - Make bucketing allocators more resistant to failing to grow arrays (feedback from Scott Phillips) - Don't use node after popping it from the list. - Avoid undefined behavior in canonicalization by reusing new helper - Comment updates (feedback from myself) - Avoid __vma_alloc vs. vma_alloc by making a zero_high_bits helper to return a non-canonical address with the high bits zeroed. - Don't shadow loop variable 'i' when destroying things (ugly; worked) v3: - Replace zero_high_bits with new common gen_48b_address helper. Reviewed-by: Scott D Phillips Reviewed-by: Jordan Justen --- diff --git a/src/mesa/drivers/dri/i965/brw_bufmgr.c b/src/mesa/drivers/dri/i965/brw_bufmgr.c index 66828f319be..121e952b9ed 100644 --- a/src/mesa/drivers/dri/i965/brw_bufmgr.c +++ b/src/mesa/drivers/dri/i965/brw_bufmgr.c @@ -54,12 +54,15 @@ #endif #include "common/gen_clflush.h" #include "common/gen_debug.h" +#include "common/gen_gem.h" #include "dev/gen_device_info.h" #include "libdrm_macros.h" #include "main/macros.h" #include "util/macros.h" #include "util/hash_table.h" #include "util/list.h" +#include "util/u_dynarray.h" +#include "util/vma.h" #include "brw_bufmgr.h" #include "brw_context.h" #include "string.h" @@ -98,9 +101,41 @@ atomic_add_unless(int *v, int add, int unless) return c == unless; } +/** + * i965 fixed-size bucketing VMA allocator. + * + * The BO cache maintains "cache buckets" for buffers of various sizes. + * All buffers in a given bucket are identically sized - when allocating, + * we always round up to the bucket size. This means that virtually all + * allocations are fixed-size; only buffers which are too large to fit in + * a bucket can be variably-sized. + * + * We create an allocator for each bucket. Each contains a free-list, where + * each node contains a pair. Each bit + * represents a bucket-sized block of memory. (At the first level, each + * bit corresponds to a page. For the second bucket, bits correspond to + * two pages, and so on.) 1 means a block is free, and 0 means it's in-use. + * The lowest bit in the bitmap is for the first block. + * + * This makes allocations cheap - any bit of any node will do. We can pick + * the head of the list and use ffs() to find a free block. If there are + * none, we allocate 64 blocks from a larger allocator - either a bigger + * bucketing allocator, or a fallback top-level allocator for large objects. + */ +struct vma_bucket_node { + uint64_t start_address; + uint64_t bitmap; +}; + struct bo_cache_bucket { + /** List of cached BOs. */ struct list_head head; + + /** Size of this bucket, in bytes. */ uint64_t size; + + /** List of vma_bucket_nodes. */ + struct util_dynarray vma_list[BRW_MEMZONE_COUNT]; }; struct brw_bufmgr { @@ -116,6 +151,8 @@ struct brw_bufmgr { struct hash_table *name_table; struct hash_table *handle_table; + struct util_vma_heap vma_allocator[BRW_MEMZONE_COUNT]; + bool has_llc:1; bool has_mmap_wc:1; bool bo_reuse:1; @@ -128,6 +165,10 @@ static int bo_set_tiling_internal(struct brw_bo *bo, uint32_t tiling_mode, static void bo_free(struct brw_bo *bo); +static uint64_t vma_alloc(struct brw_bufmgr *bufmgr, + enum brw_memory_zone memzone, + uint64_t size, uint64_t alignment); + static uint32_t key_hash_uint(const void *key) { @@ -222,6 +263,187 @@ bucket_for_size(struct brw_bufmgr *bufmgr, uint64_t size) &bufmgr->cache_bucket[index] : NULL; } +static enum brw_memory_zone +memzone_for_address(uint64_t address) +{ + const uint64_t _4GB = 1ull << 32; + + if (address >= _4GB) + return BRW_MEMZONE_OTHER; + + return BRW_MEMZONE_LOW_4G; +} + +static uint64_t +bucket_vma_alloc(struct brw_bufmgr *bufmgr, + struct bo_cache_bucket *bucket, + enum brw_memory_zone memzone) +{ + struct util_dynarray *vma_list = &bucket->vma_list[memzone]; + struct vma_bucket_node *node; + + if (vma_list->size == 0) { + /* This bucket allocator is out of space - allocate a new block of + * memory for 64 blocks from a larger allocator (either a larger + * bucket or util_vma). + * + * We align the address to the node size (64 blocks) so that + * bucket_vma_free can easily compute the starting address of this + * block by rounding any address we return down to the node size. + * + * Set the first bit used, and return the start address. + */ + uint64_t node_size = 64ull * bucket->size; + node = util_dynarray_grow(vma_list, sizeof(struct vma_bucket_node)); + + if (unlikely(!node)) + return 0ull; + + uint64_t addr = vma_alloc(bufmgr, memzone, node_size, node_size); + node->start_address = gen_48b_address(addr); + node->bitmap = ~1ull; + return node->start_address; + } + + /* Pick any bit from any node - they're all the right size and free. */ + node = util_dynarray_top_ptr(vma_list, struct vma_bucket_node); + int bit = ffsll(node->bitmap) - 1; + assert(bit >= 0 && bit <= 63); + + /* Reserve the memory by clearing the bit. */ + assert((node->bitmap & (1ull << bit)) != 0ull); + node->bitmap &= ~(1ull << bit); + + uint64_t addr = node->start_address + bit * bucket->size; + + /* If this node is now completely full, remove it from the free list. */ + if (node->bitmap == 0ull) { + (void) util_dynarray_pop(vma_list, struct vma_bucket_node); + } + + return addr; +} + +static void +bucket_vma_free(struct bo_cache_bucket *bucket, uint64_t address) +{ + enum brw_memory_zone memzone = memzone_for_address(address); + struct util_dynarray *vma_list = &bucket->vma_list[memzone]; + const uint64_t node_bytes = 64ull * bucket->size; + struct vma_bucket_node *node = NULL; + + /* bucket_vma_alloc allocates 64 blocks at a time, and aligns it to + * that 64 block size. So, we can round down to get the starting address. + */ + uint64_t start = (address / node_bytes) * node_bytes; + + /* Dividing the offset from start by bucket size gives us the bit index. */ + int bit = (address - start) / bucket->size; + + assert(start + bit * bucket->size == address); + + util_dynarray_foreach(vma_list, struct vma_bucket_node, cur) { + if (cur->start_address == start) { + node = cur; + break; + } + } + + if (!node) { + /* No node - the whole group of 64 blocks must have been in-use. */ + node = util_dynarray_grow(vma_list, sizeof(struct vma_bucket_node)); + + if (unlikely(!node)) + return; /* bogus, leaks some GPU VMA, but nothing we can do... */ + + node->start_address = start; + node->bitmap = 0ull; + } + + /* Set the bit to return the memory. */ + assert((node->bitmap & (1ull << bit)) == 0ull); + node->bitmap |= 1ull << bit; + + /* The block might be entirely free now, and if so, we could return it + * to the larger allocator. But we may as well hang on to it, in case + * we get more allocations at this block size. + */ +} + +static struct bo_cache_bucket * +get_bucket_allocator(struct brw_bufmgr *bufmgr, uint64_t size) +{ + /* Skip using the bucket allocator for very large sizes, as it allocates + * 64 of them and this can balloon rather quickly. + */ + if (size > 1024 * PAGE_SIZE) + return NULL; + + struct bo_cache_bucket *bucket = bucket_for_size(bufmgr, size); + + if (bucket && bucket->size == size) + return bucket; + + return NULL; +} + +/** + * Allocate a section of virtual memory for a buffer, assigning an address. + * + * This uses either the bucket allocator for the given size, or the large + * object allocator (util_vma). + */ +static uint64_t +vma_alloc(struct brw_bufmgr *bufmgr, + enum brw_memory_zone memzone, + uint64_t size, + uint64_t alignment) +{ + /* Without softpin support, we let the kernel assign addresses. */ + assert(brw_using_softpin(bufmgr)); + + struct bo_cache_bucket *bucket = get_bucket_allocator(bufmgr, size); + uint64_t addr; + + if (bucket) { + addr = bucket_vma_alloc(bufmgr, bucket, memzone); + } else { + addr = util_vma_heap_alloc(&bufmgr->vma_allocator[memzone], size, + alignment); + } + + assert((addr >> 48ull) == 0); + assert((addr % alignment) == 0); + + return gen_canonical_address(addr); +} + +/** + * Free a virtual memory area, allowing the address to be reused. + */ +static void +vma_free(struct brw_bufmgr *bufmgr, + uint64_t address, + uint64_t size) +{ + assert(brw_using_softpin(bufmgr)); + + /* Un-canonicalize the address. */ + address = gen_48b_address(address); + + if (address == 0ull) + return; + + struct bo_cache_bucket *bucket = get_bucket_allocator(bufmgr, size); + + if (bucket) { + bucket_vma_free(bucket, address); + } else { + enum brw_memory_zone memzone = memzone_for_address(address); + util_vma_heap_free(&bufmgr->vma_allocator[memzone], address, size); + } +} + int brw_bo_busy(struct brw_bo *bo) { @@ -360,7 +582,16 @@ retry: } } - if (!alloc_from_cache) { + if (alloc_from_cache) { + /* If the cache BO isn't in the right memory zone, free the old + * memory and assign it a new address. + */ + if ((bo->kflags & EXEC_OBJECT_PINNED) && + memzone != memzone_for_address(bo->gtt_offset)) { + vma_free(bufmgr, bo->gtt_offset, bo->size); + bo->gtt_offset = 0ull; + } + } else { bo = calloc(1, sizeof(*bo)); if (!bo) goto err; @@ -411,6 +642,13 @@ retry: bo->index = -1; bo->kflags = bufmgr->initial_kflags; + if ((bo->kflags & EXEC_OBJECT_PINNED) && bo->gtt_offset == 0ull) { + bo->gtt_offset = vma_alloc(bufmgr, memzone, bo->size, 1); + + if (bo->gtt_offset == 0ull) + goto err_free; + } + mtx_unlock(&bufmgr->lock); DBG("bo_create: buf %d (%s) %llub\n", bo->gem_handle, bo->name, @@ -545,6 +783,9 @@ brw_bo_gem_create_from_name(struct brw_bufmgr *bufmgr, bo->external = true; bo->kflags = bufmgr->initial_kflags; + if (bo->kflags & EXEC_OBJECT_PINNED) + bo->gtt_offset = vma_alloc(bufmgr, BRW_MEMZONE_OTHER, bo->size, 1); + _mesa_hash_table_insert(bufmgr->handle_table, &bo->gem_handle, bo); _mesa_hash_table_insert(bufmgr->name_table, &bo->global_name, bo); @@ -605,6 +846,10 @@ bo_free(struct brw_bo *bo) DBG("DRM_IOCTL_GEM_CLOSE %d failed (%s): %s\n", bo->gem_handle, bo->name, strerror(errno)); } + + if (bo->kflags & EXEC_OBJECT_PINNED) + vma_free(bo->bufmgr, bo->gtt_offset, bo->size); + free(bo); } @@ -1063,11 +1308,23 @@ brw_bufmgr_destroy(struct brw_bufmgr *bufmgr) bo_free(bo); } + + if (brw_using_softpin(bufmgr)) { + for (int z = 0; z < BRW_MEMZONE_COUNT; z++) { + util_dynarray_fini(&bucket->vma_list[z]); + } + } } _mesa_hash_table_destroy(bufmgr->name_table, NULL); _mesa_hash_table_destroy(bufmgr->handle_table, NULL); + if (brw_using_softpin(bufmgr)) { + for (int z = 0; z < BRW_MEMZONE_COUNT; z++) { + util_vma_heap_finish(&bufmgr->vma_allocator[z]); + } + } + free(bufmgr); } @@ -1165,6 +1422,11 @@ brw_bo_gem_create_from_prime_internal(struct brw_bufmgr *bufmgr, int prime_fd, bo->external = true; bo->kflags = bufmgr->initial_kflags; + if (bo->kflags & EXEC_OBJECT_PINNED) { + assert(bo->size > 0); + bo->gtt_offset = vma_alloc(bufmgr, BRW_MEMZONE_OTHER, bo->size, 1); + } + if (tiling_mode < 0) { struct drm_i915_gem_get_tiling get_tiling = { .handle = bo->gem_handle }; if (drmIoctl(bufmgr->fd, DRM_IOCTL_I915_GEM_GET_TILING, &get_tiling)) @@ -1291,6 +1553,10 @@ add_bucket(struct brw_bufmgr *bufmgr, int size) assert(i < ARRAY_SIZE(bufmgr->cache_bucket)); list_inithead(&bufmgr->cache_bucket[i].head); + if (brw_using_softpin(bufmgr)) { + for (int z = 0; z < BRW_MEMZONE_COUNT; z++) + util_dynarray_init(&bufmgr->cache_bucket[i].vma_list[z], NULL); + } bufmgr->cache_bucket[i].size = size; bufmgr->num_buckets++; @@ -1408,6 +1674,12 @@ gem_context_getparam(int fd, uint32_t context, uint64_t param, uint64_t *value) return 0; } +bool +brw_using_softpin(struct brw_bufmgr *bufmgr) +{ + return bufmgr->initial_kflags & EXEC_OBJECT_PINNED; +} + /** * Initializes the GEM buffer manager, which uses the kernel to allocate, map, * and manage map buffer objections. @@ -1450,6 +1722,17 @@ brw_bufmgr_init(struct gen_device_info *devinfo, int fd) if (devinfo->gen >= 8 && gtt_size > _4GB) { bufmgr->initial_kflags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS; + + /* Allocate VMA in userspace if we have softpin and full PPGTT. */ + if (false && gem_param(fd, I915_PARAM_HAS_EXEC_SOFTPIN) > 0 && + gem_param(fd, I915_PARAM_HAS_ALIASING_PPGTT) > 1) { + bufmgr->initial_kflags |= EXEC_OBJECT_PINNED; + + util_vma_heap_init(&bufmgr->vma_allocator[BRW_MEMZONE_LOW_4G], + 4096, _4GB); + util_vma_heap_init(&bufmgr->vma_allocator[BRW_MEMZONE_OTHER], + 1 * _4GB, gtt_size - 1 * _4GB); + } } init_cache_buckets(bufmgr); diff --git a/src/mesa/drivers/dri/i965/brw_bufmgr.h b/src/mesa/drivers/dri/i965/brw_bufmgr.h index 9ad129744b2..8cdf944e3da 100644 --- a/src/mesa/drivers/dri/i965/brw_bufmgr.h +++ b/src/mesa/drivers/dri/i965/brw_bufmgr.h @@ -370,6 +370,8 @@ uint32_t brw_bo_export_gem_handle(struct brw_bo *bo); int brw_reg_read(struct brw_bufmgr *bufmgr, uint32_t offset, uint64_t *result); +bool brw_using_softpin(struct brw_bufmgr *bufmgr); + /** @{ */ #if defined(__cplusplus)