* IN THE SOFTWARE.
*/
-#define _DEFAULT_SOURCE
-
#include <stdint.h>
#include <stdlib.h>
#include <unistd.h>
* means that the bottom 12 bits should all be zero.
*/
#define PFL_COUNT(x) ((uintptr_t)(x) & 0xfff)
-#define PFL_PTR(x) ((void *)((uintptr_t)(x) & ~0xfff))
+#define PFL_PTR(x) ((void *)((uintptr_t)(x) & ~(uintptr_t)0xfff))
#define PFL_PACK(ptr, count) ({ \
- assert(((uintptr_t)(ptr) & 0xfff) == 0); \
- (void *)((uintptr_t)(ptr) | (uintptr_t)((count) & 0xfff)); \
+ (void *)(((uintptr_t)(ptr) & ~(uintptr_t)0xfff) | ((count) & 0xfff)); \
})
static bool
void *old, *current;
void **next_ptr = elem;
+ /* The pointer-based free list requires that the pointer be
+ * page-aligned. This is because we use the bottom 12 bits of the
+ * pointer to store a counter to solve the ABA concurrency problem.
+ */
+ assert(((uintptr_t)elem & 0xfff) == 0);
+
old = *list;
do {
current = old;
pool->bo.gem_handle = 0;
pool->bo.offset = 0;
pool->bo.size = 0;
+ pool->bo.is_winsys_bo = false;
pool->block_size = block_size;
pool->free_list = ANV_FREE_LIST_EMPTY;
pool->back_free_list = ANV_FREE_LIST_EMPTY;
};
void
-anv_bo_pool_init(struct anv_bo_pool *pool,
- struct anv_device *device, uint32_t bo_size)
+anv_bo_pool_init(struct anv_bo_pool *pool, struct anv_device *device)
{
pool->device = device;
- pool->bo_size = bo_size;
- pool->free_list = NULL;
+ memset(pool->free_list, 0, sizeof(pool->free_list));
VG(VALGRIND_CREATE_MEMPOOL(pool, 0, false));
}
void
anv_bo_pool_finish(struct anv_bo_pool *pool)
{
- struct bo_pool_bo_link *link = PFL_PTR(pool->free_list);
- while (link != NULL) {
- struct bo_pool_bo_link link_copy = VG_NOACCESS_READ(link);
-
- anv_gem_munmap(link_copy.bo.map, pool->bo_size);
- anv_gem_close(pool->device, link_copy.bo.gem_handle);
- link = link_copy.next;
+ for (unsigned i = 0; i < ARRAY_SIZE(pool->free_list); i++) {
+ struct bo_pool_bo_link *link = PFL_PTR(pool->free_list[i]);
+ while (link != NULL) {
+ struct bo_pool_bo_link link_copy = VG_NOACCESS_READ(link);
+
+ anv_gem_munmap(link_copy.bo.map, link_copy.bo.size);
+ anv_gem_close(pool->device, link_copy.bo.gem_handle);
+ link = link_copy.next;
+ }
}
VG(VALGRIND_DESTROY_MEMPOOL(pool));
}
VkResult
-anv_bo_pool_alloc(struct anv_bo_pool *pool, struct anv_bo *bo)
+anv_bo_pool_alloc(struct anv_bo_pool *pool, struct anv_bo *bo, uint32_t size)
{
VkResult result;
+ const unsigned size_log2 = size < 4096 ? 12 : ilog2_round_up(size);
+ const unsigned pow2_size = 1 << size_log2;
+ const unsigned bucket = size_log2 - 12;
+ assert(bucket < ARRAY_SIZE(pool->free_list));
+
void *next_free_void;
- if (anv_ptr_free_list_pop(&pool->free_list, &next_free_void)) {
+ if (anv_ptr_free_list_pop(&pool->free_list[bucket], &next_free_void)) {
struct bo_pool_bo_link *next_free = next_free_void;
*bo = VG_NOACCESS_READ(&next_free->bo);
assert(bo->map == next_free);
- assert(bo->size == pool->bo_size);
+ assert(size <= bo->size);
- VG(VALGRIND_MEMPOOL_ALLOC(pool, bo->map, pool->bo_size));
+ VG(VALGRIND_MEMPOOL_ALLOC(pool, bo->map, size));
return VK_SUCCESS;
}
struct anv_bo new_bo;
- result = anv_bo_init_new(&new_bo, pool->device, pool->bo_size);
+ result = anv_bo_init_new(&new_bo, pool->device, pow2_size);
if (result != VK_SUCCESS)
return result;
- assert(new_bo.size == pool->bo_size);
+ assert(new_bo.size == pow2_size);
- new_bo.map = anv_gem_mmap(pool->device, new_bo.gem_handle, 0, pool->bo_size, 0);
+ new_bo.map = anv_gem_mmap(pool->device, new_bo.gem_handle, 0, pow2_size, 0);
if (new_bo.map == NULL) {
anv_gem_close(pool->device, new_bo.gem_handle);
return vk_error(VK_ERROR_MEMORY_MAP_FAILED);
*bo = new_bo;
- VG(VALGRIND_MEMPOOL_ALLOC(pool, bo->map, pool->bo_size));
+ VG(VALGRIND_MEMPOOL_ALLOC(pool, bo->map, size));
return VK_SUCCESS;
}
void
-anv_bo_pool_free(struct anv_bo_pool *pool, const struct anv_bo *bo)
+anv_bo_pool_free(struct anv_bo_pool *pool, const struct anv_bo *bo_in)
+{
+ /* Make a copy in case the anv_bo happens to be storred in the BO */
+ struct anv_bo bo = *bo_in;
+ struct bo_pool_bo_link *link = bo.map;
+ link->bo = bo;
+
+ assert(util_is_power_of_two(bo.size));
+ const unsigned size_log2 = ilog2_round_up(bo.size);
+ const unsigned bucket = size_log2 - 12;
+ assert(bucket < ARRAY_SIZE(pool->free_list));
+
+ VG(VALGRIND_MEMPOOL_FREE(pool, bo.map));
+ anv_ptr_free_list_push(&pool->free_list[bucket], link);
+}
+
+// Scratch pool
+
+void
+anv_scratch_pool_init(struct anv_device *device, struct anv_scratch_pool *pool)
{
- struct bo_pool_bo_link *link = bo->map;
- link->bo = *bo;
+ memset(pool, 0, sizeof(*pool));
+}
+
+void
+anv_scratch_pool_finish(struct anv_device *device, struct anv_scratch_pool *pool)
+{
+ for (unsigned s = 0; s < MESA_SHADER_STAGES; s++) {
+ for (unsigned i = 0; i < 16; i++) {
+ struct anv_bo *bo = &pool->bos[i][s];
+ if (bo->size > 0)
+ anv_gem_close(device, bo->gem_handle);
+ }
+ }
+}
+
+struct anv_bo *
+anv_scratch_pool_alloc(struct anv_device *device, struct anv_scratch_pool *pool,
+ gl_shader_stage stage, unsigned per_thread_scratch)
+{
+ if (per_thread_scratch == 0)
+ return NULL;
+
+ unsigned scratch_size_log2 = ffs(per_thread_scratch / 2048);
+ assert(scratch_size_log2 < 16);
+
+ struct anv_bo *bo = &pool->bos[scratch_size_log2][stage];
+
+ /* From now on, we go into a critical section. In order to remain
+ * thread-safe, we use the bo size as a lock. A value of 0 means we don't
+ * have a valid BO yet. A value of 1 means locked. A value greater than 1
+ * means we have a bo of the given size.
+ */
+
+ if (bo->size > 1)
+ return bo;
+
+ uint64_t size = __sync_val_compare_and_swap(&bo->size, 0, 1);
+ if (size == 0) {
+ /* We own the lock. Allocate a buffer */
+
+ struct brw_device_info *devinfo = &device->info;
+ uint32_t max_threads[] = {
+ [MESA_SHADER_VERTEX] = devinfo->max_vs_threads,
+ [MESA_SHADER_TESS_CTRL] = devinfo->max_hs_threads,
+ [MESA_SHADER_TESS_EVAL] = devinfo->max_ds_threads,
+ [MESA_SHADER_GEOMETRY] = devinfo->max_gs_threads,
+ [MESA_SHADER_FRAGMENT] = devinfo->max_wm_threads,
+ [MESA_SHADER_COMPUTE] = devinfo->max_cs_threads,
+ };
+
+ size = per_thread_scratch * max_threads[stage];
+
+ struct anv_bo new_bo;
+ anv_bo_init_new(&new_bo, device, size);
+
+ bo->gem_handle = new_bo.gem_handle;
+
+ /* Set the size last because we use it as a lock */
+ __sync_synchronize();
+ bo->size = size;
+
+ futex_wake((uint32_t *)&bo->size, INT_MAX);
+ } else {
+ /* Someone else got here first */
+ while (bo->size == 1)
+ futex_wait((uint32_t *)&bo->size, 1);
+ }
- VG(VALGRIND_MEMPOOL_FREE(pool, bo->map));
- anv_ptr_free_list_push(&pool->free_list, link);
+ return bo;
}
projects / mesa.git / blobdiff