#include "private.h"
+#ifdef HAVE_VALGRIND
+#include <valgrind.h>
+#include <memcheck.h>
+#define VG(x) x
+#define VG_NOACCESS_READ(__ptr) ({ \
+ VALGRIND_MAKE_MEM_DEFINED((__ptr), sizeof(*(__ptr))); \
+ __typeof(*(__ptr)) __val = *(__ptr); \
+ VALGRIND_MAKE_MEM_NOACCESS((__ptr), sizeof(*(__ptr)));\
+ __val; \
+})
+#define VG_NOACCESS_WRITE(__ptr, __val) ({ \
+ VALGRIND_MAKE_MEM_UNDEFINED((__ptr), sizeof(*(__ptr))); \
+ *(__ptr) = (__val); \
+ VALGRIND_MAKE_MEM_NOACCESS((__ptr), sizeof(*(__ptr))); \
+})
+#else
+#define VG(x)
+#define VG_NOACCESS_READ(__ptr) (*(__ptr))
+#define VG_NOACCESS_WRITE(__ptr, __val) (*(__ptr) = (__val))
+#endif
+
/* Design goals:
*
* - Lock free (except when resizing underlying bos)
*/
__sync_synchronize();
- next.offset = *(uint32_t *)(*map + current.offset);
+ uint32_t *next_ptr = *map + current.offset;
+ next.offset = VG_NOACCESS_READ(next_ptr);
next.count = current.count + 1;
old.u64 = __sync_val_compare_and_swap(&list->u64, current.u64, next.u64);
if (old.u64 == current.u64) {
old = *list;
do {
current = old;
- *next_ptr = current.offset;
+ VG_NOACCESS_WRITE(next_ptr, current.offset);
new.offset = offset;
new.count = current.count + 1;
old.u64 = __sync_val_compare_and_swap(&list->u64, current.u64, new.u64);
} while (old.u64 != current.u64);
}
+/* All pointers in the ptr_free_list are assumed to be page-aligned. This
+ * means that the bottom 12 bits should all be zero.
+ */
+#define PFL_COUNT(x) ((uintptr_t)(x) & 0xfff)
+#define PFL_PTR(x) ((void *)((uintptr_t)(x) & ~0xfff))
+#define PFL_PACK(ptr, count) ({ \
+ assert(((uintptr_t)(ptr) & 0xfff) == 0); \
+ (void *)((uintptr_t)(ptr) | (uintptr_t)((count) & 0xfff)); \
+})
+
+static bool
+anv_ptr_free_list_pop(void **list, void **elem)
+{
+ void *current = *list;
+ while (PFL_PTR(current) != NULL) {
+ void **next_ptr = PFL_PTR(current);
+ void *new_ptr = VG_NOACCESS_READ(next_ptr);
+ unsigned new_count = PFL_COUNT(current) + 1;
+ void *new = PFL_PACK(new_ptr, new_count);
+ void *old = __sync_val_compare_and_swap(list, current, new);
+ if (old == current) {
+ *elem = PFL_PTR(current);
+ return true;
+ }
+ current = old;
+ }
+
+ return false;
+}
+
+static void
+anv_ptr_free_list_push(void **list, void *elem)
+{
+ void *old, *current;
+ void **next_ptr = elem;
+
+ old = *list;
+ do {
+ current = old;
+ VG_NOACCESS_WRITE(next_ptr, PFL_PTR(current));
+ unsigned new_count = PFL_COUNT(current) + 1;
+ void *new = PFL_PACK(elem, new_count);
+ old = __sync_val_compare_and_swap(list, current, new);
+ } while (old != current);
+}
+
static int
anv_block_pool_grow(struct anv_block_pool *pool);
uint32_t offset, block, size;
/* Try free list first. */
- if (anv_free_list_pop(&pool->free_list, &pool->map, &offset))
+ if (anv_free_list_pop(&pool->free_list, &pool->map, &offset)) {
+ assert(pool->map);
return offset;
+ }
restart:
size = pool->size;
block = __sync_fetch_and_add(&pool->next_block, pool->block_size);
if (block < size) {
+ assert(pool->map);
return block;
} else if (block == size) {
/* We allocated the first block outside the pool, we have to grow it.
* pool->next_block acts a mutex: threads who try to allocate now will
* get block indexes above the current limit and hit futex_wait
* below. */
- anv_block_pool_grow(pool);
+ int err = anv_block_pool_grow(pool);
+ assert(err == 0);
+ (void) err;
futex_wake(&pool->size, INT_MAX);
} else {
futex_wait(&pool->size, size);
state.offset = anv_fixed_size_state_pool_alloc(&pool->buckets[bucket],
pool->block_pool);
state.map = pool->block_pool->map + state.offset;
+ VG(VALGRIND_MALLOCLIKE_BLOCK(state.map, size, 0, false));
return state;
}
size_log2 <= ANV_MAX_STATE_SIZE_LOG2);
unsigned bucket = size_log2 - ANV_MIN_STATE_SIZE_LOG2;
+ VG(VALGRIND_FREELIKE_BLOCK(state.map, 0));
anv_fixed_size_state_pool_free(&pool->buckets[bucket],
pool->block_pool, state.offset);
}
#define NULL_BLOCK 1
struct stream_block {
uint32_t next;
+
+ /* The map for the BO at the time the block was givne to us */
+ void *current_map;
+
+#ifdef HAVE_VALGRIND
+ void *_vg_ptr;
+#endif
};
/* The state stream allocator is a one-shot, single threaded allocator for
uint32_t block, next_block;
block = stream->current_block;
- while (block != 1) {
+ while (block != NULL_BLOCK) {
sb = stream->block_pool->map + block;
- next_block = sb->next;
+ next_block = VG_NOACCESS_READ(&sb->next);
+ VG(VALGRIND_FREELIKE_BLOCK(VG_NOACCESS_READ(&sb->_vg_ptr), 0));
anv_block_pool_free(stream->block_pool, block);
block = next_block;
}
state.offset = ALIGN_U32(stream->next, alignment);
if (state.offset + size > stream->end) {
block = anv_block_pool_alloc(stream->block_pool);
- sb = stream->block_pool->map + block;
- sb->next = stream->current_block;
+ void *current_map = stream->block_pool->map;
+ sb = current_map + block;
+ VG_NOACCESS_WRITE(&sb->current_map, current_map);
+ VG_NOACCESS_WRITE(&sb->next, stream->current_block);
+ VG(VG_NOACCESS_WRITE(&sb->_vg_ptr, 0));
stream->current_block = block;
stream->next = block + sizeof(*sb);
stream->end = block + stream->block_pool->block_size;
assert(state.offset + size <= stream->end);
}
- stream->next = state.offset + size;
+ sb = stream->block_pool->map + stream->current_block;
+ void *current_map = VG_NOACCESS_READ(&sb->current_map);
+ state.map = current_map + state.offset;
state.alloc_size = size;
- state.map = stream->block_pool->map + state.offset;
+
+#ifdef HAVE_VALGRIND
+ void *vg_ptr = VG_NOACCESS_READ(&sb->_vg_ptr);
+ if (vg_ptr == NULL) {
+ vg_ptr = state.map;
+ VG_NOACCESS_WRITE(&sb->_vg_ptr, vg_ptr);
+ VALGRIND_MALLOCLIKE_BLOCK(vg_ptr, size, 0, false);
+ } else {
+ ptrdiff_t vg_offset = vg_ptr - current_map;
+ assert(vg_offset >= stream->current_block &&
+ vg_offset < stream->end);
+ VALGRIND_RESIZEINPLACE_BLOCK(vg_ptr,
+ stream->next - vg_offset,
+ (state.offset + size) - vg_offset,
+ 0);
+ }
+#endif
+
+ stream->next = state.offset + size;
return state;
}
+
+struct bo_pool_bo_link {
+ struct bo_pool_bo_link *next;
+ struct anv_bo bo;
+};
+
+void
+anv_bo_pool_init(struct anv_bo_pool *pool,
+ struct anv_device *device, uint32_t bo_size)
+{
+ pool->device = device;
+ pool->bo_size = bo_size;
+ pool->free_list = NULL;
+}
+
+void
+anv_bo_pool_finish(struct anv_bo_pool *pool)
+{
+ struct bo_pool_bo_link *link = PFL_PTR(pool->free_list);
+ while (link != NULL) {
+ struct bo_pool_bo_link link_copy = VG_NOACCESS_READ(link);
+
+ /* The anv_gem_m[un]map() functions are also valgrind-safe so they
+ * act as an alloc/free. In order to avoid a double-free warning, we
+ * need to mark thiss as malloc'd before we unmap it.
+ */
+ VG(VALGRIND_MALLOCLIKE_BLOCK(link_copy.bo.map, pool->bo_size, 0, false));
+
+ anv_gem_munmap(link_copy.bo.map, pool->bo_size);
+ anv_gem_close(pool->device, link_copy.bo.gem_handle);
+ link = link_copy.next;
+ }
+}
+
+VkResult
+anv_bo_pool_alloc(struct anv_bo_pool *pool, struct anv_bo *bo)
+{
+ VkResult result;
+
+ void *next_free_void;
+ if (anv_ptr_free_list_pop(&pool->free_list, &next_free_void)) {
+ struct bo_pool_bo_link *next_free = next_free_void;
+ *bo = VG_NOACCESS_READ(&next_free->bo);
+ assert(bo->map == next_free);
+ assert(bo->size == pool->bo_size);
+
+ VG(VALGRIND_MALLOCLIKE_BLOCK(bo->map, pool->bo_size, 0, false));
+
+ return VK_SUCCESS;
+ }
+
+ struct anv_bo new_bo;
+
+ result = anv_bo_init_new(&new_bo, pool->device, pool->bo_size);
+ if (result != VK_SUCCESS)
+ return result;
+
+ assert(new_bo.size == pool->bo_size);
+
+ new_bo.map = anv_gem_mmap(pool->device, new_bo.gem_handle, 0, pool->bo_size);
+ if (new_bo.map == NULL) {
+ anv_gem_close(pool->device, new_bo.gem_handle);
+ return vk_error(VK_ERROR_MEMORY_MAP_FAILED);
+ }
+
+ /* We don't need to call VALGRIND_MALLOCLIKE_BLOCK here because gem_mmap
+ * calls it for us. If we really want to be pedantic we could do a
+ * VALGRIND_FREELIKE_BLOCK right after the mmap, but there's no good
+ * reason.
+ */
+
+ *bo = new_bo;
+ return VK_SUCCESS;
+}
+
+void
+anv_bo_pool_free(struct anv_bo_pool *pool, const struct anv_bo *bo)
+{
+ struct bo_pool_bo_link *link = bo->map;
+ link->bo = *bo;
+
+ VG(VALGRIND_FREELIKE_BLOCK(bo->map, 0));
+ anv_ptr_free_list_push(&pool->free_list, link);
+}
projects / mesa.git / blobdiff