{
size_t size;
void *map;
- int gem_handle;
+ uint32_t gem_handle;
struct anv_mmap_cleanup *cleanup;
pthread_mutex_lock(&pool->device->mutex);
goto fail;
*cleanup = ANV_MMAP_CLEANUP_INIT;
- /* First try to see if mremap can grow the map in place. */
- map = MAP_FAILED;
- if (old_size > 0 && center_bo_offset == 0)
- map = mremap(pool->map, old_size, size, 0);
- if (map == MAP_FAILED) {
- /* Just leak the old map until we destroy the pool. We can't munmap it
- * without races or imposing locking on the block allocate fast path. On
- * the whole the leaked maps adds up to less than the size of the
- * current map. MAP_POPULATE seems like the right thing to do, but we
- * should try to get some numbers.
- */
- map = mmap(NULL, size, PROT_READ | PROT_WRITE,
- MAP_SHARED | MAP_POPULATE, pool->fd,
- BLOCK_POOL_MEMFD_CENTER - center_bo_offset);
- cleanup->map = map;
- cleanup->size = size;
- }
+ /* Just leak the old map until we destroy the pool. We can't munmap it
+ * without races or imposing locking on the block allocate fast path. On
+ * the whole the leaked maps adds up to less than the size of the
+ * current map. MAP_POPULATE seems like the right thing to do, but we
+ * should try to get some numbers.
+ */
+ map = mmap(NULL, size, PROT_READ | PROT_WRITE,
+ MAP_SHARED | MAP_POPULATE, pool->fd,
+ BLOCK_POOL_MEMFD_CENTER - center_bo_offset);
+ cleanup->map = map;
+ cleanup->size = size;
+
if (map == MAP_FAILED)
goto fail;
goto fail;
cleanup->gem_handle = gem_handle;
+ /* Regular objects are created I915_CACHING_CACHED on LLC platforms and
+ * I915_CACHING_NONE on non-LLC platforms. However, userptr objects are
+ * always created as I915_CACHING_CACHED, which on non-LLC means
+ * snooped. That can be useful but comes with a bit of overheard. Since
+ * we're eplicitly clflushing and don't want the overhead we need to turn
+ * it off. */
+ if (!pool->device->info.has_llc) {
+ anv_gem_set_caching(pool->device, gem_handle, I915_CACHING_NONE);
+ anv_gem_set_domain(pool->device, gem_handle,
+ I915_GEM_DOMAIN_GTT, I915_GEM_DOMAIN_GTT);
+ }
+
/* Now that we successfull allocated everything, we can write the new
* values back into pool. */
pool->map = map + center_bo_offset;
}
#define NULL_BLOCK 1
-struct stream_block {
- uint32_t next;
+struct anv_state_stream_block {
+ /* The next block */
+ struct anv_state_stream_block *next;
- /* The map for the BO at the time the block was givne to us */
- void *current_map;
+ /* The offset into the block pool at which this block starts */
+ uint32_t offset;
#ifdef HAVE_VALGRIND
+ /* A pointer to the first user-allocated thing in this block. This is
+ * what valgrind sees as the start of the block.
+ */
void *_vg_ptr;
#endif
};
struct anv_block_pool *block_pool)
{
stream->block_pool = block_pool;
- stream->next = 0;
+ stream->block = NULL;
+
+ /* Ensure that next + whatever > end. This way the first call to
+ * state_stream_alloc fetches a new block.
+ */
+ stream->next = 1;
stream->end = 0;
- stream->current_block = NULL_BLOCK;
VG(VALGRIND_CREATE_MEMPOOL(stream, 0, false));
}
void
anv_state_stream_finish(struct anv_state_stream *stream)
{
- struct stream_block *sb;
- uint32_t block, next_block;
-
- block = stream->current_block;
- while (block != NULL_BLOCK) {
- sb = stream->block_pool->map + block;
- next_block = VG_NOACCESS_READ(&sb->next);
- VG(VALGRIND_MEMPOOL_FREE(stream, VG_NOACCESS_READ(&sb->_vg_ptr)));
- anv_block_pool_free(stream->block_pool, block);
- block = next_block;
+ VG(const uint32_t block_size = stream->block_pool->block_size);
+
+ struct anv_state_stream_block *next = stream->block;
+ while (next != NULL) {
+ VG(VALGRIND_MAKE_MEM_DEFINED(next, sizeof(*next)));
+ struct anv_state_stream_block sb = VG_NOACCESS_READ(next);
+ VG(VALGRIND_MEMPOOL_FREE(stream, sb._vg_ptr));
+ VG(VALGRIND_MAKE_MEM_UNDEFINED(next, block_size));
+ anv_block_pool_free(stream->block_pool, sb.offset);
+ next = sb.next;
}
VG(VALGRIND_DESTROY_MEMPOOL(stream));
anv_state_stream_alloc(struct anv_state_stream *stream,
uint32_t size, uint32_t alignment)
{
- struct stream_block *sb;
+ struct anv_state_stream_block *sb = stream->block;
+
struct anv_state state;
- uint32_t block;
state.offset = align_u32(stream->next, alignment);
if (state.offset + size > stream->end) {
- block = anv_block_pool_alloc(stream->block_pool);
- void *current_map = stream->block_pool->map;
- sb = current_map + block;
- VG_NOACCESS_WRITE(&sb->current_map, current_map);
- VG_NOACCESS_WRITE(&sb->next, stream->current_block);
- VG(VG_NOACCESS_WRITE(&sb->_vg_ptr, 0));
- stream->current_block = block;
+ uint32_t block = anv_block_pool_alloc(stream->block_pool);
+ sb = stream->block_pool->map + block;
+
+ VG(VALGRIND_MAKE_MEM_UNDEFINED(sb, sizeof(*sb)));
+ sb->next = stream->block;
+ sb->offset = block;
+ VG(sb->_vg_ptr = NULL);
+ VG(VALGRIND_MAKE_MEM_NOACCESS(sb, stream->block_pool->block_size));
+
+ stream->block = sb;
+ stream->start = block;
stream->next = block + sizeof(*sb);
stream->end = block + stream->block_pool->block_size;
+
state.offset = align_u32(stream->next, alignment);
assert(state.offset + size <= stream->end);
}
- sb = stream->block_pool->map + stream->current_block;
- void *current_map = VG_NOACCESS_READ(&sb->current_map);
-
- state.map = current_map + state.offset;
+ assert(state.offset > stream->start);
+ state.map = (void *)sb + (state.offset - stream->start);
state.alloc_size = size;
#ifdef HAVE_VALGRIND
VG_NOACCESS_WRITE(&sb->_vg_ptr, vg_ptr);
VALGRIND_MEMPOOL_ALLOC(stream, vg_ptr, size);
} else {
- ptrdiff_t vg_offset = vg_ptr - current_map;
- assert(vg_offset >= stream->current_block &&
- vg_offset < stream->end);
- VALGRIND_MEMPOOL_CHANGE(stream, vg_ptr, vg_ptr,
- (state.offset + size) - vg_offset);
+ void *state_end = state.map + state.alloc_size;
+ /* This only updates the mempool. The newly allocated chunk is still
+ * marked as NOACCESS. */
+ VALGRIND_MEMPOOL_CHANGE(stream, vg_ptr, vg_ptr, state_end - vg_ptr);
+ /* Mark the newly allocated chunk as undefined */
+ VALGRIND_MAKE_MEM_UNDEFINED(state.map, state.alloc_size);
}
#endif
assert(new_bo.size == pool->bo_size);
- new_bo.map = anv_gem_mmap(pool->device, new_bo.gem_handle, 0, pool->bo_size);
+ new_bo.map = anv_gem_mmap(pool->device, new_bo.gem_handle, 0, pool->bo_size, 0);
if (new_bo.map == NULL) {
anv_gem_close(pool->device, new_bo.gem_handle);
return vk_error(VK_ERROR_MEMORY_MAP_FAILED);
projects / mesa.git / blobdiff