static uint32_t
anv_block_pool_alloc_new(struct anv_block_pool *pool,
struct anv_block_state *pool_state,
- uint32_t block_size)
+ uint32_t block_size, uint32_t *padding)
{
struct anv_block_state state, old, new;
+ /* Most allocations won't generate any padding */
+ if (padding)
+ *padding = 0;
+
while (1) {
state.u64 = __sync_fetch_and_add(&pool_state->u64, block_size);
if (state.next + block_size <= state.end) {
return state.next;
} else if (state.next <= state.end) {
+ if (pool->bo_flags & EXEC_OBJECT_PINNED && state.next < state.end) {
+ /* We need to grow the block pool, but still have some leftover
+ * space that can't be used by that particular allocation. So we
+ * add that as a "padding", and return it.
+ */
+ uint32_t leftover = state.end - state.next;
+
+ /* If there is some leftover space in the pool, the caller must
+ * deal with it.
+ */
+ assert(leftover == 0 || padding);
+ if (padding)
+ *padding = leftover;
+ state.next += leftover;
+ }
+
/* We allocated the first block outside the pool so we have to grow
* the pool. pool_state->next acts a mutex: threads who try to
* allocate now will get block indexes above the current limit and
int32_t
anv_block_pool_alloc(struct anv_block_pool *pool,
- uint32_t block_size)
+ uint32_t block_size, uint32_t *padding)
{
- return anv_block_pool_alloc_new(pool, &pool->state, block_size);
+ uint32_t offset;
+
+ offset = anv_block_pool_alloc_new(pool, &pool->state, block_size, padding);
+
+ return offset;
}
/* Allocates a block out of the back of the block pool.
uint32_t block_size)
{
int32_t offset = anv_block_pool_alloc_new(pool, &pool->back_state,
- block_size);
+ block_size, NULL);
/* The offset we get out of anv_block_pool_alloc_new() is actually the
* number of bytes downwards from the middle to the end of the block.
anv_fixed_size_state_pool_alloc_new(struct anv_fixed_size_state_pool *pool,
struct anv_block_pool *block_pool,
uint32_t state_size,
- uint32_t block_size)
+ uint32_t block_size,
+ uint32_t *padding)
{
struct anv_block_state block, old, new;
uint32_t offset;
+ /* We don't always use anv_block_pool_alloc(), which would set *padding to
+ * zero for us. So if we have a pointer to padding, we must zero it out
+ * ourselves here, to make sure we always return some sensible value.
+ */
+ if (padding)
+ *padding = 0;
+
/* If our state is large, we don't need any sub-allocation from a block.
* Instead, we just grab whole (potentially large) blocks.
*/
if (state_size >= block_size)
- return anv_block_pool_alloc(block_pool, state_size);
+ return anv_block_pool_alloc(block_pool, state_size, padding);
restart:
block.u64 = __sync_fetch_and_add(&pool->block.u64, state_size);
if (block.next < block.end) {
return block.next;
} else if (block.next == block.end) {
- offset = anv_block_pool_alloc(block_pool, block_size);
+ offset = anv_block_pool_alloc(block_pool, block_size, padding);
new.next = offset + state_size;
new.end = offset + block_size;
old.u64 = __sync_lock_test_and_set(&pool->block.u64, new.u64);
uint32_t chunk_offset, uint32_t count,
uint32_t block_size)
{
- if (count == 0)
- return;
+ /* Disallow returning 0 chunks */
+ assert(count != 0);
/* Make sure we always return chunks aligned to the block_size */
assert(chunk_offset % block_size == 0);
&pool->table, st_idx, count);
}
+/** Returns a chunk of memory back to the state pool.
+ *
+ * Do a two-level split. If chunk_size is bigger than divisor
+ * (pool->block_size), we return as many divisor sized blocks as we can, from
+ * the end of the chunk.
+ *
+ * The remaining is then split into smaller blocks (starting at small_size if
+ * it is non-zero), with larger blocks always being taken from the end of the
+ * chunk.
+ */
+static void
+anv_state_pool_return_chunk(struct anv_state_pool *pool,
+ uint32_t chunk_offset, uint32_t chunk_size,
+ uint32_t small_size)
+{
+ uint32_t divisor = pool->block_size;
+ uint32_t nblocks = chunk_size / divisor;
+ uint32_t rest = chunk_size - nblocks * divisor;
+
+ if (nblocks > 0) {
+ /* First return divisor aligned and sized chunks. We start returning
+ * larger blocks from the end fo the chunk, since they should already be
+ * aligned to divisor. Also anv_state_pool_return_blocks() only accepts
+ * aligned chunks.
+ */
+ uint32_t offset = chunk_offset + rest;
+ anv_state_pool_return_blocks(pool, offset, nblocks, divisor);
+ }
+
+ chunk_size = rest;
+ divisor /= 2;
+
+ if (small_size > 0 && small_size < divisor)
+ divisor = small_size;
+
+ uint32_t min_size = 1 << ANV_MIN_STATE_SIZE_LOG2;
+
+ /* Just as before, return larger divisor aligned blocks from the end of the
+ * chunk first.
+ */
+ while (chunk_size > 0 && divisor >= min_size) {
+ nblocks = chunk_size / divisor;
+ rest = chunk_size - nblocks * divisor;
+ if (nblocks > 0) {
+ anv_state_pool_return_blocks(pool, chunk_offset + rest,
+ nblocks, divisor);
+ chunk_size = rest;
+ }
+ divisor /= 2;
+ }
+}
+
static struct anv_state
anv_state_pool_alloc_no_vg(struct anv_state_pool *pool,
uint32_t size, uint32_t align)
*/
state->alloc_size = alloc_size;
- /* We've found a chunk that's larger than the requested state size.
+ /* Now return the unused part of the chunk back to the pool as free
+ * blocks
+ *
* There are a couple of options as to what we do with it:
*
* 1) We could fully split the chunk into state.alloc_size sized
* two-level split. If it's bigger than some fixed block_size,
* we split it into block_size sized chunks and return all but
* one of them. Then we split what remains into
- * state.alloc_size sized chunks and return all but one.
+ * state.alloc_size sized chunks and return them.
*
- * We choose option (3).
+ * We choose something close to option (3), which is implemented with
+ * anv_state_pool_return_chunk(). That is done by returning the
+ * remaining of the chunk, with alloc_size as a hint of the size that
+ * we want the smaller chunk split into.
*/
- if (chunk_size > pool->block_size &&
- alloc_size < pool->block_size) {
- assert(chunk_size % pool->block_size == 0);
- /* We don't want to split giant chunks into tiny chunks. Instead,
- * break anything bigger than a block into block-sized chunks and
- * then break it down into bucket-sized chunks from there. Return
- * all but the first block of the chunk to the block bucket.
- */
- uint32_t push_back = (chunk_size / pool->block_size) - 1;
- anv_state_pool_return_blocks(pool, chunk_offset + pool->block_size,
- push_back, pool->block_size);
- chunk_size = pool->block_size;
- }
-
- assert(chunk_size % alloc_size == 0);
- uint32_t push_back = (chunk_size / alloc_size) - 1;
- anv_state_pool_return_blocks(pool, chunk_offset + alloc_size,
- push_back, alloc_size);
+ anv_state_pool_return_chunk(pool, chunk_offset + alloc_size,
+ chunk_size - alloc_size, alloc_size);
goto done;
}
}
+ uint32_t padding;
offset = anv_fixed_size_state_pool_alloc_new(&pool->buckets[bucket],
&pool->block_pool,
alloc_size,
- pool->block_size);
+ pool->block_size,
+ &padding);
/* Everytime we allocate a new state, add it to the state pool */
uint32_t idx;
VkResult result = anv_state_table_add(&pool->table, &idx, 1);
state->alloc_size = alloc_size;
state->map = anv_block_pool_map(&pool->block_pool, offset);
+ if (padding > 0) {
+ uint32_t return_offset = offset - padding;
+ anv_state_pool_return_chunk(pool, return_offset, padding, 0);
+ }
+
done:
return *state;
}
projects / mesa.git / blobdiff