return -(offset + block_size);
}
-void
+VkResult
anv_state_pool_init(struct anv_state_pool *pool,
- struct anv_block_pool *block_pool,
+ struct anv_device *device,
uint32_t block_size)
{
- pool->block_pool = block_pool;
+ VkResult result = anv_block_pool_init(&pool->block_pool, device,
+ block_size * 16);
+ if (result != VK_SUCCESS)
+ return result;
+
assert(util_is_power_of_two(block_size));
pool->block_size = block_size;
pool->back_alloc_free_list = ANV_FREE_LIST_EMPTY;
pool->buckets[i].block.end = 0;
}
VG(VALGRIND_CREATE_MEMPOOL(pool, 0, false));
+
+ return VK_SUCCESS;
}
void
anv_state_pool_finish(struct anv_state_pool *pool)
{
VG(VALGRIND_DESTROY_MEMPOOL(pool));
+ anv_block_pool_finish(&pool->block_pool);
}
static uint32_t
/* Try free list first. */
if (anv_free_list_pop(&pool->buckets[bucket].free_list,
- &pool->block_pool->map, &state.offset)) {
+ &pool->block_pool.map, &state.offset)) {
assert(state.offset >= 0);
goto done;
}
state.offset = anv_fixed_size_state_pool_alloc_new(&pool->buckets[bucket],
- pool->block_pool,
+ &pool->block_pool,
state.alloc_size,
pool->block_size);
done:
- state.map = pool->block_pool->map + state.offset;
+ state.map = pool->block_pool.map + state.offset;
return state;
}
state.alloc_size = pool->block_size;
if (anv_free_list_pop(&pool->back_alloc_free_list,
- &pool->block_pool->map, &state.offset)) {
+ &pool->block_pool.map, &state.offset)) {
assert(state.offset < 0);
goto done;
}
- state.offset = anv_block_pool_alloc_back(pool->block_pool, pool->block_size);
+ state.offset = anv_block_pool_alloc_back(&pool->block_pool,
+ pool->block_size);
done:
- state.map = pool->block_pool->map + state.offset;
+ state.map = pool->block_pool.map + state.offset;
VG(VALGRIND_MEMPOOL_ALLOC(pool, state.map, state.alloc_size));
return state;
}
if (state.offset < 0) {
assert(state.alloc_size == pool->block_size);
anv_free_list_push(&pool->back_alloc_free_list,
- pool->block_pool->map, state.offset);
+ pool->block_pool.map, state.offset);
} else {
anv_free_list_push(&pool->buckets[bucket].free_list,
- pool->block_pool->map, state.offset);
+ pool->block_pool.map, state.offset);
}
}
{
struct anv_state *bt_block = u_vector_head(&cmd_buffer->bt_block_states);
return (struct anv_address) {
- .bo = &cmd_buffer->device->surface_state_block_pool.bo,
+ .bo = &cmd_buffer->device->surface_state_pool.block_pool.bo,
.offset = bt_block->offset,
};
}
return (struct anv_state) { 0 };
state.offset = cmd_buffer->bt_next;
- state.map = state_pool->block_pool->map + bt_block->offset + state.offset;
+ state.map = state_pool->block_pool.map + bt_block->offset + state.offset;
cmd_buffer->bt_next += state.alloc_size;
}
static void
-adjust_relocations_from_state_pool(struct anv_block_pool *pool,
+adjust_relocations_from_state_pool(struct anv_state_pool *pool,
struct anv_reloc_list *relocs,
uint32_t last_pool_center_bo_offset)
{
- assert(last_pool_center_bo_offset <= pool->center_bo_offset);
- uint32_t delta = pool->center_bo_offset - last_pool_center_bo_offset;
+ assert(last_pool_center_bo_offset <= pool->block_pool.center_bo_offset);
+ uint32_t delta = pool->block_pool.center_bo_offset - last_pool_center_bo_offset;
for (size_t i = 0; i < relocs->num_relocs; i++) {
/* All of the relocations from this block pool to other BO's should
}
static void
-adjust_relocations_to_state_pool(struct anv_block_pool *pool,
+adjust_relocations_to_state_pool(struct anv_state_pool *pool,
struct anv_bo *from_bo,
struct anv_reloc_list *relocs,
uint32_t last_pool_center_bo_offset)
{
- assert(last_pool_center_bo_offset <= pool->center_bo_offset);
- uint32_t delta = pool->center_bo_offset - last_pool_center_bo_offset;
+ assert(last_pool_center_bo_offset <= pool->block_pool.center_bo_offset);
+ uint32_t delta = pool->block_pool.center_bo_offset - last_pool_center_bo_offset;
/* When we initially emit relocations into a block pool, we don't
* actually know what the final center_bo_offset will be so we just emit
* relocations that point to the pool bo with the correct offset.
*/
for (size_t i = 0; i < relocs->num_relocs; i++) {
- if (relocs->reloc_bos[i] == &pool->bo) {
+ if (relocs->reloc_bos[i] == &pool->block_pool.bo) {
/* Adjust the delta value in the relocation to correctly
* correspond to the new delta. Initially, this value may have
* been negative (if treated as unsigned), but we trust in
* use by the GPU at the moment.
*/
assert(relocs->relocs[i].offset < from_bo->size);
- write_reloc(pool->device, from_bo->map + relocs->relocs[i].offset,
+ write_reloc(pool->block_pool.device,
+ from_bo->map + relocs->relocs[i].offset,
relocs->relocs[i].presumed_offset +
relocs->relocs[i].delta, false);
}
* given time. The only option is to always relocate them.
*/
anv_reloc_list_apply(cmd_buffer->device, &cmd_buffer->surface_relocs,
- &cmd_buffer->device->surface_state_block_pool.bo,
+ &cmd_buffer->device->surface_state_pool.block_pool.bo,
true /* always relocate surface states */);
/* Since we own all of the batch buffers, we know what values are stored
struct anv_cmd_buffer *cmd_buffer)
{
struct anv_batch *batch = &cmd_buffer->batch;
- struct anv_block_pool *ss_pool =
- &cmd_buffer->device->surface_state_block_pool;
+ struct anv_state_pool *ss_pool =
+ &cmd_buffer->device->surface_state_pool;
adjust_relocations_from_state_pool(ss_pool, &cmd_buffer->surface_relocs,
cmd_buffer->last_ss_pool_center);
- VkResult result =
- anv_execbuf_add_bo(execbuf, &ss_pool->bo, &cmd_buffer->surface_relocs, 0,
- &cmd_buffer->device->alloc);
+ VkResult result = anv_execbuf_add_bo(execbuf, &ss_pool->block_pool.bo,
+ &cmd_buffer->surface_relocs, 0,
+ &cmd_buffer->device->alloc);
if (result != VK_SUCCESS)
return result;
* record the surface state pool center so future executions of the command
* buffer can adjust correctly.
*/
- cmd_buffer->last_ss_pool_center = ss_pool->center_bo_offset;
+ cmd_buffer->last_ss_pool_center = ss_pool->block_pool.center_bo_offset;
struct anv_batch_bo *first_batch_bo =
list_first_entry(&cmd_buffer->batch_bos, struct anv_batch_bo, link);
bs = gcd_pow2_u64(bs, copy_size);
do_buffer_copy(&batch,
- &cmd_buffer->device->dynamic_state_block_pool.bo,
+ &cmd_buffer->device->dynamic_state_pool.block_pool.bo,
tmp_data.offset,
dst_buffer->bo, dst_buffer->offset + dstOffset,
copy_size / bs, 1, bs);
state = anv_state_pool_alloc(pool, size, align);
memcpy(state.map, p, size);
- anv_state_flush(pool->block_pool->device, state);
+ anv_state_flush(pool->block_pool.device, state);
return state;
}
if (result != VK_SUCCESS)
goto fail_batch_bo_pool;
- result = anv_block_pool_init(&device->dynamic_state_block_pool, device,
- 16384 * 16);
+ result = anv_state_pool_init(&device->dynamic_state_pool, device, 16384);
if (result != VK_SUCCESS)
goto fail_bo_cache;
- anv_state_pool_init(&device->dynamic_state_pool,
- &device->dynamic_state_block_pool,
- 16384);
-
- result = anv_block_pool_init(&device->instruction_block_pool, device,
- 1024 * 1024 * 16);
+ result = anv_state_pool_init(&device->instruction_state_pool, device,
+ 1024 * 1024);
if (result != VK_SUCCESS)
goto fail_dynamic_state_pool;
- anv_state_pool_init(&device->instruction_state_pool,
- &device->instruction_block_pool,
- 1024 * 1024);
-
- result = anv_block_pool_init(&device->surface_state_block_pool, device,
- 4096 * 16);
+ result = anv_state_pool_init(&device->surface_state_pool, device, 4096);
if (result != VK_SUCCESS)
goto fail_instruction_state_pool;
- anv_state_pool_init(&device->surface_state_pool,
- &device->surface_state_block_pool,
- 4096);
-
result = anv_bo_init_new(&device->workaround_bo, device, 1024);
if (result != VK_SUCCESS)
goto fail_surface_state_pool;
anv_gem_close(device, device->workaround_bo.gem_handle);
fail_surface_state_pool:
anv_state_pool_finish(&device->surface_state_pool);
- anv_block_pool_finish(&device->surface_state_block_pool);
fail_instruction_state_pool:
anv_state_pool_finish(&device->instruction_state_pool);
- anv_block_pool_finish(&device->instruction_block_pool);
fail_dynamic_state_pool:
anv_state_pool_finish(&device->dynamic_state_pool);
- anv_block_pool_finish(&device->dynamic_state_block_pool);
fail_bo_cache:
anv_bo_cache_finish(&device->bo_cache);
fail_batch_bo_pool:
anv_gem_close(device, device->workaround_bo.gem_handle);
anv_state_pool_finish(&device->surface_state_pool);
- anv_block_pool_finish(&device->surface_state_block_pool);
anv_state_pool_finish(&device->instruction_state_pool);
- anv_block_pool_finish(&device->instruction_block_pool);
anv_state_pool_finish(&device->dynamic_state_pool);
- anv_block_pool_finish(&device->dynamic_state_block_pool);
anv_bo_cache_finish(&device->bo_cache);
#define ANV_STATE_BUCKETS (ANV_MAX_STATE_SIZE_LOG2 - ANV_MIN_STATE_SIZE_LOG2 + 1)
struct anv_state_pool {
- struct anv_block_pool *block_pool;
+ struct anv_block_pool block_pool;
/* The size of blocks which will be allocated from the block pool */
uint32_t block_size;
__builtin_ia32_mfence();
}
+/* The block_pool functions exported for testing only. The block pool should
+ * only be used via a state pool (see below).
+ */
VkResult anv_block_pool_init(struct anv_block_pool *pool,
struct anv_device *device,
uint32_t initial_size);
uint32_t block_size);
int32_t anv_block_pool_alloc_back(struct anv_block_pool *pool,
uint32_t block_size);
-void anv_state_pool_init(struct anv_state_pool *pool,
- struct anv_block_pool *block_pool,
- uint32_t block_size);
+
+VkResult anv_state_pool_init(struct anv_state_pool *pool,
+ struct anv_device *device,
+ uint32_t block_size);
void anv_state_pool_finish(struct anv_state_pool *pool);
struct anv_state anv_state_pool_alloc(struct anv_state_pool *pool,
uint32_t state_size, uint32_t alignment);
struct anv_bo_cache bo_cache;
- struct anv_block_pool dynamic_state_block_pool;
struct anv_state_pool dynamic_state_pool;
-
- struct anv_block_pool instruction_block_pool;
struct anv_state_pool instruction_state_pool;
-
- struct anv_block_pool surface_state_block_pool;
struct anv_state_pool surface_state_pool;
struct anv_bo workaround_bo;
pc.DestinationAddressType = DAT_PPGTT,
pc.PostSyncOperation = WriteImmediateData,
pc.Address = (struct anv_address) {
- &cmd_buffer->device->dynamic_state_block_pool.bo,
+ &cmd_buffer->device->dynamic_state_pool.block_pool.bo,
event->state.offset
};
pc.ImmediateData = VK_EVENT_SET;
pc.DestinationAddressType = DAT_PPGTT;
pc.PostSyncOperation = WriteImmediateData;
pc.Address = (struct anv_address) {
- &cmd_buffer->device->dynamic_state_block_pool.bo,
+ &cmd_buffer->device->dynamic_state_pool.block_pool.bo,
event->state.offset
};
pc.ImmediateData = VK_EVENT_RESET;
sem.CompareOperation = COMPARE_SAD_EQUAL_SDD,
sem.SemaphoreDataDword = VK_EVENT_SET,
sem.SemaphoreAddress = (struct anv_address) {
- &cmd_buffer->device->dynamic_state_block_pool.bo,
+ &cmd_buffer->device->dynamic_state_pool.block_pool.bo,
event->state.offset
};
}
anv_cmd_buffer_alloc_dynamic_state(cmd_buffer, size, 64);
*addr = (struct blorp_address) {
- .buffer = &cmd_buffer->device->dynamic_state_block_pool.bo,
+ .buffer = &cmd_buffer->device->dynamic_state_pool.block_pool.bo,
.offset = vb_state.offset,
};
sba.SurfaceStateBaseAddressModifyEnable = true;
sba.DynamicStateBaseAddress =
- (struct anv_address) { &device->dynamic_state_block_pool.bo, 0 };
+ (struct anv_address) { &device->dynamic_state_pool.block_pool.bo, 0 };
sba.DynamicStateMemoryObjectControlState = GENX(MOCS);
sba.DynamicStateBaseAddressModifyEnable = true;
sba.IndirectObjectBaseAddressModifyEnable = true;
sba.InstructionBaseAddress =
- (struct anv_address) { &device->instruction_block_pool.bo, 0 };
+ (struct anv_address) { &device->instruction_state_pool.block_pool.bo, 0 };
sba.InstructionMemoryObjectControlState = GENX(MOCS);
sba.InstructionBaseAddressModifyEnable = true;
* copy the surface states for the current subpass into the storage
* we allocated for them in BeginCommandBuffer.
*/
- struct anv_bo *ss_bo = &primary->device->surface_state_block_pool.bo;
+ struct anv_bo *ss_bo =
+ &primary->device->surface_state_pool.block_pool.bo;
struct anv_state src_state = primary->state.render_pass_states;
struct anv_state dst_state = secondary->state.render_pass_states;
assert(src_state.alloc_size == dst_state.alloc_size);
c._3DCommandSubOpcode = push_constant_opcodes[stage],
c.ConstantBody = (struct GENX(3DSTATE_CONSTANT_BODY)) {
#if GEN_GEN >= 9
- .PointerToConstantBuffer2 = { &cmd_buffer->device->dynamic_state_block_pool.bo, state.offset },
+ .PointerToConstantBuffer2 = { &cmd_buffer->device->dynamic_state_pool.block_pool.bo, state.offset },
.ConstantBuffer2ReadLength = DIV_ROUND_UP(state.alloc_size, 32),
#else
.PointerToConstantBuffer0 = { .offset = state.offset },
anv_state_flush(cmd_buffer->device, id_state);
emit_base_vertex_instance_bo(cmd_buffer,
- &cmd_buffer->device->dynamic_state_block_pool.bo, id_state.offset);
+ &cmd_buffer->device->dynamic_state_pool.block_pool.bo, id_state.offset);
}
static void
anv_state_flush(cmd_buffer->device, state);
emit_vertex_bo(cmd_buffer,
- &cmd_buffer->device->dynamic_state_block_pool.bo,
+ &cmd_buffer->device->dynamic_state_pool.block_pool.bo,
state.offset, 4, ANV_DRAWID_VB_INDEX);
}
anv_state_flush(cmd_buffer->device, state);
cmd_buffer->state.num_workgroups_offset = state.offset;
cmd_buffer->state.num_workgroups_bo =
- &cmd_buffer->device->dynamic_state_block_pool.bo;
+ &cmd_buffer->device->dynamic_state_pool.block_pool.bo;
}
genX(cmd_buffer_flush_compute_state)(cmd_buffer);
struct anv_device device = {
.instance = &instance,
};
- struct anv_block_pool block_pool;
struct anv_state_pool state_pool;
pthread_mutex_init(&device.mutex, NULL);
for (unsigned i = 0; i < NUM_RUNS; i++) {
- anv_block_pool_init(&block_pool, &device, 4096);
- anv_state_pool_init(&state_pool, &block_pool, 256);
+ anv_state_pool_init(&state_pool, &device, 256);
/* Grab one so a zero offset is impossible */
anv_state_pool_alloc(&state_pool, 16, 16);
run_state_pool_test(&state_pool);
anv_state_pool_finish(&state_pool);
- anv_block_pool_finish(&block_pool);
}
pthread_mutex_destroy(&device.mutex);
struct anv_device device = {
.instance = &instance,
};
- struct anv_block_pool block_pool;
struct anv_state_pool state_pool;
pthread_mutex_init(&device.mutex, NULL);
- anv_block_pool_init(&block_pool, &device, 4096);
- anv_state_pool_init(&state_pool, &block_pool, 4096);
+ anv_state_pool_init(&state_pool, &device, 4096);
/* Grab one so a zero offset is impossible */
anv_state_pool_alloc(&state_pool, 16, 16);
run_state_pool_test(&state_pool);
anv_state_pool_finish(&state_pool);
- anv_block_pool_finish(&block_pool);
pthread_mutex_destroy(&device.mutex);
}
struct anv_device device = {
.instance = &instance,
};
- struct anv_block_pool block_pool;
struct anv_state_pool state_pool;
pthread_mutex_init(&device.mutex, NULL);
- anv_block_pool_init(&block_pool, &device, 4096);
- anv_state_pool_init(&state_pool, &block_pool, 64);
+ anv_state_pool_init(&state_pool, &device, 64);
pthread_barrier_init(&barrier, NULL, NUM_THREADS);
}
anv_state_pool_finish(&state_pool);
- anv_block_pool_finish(&block_pool);
pthread_mutex_destroy(&device.mutex);
}