break;
}
+ case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SEPARATE_DEPTH_STENCIL_LAYOUTS_FEATURES_KHR: {
+ VkPhysicalDeviceSeparateDepthStencilLayoutsFeaturesKHR *features =
+ (VkPhysicalDeviceSeparateDepthStencilLayoutsFeaturesKHR *)ext;
+ features->separateDepthStencilLayouts = true;
+ break;
+ }
+
case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_INT64_FEATURES_KHR: {
VkPhysicalDeviceShaderAtomicInt64FeaturesKHR *features = (void *)ext;
features->shaderBufferInt64Atomics =
break;
}
+ case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES_KHR: {
+ VkPhysicalDeviceTimelineSemaphoreFeaturesKHR *features =
+ (VkPhysicalDeviceTimelineSemaphoreFeaturesKHR *) ext;
+ features->timelineSemaphore = true;
+ break;
+ }
+
case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VARIABLE_POINTERS_FEATURES: {
VkPhysicalDeviceVariablePointersFeatures *features = (void *)ext;
features->variablePointersStorageBuffer = true;
break;
}
+ case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_PROPERTIES_KHR: {
+ VkPhysicalDeviceTimelineSemaphorePropertiesKHR *props =
+ (VkPhysicalDeviceTimelineSemaphorePropertiesKHR *) ext;
+ props->maxTimelineSemaphoreValueDifference = UINT64_MAX;
+ break;
+ }
+
case VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TRANSFORM_FEEDBACK_PROPERTIES_EXT: {
VkPhysicalDeviceTransformFeedbackPropertiesEXT *props =
(VkPhysicalDeviceTransformFeedbackPropertiesEXT *)ext;
object, location, messageCode, pLayerPrefix, pMessage);
}
-static void
-anv_queue_init(struct anv_device *device, struct anv_queue *queue)
-{
- queue->_loader_data.loaderMagic = ICD_LOADER_MAGIC;
- queue->device = device;
- queue->flags = 0;
-}
-
-static void
-anv_queue_finish(struct anv_queue *queue)
-{
-}
-
static struct anv_state
anv_state_pool_emit_data(struct anv_state_pool *pool, size_t size, size_t align, const void *p)
{
}
}
-static void
+static VkResult
anv_device_init_trivial_batch(struct anv_device *device)
{
- anv_bo_init_new(&device->trivial_batch_bo, device, 4096);
-
- if (device->instance->physicalDevice.has_exec_async)
- device->trivial_batch_bo.flags |= EXEC_OBJECT_ASYNC;
-
- if (device->instance->physicalDevice.use_softpin)
- device->trivial_batch_bo.flags |= EXEC_OBJECT_PINNED;
-
- anv_vma_alloc(device, &device->trivial_batch_bo);
-
- void *map = anv_gem_mmap(device, device->trivial_batch_bo.gem_handle,
- 0, 4096, 0);
+ VkResult result = anv_device_alloc_bo(device, 4096,
+ ANV_BO_ALLOC_MAPPED,
+ &device->trivial_batch_bo);
+ if (result != VK_SUCCESS)
+ return result;
struct anv_batch batch = {
- .start = map,
- .next = map,
- .end = map + 4096,
+ .start = device->trivial_batch_bo->map,
+ .next = device->trivial_batch_bo->map,
+ .end = device->trivial_batch_bo->map + 4096,
};
anv_batch_emit(&batch, GEN7_MI_BATCH_BUFFER_END, bbe);
anv_batch_emit(&batch, GEN7_MI_NOOP, noop);
if (!device->info.has_llc)
- gen_clflush_range(map, batch.next - map);
+ gen_clflush_range(batch.start, batch.next - batch.start);
- anv_gem_munmap(map, device->trivial_batch_bo.size);
+ return VK_SUCCESS;
}
VkResult anv_EnumerateDeviceExtensionProperties(
}
}
-static void
+static VkResult
anv_device_init_hiz_clear_value_bo(struct anv_device *device)
{
- anv_bo_init_new(&device->hiz_clear_bo, device, 4096);
-
- if (device->instance->physicalDevice.has_exec_async)
- device->hiz_clear_bo.flags |= EXEC_OBJECT_ASYNC;
-
- if (device->instance->physicalDevice.use_softpin)
- device->hiz_clear_bo.flags |= EXEC_OBJECT_PINNED;
-
- anv_vma_alloc(device, &device->hiz_clear_bo);
-
- uint32_t *map = anv_gem_mmap(device, device->hiz_clear_bo.gem_handle,
- 0, 4096, 0);
+ VkResult result = anv_device_alloc_bo(device, 4096,
+ ANV_BO_ALLOC_MAPPED,
+ &device->hiz_clear_bo);
+ if (result != VK_SUCCESS)
+ return result;
union isl_color_value hiz_clear = { .u32 = { 0, } };
hiz_clear.f32[0] = ANV_HZ_FC_VAL;
- memcpy(map, hiz_clear.u32, sizeof(hiz_clear.u32));
- anv_gem_munmap(map, device->hiz_clear_bo.size);
+ memcpy(device->hiz_clear_bo->map, hiz_clear.u32, sizeof(hiz_clear.u32));
+
+ if (!device->info.has_llc)
+ gen_clflush_range(device->hiz_clear_bo->map, sizeof(hiz_clear.u32));
+
+ return VK_SUCCESS;
}
static bool
struct anv_block_pool *pool,
uint64_t address)
{
- for (uint32_t i = 0; i < pool->nbos; i++) {
- uint64_t bo_address = pool->bos[i].offset & (~0ull >> 16);
- uint32_t bo_size = pool->bos[i].size;
- if (address >= bo_address && address < (bo_address + bo_size)) {
+ anv_block_pool_foreach_bo(bo, pool) {
+ uint64_t bo_address = gen_48b_address(bo->offset);
+ if (address >= bo_address && address < (bo_address + bo->size)) {
*ret = (struct gen_batch_decode_bo) {
.addr = bo_address,
- .size = bo_size,
- .map = pool->bos[i].map,
+ .size = bo->size,
+ .map = bo->map,
};
return true;
}
u_vector_foreach(bo, &device->cmd_buffer_being_decoded->seen_bbos) {
/* The decoder zeroes out the top 16 bits, so we need to as well */
- uint64_t bo_address = (*bo)->bo.offset & (~0ull >> 16);
+ uint64_t bo_address = (*bo)->bo->offset & (~0ull >> 16);
- if (address >= bo_address && address < bo_address + (*bo)->bo.size) {
+ if (address >= bo_address && address < bo_address + (*bo)->bo->size) {
return (struct gen_batch_decode_bo) {
.addr = bo_address,
- .size = (*bo)->bo.size,
- .map = (*bo)->bo.map,
+ .size = (*bo)->bo->size,
+ .map = (*bo)->bo->map,
};
}
}
goto fail_fd;
}
+ result = anv_queue_init(device, &device->queue);
+ if (result != VK_SUCCESS)
+ goto fail_context_id;
+
if (physical_device->use_softpin) {
if (pthread_mutex_init(&device->vma_mutex, NULL) != 0) {
result = vk_error(VK_ERROR_INITIALIZATION_FAILED);
- goto fail_context_id;
+ goto fail_queue;
}
/* keep the page with address zero out of the allocator */
if (pthread_mutex_init(&device->mutex, NULL) != 0) {
result = vk_error(VK_ERROR_INITIALIZATION_FAILED);
- goto fail_context_id;
+ goto fail_queue;
}
pthread_condattr_t condattr;
(physical_device->has_exec_capture ? EXEC_OBJECT_CAPTURE : 0) |
(physical_device->use_softpin ? EXEC_OBJECT_PINNED : 0);
- anv_bo_pool_init(&device->batch_bo_pool, device, bo_flags);
-
result = anv_bo_cache_init(&device->bo_cache);
if (result != VK_SUCCESS)
- goto fail_batch_bo_pool;
+ goto fail_queue_cond;
- /* For state pool BOs we have to be a bit careful about where we place them
- * in the GTT. There are two documented workarounds for state base address
- * placement : Wa32bitGeneralStateOffset and Wa32bitInstructionBaseOffset
- * which state that those two base addresses do not support 48-bit
- * addresses and need to be placed in the bottom 32-bit range.
- * Unfortunately, this is not quite accurate.
- *
- * The real problem is that we always set the size of our state pools in
- * STATE_BASE_ADDRESS to 0xfffff (the maximum) even though the BO is most
- * likely significantly smaller. We do this because we do not no at the
- * time we emit STATE_BASE_ADDRESS whether or not we will need to expand
- * the pool during command buffer building so we don't actually have a
- * valid final size. If the address + size, as seen by STATE_BASE_ADDRESS
- * overflows 48 bits, the GPU appears to treat all accesses to the buffer
- * as being out of bounds and returns zero. For dynamic state, this
- * usually just leads to rendering corruptions, but shaders that are all
- * zero hang the GPU immediately.
- *
- * The easiest solution to do is exactly what the bogus workarounds say to
- * do: restrict these buffers to 32-bit addresses. We could also pin the
- * BO to some particular location of our choosing, but that's significantly
- * more work than just not setting a flag. So, we explicitly DO NOT set
- * the EXEC_OBJECT_SUPPORTS_48B_ADDRESS flag and the kernel does all of the
- * hard work for us.
- */
- if (!physical_device->use_softpin)
- bo_flags &= ~EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
+ anv_bo_pool_init(&device->batch_bo_pool, device, bo_flags);
result = anv_state_pool_init(&device->dynamic_state_pool, device,
- DYNAMIC_STATE_POOL_MIN_ADDRESS,
- 16384,
- bo_flags);
+ DYNAMIC_STATE_POOL_MIN_ADDRESS, 16384);
if (result != VK_SUCCESS)
- goto fail_bo_cache;
+ goto fail_batch_bo_pool;
result = anv_state_pool_init(&device->instruction_state_pool, device,
- INSTRUCTION_STATE_POOL_MIN_ADDRESS,
- 16384,
- bo_flags);
+ INSTRUCTION_STATE_POOL_MIN_ADDRESS, 16384);
if (result != VK_SUCCESS)
goto fail_dynamic_state_pool;
result = anv_state_pool_init(&device->surface_state_pool, device,
- SURFACE_STATE_POOL_MIN_ADDRESS,
- 4096,
- bo_flags);
+ SURFACE_STATE_POOL_MIN_ADDRESS, 4096);
if (result != VK_SUCCESS)
goto fail_instruction_state_pool;
if (physical_device->use_softpin) {
result = anv_state_pool_init(&device->binding_table_pool, device,
- BINDING_TABLE_POOL_MIN_ADDRESS,
- 4096,
- bo_flags);
+ BINDING_TABLE_POOL_MIN_ADDRESS, 4096);
if (result != VK_SUCCESS)
goto fail_surface_state_pool;
}
goto fail_binding_table_pool;
}
- result = anv_bo_init_new(&device->workaround_bo, device, 4096);
+ result = anv_device_alloc_bo(device, 4096, 0, &device->workaround_bo);
if (result != VK_SUCCESS)
goto fail_surface_aux_map_pool;
- if (physical_device->use_softpin)
- device->workaround_bo.flags |= EXEC_OBJECT_PINNED;
-
- if (!anv_vma_alloc(device, &device->workaround_bo))
+ result = anv_device_init_trivial_batch(device);
+ if (result != VK_SUCCESS)
goto fail_workaround_bo;
- anv_device_init_trivial_batch(device);
-
- if (device->info.gen >= 10)
- anv_device_init_hiz_clear_value_bo(device);
+ if (device->info.gen >= 10) {
+ result = anv_device_init_hiz_clear_value_bo(device);
+ if (result != VK_SUCCESS)
+ goto fail_trivial_batch_bo;
+ }
anv_scratch_pool_init(device, &device->scratch_pool);
- anv_queue_init(device, &device->queue);
-
switch (device->info.gen) {
case 7:
if (!device->info.is_haswell)
return VK_SUCCESS;
fail_workaround_bo:
- anv_queue_finish(&device->queue);
anv_scratch_pool_finish(device, &device->scratch_pool);
- anv_gem_munmap(device->workaround_bo.map, device->workaround_bo.size);
- anv_gem_close(device, device->workaround_bo.gem_handle);
+ if (device->info.gen >= 10)
+ anv_device_release_bo(device, device->hiz_clear_bo);
+ anv_device_release_bo(device, device->workaround_bo);
+ fail_trivial_batch_bo:
+ anv_device_release_bo(device, device->trivial_batch_bo);
fail_surface_aux_map_pool:
if (device->info.gen >= 12) {
gen_aux_map_finish(device->aux_map_ctx);
anv_state_pool_finish(&device->instruction_state_pool);
fail_dynamic_state_pool:
anv_state_pool_finish(&device->dynamic_state_pool);
- fail_bo_cache:
- anv_bo_cache_finish(&device->bo_cache);
fail_batch_bo_pool:
anv_bo_pool_finish(&device->batch_bo_pool);
+ anv_bo_cache_finish(&device->bo_cache);
+ fail_queue_cond:
pthread_cond_destroy(&device->queue_submit);
fail_mutex:
pthread_mutex_destroy(&device->mutex);
util_vma_heap_finish(&device->vma_hi);
util_vma_heap_finish(&device->vma_lo);
}
+ fail_queue:
+ anv_queue_finish(&device->queue);
fail_context_id:
anv_gem_destroy_context(device, device->context_id);
fail_fd:
anv_scratch_pool_finish(device, &device->scratch_pool);
- anv_gem_munmap(device->workaround_bo.map, device->workaround_bo.size);
- anv_vma_free(device, &device->workaround_bo);
- anv_gem_close(device, device->workaround_bo.gem_handle);
-
- anv_vma_free(device, &device->trivial_batch_bo);
- anv_gem_close(device, device->trivial_batch_bo.gem_handle);
+ anv_device_release_bo(device, device->workaround_bo);
+ anv_device_release_bo(device, device->trivial_batch_bo);
if (device->info.gen >= 10)
- anv_gem_close(device, device->hiz_clear_bo.gem_handle);
+ anv_device_release_bo(device, device->hiz_clear_bo);
if (device->info.gen >= 12) {
gen_aux_map_finish(device->aux_map_ctx);
anv_state_pool_finish(&device->instruction_state_pool);
anv_state_pool_finish(&device->dynamic_state_pool);
- anv_bo_cache_finish(&device->bo_cache);
-
anv_bo_pool_finish(&device->batch_bo_pool);
+ anv_bo_cache_finish(&device->bo_cache);
+
if (physical_device->use_softpin) {
util_vma_heap_finish(&device->vma_hi);
util_vma_heap_finish(&device->vma_lo);
uint32_t queueIndex,
VkQueue* pQueue)
{
- ANV_FROM_HANDLE(anv_device, device, _device);
-
- assert(queueIndex == 0);
+ const VkDeviceQueueInfo2 info = {
+ .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_INFO_2,
+ .pNext = NULL,
+ .flags = 0,
+ .queueFamilyIndex = queueNodeIndex,
+ .queueIndex = queueIndex,
+ };
- *pQueue = anv_queue_to_handle(&device->queue);
+ anv_GetDeviceQueue2(_device, &info, pQueue);
}
void anv_GetDeviceQueue2(
VkResult err;
va_list ap;
- device->_lost = true;
+ p_atomic_inc(&device->_lost);
va_start(ap, msg);
err = __vk_errorv(device->instance, device,
return err;
}
+VkResult
+_anv_queue_set_lost(struct anv_queue *queue,
+ const char *file, int line,
+ const char *msg, ...)
+{
+ VkResult err;
+ va_list ap;
+
+ p_atomic_inc(&queue->device->_lost);
+
+ va_start(ap, msg);
+ err = __vk_errorv(queue->device->instance, queue->device,
+ VK_DEBUG_REPORT_OBJECT_TYPE_DEVICE_EXT,
+ VK_ERROR_DEVICE_LOST, file, line, msg, ap);
+ va_end(ap);
+
+ if (env_var_as_boolean("ANV_ABORT_ON_DEVICE_LOSS", false))
+ abort();
+
+ return err;
+}
+
VkResult
anv_device_query_status(struct anv_device *device)
{
VkDevice _device)
{
ANV_FROM_HANDLE(anv_device, device, _device);
+
if (anv_device_is_lost(device))
return VK_ERROR_DEVICE_LOST;
- struct anv_batch batch;
-
- uint32_t cmds[8];
- batch.start = batch.next = cmds;
- batch.end = (void *) cmds + sizeof(cmds);
-
- anv_batch_emit(&batch, GEN7_MI_BATCH_BUFFER_END, bbe);
- anv_batch_emit(&batch, GEN7_MI_NOOP, noop);
-
- return anv_device_submit_simple_batch(device, &batch);
+ return anv_queue_submit_simple_batch(&device->queue, NULL);
}
bool
bo->offset = 0;
}
-VkResult
-anv_bo_init_new(struct anv_bo *bo, struct anv_device *device, uint64_t size)
-{
- uint32_t gem_handle = anv_gem_create(device, size);
- if (!gem_handle)
- return vk_error(VK_ERROR_OUT_OF_DEVICE_MEMORY);
-
- anv_bo_init(bo, gem_handle, size);
-
- return VK_SUCCESS;
-}
-
VkResult anv_AllocateMemory(
VkDevice _device,
const VkMemoryAllocateInfo* pAllocateInfo,
mem->ahw = NULL;
mem->host_ptr = NULL;
- uint64_t bo_flags = 0;
+ enum anv_bo_alloc_flags alloc_flags = 0;
assert(mem->type->heapIndex < pdevice->memory.heap_count);
- if (pdevice->memory.heaps[mem->type->heapIndex].supports_48bit_addresses)
- bo_flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
+ if (!pdevice->memory.heaps[mem->type->heapIndex].supports_48bit_addresses)
+ alloc_flags |= ANV_BO_ALLOC_32BIT_ADDRESS;
const struct wsi_memory_allocate_info *wsi_info =
vk_find_struct_const(pAllocateInfo->pNext, WSI_MEMORY_ALLOCATE_INFO_MESA);
* will know we're writing to them and synchronize uses on other rings
* (eg if the display server uses the blitter ring).
*/
- bo_flags |= EXEC_OBJECT_WRITE;
- } else if (pdevice->has_exec_async) {
- bo_flags |= EXEC_OBJECT_ASYNC;
+ alloc_flags |= ANV_BO_ALLOC_IMPLICIT_SYNC |
+ ANV_BO_ALLOC_IMPLICIT_WRITE;
}
- if (pdevice->use_softpin)
- bo_flags |= EXEC_OBJECT_PINNED;
-
const VkExportMemoryAllocateInfo *export_info =
vk_find_struct_const(pAllocateInfo->pNext, EXPORT_MEMORY_ALLOCATE_INFO);
fd_info->handleType ==
VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
- result = anv_bo_cache_import(device, &device->bo_cache, fd_info->fd,
- bo_flags, &mem->bo);
+ result = anv_device_import_bo(device, fd_info->fd, alloc_flags,
+ &mem->bo);
if (result != VK_SUCCESS)
goto fail;
"VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT: "
"%"PRIu64"B > %"PRIu64"B",
aligned_alloc_size, mem->bo->size);
- anv_bo_cache_release(device, &device->bo_cache, mem->bo);
+ anv_device_release_bo(device, mem->bo);
goto fail;
}
assert(host_ptr_info->handleType ==
VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT);
- result = anv_bo_cache_import_host_ptr(
- device, &device->bo_cache, host_ptr_info->pHostPointer,
- pAllocateInfo->allocationSize, bo_flags, &mem->bo);
+ result = anv_device_import_bo_from_host_ptr(device,
+ host_ptr_info->pHostPointer,
+ pAllocateInfo->allocationSize,
+ alloc_flags,
+ &mem->bo);
if (result != VK_SUCCESS)
goto fail;
/* Regular allocate (not importing memory). */
- bool is_external = export_info && export_info->handleTypes;
- result = anv_bo_cache_alloc(device, &device->bo_cache,
- pAllocateInfo->allocationSize,
- bo_flags, is_external,
- &mem->bo);
+ if (export_info && export_info->handleTypes)
+ alloc_flags |= ANV_BO_ALLOC_EXTERNAL;
+
+ result = anv_device_alloc_bo(device, pAllocateInfo->allocationSize,
+ alloc_flags, &mem->bo);
if (result != VK_SUCCESS)
goto fail;
image->planes[0].surface.isl.row_pitch_B,
i915_tiling);
if (ret) {
- anv_bo_cache_release(device, &device->bo_cache, mem->bo);
+ anv_device_release_bo(device, mem->bo);
return vk_errorf(device->instance, NULL,
VK_ERROR_OUT_OF_DEVICE_MEMORY,
"failed to set BO tiling: %m");
assert(pGetFdInfo->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT ||
pGetFdInfo->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
- return anv_bo_cache_export(dev, &dev->bo_cache, mem->bo, pFd);
+ return anv_device_export_bo(dev, mem->bo, pFd);
}
VkResult anv_GetMemoryFdPropertiesKHR(
p_atomic_add(&pdevice->memory.heaps[mem->type->heapIndex].used,
-mem->bo->size);
- anv_bo_cache_release(device, &device->bo_cache, mem->bo);
+ anv_device_release_bo(device, mem->bo);
#if defined(ANDROID) && ANDROID_API_LEVEL >= 26
if (mem->ahw)
projects / mesa.git / blobdiff