pthread_mutex_destroy(&cache->mutex);
}
-static struct anv_bo *
-anv_bo_cache_lookup(struct anv_bo_cache *cache, uint32_t gem_handle)
-{
- return util_sparse_array_get(&cache->bo_map, gem_handle);
-}
-
#define ANV_BO_CACHE_SUPPORTED_FLAGS \
(EXEC_OBJECT_WRITE | \
EXEC_OBJECT_ASYNC | \
EXEC_OBJECT_SUPPORTS_48B_ADDRESS | \
- EXEC_OBJECT_PINNED)
+ EXEC_OBJECT_PINNED | \
+ EXEC_OBJECT_CAPTURE)
+
+static uint32_t
+anv_bo_alloc_flags_to_bo_flags(struct anv_device *device,
+ enum anv_bo_alloc_flags alloc_flags)
+{
+ struct anv_physical_device *pdevice = &device->instance->physicalDevice;
+
+ uint64_t bo_flags = 0;
+ if (!(alloc_flags & ANV_BO_ALLOC_32BIT_ADDRESS) &&
+ pdevice->supports_48bit_addresses)
+ bo_flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
+
+ if ((alloc_flags & ANV_BO_ALLOC_CAPTURE) && pdevice->has_exec_capture)
+ bo_flags |= EXEC_OBJECT_CAPTURE;
+
+ if (alloc_flags & ANV_BO_ALLOC_IMPLICIT_WRITE) {
+ assert(alloc_flags & ANV_BO_ALLOC_IMPLICIT_SYNC);
+ bo_flags |= EXEC_OBJECT_WRITE;
+ }
+
+ if (!(alloc_flags & ANV_BO_ALLOC_IMPLICIT_SYNC) && pdevice->has_exec_async)
+ bo_flags |= EXEC_OBJECT_ASYNC;
+
+ if (pdevice->use_softpin)
+ bo_flags |= EXEC_OBJECT_PINNED;
+
+ return bo_flags;
+}
VkResult
-anv_bo_cache_alloc(struct anv_device *device,
- struct anv_bo_cache *cache,
- uint64_t size, uint64_t bo_flags,
- bool is_external,
- struct anv_bo **bo_out)
+anv_device_alloc_bo(struct anv_device *device,
+ uint64_t size,
+ enum anv_bo_alloc_flags alloc_flags,
+ struct anv_bo **bo_out)
{
+ const uint32_t bo_flags =
+ anv_bo_alloc_flags_to_bo_flags(device, alloc_flags);
assert(bo_flags == (bo_flags & ANV_BO_CACHE_SUPPORTED_FLAGS));
/* The kernel is going to give us whole pages anyway */
return result;
new_bo.flags = bo_flags;
- new_bo.is_external = is_external;
+ new_bo.is_external = (alloc_flags & ANV_BO_ALLOC_EXTERNAL);
- if (!anv_vma_alloc(device, &new_bo)) {
- anv_gem_close(device, new_bo.gem_handle);
- return vk_errorf(device->instance, NULL,
- VK_ERROR_OUT_OF_DEVICE_MEMORY,
- "failed to allocate virtual address for BO");
+ if (alloc_flags & ANV_BO_ALLOC_MAPPED) {
+ new_bo.map = anv_gem_mmap(device, new_bo.gem_handle, 0, size, 0);
+ if (new_bo.map == MAP_FAILED) {
+ anv_gem_close(device, new_bo.gem_handle);
+ return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
+ }
+ }
+
+ if (alloc_flags & ANV_BO_ALLOC_SNOOPED) {
+ assert(alloc_flags & ANV_BO_ALLOC_MAPPED);
+ /* We don't want to change these defaults if it's going to be shared
+ * with another process.
+ */
+ assert(!(alloc_flags & ANV_BO_ALLOC_EXTERNAL));
+
+ /* Regular objects are created I915_CACHING_CACHED on LLC platforms and
+ * I915_CACHING_NONE on non-LLC platforms. For many internal state
+ * objects, we'd rather take the snooping overhead than risk forgetting
+ * a CLFLUSH somewhere. Userptr objects are always created as
+ * I915_CACHING_CACHED, which on non-LLC means snooped so there's no
+ * need to do this there.
+ */
+ if (!device->info.has_llc) {
+ anv_gem_set_caching(device, new_bo.gem_handle,
+ I915_CACHING_CACHED);
+ }
+ }
+
+ if (alloc_flags & ANV_BO_ALLOC_FIXED_ADDRESS) {
+ new_bo.has_fixed_address = true;
+ } else {
+ if (!anv_vma_alloc(device, &new_bo)) {
+ if (new_bo.map)
+ anv_gem_munmap(new_bo.map, size);
+ anv_gem_close(device, new_bo.gem_handle);
+ return vk_errorf(device->instance, NULL,
+ VK_ERROR_OUT_OF_DEVICE_MEMORY,
+ "failed to allocate virtual address for BO");
+ }
}
assert(new_bo.gem_handle);
/* If we just got this gem_handle from anv_bo_init_new then we know no one
* else is touching this BO at the moment so we don't need to lock here.
*/
- struct anv_bo *bo = anv_bo_cache_lookup(cache, new_bo.gem_handle);
+ struct anv_bo *bo = anv_device_lookup_bo(device, new_bo.gem_handle);
*bo = new_bo;
*bo_out = bo;
}
VkResult
-anv_bo_cache_import_host_ptr(struct anv_device *device,
- struct anv_bo_cache *cache,
- void *host_ptr, uint32_t size,
- uint64_t bo_flags, struct anv_bo **bo_out)
+anv_device_import_bo_from_host_ptr(struct anv_device *device,
+ void *host_ptr, uint32_t size,
+ enum anv_bo_alloc_flags alloc_flags,
+ struct anv_bo **bo_out)
{
+ assert(!(alloc_flags & (ANV_BO_ALLOC_MAPPED |
+ ANV_BO_ALLOC_SNOOPED |
+ ANV_BO_ALLOC_FIXED_ADDRESS)));
+
+ struct anv_bo_cache *cache = &device->bo_cache;
+ const uint32_t bo_flags =
+ anv_bo_alloc_flags_to_bo_flags(device, alloc_flags);
assert(bo_flags == (bo_flags & ANV_BO_CACHE_SUPPORTED_FLAGS));
uint32_t gem_handle = anv_gem_userptr(device, host_ptr, size);
pthread_mutex_lock(&cache->mutex);
- struct anv_bo *bo = anv_bo_cache_lookup(cache, gem_handle);
+ struct anv_bo *bo = anv_device_lookup_bo(device, gem_handle);
if (bo->refcount > 0) {
/* VK_EXT_external_memory_host doesn't require handling importing the
* same pointer twice at the same time, but we don't get in the way. If
} else {
struct anv_bo new_bo;
anv_bo_init(&new_bo, gem_handle, size);
+ new_bo.map = host_ptr;
new_bo.flags = bo_flags;
new_bo.is_external = true;
+ new_bo.from_host_ptr = true;
if (!anv_vma_alloc(device, &new_bo)) {
anv_gem_close(device, new_bo.gem_handle);
}
VkResult
-anv_bo_cache_import(struct anv_device *device,
- struct anv_bo_cache *cache,
- int fd, uint64_t bo_flags,
- struct anv_bo **bo_out)
+anv_device_import_bo(struct anv_device *device,
+ int fd,
+ enum anv_bo_alloc_flags alloc_flags,
+ struct anv_bo **bo_out)
{
+ assert(!(alloc_flags & (ANV_BO_ALLOC_MAPPED |
+ ANV_BO_ALLOC_SNOOPED |
+ ANV_BO_ALLOC_FIXED_ADDRESS)));
+
+ struct anv_bo_cache *cache = &device->bo_cache;
+ const uint32_t bo_flags =
+ anv_bo_alloc_flags_to_bo_flags(device, alloc_flags);
assert(bo_flags == (bo_flags & ANV_BO_CACHE_SUPPORTED_FLAGS));
pthread_mutex_lock(&cache->mutex);
return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE);
}
- struct anv_bo *bo = anv_bo_cache_lookup(cache, gem_handle);
+ struct anv_bo *bo = anv_device_lookup_bo(device, gem_handle);
if (bo->refcount > 0) {
/* We have to be careful how we combine flags so that it makes sense.
* Really, though, if we get to this case and it actually matters, the
new_flags |= (bo->flags & bo_flags) & EXEC_OBJECT_ASYNC;
new_flags |= (bo->flags & bo_flags) & EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
new_flags |= (bo->flags | bo_flags) & EXEC_OBJECT_PINNED;
+ new_flags |= (bo->flags | bo_flags) & EXEC_OBJECT_CAPTURE;
/* It's theoretically possible for a BO to get imported such that it's
* both pinned and not pinned. The only way this can happen is if it
}
VkResult
-anv_bo_cache_export(struct anv_device *device,
- struct anv_bo_cache *cache,
- struct anv_bo *bo, int *fd_out)
+anv_device_export_bo(struct anv_device *device,
+ struct anv_bo *bo, int *fd_out)
{
- assert(anv_bo_cache_lookup(cache, bo->gem_handle) == bo);
+ assert(anv_device_lookup_bo(device, bo->gem_handle) == bo);
/* This BO must have been flagged external in order for us to be able
* to export it. This is done based on external options passed into
}
void
-anv_bo_cache_release(struct anv_device *device,
- struct anv_bo_cache *cache,
- struct anv_bo *bo)
+anv_device_release_bo(struct anv_device *device,
+ struct anv_bo *bo)
{
- assert(anv_bo_cache_lookup(cache, bo->gem_handle) == bo);
+ struct anv_bo_cache *cache = &device->bo_cache;
+ assert(anv_device_lookup_bo(device, bo->gem_handle) == bo);
/* Try to decrement the counter but don't go below one. If this succeeds
* then the refcount has been decremented and we are not the last
}
assert(bo->refcount == 0);
- if (bo->map)
+ if (bo->map && !bo->from_host_ptr)
anv_gem_munmap(bo->map, bo->size);
- anv_vma_free(device, bo);
+ if (!bo->has_fixed_address)
+ anv_vma_free(device, bo);
anv_gem_close(device, bo->gem_handle);
mem->ahw = NULL;
mem->host_ptr = NULL;
- uint64_t bo_flags = 0;
+ enum anv_bo_alloc_flags alloc_flags = 0;
assert(mem->type->heapIndex < pdevice->memory.heap_count);
- if (pdevice->memory.heaps[mem->type->heapIndex].supports_48bit_addresses)
- bo_flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
+ if (!pdevice->memory.heaps[mem->type->heapIndex].supports_48bit_addresses)
+ alloc_flags |= ANV_BO_ALLOC_32BIT_ADDRESS;
const struct wsi_memory_allocate_info *wsi_info =
vk_find_struct_const(pAllocateInfo->pNext, WSI_MEMORY_ALLOCATE_INFO_MESA);
* will know we're writing to them and synchronize uses on other rings
* (eg if the display server uses the blitter ring).
*/
- bo_flags |= EXEC_OBJECT_WRITE;
- } else if (pdevice->has_exec_async) {
- bo_flags |= EXEC_OBJECT_ASYNC;
+ alloc_flags |= ANV_BO_ALLOC_IMPLICIT_SYNC |
+ ANV_BO_ALLOC_IMPLICIT_WRITE;
}
- if (pdevice->use_softpin)
- bo_flags |= EXEC_OBJECT_PINNED;
-
const VkExportMemoryAllocateInfo *export_info =
vk_find_struct_const(pAllocateInfo->pNext, EXPORT_MEMORY_ALLOCATE_INFO);
fd_info->handleType ==
VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
- result = anv_bo_cache_import(device, &device->bo_cache, fd_info->fd,
- bo_flags, &mem->bo);
+ result = anv_device_import_bo(device, fd_info->fd, alloc_flags,
+ &mem->bo);
if (result != VK_SUCCESS)
goto fail;
"VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT: "
"%"PRIu64"B > %"PRIu64"B",
aligned_alloc_size, mem->bo->size);
- anv_bo_cache_release(device, &device->bo_cache, mem->bo);
+ anv_device_release_bo(device, mem->bo);
goto fail;
}
assert(host_ptr_info->handleType ==
VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT);
- result = anv_bo_cache_import_host_ptr(
- device, &device->bo_cache, host_ptr_info->pHostPointer,
- pAllocateInfo->allocationSize, bo_flags, &mem->bo);
+ result = anv_device_import_bo_from_host_ptr(device,
+ host_ptr_info->pHostPointer,
+ pAllocateInfo->allocationSize,
+ alloc_flags,
+ &mem->bo);
if (result != VK_SUCCESS)
goto fail;
/* Regular allocate (not importing memory). */
- bool is_external = export_info && export_info->handleTypes;
- result = anv_bo_cache_alloc(device, &device->bo_cache,
- pAllocateInfo->allocationSize,
- bo_flags, is_external,
- &mem->bo);
+ if (export_info && export_info->handleTypes)
+ alloc_flags |= ANV_BO_ALLOC_EXTERNAL;
+
+ result = anv_device_alloc_bo(device, pAllocateInfo->allocationSize,
+ alloc_flags, &mem->bo);
if (result != VK_SUCCESS)
goto fail;
image->planes[0].surface.isl.row_pitch_B,
i915_tiling);
if (ret) {
- anv_bo_cache_release(device, &device->bo_cache, mem->bo);
+ anv_device_release_bo(device, mem->bo);
return vk_errorf(device->instance, NULL,
VK_ERROR_OUT_OF_DEVICE_MEMORY,
"failed to set BO tiling: %m");
assert(pGetFdInfo->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT ||
pGetFdInfo->handleType == VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT);
- return anv_bo_cache_export(dev, &dev->bo_cache, mem->bo, pFd);
+ return anv_device_export_bo(dev, mem->bo, pFd);
}
VkResult anv_GetMemoryFdPropertiesKHR(
p_atomic_add(&pdevice->memory.heaps[mem->type->heapIndex].used,
-mem->bo->size);
- anv_bo_cache_release(device, &device->bo_cache, mem->bo);
+ anv_device_release_bo(device, mem->bo);
#if defined(ANDROID) && ANDROID_API_LEVEL >= 26
if (mem->ahw)
}
if (image->planes[p].bo_is_owned) {
assert(image->planes[p].address.bo != NULL);
- anv_bo_cache_release(device, &device->bo_cache,
- image->planes[p].address.bo);
+ anv_device_release_bo(device, image->planes[p].address.bo);
}
}
image = anv_image_from_handle(image_h);
- uint64_t bo_flags = 0;
- if (device->instance->physicalDevice.supports_48bit_addresses)
- bo_flags |= EXEC_OBJECT_SUPPORTS_48B_ADDRESS;
- if (device->instance->physicalDevice.use_softpin)
- bo_flags |= EXEC_OBJECT_PINNED;
-
- result = anv_bo_cache_import(device, &device->bo_cache,
- pCreateInfo->fd, bo_flags, &mem->bo);
+ result = anv_device_import_bo(device, pCreateInfo->fd,
+ ANV_BO_ALLOC_IMPLICIT_SYNC,
+ &mem->bo);
if (result != VK_SUCCESS)
goto fail_import;
"dma-buf too small for image in "
"vkCreateDmaBufImageINTEL: %"PRIu64"B < %"PRIu64"B",
mem->bo->size, aligned_image_size);
- anv_bo_cache_release(device, &device->bo_cache, mem->bo);
+ anv_device_release_bo(device, mem->bo);
goto fail_import;
}
* is set in the physical device.
*/
bool is_wrapper:1;
+
+ /** See also ANV_BO_ALLOC_FIXED_ADDRESS */
+ bool has_fixed_address:1;
+
+ /** True if this BO wraps a host pointer */
+ bool from_host_ptr:1;
};
static inline void
bo->flags = 0;
bo->is_external = false;
bo->is_wrapper = false;
+ bo->has_fixed_address = false;
+ bo->from_host_ptr = false;
}
static inline struct anv_bo *
VkResult anv_bo_cache_init(struct anv_bo_cache *cache);
void anv_bo_cache_finish(struct anv_bo_cache *cache);
-VkResult anv_bo_cache_alloc(struct anv_device *device,
- struct anv_bo_cache *cache,
- uint64_t size, uint64_t bo_flags,
- bool is_external,
- struct anv_bo **bo);
-VkResult anv_bo_cache_import_host_ptr(struct anv_device *device,
- struct anv_bo_cache *cache,
- void *host_ptr, uint32_t size,
- uint64_t bo_flags, struct anv_bo **bo_out);
-VkResult anv_bo_cache_import(struct anv_device *device,
- struct anv_bo_cache *cache,
- int fd, uint64_t bo_flags,
- struct anv_bo **bo);
-VkResult anv_bo_cache_export(struct anv_device *device,
- struct anv_bo_cache *cache,
- struct anv_bo *bo_in, int *fd_out);
-void anv_bo_cache_release(struct anv_device *device,
- struct anv_bo_cache *cache,
- struct anv_bo *bo);
struct anv_memory_type {
/* Standard bits passed on to the client */
struct drm_i915_gem_execbuffer2 *execbuf,
struct anv_bo **execbuf_bos);
VkResult anv_device_query_status(struct anv_device *device);
+
+
+enum anv_bo_alloc_flags {
+ /** Specifies that the BO must have a 32-bit address
+ *
+ * This is the opposite of EXEC_OBJECT_SUPPORTS_48B_ADDRESS.
+ */
+ ANV_BO_ALLOC_32BIT_ADDRESS = (1 << 0),
+
+ /** Specifies that the BO may be shared externally */
+ ANV_BO_ALLOC_EXTERNAL = (1 << 1),
+
+ /** Specifies that the BO should be mapped */
+ ANV_BO_ALLOC_MAPPED = (1 << 2),
+
+ /** Specifies that the BO should be snooped so we get coherency */
+ ANV_BO_ALLOC_SNOOPED = (1 << 3),
+
+ /** Specifies that the BO should be captured in error states */
+ ANV_BO_ALLOC_CAPTURE = (1 << 4),
+
+ /** Specifies that the BO will have an address assigned by the caller */
+ ANV_BO_ALLOC_FIXED_ADDRESS = (1 << 5),
+
+ /** Enables implicit synchronization on the BO
+ *
+ * This is the opposite of EXEC_OBJECT_ASYNC.
+ */
+ ANV_BO_ALLOC_IMPLICIT_SYNC = (1 << 6),
+
+ /** Enables implicit synchronization on the BO
+ *
+ * This is equivalent to EXEC_OBJECT_WRITE.
+ */
+ ANV_BO_ALLOC_IMPLICIT_WRITE = (1 << 7),
+};
+
+VkResult anv_device_alloc_bo(struct anv_device *device, uint64_t size,
+ enum anv_bo_alloc_flags alloc_flags,
+ struct anv_bo **bo);
+VkResult anv_device_import_bo_from_host_ptr(struct anv_device *device,
+ void *host_ptr, uint32_t size,
+ enum anv_bo_alloc_flags alloc_flags,
+ struct anv_bo **bo_out);
+VkResult anv_device_import_bo(struct anv_device *device, int fd,
+ enum anv_bo_alloc_flags alloc_flags,
+ struct anv_bo **bo);
+VkResult anv_device_export_bo(struct anv_device *device,
+ struct anv_bo *bo, int *fd_out);
+void anv_device_release_bo(struct anv_device *device,
+ struct anv_bo *bo);
+
+static inline struct anv_bo *
+anv_device_lookup_bo(struct anv_device *device, uint32_t gem_handle)
+{
+ return util_sparse_array_get(&device->bo_cache.bo_map, gem_handle);
+}
+
VkResult anv_device_bo_busy(struct anv_device *device, struct anv_bo *bo);
VkResult anv_device_wait(struct anv_device *device, struct anv_bo *bo,
int64_t timeout);
}
} else {
semaphore->permanent.type = ANV_SEMAPHORE_TYPE_BO;
- VkResult result = anv_bo_cache_alloc(device, &device->bo_cache,
- 4096, 0 /* flags */,
- true /* is_external */,
- &semaphore->permanent.bo);
+ VkResult result = anv_device_alloc_bo(device, 4096,
+ ANV_BO_ALLOC_EXTERNAL |
+ ANV_BO_ALLOC_IMPLICIT_SYNC,
+ &semaphore->permanent.bo);
if (result != VK_SUCCESS) {
vk_free2(&device->alloc, pAllocator, semaphore);
return result;
break;
case ANV_SEMAPHORE_TYPE_BO:
- anv_bo_cache_release(device, &device->bo_cache, impl->bo);
+ anv_device_release_bo(device, impl->bo);
break;
case ANV_SEMAPHORE_TYPE_SYNC_FILE:
} else {
new_impl.type = ANV_SEMAPHORE_TYPE_BO;
- VkResult result = anv_bo_cache_import(device, &device->bo_cache,
- fd, 0 /* flags */,
- &new_impl.bo);
+ VkResult result = anv_device_import_bo(device, fd,
+ ANV_BO_ALLOC_EXTERNAL |
+ ANV_BO_ALLOC_IMPLICIT_SYNC,
+ &new_impl.bo);
if (result != VK_SUCCESS)
return result;
if (new_impl.bo->size < 4096) {
- anv_bo_cache_release(device, &device->bo_cache, new_impl.bo);
+ anv_device_release_bo(device, new_impl.bo);
return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE);
}
switch (impl->type) {
case ANV_SEMAPHORE_TYPE_BO:
- result = anv_bo_cache_export(device, &device->bo_cache, impl->bo, pFd);
+ result = anv_device_export_bo(device, impl->bo, pFd);
if (result != VK_SUCCESS)
return result;
break;
projects / mesa.git / commitdiff