X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fintel%2Fvulkan%2Fanv_queue.c;h=2a8ed2eb4ed135e86c2ed6cd96c4fbfecae0c80a;hb=1bd4f8fefc2728963fc37900fe75210ee24e09d1;hp=be7fd31008106db4e7244942d8b2215a578db4eb;hpb=00df1cd9d6234cdfc9fb2bf3615196ff83a3c956;p=mesa.git diff --git a/src/intel/vulkan/anv_queue.c b/src/intel/vulkan/anv_queue.c index be7fd310081..2a8ed2eb4ed 100644 --- a/src/intel/vulkan/anv_queue.c +++ b/src/intel/vulkan/anv_queue.c @@ -30,7 +30,7 @@ #include #include "anv_private.h" -#include "util/vk_util.h" +#include "vk_util.h" #include "genxml/gen7_pack.h" @@ -39,17 +39,19 @@ anv_device_execbuf(struct anv_device *device, struct drm_i915_gem_execbuffer2 *execbuf, struct anv_bo **execbuf_bos) { - int ret = anv_gem_execbuffer(device, execbuf); + int ret = device->no_hw ? 0 : anv_gem_execbuffer(device, execbuf); if (ret != 0) { /* We don't know the real error. */ - device->lost = true; - return vk_errorf(VK_ERROR_DEVICE_LOST, "execbuf2 failed: %m"); + return anv_device_set_lost(device, "execbuf2 failed: %m"); } struct drm_i915_gem_exec_object2 *objects = (void *)(uintptr_t)execbuf->buffers_ptr; - for (uint32_t k = 0; k < execbuf->buffer_count; k++) + for (uint32_t k = 0; k < execbuf->buffer_count; k++) { + if (execbuf_bos[k]->flags & EXEC_OBJECT_PINNED) + assert(execbuf_bos[k]->offset == objects[k].offset); execbuf_bos[k]->offset = objects[k].offset; + } return VK_SUCCESS; } @@ -72,7 +74,7 @@ anv_device_submit_simple_batch(struct anv_device *device, memcpy(bo.map, batch->start, size); if (!device->info.has_llc) - anv_flush_range(bo.map, size); + gen_flush_range(bo.map, size); exec_bos[0] = &bo; exec2_objects[0].handle = bo.gem_handle; @@ -80,7 +82,7 @@ anv_device_submit_simple_batch(struct anv_device *device, exec2_objects[0].relocs_ptr = 0; exec2_objects[0].alignment = 0; exec2_objects[0].offset = bo.offset; - exec2_objects[0].flags = 0; + exec2_objects[0].flags = bo.flags; exec2_objects[0].rsvd1 = 0; exec2_objects[0].rsvd2 = 0; @@ -114,10 +116,9 @@ VkResult anv_QueueSubmit( VkQueue _queue, uint32_t submitCount, const VkSubmitInfo* pSubmits, - VkFence _fence) + VkFence fence) { ANV_FROM_HANDLE(anv_queue, queue, _queue); - ANV_FROM_HANDLE(anv_fence, fence, _fence); struct anv_device *device = queue->device; /* Query for device status prior to submitting. Technically, we don't need @@ -158,13 +159,49 @@ VkResult anv_QueueSubmit( */ pthread_mutex_lock(&device->mutex); + if (fence && submitCount == 0) { + /* If we don't have any command buffers, we need to submit a dummy + * batch to give GEM something to wait on. We could, potentially, + * come up with something more efficient but this shouldn't be a + * common case. + */ + result = anv_cmd_buffer_execbuf(device, NULL, NULL, 0, NULL, 0, fence); + goto out; + } + for (uint32_t i = 0; i < submitCount; i++) { + /* Fence for this submit. NULL for all but the last one */ + VkFence submit_fence = (i == submitCount - 1) ? fence : VK_NULL_HANDLE; + + if (pSubmits[i].commandBufferCount == 0) { + /* If we don't have any command buffers, we need to submit a dummy + * batch to give GEM something to wait on. We could, potentially, + * come up with something more efficient but this shouldn't be a + * common case. + */ + result = anv_cmd_buffer_execbuf(device, NULL, + pSubmits[i].pWaitSemaphores, + pSubmits[i].waitSemaphoreCount, + pSubmits[i].pSignalSemaphores, + pSubmits[i].signalSemaphoreCount, + submit_fence); + if (result != VK_SUCCESS) + goto out; + + continue; + } + for (uint32_t j = 0; j < pSubmits[i].commandBufferCount; j++) { ANV_FROM_HANDLE(anv_cmd_buffer, cmd_buffer, pSubmits[i].pCommandBuffers[j]); assert(cmd_buffer->level == VK_COMMAND_BUFFER_LEVEL_PRIMARY); assert(!anv_batch_has_error(&cmd_buffer->batch)); + /* Fence for this execbuf. NULL for all but the last one */ + VkFence execbuf_fence = + (j == pSubmits[i].commandBufferCount - 1) ? + submit_fence : VK_NULL_HANDLE; + const VkSemaphore *in_semaphores = NULL, *out_semaphores = NULL; uint32_t num_in_semaphores = 0, num_out_semaphores = 0; if (j == 0) { @@ -181,23 +218,14 @@ VkResult anv_QueueSubmit( result = anv_cmd_buffer_execbuf(device, cmd_buffer, in_semaphores, num_in_semaphores, - out_semaphores, num_out_semaphores); + out_semaphores, num_out_semaphores, + execbuf_fence); if (result != VK_SUCCESS) goto out; } } - if (fence) { - struct anv_bo *fence_bo = &fence->bo; - result = anv_device_execbuf(device, &fence->execbuf, &fence_bo); - if (result != VK_SUCCESS) - goto out; - - /* Update the fence and wake up any waiters */ - assert(fence->state == ANV_FENCE_STATE_RESET); - fence->state = ANV_FENCE_STATE_SUBMITTED; - pthread_cond_broadcast(&device->queue_submit); - } + pthread_cond_broadcast(&device->queue_submit); out: if (result != VK_SUCCESS) { @@ -213,17 +241,7 @@ out: * VK_ERROR_DEVICE_LOST to ensure that clients do not attempt to * submit the same job again to this device. */ - result = vk_errorf(VK_ERROR_DEVICE_LOST, "vkQueueSubmit() failed"); - device->lost = true; - - /* If we return VK_ERROR_DEVICE LOST here, we need to ensure that - * vkWaitForFences() and vkGetFenceStatus() return a valid result - * (VK_SUCCESS or VK_ERROR_DEVICE_LOST) in a finite amount of time. - * Setting the fence status to SIGNALED ensures this will happen in - * any case. - */ - if (fence) - fence->state = ANV_FENCE_STATE_SIGNALED; + result = anv_device_set_lost(device, "vkQueueSubmit() failed"); } pthread_mutex_unlock(&device->mutex); @@ -246,62 +264,38 @@ VkResult anv_CreateFence( VkFence* pFence) { ANV_FROM_HANDLE(anv_device, device, _device); - struct anv_bo fence_bo; struct anv_fence *fence; - struct anv_batch batch; - VkResult result; assert(pCreateInfo->sType == VK_STRUCTURE_TYPE_FENCE_CREATE_INFO); - result = anv_bo_pool_alloc(&device->batch_bo_pool, &fence_bo, 4096); - if (result != VK_SUCCESS) - return result; + fence = vk_zalloc2(&device->alloc, pAllocator, sizeof(*fence), 8, + VK_SYSTEM_ALLOCATION_SCOPE_OBJECT); + if (fence == NULL) + return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); - /* Fences are small. Just store the CPU data structure in the BO. */ - fence = fence_bo.map; - fence->bo = fence_bo; - - /* Place the batch after the CPU data but on its own cache line. */ - const uint32_t batch_offset = align_u32(sizeof(*fence), CACHELINE_SIZE); - batch.next = batch.start = fence->bo.map + batch_offset; - batch.end = fence->bo.map + fence->bo.size; - anv_batch_emit(&batch, GEN7_MI_BATCH_BUFFER_END, bbe); - anv_batch_emit(&batch, GEN7_MI_NOOP, noop); - - if (!device->info.has_llc) { - assert(((uintptr_t) batch.start & CACHELINE_MASK) == 0); - assert(batch.next - batch.start <= CACHELINE_SIZE); - __builtin_ia32_mfence(); - __builtin_ia32_clflush(batch.start); - } + if (device->instance->physicalDevice.has_syncobj_wait) { + fence->permanent.type = ANV_FENCE_TYPE_SYNCOBJ; - fence->exec2_objects[0].handle = fence->bo.gem_handle; - fence->exec2_objects[0].relocation_count = 0; - fence->exec2_objects[0].relocs_ptr = 0; - fence->exec2_objects[0].alignment = 0; - fence->exec2_objects[0].offset = fence->bo.offset; - fence->exec2_objects[0].flags = 0; - fence->exec2_objects[0].rsvd1 = 0; - fence->exec2_objects[0].rsvd2 = 0; - - fence->execbuf.buffers_ptr = (uintptr_t) fence->exec2_objects; - fence->execbuf.buffer_count = 1; - fence->execbuf.batch_start_offset = batch.start - fence->bo.map; - fence->execbuf.batch_len = batch.next - batch.start; - fence->execbuf.cliprects_ptr = 0; - fence->execbuf.num_cliprects = 0; - fence->execbuf.DR1 = 0; - fence->execbuf.DR4 = 0; - - fence->execbuf.flags = - I915_EXEC_HANDLE_LUT | I915_EXEC_NO_RELOC | I915_EXEC_RENDER; - fence->execbuf.rsvd1 = device->context_id; - fence->execbuf.rsvd2 = 0; + uint32_t create_flags = 0; + if (pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT) + create_flags |= DRM_SYNCOBJ_CREATE_SIGNALED; - if (pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT) { - fence->state = ANV_FENCE_STATE_SIGNALED; + fence->permanent.syncobj = anv_gem_syncobj_create(device, create_flags); + if (!fence->permanent.syncobj) + return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); } else { - fence->state = ANV_FENCE_STATE_RESET; + fence->permanent.type = ANV_FENCE_TYPE_BO; + + VkResult result = anv_bo_pool_alloc(&device->batch_bo_pool, + &fence->permanent.bo.bo, 4096); + if (result != VK_SUCCESS) + return result; + + if (pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT) { + fence->permanent.bo.state = ANV_BO_FENCE_STATE_SIGNALED; + } else { + fence->permanent.bo.state = ANV_BO_FENCE_STATE_RESET; + } } *pFence = anv_fence_to_handle(fence); @@ -309,6 +303,34 @@ VkResult anv_CreateFence( return VK_SUCCESS; } +static void +anv_fence_impl_cleanup(struct anv_device *device, + struct anv_fence_impl *impl) +{ + switch (impl->type) { + case ANV_FENCE_TYPE_NONE: + /* Dummy. Nothing to do */ + break; + + case ANV_FENCE_TYPE_BO: + anv_bo_pool_free(&device->batch_bo_pool, &impl->bo.bo); + break; + + case ANV_FENCE_TYPE_SYNCOBJ: + anv_gem_syncobj_destroy(device, impl->syncobj); + break; + + case ANV_FENCE_TYPE_WSI: + impl->fence_wsi->destroy(impl->fence_wsi); + break; + + default: + unreachable("Invalid fence type"); + } + + impl->type = ANV_FENCE_TYPE_NONE; +} + void anv_DestroyFence( VkDevice _device, VkFence _fence, @@ -320,8 +342,10 @@ void anv_DestroyFence( if (!fence) return; - assert(fence->bo.map == fence); - anv_bo_pool_free(&device->batch_bo_pool, &fence->bo); + anv_fence_impl_cleanup(device, &fence->temporary); + anv_fence_impl_cleanup(device, &fence->permanent); + + vk_free2(&device->alloc, pAllocator, fence); } VkResult anv_ResetFences( @@ -329,9 +353,35 @@ VkResult anv_ResetFences( uint32_t fenceCount, const VkFence* pFences) { + ANV_FROM_HANDLE(anv_device, device, _device); + for (uint32_t i = 0; i < fenceCount; i++) { ANV_FROM_HANDLE(anv_fence, fence, pFences[i]); - fence->state = ANV_FENCE_STATE_RESET; + + /* From the Vulkan 1.0.53 spec: + * + * "If any member of pFences currently has its payload imported with + * temporary permanence, that fence’s prior permanent payload is + * first restored. The remaining operations described therefore + * operate on the restored payload. + */ + if (fence->temporary.type != ANV_FENCE_TYPE_NONE) + anv_fence_impl_cleanup(device, &fence->temporary); + + struct anv_fence_impl *impl = &fence->permanent; + + switch (impl->type) { + case ANV_FENCE_TYPE_BO: + impl->bo.state = ANV_BO_FENCE_STATE_RESET; + break; + + case ANV_FENCE_TYPE_SYNCOBJ: + anv_gem_syncobj_reset(device, impl->syncobj); + break; + + default: + unreachable("Invalid fence type"); + } } return VK_SUCCESS; @@ -344,57 +394,161 @@ VkResult anv_GetFenceStatus( ANV_FROM_HANDLE(anv_device, device, _device); ANV_FROM_HANDLE(anv_fence, fence, _fence); - if (unlikely(device->lost)) + if (anv_device_is_lost(device)) return VK_ERROR_DEVICE_LOST; - switch (fence->state) { - case ANV_FENCE_STATE_RESET: - /* If it hasn't even been sent off to the GPU yet, it's not ready */ - return VK_NOT_READY; + struct anv_fence_impl *impl = + fence->temporary.type != ANV_FENCE_TYPE_NONE ? + &fence->temporary : &fence->permanent; - case ANV_FENCE_STATE_SIGNALED: - /* It's been signaled, return success */ - return VK_SUCCESS; - - case ANV_FENCE_STATE_SUBMITTED: { - VkResult result = anv_device_bo_busy(device, &fence->bo); - if (result == VK_SUCCESS) { - fence->state = ANV_FENCE_STATE_SIGNALED; + switch (impl->type) { + case ANV_FENCE_TYPE_BO: + /* BO fences don't support import/export */ + assert(fence->temporary.type == ANV_FENCE_TYPE_NONE); + switch (impl->bo.state) { + case ANV_BO_FENCE_STATE_RESET: + /* If it hasn't even been sent off to the GPU yet, it's not ready */ + return VK_NOT_READY; + + case ANV_BO_FENCE_STATE_SIGNALED: + /* It's been signaled, return success */ return VK_SUCCESS; + + case ANV_BO_FENCE_STATE_SUBMITTED: { + VkResult result = anv_device_bo_busy(device, &impl->bo.bo); + if (result == VK_SUCCESS) { + impl->bo.state = ANV_BO_FENCE_STATE_SIGNALED; + return VK_SUCCESS; + } else { + return result; + } + } + default: + unreachable("Invalid fence status"); + } + + case ANV_FENCE_TYPE_SYNCOBJ: { + int ret = anv_gem_syncobj_wait(device, &impl->syncobj, 1, 0, true); + if (ret == -1) { + if (errno == ETIME) { + return VK_NOT_READY; + } else { + /* We don't know the real error. */ + return anv_device_set_lost(device, "drm_syncobj_wait failed: %m"); + } } else { - return result; + return VK_SUCCESS; } } + default: - unreachable("Invalid fence status"); + unreachable("Invalid fence type"); } } #define NSEC_PER_SEC 1000000000 #define INT_TYPE_MAX(type) ((1ull << (sizeof(type) * 8 - 1)) - 1) -VkResult anv_WaitForFences( - VkDevice _device, - uint32_t fenceCount, - const VkFence* pFences, - VkBool32 waitAll, - uint64_t _timeout) +static uint64_t +gettime_ns(void) { - ANV_FROM_HANDLE(anv_device, device, _device); - int ret; + struct timespec current; + clock_gettime(CLOCK_MONOTONIC, ¤t); + return (uint64_t)current.tv_sec * NSEC_PER_SEC + current.tv_nsec; +} - if (unlikely(device->lost)) - return VK_ERROR_DEVICE_LOST; +static uint64_t anv_get_absolute_timeout(uint64_t timeout) +{ + if (timeout == 0) + return 0; + uint64_t current_time = gettime_ns(); + uint64_t max_timeout = (uint64_t) INT64_MAX - current_time; + + timeout = MIN2(max_timeout, timeout); + + return (current_time + timeout); +} + +static int64_t anv_get_relative_timeout(uint64_t abs_timeout) +{ + uint64_t now = gettime_ns(); + + /* We don't want negative timeouts. + * + * DRM_IOCTL_I915_GEM_WAIT uses a signed 64 bit timeout and is + * supposed to block indefinitely timeouts < 0. Unfortunately, + * this was broken for a couple of kernel releases. Since there's + * no way to know whether or not the kernel we're using is one of + * the broken ones, the best we can do is to clamp the timeout to + * INT64_MAX. This limits the maximum timeout from 584 years to + * 292 years - likely not a big deal. + */ + if (abs_timeout < now) + return 0; + + uint64_t rel_timeout = abs_timeout - now; + if (rel_timeout > (uint64_t) INT64_MAX) + rel_timeout = INT64_MAX; + + return rel_timeout; +} + +static VkResult +anv_wait_for_syncobj_fences(struct anv_device *device, + uint32_t fenceCount, + const VkFence *pFences, + bool waitAll, + uint64_t abs_timeout_ns) +{ + uint32_t *syncobjs = vk_zalloc(&device->alloc, + sizeof(*syncobjs) * fenceCount, 8, + VK_SYSTEM_ALLOCATION_SCOPE_COMMAND); + if (!syncobjs) + return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); + + for (uint32_t i = 0; i < fenceCount; i++) { + ANV_FROM_HANDLE(anv_fence, fence, pFences[i]); + assert(fence->permanent.type == ANV_FENCE_TYPE_SYNCOBJ); + + struct anv_fence_impl *impl = + fence->temporary.type != ANV_FENCE_TYPE_NONE ? + &fence->temporary : &fence->permanent; - /* DRM_IOCTL_I915_GEM_WAIT uses a signed 64 bit timeout and is supposed - * to block indefinitely timeouts <= 0. Unfortunately, this was broken - * for a couple of kernel releases. Since there's no way to know - * whether or not the kernel we're using is one of the broken ones, the - * best we can do is to clamp the timeout to INT64_MAX. This limits the - * maximum timeout from 584 years to 292 years - likely not a big deal. + assert(impl->type == ANV_FENCE_TYPE_SYNCOBJ); + syncobjs[i] = impl->syncobj; + } + + /* The gem_syncobj_wait ioctl may return early due to an inherent + * limitation in the way it computes timeouts. Loop until we've actually + * passed the timeout. */ - int64_t timeout = MIN2(_timeout, INT64_MAX); + int ret; + do { + ret = anv_gem_syncobj_wait(device, syncobjs, fenceCount, + abs_timeout_ns, waitAll); + } while (ret == -1 && errno == ETIME && gettime_ns() < abs_timeout_ns); + vk_free(&device->alloc, syncobjs); + + if (ret == -1) { + if (errno == ETIME) { + return VK_TIMEOUT; + } else { + /* We don't know the real error. */ + return anv_device_set_lost(device, "drm_syncobj_wait failed: %m"); + } + } else { + return VK_SUCCESS; + } +} + +static VkResult +anv_wait_for_bo_fences(struct anv_device *device, + uint32_t fenceCount, + const VkFence *pFences, + bool waitAll, + uint64_t abs_timeout_ns) +{ VkResult result = VK_SUCCESS; uint32_t pending_fences = fenceCount; while (pending_fences) { @@ -402,8 +556,17 @@ VkResult anv_WaitForFences( bool signaled_fences = false; for (uint32_t i = 0; i < fenceCount; i++) { ANV_FROM_HANDLE(anv_fence, fence, pFences[i]); - switch (fence->state) { - case ANV_FENCE_STATE_RESET: + + /* This function assumes that all fences are BO fences and that they + * have no temporary state. Since BO fences will never be exported, + * this should be a safe assumption. + */ + assert(fence->permanent.type == ANV_FENCE_TYPE_BO); + assert(fence->temporary.type == ANV_FENCE_TYPE_NONE); + struct anv_fence_impl *impl = &fence->permanent; + + switch (impl->bo.state) { + case ANV_BO_FENCE_STATE_RESET: /* This fence hasn't been submitted yet, we'll catch it the next * time around. Yes, this may mean we dead-loop but, short of * lots of locking and a condition variable, there's not much that @@ -412,7 +575,7 @@ VkResult anv_WaitForFences( pending_fences++; continue; - case ANV_FENCE_STATE_SIGNALED: + case ANV_BO_FENCE_STATE_SIGNALED: /* This fence is not pending. If waitAll isn't set, we can return * early. Otherwise, we have to keep going. */ @@ -422,14 +585,15 @@ VkResult anv_WaitForFences( } continue; - case ANV_FENCE_STATE_SUBMITTED: + case ANV_BO_FENCE_STATE_SUBMITTED: /* These are the fences we really care about. Go ahead and wait * on it until we hit a timeout. */ - result = anv_device_wait(device, &fence->bo, timeout); + result = anv_device_wait(device, &impl->bo.bo, + anv_get_relative_timeout(abs_timeout_ns)); switch (result) { case VK_SUCCESS: - fence->state = ANV_FENCE_STATE_SIGNALED; + impl->bo.state = ANV_BO_FENCE_STATE_SIGNALED; signaled_fences = true; if (!waitAll) goto done; @@ -459,44 +623,26 @@ VkResult anv_WaitForFences( uint32_t now_pending_fences = 0; for (uint32_t i = 0; i < fenceCount; i++) { ANV_FROM_HANDLE(anv_fence, fence, pFences[i]); - if (fence->state == ANV_FENCE_STATE_RESET) + if (fence->permanent.bo.state == ANV_BO_FENCE_STATE_RESET) now_pending_fences++; } assert(now_pending_fences <= pending_fences); if (now_pending_fences == pending_fences) { - struct timespec before; - clock_gettime(CLOCK_MONOTONIC, &before); - - uint32_t abs_nsec = before.tv_nsec + timeout % NSEC_PER_SEC; - uint64_t abs_sec = before.tv_sec + (abs_nsec / NSEC_PER_SEC) + - (timeout / NSEC_PER_SEC); - abs_nsec %= NSEC_PER_SEC; - - /* Avoid roll-over in tv_sec on 32-bit systems if the user - * provided timeout is UINT64_MAX - */ - struct timespec abstime; - abstime.tv_nsec = abs_nsec; - abstime.tv_sec = MIN2(abs_sec, INT_TYPE_MAX(abstime.tv_sec)); + struct timespec abstime = { + .tv_sec = abs_timeout_ns / NSEC_PER_SEC, + .tv_nsec = abs_timeout_ns % NSEC_PER_SEC, + }; + MAYBE_UNUSED int ret; ret = pthread_cond_timedwait(&device->queue_submit, &device->mutex, &abstime); assert(ret != EINVAL); - - struct timespec after; - clock_gettime(CLOCK_MONOTONIC, &after); - uint64_t time_elapsed = - ((uint64_t)after.tv_sec * NSEC_PER_SEC + after.tv_nsec) - - ((uint64_t)before.tv_sec * NSEC_PER_SEC + before.tv_nsec); - - if (time_elapsed >= timeout) { + if (gettime_ns() >= abs_timeout_ns) { pthread_mutex_unlock(&device->mutex); result = VK_TIMEOUT; goto done; } - - timeout -= time_elapsed; } pthread_mutex_unlock(&device->mutex); @@ -504,12 +650,265 @@ VkResult anv_WaitForFences( } done: - if (unlikely(device->lost)) + if (anv_device_is_lost(device)) return VK_ERROR_DEVICE_LOST; return result; } +static VkResult +anv_wait_for_wsi_fence(struct anv_device *device, + const VkFence _fence, + uint64_t abs_timeout) +{ + ANV_FROM_HANDLE(anv_fence, fence, _fence); + struct anv_fence_impl *impl = &fence->permanent; + + return impl->fence_wsi->wait(impl->fence_wsi, abs_timeout); +} + +static VkResult +anv_wait_for_fences(struct anv_device *device, + uint32_t fenceCount, + const VkFence *pFences, + bool waitAll, + uint64_t abs_timeout) +{ + VkResult result = VK_SUCCESS; + + if (fenceCount <= 1 || waitAll) { + for (uint32_t i = 0; i < fenceCount; i++) { + ANV_FROM_HANDLE(anv_fence, fence, pFences[i]); + switch (fence->permanent.type) { + case ANV_FENCE_TYPE_BO: + result = anv_wait_for_bo_fences(device, 1, &pFences[i], + true, abs_timeout); + break; + case ANV_FENCE_TYPE_SYNCOBJ: + result = anv_wait_for_syncobj_fences(device, 1, &pFences[i], + true, abs_timeout); + break; + case ANV_FENCE_TYPE_WSI: + result = anv_wait_for_wsi_fence(device, pFences[i], abs_timeout); + break; + case ANV_FENCE_TYPE_NONE: + result = VK_SUCCESS; + break; + } + if (result != VK_SUCCESS) + return result; + } + } else { + do { + for (uint32_t i = 0; i < fenceCount; i++) { + if (anv_wait_for_fences(device, 1, &pFences[i], true, 0) == VK_SUCCESS) + return VK_SUCCESS; + } + } while (gettime_ns() < abs_timeout); + result = VK_TIMEOUT; + } + return result; +} + +static bool anv_all_fences_syncobj(uint32_t fenceCount, const VkFence *pFences) +{ + for (uint32_t i = 0; i < fenceCount; ++i) { + ANV_FROM_HANDLE(anv_fence, fence, pFences[i]); + if (fence->permanent.type != ANV_FENCE_TYPE_SYNCOBJ) + return false; + } + return true; +} + +static bool anv_all_fences_bo(uint32_t fenceCount, const VkFence *pFences) +{ + for (uint32_t i = 0; i < fenceCount; ++i) { + ANV_FROM_HANDLE(anv_fence, fence, pFences[i]); + if (fence->permanent.type != ANV_FENCE_TYPE_BO) + return false; + } + return true; +} + +VkResult anv_WaitForFences( + VkDevice _device, + uint32_t fenceCount, + const VkFence* pFences, + VkBool32 waitAll, + uint64_t timeout) +{ + ANV_FROM_HANDLE(anv_device, device, _device); + + if (anv_device_is_lost(device)) + return VK_ERROR_DEVICE_LOST; + + uint64_t abs_timeout = anv_get_absolute_timeout(timeout); + if (anv_all_fences_syncobj(fenceCount, pFences)) { + return anv_wait_for_syncobj_fences(device, fenceCount, pFences, + waitAll, abs_timeout); + } else if (anv_all_fences_bo(fenceCount, pFences)) { + return anv_wait_for_bo_fences(device, fenceCount, pFences, + waitAll, abs_timeout); + } else { + return anv_wait_for_fences(device, fenceCount, pFences, + waitAll, abs_timeout); + } +} + +void anv_GetPhysicalDeviceExternalFenceProperties( + VkPhysicalDevice physicalDevice, + const VkPhysicalDeviceExternalFenceInfoKHR* pExternalFenceInfo, + VkExternalFencePropertiesKHR* pExternalFenceProperties) +{ + ANV_FROM_HANDLE(anv_physical_device, device, physicalDevice); + + switch (pExternalFenceInfo->handleType) { + case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT: + case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT: + if (device->has_syncobj_wait) { + pExternalFenceProperties->exportFromImportedHandleTypes = + VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT | + VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT; + pExternalFenceProperties->compatibleHandleTypes = + VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT | + VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT; + pExternalFenceProperties->externalFenceFeatures = + VK_EXTERNAL_FENCE_FEATURE_EXPORTABLE_BIT | + VK_EXTERNAL_FENCE_FEATURE_IMPORTABLE_BIT; + return; + } + break; + + default: + break; + } + + pExternalFenceProperties->exportFromImportedHandleTypes = 0; + pExternalFenceProperties->compatibleHandleTypes = 0; + pExternalFenceProperties->externalFenceFeatures = 0; +} + +VkResult anv_ImportFenceFdKHR( + VkDevice _device, + const VkImportFenceFdInfoKHR* pImportFenceFdInfo) +{ + ANV_FROM_HANDLE(anv_device, device, _device); + ANV_FROM_HANDLE(anv_fence, fence, pImportFenceFdInfo->fence); + int fd = pImportFenceFdInfo->fd; + + assert(pImportFenceFdInfo->sType == + VK_STRUCTURE_TYPE_IMPORT_FENCE_FD_INFO_KHR); + + struct anv_fence_impl new_impl = { + .type = ANV_FENCE_TYPE_NONE, + }; + + switch (pImportFenceFdInfo->handleType) { + case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT: + new_impl.type = ANV_FENCE_TYPE_SYNCOBJ; + + new_impl.syncobj = anv_gem_syncobj_fd_to_handle(device, fd); + if (!new_impl.syncobj) + return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); + + break; + + case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT: + /* Sync files are a bit tricky. Because we want to continue using the + * syncobj implementation of WaitForFences, we don't use the sync file + * directly but instead import it into a syncobj. + */ + new_impl.type = ANV_FENCE_TYPE_SYNCOBJ; + + new_impl.syncobj = anv_gem_syncobj_create(device, 0); + if (!new_impl.syncobj) + return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); + + if (anv_gem_syncobj_import_sync_file(device, new_impl.syncobj, fd)) { + anv_gem_syncobj_destroy(device, new_impl.syncobj); + return vk_errorf(device->instance, NULL, + VK_ERROR_INVALID_EXTERNAL_HANDLE, + "syncobj sync file import failed: %m"); + } + break; + + default: + return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); + } + + /* From the Vulkan 1.0.53 spec: + * + * "Importing a fence payload from a file descriptor transfers + * ownership of the file descriptor from the application to the + * Vulkan implementation. The application must not perform any + * operations on the file descriptor after a successful import." + * + * If the import fails, we leave the file descriptor open. + */ + close(fd); + + if (pImportFenceFdInfo->flags & VK_FENCE_IMPORT_TEMPORARY_BIT) { + anv_fence_impl_cleanup(device, &fence->temporary); + fence->temporary = new_impl; + } else { + anv_fence_impl_cleanup(device, &fence->permanent); + fence->permanent = new_impl; + } + + return VK_SUCCESS; +} + +VkResult anv_GetFenceFdKHR( + VkDevice _device, + const VkFenceGetFdInfoKHR* pGetFdInfo, + int* pFd) +{ + ANV_FROM_HANDLE(anv_device, device, _device); + ANV_FROM_HANDLE(anv_fence, fence, pGetFdInfo->fence); + + assert(pGetFdInfo->sType == VK_STRUCTURE_TYPE_FENCE_GET_FD_INFO_KHR); + + struct anv_fence_impl *impl = + fence->temporary.type != ANV_FENCE_TYPE_NONE ? + &fence->temporary : &fence->permanent; + + assert(impl->type == ANV_FENCE_TYPE_SYNCOBJ); + switch (pGetFdInfo->handleType) { + case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT: { + int fd = anv_gem_syncobj_handle_to_fd(device, impl->syncobj); + if (fd < 0) + return vk_error(VK_ERROR_TOO_MANY_OBJECTS); + + *pFd = fd; + break; + } + + case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT: { + int fd = anv_gem_syncobj_export_sync_file(device, impl->syncobj); + if (fd < 0) + return vk_error(VK_ERROR_TOO_MANY_OBJECTS); + + *pFd = fd; + break; + } + + default: + unreachable("Invalid fence export handle type"); + } + + /* From the Vulkan 1.0.53 spec: + * + * "Export operations have the same transference as the specified handle + * type’s import operations. [...] If the fence was using a + * temporarily imported payload, the fence’s prior permanent payload + * will be restored. + */ + if (impl == &fence->temporary) + anv_fence_impl_cleanup(device, impl); + + return VK_SUCCESS; +} + // Queue semaphore functions VkResult anv_CreateSemaphore( @@ -528,9 +927,9 @@ VkResult anv_CreateSemaphore( if (semaphore == NULL) return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); - const VkExportSemaphoreCreateInfoKHX *export = - vk_find_struct_const(pCreateInfo->pNext, EXPORT_SEMAPHORE_CREATE_INFO_KHX); - VkExternalSemaphoreHandleTypeFlagsKHX handleTypes = + const VkExportSemaphoreCreateInfoKHR *export = + vk_find_struct_const(pCreateInfo->pNext, EXPORT_SEMAPHORE_CREATE_INFO); + VkExternalSemaphoreHandleTypeFlagsKHR handleTypes = export ? export->handleTypes : 0; if (handleTypes == 0) { @@ -539,25 +938,39 @@ VkResult anv_CreateSemaphore( * queue, a dummy no-op semaphore is a perfectly valid implementation. */ semaphore->permanent.type = ANV_SEMAPHORE_TYPE_DUMMY; - } else if (handleTypes & VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT_KHX) { - assert(handleTypes == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT_KHX); - - semaphore->permanent.type = ANV_SEMAPHORE_TYPE_BO; - VkResult result = anv_bo_cache_alloc(device, &device->bo_cache, - 4096, &semaphore->permanent.bo); - if (result != VK_SUCCESS) { - vk_free2(&device->alloc, pAllocator, semaphore); - return result; + } else if (handleTypes & VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT) { + assert(handleTypes == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT); + if (device->instance->physicalDevice.has_syncobj) { + semaphore->permanent.type = ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ; + semaphore->permanent.syncobj = anv_gem_syncobj_create(device, 0); + if (!semaphore->permanent.syncobj) { + vk_free2(&device->alloc, pAllocator, semaphore); + return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY); + } + } else { + semaphore->permanent.type = ANV_SEMAPHORE_TYPE_BO; + VkResult result = anv_bo_cache_alloc(device, &device->bo_cache, + 4096, ANV_BO_EXTERNAL, + &semaphore->permanent.bo); + if (result != VK_SUCCESS) { + vk_free2(&device->alloc, pAllocator, semaphore); + return result; + } + + /* If we're going to use this as a fence, we need to *not* have the + * EXEC_OBJECT_ASYNC bit set. + */ + assert(!(semaphore->permanent.bo->flags & EXEC_OBJECT_ASYNC)); } + } else if (handleTypes & VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT) { + assert(handleTypes == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT); - /* If we're going to use this as a fence, we need to *not* have the - * EXEC_OBJECT_ASYNC bit set. - */ - assert(!(semaphore->permanent.bo->flags & EXEC_OBJECT_ASYNC)); + semaphore->permanent.type = ANV_SEMAPHORE_TYPE_SYNC_FILE; + semaphore->permanent.fd = -1; } else { assert(!"Unknown handle type"); vk_free2(&device->alloc, pAllocator, semaphore); - return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE_KHX); + return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); } semaphore->temporary.type = ANV_SEMAPHORE_TYPE_NONE; @@ -575,14 +988,35 @@ anv_semaphore_impl_cleanup(struct anv_device *device, case ANV_SEMAPHORE_TYPE_NONE: case ANV_SEMAPHORE_TYPE_DUMMY: /* Dummy. Nothing to do */ - return; + break; case ANV_SEMAPHORE_TYPE_BO: anv_bo_cache_release(device, &device->bo_cache, impl->bo); - return; + break; + + case ANV_SEMAPHORE_TYPE_SYNC_FILE: + close(impl->fd); + break; + + case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ: + anv_gem_syncobj_destroy(device, impl->syncobj); + break; + + default: + unreachable("Invalid semaphore type"); } - unreachable("Invalid semaphore type"); + impl->type = ANV_SEMAPHORE_TYPE_NONE; +} + +void +anv_semaphore_reset_temporary(struct anv_device *device, + struct anv_semaphore *semaphore) +{ + if (semaphore->temporary.type == ANV_SEMAPHORE_TYPE_NONE) + return; + + anv_semaphore_impl_cleanup(device, &semaphore->temporary); } void anv_DestroySemaphore( @@ -602,80 +1036,190 @@ void anv_DestroySemaphore( vk_free2(&device->alloc, pAllocator, semaphore); } -void anv_GetPhysicalDeviceExternalSemaphorePropertiesKHX( +void anv_GetPhysicalDeviceExternalSemaphoreProperties( VkPhysicalDevice physicalDevice, - const VkPhysicalDeviceExternalSemaphoreInfoKHX* pExternalSemaphoreInfo, - VkExternalSemaphorePropertiesKHX* pExternalSemaphoreProperties) + const VkPhysicalDeviceExternalSemaphoreInfoKHR* pExternalSemaphoreInfo, + VkExternalSemaphorePropertiesKHR* pExternalSemaphoreProperties) { + ANV_FROM_HANDLE(anv_physical_device, device, physicalDevice); + switch (pExternalSemaphoreInfo->handleType) { - case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT_KHX: + case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT: pExternalSemaphoreProperties->exportFromImportedHandleTypes = - VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT_KHX; + VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; pExternalSemaphoreProperties->compatibleHandleTypes = - VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT_KHX; + VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT; pExternalSemaphoreProperties->externalSemaphoreFeatures = - VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT_KHX | - VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT_KHX; + VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT | + VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT; + return; + + case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT: + if (device->has_exec_fence) { + pExternalSemaphoreProperties->exportFromImportedHandleTypes = 0; + pExternalSemaphoreProperties->compatibleHandleTypes = + VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT; + pExternalSemaphoreProperties->externalSemaphoreFeatures = + VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT | + VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT; + return; + } break; default: - pExternalSemaphoreProperties->exportFromImportedHandleTypes = 0; - pExternalSemaphoreProperties->compatibleHandleTypes = 0; - pExternalSemaphoreProperties->externalSemaphoreFeatures = 0; + break; } + + pExternalSemaphoreProperties->exportFromImportedHandleTypes = 0; + pExternalSemaphoreProperties->compatibleHandleTypes = 0; + pExternalSemaphoreProperties->externalSemaphoreFeatures = 0; } -VkResult anv_ImportSemaphoreFdKHX( +VkResult anv_ImportSemaphoreFdKHR( VkDevice _device, - const VkImportSemaphoreFdInfoKHX* pImportSemaphoreFdInfo) + const VkImportSemaphoreFdInfoKHR* pImportSemaphoreFdInfo) { ANV_FROM_HANDLE(anv_device, device, _device); ANV_FROM_HANDLE(anv_semaphore, semaphore, pImportSemaphoreFdInfo->semaphore); + int fd = pImportSemaphoreFdInfo->fd; + + struct anv_semaphore_impl new_impl = { + .type = ANV_SEMAPHORE_TYPE_NONE, + }; switch (pImportSemaphoreFdInfo->handleType) { - case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT_KHX: { - struct anv_bo *bo; - VkResult result = anv_bo_cache_import(device, &device->bo_cache, - pImportSemaphoreFdInfo->fd, 4096, - &bo); - if (result != VK_SUCCESS) - return result; + case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT: + if (device->instance->physicalDevice.has_syncobj) { + new_impl.type = ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ; - /* If we're going to use this as a fence, we need to *not* have the - * EXEC_OBJECT_ASYNC bit set. - */ - assert(!(bo->flags & EXEC_OBJECT_ASYNC)); + new_impl.syncobj = anv_gem_syncobj_fd_to_handle(device, fd); + if (!new_impl.syncobj) + return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); + } else { + new_impl.type = ANV_SEMAPHORE_TYPE_BO; - anv_semaphore_impl_cleanup(device, &semaphore->permanent); + VkResult result = anv_bo_cache_import(device, &device->bo_cache, + fd, ANV_BO_EXTERNAL, + &new_impl.bo); + if (result != VK_SUCCESS) + return result; - semaphore->permanent.type = ANV_SEMAPHORE_TYPE_BO; - semaphore->permanent.bo = bo; + if (new_impl.bo->size < 4096) { + anv_bo_cache_release(device, &device->bo_cache, new_impl.bo); + return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE_KHR); + } - return VK_SUCCESS; - } + /* If we're going to use this as a fence, we need to *not* have the + * EXEC_OBJECT_ASYNC bit set. + */ + assert(!(new_impl.bo->flags & EXEC_OBJECT_ASYNC)); + } + + /* From the Vulkan spec: + * + * "Importing semaphore state from a file descriptor transfers + * ownership of the file descriptor from the application to the + * Vulkan implementation. The application must not perform any + * operations on the file descriptor after a successful import." + * + * If the import fails, we leave the file descriptor open. + */ + close(fd); + break; + + case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT: + new_impl = (struct anv_semaphore_impl) { + .type = ANV_SEMAPHORE_TYPE_SYNC_FILE, + .fd = fd, + }; + break; default: - return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE_KHX); + return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); } + + if (pImportSemaphoreFdInfo->flags & VK_SEMAPHORE_IMPORT_TEMPORARY_BIT) { + anv_semaphore_impl_cleanup(device, &semaphore->temporary); + semaphore->temporary = new_impl; + } else { + anv_semaphore_impl_cleanup(device, &semaphore->permanent); + semaphore->permanent = new_impl; + } + + return VK_SUCCESS; } -VkResult anv_GetSemaphoreFdKHX( +VkResult anv_GetSemaphoreFdKHR( VkDevice _device, - VkSemaphore _semaphore, - VkExternalSemaphoreHandleTypeFlagBitsKHX handleType, + const VkSemaphoreGetFdInfoKHR* pGetFdInfo, int* pFd) { ANV_FROM_HANDLE(anv_device, device, _device); - ANV_FROM_HANDLE(anv_semaphore, semaphore, _semaphore); + ANV_FROM_HANDLE(anv_semaphore, semaphore, pGetFdInfo->semaphore); + VkResult result; + int fd; + + assert(pGetFdInfo->sType == VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR); + + struct anv_semaphore_impl *impl = + semaphore->temporary.type != ANV_SEMAPHORE_TYPE_NONE ? + &semaphore->temporary : &semaphore->permanent; - switch (semaphore->permanent.type) { + switch (impl->type) { case ANV_SEMAPHORE_TYPE_BO: - return anv_bo_cache_export(device, &device->bo_cache, - semaphore->permanent.bo, pFd); + result = anv_bo_cache_export(device, &device->bo_cache, impl->bo, pFd); + if (result != VK_SUCCESS) + return result; + break; + + case ANV_SEMAPHORE_TYPE_SYNC_FILE: + /* There are two reasons why this could happen: + * + * 1) The user is trying to export without submitting something that + * signals the semaphore. If this is the case, it's their bug so + * what we return here doesn't matter. + * + * 2) The kernel didn't give us a file descriptor. The most likely + * reason for this is running out of file descriptors. + */ + if (impl->fd < 0) + return vk_error(VK_ERROR_TOO_MANY_OBJECTS); + + *pFd = impl->fd; + + /* From the Vulkan 1.0.53 spec: + * + * "...exporting a semaphore payload to a handle with copy + * transference has the same side effects on the source + * semaphore’s payload as executing a semaphore wait operation." + * + * In other words, it may still be a SYNC_FD semaphore, but it's now + * considered to have been waited on and no longer has a sync file + * attached. + */ + impl->fd = -1; + return VK_SUCCESS; + + case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ: + fd = anv_gem_syncobj_handle_to_fd(device, impl->syncobj); + if (fd < 0) + return vk_error(VK_ERROR_TOO_MANY_OBJECTS); + *pFd = fd; + break; default: - return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE_KHX); + return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE); } + /* From the Vulkan 1.0.53 spec: + * + * "Export operations have the same transference as the specified handle + * type’s import operations. [...] If the semaphore was using a + * temporarily imported payload, the semaphore’s prior permanent payload + * will be restored. + */ + if (impl == &semaphore->temporary) + anv_semaphore_impl_cleanup(device, impl); + return VK_SUCCESS; }