2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 * This file implements VkQueue, VkFence, and VkSemaphore
31 #include "anv_private.h"
34 #include "genxml/gen7_pack.h"
37 anv_device_execbuf(struct anv_device
*device
,
38 struct drm_i915_gem_execbuffer2
*execbuf
,
39 struct anv_bo
**execbuf_bos
)
41 int ret
= device
->no_hw
? 0 : anv_gem_execbuffer(device
, execbuf
);
43 /* We don't know the real error. */
44 return anv_device_set_lost(device
, "execbuf2 failed: %m");
47 struct drm_i915_gem_exec_object2
*objects
=
48 (void *)(uintptr_t)execbuf
->buffers_ptr
;
49 for (uint32_t k
= 0; k
< execbuf
->buffer_count
; k
++) {
50 if (execbuf_bos
[k
]->flags
& EXEC_OBJECT_PINNED
)
51 assert(execbuf_bos
[k
]->offset
== objects
[k
].offset
);
52 execbuf_bos
[k
]->offset
= objects
[k
].offset
;
59 anv_device_submit_simple_batch(struct anv_device
*device
,
60 struct anv_batch
*batch
)
62 struct drm_i915_gem_execbuffer2 execbuf
;
63 struct drm_i915_gem_exec_object2 exec2_objects
[1];
65 VkResult result
= VK_SUCCESS
;
68 /* Kernel driver requires 8 byte aligned batch length */
69 size
= align_u32(batch
->next
- batch
->start
, 8);
70 result
= anv_bo_pool_alloc(&device
->batch_bo_pool
, size
, &bo
);
71 if (result
!= VK_SUCCESS
)
74 memcpy(bo
->map
, batch
->start
, size
);
75 if (!device
->info
.has_llc
)
76 gen_flush_range(bo
->map
, size
);
78 exec2_objects
[0].handle
= bo
->gem_handle
;
79 exec2_objects
[0].relocation_count
= 0;
80 exec2_objects
[0].relocs_ptr
= 0;
81 exec2_objects
[0].alignment
= 0;
82 exec2_objects
[0].offset
= bo
->offset
;
83 exec2_objects
[0].flags
= bo
->flags
;
84 exec2_objects
[0].rsvd1
= 0;
85 exec2_objects
[0].rsvd2
= 0;
87 execbuf
.buffers_ptr
= (uintptr_t) exec2_objects
;
88 execbuf
.buffer_count
= 1;
89 execbuf
.batch_start_offset
= 0;
90 execbuf
.batch_len
= size
;
91 execbuf
.cliprects_ptr
= 0;
92 execbuf
.num_cliprects
= 0;
97 I915_EXEC_HANDLE_LUT
| I915_EXEC_NO_RELOC
| I915_EXEC_RENDER
;
98 execbuf
.rsvd1
= device
->context_id
;
101 if (unlikely(INTEL_DEBUG
& DEBUG_BATCH
)) {
102 gen_print_batch(&device
->decoder_ctx
, bo
->map
,
103 bo
->size
, bo
->offset
, false);
106 result
= anv_device_execbuf(device
, &execbuf
, &bo
);
107 if (result
!= VK_SUCCESS
)
110 result
= anv_device_wait(device
, bo
, INT64_MAX
);
113 anv_bo_pool_free(&device
->batch_bo_pool
, bo
);
118 VkResult
anv_QueueSubmit(
120 uint32_t submitCount
,
121 const VkSubmitInfo
* pSubmits
,
124 ANV_FROM_HANDLE(anv_queue
, queue
, _queue
);
125 struct anv_device
*device
= queue
->device
;
127 /* Query for device status prior to submitting. Technically, we don't need
128 * to do this. However, if we have a client that's submitting piles of
129 * garbage, we would rather break as early as possible to keep the GPU
130 * hanging contained. If we don't check here, we'll either be waiting for
131 * the kernel to kick us or we'll have to wait until the client waits on a
132 * fence before we actually know whether or not we've hung.
134 VkResult result
= anv_device_query_status(device
);
135 if (result
!= VK_SUCCESS
)
138 /* We lock around QueueSubmit for three main reasons:
140 * 1) When a block pool is resized, we create a new gem handle with a
141 * different size and, in the case of surface states, possibly a
142 * different center offset but we re-use the same anv_bo struct when
143 * we do so. If this happens in the middle of setting up an execbuf,
144 * we could end up with our list of BOs out of sync with our list of
147 * 2) The algorithm we use for building the list of unique buffers isn't
148 * thread-safe. While the client is supposed to syncronize around
149 * QueueSubmit, this would be extremely difficult to debug if it ever
150 * came up in the wild due to a broken app. It's better to play it
151 * safe and just lock around QueueSubmit.
153 * 3) The anv_cmd_buffer_execbuf function may perform relocations in
154 * userspace. Due to the fact that the surface state buffer is shared
155 * between batches, we can't afford to have that happen from multiple
156 * threads at the same time. Even though the user is supposed to
157 * ensure this doesn't happen, we play it safe as in (2) above.
159 * Since the only other things that ever take the device lock such as block
160 * pool resize only rarely happen, this will almost never be contended so
161 * taking a lock isn't really an expensive operation in this case.
163 pthread_mutex_lock(&device
->mutex
);
165 if (fence
&& submitCount
== 0) {
166 /* If we don't have any command buffers, we need to submit a dummy
167 * batch to give GEM something to wait on. We could, potentially,
168 * come up with something more efficient but this shouldn't be a
171 result
= anv_cmd_buffer_execbuf(device
, NULL
, NULL
, 0, NULL
, 0, fence
);
175 for (uint32_t i
= 0; i
< submitCount
; i
++) {
176 /* Fence for this submit. NULL for all but the last one */
177 VkFence submit_fence
= (i
== submitCount
- 1) ? fence
: VK_NULL_HANDLE
;
179 if (pSubmits
[i
].commandBufferCount
== 0) {
180 /* If we don't have any command buffers, we need to submit a dummy
181 * batch to give GEM something to wait on. We could, potentially,
182 * come up with something more efficient but this shouldn't be a
185 result
= anv_cmd_buffer_execbuf(device
, NULL
,
186 pSubmits
[i
].pWaitSemaphores
,
187 pSubmits
[i
].waitSemaphoreCount
,
188 pSubmits
[i
].pSignalSemaphores
,
189 pSubmits
[i
].signalSemaphoreCount
,
191 if (result
!= VK_SUCCESS
)
197 for (uint32_t j
= 0; j
< pSubmits
[i
].commandBufferCount
; j
++) {
198 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
,
199 pSubmits
[i
].pCommandBuffers
[j
]);
200 assert(cmd_buffer
->level
== VK_COMMAND_BUFFER_LEVEL_PRIMARY
);
201 assert(!anv_batch_has_error(&cmd_buffer
->batch
));
203 /* Fence for this execbuf. NULL for all but the last one */
204 VkFence execbuf_fence
=
205 (j
== pSubmits
[i
].commandBufferCount
- 1) ?
206 submit_fence
: VK_NULL_HANDLE
;
208 const VkSemaphore
*in_semaphores
= NULL
, *out_semaphores
= NULL
;
209 uint32_t num_in_semaphores
= 0, num_out_semaphores
= 0;
211 /* Only the first batch gets the in semaphores */
212 in_semaphores
= pSubmits
[i
].pWaitSemaphores
;
213 num_in_semaphores
= pSubmits
[i
].waitSemaphoreCount
;
216 if (j
== pSubmits
[i
].commandBufferCount
- 1) {
217 /* Only the last batch gets the out semaphores */
218 out_semaphores
= pSubmits
[i
].pSignalSemaphores
;
219 num_out_semaphores
= pSubmits
[i
].signalSemaphoreCount
;
222 result
= anv_cmd_buffer_execbuf(device
, cmd_buffer
,
223 in_semaphores
, num_in_semaphores
,
224 out_semaphores
, num_out_semaphores
,
226 if (result
!= VK_SUCCESS
)
231 pthread_cond_broadcast(&device
->queue_submit
);
234 if (result
!= VK_SUCCESS
) {
235 /* In the case that something has gone wrong we may end up with an
236 * inconsistent state from which it may not be trivial to recover.
237 * For example, we might have computed address relocations and
238 * any future attempt to re-submit this job will need to know about
239 * this and avoid computing relocation addresses again.
241 * To avoid this sort of issues, we assume that if something was
242 * wrong during submission we must already be in a really bad situation
243 * anyway (such us being out of memory) and return
244 * VK_ERROR_DEVICE_LOST to ensure that clients do not attempt to
245 * submit the same job again to this device.
247 result
= anv_device_set_lost(device
, "vkQueueSubmit() failed");
250 pthread_mutex_unlock(&device
->mutex
);
255 VkResult
anv_QueueWaitIdle(
258 ANV_FROM_HANDLE(anv_queue
, queue
, _queue
);
260 return anv_DeviceWaitIdle(anv_device_to_handle(queue
->device
));
263 VkResult
anv_CreateFence(
265 const VkFenceCreateInfo
* pCreateInfo
,
266 const VkAllocationCallbacks
* pAllocator
,
269 ANV_FROM_HANDLE(anv_device
, device
, _device
);
270 struct anv_fence
*fence
;
272 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_FENCE_CREATE_INFO
);
274 fence
= vk_zalloc2(&device
->alloc
, pAllocator
, sizeof(*fence
), 8,
275 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
277 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
279 if (device
->instance
->physicalDevice
.has_syncobj_wait
) {
280 fence
->permanent
.type
= ANV_FENCE_TYPE_SYNCOBJ
;
282 uint32_t create_flags
= 0;
283 if (pCreateInfo
->flags
& VK_FENCE_CREATE_SIGNALED_BIT
)
284 create_flags
|= DRM_SYNCOBJ_CREATE_SIGNALED
;
286 fence
->permanent
.syncobj
= anv_gem_syncobj_create(device
, create_flags
);
287 if (!fence
->permanent
.syncobj
)
288 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
290 fence
->permanent
.type
= ANV_FENCE_TYPE_BO
;
292 VkResult result
= anv_bo_pool_alloc(&device
->batch_bo_pool
, 4096,
293 &fence
->permanent
.bo
.bo
);
294 if (result
!= VK_SUCCESS
)
297 if (pCreateInfo
->flags
& VK_FENCE_CREATE_SIGNALED_BIT
) {
298 fence
->permanent
.bo
.state
= ANV_BO_FENCE_STATE_SIGNALED
;
300 fence
->permanent
.bo
.state
= ANV_BO_FENCE_STATE_RESET
;
304 *pFence
= anv_fence_to_handle(fence
);
310 anv_fence_impl_cleanup(struct anv_device
*device
,
311 struct anv_fence_impl
*impl
)
313 switch (impl
->type
) {
314 case ANV_FENCE_TYPE_NONE
:
315 /* Dummy. Nothing to do */
318 case ANV_FENCE_TYPE_BO
:
319 anv_bo_pool_free(&device
->batch_bo_pool
, impl
->bo
.bo
);
322 case ANV_FENCE_TYPE_SYNCOBJ
:
323 anv_gem_syncobj_destroy(device
, impl
->syncobj
);
326 case ANV_FENCE_TYPE_WSI
:
327 impl
->fence_wsi
->destroy(impl
->fence_wsi
);
331 unreachable("Invalid fence type");
334 impl
->type
= ANV_FENCE_TYPE_NONE
;
337 void anv_DestroyFence(
340 const VkAllocationCallbacks
* pAllocator
)
342 ANV_FROM_HANDLE(anv_device
, device
, _device
);
343 ANV_FROM_HANDLE(anv_fence
, fence
, _fence
);
348 anv_fence_impl_cleanup(device
, &fence
->temporary
);
349 anv_fence_impl_cleanup(device
, &fence
->permanent
);
351 vk_free2(&device
->alloc
, pAllocator
, fence
);
354 VkResult
anv_ResetFences(
357 const VkFence
* pFences
)
359 ANV_FROM_HANDLE(anv_device
, device
, _device
);
361 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
362 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
364 /* From the Vulkan 1.0.53 spec:
366 * "If any member of pFences currently has its payload imported with
367 * temporary permanence, that fence’s prior permanent payload is
368 * first restored. The remaining operations described therefore
369 * operate on the restored payload.
371 if (fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
)
372 anv_fence_impl_cleanup(device
, &fence
->temporary
);
374 struct anv_fence_impl
*impl
= &fence
->permanent
;
376 switch (impl
->type
) {
377 case ANV_FENCE_TYPE_BO
:
378 impl
->bo
.state
= ANV_BO_FENCE_STATE_RESET
;
381 case ANV_FENCE_TYPE_SYNCOBJ
:
382 anv_gem_syncobj_reset(device
, impl
->syncobj
);
386 unreachable("Invalid fence type");
393 VkResult
anv_GetFenceStatus(
397 ANV_FROM_HANDLE(anv_device
, device
, _device
);
398 ANV_FROM_HANDLE(anv_fence
, fence
, _fence
);
400 if (anv_device_is_lost(device
))
401 return VK_ERROR_DEVICE_LOST
;
403 struct anv_fence_impl
*impl
=
404 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
405 &fence
->temporary
: &fence
->permanent
;
407 switch (impl
->type
) {
408 case ANV_FENCE_TYPE_BO
:
409 /* BO fences don't support import/export */
410 assert(fence
->temporary
.type
== ANV_FENCE_TYPE_NONE
);
411 switch (impl
->bo
.state
) {
412 case ANV_BO_FENCE_STATE_RESET
:
413 /* If it hasn't even been sent off to the GPU yet, it's not ready */
416 case ANV_BO_FENCE_STATE_SIGNALED
:
417 /* It's been signaled, return success */
420 case ANV_BO_FENCE_STATE_SUBMITTED
: {
421 VkResult result
= anv_device_bo_busy(device
, impl
->bo
.bo
);
422 if (result
== VK_SUCCESS
) {
423 impl
->bo
.state
= ANV_BO_FENCE_STATE_SIGNALED
;
430 unreachable("Invalid fence status");
433 case ANV_FENCE_TYPE_SYNCOBJ
: {
434 int ret
= anv_gem_syncobj_wait(device
, &impl
->syncobj
, 1, 0, true);
436 if (errno
== ETIME
) {
439 /* We don't know the real error. */
440 return anv_device_set_lost(device
, "drm_syncobj_wait failed: %m");
448 unreachable("Invalid fence type");
452 #define NSEC_PER_SEC 1000000000
453 #define INT_TYPE_MAX(type) ((1ull << (sizeof(type) * 8 - 1)) - 1)
458 struct timespec current
;
459 clock_gettime(CLOCK_MONOTONIC
, ¤t
);
460 return (uint64_t)current
.tv_sec
* NSEC_PER_SEC
+ current
.tv_nsec
;
463 static uint64_t anv_get_absolute_timeout(uint64_t timeout
)
467 uint64_t current_time
= gettime_ns();
468 uint64_t max_timeout
= (uint64_t) INT64_MAX
- current_time
;
470 timeout
= MIN2(max_timeout
, timeout
);
472 return (current_time
+ timeout
);
475 static int64_t anv_get_relative_timeout(uint64_t abs_timeout
)
477 uint64_t now
= gettime_ns();
479 /* We don't want negative timeouts.
481 * DRM_IOCTL_I915_GEM_WAIT uses a signed 64 bit timeout and is
482 * supposed to block indefinitely timeouts < 0. Unfortunately,
483 * this was broken for a couple of kernel releases. Since there's
484 * no way to know whether or not the kernel we're using is one of
485 * the broken ones, the best we can do is to clamp the timeout to
486 * INT64_MAX. This limits the maximum timeout from 584 years to
487 * 292 years - likely not a big deal.
489 if (abs_timeout
< now
)
492 uint64_t rel_timeout
= abs_timeout
- now
;
493 if (rel_timeout
> (uint64_t) INT64_MAX
)
494 rel_timeout
= INT64_MAX
;
500 anv_wait_for_syncobj_fences(struct anv_device
*device
,
502 const VkFence
*pFences
,
504 uint64_t abs_timeout_ns
)
506 uint32_t *syncobjs
= vk_zalloc(&device
->alloc
,
507 sizeof(*syncobjs
) * fenceCount
, 8,
508 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND
);
510 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
512 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
513 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
514 assert(fence
->permanent
.type
== ANV_FENCE_TYPE_SYNCOBJ
);
516 struct anv_fence_impl
*impl
=
517 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
518 &fence
->temporary
: &fence
->permanent
;
520 assert(impl
->type
== ANV_FENCE_TYPE_SYNCOBJ
);
521 syncobjs
[i
] = impl
->syncobj
;
524 /* The gem_syncobj_wait ioctl may return early due to an inherent
525 * limitation in the way it computes timeouts. Loop until we've actually
526 * passed the timeout.
530 ret
= anv_gem_syncobj_wait(device
, syncobjs
, fenceCount
,
531 abs_timeout_ns
, waitAll
);
532 } while (ret
== -1 && errno
== ETIME
&& gettime_ns() < abs_timeout_ns
);
534 vk_free(&device
->alloc
, syncobjs
);
537 if (errno
== ETIME
) {
540 /* We don't know the real error. */
541 return anv_device_set_lost(device
, "drm_syncobj_wait failed: %m");
549 anv_wait_for_bo_fences(struct anv_device
*device
,
551 const VkFence
*pFences
,
553 uint64_t abs_timeout_ns
)
555 VkResult result
= VK_SUCCESS
;
556 uint32_t pending_fences
= fenceCount
;
557 while (pending_fences
) {
559 bool signaled_fences
= false;
560 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
561 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
563 /* This function assumes that all fences are BO fences and that they
564 * have no temporary state. Since BO fences will never be exported,
565 * this should be a safe assumption.
567 assert(fence
->permanent
.type
== ANV_FENCE_TYPE_BO
);
568 assert(fence
->temporary
.type
== ANV_FENCE_TYPE_NONE
);
569 struct anv_fence_impl
*impl
= &fence
->permanent
;
571 switch (impl
->bo
.state
) {
572 case ANV_BO_FENCE_STATE_RESET
:
573 /* This fence hasn't been submitted yet, we'll catch it the next
574 * time around. Yes, this may mean we dead-loop but, short of
575 * lots of locking and a condition variable, there's not much that
576 * we can do about that.
581 case ANV_BO_FENCE_STATE_SIGNALED
:
582 /* This fence is not pending. If waitAll isn't set, we can return
583 * early. Otherwise, we have to keep going.
591 case ANV_BO_FENCE_STATE_SUBMITTED
:
592 /* These are the fences we really care about. Go ahead and wait
593 * on it until we hit a timeout.
595 result
= anv_device_wait(device
, impl
->bo
.bo
,
596 anv_get_relative_timeout(abs_timeout_ns
));
599 impl
->bo
.state
= ANV_BO_FENCE_STATE_SIGNALED
;
600 signaled_fences
= true;
614 if (pending_fences
&& !signaled_fences
) {
615 /* If we've hit this then someone decided to vkWaitForFences before
616 * they've actually submitted any of them to a queue. This is a
617 * fairly pessimal case, so it's ok to lock here and use a standard
618 * pthreads condition variable.
620 pthread_mutex_lock(&device
->mutex
);
622 /* It's possible that some of the fences have changed state since the
623 * last time we checked. Now that we have the lock, check for
624 * pending fences again and don't wait if it's changed.
626 uint32_t now_pending_fences
= 0;
627 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
628 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
629 if (fence
->permanent
.bo
.state
== ANV_BO_FENCE_STATE_RESET
)
630 now_pending_fences
++;
632 assert(now_pending_fences
<= pending_fences
);
634 if (now_pending_fences
== pending_fences
) {
635 struct timespec abstime
= {
636 .tv_sec
= abs_timeout_ns
/ NSEC_PER_SEC
,
637 .tv_nsec
= abs_timeout_ns
% NSEC_PER_SEC
,
641 ret
= pthread_cond_timedwait(&device
->queue_submit
,
642 &device
->mutex
, &abstime
);
643 assert(ret
!= EINVAL
);
644 if (gettime_ns() >= abs_timeout_ns
) {
645 pthread_mutex_unlock(&device
->mutex
);
651 pthread_mutex_unlock(&device
->mutex
);
656 if (anv_device_is_lost(device
))
657 return VK_ERROR_DEVICE_LOST
;
663 anv_wait_for_wsi_fence(struct anv_device
*device
,
664 const VkFence _fence
,
665 uint64_t abs_timeout
)
667 ANV_FROM_HANDLE(anv_fence
, fence
, _fence
);
668 struct anv_fence_impl
*impl
= &fence
->permanent
;
670 return impl
->fence_wsi
->wait(impl
->fence_wsi
, abs_timeout
);
674 anv_wait_for_fences(struct anv_device
*device
,
676 const VkFence
*pFences
,
678 uint64_t abs_timeout
)
680 VkResult result
= VK_SUCCESS
;
682 if (fenceCount
<= 1 || waitAll
) {
683 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
684 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
685 switch (fence
->permanent
.type
) {
686 case ANV_FENCE_TYPE_BO
:
687 result
= anv_wait_for_bo_fences(device
, 1, &pFences
[i
],
690 case ANV_FENCE_TYPE_SYNCOBJ
:
691 result
= anv_wait_for_syncobj_fences(device
, 1, &pFences
[i
],
694 case ANV_FENCE_TYPE_WSI
:
695 result
= anv_wait_for_wsi_fence(device
, pFences
[i
], abs_timeout
);
697 case ANV_FENCE_TYPE_NONE
:
701 if (result
!= VK_SUCCESS
)
706 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
707 if (anv_wait_for_fences(device
, 1, &pFences
[i
], true, 0) == VK_SUCCESS
)
710 } while (gettime_ns() < abs_timeout
);
716 static bool anv_all_fences_syncobj(uint32_t fenceCount
, const VkFence
*pFences
)
718 for (uint32_t i
= 0; i
< fenceCount
; ++i
) {
719 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
720 if (fence
->permanent
.type
!= ANV_FENCE_TYPE_SYNCOBJ
)
726 static bool anv_all_fences_bo(uint32_t fenceCount
, const VkFence
*pFences
)
728 for (uint32_t i
= 0; i
< fenceCount
; ++i
) {
729 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
730 if (fence
->permanent
.type
!= ANV_FENCE_TYPE_BO
)
736 VkResult
anv_WaitForFences(
739 const VkFence
* pFences
,
743 ANV_FROM_HANDLE(anv_device
, device
, _device
);
745 if (anv_device_is_lost(device
))
746 return VK_ERROR_DEVICE_LOST
;
748 uint64_t abs_timeout
= anv_get_absolute_timeout(timeout
);
749 if (anv_all_fences_syncobj(fenceCount
, pFences
)) {
750 return anv_wait_for_syncobj_fences(device
, fenceCount
, pFences
,
751 waitAll
, abs_timeout
);
752 } else if (anv_all_fences_bo(fenceCount
, pFences
)) {
753 return anv_wait_for_bo_fences(device
, fenceCount
, pFences
,
754 waitAll
, abs_timeout
);
756 return anv_wait_for_fences(device
, fenceCount
, pFences
,
757 waitAll
, abs_timeout
);
761 void anv_GetPhysicalDeviceExternalFenceProperties(
762 VkPhysicalDevice physicalDevice
,
763 const VkPhysicalDeviceExternalFenceInfo
* pExternalFenceInfo
,
764 VkExternalFenceProperties
* pExternalFenceProperties
)
766 ANV_FROM_HANDLE(anv_physical_device
, device
, physicalDevice
);
768 switch (pExternalFenceInfo
->handleType
) {
769 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
:
770 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
:
771 if (device
->has_syncobj_wait
) {
772 pExternalFenceProperties
->exportFromImportedHandleTypes
=
773 VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
|
774 VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
;
775 pExternalFenceProperties
->compatibleHandleTypes
=
776 VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
|
777 VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
;
778 pExternalFenceProperties
->externalFenceFeatures
=
779 VK_EXTERNAL_FENCE_FEATURE_EXPORTABLE_BIT
|
780 VK_EXTERNAL_FENCE_FEATURE_IMPORTABLE_BIT
;
789 pExternalFenceProperties
->exportFromImportedHandleTypes
= 0;
790 pExternalFenceProperties
->compatibleHandleTypes
= 0;
791 pExternalFenceProperties
->externalFenceFeatures
= 0;
794 VkResult
anv_ImportFenceFdKHR(
796 const VkImportFenceFdInfoKHR
* pImportFenceFdInfo
)
798 ANV_FROM_HANDLE(anv_device
, device
, _device
);
799 ANV_FROM_HANDLE(anv_fence
, fence
, pImportFenceFdInfo
->fence
);
800 int fd
= pImportFenceFdInfo
->fd
;
802 assert(pImportFenceFdInfo
->sType
==
803 VK_STRUCTURE_TYPE_IMPORT_FENCE_FD_INFO_KHR
);
805 struct anv_fence_impl new_impl
= {
806 .type
= ANV_FENCE_TYPE_NONE
,
809 switch (pImportFenceFdInfo
->handleType
) {
810 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
:
811 new_impl
.type
= ANV_FENCE_TYPE_SYNCOBJ
;
813 new_impl
.syncobj
= anv_gem_syncobj_fd_to_handle(device
, fd
);
814 if (!new_impl
.syncobj
)
815 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
819 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
:
820 /* Sync files are a bit tricky. Because we want to continue using the
821 * syncobj implementation of WaitForFences, we don't use the sync file
822 * directly but instead import it into a syncobj.
824 new_impl
.type
= ANV_FENCE_TYPE_SYNCOBJ
;
826 new_impl
.syncobj
= anv_gem_syncobj_create(device
, 0);
827 if (!new_impl
.syncobj
)
828 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
830 if (anv_gem_syncobj_import_sync_file(device
, new_impl
.syncobj
, fd
)) {
831 anv_gem_syncobj_destroy(device
, new_impl
.syncobj
);
832 return vk_errorf(device
->instance
, NULL
,
833 VK_ERROR_INVALID_EXTERNAL_HANDLE
,
834 "syncobj sync file import failed: %m");
839 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
842 /* From the Vulkan 1.0.53 spec:
844 * "Importing a fence payload from a file descriptor transfers
845 * ownership of the file descriptor from the application to the
846 * Vulkan implementation. The application must not perform any
847 * operations on the file descriptor after a successful import."
849 * If the import fails, we leave the file descriptor open.
853 if (pImportFenceFdInfo
->flags
& VK_FENCE_IMPORT_TEMPORARY_BIT
) {
854 anv_fence_impl_cleanup(device
, &fence
->temporary
);
855 fence
->temporary
= new_impl
;
857 anv_fence_impl_cleanup(device
, &fence
->permanent
);
858 fence
->permanent
= new_impl
;
864 VkResult
anv_GetFenceFdKHR(
866 const VkFenceGetFdInfoKHR
* pGetFdInfo
,
869 ANV_FROM_HANDLE(anv_device
, device
, _device
);
870 ANV_FROM_HANDLE(anv_fence
, fence
, pGetFdInfo
->fence
);
872 assert(pGetFdInfo
->sType
== VK_STRUCTURE_TYPE_FENCE_GET_FD_INFO_KHR
);
874 struct anv_fence_impl
*impl
=
875 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
876 &fence
->temporary
: &fence
->permanent
;
878 assert(impl
->type
== ANV_FENCE_TYPE_SYNCOBJ
);
879 switch (pGetFdInfo
->handleType
) {
880 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
: {
881 int fd
= anv_gem_syncobj_handle_to_fd(device
, impl
->syncobj
);
883 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
889 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
: {
890 int fd
= anv_gem_syncobj_export_sync_file(device
, impl
->syncobj
);
892 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
899 unreachable("Invalid fence export handle type");
902 /* From the Vulkan 1.0.53 spec:
904 * "Export operations have the same transference as the specified handle
905 * type’s import operations. [...] If the fence was using a
906 * temporarily imported payload, the fence’s prior permanent payload
909 if (impl
== &fence
->temporary
)
910 anv_fence_impl_cleanup(device
, impl
);
915 // Queue semaphore functions
917 VkResult
anv_CreateSemaphore(
919 const VkSemaphoreCreateInfo
* pCreateInfo
,
920 const VkAllocationCallbacks
* pAllocator
,
921 VkSemaphore
* pSemaphore
)
923 ANV_FROM_HANDLE(anv_device
, device
, _device
);
924 struct anv_semaphore
*semaphore
;
926 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO
);
928 semaphore
= vk_alloc2(&device
->alloc
, pAllocator
, sizeof(*semaphore
), 8,
929 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
930 if (semaphore
== NULL
)
931 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
933 const VkExportSemaphoreCreateInfo
*export
=
934 vk_find_struct_const(pCreateInfo
->pNext
, EXPORT_SEMAPHORE_CREATE_INFO
);
935 VkExternalSemaphoreHandleTypeFlags handleTypes
=
936 export
? export
->handleTypes
: 0;
938 if (handleTypes
== 0) {
939 /* The DRM execbuffer ioctl always execute in-oder so long as you stay
940 * on the same ring. Since we don't expose the blit engine as a DMA
941 * queue, a dummy no-op semaphore is a perfectly valid implementation.
943 semaphore
->permanent
.type
= ANV_SEMAPHORE_TYPE_DUMMY
;
944 } else if (handleTypes
& VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
) {
945 assert(handleTypes
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
);
946 if (device
->instance
->physicalDevice
.has_syncobj
) {
947 semaphore
->permanent
.type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
;
948 semaphore
->permanent
.syncobj
= anv_gem_syncobj_create(device
, 0);
949 if (!semaphore
->permanent
.syncobj
) {
950 vk_free2(&device
->alloc
, pAllocator
, semaphore
);
951 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
954 semaphore
->permanent
.type
= ANV_SEMAPHORE_TYPE_BO
;
955 VkResult result
= anv_device_alloc_bo(device
, 4096,
956 ANV_BO_ALLOC_EXTERNAL
|
957 ANV_BO_ALLOC_IMPLICIT_SYNC
,
958 &semaphore
->permanent
.bo
);
959 if (result
!= VK_SUCCESS
) {
960 vk_free2(&device
->alloc
, pAllocator
, semaphore
);
964 /* If we're going to use this as a fence, we need to *not* have the
965 * EXEC_OBJECT_ASYNC bit set.
967 assert(!(semaphore
->permanent
.bo
->flags
& EXEC_OBJECT_ASYNC
));
969 } else if (handleTypes
& VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
) {
970 assert(handleTypes
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
);
971 if (device
->instance
->physicalDevice
.has_syncobj
) {
972 semaphore
->permanent
.type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
;
973 semaphore
->permanent
.syncobj
= anv_gem_syncobj_create(device
, 0);
975 semaphore
->permanent
.type
= ANV_SEMAPHORE_TYPE_SYNC_FILE
;
976 semaphore
->permanent
.fd
= -1;
979 assert(!"Unknown handle type");
980 vk_free2(&device
->alloc
, pAllocator
, semaphore
);
981 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
984 semaphore
->temporary
.type
= ANV_SEMAPHORE_TYPE_NONE
;
986 *pSemaphore
= anv_semaphore_to_handle(semaphore
);
992 anv_semaphore_impl_cleanup(struct anv_device
*device
,
993 struct anv_semaphore_impl
*impl
)
995 switch (impl
->type
) {
996 case ANV_SEMAPHORE_TYPE_NONE
:
997 case ANV_SEMAPHORE_TYPE_DUMMY
:
998 /* Dummy. Nothing to do */
1001 case ANV_SEMAPHORE_TYPE_BO
:
1002 anv_device_release_bo(device
, impl
->bo
);
1005 case ANV_SEMAPHORE_TYPE_SYNC_FILE
:
1009 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
:
1010 anv_gem_syncobj_destroy(device
, impl
->syncobj
);
1014 unreachable("Invalid semaphore type");
1017 impl
->type
= ANV_SEMAPHORE_TYPE_NONE
;
1021 anv_semaphore_reset_temporary(struct anv_device
*device
,
1022 struct anv_semaphore
*semaphore
)
1024 if (semaphore
->temporary
.type
== ANV_SEMAPHORE_TYPE_NONE
)
1027 anv_semaphore_impl_cleanup(device
, &semaphore
->temporary
);
1030 void anv_DestroySemaphore(
1032 VkSemaphore _semaphore
,
1033 const VkAllocationCallbacks
* pAllocator
)
1035 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1036 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, _semaphore
);
1038 if (semaphore
== NULL
)
1041 anv_semaphore_impl_cleanup(device
, &semaphore
->temporary
);
1042 anv_semaphore_impl_cleanup(device
, &semaphore
->permanent
);
1044 vk_free2(&device
->alloc
, pAllocator
, semaphore
);
1047 void anv_GetPhysicalDeviceExternalSemaphoreProperties(
1048 VkPhysicalDevice physicalDevice
,
1049 const VkPhysicalDeviceExternalSemaphoreInfo
* pExternalSemaphoreInfo
,
1050 VkExternalSemaphoreProperties
* pExternalSemaphoreProperties
)
1052 ANV_FROM_HANDLE(anv_physical_device
, device
, physicalDevice
);
1054 switch (pExternalSemaphoreInfo
->handleType
) {
1055 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1056 pExternalSemaphoreProperties
->exportFromImportedHandleTypes
=
1057 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
;
1058 pExternalSemaphoreProperties
->compatibleHandleTypes
=
1059 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
;
1060 pExternalSemaphoreProperties
->externalSemaphoreFeatures
=
1061 VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT
|
1062 VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT
;
1065 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
:
1066 if (device
->has_exec_fence
) {
1067 pExternalSemaphoreProperties
->exportFromImportedHandleTypes
=
1068 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
;
1069 pExternalSemaphoreProperties
->compatibleHandleTypes
=
1070 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
;
1071 pExternalSemaphoreProperties
->externalSemaphoreFeatures
=
1072 VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT
|
1073 VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT
;
1082 pExternalSemaphoreProperties
->exportFromImportedHandleTypes
= 0;
1083 pExternalSemaphoreProperties
->compatibleHandleTypes
= 0;
1084 pExternalSemaphoreProperties
->externalSemaphoreFeatures
= 0;
1087 VkResult
anv_ImportSemaphoreFdKHR(
1089 const VkImportSemaphoreFdInfoKHR
* pImportSemaphoreFdInfo
)
1091 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1092 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pImportSemaphoreFdInfo
->semaphore
);
1093 int fd
= pImportSemaphoreFdInfo
->fd
;
1095 struct anv_semaphore_impl new_impl
= {
1096 .type
= ANV_SEMAPHORE_TYPE_NONE
,
1099 switch (pImportSemaphoreFdInfo
->handleType
) {
1100 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1101 if (device
->instance
->physicalDevice
.has_syncobj
) {
1102 new_impl
.type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
;
1104 new_impl
.syncobj
= anv_gem_syncobj_fd_to_handle(device
, fd
);
1105 if (!new_impl
.syncobj
)
1106 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1108 new_impl
.type
= ANV_SEMAPHORE_TYPE_BO
;
1110 VkResult result
= anv_device_import_bo(device
, fd
,
1111 ANV_BO_ALLOC_EXTERNAL
|
1112 ANV_BO_ALLOC_IMPLICIT_SYNC
,
1114 if (result
!= VK_SUCCESS
)
1117 if (new_impl
.bo
->size
< 4096) {
1118 anv_device_release_bo(device
, new_impl
.bo
);
1119 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1122 /* If we're going to use this as a fence, we need to *not* have the
1123 * EXEC_OBJECT_ASYNC bit set.
1125 assert(!(new_impl
.bo
->flags
& EXEC_OBJECT_ASYNC
));
1128 /* From the Vulkan spec:
1130 * "Importing semaphore state from a file descriptor transfers
1131 * ownership of the file descriptor from the application to the
1132 * Vulkan implementation. The application must not perform any
1133 * operations on the file descriptor after a successful import."
1135 * If the import fails, we leave the file descriptor open.
1140 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
:
1141 if (device
->instance
->physicalDevice
.has_syncobj
) {
1142 new_impl
= (struct anv_semaphore_impl
) {
1143 .type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
,
1144 .syncobj
= anv_gem_syncobj_create(device
, 0),
1146 if (!new_impl
.syncobj
)
1147 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1148 if (anv_gem_syncobj_import_sync_file(device
, new_impl
.syncobj
, fd
)) {
1149 anv_gem_syncobj_destroy(device
, new_impl
.syncobj
);
1150 return vk_errorf(device
->instance
, NULL
,
1151 VK_ERROR_INVALID_EXTERNAL_HANDLE
,
1152 "syncobj sync file import failed: %m");
1154 /* Ownership of the FD is transfered to Anv. Since we don't need it
1155 * anymore because the associated fence has been put into a syncobj,
1156 * we must close the FD.
1160 new_impl
= (struct anv_semaphore_impl
) {
1161 .type
= ANV_SEMAPHORE_TYPE_SYNC_FILE
,
1168 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1171 if (pImportSemaphoreFdInfo
->flags
& VK_SEMAPHORE_IMPORT_TEMPORARY_BIT
) {
1172 anv_semaphore_impl_cleanup(device
, &semaphore
->temporary
);
1173 semaphore
->temporary
= new_impl
;
1175 anv_semaphore_impl_cleanup(device
, &semaphore
->permanent
);
1176 semaphore
->permanent
= new_impl
;
1182 VkResult
anv_GetSemaphoreFdKHR(
1184 const VkSemaphoreGetFdInfoKHR
* pGetFdInfo
,
1187 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1188 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pGetFdInfo
->semaphore
);
1192 assert(pGetFdInfo
->sType
== VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR
);
1194 struct anv_semaphore_impl
*impl
=
1195 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
1196 &semaphore
->temporary
: &semaphore
->permanent
;
1198 switch (impl
->type
) {
1199 case ANV_SEMAPHORE_TYPE_BO
:
1200 result
= anv_device_export_bo(device
, impl
->bo
, pFd
);
1201 if (result
!= VK_SUCCESS
)
1205 case ANV_SEMAPHORE_TYPE_SYNC_FILE
:
1206 /* There are two reasons why this could happen:
1208 * 1) The user is trying to export without submitting something that
1209 * signals the semaphore. If this is the case, it's their bug so
1210 * what we return here doesn't matter.
1212 * 2) The kernel didn't give us a file descriptor. The most likely
1213 * reason for this is running out of file descriptors.
1216 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
1220 /* From the Vulkan 1.0.53 spec:
1222 * "...exporting a semaphore payload to a handle with copy
1223 * transference has the same side effects on the source
1224 * semaphore’s payload as executing a semaphore wait operation."
1226 * In other words, it may still be a SYNC_FD semaphore, but it's now
1227 * considered to have been waited on and no longer has a sync file
1233 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
:
1234 if (pGetFdInfo
->handleType
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
)
1235 fd
= anv_gem_syncobj_export_sync_file(device
, impl
->syncobj
);
1237 assert(pGetFdInfo
->handleType
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
);
1238 fd
= anv_gem_syncobj_handle_to_fd(device
, impl
->syncobj
);
1241 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
1246 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1249 /* From the Vulkan 1.0.53 spec:
1251 * "Export operations have the same transference as the specified handle
1252 * type’s import operations. [...] If the semaphore was using a
1253 * temporarily imported payload, the semaphore’s prior permanent payload
1256 if (impl
== &semaphore
->temporary
)
1257 anv_semaphore_impl_cleanup(device
, impl
);