2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 * This file implements VkQueue, VkFence, and VkSemaphore
32 #include "util/os_file.h"
34 #include "anv_private.h"
37 #include "genxml/gen7_pack.h"
39 uint64_t anv_gettime_ns(void)
41 struct timespec current
;
42 clock_gettime(CLOCK_MONOTONIC
, ¤t
);
43 return (uint64_t)current
.tv_sec
* NSEC_PER_SEC
+ current
.tv_nsec
;
46 uint64_t anv_get_absolute_timeout(uint64_t timeout
)
50 uint64_t current_time
= anv_gettime_ns();
51 uint64_t max_timeout
= (uint64_t) INT64_MAX
- current_time
;
53 timeout
= MIN2(max_timeout
, timeout
);
55 return (current_time
+ timeout
);
58 static int64_t anv_get_relative_timeout(uint64_t abs_timeout
)
60 uint64_t now
= anv_gettime_ns();
62 /* We don't want negative timeouts.
64 * DRM_IOCTL_I915_GEM_WAIT uses a signed 64 bit timeout and is
65 * supposed to block indefinitely timeouts < 0. Unfortunately,
66 * this was broken for a couple of kernel releases. Since there's
67 * no way to know whether or not the kernel we're using is one of
68 * the broken ones, the best we can do is to clamp the timeout to
69 * INT64_MAX. This limits the maximum timeout from 584 years to
70 * 292 years - likely not a big deal.
72 if (abs_timeout
< now
)
75 uint64_t rel_timeout
= abs_timeout
- now
;
76 if (rel_timeout
> (uint64_t) INT64_MAX
)
77 rel_timeout
= INT64_MAX
;
82 static struct anv_semaphore
*anv_semaphore_ref(struct anv_semaphore
*semaphore
);
83 static void anv_semaphore_unref(struct anv_device
*device
, struct anv_semaphore
*semaphore
);
84 static void anv_semaphore_impl_cleanup(struct anv_device
*device
,
85 struct anv_semaphore_impl
*impl
);
88 anv_queue_submit_free(struct anv_device
*device
,
89 struct anv_queue_submit
*submit
)
91 const VkAllocationCallbacks
*alloc
= submit
->alloc
;
93 for (uint32_t i
= 0; i
< submit
->temporary_semaphore_count
; i
++)
94 anv_semaphore_impl_cleanup(device
, &submit
->temporary_semaphores
[i
]);
95 for (uint32_t i
= 0; i
< submit
->sync_fd_semaphore_count
; i
++)
96 anv_semaphore_unref(device
, submit
->sync_fd_semaphores
[i
]);
97 /* Execbuf does not consume the in_fence. It's our job to close it. */
98 if (submit
->in_fence
!= -1)
99 close(submit
->in_fence
);
100 if (submit
->out_fence
!= -1)
101 close(submit
->out_fence
);
102 vk_free(alloc
, submit
->fences
);
103 vk_free(alloc
, submit
->temporary_semaphores
);
104 vk_free(alloc
, submit
->wait_timelines
);
105 vk_free(alloc
, submit
->wait_timeline_values
);
106 vk_free(alloc
, submit
->signal_timelines
);
107 vk_free(alloc
, submit
->signal_timeline_values
);
108 vk_free(alloc
, submit
->fence_bos
);
109 vk_free(alloc
, submit
);
113 anv_queue_submit_ready_locked(struct anv_queue_submit
*submit
)
115 for (uint32_t i
= 0; i
< submit
->wait_timeline_count
; i
++) {
116 if (submit
->wait_timeline_values
[i
] > submit
->wait_timelines
[i
]->highest_pending
)
124 anv_timeline_init(struct anv_device
*device
,
125 struct anv_timeline
*timeline
,
126 uint64_t initial_value
)
128 timeline
->highest_past
=
129 timeline
->highest_pending
= initial_value
;
130 list_inithead(&timeline
->points
);
131 list_inithead(&timeline
->free_points
);
137 anv_timeline_finish(struct anv_device
*device
,
138 struct anv_timeline
*timeline
)
140 list_for_each_entry_safe(struct anv_timeline_point
, point
,
141 &timeline
->free_points
, link
) {
142 list_del(&point
->link
);
143 anv_device_release_bo(device
, point
->bo
);
144 vk_free(&device
->vk
.alloc
, point
);
146 list_for_each_entry_safe(struct anv_timeline_point
, point
,
147 &timeline
->points
, link
) {
148 list_del(&point
->link
);
149 anv_device_release_bo(device
, point
->bo
);
150 vk_free(&device
->vk
.alloc
, point
);
155 anv_timeline_add_point_locked(struct anv_device
*device
,
156 struct anv_timeline
*timeline
,
158 struct anv_timeline_point
**point
)
160 VkResult result
= VK_SUCCESS
;
162 if (list_is_empty(&timeline
->free_points
)) {
164 vk_zalloc(&device
->vk
.alloc
, sizeof(**point
),
165 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE
);
167 result
= vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
168 if (result
== VK_SUCCESS
) {
169 result
= anv_device_alloc_bo(device
, 4096,
170 ANV_BO_ALLOC_EXTERNAL
|
171 ANV_BO_ALLOC_IMPLICIT_SYNC
,
172 0 /* explicit_address */,
174 if (result
!= VK_SUCCESS
)
175 vk_free(&device
->vk
.alloc
, *point
);
178 *point
= list_first_entry(&timeline
->free_points
,
179 struct anv_timeline_point
, link
);
180 list_del(&(*point
)->link
);
183 if (result
== VK_SUCCESS
) {
184 (*point
)->serial
= value
;
185 list_addtail(&(*point
)->link
, &timeline
->points
);
192 anv_timeline_gc_locked(struct anv_device
*device
,
193 struct anv_timeline
*timeline
)
195 list_for_each_entry_safe(struct anv_timeline_point
, point
,
196 &timeline
->points
, link
) {
197 /* timeline->higest_pending is only incremented once submission has
198 * happened. If this point has a greater serial, it means the point
199 * hasn't been submitted yet.
201 if (point
->serial
> timeline
->highest_pending
)
204 /* If someone is waiting on this time point, consider it busy and don't
205 * try to recycle it. There's a slim possibility that it's no longer
206 * busy by the time we look at it but we would be recycling it out from
207 * under a waiter and that can lead to weird races.
209 * We walk the list in-order so if this time point is still busy so is
210 * every following time point
212 assert(point
->waiting
>= 0);
216 /* Garbage collect any signaled point. */
217 VkResult result
= anv_device_bo_busy(device
, point
->bo
);
218 if (result
== VK_NOT_READY
) {
219 /* We walk the list in-order so if this time point is still busy so
220 * is every following time point
223 } else if (result
!= VK_SUCCESS
) {
227 assert(timeline
->highest_past
< point
->serial
);
228 timeline
->highest_past
= point
->serial
;
230 list_del(&point
->link
);
231 list_add(&point
->link
, &timeline
->free_points
);
237 static VkResult
anv_queue_submit_add_fence_bo(struct anv_queue_submit
*submit
,
242 anv_queue_submit_timeline_locked(struct anv_queue
*queue
,
243 struct anv_queue_submit
*submit
)
247 for (uint32_t i
= 0; i
< submit
->wait_timeline_count
; i
++) {
248 struct anv_timeline
*timeline
= submit
->wait_timelines
[i
];
249 uint64_t wait_value
= submit
->wait_timeline_values
[i
];
251 if (timeline
->highest_past
>= wait_value
)
254 list_for_each_entry(struct anv_timeline_point
, point
, &timeline
->points
, link
) {
255 if (point
->serial
< wait_value
)
257 result
= anv_queue_submit_add_fence_bo(submit
, point
->bo
, false);
258 if (result
!= VK_SUCCESS
)
263 for (uint32_t i
= 0; i
< submit
->signal_timeline_count
; i
++) {
264 struct anv_timeline
*timeline
= submit
->signal_timelines
[i
];
265 uint64_t signal_value
= submit
->signal_timeline_values
[i
];
266 struct anv_timeline_point
*point
;
268 result
= anv_timeline_add_point_locked(queue
->device
, timeline
,
269 signal_value
, &point
);
270 if (result
!= VK_SUCCESS
)
273 result
= anv_queue_submit_add_fence_bo(submit
, point
->bo
, true);
274 if (result
!= VK_SUCCESS
)
278 result
= anv_queue_execbuf_locked(queue
, submit
);
280 if (result
== VK_SUCCESS
) {
281 /* Update the pending values in the timeline objects. */
282 for (uint32_t i
= 0; i
< submit
->signal_timeline_count
; i
++) {
283 struct anv_timeline
*timeline
= submit
->signal_timelines
[i
];
284 uint64_t signal_value
= submit
->signal_timeline_values
[i
];
286 assert(signal_value
> timeline
->highest_pending
);
287 timeline
->highest_pending
= signal_value
;
290 /* Update signaled semaphores backed by syncfd. */
291 for (uint32_t i
= 0; i
< submit
->sync_fd_semaphore_count
; i
++) {
292 struct anv_semaphore
*semaphore
= submit
->sync_fd_semaphores
[i
];
293 /* Out fences can't have temporary state because that would imply
294 * that we imported a sync file and are trying to signal it.
296 assert(semaphore
->temporary
.type
== ANV_SEMAPHORE_TYPE_NONE
);
297 struct anv_semaphore_impl
*impl
= &semaphore
->permanent
;
299 assert(impl
->type
== ANV_SEMAPHORE_TYPE_SYNC_FILE
);
300 impl
->fd
= os_dupfd_cloexec(submit
->out_fence
);
303 /* Unblock any waiter by signaling the points, the application will get
304 * a device lost error code.
306 for (uint32_t i
= 0; i
< submit
->signal_timeline_count
; i
++) {
307 struct anv_timeline
*timeline
= submit
->signal_timelines
[i
];
308 uint64_t signal_value
= submit
->signal_timeline_values
[i
];
310 assert(signal_value
> timeline
->highest_pending
);
311 timeline
->highest_past
= timeline
->highest_pending
= signal_value
;
319 anv_queue_submit_deferred_locked(struct anv_queue
*queue
, uint32_t *advance
)
321 VkResult result
= VK_SUCCESS
;
323 /* Go through all the queued submissions and submit then until we find one
324 * that's waiting on a point that hasn't materialized yet.
326 list_for_each_entry_safe(struct anv_queue_submit
, submit
,
327 &queue
->queued_submits
, link
) {
328 if (!anv_queue_submit_ready_locked(submit
))
332 list_del(&submit
->link
);
334 result
= anv_queue_submit_timeline_locked(queue
, submit
);
336 anv_queue_submit_free(queue
->device
, submit
);
338 if (result
!= VK_SUCCESS
)
346 anv_device_submit_deferred_locked(struct anv_device
*device
)
348 uint32_t advance
= 0;
349 return anv_queue_submit_deferred_locked(&device
->queue
, &advance
);
353 _anv_queue_submit(struct anv_queue
*queue
, struct anv_queue_submit
**_submit
,
356 struct anv_queue_submit
*submit
= *_submit
;
358 /* Wait before signal behavior means we might keep alive the
359 * anv_queue_submit object a bit longer, so transfer the ownership to the
364 pthread_mutex_lock(&queue
->device
->mutex
);
365 list_addtail(&submit
->link
, &queue
->queued_submits
);
366 VkResult result
= anv_device_submit_deferred_locked(queue
->device
);
368 while (result
== VK_SUCCESS
&& !list_is_empty(&queue
->queued_submits
)) {
369 int ret
= pthread_cond_wait(&queue
->device
->queue_submit
,
370 &queue
->device
->mutex
);
372 result
= anv_device_set_lost(queue
->device
, "wait timeout");
376 result
= anv_device_submit_deferred_locked(queue
->device
);
379 pthread_mutex_unlock(&queue
->device
->mutex
);
384 anv_queue_init(struct anv_device
*device
, struct anv_queue
*queue
)
386 vk_object_base_init(&device
->vk
, &queue
->base
, VK_OBJECT_TYPE_QUEUE
);
387 queue
->device
= device
;
390 list_inithead(&queue
->queued_submits
);
396 anv_queue_finish(struct anv_queue
*queue
)
398 vk_object_base_finish(&queue
->base
);
402 anv_queue_submit_add_fence_bo(struct anv_queue_submit
*submit
,
406 if (submit
->fence_bo_count
>= submit
->fence_bo_array_length
) {
407 uint32_t new_len
= MAX2(submit
->fence_bo_array_length
* 2, 64);
410 vk_realloc(submit
->alloc
,
411 submit
->fence_bos
, new_len
* sizeof(*submit
->fence_bos
),
412 8, submit
->alloc_scope
);
413 if (submit
->fence_bos
== NULL
)
414 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
416 submit
->fence_bo_array_length
= new_len
;
419 /* Take advantage that anv_bo are allocated at 8 byte alignement so we can
420 * use the lowest bit to store whether this is a BO we need to signal.
422 submit
->fence_bos
[submit
->fence_bo_count
++] = anv_pack_ptr(bo
, 1, signal
);
428 anv_queue_submit_add_syncobj(struct anv_queue_submit
* submit
,
429 struct anv_device
*device
,
430 uint32_t handle
, uint32_t flags
)
434 if (submit
->fence_count
>= submit
->fence_array_length
) {
435 uint32_t new_len
= MAX2(submit
->fence_array_length
* 2, 64);
438 vk_realloc(submit
->alloc
,
439 submit
->fences
, new_len
* sizeof(*submit
->fences
),
440 8, submit
->alloc_scope
);
441 if (submit
->fences
== NULL
)
442 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
444 submit
->fence_array_length
= new_len
;
447 submit
->fences
[submit
->fence_count
++] = (struct drm_i915_gem_exec_fence
) {
456 anv_queue_submit_add_sync_fd_fence(struct anv_queue_submit
*submit
,
457 struct anv_semaphore
*semaphore
)
459 if (submit
->sync_fd_semaphore_count
>= submit
->sync_fd_semaphore_array_length
) {
460 uint32_t new_len
= MAX2(submit
->sync_fd_semaphore_array_length
* 2, 64);
461 struct anv_semaphore
**new_semaphores
=
462 vk_realloc(submit
->alloc
, submit
->sync_fd_semaphores
,
463 new_len
* sizeof(*submit
->sync_fd_semaphores
), 8,
464 submit
->alloc_scope
);
465 if (new_semaphores
== NULL
)
466 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
468 submit
->sync_fd_semaphores
= new_semaphores
;
471 submit
->sync_fd_semaphores
[submit
->sync_fd_semaphore_count
++] =
472 anv_semaphore_ref(semaphore
);
473 submit
->need_out_fence
= true;
479 anv_queue_submit_add_timeline_wait(struct anv_queue_submit
* submit
,
480 struct anv_device
*device
,
481 struct anv_timeline
*timeline
,
484 if (submit
->wait_timeline_count
>= submit
->wait_timeline_array_length
) {
485 uint32_t new_len
= MAX2(submit
->wait_timeline_array_length
* 2, 64);
487 submit
->wait_timelines
=
488 vk_realloc(submit
->alloc
,
489 submit
->wait_timelines
, new_len
* sizeof(*submit
->wait_timelines
),
490 8, submit
->alloc_scope
);
491 if (submit
->wait_timelines
== NULL
)
492 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
494 submit
->wait_timeline_values
=
495 vk_realloc(submit
->alloc
,
496 submit
->wait_timeline_values
, new_len
* sizeof(*submit
->wait_timeline_values
),
497 8, submit
->alloc_scope
);
498 if (submit
->wait_timeline_values
== NULL
)
499 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
501 submit
->wait_timeline_array_length
= new_len
;
504 submit
->wait_timelines
[submit
->wait_timeline_count
] = timeline
;
505 submit
->wait_timeline_values
[submit
->wait_timeline_count
] = value
;
507 submit
->wait_timeline_count
++;
513 anv_queue_submit_add_timeline_signal(struct anv_queue_submit
* submit
,
514 struct anv_device
*device
,
515 struct anv_timeline
*timeline
,
518 assert(timeline
->highest_pending
< value
);
520 if (submit
->signal_timeline_count
>= submit
->signal_timeline_array_length
) {
521 uint32_t new_len
= MAX2(submit
->signal_timeline_array_length
* 2, 64);
523 submit
->signal_timelines
=
524 vk_realloc(submit
->alloc
,
525 submit
->signal_timelines
, new_len
* sizeof(*submit
->signal_timelines
),
526 8, submit
->alloc_scope
);
527 if (submit
->signal_timelines
== NULL
)
528 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
530 submit
->signal_timeline_values
=
531 vk_realloc(submit
->alloc
,
532 submit
->signal_timeline_values
, new_len
* sizeof(*submit
->signal_timeline_values
),
533 8, submit
->alloc_scope
);
534 if (submit
->signal_timeline_values
== NULL
)
535 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
537 submit
->signal_timeline_array_length
= new_len
;
540 submit
->signal_timelines
[submit
->signal_timeline_count
] = timeline
;
541 submit
->signal_timeline_values
[submit
->signal_timeline_count
] = value
;
543 submit
->signal_timeline_count
++;
548 static struct anv_queue_submit
*
549 anv_queue_submit_alloc(struct anv_device
*device
, int perf_query_pass
)
551 const VkAllocationCallbacks
*alloc
= &device
->vk
.alloc
;
552 VkSystemAllocationScope alloc_scope
= VK_SYSTEM_ALLOCATION_SCOPE_DEVICE
;
554 struct anv_queue_submit
*submit
= vk_zalloc(alloc
, sizeof(*submit
), 8, alloc_scope
);
558 submit
->alloc
= alloc
;
559 submit
->alloc_scope
= alloc_scope
;
560 submit
->in_fence
= -1;
561 submit
->out_fence
= -1;
562 submit
->perf_query_pass
= perf_query_pass
;
568 anv_queue_submit_simple_batch(struct anv_queue
*queue
,
569 struct anv_batch
*batch
)
571 if (queue
->device
->no_hw
)
574 struct anv_device
*device
= queue
->device
;
575 struct anv_queue_submit
*submit
= anv_queue_submit_alloc(device
, -1);
577 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
579 bool has_syncobj_wait
= device
->physical
->has_syncobj_wait
;
582 struct anv_bo
*batch_bo
, *sync_bo
;
584 if (has_syncobj_wait
) {
585 syncobj
= anv_gem_syncobj_create(device
, 0);
587 result
= vk_error(VK_ERROR_OUT_OF_DEVICE_MEMORY
);
588 goto err_free_submit
;
591 result
= anv_queue_submit_add_syncobj(submit
, device
, syncobj
,
592 I915_EXEC_FENCE_SIGNAL
);
594 result
= anv_device_alloc_bo(device
, 4096,
595 ANV_BO_ALLOC_EXTERNAL
|
596 ANV_BO_ALLOC_IMPLICIT_SYNC
,
597 0 /* explicit_address */,
599 if (result
!= VK_SUCCESS
)
600 goto err_free_submit
;
602 result
= anv_queue_submit_add_fence_bo(submit
, sync_bo
, true /* signal */);
605 if (result
!= VK_SUCCESS
)
606 goto err_destroy_sync_primitive
;
609 uint32_t size
= align_u32(batch
->next
- batch
->start
, 8);
610 result
= anv_bo_pool_alloc(&device
->batch_bo_pool
, size
, &batch_bo
);
611 if (result
!= VK_SUCCESS
)
612 goto err_destroy_sync_primitive
;
614 memcpy(batch_bo
->map
, batch
->start
, size
);
615 if (!device
->info
.has_llc
)
616 gen_flush_range(batch_bo
->map
, size
);
618 submit
->simple_bo
= batch_bo
;
619 submit
->simple_bo_size
= size
;
622 result
= _anv_queue_submit(queue
, &submit
, true);
624 if (result
== VK_SUCCESS
) {
625 if (has_syncobj_wait
) {
626 if (anv_gem_syncobj_wait(device
, &syncobj
, 1,
627 anv_get_absolute_timeout(INT64_MAX
), true))
628 result
= anv_device_set_lost(device
, "anv_gem_syncobj_wait failed: %m");
629 anv_gem_syncobj_destroy(device
, syncobj
);
631 result
= anv_device_wait(device
, sync_bo
,
632 anv_get_relative_timeout(INT64_MAX
));
633 anv_device_release_bo(device
, sync_bo
);
638 anv_bo_pool_free(&device
->batch_bo_pool
, batch_bo
);
641 anv_queue_submit_free(device
, submit
);
645 err_destroy_sync_primitive
:
646 if (has_syncobj_wait
)
647 anv_gem_syncobj_destroy(device
, syncobj
);
649 anv_device_release_bo(device
, sync_bo
);
652 anv_queue_submit_free(device
, submit
);
657 /* Transfer ownership of temporary semaphores from the VkSemaphore object to
658 * the anv_queue_submit object. Those temporary semaphores are then freed in
659 * anv_queue_submit_free() once the driver is finished with them.
662 maybe_transfer_temporary_semaphore(struct anv_queue_submit
*submit
,
663 struct anv_semaphore
*semaphore
,
664 struct anv_semaphore_impl
**out_impl
)
666 struct anv_semaphore_impl
*impl
= &semaphore
->temporary
;
668 if (impl
->type
== ANV_SEMAPHORE_TYPE_NONE
) {
669 *out_impl
= &semaphore
->permanent
;
673 /* BO backed timeline semaphores cannot be temporary. */
674 assert(impl
->type
!= ANV_SEMAPHORE_TYPE_TIMELINE
);
677 * There is a requirement to reset semaphore to their permanent state after
678 * submission. From the Vulkan 1.0.53 spec:
680 * "If the import is temporary, the implementation must restore the
681 * semaphore to its prior permanent state after submitting the next
682 * semaphore wait operation."
684 * In the case we defer the actual submission to a thread because of the
685 * wait-before-submit behavior required for timeline semaphores, we need to
686 * make copies of the temporary syncobj to ensure they stay alive until we
687 * do the actual execbuffer ioctl.
689 if (submit
->temporary_semaphore_count
>= submit
->temporary_semaphore_array_length
) {
690 uint32_t new_len
= MAX2(submit
->temporary_semaphore_array_length
* 2, 8);
691 /* Make sure that if the realloc fails, we still have the old semaphore
692 * array around to properly clean things up on failure.
694 struct anv_semaphore_impl
*new_array
=
695 vk_realloc(submit
->alloc
,
696 submit
->temporary_semaphores
,
697 new_len
* sizeof(*submit
->temporary_semaphores
),
698 8, submit
->alloc_scope
);
699 if (new_array
== NULL
)
700 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
702 submit
->temporary_semaphores
= new_array
;
703 submit
->temporary_semaphore_array_length
= new_len
;
706 /* Copy anv_semaphore_impl into anv_queue_submit. */
707 submit
->temporary_semaphores
[submit
->temporary_semaphore_count
++] = *impl
;
708 *out_impl
= &submit
->temporary_semaphores
[submit
->temporary_semaphore_count
- 1];
710 /* Clear the incoming semaphore */
711 impl
->type
= ANV_SEMAPHORE_TYPE_NONE
;
717 anv_queue_submit(struct anv_queue
*queue
,
718 struct anv_cmd_buffer
*cmd_buffer
,
719 const VkSemaphore
*in_semaphores
,
720 const uint64_t *in_values
,
721 uint32_t num_in_semaphores
,
722 const VkSemaphore
*out_semaphores
,
723 const uint64_t *out_values
,
724 uint32_t num_out_semaphores
,
725 struct anv_bo
*wsi_signal_bo
,
729 ANV_FROM_HANDLE(anv_fence
, fence
, _fence
);
730 struct anv_device
*device
= queue
->device
;
731 UNUSED
struct anv_physical_device
*pdevice
= device
->physical
;
732 struct anv_queue_submit
*submit
= anv_queue_submit_alloc(device
, perf_query_pass
);
734 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
736 submit
->cmd_buffer
= cmd_buffer
;
738 VkResult result
= VK_SUCCESS
;
740 for (uint32_t i
= 0; i
< num_in_semaphores
; i
++) {
741 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, in_semaphores
[i
]);
742 struct anv_semaphore_impl
*impl
;
744 result
= maybe_transfer_temporary_semaphore(submit
, semaphore
, &impl
);
745 if (result
!= VK_SUCCESS
)
748 switch (impl
->type
) {
749 case ANV_SEMAPHORE_TYPE_BO
:
750 assert(!pdevice
->has_syncobj
);
751 result
= anv_queue_submit_add_fence_bo(submit
, impl
->bo
, false /* signal */);
752 if (result
!= VK_SUCCESS
)
756 case ANV_SEMAPHORE_TYPE_WSI_BO
:
757 /* When using a window-system buffer as a semaphore, always enable
758 * EXEC_OBJECT_WRITE. This gives us a WaR hazard with the display or
759 * compositor's read of the buffer and enforces that we don't start
760 * rendering until they are finished. This is exactly the
761 * synchronization we want with vkAcquireNextImage.
763 result
= anv_queue_submit_add_fence_bo(submit
, impl
->bo
, true /* signal */);
764 if (result
!= VK_SUCCESS
)
768 case ANV_SEMAPHORE_TYPE_SYNC_FILE
:
769 assert(!pdevice
->has_syncobj
);
770 if (submit
->in_fence
== -1) {
771 submit
->in_fence
= impl
->fd
;
772 if (submit
->in_fence
== -1) {
773 result
= vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
778 int merge
= anv_gem_sync_file_merge(device
, submit
->in_fence
, impl
->fd
);
780 result
= vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
784 close(submit
->in_fence
);
786 submit
->in_fence
= merge
;
790 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
: {
791 result
= anv_queue_submit_add_syncobj(submit
, device
,
793 I915_EXEC_FENCE_WAIT
);
794 if (result
!= VK_SUCCESS
)
799 case ANV_SEMAPHORE_TYPE_TIMELINE
:
800 result
= anv_queue_submit_add_timeline_wait(submit
, device
,
802 in_values
? in_values
[i
] : 0);
803 if (result
!= VK_SUCCESS
)
812 for (uint32_t i
= 0; i
< num_out_semaphores
; i
++) {
813 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, out_semaphores
[i
]);
815 /* Under most circumstances, out fences won't be temporary. However,
816 * the spec does allow it for opaque_fd. From the Vulkan 1.0.53 spec:
818 * "If the import is temporary, the implementation must restore the
819 * semaphore to its prior permanent state after submitting the next
820 * semaphore wait operation."
822 * The spec says nothing whatsoever about signal operations on
823 * temporarily imported semaphores so it appears they are allowed.
824 * There are also CTS tests that require this to work.
826 struct anv_semaphore_impl
*impl
=
827 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
828 &semaphore
->temporary
: &semaphore
->permanent
;
830 switch (impl
->type
) {
831 case ANV_SEMAPHORE_TYPE_BO
:
832 assert(!pdevice
->has_syncobj
);
833 result
= anv_queue_submit_add_fence_bo(submit
, impl
->bo
, true /* signal */);
834 if (result
!= VK_SUCCESS
)
838 case ANV_SEMAPHORE_TYPE_SYNC_FILE
:
839 assert(!pdevice
->has_syncobj
);
840 result
= anv_queue_submit_add_sync_fd_fence(submit
, semaphore
);
841 if (result
!= VK_SUCCESS
)
845 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
: {
846 result
= anv_queue_submit_add_syncobj(submit
, device
, impl
->syncobj
,
847 I915_EXEC_FENCE_SIGNAL
);
848 if (result
!= VK_SUCCESS
)
853 case ANV_SEMAPHORE_TYPE_TIMELINE
:
854 result
= anv_queue_submit_add_timeline_signal(submit
, device
,
856 out_values
? out_values
[i
] : 0);
857 if (result
!= VK_SUCCESS
)
867 result
= anv_queue_submit_add_fence_bo(submit
, wsi_signal_bo
, true /* signal */);
868 if (result
!= VK_SUCCESS
)
873 /* Under most circumstances, out fences won't be temporary. However,
874 * the spec does allow it for opaque_fd. From the Vulkan 1.0.53 spec:
876 * "If the import is temporary, the implementation must restore the
877 * semaphore to its prior permanent state after submitting the next
878 * semaphore wait operation."
880 * The spec says nothing whatsoever about signal operations on
881 * temporarily imported semaphores so it appears they are allowed.
882 * There are also CTS tests that require this to work.
884 struct anv_fence_impl
*impl
=
885 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
886 &fence
->temporary
: &fence
->permanent
;
888 switch (impl
->type
) {
889 case ANV_FENCE_TYPE_BO
:
890 result
= anv_queue_submit_add_fence_bo(submit
, impl
->bo
.bo
, true /* signal */);
891 if (result
!= VK_SUCCESS
)
895 case ANV_FENCE_TYPE_SYNCOBJ
: {
897 * For the same reason we reset the signaled binary syncobj above,
898 * also reset the fence's syncobj so that they don't contain a
899 * signaled dma-fence.
901 result
= anv_queue_submit_add_syncobj(submit
, device
, impl
->syncobj
,
902 I915_EXEC_FENCE_SIGNAL
);
903 if (result
!= VK_SUCCESS
)
909 unreachable("Invalid fence type");
913 result
= _anv_queue_submit(queue
, &submit
, false);
914 if (result
!= VK_SUCCESS
)
917 if (fence
&& fence
->permanent
.type
== ANV_FENCE_TYPE_BO
) {
918 /* If we have permanent BO fence, the only type of temporary possible
919 * would be BO_WSI (because BO fences are not shareable). The Vulkan spec
920 * also requires that the fence passed to vkQueueSubmit() be :
923 * * not be associated with any other queue command that has not yet
924 * completed execution on that queue
926 * So the only acceptable type for the temporary is NONE.
928 assert(fence
->temporary
.type
== ANV_FENCE_TYPE_NONE
);
930 /* Once the execbuf has returned, we need to set the fence state to
931 * SUBMITTED. We can't do this before calling execbuf because
932 * anv_GetFenceStatus does take the global device lock before checking
935 * We set the fence state to SUBMITTED regardless of whether or not the
936 * execbuf succeeds because we need to ensure that vkWaitForFences() and
937 * vkGetFenceStatus() return a valid result (VK_ERROR_DEVICE_LOST or
938 * VK_SUCCESS) in a finite amount of time even if execbuf fails.
940 fence
->permanent
.bo
.state
= ANV_BO_FENCE_STATE_SUBMITTED
;
945 anv_queue_submit_free(device
, submit
);
950 VkResult
anv_QueueSubmit(
952 uint32_t submitCount
,
953 const VkSubmitInfo
* pSubmits
,
956 ANV_FROM_HANDLE(anv_queue
, queue
, _queue
);
958 if (queue
->device
->no_hw
)
961 /* Query for device status prior to submitting. Technically, we don't need
962 * to do this. However, if we have a client that's submitting piles of
963 * garbage, we would rather break as early as possible to keep the GPU
964 * hanging contained. If we don't check here, we'll either be waiting for
965 * the kernel to kick us or we'll have to wait until the client waits on a
966 * fence before we actually know whether or not we've hung.
968 VkResult result
= anv_device_query_status(queue
->device
);
969 if (result
!= VK_SUCCESS
)
972 if (fence
&& submitCount
== 0) {
973 /* If we don't have any command buffers, we need to submit a dummy
974 * batch to give GEM something to wait on. We could, potentially,
975 * come up with something more efficient but this shouldn't be a
978 result
= anv_queue_submit(queue
, NULL
, NULL
, NULL
, 0, NULL
, NULL
, 0,
983 for (uint32_t i
= 0; i
< submitCount
; i
++) {
984 /* Fence for this submit. NULL for all but the last one */
985 VkFence submit_fence
= (i
== submitCount
- 1) ? fence
: VK_NULL_HANDLE
;
987 const struct wsi_memory_signal_submit_info
*mem_signal_info
=
988 vk_find_struct_const(pSubmits
[i
].pNext
,
989 WSI_MEMORY_SIGNAL_SUBMIT_INFO_MESA
);
990 struct anv_bo
*wsi_signal_bo
=
991 mem_signal_info
&& mem_signal_info
->memory
!= VK_NULL_HANDLE
?
992 anv_device_memory_from_handle(mem_signal_info
->memory
)->bo
: NULL
;
994 const VkTimelineSemaphoreSubmitInfoKHR
*timeline_info
=
995 vk_find_struct_const(pSubmits
[i
].pNext
,
996 TIMELINE_SEMAPHORE_SUBMIT_INFO_KHR
);
997 const VkPerformanceQuerySubmitInfoKHR
*perf_info
=
998 vk_find_struct_const(pSubmits
[i
].pNext
,
999 PERFORMANCE_QUERY_SUBMIT_INFO_KHR
);
1000 const uint64_t *wait_values
=
1001 timeline_info
&& timeline_info
->waitSemaphoreValueCount
?
1002 timeline_info
->pWaitSemaphoreValues
: NULL
;
1003 const uint64_t *signal_values
=
1004 timeline_info
&& timeline_info
->signalSemaphoreValueCount
?
1005 timeline_info
->pSignalSemaphoreValues
: NULL
;
1007 if (pSubmits
[i
].commandBufferCount
== 0) {
1008 /* If we don't have any command buffers, we need to submit a dummy
1009 * batch to give GEM something to wait on. We could, potentially,
1010 * come up with something more efficient but this shouldn't be a
1013 result
= anv_queue_submit(queue
, NULL
,
1014 pSubmits
[i
].pWaitSemaphores
,
1016 pSubmits
[i
].waitSemaphoreCount
,
1017 pSubmits
[i
].pSignalSemaphores
,
1019 pSubmits
[i
].signalSemaphoreCount
,
1023 if (result
!= VK_SUCCESS
)
1029 for (uint32_t j
= 0; j
< pSubmits
[i
].commandBufferCount
; j
++) {
1030 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
,
1031 pSubmits
[i
].pCommandBuffers
[j
]);
1032 assert(cmd_buffer
->level
== VK_COMMAND_BUFFER_LEVEL_PRIMARY
);
1033 assert(!anv_batch_has_error(&cmd_buffer
->batch
));
1035 /* Fence for this execbuf. NULL for all but the last one */
1036 VkFence execbuf_fence
=
1037 (j
== pSubmits
[i
].commandBufferCount
- 1) ?
1038 submit_fence
: VK_NULL_HANDLE
;
1040 const VkSemaphore
*in_semaphores
= NULL
, *out_semaphores
= NULL
;
1041 const uint64_t *in_values
= NULL
, *out_values
= NULL
;
1042 uint32_t num_in_semaphores
= 0, num_out_semaphores
= 0;
1044 /* Only the first batch gets the in semaphores */
1045 in_semaphores
= pSubmits
[i
].pWaitSemaphores
;
1046 in_values
= wait_values
;
1047 num_in_semaphores
= pSubmits
[i
].waitSemaphoreCount
;
1050 if (j
== pSubmits
[i
].commandBufferCount
- 1) {
1051 /* Only the last batch gets the out semaphores */
1052 out_semaphores
= pSubmits
[i
].pSignalSemaphores
;
1053 out_values
= signal_values
;
1054 num_out_semaphores
= pSubmits
[i
].signalSemaphoreCount
;
1057 result
= anv_queue_submit(queue
, cmd_buffer
,
1058 in_semaphores
, in_values
, num_in_semaphores
,
1059 out_semaphores
, out_values
, num_out_semaphores
,
1060 wsi_signal_bo
, execbuf_fence
,
1061 perf_info
? perf_info
->counterPassIndex
: 0);
1062 if (result
!= VK_SUCCESS
)
1068 if (result
!= VK_SUCCESS
&& result
!= VK_ERROR_DEVICE_LOST
) {
1069 /* In the case that something has gone wrong we may end up with an
1070 * inconsistent state from which it may not be trivial to recover.
1071 * For example, we might have computed address relocations and
1072 * any future attempt to re-submit this job will need to know about
1073 * this and avoid computing relocation addresses again.
1075 * To avoid this sort of issues, we assume that if something was
1076 * wrong during submission we must already be in a really bad situation
1077 * anyway (such us being out of memory) and return
1078 * VK_ERROR_DEVICE_LOST to ensure that clients do not attempt to
1079 * submit the same job again to this device.
1081 * We skip doing this on VK_ERROR_DEVICE_LOST because
1082 * anv_device_set_lost() would have been called already by a callee of
1083 * anv_queue_submit().
1085 result
= anv_device_set_lost(queue
->device
, "vkQueueSubmit() failed");
1091 VkResult
anv_QueueWaitIdle(
1094 ANV_FROM_HANDLE(anv_queue
, queue
, _queue
);
1096 if (anv_device_is_lost(queue
->device
))
1097 return VK_ERROR_DEVICE_LOST
;
1099 return anv_queue_submit_simple_batch(queue
, NULL
);
1102 VkResult
anv_CreateFence(
1104 const VkFenceCreateInfo
* pCreateInfo
,
1105 const VkAllocationCallbacks
* pAllocator
,
1108 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1109 struct anv_fence
*fence
;
1111 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_FENCE_CREATE_INFO
);
1113 fence
= vk_zalloc2(&device
->vk
.alloc
, pAllocator
, sizeof(*fence
), 8,
1114 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
1116 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1118 vk_object_base_init(&device
->vk
, &fence
->base
, VK_OBJECT_TYPE_FENCE
);
1120 if (device
->physical
->has_syncobj_wait
) {
1121 fence
->permanent
.type
= ANV_FENCE_TYPE_SYNCOBJ
;
1123 uint32_t create_flags
= 0;
1124 if (pCreateInfo
->flags
& VK_FENCE_CREATE_SIGNALED_BIT
)
1125 create_flags
|= DRM_SYNCOBJ_CREATE_SIGNALED
;
1127 fence
->permanent
.syncobj
= anv_gem_syncobj_create(device
, create_flags
);
1128 if (!fence
->permanent
.syncobj
)
1129 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1131 fence
->permanent
.type
= ANV_FENCE_TYPE_BO
;
1133 VkResult result
= anv_bo_pool_alloc(&device
->batch_bo_pool
, 4096,
1134 &fence
->permanent
.bo
.bo
);
1135 if (result
!= VK_SUCCESS
)
1138 if (pCreateInfo
->flags
& VK_FENCE_CREATE_SIGNALED_BIT
) {
1139 fence
->permanent
.bo
.state
= ANV_BO_FENCE_STATE_SIGNALED
;
1141 fence
->permanent
.bo
.state
= ANV_BO_FENCE_STATE_RESET
;
1145 *pFence
= anv_fence_to_handle(fence
);
1151 anv_fence_impl_cleanup(struct anv_device
*device
,
1152 struct anv_fence_impl
*impl
)
1154 switch (impl
->type
) {
1155 case ANV_FENCE_TYPE_NONE
:
1156 /* Dummy. Nothing to do */
1159 case ANV_FENCE_TYPE_BO
:
1160 anv_bo_pool_free(&device
->batch_bo_pool
, impl
->bo
.bo
);
1163 case ANV_FENCE_TYPE_WSI_BO
:
1164 anv_device_release_bo(device
, impl
->bo
.bo
);
1167 case ANV_FENCE_TYPE_SYNCOBJ
:
1168 anv_gem_syncobj_destroy(device
, impl
->syncobj
);
1171 case ANV_FENCE_TYPE_WSI
:
1172 impl
->fence_wsi
->destroy(impl
->fence_wsi
);
1176 unreachable("Invalid fence type");
1179 impl
->type
= ANV_FENCE_TYPE_NONE
;
1183 anv_fence_reset_temporary(struct anv_device
*device
,
1184 struct anv_fence
*fence
)
1186 if (fence
->temporary
.type
== ANV_FENCE_TYPE_NONE
)
1189 anv_fence_impl_cleanup(device
, &fence
->temporary
);
1192 void anv_DestroyFence(
1195 const VkAllocationCallbacks
* pAllocator
)
1197 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1198 ANV_FROM_HANDLE(anv_fence
, fence
, _fence
);
1203 anv_fence_impl_cleanup(device
, &fence
->temporary
);
1204 anv_fence_impl_cleanup(device
, &fence
->permanent
);
1206 vk_object_base_finish(&fence
->base
);
1207 vk_free2(&device
->vk
.alloc
, pAllocator
, fence
);
1210 VkResult
anv_ResetFences(
1212 uint32_t fenceCount
,
1213 const VkFence
* pFences
)
1215 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1217 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1218 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1220 /* From the Vulkan 1.0.53 spec:
1222 * "If any member of pFences currently has its payload imported with
1223 * temporary permanence, that fence’s prior permanent payload is
1224 * first restored. The remaining operations described therefore
1225 * operate on the restored payload.
1227 anv_fence_reset_temporary(device
, fence
);
1229 struct anv_fence_impl
*impl
= &fence
->permanent
;
1231 switch (impl
->type
) {
1232 case ANV_FENCE_TYPE_BO
:
1233 impl
->bo
.state
= ANV_BO_FENCE_STATE_RESET
;
1236 case ANV_FENCE_TYPE_SYNCOBJ
:
1237 anv_gem_syncobj_reset(device
, impl
->syncobj
);
1241 unreachable("Invalid fence type");
1248 VkResult
anv_GetFenceStatus(
1252 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1253 ANV_FROM_HANDLE(anv_fence
, fence
, _fence
);
1255 if (anv_device_is_lost(device
))
1256 return VK_ERROR_DEVICE_LOST
;
1258 struct anv_fence_impl
*impl
=
1259 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1260 &fence
->temporary
: &fence
->permanent
;
1262 switch (impl
->type
) {
1263 case ANV_FENCE_TYPE_BO
:
1264 case ANV_FENCE_TYPE_WSI_BO
:
1265 switch (impl
->bo
.state
) {
1266 case ANV_BO_FENCE_STATE_RESET
:
1267 /* If it hasn't even been sent off to the GPU yet, it's not ready */
1268 return VK_NOT_READY
;
1270 case ANV_BO_FENCE_STATE_SIGNALED
:
1271 /* It's been signaled, return success */
1274 case ANV_BO_FENCE_STATE_SUBMITTED
: {
1275 VkResult result
= anv_device_bo_busy(device
, impl
->bo
.bo
);
1276 if (result
== VK_SUCCESS
) {
1277 impl
->bo
.state
= ANV_BO_FENCE_STATE_SIGNALED
;
1284 unreachable("Invalid fence status");
1287 case ANV_FENCE_TYPE_SYNCOBJ
: {
1288 int ret
= anv_gem_syncobj_wait(device
, &impl
->syncobj
, 1, 0, true);
1290 if (errno
== ETIME
) {
1291 return VK_NOT_READY
;
1293 /* We don't know the real error. */
1294 return anv_device_set_lost(device
, "drm_syncobj_wait failed: %m");
1302 unreachable("Invalid fence type");
1307 anv_wait_for_syncobj_fences(struct anv_device
*device
,
1308 uint32_t fenceCount
,
1309 const VkFence
*pFences
,
1311 uint64_t abs_timeout_ns
)
1313 uint32_t *syncobjs
= vk_zalloc(&device
->vk
.alloc
,
1314 sizeof(*syncobjs
) * fenceCount
, 8,
1315 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND
);
1317 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1319 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1320 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1321 assert(fence
->permanent
.type
== ANV_FENCE_TYPE_SYNCOBJ
);
1323 struct anv_fence_impl
*impl
=
1324 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1325 &fence
->temporary
: &fence
->permanent
;
1327 assert(impl
->type
== ANV_FENCE_TYPE_SYNCOBJ
);
1328 syncobjs
[i
] = impl
->syncobj
;
1331 /* The gem_syncobj_wait ioctl may return early due to an inherent
1332 * limitation in the way it computes timeouts. Loop until we've actually
1333 * passed the timeout.
1337 ret
= anv_gem_syncobj_wait(device
, syncobjs
, fenceCount
,
1338 abs_timeout_ns
, waitAll
);
1339 } while (ret
== -1 && errno
== ETIME
&& anv_gettime_ns() < abs_timeout_ns
);
1341 vk_free(&device
->vk
.alloc
, syncobjs
);
1344 if (errno
== ETIME
) {
1347 /* We don't know the real error. */
1348 return anv_device_set_lost(device
, "drm_syncobj_wait failed: %m");
1356 anv_wait_for_bo_fences(struct anv_device
*device
,
1357 uint32_t fenceCount
,
1358 const VkFence
*pFences
,
1360 uint64_t abs_timeout_ns
)
1362 VkResult result
= VK_SUCCESS
;
1363 uint32_t pending_fences
= fenceCount
;
1364 while (pending_fences
) {
1366 bool signaled_fences
= false;
1367 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1368 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1370 struct anv_fence_impl
*impl
=
1371 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1372 &fence
->temporary
: &fence
->permanent
;
1373 assert(impl
->type
== ANV_FENCE_TYPE_BO
||
1374 impl
->type
== ANV_FENCE_TYPE_WSI_BO
);
1376 switch (impl
->bo
.state
) {
1377 case ANV_BO_FENCE_STATE_RESET
:
1378 /* This fence hasn't been submitted yet, we'll catch it the next
1379 * time around. Yes, this may mean we dead-loop but, short of
1380 * lots of locking and a condition variable, there's not much that
1381 * we can do about that.
1386 case ANV_BO_FENCE_STATE_SIGNALED
:
1387 /* This fence is not pending. If waitAll isn't set, we can return
1388 * early. Otherwise, we have to keep going.
1391 result
= VK_SUCCESS
;
1396 case ANV_BO_FENCE_STATE_SUBMITTED
:
1397 /* These are the fences we really care about. Go ahead and wait
1398 * on it until we hit a timeout.
1400 result
= anv_device_wait(device
, impl
->bo
.bo
,
1401 anv_get_relative_timeout(abs_timeout_ns
));
1404 impl
->bo
.state
= ANV_BO_FENCE_STATE_SIGNALED
;
1405 signaled_fences
= true;
1419 if (pending_fences
&& !signaled_fences
) {
1420 /* If we've hit this then someone decided to vkWaitForFences before
1421 * they've actually submitted any of them to a queue. This is a
1422 * fairly pessimal case, so it's ok to lock here and use a standard
1423 * pthreads condition variable.
1425 pthread_mutex_lock(&device
->mutex
);
1427 /* It's possible that some of the fences have changed state since the
1428 * last time we checked. Now that we have the lock, check for
1429 * pending fences again and don't wait if it's changed.
1431 uint32_t now_pending_fences
= 0;
1432 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1433 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1434 if (fence
->permanent
.bo
.state
== ANV_BO_FENCE_STATE_RESET
)
1435 now_pending_fences
++;
1437 assert(now_pending_fences
<= pending_fences
);
1439 if (now_pending_fences
== pending_fences
) {
1440 struct timespec abstime
= {
1441 .tv_sec
= abs_timeout_ns
/ NSEC_PER_SEC
,
1442 .tv_nsec
= abs_timeout_ns
% NSEC_PER_SEC
,
1446 ret
= pthread_cond_timedwait(&device
->queue_submit
,
1447 &device
->mutex
, &abstime
);
1448 assert(ret
!= EINVAL
);
1449 if (anv_gettime_ns() >= abs_timeout_ns
) {
1450 pthread_mutex_unlock(&device
->mutex
);
1451 result
= VK_TIMEOUT
;
1456 pthread_mutex_unlock(&device
->mutex
);
1461 if (anv_device_is_lost(device
))
1462 return VK_ERROR_DEVICE_LOST
;
1468 anv_wait_for_wsi_fence(struct anv_device
*device
,
1469 struct anv_fence_impl
*impl
,
1470 uint64_t abs_timeout
)
1472 return impl
->fence_wsi
->wait(impl
->fence_wsi
, abs_timeout
);
1476 anv_wait_for_fences(struct anv_device
*device
,
1477 uint32_t fenceCount
,
1478 const VkFence
*pFences
,
1480 uint64_t abs_timeout
)
1482 VkResult result
= VK_SUCCESS
;
1484 if (fenceCount
<= 1 || waitAll
) {
1485 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1486 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1487 struct anv_fence_impl
*impl
=
1488 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1489 &fence
->temporary
: &fence
->permanent
;
1491 switch (impl
->type
) {
1492 case ANV_FENCE_TYPE_BO
:
1493 case ANV_FENCE_TYPE_WSI_BO
:
1494 result
= anv_wait_for_bo_fences(device
, 1, &pFences
[i
],
1497 case ANV_FENCE_TYPE_SYNCOBJ
:
1498 result
= anv_wait_for_syncobj_fences(device
, 1, &pFences
[i
],
1501 case ANV_FENCE_TYPE_WSI
:
1502 result
= anv_wait_for_wsi_fence(device
, impl
, abs_timeout
);
1504 case ANV_FENCE_TYPE_NONE
:
1505 result
= VK_SUCCESS
;
1508 if (result
!= VK_SUCCESS
)
1513 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1514 if (anv_wait_for_fences(device
, 1, &pFences
[i
], true, 0) == VK_SUCCESS
)
1517 } while (anv_gettime_ns() < abs_timeout
);
1518 result
= VK_TIMEOUT
;
1523 static bool anv_all_fences_syncobj(uint32_t fenceCount
, const VkFence
*pFences
)
1525 for (uint32_t i
= 0; i
< fenceCount
; ++i
) {
1526 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1527 struct anv_fence_impl
*impl
=
1528 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1529 &fence
->temporary
: &fence
->permanent
;
1530 if (impl
->type
!= ANV_FENCE_TYPE_SYNCOBJ
)
1536 static bool anv_all_fences_bo(uint32_t fenceCount
, const VkFence
*pFences
)
1538 for (uint32_t i
= 0; i
< fenceCount
; ++i
) {
1539 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1540 struct anv_fence_impl
*impl
=
1541 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1542 &fence
->temporary
: &fence
->permanent
;
1543 if (impl
->type
!= ANV_FENCE_TYPE_BO
&&
1544 impl
->type
!= ANV_FENCE_TYPE_WSI_BO
)
1550 VkResult
anv_WaitForFences(
1552 uint32_t fenceCount
,
1553 const VkFence
* pFences
,
1557 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1562 if (anv_device_is_lost(device
))
1563 return VK_ERROR_DEVICE_LOST
;
1565 uint64_t abs_timeout
= anv_get_absolute_timeout(timeout
);
1566 if (anv_all_fences_syncobj(fenceCount
, pFences
)) {
1567 return anv_wait_for_syncobj_fences(device
, fenceCount
, pFences
,
1568 waitAll
, abs_timeout
);
1569 } else if (anv_all_fences_bo(fenceCount
, pFences
)) {
1570 return anv_wait_for_bo_fences(device
, fenceCount
, pFences
,
1571 waitAll
, abs_timeout
);
1573 return anv_wait_for_fences(device
, fenceCount
, pFences
,
1574 waitAll
, abs_timeout
);
1578 void anv_GetPhysicalDeviceExternalFenceProperties(
1579 VkPhysicalDevice physicalDevice
,
1580 const VkPhysicalDeviceExternalFenceInfo
* pExternalFenceInfo
,
1581 VkExternalFenceProperties
* pExternalFenceProperties
)
1583 ANV_FROM_HANDLE(anv_physical_device
, device
, physicalDevice
);
1585 switch (pExternalFenceInfo
->handleType
) {
1586 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1587 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
:
1588 if (device
->has_syncobj_wait
) {
1589 pExternalFenceProperties
->exportFromImportedHandleTypes
=
1590 VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
|
1591 VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
;
1592 pExternalFenceProperties
->compatibleHandleTypes
=
1593 VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
|
1594 VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
;
1595 pExternalFenceProperties
->externalFenceFeatures
=
1596 VK_EXTERNAL_FENCE_FEATURE_EXPORTABLE_BIT
|
1597 VK_EXTERNAL_FENCE_FEATURE_IMPORTABLE_BIT
;
1606 pExternalFenceProperties
->exportFromImportedHandleTypes
= 0;
1607 pExternalFenceProperties
->compatibleHandleTypes
= 0;
1608 pExternalFenceProperties
->externalFenceFeatures
= 0;
1611 VkResult
anv_ImportFenceFdKHR(
1613 const VkImportFenceFdInfoKHR
* pImportFenceFdInfo
)
1615 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1616 ANV_FROM_HANDLE(anv_fence
, fence
, pImportFenceFdInfo
->fence
);
1617 int fd
= pImportFenceFdInfo
->fd
;
1619 assert(pImportFenceFdInfo
->sType
==
1620 VK_STRUCTURE_TYPE_IMPORT_FENCE_FD_INFO_KHR
);
1622 struct anv_fence_impl new_impl
= {
1623 .type
= ANV_FENCE_TYPE_NONE
,
1626 switch (pImportFenceFdInfo
->handleType
) {
1627 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1628 new_impl
.type
= ANV_FENCE_TYPE_SYNCOBJ
;
1630 new_impl
.syncobj
= anv_gem_syncobj_fd_to_handle(device
, fd
);
1631 if (!new_impl
.syncobj
)
1632 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1636 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
:
1637 /* Sync files are a bit tricky. Because we want to continue using the
1638 * syncobj implementation of WaitForFences, we don't use the sync file
1639 * directly but instead import it into a syncobj.
1641 new_impl
.type
= ANV_FENCE_TYPE_SYNCOBJ
;
1643 new_impl
.syncobj
= anv_gem_syncobj_create(device
, 0);
1644 if (!new_impl
.syncobj
)
1645 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1647 if (anv_gem_syncobj_import_sync_file(device
, new_impl
.syncobj
, fd
)) {
1648 anv_gem_syncobj_destroy(device
, new_impl
.syncobj
);
1649 return vk_errorf(device
, NULL
, VK_ERROR_INVALID_EXTERNAL_HANDLE
,
1650 "syncobj sync file import failed: %m");
1655 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1658 /* From the Vulkan 1.0.53 spec:
1660 * "Importing a fence payload from a file descriptor transfers
1661 * ownership of the file descriptor from the application to the
1662 * Vulkan implementation. The application must not perform any
1663 * operations on the file descriptor after a successful import."
1665 * If the import fails, we leave the file descriptor open.
1669 if (pImportFenceFdInfo
->flags
& VK_FENCE_IMPORT_TEMPORARY_BIT
) {
1670 anv_fence_impl_cleanup(device
, &fence
->temporary
);
1671 fence
->temporary
= new_impl
;
1673 anv_fence_impl_cleanup(device
, &fence
->permanent
);
1674 fence
->permanent
= new_impl
;
1680 VkResult
anv_GetFenceFdKHR(
1682 const VkFenceGetFdInfoKHR
* pGetFdInfo
,
1685 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1686 ANV_FROM_HANDLE(anv_fence
, fence
, pGetFdInfo
->fence
);
1688 assert(pGetFdInfo
->sType
== VK_STRUCTURE_TYPE_FENCE_GET_FD_INFO_KHR
);
1690 struct anv_fence_impl
*impl
=
1691 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1692 &fence
->temporary
: &fence
->permanent
;
1694 assert(impl
->type
== ANV_FENCE_TYPE_SYNCOBJ
);
1695 switch (pGetFdInfo
->handleType
) {
1696 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
: {
1697 int fd
= anv_gem_syncobj_handle_to_fd(device
, impl
->syncobj
);
1699 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
1705 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
: {
1706 int fd
= anv_gem_syncobj_export_sync_file(device
, impl
->syncobj
);
1708 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
1715 unreachable("Invalid fence export handle type");
1718 /* From the Vulkan 1.0.53 spec:
1720 * "Export operations have the same transference as the specified handle
1721 * type’s import operations. [...] If the fence was using a
1722 * temporarily imported payload, the fence’s prior permanent payload
1725 if (impl
== &fence
->temporary
)
1726 anv_fence_impl_cleanup(device
, impl
);
1731 // Queue semaphore functions
1733 static VkSemaphoreTypeKHR
1734 get_semaphore_type(const void *pNext
, uint64_t *initial_value
)
1736 const VkSemaphoreTypeCreateInfoKHR
*type_info
=
1737 vk_find_struct_const(pNext
, SEMAPHORE_TYPE_CREATE_INFO_KHR
);
1740 return VK_SEMAPHORE_TYPE_BINARY_KHR
;
1743 *initial_value
= type_info
->initialValue
;
1744 return type_info
->semaphoreType
;
1748 binary_semaphore_create(struct anv_device
*device
,
1749 struct anv_semaphore_impl
*impl
,
1752 if (device
->physical
->has_syncobj
) {
1753 impl
->type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
;
1754 impl
->syncobj
= anv_gem_syncobj_create(device
, 0);
1756 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1759 impl
->type
= ANV_SEMAPHORE_TYPE_BO
;
1761 anv_device_alloc_bo(device
, 4096,
1762 ANV_BO_ALLOC_EXTERNAL
|
1763 ANV_BO_ALLOC_IMPLICIT_SYNC
,
1764 0 /* explicit_address */,
1766 /* If we're going to use this as a fence, we need to *not* have the
1767 * EXEC_OBJECT_ASYNC bit set.
1769 assert(!(impl
->bo
->flags
& EXEC_OBJECT_ASYNC
));
1775 timeline_semaphore_create(struct anv_device
*device
,
1776 struct anv_semaphore_impl
*impl
,
1777 uint64_t initial_value
)
1779 impl
->type
= ANV_SEMAPHORE_TYPE_TIMELINE
;
1780 anv_timeline_init(device
, &impl
->timeline
, initial_value
);
1784 VkResult
anv_CreateSemaphore(
1786 const VkSemaphoreCreateInfo
* pCreateInfo
,
1787 const VkAllocationCallbacks
* pAllocator
,
1788 VkSemaphore
* pSemaphore
)
1790 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1791 struct anv_semaphore
*semaphore
;
1793 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO
);
1795 uint64_t timeline_value
= 0;
1796 VkSemaphoreTypeKHR sem_type
= get_semaphore_type(pCreateInfo
->pNext
, &timeline_value
);
1798 semaphore
= vk_alloc(&device
->vk
.alloc
, sizeof(*semaphore
), 8,
1799 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE
);
1800 if (semaphore
== NULL
)
1801 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1803 vk_object_base_init(&device
->vk
, &semaphore
->base
, VK_OBJECT_TYPE_SEMAPHORE
);
1805 p_atomic_set(&semaphore
->refcount
, 1);
1807 const VkExportSemaphoreCreateInfo
*export
=
1808 vk_find_struct_const(pCreateInfo
->pNext
, EXPORT_SEMAPHORE_CREATE_INFO
);
1809 VkExternalSemaphoreHandleTypeFlags handleTypes
=
1810 export
? export
->handleTypes
: 0;
1813 if (handleTypes
== 0) {
1814 if (sem_type
== VK_SEMAPHORE_TYPE_BINARY_KHR
)
1815 result
= binary_semaphore_create(device
, &semaphore
->permanent
, false);
1817 result
= timeline_semaphore_create(device
, &semaphore
->permanent
, timeline_value
);
1818 if (result
!= VK_SUCCESS
) {
1819 vk_free2(&device
->vk
.alloc
, pAllocator
, semaphore
);
1822 } else if (handleTypes
& VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
) {
1823 assert(handleTypes
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
);
1824 assert(sem_type
== VK_SEMAPHORE_TYPE_BINARY_KHR
);
1825 result
= binary_semaphore_create(device
, &semaphore
->permanent
, true);
1826 if (result
!= VK_SUCCESS
) {
1827 vk_free2(&device
->vk
.alloc
, pAllocator
, semaphore
);
1830 } else if (handleTypes
& VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
) {
1831 assert(handleTypes
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
);
1832 assert(sem_type
== VK_SEMAPHORE_TYPE_BINARY_KHR
);
1833 if (device
->physical
->has_syncobj
) {
1834 semaphore
->permanent
.type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
;
1835 semaphore
->permanent
.syncobj
= anv_gem_syncobj_create(device
, 0);
1836 if (!semaphore
->permanent
.syncobj
) {
1837 vk_free2(&device
->vk
.alloc
, pAllocator
, semaphore
);
1838 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1841 semaphore
->permanent
.type
= ANV_SEMAPHORE_TYPE_SYNC_FILE
;
1842 semaphore
->permanent
.fd
= -1;
1845 assert(!"Unknown handle type");
1846 vk_free2(&device
->vk
.alloc
, pAllocator
, semaphore
);
1847 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1850 semaphore
->temporary
.type
= ANV_SEMAPHORE_TYPE_NONE
;
1852 *pSemaphore
= anv_semaphore_to_handle(semaphore
);
1858 anv_semaphore_impl_cleanup(struct anv_device
*device
,
1859 struct anv_semaphore_impl
*impl
)
1861 switch (impl
->type
) {
1862 case ANV_SEMAPHORE_TYPE_NONE
:
1863 case ANV_SEMAPHORE_TYPE_DUMMY
:
1864 /* Dummy. Nothing to do */
1867 case ANV_SEMAPHORE_TYPE_BO
:
1868 case ANV_SEMAPHORE_TYPE_WSI_BO
:
1869 anv_device_release_bo(device
, impl
->bo
);
1872 case ANV_SEMAPHORE_TYPE_SYNC_FILE
:
1877 case ANV_SEMAPHORE_TYPE_TIMELINE
:
1878 anv_timeline_finish(device
, &impl
->timeline
);
1881 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
:
1882 anv_gem_syncobj_destroy(device
, impl
->syncobj
);
1886 unreachable("Invalid semaphore type");
1889 impl
->type
= ANV_SEMAPHORE_TYPE_NONE
;
1893 anv_semaphore_reset_temporary(struct anv_device
*device
,
1894 struct anv_semaphore
*semaphore
)
1896 if (semaphore
->temporary
.type
== ANV_SEMAPHORE_TYPE_NONE
)
1899 anv_semaphore_impl_cleanup(device
, &semaphore
->temporary
);
1902 static struct anv_semaphore
*
1903 anv_semaphore_ref(struct anv_semaphore
*semaphore
)
1905 assert(semaphore
->refcount
);
1906 p_atomic_inc(&semaphore
->refcount
);
1911 anv_semaphore_unref(struct anv_device
*device
, struct anv_semaphore
*semaphore
)
1913 if (!p_atomic_dec_zero(&semaphore
->refcount
))
1916 anv_semaphore_impl_cleanup(device
, &semaphore
->temporary
);
1917 anv_semaphore_impl_cleanup(device
, &semaphore
->permanent
);
1919 vk_object_base_finish(&semaphore
->base
);
1920 vk_free(&device
->vk
.alloc
, semaphore
);
1923 void anv_DestroySemaphore(
1925 VkSemaphore _semaphore
,
1926 const VkAllocationCallbacks
* pAllocator
)
1928 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1929 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, _semaphore
);
1931 if (semaphore
== NULL
)
1934 anv_semaphore_unref(device
, semaphore
);
1937 void anv_GetPhysicalDeviceExternalSemaphoreProperties(
1938 VkPhysicalDevice physicalDevice
,
1939 const VkPhysicalDeviceExternalSemaphoreInfo
* pExternalSemaphoreInfo
,
1940 VkExternalSemaphoreProperties
* pExternalSemaphoreProperties
)
1942 ANV_FROM_HANDLE(anv_physical_device
, device
, physicalDevice
);
1944 VkSemaphoreTypeKHR sem_type
=
1945 get_semaphore_type(pExternalSemaphoreInfo
->pNext
, NULL
);
1947 switch (pExternalSemaphoreInfo
->handleType
) {
1948 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1949 /* Timeline semaphores are not exportable. */
1950 if (sem_type
== VK_SEMAPHORE_TYPE_TIMELINE_KHR
)
1952 pExternalSemaphoreProperties
->exportFromImportedHandleTypes
=
1953 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
;
1954 pExternalSemaphoreProperties
->compatibleHandleTypes
=
1955 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
;
1956 pExternalSemaphoreProperties
->externalSemaphoreFeatures
=
1957 VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT
|
1958 VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT
;
1961 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
:
1962 if (sem_type
== VK_SEMAPHORE_TYPE_TIMELINE_KHR
)
1964 if (!device
->has_exec_fence
)
1966 pExternalSemaphoreProperties
->exportFromImportedHandleTypes
=
1967 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
;
1968 pExternalSemaphoreProperties
->compatibleHandleTypes
=
1969 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
;
1970 pExternalSemaphoreProperties
->externalSemaphoreFeatures
=
1971 VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT
|
1972 VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT
;
1979 pExternalSemaphoreProperties
->exportFromImportedHandleTypes
= 0;
1980 pExternalSemaphoreProperties
->compatibleHandleTypes
= 0;
1981 pExternalSemaphoreProperties
->externalSemaphoreFeatures
= 0;
1984 VkResult
anv_ImportSemaphoreFdKHR(
1986 const VkImportSemaphoreFdInfoKHR
* pImportSemaphoreFdInfo
)
1988 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1989 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pImportSemaphoreFdInfo
->semaphore
);
1990 int fd
= pImportSemaphoreFdInfo
->fd
;
1992 struct anv_semaphore_impl new_impl
= {
1993 .type
= ANV_SEMAPHORE_TYPE_NONE
,
1996 switch (pImportSemaphoreFdInfo
->handleType
) {
1997 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1998 if (device
->physical
->has_syncobj
) {
1999 new_impl
.type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
;
2001 new_impl
.syncobj
= anv_gem_syncobj_fd_to_handle(device
, fd
);
2002 if (!new_impl
.syncobj
)
2003 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
2005 new_impl
.type
= ANV_SEMAPHORE_TYPE_BO
;
2007 VkResult result
= anv_device_import_bo(device
, fd
,
2008 ANV_BO_ALLOC_EXTERNAL
|
2009 ANV_BO_ALLOC_IMPLICIT_SYNC
,
2010 0 /* client_address */,
2012 if (result
!= VK_SUCCESS
)
2015 if (new_impl
.bo
->size
< 4096) {
2016 anv_device_release_bo(device
, new_impl
.bo
);
2017 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
2020 /* If we're going to use this as a fence, we need to *not* have the
2021 * EXEC_OBJECT_ASYNC bit set.
2023 assert(!(new_impl
.bo
->flags
& EXEC_OBJECT_ASYNC
));
2026 /* From the Vulkan spec:
2028 * "Importing semaphore state from a file descriptor transfers
2029 * ownership of the file descriptor from the application to the
2030 * Vulkan implementation. The application must not perform any
2031 * operations on the file descriptor after a successful import."
2033 * If the import fails, we leave the file descriptor open.
2038 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
:
2039 if (device
->physical
->has_syncobj
) {
2040 new_impl
= (struct anv_semaphore_impl
) {
2041 .type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
,
2042 .syncobj
= anv_gem_syncobj_create(device
, 0),
2044 if (!new_impl
.syncobj
)
2045 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
2046 if (anv_gem_syncobj_import_sync_file(device
, new_impl
.syncobj
, fd
)) {
2047 anv_gem_syncobj_destroy(device
, new_impl
.syncobj
);
2048 return vk_errorf(device
, NULL
, VK_ERROR_INVALID_EXTERNAL_HANDLE
,
2049 "syncobj sync file import failed: %m");
2051 /* Ownership of the FD is transfered to Anv. Since we don't need it
2052 * anymore because the associated fence has been put into a syncobj,
2053 * we must close the FD.
2057 new_impl
= (struct anv_semaphore_impl
) {
2058 .type
= ANV_SEMAPHORE_TYPE_SYNC_FILE
,
2065 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
2068 if (pImportSemaphoreFdInfo
->flags
& VK_SEMAPHORE_IMPORT_TEMPORARY_BIT
) {
2069 anv_semaphore_impl_cleanup(device
, &semaphore
->temporary
);
2070 semaphore
->temporary
= new_impl
;
2072 anv_semaphore_impl_cleanup(device
, &semaphore
->permanent
);
2073 semaphore
->permanent
= new_impl
;
2079 VkResult
anv_GetSemaphoreFdKHR(
2081 const VkSemaphoreGetFdInfoKHR
* pGetFdInfo
,
2084 ANV_FROM_HANDLE(anv_device
, device
, _device
);
2085 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pGetFdInfo
->semaphore
);
2089 assert(pGetFdInfo
->sType
== VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR
);
2091 struct anv_semaphore_impl
*impl
=
2092 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
2093 &semaphore
->temporary
: &semaphore
->permanent
;
2095 switch (impl
->type
) {
2096 case ANV_SEMAPHORE_TYPE_BO
:
2097 result
= anv_device_export_bo(device
, impl
->bo
, pFd
);
2098 if (result
!= VK_SUCCESS
)
2102 case ANV_SEMAPHORE_TYPE_SYNC_FILE
: {
2103 /* There's a potential race here with vkQueueSubmit if you are trying
2104 * to export a semaphore Fd while the queue submit is still happening.
2105 * This can happen if we see all dependencies get resolved via timeline
2106 * semaphore waits completing before the execbuf completes and we
2107 * process the resulting out fence. To work around this, take a lock
2108 * around grabbing the fd.
2110 pthread_mutex_lock(&device
->mutex
);
2112 /* From the Vulkan 1.0.53 spec:
2114 * "...exporting a semaphore payload to a handle with copy
2115 * transference has the same side effects on the source
2116 * semaphore’s payload as executing a semaphore wait operation."
2118 * In other words, it may still be a SYNC_FD semaphore, but it's now
2119 * considered to have been waited on and no longer has a sync file
2125 pthread_mutex_unlock(&device
->mutex
);
2127 /* There are two reasons why this could happen:
2129 * 1) The user is trying to export without submitting something that
2130 * signals the semaphore. If this is the case, it's their bug so
2131 * what we return here doesn't matter.
2133 * 2) The kernel didn't give us a file descriptor. The most likely
2134 * reason for this is running out of file descriptors.
2137 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
2143 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
:
2144 if (pGetFdInfo
->handleType
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
)
2145 fd
= anv_gem_syncobj_export_sync_file(device
, impl
->syncobj
);
2147 assert(pGetFdInfo
->handleType
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
);
2148 fd
= anv_gem_syncobj_handle_to_fd(device
, impl
->syncobj
);
2151 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
2156 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
2159 /* From the Vulkan 1.0.53 spec:
2161 * "Export operations have the same transference as the specified handle
2162 * type’s import operations. [...] If the semaphore was using a
2163 * temporarily imported payload, the semaphore’s prior permanent payload
2166 if (impl
== &semaphore
->temporary
)
2167 anv_semaphore_impl_cleanup(device
, impl
);
2172 VkResult
anv_GetSemaphoreCounterValue(
2174 VkSemaphore _semaphore
,
2177 ANV_FROM_HANDLE(anv_device
, device
, _device
);
2178 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, _semaphore
);
2180 struct anv_semaphore_impl
*impl
=
2181 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
2182 &semaphore
->temporary
: &semaphore
->permanent
;
2184 switch (impl
->type
) {
2185 case ANV_SEMAPHORE_TYPE_TIMELINE
: {
2186 pthread_mutex_lock(&device
->mutex
);
2187 *pValue
= impl
->timeline
.highest_past
;
2188 pthread_mutex_unlock(&device
->mutex
);
2193 unreachable("Invalid semaphore type");
2198 anv_timeline_wait_locked(struct anv_device
*device
,
2199 struct anv_timeline
*timeline
,
2200 uint64_t serial
, uint64_t abs_timeout_ns
)
2202 /* Wait on the queue_submit condition variable until the timeline has a
2203 * time point pending that's at least as high as serial.
2205 while (timeline
->highest_pending
< serial
) {
2206 struct timespec abstime
= {
2207 .tv_sec
= abs_timeout_ns
/ NSEC_PER_SEC
,
2208 .tv_nsec
= abs_timeout_ns
% NSEC_PER_SEC
,
2211 int ret
= pthread_cond_timedwait(&device
->queue_submit
,
2212 &device
->mutex
, &abstime
);
2213 assert(ret
!= EINVAL
);
2214 if (anv_gettime_ns() >= abs_timeout_ns
&&
2215 timeline
->highest_pending
< serial
)
2220 VkResult result
= anv_timeline_gc_locked(device
, timeline
);
2221 if (result
!= VK_SUCCESS
)
2224 if (timeline
->highest_past
>= serial
)
2227 /* If we got here, our earliest time point has a busy BO */
2228 struct anv_timeline_point
*point
=
2229 list_first_entry(&timeline
->points
,
2230 struct anv_timeline_point
, link
);
2232 /* Drop the lock while we wait. */
2234 pthread_mutex_unlock(&device
->mutex
);
2236 result
= anv_device_wait(device
, point
->bo
,
2237 anv_get_relative_timeout(abs_timeout_ns
));
2239 /* Pick the mutex back up */
2240 pthread_mutex_lock(&device
->mutex
);
2243 /* This covers both VK_TIMEOUT and VK_ERROR_DEVICE_LOST */
2244 if (result
!= VK_SUCCESS
)
2250 anv_timelines_wait(struct anv_device
*device
,
2251 struct anv_timeline
**timelines
,
2252 const uint64_t *serials
,
2253 uint32_t n_timelines
,
2255 uint64_t abs_timeout_ns
)
2257 if (!wait_all
&& n_timelines
> 1) {
2258 pthread_mutex_lock(&device
->mutex
);
2262 for (uint32_t i
= 0; i
< n_timelines
; i
++) {
2264 anv_timeline_wait_locked(device
, timelines
[i
], serials
[i
], 0);
2265 if (result
!= VK_TIMEOUT
)
2269 if (result
!= VK_TIMEOUT
||
2270 anv_gettime_ns() >= abs_timeout_ns
) {
2271 pthread_mutex_unlock(&device
->mutex
);
2275 /* If none of them are ready do a short wait so we don't completely
2276 * spin while holding the lock. The 10us is completely arbitrary.
2278 uint64_t abs_short_wait_ns
=
2279 anv_get_absolute_timeout(
2280 MIN2((anv_gettime_ns() - abs_timeout_ns
) / 10, 10 * 1000));
2281 struct timespec abstime
= {
2282 .tv_sec
= abs_short_wait_ns
/ NSEC_PER_SEC
,
2283 .tv_nsec
= abs_short_wait_ns
% NSEC_PER_SEC
,
2286 ret
= pthread_cond_timedwait(&device
->queue_submit
,
2287 &device
->mutex
, &abstime
);
2288 assert(ret
!= EINVAL
);
2291 VkResult result
= VK_SUCCESS
;
2292 pthread_mutex_lock(&device
->mutex
);
2293 for (uint32_t i
= 0; i
< n_timelines
; i
++) {
2295 anv_timeline_wait_locked(device
, timelines
[i
],
2296 serials
[i
], abs_timeout_ns
);
2297 if (result
!= VK_SUCCESS
)
2300 pthread_mutex_unlock(&device
->mutex
);
2305 VkResult
anv_WaitSemaphores(
2307 const VkSemaphoreWaitInfoKHR
* pWaitInfo
,
2310 ANV_FROM_HANDLE(anv_device
, device
, _device
);
2315 struct anv_timeline
**timelines
=
2316 vk_alloc(&device
->vk
.alloc
,
2317 pWaitInfo
->semaphoreCount
* sizeof(*timelines
),
2318 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND
);
2320 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
2322 uint64_t *values
= vk_alloc(&device
->vk
.alloc
,
2323 pWaitInfo
->semaphoreCount
* sizeof(*values
),
2324 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND
);
2326 vk_free(&device
->vk
.alloc
, timelines
);
2327 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
2330 uint32_t handle_count
= 0;
2331 for (uint32_t i
= 0; i
< pWaitInfo
->semaphoreCount
; i
++) {
2332 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pWaitInfo
->pSemaphores
[i
]);
2333 struct anv_semaphore_impl
*impl
=
2334 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
2335 &semaphore
->temporary
: &semaphore
->permanent
;
2337 assert(impl
->type
== ANV_SEMAPHORE_TYPE_TIMELINE
);
2339 if (pWaitInfo
->pValues
[i
] == 0)
2342 timelines
[handle_count
] = &impl
->timeline
;
2343 values
[handle_count
] = pWaitInfo
->pValues
[i
];
2347 VkResult result
= VK_SUCCESS
;
2348 if (handle_count
> 0) {
2349 result
= anv_timelines_wait(device
, timelines
, values
, handle_count
,
2350 !(pWaitInfo
->flags
& VK_SEMAPHORE_WAIT_ANY_BIT_KHR
),
2351 anv_get_absolute_timeout(timeout
));
2354 vk_free(&device
->vk
.alloc
, timelines
);
2355 vk_free(&device
->vk
.alloc
, values
);
2360 VkResult
anv_SignalSemaphore(
2362 const VkSemaphoreSignalInfoKHR
* pSignalInfo
)
2364 ANV_FROM_HANDLE(anv_device
, device
, _device
);
2365 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pSignalInfo
->semaphore
);
2367 struct anv_semaphore_impl
*impl
=
2368 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
2369 &semaphore
->temporary
: &semaphore
->permanent
;
2371 switch (impl
->type
) {
2372 case ANV_SEMAPHORE_TYPE_TIMELINE
: {
2373 pthread_mutex_lock(&device
->mutex
);
2375 VkResult result
= anv_timeline_gc_locked(device
, &impl
->timeline
);
2377 assert(pSignalInfo
->value
> impl
->timeline
.highest_pending
);
2379 impl
->timeline
.highest_pending
= impl
->timeline
.highest_past
= pSignalInfo
->value
;
2381 if (result
== VK_SUCCESS
)
2382 result
= anv_device_submit_deferred_locked(device
);
2384 pthread_cond_broadcast(&device
->queue_submit
);
2385 pthread_mutex_unlock(&device
->mutex
);
2390 unreachable("Invalid semaphore type");