2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 * This file implements VkQueue, VkFence, and VkSemaphore
32 #include "util/os_file.h"
34 #include "anv_private.h"
37 #include "genxml/gen7_pack.h"
39 uint64_t anv_gettime_ns(void)
41 struct timespec current
;
42 clock_gettime(CLOCK_MONOTONIC
, ¤t
);
43 return (uint64_t)current
.tv_sec
* NSEC_PER_SEC
+ current
.tv_nsec
;
46 uint64_t anv_get_absolute_timeout(uint64_t timeout
)
50 uint64_t current_time
= anv_gettime_ns();
51 uint64_t max_timeout
= (uint64_t) INT64_MAX
- current_time
;
53 timeout
= MIN2(max_timeout
, timeout
);
55 return (current_time
+ timeout
);
58 static int64_t anv_get_relative_timeout(uint64_t abs_timeout
)
60 uint64_t now
= anv_gettime_ns();
62 /* We don't want negative timeouts.
64 * DRM_IOCTL_I915_GEM_WAIT uses a signed 64 bit timeout and is
65 * supposed to block indefinitely timeouts < 0. Unfortunately,
66 * this was broken for a couple of kernel releases. Since there's
67 * no way to know whether or not the kernel we're using is one of
68 * the broken ones, the best we can do is to clamp the timeout to
69 * INT64_MAX. This limits the maximum timeout from 584 years to
70 * 292 years - likely not a big deal.
72 if (abs_timeout
< now
)
75 uint64_t rel_timeout
= abs_timeout
- now
;
76 if (rel_timeout
> (uint64_t) INT64_MAX
)
77 rel_timeout
= INT64_MAX
;
82 static struct anv_semaphore
*anv_semaphore_ref(struct anv_semaphore
*semaphore
);
83 static void anv_semaphore_unref(struct anv_device
*device
, struct anv_semaphore
*semaphore
);
84 static void anv_semaphore_impl_cleanup(struct anv_device
*device
,
85 struct anv_semaphore_impl
*impl
);
88 anv_queue_submit_free(struct anv_device
*device
,
89 struct anv_queue_submit
*submit
)
91 const VkAllocationCallbacks
*alloc
= submit
->alloc
;
93 for (uint32_t i
= 0; i
< submit
->temporary_semaphore_count
; i
++)
94 anv_semaphore_impl_cleanup(device
, &submit
->temporary_semaphores
[i
]);
95 for (uint32_t i
= 0; i
< submit
->sync_fd_semaphore_count
; i
++)
96 anv_semaphore_unref(device
, submit
->sync_fd_semaphores
[i
]);
97 /* Execbuf does not consume the in_fence. It's our job to close it. */
98 if (submit
->in_fence
!= -1)
99 close(submit
->in_fence
);
100 if (submit
->out_fence
!= -1)
101 close(submit
->out_fence
);
102 vk_free(alloc
, submit
->fences
);
103 vk_free(alloc
, submit
->temporary_semaphores
);
104 vk_free(alloc
, submit
->wait_timelines
);
105 vk_free(alloc
, submit
->wait_timeline_values
);
106 vk_free(alloc
, submit
->signal_timelines
);
107 vk_free(alloc
, submit
->signal_timeline_values
);
108 vk_free(alloc
, submit
->fence_bos
);
109 vk_free(alloc
, submit
);
113 anv_queue_submit_ready_locked(struct anv_queue_submit
*submit
)
115 for (uint32_t i
= 0; i
< submit
->wait_timeline_count
; i
++) {
116 if (submit
->wait_timeline_values
[i
] > submit
->wait_timelines
[i
]->highest_pending
)
124 anv_timeline_init(struct anv_device
*device
,
125 struct anv_timeline
*timeline
,
126 uint64_t initial_value
)
128 timeline
->highest_past
=
129 timeline
->highest_pending
= initial_value
;
130 list_inithead(&timeline
->points
);
131 list_inithead(&timeline
->free_points
);
137 anv_timeline_finish(struct anv_device
*device
,
138 struct anv_timeline
*timeline
)
140 list_for_each_entry_safe(struct anv_timeline_point
, point
,
141 &timeline
->free_points
, link
) {
142 list_del(&point
->link
);
143 anv_device_release_bo(device
, point
->bo
);
144 vk_free(&device
->vk
.alloc
, point
);
146 list_for_each_entry_safe(struct anv_timeline_point
, point
,
147 &timeline
->points
, link
) {
148 list_del(&point
->link
);
149 anv_device_release_bo(device
, point
->bo
);
150 vk_free(&device
->vk
.alloc
, point
);
155 anv_timeline_add_point_locked(struct anv_device
*device
,
156 struct anv_timeline
*timeline
,
158 struct anv_timeline_point
**point
)
160 VkResult result
= VK_SUCCESS
;
162 if (list_is_empty(&timeline
->free_points
)) {
164 vk_zalloc(&device
->vk
.alloc
, sizeof(**point
),
165 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE
);
167 result
= vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
168 if (result
== VK_SUCCESS
) {
169 result
= anv_device_alloc_bo(device
, 4096,
170 ANV_BO_ALLOC_EXTERNAL
|
171 ANV_BO_ALLOC_IMPLICIT_SYNC
,
172 0 /* explicit_address */,
174 if (result
!= VK_SUCCESS
)
175 vk_free(&device
->vk
.alloc
, *point
);
178 *point
= list_first_entry(&timeline
->free_points
,
179 struct anv_timeline_point
, link
);
180 list_del(&(*point
)->link
);
183 if (result
== VK_SUCCESS
) {
184 (*point
)->serial
= value
;
185 list_addtail(&(*point
)->link
, &timeline
->points
);
192 anv_timeline_gc_locked(struct anv_device
*device
,
193 struct anv_timeline
*timeline
)
195 list_for_each_entry_safe(struct anv_timeline_point
, point
,
196 &timeline
->points
, link
) {
197 /* timeline->higest_pending is only incremented once submission has
198 * happened. If this point has a greater serial, it means the point
199 * hasn't been submitted yet.
201 if (point
->serial
> timeline
->highest_pending
)
204 /* If someone is waiting on this time point, consider it busy and don't
205 * try to recycle it. There's a slim possibility that it's no longer
206 * busy by the time we look at it but we would be recycling it out from
207 * under a waiter and that can lead to weird races.
209 * We walk the list in-order so if this time point is still busy so is
210 * every following time point
212 assert(point
->waiting
>= 0);
216 /* Garbage collect any signaled point. */
217 VkResult result
= anv_device_bo_busy(device
, point
->bo
);
218 if (result
== VK_NOT_READY
) {
219 /* We walk the list in-order so if this time point is still busy so
220 * is every following time point
223 } else if (result
!= VK_SUCCESS
) {
227 assert(timeline
->highest_past
< point
->serial
);
228 timeline
->highest_past
= point
->serial
;
230 list_del(&point
->link
);
231 list_add(&point
->link
, &timeline
->free_points
);
237 static VkResult
anv_queue_submit_add_fence_bo(struct anv_queue_submit
*submit
,
242 anv_queue_submit_timeline_locked(struct anv_queue
*queue
,
243 struct anv_queue_submit
*submit
)
247 for (uint32_t i
= 0; i
< submit
->wait_timeline_count
; i
++) {
248 struct anv_timeline
*timeline
= submit
->wait_timelines
[i
];
249 uint64_t wait_value
= submit
->wait_timeline_values
[i
];
251 if (timeline
->highest_past
>= wait_value
)
254 list_for_each_entry(struct anv_timeline_point
, point
, &timeline
->points
, link
) {
255 if (point
->serial
< wait_value
)
257 result
= anv_queue_submit_add_fence_bo(submit
, point
->bo
, false);
258 if (result
!= VK_SUCCESS
)
263 for (uint32_t i
= 0; i
< submit
->signal_timeline_count
; i
++) {
264 struct anv_timeline
*timeline
= submit
->signal_timelines
[i
];
265 uint64_t signal_value
= submit
->signal_timeline_values
[i
];
266 struct anv_timeline_point
*point
;
268 result
= anv_timeline_add_point_locked(queue
->device
, timeline
,
269 signal_value
, &point
);
270 if (result
!= VK_SUCCESS
)
273 result
= anv_queue_submit_add_fence_bo(submit
, point
->bo
, true);
274 if (result
!= VK_SUCCESS
)
278 result
= anv_queue_execbuf_locked(queue
, submit
);
280 if (result
== VK_SUCCESS
) {
281 /* Update the pending values in the timeline objects. */
282 for (uint32_t i
= 0; i
< submit
->signal_timeline_count
; i
++) {
283 struct anv_timeline
*timeline
= submit
->signal_timelines
[i
];
284 uint64_t signal_value
= submit
->signal_timeline_values
[i
];
286 assert(signal_value
> timeline
->highest_pending
);
287 timeline
->highest_pending
= signal_value
;
290 /* Update signaled semaphores backed by syncfd. */
291 for (uint32_t i
= 0; i
< submit
->sync_fd_semaphore_count
; i
++) {
292 struct anv_semaphore
*semaphore
= submit
->sync_fd_semaphores
[i
];
293 /* Out fences can't have temporary state because that would imply
294 * that we imported a sync file and are trying to signal it.
296 assert(semaphore
->temporary
.type
== ANV_SEMAPHORE_TYPE_NONE
);
297 struct anv_semaphore_impl
*impl
= &semaphore
->permanent
;
299 assert(impl
->type
== ANV_SEMAPHORE_TYPE_SYNC_FILE
);
300 impl
->fd
= os_dupfd_cloexec(submit
->out_fence
);
303 /* Unblock any waiter by signaling the points, the application will get
304 * a device lost error code.
306 for (uint32_t i
= 0; i
< submit
->signal_timeline_count
; i
++) {
307 struct anv_timeline
*timeline
= submit
->signal_timelines
[i
];
308 uint64_t signal_value
= submit
->signal_timeline_values
[i
];
310 assert(signal_value
> timeline
->highest_pending
);
311 timeline
->highest_past
= timeline
->highest_pending
= signal_value
;
319 anv_queue_submit_deferred_locked(struct anv_queue
*queue
, uint32_t *advance
)
321 VkResult result
= VK_SUCCESS
;
323 /* Go through all the queued submissions and submit then until we find one
324 * that's waiting on a point that hasn't materialized yet.
326 list_for_each_entry_safe(struct anv_queue_submit
, submit
,
327 &queue
->queued_submits
, link
) {
328 if (!anv_queue_submit_ready_locked(submit
))
332 list_del(&submit
->link
);
334 result
= anv_queue_submit_timeline_locked(queue
, submit
);
336 anv_queue_submit_free(queue
->device
, submit
);
338 if (result
!= VK_SUCCESS
)
346 anv_device_submit_deferred_locked(struct anv_device
*device
)
348 uint32_t advance
= 0;
349 return anv_queue_submit_deferred_locked(&device
->queue
, &advance
);
353 _anv_queue_submit(struct anv_queue
*queue
, struct anv_queue_submit
**_submit
,
356 struct anv_queue_submit
*submit
= *_submit
;
358 /* Wait before signal behavior means we might keep alive the
359 * anv_queue_submit object a bit longer, so transfer the ownership to the
364 pthread_mutex_lock(&queue
->device
->mutex
);
365 list_addtail(&submit
->link
, &queue
->queued_submits
);
366 VkResult result
= anv_device_submit_deferred_locked(queue
->device
);
368 while (result
== VK_SUCCESS
&& !list_is_empty(&queue
->queued_submits
)) {
369 int ret
= pthread_cond_wait(&queue
->device
->queue_submit
,
370 &queue
->device
->mutex
);
372 result
= anv_device_set_lost(queue
->device
, "wait timeout");
376 result
= anv_device_submit_deferred_locked(queue
->device
);
379 pthread_mutex_unlock(&queue
->device
->mutex
);
384 anv_queue_init(struct anv_device
*device
, struct anv_queue
*queue
)
386 vk_object_base_init(&device
->vk
, &queue
->base
, VK_OBJECT_TYPE_QUEUE
);
387 queue
->device
= device
;
390 list_inithead(&queue
->queued_submits
);
396 anv_queue_finish(struct anv_queue
*queue
)
398 vk_object_base_finish(&queue
->base
);
402 anv_queue_submit_add_fence_bo(struct anv_queue_submit
*submit
,
406 if (submit
->fence_bo_count
>= submit
->fence_bo_array_length
) {
407 uint32_t new_len
= MAX2(submit
->fence_bo_array_length
* 2, 64);
408 uintptr_t *new_fence_bos
=
409 vk_realloc(submit
->alloc
,
410 submit
->fence_bos
, new_len
* sizeof(*submit
->fence_bos
),
411 8, submit
->alloc_scope
);
412 if (new_fence_bos
== NULL
)
413 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
415 submit
->fence_bos
= new_fence_bos
;
416 submit
->fence_bo_array_length
= new_len
;
419 /* Take advantage that anv_bo are allocated at 8 byte alignement so we can
420 * use the lowest bit to store whether this is a BO we need to signal.
422 submit
->fence_bos
[submit
->fence_bo_count
++] = anv_pack_ptr(bo
, 1, signal
);
428 anv_queue_submit_add_syncobj(struct anv_queue_submit
* submit
,
429 struct anv_device
*device
,
430 uint32_t handle
, uint32_t flags
)
434 if (submit
->fence_count
>= submit
->fence_array_length
) {
435 uint32_t new_len
= MAX2(submit
->fence_array_length
* 2, 64);
436 struct drm_i915_gem_exec_fence
*new_fences
=
437 vk_realloc(submit
->alloc
,
438 submit
->fences
, new_len
* sizeof(*submit
->fences
),
439 8, submit
->alloc_scope
);
440 if (new_fences
== NULL
)
441 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
443 submit
->fences
= new_fences
;
444 submit
->fence_array_length
= new_len
;
447 submit
->fences
[submit
->fence_count
++] = (struct drm_i915_gem_exec_fence
) {
456 anv_queue_submit_add_sync_fd_fence(struct anv_queue_submit
*submit
,
457 struct anv_semaphore
*semaphore
)
459 if (submit
->sync_fd_semaphore_count
>= submit
->sync_fd_semaphore_array_length
) {
460 uint32_t new_len
= MAX2(submit
->sync_fd_semaphore_array_length
* 2, 64);
461 struct anv_semaphore
**new_semaphores
=
462 vk_realloc(submit
->alloc
, submit
->sync_fd_semaphores
,
463 new_len
* sizeof(*submit
->sync_fd_semaphores
), 8,
464 submit
->alloc_scope
);
465 if (new_semaphores
== NULL
)
466 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
468 submit
->sync_fd_semaphores
= new_semaphores
;
471 submit
->sync_fd_semaphores
[submit
->sync_fd_semaphore_count
++] =
472 anv_semaphore_ref(semaphore
);
473 submit
->need_out_fence
= true;
479 anv_queue_submit_add_timeline_wait(struct anv_queue_submit
* submit
,
480 struct anv_device
*device
,
481 struct anv_timeline
*timeline
,
484 if (submit
->wait_timeline_count
>= submit
->wait_timeline_array_length
) {
485 uint32_t new_len
= MAX2(submit
->wait_timeline_array_length
* 2, 64);
486 struct anv_timeline
**new_wait_timelines
=
487 vk_realloc(submit
->alloc
,
488 submit
->wait_timelines
, new_len
* sizeof(*submit
->wait_timelines
),
489 8, submit
->alloc_scope
);
490 if (new_wait_timelines
== NULL
)
491 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
493 submit
->wait_timelines
= new_wait_timelines
;
495 uint64_t *new_wait_timeline_values
=
496 vk_realloc(submit
->alloc
,
497 submit
->wait_timeline_values
, new_len
* sizeof(*submit
->wait_timeline_values
),
498 8, submit
->alloc_scope
);
499 if (new_wait_timeline_values
== NULL
)
500 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
502 submit
->wait_timeline_values
= new_wait_timeline_values
;
504 submit
->wait_timeline_array_length
= new_len
;
507 submit
->wait_timelines
[submit
->wait_timeline_count
] = timeline
;
508 submit
->wait_timeline_values
[submit
->wait_timeline_count
] = value
;
510 submit
->wait_timeline_count
++;
516 anv_queue_submit_add_timeline_signal(struct anv_queue_submit
* submit
,
517 struct anv_device
*device
,
518 struct anv_timeline
*timeline
,
521 assert(timeline
->highest_pending
< value
);
523 if (submit
->signal_timeline_count
>= submit
->signal_timeline_array_length
) {
524 uint32_t new_len
= MAX2(submit
->signal_timeline_array_length
* 2, 64);
525 struct anv_timeline
**new_signal_timelines
=
526 vk_realloc(submit
->alloc
,
527 submit
->signal_timelines
, new_len
* sizeof(*submit
->signal_timelines
),
528 8, submit
->alloc_scope
);
529 if (new_signal_timelines
== NULL
)
530 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
532 submit
->signal_timelines
= new_signal_timelines
;
534 uint64_t *new_signal_timeline_values
=
535 vk_realloc(submit
->alloc
,
536 submit
->signal_timeline_values
, new_len
* sizeof(*submit
->signal_timeline_values
),
537 8, submit
->alloc_scope
);
538 if (new_signal_timeline_values
== NULL
)
539 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
541 submit
->signal_timeline_values
= new_signal_timeline_values
;
543 submit
->signal_timeline_array_length
= new_len
;
546 submit
->signal_timelines
[submit
->signal_timeline_count
] = timeline
;
547 submit
->signal_timeline_values
[submit
->signal_timeline_count
] = value
;
549 submit
->signal_timeline_count
++;
554 static struct anv_queue_submit
*
555 anv_queue_submit_alloc(struct anv_device
*device
, int perf_query_pass
)
557 const VkAllocationCallbacks
*alloc
= &device
->vk
.alloc
;
558 VkSystemAllocationScope alloc_scope
= VK_SYSTEM_ALLOCATION_SCOPE_DEVICE
;
560 struct anv_queue_submit
*submit
= vk_zalloc(alloc
, sizeof(*submit
), 8, alloc_scope
);
564 submit
->alloc
= alloc
;
565 submit
->alloc_scope
= alloc_scope
;
566 submit
->in_fence
= -1;
567 submit
->out_fence
= -1;
568 submit
->perf_query_pass
= perf_query_pass
;
574 anv_queue_submit_simple_batch(struct anv_queue
*queue
,
575 struct anv_batch
*batch
)
577 if (queue
->device
->no_hw
)
580 struct anv_device
*device
= queue
->device
;
581 struct anv_queue_submit
*submit
= anv_queue_submit_alloc(device
, -1);
583 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
585 bool has_syncobj_wait
= device
->physical
->has_syncobj_wait
;
588 struct anv_bo
*batch_bo
, *sync_bo
;
590 if (has_syncobj_wait
) {
591 syncobj
= anv_gem_syncobj_create(device
, 0);
593 result
= vk_error(VK_ERROR_OUT_OF_DEVICE_MEMORY
);
594 goto err_free_submit
;
597 result
= anv_queue_submit_add_syncobj(submit
, device
, syncobj
,
598 I915_EXEC_FENCE_SIGNAL
);
600 result
= anv_device_alloc_bo(device
, 4096,
601 ANV_BO_ALLOC_EXTERNAL
|
602 ANV_BO_ALLOC_IMPLICIT_SYNC
,
603 0 /* explicit_address */,
605 if (result
!= VK_SUCCESS
)
606 goto err_free_submit
;
608 result
= anv_queue_submit_add_fence_bo(submit
, sync_bo
, true /* signal */);
611 if (result
!= VK_SUCCESS
)
612 goto err_destroy_sync_primitive
;
615 uint32_t size
= align_u32(batch
->next
- batch
->start
, 8);
616 result
= anv_bo_pool_alloc(&device
->batch_bo_pool
, size
, &batch_bo
);
617 if (result
!= VK_SUCCESS
)
618 goto err_destroy_sync_primitive
;
620 memcpy(batch_bo
->map
, batch
->start
, size
);
621 if (!device
->info
.has_llc
)
622 gen_flush_range(batch_bo
->map
, size
);
624 submit
->simple_bo
= batch_bo
;
625 submit
->simple_bo_size
= size
;
628 result
= _anv_queue_submit(queue
, &submit
, true);
630 if (result
== VK_SUCCESS
) {
631 if (has_syncobj_wait
) {
632 if (anv_gem_syncobj_wait(device
, &syncobj
, 1,
633 anv_get_absolute_timeout(INT64_MAX
), true))
634 result
= anv_device_set_lost(device
, "anv_gem_syncobj_wait failed: %m");
635 anv_gem_syncobj_destroy(device
, syncobj
);
637 result
= anv_device_wait(device
, sync_bo
,
638 anv_get_relative_timeout(INT64_MAX
));
639 anv_device_release_bo(device
, sync_bo
);
644 anv_bo_pool_free(&device
->batch_bo_pool
, batch_bo
);
647 anv_queue_submit_free(device
, submit
);
651 err_destroy_sync_primitive
:
652 if (has_syncobj_wait
)
653 anv_gem_syncobj_destroy(device
, syncobj
);
655 anv_device_release_bo(device
, sync_bo
);
658 anv_queue_submit_free(device
, submit
);
663 /* Transfer ownership of temporary semaphores from the VkSemaphore object to
664 * the anv_queue_submit object. Those temporary semaphores are then freed in
665 * anv_queue_submit_free() once the driver is finished with them.
668 maybe_transfer_temporary_semaphore(struct anv_queue_submit
*submit
,
669 struct anv_semaphore
*semaphore
,
670 struct anv_semaphore_impl
**out_impl
)
672 struct anv_semaphore_impl
*impl
= &semaphore
->temporary
;
674 if (impl
->type
== ANV_SEMAPHORE_TYPE_NONE
) {
675 *out_impl
= &semaphore
->permanent
;
679 /* BO backed timeline semaphores cannot be temporary. */
680 assert(impl
->type
!= ANV_SEMAPHORE_TYPE_TIMELINE
);
683 * There is a requirement to reset semaphore to their permanent state after
684 * submission. From the Vulkan 1.0.53 spec:
686 * "If the import is temporary, the implementation must restore the
687 * semaphore to its prior permanent state after submitting the next
688 * semaphore wait operation."
690 * In the case we defer the actual submission to a thread because of the
691 * wait-before-submit behavior required for timeline semaphores, we need to
692 * make copies of the temporary syncobj to ensure they stay alive until we
693 * do the actual execbuffer ioctl.
695 if (submit
->temporary_semaphore_count
>= submit
->temporary_semaphore_array_length
) {
696 uint32_t new_len
= MAX2(submit
->temporary_semaphore_array_length
* 2, 8);
697 /* Make sure that if the realloc fails, we still have the old semaphore
698 * array around to properly clean things up on failure.
700 struct anv_semaphore_impl
*new_array
=
701 vk_realloc(submit
->alloc
,
702 submit
->temporary_semaphores
,
703 new_len
* sizeof(*submit
->temporary_semaphores
),
704 8, submit
->alloc_scope
);
705 if (new_array
== NULL
)
706 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
708 submit
->temporary_semaphores
= new_array
;
709 submit
->temporary_semaphore_array_length
= new_len
;
712 /* Copy anv_semaphore_impl into anv_queue_submit. */
713 submit
->temporary_semaphores
[submit
->temporary_semaphore_count
++] = *impl
;
714 *out_impl
= &submit
->temporary_semaphores
[submit
->temporary_semaphore_count
- 1];
716 /* Clear the incoming semaphore */
717 impl
->type
= ANV_SEMAPHORE_TYPE_NONE
;
723 anv_queue_submit(struct anv_queue
*queue
,
724 struct anv_cmd_buffer
*cmd_buffer
,
725 const VkSemaphore
*in_semaphores
,
726 const uint64_t *in_values
,
727 uint32_t num_in_semaphores
,
728 const VkSemaphore
*out_semaphores
,
729 const uint64_t *out_values
,
730 uint32_t num_out_semaphores
,
731 struct anv_bo
*wsi_signal_bo
,
735 ANV_FROM_HANDLE(anv_fence
, fence
, _fence
);
736 struct anv_device
*device
= queue
->device
;
737 UNUSED
struct anv_physical_device
*pdevice
= device
->physical
;
738 struct anv_queue_submit
*submit
= anv_queue_submit_alloc(device
, perf_query_pass
);
740 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
742 submit
->cmd_buffer
= cmd_buffer
;
744 VkResult result
= VK_SUCCESS
;
746 for (uint32_t i
= 0; i
< num_in_semaphores
; i
++) {
747 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, in_semaphores
[i
]);
748 struct anv_semaphore_impl
*impl
;
750 result
= maybe_transfer_temporary_semaphore(submit
, semaphore
, &impl
);
751 if (result
!= VK_SUCCESS
)
754 switch (impl
->type
) {
755 case ANV_SEMAPHORE_TYPE_BO
:
756 assert(!pdevice
->has_syncobj
);
757 result
= anv_queue_submit_add_fence_bo(submit
, impl
->bo
, false /* signal */);
758 if (result
!= VK_SUCCESS
)
762 case ANV_SEMAPHORE_TYPE_WSI_BO
:
763 /* When using a window-system buffer as a semaphore, always enable
764 * EXEC_OBJECT_WRITE. This gives us a WaR hazard with the display or
765 * compositor's read of the buffer and enforces that we don't start
766 * rendering until they are finished. This is exactly the
767 * synchronization we want with vkAcquireNextImage.
769 result
= anv_queue_submit_add_fence_bo(submit
, impl
->bo
, true /* signal */);
770 if (result
!= VK_SUCCESS
)
774 case ANV_SEMAPHORE_TYPE_SYNC_FILE
:
775 assert(!pdevice
->has_syncobj
);
776 if (submit
->in_fence
== -1) {
777 submit
->in_fence
= impl
->fd
;
778 if (submit
->in_fence
== -1) {
779 result
= vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
784 int merge
= anv_gem_sync_file_merge(device
, submit
->in_fence
, impl
->fd
);
786 result
= vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
790 close(submit
->in_fence
);
792 submit
->in_fence
= merge
;
796 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
: {
797 result
= anv_queue_submit_add_syncobj(submit
, device
,
799 I915_EXEC_FENCE_WAIT
);
800 if (result
!= VK_SUCCESS
)
805 case ANV_SEMAPHORE_TYPE_TIMELINE
:
806 result
= anv_queue_submit_add_timeline_wait(submit
, device
,
808 in_values
? in_values
[i
] : 0);
809 if (result
!= VK_SUCCESS
)
818 for (uint32_t i
= 0; i
< num_out_semaphores
; i
++) {
819 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, out_semaphores
[i
]);
821 /* Under most circumstances, out fences won't be temporary. However,
822 * the spec does allow it for opaque_fd. From the Vulkan 1.0.53 spec:
824 * "If the import is temporary, the implementation must restore the
825 * semaphore to its prior permanent state after submitting the next
826 * semaphore wait operation."
828 * The spec says nothing whatsoever about signal operations on
829 * temporarily imported semaphores so it appears they are allowed.
830 * There are also CTS tests that require this to work.
832 struct anv_semaphore_impl
*impl
=
833 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
834 &semaphore
->temporary
: &semaphore
->permanent
;
836 switch (impl
->type
) {
837 case ANV_SEMAPHORE_TYPE_BO
:
838 assert(!pdevice
->has_syncobj
);
839 result
= anv_queue_submit_add_fence_bo(submit
, impl
->bo
, true /* signal */);
840 if (result
!= VK_SUCCESS
)
844 case ANV_SEMAPHORE_TYPE_SYNC_FILE
:
845 assert(!pdevice
->has_syncobj
);
846 result
= anv_queue_submit_add_sync_fd_fence(submit
, semaphore
);
847 if (result
!= VK_SUCCESS
)
851 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
: {
852 result
= anv_queue_submit_add_syncobj(submit
, device
, impl
->syncobj
,
853 I915_EXEC_FENCE_SIGNAL
);
854 if (result
!= VK_SUCCESS
)
859 case ANV_SEMAPHORE_TYPE_TIMELINE
:
860 result
= anv_queue_submit_add_timeline_signal(submit
, device
,
862 out_values
? out_values
[i
] : 0);
863 if (result
!= VK_SUCCESS
)
873 result
= anv_queue_submit_add_fence_bo(submit
, wsi_signal_bo
, true /* signal */);
874 if (result
!= VK_SUCCESS
)
879 /* Under most circumstances, out fences won't be temporary. However,
880 * the spec does allow it for opaque_fd. From the Vulkan 1.0.53 spec:
882 * "If the import is temporary, the implementation must restore the
883 * semaphore to its prior permanent state after submitting the next
884 * semaphore wait operation."
886 * The spec says nothing whatsoever about signal operations on
887 * temporarily imported semaphores so it appears they are allowed.
888 * There are also CTS tests that require this to work.
890 struct anv_fence_impl
*impl
=
891 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
892 &fence
->temporary
: &fence
->permanent
;
894 switch (impl
->type
) {
895 case ANV_FENCE_TYPE_BO
:
896 result
= anv_queue_submit_add_fence_bo(submit
, impl
->bo
.bo
, true /* signal */);
897 if (result
!= VK_SUCCESS
)
901 case ANV_FENCE_TYPE_SYNCOBJ
: {
903 * For the same reason we reset the signaled binary syncobj above,
904 * also reset the fence's syncobj so that they don't contain a
905 * signaled dma-fence.
907 result
= anv_queue_submit_add_syncobj(submit
, device
, impl
->syncobj
,
908 I915_EXEC_FENCE_SIGNAL
);
909 if (result
!= VK_SUCCESS
)
915 unreachable("Invalid fence type");
919 result
= _anv_queue_submit(queue
, &submit
, false);
920 if (result
!= VK_SUCCESS
)
923 if (fence
&& fence
->permanent
.type
== ANV_FENCE_TYPE_BO
) {
924 /* If we have permanent BO fence, the only type of temporary possible
925 * would be BO_WSI (because BO fences are not shareable). The Vulkan spec
926 * also requires that the fence passed to vkQueueSubmit() be :
929 * * not be associated with any other queue command that has not yet
930 * completed execution on that queue
932 * So the only acceptable type for the temporary is NONE.
934 assert(fence
->temporary
.type
== ANV_FENCE_TYPE_NONE
);
936 /* Once the execbuf has returned, we need to set the fence state to
937 * SUBMITTED. We can't do this before calling execbuf because
938 * anv_GetFenceStatus does take the global device lock before checking
941 * We set the fence state to SUBMITTED regardless of whether or not the
942 * execbuf succeeds because we need to ensure that vkWaitForFences() and
943 * vkGetFenceStatus() return a valid result (VK_ERROR_DEVICE_LOST or
944 * VK_SUCCESS) in a finite amount of time even if execbuf fails.
946 fence
->permanent
.bo
.state
= ANV_BO_FENCE_STATE_SUBMITTED
;
951 anv_queue_submit_free(device
, submit
);
956 VkResult
anv_QueueSubmit(
958 uint32_t submitCount
,
959 const VkSubmitInfo
* pSubmits
,
962 ANV_FROM_HANDLE(anv_queue
, queue
, _queue
);
964 if (queue
->device
->no_hw
)
967 /* Query for device status prior to submitting. Technically, we don't need
968 * to do this. However, if we have a client that's submitting piles of
969 * garbage, we would rather break as early as possible to keep the GPU
970 * hanging contained. If we don't check here, we'll either be waiting for
971 * the kernel to kick us or we'll have to wait until the client waits on a
972 * fence before we actually know whether or not we've hung.
974 VkResult result
= anv_device_query_status(queue
->device
);
975 if (result
!= VK_SUCCESS
)
978 if (fence
&& submitCount
== 0) {
979 /* If we don't have any command buffers, we need to submit a dummy
980 * batch to give GEM something to wait on. We could, potentially,
981 * come up with something more efficient but this shouldn't be a
984 result
= anv_queue_submit(queue
, NULL
, NULL
, NULL
, 0, NULL
, NULL
, 0,
989 for (uint32_t i
= 0; i
< submitCount
; i
++) {
990 /* Fence for this submit. NULL for all but the last one */
991 VkFence submit_fence
= (i
== submitCount
- 1) ? fence
: VK_NULL_HANDLE
;
993 const struct wsi_memory_signal_submit_info
*mem_signal_info
=
994 vk_find_struct_const(pSubmits
[i
].pNext
,
995 WSI_MEMORY_SIGNAL_SUBMIT_INFO_MESA
);
996 struct anv_bo
*wsi_signal_bo
=
997 mem_signal_info
&& mem_signal_info
->memory
!= VK_NULL_HANDLE
?
998 anv_device_memory_from_handle(mem_signal_info
->memory
)->bo
: NULL
;
1000 const VkTimelineSemaphoreSubmitInfoKHR
*timeline_info
=
1001 vk_find_struct_const(pSubmits
[i
].pNext
,
1002 TIMELINE_SEMAPHORE_SUBMIT_INFO_KHR
);
1003 const VkPerformanceQuerySubmitInfoKHR
*perf_info
=
1004 vk_find_struct_const(pSubmits
[i
].pNext
,
1005 PERFORMANCE_QUERY_SUBMIT_INFO_KHR
);
1006 const uint64_t *wait_values
=
1007 timeline_info
&& timeline_info
->waitSemaphoreValueCount
?
1008 timeline_info
->pWaitSemaphoreValues
: NULL
;
1009 const uint64_t *signal_values
=
1010 timeline_info
&& timeline_info
->signalSemaphoreValueCount
?
1011 timeline_info
->pSignalSemaphoreValues
: NULL
;
1013 if (pSubmits
[i
].commandBufferCount
== 0) {
1014 /* If we don't have any command buffers, we need to submit a dummy
1015 * batch to give GEM something to wait on. We could, potentially,
1016 * come up with something more efficient but this shouldn't be a
1019 result
= anv_queue_submit(queue
, NULL
,
1020 pSubmits
[i
].pWaitSemaphores
,
1022 pSubmits
[i
].waitSemaphoreCount
,
1023 pSubmits
[i
].pSignalSemaphores
,
1025 pSubmits
[i
].signalSemaphoreCount
,
1029 if (result
!= VK_SUCCESS
)
1035 for (uint32_t j
= 0; j
< pSubmits
[i
].commandBufferCount
; j
++) {
1036 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
,
1037 pSubmits
[i
].pCommandBuffers
[j
]);
1038 assert(cmd_buffer
->level
== VK_COMMAND_BUFFER_LEVEL_PRIMARY
);
1039 assert(!anv_batch_has_error(&cmd_buffer
->batch
));
1041 /* Fence for this execbuf. NULL for all but the last one */
1042 VkFence execbuf_fence
=
1043 (j
== pSubmits
[i
].commandBufferCount
- 1) ?
1044 submit_fence
: VK_NULL_HANDLE
;
1046 const VkSemaphore
*in_semaphores
= NULL
, *out_semaphores
= NULL
;
1047 const uint64_t *in_values
= NULL
, *out_values
= NULL
;
1048 uint32_t num_in_semaphores
= 0, num_out_semaphores
= 0;
1050 /* Only the first batch gets the in semaphores */
1051 in_semaphores
= pSubmits
[i
].pWaitSemaphores
;
1052 in_values
= wait_values
;
1053 num_in_semaphores
= pSubmits
[i
].waitSemaphoreCount
;
1056 if (j
== pSubmits
[i
].commandBufferCount
- 1) {
1057 /* Only the last batch gets the out semaphores */
1058 out_semaphores
= pSubmits
[i
].pSignalSemaphores
;
1059 out_values
= signal_values
;
1060 num_out_semaphores
= pSubmits
[i
].signalSemaphoreCount
;
1063 result
= anv_queue_submit(queue
, cmd_buffer
,
1064 in_semaphores
, in_values
, num_in_semaphores
,
1065 out_semaphores
, out_values
, num_out_semaphores
,
1066 wsi_signal_bo
, execbuf_fence
,
1067 perf_info
? perf_info
->counterPassIndex
: 0);
1068 if (result
!= VK_SUCCESS
)
1074 if (result
!= VK_SUCCESS
&& result
!= VK_ERROR_DEVICE_LOST
) {
1075 /* In the case that something has gone wrong we may end up with an
1076 * inconsistent state from which it may not be trivial to recover.
1077 * For example, we might have computed address relocations and
1078 * any future attempt to re-submit this job will need to know about
1079 * this and avoid computing relocation addresses again.
1081 * To avoid this sort of issues, we assume that if something was
1082 * wrong during submission we must already be in a really bad situation
1083 * anyway (such us being out of memory) and return
1084 * VK_ERROR_DEVICE_LOST to ensure that clients do not attempt to
1085 * submit the same job again to this device.
1087 * We skip doing this on VK_ERROR_DEVICE_LOST because
1088 * anv_device_set_lost() would have been called already by a callee of
1089 * anv_queue_submit().
1091 result
= anv_device_set_lost(queue
->device
, "vkQueueSubmit() failed");
1097 VkResult
anv_QueueWaitIdle(
1100 ANV_FROM_HANDLE(anv_queue
, queue
, _queue
);
1102 if (anv_device_is_lost(queue
->device
))
1103 return VK_ERROR_DEVICE_LOST
;
1105 return anv_queue_submit_simple_batch(queue
, NULL
);
1108 VkResult
anv_CreateFence(
1110 const VkFenceCreateInfo
* pCreateInfo
,
1111 const VkAllocationCallbacks
* pAllocator
,
1114 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1115 struct anv_fence
*fence
;
1117 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_FENCE_CREATE_INFO
);
1119 fence
= vk_zalloc2(&device
->vk
.alloc
, pAllocator
, sizeof(*fence
), 8,
1120 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
1122 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1124 vk_object_base_init(&device
->vk
, &fence
->base
, VK_OBJECT_TYPE_FENCE
);
1126 if (device
->physical
->has_syncobj_wait
) {
1127 fence
->permanent
.type
= ANV_FENCE_TYPE_SYNCOBJ
;
1129 uint32_t create_flags
= 0;
1130 if (pCreateInfo
->flags
& VK_FENCE_CREATE_SIGNALED_BIT
)
1131 create_flags
|= DRM_SYNCOBJ_CREATE_SIGNALED
;
1133 fence
->permanent
.syncobj
= anv_gem_syncobj_create(device
, create_flags
);
1134 if (!fence
->permanent
.syncobj
)
1135 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1137 fence
->permanent
.type
= ANV_FENCE_TYPE_BO
;
1139 VkResult result
= anv_bo_pool_alloc(&device
->batch_bo_pool
, 4096,
1140 &fence
->permanent
.bo
.bo
);
1141 if (result
!= VK_SUCCESS
)
1144 if (pCreateInfo
->flags
& VK_FENCE_CREATE_SIGNALED_BIT
) {
1145 fence
->permanent
.bo
.state
= ANV_BO_FENCE_STATE_SIGNALED
;
1147 fence
->permanent
.bo
.state
= ANV_BO_FENCE_STATE_RESET
;
1151 *pFence
= anv_fence_to_handle(fence
);
1157 anv_fence_impl_cleanup(struct anv_device
*device
,
1158 struct anv_fence_impl
*impl
)
1160 switch (impl
->type
) {
1161 case ANV_FENCE_TYPE_NONE
:
1162 /* Dummy. Nothing to do */
1165 case ANV_FENCE_TYPE_BO
:
1166 anv_bo_pool_free(&device
->batch_bo_pool
, impl
->bo
.bo
);
1169 case ANV_FENCE_TYPE_WSI_BO
:
1170 anv_device_release_bo(device
, impl
->bo
.bo
);
1173 case ANV_FENCE_TYPE_SYNCOBJ
:
1174 anv_gem_syncobj_destroy(device
, impl
->syncobj
);
1177 case ANV_FENCE_TYPE_WSI
:
1178 impl
->fence_wsi
->destroy(impl
->fence_wsi
);
1182 unreachable("Invalid fence type");
1185 impl
->type
= ANV_FENCE_TYPE_NONE
;
1189 anv_fence_reset_temporary(struct anv_device
*device
,
1190 struct anv_fence
*fence
)
1192 if (fence
->temporary
.type
== ANV_FENCE_TYPE_NONE
)
1195 anv_fence_impl_cleanup(device
, &fence
->temporary
);
1198 void anv_DestroyFence(
1201 const VkAllocationCallbacks
* pAllocator
)
1203 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1204 ANV_FROM_HANDLE(anv_fence
, fence
, _fence
);
1209 anv_fence_impl_cleanup(device
, &fence
->temporary
);
1210 anv_fence_impl_cleanup(device
, &fence
->permanent
);
1212 vk_object_base_finish(&fence
->base
);
1213 vk_free2(&device
->vk
.alloc
, pAllocator
, fence
);
1216 VkResult
anv_ResetFences(
1218 uint32_t fenceCount
,
1219 const VkFence
* pFences
)
1221 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1223 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1224 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1226 /* From the Vulkan 1.0.53 spec:
1228 * "If any member of pFences currently has its payload imported with
1229 * temporary permanence, that fence’s prior permanent payload is
1230 * first restored. The remaining operations described therefore
1231 * operate on the restored payload.
1233 anv_fence_reset_temporary(device
, fence
);
1235 struct anv_fence_impl
*impl
= &fence
->permanent
;
1237 switch (impl
->type
) {
1238 case ANV_FENCE_TYPE_BO
:
1239 impl
->bo
.state
= ANV_BO_FENCE_STATE_RESET
;
1242 case ANV_FENCE_TYPE_SYNCOBJ
:
1243 anv_gem_syncobj_reset(device
, impl
->syncobj
);
1247 unreachable("Invalid fence type");
1254 VkResult
anv_GetFenceStatus(
1258 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1259 ANV_FROM_HANDLE(anv_fence
, fence
, _fence
);
1261 if (anv_device_is_lost(device
))
1262 return VK_ERROR_DEVICE_LOST
;
1264 struct anv_fence_impl
*impl
=
1265 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1266 &fence
->temporary
: &fence
->permanent
;
1268 switch (impl
->type
) {
1269 case ANV_FENCE_TYPE_BO
:
1270 case ANV_FENCE_TYPE_WSI_BO
:
1271 switch (impl
->bo
.state
) {
1272 case ANV_BO_FENCE_STATE_RESET
:
1273 /* If it hasn't even been sent off to the GPU yet, it's not ready */
1274 return VK_NOT_READY
;
1276 case ANV_BO_FENCE_STATE_SIGNALED
:
1277 /* It's been signaled, return success */
1280 case ANV_BO_FENCE_STATE_SUBMITTED
: {
1281 VkResult result
= anv_device_bo_busy(device
, impl
->bo
.bo
);
1282 if (result
== VK_SUCCESS
) {
1283 impl
->bo
.state
= ANV_BO_FENCE_STATE_SIGNALED
;
1290 unreachable("Invalid fence status");
1293 case ANV_FENCE_TYPE_SYNCOBJ
: {
1294 int ret
= anv_gem_syncobj_wait(device
, &impl
->syncobj
, 1, 0, true);
1296 if (errno
== ETIME
) {
1297 return VK_NOT_READY
;
1299 /* We don't know the real error. */
1300 return anv_device_set_lost(device
, "drm_syncobj_wait failed: %m");
1308 unreachable("Invalid fence type");
1313 anv_wait_for_syncobj_fences(struct anv_device
*device
,
1314 uint32_t fenceCount
,
1315 const VkFence
*pFences
,
1317 uint64_t abs_timeout_ns
)
1319 uint32_t *syncobjs
= vk_zalloc(&device
->vk
.alloc
,
1320 sizeof(*syncobjs
) * fenceCount
, 8,
1321 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND
);
1323 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1325 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1326 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1327 assert(fence
->permanent
.type
== ANV_FENCE_TYPE_SYNCOBJ
);
1329 struct anv_fence_impl
*impl
=
1330 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1331 &fence
->temporary
: &fence
->permanent
;
1333 assert(impl
->type
== ANV_FENCE_TYPE_SYNCOBJ
);
1334 syncobjs
[i
] = impl
->syncobj
;
1337 /* The gem_syncobj_wait ioctl may return early due to an inherent
1338 * limitation in the way it computes timeouts. Loop until we've actually
1339 * passed the timeout.
1343 ret
= anv_gem_syncobj_wait(device
, syncobjs
, fenceCount
,
1344 abs_timeout_ns
, waitAll
);
1345 } while (ret
== -1 && errno
== ETIME
&& anv_gettime_ns() < abs_timeout_ns
);
1347 vk_free(&device
->vk
.alloc
, syncobjs
);
1350 if (errno
== ETIME
) {
1353 /* We don't know the real error. */
1354 return anv_device_set_lost(device
, "drm_syncobj_wait failed: %m");
1362 anv_wait_for_bo_fences(struct anv_device
*device
,
1363 uint32_t fenceCount
,
1364 const VkFence
*pFences
,
1366 uint64_t abs_timeout_ns
)
1368 VkResult result
= VK_SUCCESS
;
1369 uint32_t pending_fences
= fenceCount
;
1370 while (pending_fences
) {
1372 bool signaled_fences
= false;
1373 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1374 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1376 struct anv_fence_impl
*impl
=
1377 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1378 &fence
->temporary
: &fence
->permanent
;
1379 assert(impl
->type
== ANV_FENCE_TYPE_BO
||
1380 impl
->type
== ANV_FENCE_TYPE_WSI_BO
);
1382 switch (impl
->bo
.state
) {
1383 case ANV_BO_FENCE_STATE_RESET
:
1384 /* This fence hasn't been submitted yet, we'll catch it the next
1385 * time around. Yes, this may mean we dead-loop but, short of
1386 * lots of locking and a condition variable, there's not much that
1387 * we can do about that.
1392 case ANV_BO_FENCE_STATE_SIGNALED
:
1393 /* This fence is not pending. If waitAll isn't set, we can return
1394 * early. Otherwise, we have to keep going.
1397 result
= VK_SUCCESS
;
1402 case ANV_BO_FENCE_STATE_SUBMITTED
:
1403 /* These are the fences we really care about. Go ahead and wait
1404 * on it until we hit a timeout.
1406 result
= anv_device_wait(device
, impl
->bo
.bo
,
1407 anv_get_relative_timeout(abs_timeout_ns
));
1410 impl
->bo
.state
= ANV_BO_FENCE_STATE_SIGNALED
;
1411 signaled_fences
= true;
1425 if (pending_fences
&& !signaled_fences
) {
1426 /* If we've hit this then someone decided to vkWaitForFences before
1427 * they've actually submitted any of them to a queue. This is a
1428 * fairly pessimal case, so it's ok to lock here and use a standard
1429 * pthreads condition variable.
1431 pthread_mutex_lock(&device
->mutex
);
1433 /* It's possible that some of the fences have changed state since the
1434 * last time we checked. Now that we have the lock, check for
1435 * pending fences again and don't wait if it's changed.
1437 uint32_t now_pending_fences
= 0;
1438 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1439 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1440 if (fence
->permanent
.bo
.state
== ANV_BO_FENCE_STATE_RESET
)
1441 now_pending_fences
++;
1443 assert(now_pending_fences
<= pending_fences
);
1445 if (now_pending_fences
== pending_fences
) {
1446 struct timespec abstime
= {
1447 .tv_sec
= abs_timeout_ns
/ NSEC_PER_SEC
,
1448 .tv_nsec
= abs_timeout_ns
% NSEC_PER_SEC
,
1452 ret
= pthread_cond_timedwait(&device
->queue_submit
,
1453 &device
->mutex
, &abstime
);
1454 assert(ret
!= EINVAL
);
1455 if (anv_gettime_ns() >= abs_timeout_ns
) {
1456 pthread_mutex_unlock(&device
->mutex
);
1457 result
= VK_TIMEOUT
;
1462 pthread_mutex_unlock(&device
->mutex
);
1467 if (anv_device_is_lost(device
))
1468 return VK_ERROR_DEVICE_LOST
;
1474 anv_wait_for_wsi_fence(struct anv_device
*device
,
1475 struct anv_fence_impl
*impl
,
1476 uint64_t abs_timeout
)
1478 return impl
->fence_wsi
->wait(impl
->fence_wsi
, abs_timeout
);
1482 anv_wait_for_fences(struct anv_device
*device
,
1483 uint32_t fenceCount
,
1484 const VkFence
*pFences
,
1486 uint64_t abs_timeout
)
1488 VkResult result
= VK_SUCCESS
;
1490 if (fenceCount
<= 1 || waitAll
) {
1491 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1492 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1493 struct anv_fence_impl
*impl
=
1494 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1495 &fence
->temporary
: &fence
->permanent
;
1497 switch (impl
->type
) {
1498 case ANV_FENCE_TYPE_BO
:
1499 case ANV_FENCE_TYPE_WSI_BO
:
1500 result
= anv_wait_for_bo_fences(device
, 1, &pFences
[i
],
1503 case ANV_FENCE_TYPE_SYNCOBJ
:
1504 result
= anv_wait_for_syncobj_fences(device
, 1, &pFences
[i
],
1507 case ANV_FENCE_TYPE_WSI
:
1508 result
= anv_wait_for_wsi_fence(device
, impl
, abs_timeout
);
1510 case ANV_FENCE_TYPE_NONE
:
1511 result
= VK_SUCCESS
;
1514 if (result
!= VK_SUCCESS
)
1519 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1520 if (anv_wait_for_fences(device
, 1, &pFences
[i
], true, 0) == VK_SUCCESS
)
1523 } while (anv_gettime_ns() < abs_timeout
);
1524 result
= VK_TIMEOUT
;
1529 static bool anv_all_fences_syncobj(uint32_t fenceCount
, const VkFence
*pFences
)
1531 for (uint32_t i
= 0; i
< fenceCount
; ++i
) {
1532 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1533 struct anv_fence_impl
*impl
=
1534 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1535 &fence
->temporary
: &fence
->permanent
;
1536 if (impl
->type
!= ANV_FENCE_TYPE_SYNCOBJ
)
1542 static bool anv_all_fences_bo(uint32_t fenceCount
, const VkFence
*pFences
)
1544 for (uint32_t i
= 0; i
< fenceCount
; ++i
) {
1545 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1546 struct anv_fence_impl
*impl
=
1547 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1548 &fence
->temporary
: &fence
->permanent
;
1549 if (impl
->type
!= ANV_FENCE_TYPE_BO
&&
1550 impl
->type
!= ANV_FENCE_TYPE_WSI_BO
)
1556 VkResult
anv_WaitForFences(
1558 uint32_t fenceCount
,
1559 const VkFence
* pFences
,
1563 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1568 if (anv_device_is_lost(device
))
1569 return VK_ERROR_DEVICE_LOST
;
1571 uint64_t abs_timeout
= anv_get_absolute_timeout(timeout
);
1572 if (anv_all_fences_syncobj(fenceCount
, pFences
)) {
1573 return anv_wait_for_syncobj_fences(device
, fenceCount
, pFences
,
1574 waitAll
, abs_timeout
);
1575 } else if (anv_all_fences_bo(fenceCount
, pFences
)) {
1576 return anv_wait_for_bo_fences(device
, fenceCount
, pFences
,
1577 waitAll
, abs_timeout
);
1579 return anv_wait_for_fences(device
, fenceCount
, pFences
,
1580 waitAll
, abs_timeout
);
1584 void anv_GetPhysicalDeviceExternalFenceProperties(
1585 VkPhysicalDevice physicalDevice
,
1586 const VkPhysicalDeviceExternalFenceInfo
* pExternalFenceInfo
,
1587 VkExternalFenceProperties
* pExternalFenceProperties
)
1589 ANV_FROM_HANDLE(anv_physical_device
, device
, physicalDevice
);
1591 switch (pExternalFenceInfo
->handleType
) {
1592 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1593 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
:
1594 if (device
->has_syncobj_wait
) {
1595 pExternalFenceProperties
->exportFromImportedHandleTypes
=
1596 VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
|
1597 VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
;
1598 pExternalFenceProperties
->compatibleHandleTypes
=
1599 VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
|
1600 VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
;
1601 pExternalFenceProperties
->externalFenceFeatures
=
1602 VK_EXTERNAL_FENCE_FEATURE_EXPORTABLE_BIT
|
1603 VK_EXTERNAL_FENCE_FEATURE_IMPORTABLE_BIT
;
1612 pExternalFenceProperties
->exportFromImportedHandleTypes
= 0;
1613 pExternalFenceProperties
->compatibleHandleTypes
= 0;
1614 pExternalFenceProperties
->externalFenceFeatures
= 0;
1617 VkResult
anv_ImportFenceFdKHR(
1619 const VkImportFenceFdInfoKHR
* pImportFenceFdInfo
)
1621 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1622 ANV_FROM_HANDLE(anv_fence
, fence
, pImportFenceFdInfo
->fence
);
1623 int fd
= pImportFenceFdInfo
->fd
;
1625 assert(pImportFenceFdInfo
->sType
==
1626 VK_STRUCTURE_TYPE_IMPORT_FENCE_FD_INFO_KHR
);
1628 struct anv_fence_impl new_impl
= {
1629 .type
= ANV_FENCE_TYPE_NONE
,
1632 switch (pImportFenceFdInfo
->handleType
) {
1633 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1634 new_impl
.type
= ANV_FENCE_TYPE_SYNCOBJ
;
1636 new_impl
.syncobj
= anv_gem_syncobj_fd_to_handle(device
, fd
);
1637 if (!new_impl
.syncobj
)
1638 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1642 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
: {
1643 /* Sync files are a bit tricky. Because we want to continue using the
1644 * syncobj implementation of WaitForFences, we don't use the sync file
1645 * directly but instead import it into a syncobj.
1647 new_impl
.type
= ANV_FENCE_TYPE_SYNCOBJ
;
1649 /* "If handleType is VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT, the
1650 * special value -1 for fd is treated like a valid sync file descriptor
1651 * referring to an object that has already signaled. The import
1652 * operation will succeed and the VkFence will have a temporarily
1653 * imported payload as if a valid file descriptor had been provided."
1655 uint32_t create_flags
= 0;
1657 create_flags
|= DRM_SYNCOBJ_CREATE_SIGNALED
;
1659 new_impl
.syncobj
= anv_gem_syncobj_create(device
, create_flags
);
1660 if (!new_impl
.syncobj
)
1661 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1664 anv_gem_syncobj_import_sync_file(device
, new_impl
.syncobj
, fd
)) {
1665 anv_gem_syncobj_destroy(device
, new_impl
.syncobj
);
1666 return vk_errorf(device
, NULL
, VK_ERROR_INVALID_EXTERNAL_HANDLE
,
1667 "syncobj sync file import failed: %m");
1673 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1676 /* From the Vulkan 1.0.53 spec:
1678 * "Importing a fence payload from a file descriptor transfers
1679 * ownership of the file descriptor from the application to the
1680 * Vulkan implementation. The application must not perform any
1681 * operations on the file descriptor after a successful import."
1683 * If the import fails, we leave the file descriptor open.
1687 if (pImportFenceFdInfo
->flags
& VK_FENCE_IMPORT_TEMPORARY_BIT
) {
1688 anv_fence_impl_cleanup(device
, &fence
->temporary
);
1689 fence
->temporary
= new_impl
;
1691 anv_fence_impl_cleanup(device
, &fence
->permanent
);
1692 fence
->permanent
= new_impl
;
1698 VkResult
anv_GetFenceFdKHR(
1700 const VkFenceGetFdInfoKHR
* pGetFdInfo
,
1703 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1704 ANV_FROM_HANDLE(anv_fence
, fence
, pGetFdInfo
->fence
);
1706 assert(pGetFdInfo
->sType
== VK_STRUCTURE_TYPE_FENCE_GET_FD_INFO_KHR
);
1708 struct anv_fence_impl
*impl
=
1709 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1710 &fence
->temporary
: &fence
->permanent
;
1712 assert(impl
->type
== ANV_FENCE_TYPE_SYNCOBJ
);
1713 switch (pGetFdInfo
->handleType
) {
1714 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
: {
1715 int fd
= anv_gem_syncobj_handle_to_fd(device
, impl
->syncobj
);
1717 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
1723 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
: {
1724 int fd
= anv_gem_syncobj_export_sync_file(device
, impl
->syncobj
);
1726 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
1733 unreachable("Invalid fence export handle type");
1736 /* From the Vulkan 1.0.53 spec:
1738 * "Export operations have the same transference as the specified handle
1739 * type’s import operations. [...] If the fence was using a
1740 * temporarily imported payload, the fence’s prior permanent payload
1743 if (impl
== &fence
->temporary
)
1744 anv_fence_impl_cleanup(device
, impl
);
1749 // Queue semaphore functions
1751 static VkSemaphoreTypeKHR
1752 get_semaphore_type(const void *pNext
, uint64_t *initial_value
)
1754 const VkSemaphoreTypeCreateInfoKHR
*type_info
=
1755 vk_find_struct_const(pNext
, SEMAPHORE_TYPE_CREATE_INFO_KHR
);
1758 return VK_SEMAPHORE_TYPE_BINARY_KHR
;
1761 *initial_value
= type_info
->initialValue
;
1762 return type_info
->semaphoreType
;
1766 binary_semaphore_create(struct anv_device
*device
,
1767 struct anv_semaphore_impl
*impl
,
1770 if (device
->physical
->has_syncobj
) {
1771 impl
->type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
;
1772 impl
->syncobj
= anv_gem_syncobj_create(device
, 0);
1774 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1777 impl
->type
= ANV_SEMAPHORE_TYPE_BO
;
1779 anv_device_alloc_bo(device
, 4096,
1780 ANV_BO_ALLOC_EXTERNAL
|
1781 ANV_BO_ALLOC_IMPLICIT_SYNC
,
1782 0 /* explicit_address */,
1784 /* If we're going to use this as a fence, we need to *not* have the
1785 * EXEC_OBJECT_ASYNC bit set.
1787 assert(!(impl
->bo
->flags
& EXEC_OBJECT_ASYNC
));
1793 timeline_semaphore_create(struct anv_device
*device
,
1794 struct anv_semaphore_impl
*impl
,
1795 uint64_t initial_value
)
1797 impl
->type
= ANV_SEMAPHORE_TYPE_TIMELINE
;
1798 anv_timeline_init(device
, &impl
->timeline
, initial_value
);
1802 VkResult
anv_CreateSemaphore(
1804 const VkSemaphoreCreateInfo
* pCreateInfo
,
1805 const VkAllocationCallbacks
* pAllocator
,
1806 VkSemaphore
* pSemaphore
)
1808 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1809 struct anv_semaphore
*semaphore
;
1811 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO
);
1813 uint64_t timeline_value
= 0;
1814 VkSemaphoreTypeKHR sem_type
= get_semaphore_type(pCreateInfo
->pNext
, &timeline_value
);
1816 semaphore
= vk_alloc(&device
->vk
.alloc
, sizeof(*semaphore
), 8,
1817 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE
);
1818 if (semaphore
== NULL
)
1819 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1821 vk_object_base_init(&device
->vk
, &semaphore
->base
, VK_OBJECT_TYPE_SEMAPHORE
);
1823 p_atomic_set(&semaphore
->refcount
, 1);
1825 const VkExportSemaphoreCreateInfo
*export
=
1826 vk_find_struct_const(pCreateInfo
->pNext
, EXPORT_SEMAPHORE_CREATE_INFO
);
1827 VkExternalSemaphoreHandleTypeFlags handleTypes
=
1828 export
? export
->handleTypes
: 0;
1831 if (handleTypes
== 0) {
1832 if (sem_type
== VK_SEMAPHORE_TYPE_BINARY_KHR
)
1833 result
= binary_semaphore_create(device
, &semaphore
->permanent
, false);
1835 result
= timeline_semaphore_create(device
, &semaphore
->permanent
, timeline_value
);
1836 if (result
!= VK_SUCCESS
) {
1837 vk_free2(&device
->vk
.alloc
, pAllocator
, semaphore
);
1840 } else if (handleTypes
& VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
) {
1841 assert(handleTypes
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
);
1842 assert(sem_type
== VK_SEMAPHORE_TYPE_BINARY_KHR
);
1843 result
= binary_semaphore_create(device
, &semaphore
->permanent
, true);
1844 if (result
!= VK_SUCCESS
) {
1845 vk_free2(&device
->vk
.alloc
, pAllocator
, semaphore
);
1848 } else if (handleTypes
& VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
) {
1849 assert(handleTypes
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
);
1850 assert(sem_type
== VK_SEMAPHORE_TYPE_BINARY_KHR
);
1851 if (device
->physical
->has_syncobj
) {
1852 semaphore
->permanent
.type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
;
1853 semaphore
->permanent
.syncobj
= anv_gem_syncobj_create(device
, 0);
1854 if (!semaphore
->permanent
.syncobj
) {
1855 vk_free2(&device
->vk
.alloc
, pAllocator
, semaphore
);
1856 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1859 semaphore
->permanent
.type
= ANV_SEMAPHORE_TYPE_SYNC_FILE
;
1860 semaphore
->permanent
.fd
= -1;
1863 assert(!"Unknown handle type");
1864 vk_free2(&device
->vk
.alloc
, pAllocator
, semaphore
);
1865 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1868 semaphore
->temporary
.type
= ANV_SEMAPHORE_TYPE_NONE
;
1870 *pSemaphore
= anv_semaphore_to_handle(semaphore
);
1876 anv_semaphore_impl_cleanup(struct anv_device
*device
,
1877 struct anv_semaphore_impl
*impl
)
1879 switch (impl
->type
) {
1880 case ANV_SEMAPHORE_TYPE_NONE
:
1881 case ANV_SEMAPHORE_TYPE_DUMMY
:
1882 /* Dummy. Nothing to do */
1885 case ANV_SEMAPHORE_TYPE_BO
:
1886 case ANV_SEMAPHORE_TYPE_WSI_BO
:
1887 anv_device_release_bo(device
, impl
->bo
);
1890 case ANV_SEMAPHORE_TYPE_SYNC_FILE
:
1895 case ANV_SEMAPHORE_TYPE_TIMELINE
:
1896 anv_timeline_finish(device
, &impl
->timeline
);
1899 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
:
1900 anv_gem_syncobj_destroy(device
, impl
->syncobj
);
1904 unreachable("Invalid semaphore type");
1907 impl
->type
= ANV_SEMAPHORE_TYPE_NONE
;
1911 anv_semaphore_reset_temporary(struct anv_device
*device
,
1912 struct anv_semaphore
*semaphore
)
1914 if (semaphore
->temporary
.type
== ANV_SEMAPHORE_TYPE_NONE
)
1917 anv_semaphore_impl_cleanup(device
, &semaphore
->temporary
);
1920 static struct anv_semaphore
*
1921 anv_semaphore_ref(struct anv_semaphore
*semaphore
)
1923 assert(semaphore
->refcount
);
1924 p_atomic_inc(&semaphore
->refcount
);
1929 anv_semaphore_unref(struct anv_device
*device
, struct anv_semaphore
*semaphore
)
1931 if (!p_atomic_dec_zero(&semaphore
->refcount
))
1934 anv_semaphore_impl_cleanup(device
, &semaphore
->temporary
);
1935 anv_semaphore_impl_cleanup(device
, &semaphore
->permanent
);
1937 vk_object_base_finish(&semaphore
->base
);
1938 vk_free(&device
->vk
.alloc
, semaphore
);
1941 void anv_DestroySemaphore(
1943 VkSemaphore _semaphore
,
1944 const VkAllocationCallbacks
* pAllocator
)
1946 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1947 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, _semaphore
);
1949 if (semaphore
== NULL
)
1952 anv_semaphore_unref(device
, semaphore
);
1955 void anv_GetPhysicalDeviceExternalSemaphoreProperties(
1956 VkPhysicalDevice physicalDevice
,
1957 const VkPhysicalDeviceExternalSemaphoreInfo
* pExternalSemaphoreInfo
,
1958 VkExternalSemaphoreProperties
* pExternalSemaphoreProperties
)
1960 ANV_FROM_HANDLE(anv_physical_device
, device
, physicalDevice
);
1962 VkSemaphoreTypeKHR sem_type
=
1963 get_semaphore_type(pExternalSemaphoreInfo
->pNext
, NULL
);
1965 switch (pExternalSemaphoreInfo
->handleType
) {
1966 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1967 /* Timeline semaphores are not exportable. */
1968 if (sem_type
== VK_SEMAPHORE_TYPE_TIMELINE_KHR
)
1970 pExternalSemaphoreProperties
->exportFromImportedHandleTypes
=
1971 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
;
1972 pExternalSemaphoreProperties
->compatibleHandleTypes
=
1973 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
;
1974 pExternalSemaphoreProperties
->externalSemaphoreFeatures
=
1975 VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT
|
1976 VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT
;
1979 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
:
1980 if (sem_type
== VK_SEMAPHORE_TYPE_TIMELINE_KHR
)
1982 if (!device
->has_exec_fence
)
1984 pExternalSemaphoreProperties
->exportFromImportedHandleTypes
=
1985 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
;
1986 pExternalSemaphoreProperties
->compatibleHandleTypes
=
1987 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
;
1988 pExternalSemaphoreProperties
->externalSemaphoreFeatures
=
1989 VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT
|
1990 VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT
;
1997 pExternalSemaphoreProperties
->exportFromImportedHandleTypes
= 0;
1998 pExternalSemaphoreProperties
->compatibleHandleTypes
= 0;
1999 pExternalSemaphoreProperties
->externalSemaphoreFeatures
= 0;
2002 VkResult
anv_ImportSemaphoreFdKHR(
2004 const VkImportSemaphoreFdInfoKHR
* pImportSemaphoreFdInfo
)
2006 ANV_FROM_HANDLE(anv_device
, device
, _device
);
2007 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pImportSemaphoreFdInfo
->semaphore
);
2008 int fd
= pImportSemaphoreFdInfo
->fd
;
2010 struct anv_semaphore_impl new_impl
= {
2011 .type
= ANV_SEMAPHORE_TYPE_NONE
,
2014 switch (pImportSemaphoreFdInfo
->handleType
) {
2015 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
:
2016 if (device
->physical
->has_syncobj
) {
2017 new_impl
.type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
;
2019 new_impl
.syncobj
= anv_gem_syncobj_fd_to_handle(device
, fd
);
2020 if (!new_impl
.syncobj
)
2021 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
2023 new_impl
.type
= ANV_SEMAPHORE_TYPE_BO
;
2025 VkResult result
= anv_device_import_bo(device
, fd
,
2026 ANV_BO_ALLOC_EXTERNAL
|
2027 ANV_BO_ALLOC_IMPLICIT_SYNC
,
2028 0 /* client_address */,
2030 if (result
!= VK_SUCCESS
)
2033 if (new_impl
.bo
->size
< 4096) {
2034 anv_device_release_bo(device
, new_impl
.bo
);
2035 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
2038 /* If we're going to use this as a fence, we need to *not* have the
2039 * EXEC_OBJECT_ASYNC bit set.
2041 assert(!(new_impl
.bo
->flags
& EXEC_OBJECT_ASYNC
));
2044 /* From the Vulkan spec:
2046 * "Importing semaphore state from a file descriptor transfers
2047 * ownership of the file descriptor from the application to the
2048 * Vulkan implementation. The application must not perform any
2049 * operations on the file descriptor after a successful import."
2051 * If the import fails, we leave the file descriptor open.
2056 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
:
2057 if (device
->physical
->has_syncobj
) {
2058 uint32_t create_flags
= 0;
2061 create_flags
|= DRM_SYNCOBJ_CREATE_SIGNALED
;
2063 new_impl
= (struct anv_semaphore_impl
) {
2064 .type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
,
2065 .syncobj
= anv_gem_syncobj_create(device
, create_flags
),
2068 if (!new_impl
.syncobj
)
2069 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
2072 if (anv_gem_syncobj_import_sync_file(device
, new_impl
.syncobj
, fd
)) {
2073 anv_gem_syncobj_destroy(device
, new_impl
.syncobj
);
2074 return vk_errorf(device
, NULL
, VK_ERROR_INVALID_EXTERNAL_HANDLE
,
2075 "syncobj sync file import failed: %m");
2077 /* Ownership of the FD is transfered to Anv. Since we don't need it
2078 * anymore because the associated fence has been put into a syncobj,
2079 * we must close the FD.
2084 new_impl
= (struct anv_semaphore_impl
) {
2085 .type
= ANV_SEMAPHORE_TYPE_SYNC_FILE
,
2092 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
2095 if (pImportSemaphoreFdInfo
->flags
& VK_SEMAPHORE_IMPORT_TEMPORARY_BIT
) {
2096 anv_semaphore_impl_cleanup(device
, &semaphore
->temporary
);
2097 semaphore
->temporary
= new_impl
;
2099 anv_semaphore_impl_cleanup(device
, &semaphore
->permanent
);
2100 semaphore
->permanent
= new_impl
;
2106 VkResult
anv_GetSemaphoreFdKHR(
2108 const VkSemaphoreGetFdInfoKHR
* pGetFdInfo
,
2111 ANV_FROM_HANDLE(anv_device
, device
, _device
);
2112 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pGetFdInfo
->semaphore
);
2116 assert(pGetFdInfo
->sType
== VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR
);
2118 struct anv_semaphore_impl
*impl
=
2119 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
2120 &semaphore
->temporary
: &semaphore
->permanent
;
2122 switch (impl
->type
) {
2123 case ANV_SEMAPHORE_TYPE_BO
:
2124 result
= anv_device_export_bo(device
, impl
->bo
, pFd
);
2125 if (result
!= VK_SUCCESS
)
2129 case ANV_SEMAPHORE_TYPE_SYNC_FILE
: {
2130 /* There's a potential race here with vkQueueSubmit if you are trying
2131 * to export a semaphore Fd while the queue submit is still happening.
2132 * This can happen if we see all dependencies get resolved via timeline
2133 * semaphore waits completing before the execbuf completes and we
2134 * process the resulting out fence. To work around this, take a lock
2135 * around grabbing the fd.
2137 pthread_mutex_lock(&device
->mutex
);
2139 /* From the Vulkan 1.0.53 spec:
2141 * "...exporting a semaphore payload to a handle with copy
2142 * transference has the same side effects on the source
2143 * semaphore’s payload as executing a semaphore wait operation."
2145 * In other words, it may still be a SYNC_FD semaphore, but it's now
2146 * considered to have been waited on and no longer has a sync file
2152 pthread_mutex_unlock(&device
->mutex
);
2154 /* There are two reasons why this could happen:
2156 * 1) The user is trying to export without submitting something that
2157 * signals the semaphore. If this is the case, it's their bug so
2158 * what we return here doesn't matter.
2160 * 2) The kernel didn't give us a file descriptor. The most likely
2161 * reason for this is running out of file descriptors.
2164 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
2170 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
:
2171 if (pGetFdInfo
->handleType
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
)
2172 fd
= anv_gem_syncobj_export_sync_file(device
, impl
->syncobj
);
2174 assert(pGetFdInfo
->handleType
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
);
2175 fd
= anv_gem_syncobj_handle_to_fd(device
, impl
->syncobj
);
2178 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
2183 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
2186 /* From the Vulkan 1.0.53 spec:
2188 * "Export operations have the same transference as the specified handle
2189 * type’s import operations. [...] If the semaphore was using a
2190 * temporarily imported payload, the semaphore’s prior permanent payload
2193 if (impl
== &semaphore
->temporary
)
2194 anv_semaphore_impl_cleanup(device
, impl
);
2199 VkResult
anv_GetSemaphoreCounterValue(
2201 VkSemaphore _semaphore
,
2204 ANV_FROM_HANDLE(anv_device
, device
, _device
);
2205 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, _semaphore
);
2207 struct anv_semaphore_impl
*impl
=
2208 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
2209 &semaphore
->temporary
: &semaphore
->permanent
;
2211 switch (impl
->type
) {
2212 case ANV_SEMAPHORE_TYPE_TIMELINE
: {
2213 pthread_mutex_lock(&device
->mutex
);
2214 anv_timeline_gc_locked(device
, &impl
->timeline
);
2215 *pValue
= impl
->timeline
.highest_past
;
2216 pthread_mutex_unlock(&device
->mutex
);
2221 unreachable("Invalid semaphore type");
2226 anv_timeline_wait_locked(struct anv_device
*device
,
2227 struct anv_timeline
*timeline
,
2228 uint64_t serial
, uint64_t abs_timeout_ns
)
2230 /* Wait on the queue_submit condition variable until the timeline has a
2231 * time point pending that's at least as high as serial.
2233 while (timeline
->highest_pending
< serial
) {
2234 struct timespec abstime
= {
2235 .tv_sec
= abs_timeout_ns
/ NSEC_PER_SEC
,
2236 .tv_nsec
= abs_timeout_ns
% NSEC_PER_SEC
,
2239 int ret
= pthread_cond_timedwait(&device
->queue_submit
,
2240 &device
->mutex
, &abstime
);
2241 assert(ret
!= EINVAL
);
2242 if (anv_gettime_ns() >= abs_timeout_ns
&&
2243 timeline
->highest_pending
< serial
)
2248 VkResult result
= anv_timeline_gc_locked(device
, timeline
);
2249 if (result
!= VK_SUCCESS
)
2252 if (timeline
->highest_past
>= serial
)
2255 /* If we got here, our earliest time point has a busy BO */
2256 struct anv_timeline_point
*point
=
2257 list_first_entry(&timeline
->points
,
2258 struct anv_timeline_point
, link
);
2260 /* Drop the lock while we wait. */
2262 pthread_mutex_unlock(&device
->mutex
);
2264 result
= anv_device_wait(device
, point
->bo
,
2265 anv_get_relative_timeout(abs_timeout_ns
));
2267 /* Pick the mutex back up */
2268 pthread_mutex_lock(&device
->mutex
);
2271 /* This covers both VK_TIMEOUT and VK_ERROR_DEVICE_LOST */
2272 if (result
!= VK_SUCCESS
)
2278 anv_timelines_wait(struct anv_device
*device
,
2279 struct anv_timeline
**timelines
,
2280 const uint64_t *serials
,
2281 uint32_t n_timelines
,
2283 uint64_t abs_timeout_ns
)
2285 if (!wait_all
&& n_timelines
> 1) {
2286 pthread_mutex_lock(&device
->mutex
);
2290 for (uint32_t i
= 0; i
< n_timelines
; i
++) {
2292 anv_timeline_wait_locked(device
, timelines
[i
], serials
[i
], 0);
2293 if (result
!= VK_TIMEOUT
)
2297 if (result
!= VK_TIMEOUT
||
2298 anv_gettime_ns() >= abs_timeout_ns
) {
2299 pthread_mutex_unlock(&device
->mutex
);
2303 /* If none of them are ready do a short wait so we don't completely
2304 * spin while holding the lock. The 10us is completely arbitrary.
2306 uint64_t abs_short_wait_ns
=
2307 anv_get_absolute_timeout(
2308 MIN2((anv_gettime_ns() - abs_timeout_ns
) / 10, 10 * 1000));
2309 struct timespec abstime
= {
2310 .tv_sec
= abs_short_wait_ns
/ NSEC_PER_SEC
,
2311 .tv_nsec
= abs_short_wait_ns
% NSEC_PER_SEC
,
2314 ret
= pthread_cond_timedwait(&device
->queue_submit
,
2315 &device
->mutex
, &abstime
);
2316 assert(ret
!= EINVAL
);
2319 VkResult result
= VK_SUCCESS
;
2320 pthread_mutex_lock(&device
->mutex
);
2321 for (uint32_t i
= 0; i
< n_timelines
; i
++) {
2323 anv_timeline_wait_locked(device
, timelines
[i
],
2324 serials
[i
], abs_timeout_ns
);
2325 if (result
!= VK_SUCCESS
)
2328 pthread_mutex_unlock(&device
->mutex
);
2333 VkResult
anv_WaitSemaphores(
2335 const VkSemaphoreWaitInfoKHR
* pWaitInfo
,
2338 ANV_FROM_HANDLE(anv_device
, device
, _device
);
2343 struct anv_timeline
**timelines
=
2344 vk_alloc(&device
->vk
.alloc
,
2345 pWaitInfo
->semaphoreCount
* sizeof(*timelines
),
2346 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND
);
2348 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
2350 uint64_t *values
= vk_alloc(&device
->vk
.alloc
,
2351 pWaitInfo
->semaphoreCount
* sizeof(*values
),
2352 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND
);
2354 vk_free(&device
->vk
.alloc
, timelines
);
2355 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
2358 uint32_t handle_count
= 0;
2359 for (uint32_t i
= 0; i
< pWaitInfo
->semaphoreCount
; i
++) {
2360 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pWaitInfo
->pSemaphores
[i
]);
2361 struct anv_semaphore_impl
*impl
=
2362 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
2363 &semaphore
->temporary
: &semaphore
->permanent
;
2365 assert(impl
->type
== ANV_SEMAPHORE_TYPE_TIMELINE
);
2367 if (pWaitInfo
->pValues
[i
] == 0)
2370 timelines
[handle_count
] = &impl
->timeline
;
2371 values
[handle_count
] = pWaitInfo
->pValues
[i
];
2375 VkResult result
= VK_SUCCESS
;
2376 if (handle_count
> 0) {
2377 result
= anv_timelines_wait(device
, timelines
, values
, handle_count
,
2378 !(pWaitInfo
->flags
& VK_SEMAPHORE_WAIT_ANY_BIT_KHR
),
2379 anv_get_absolute_timeout(timeout
));
2382 vk_free(&device
->vk
.alloc
, timelines
);
2383 vk_free(&device
->vk
.alloc
, values
);
2388 VkResult
anv_SignalSemaphore(
2390 const VkSemaphoreSignalInfoKHR
* pSignalInfo
)
2392 ANV_FROM_HANDLE(anv_device
, device
, _device
);
2393 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pSignalInfo
->semaphore
);
2395 struct anv_semaphore_impl
*impl
=
2396 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
2397 &semaphore
->temporary
: &semaphore
->permanent
;
2399 switch (impl
->type
) {
2400 case ANV_SEMAPHORE_TYPE_TIMELINE
: {
2401 pthread_mutex_lock(&device
->mutex
);
2403 VkResult result
= anv_timeline_gc_locked(device
, &impl
->timeline
);
2405 assert(pSignalInfo
->value
> impl
->timeline
.highest_pending
);
2407 impl
->timeline
.highest_pending
= impl
->timeline
.highest_past
= pSignalInfo
->value
;
2409 if (result
== VK_SUCCESS
)
2410 result
= anv_device_submit_deferred_locked(device
);
2412 pthread_cond_broadcast(&device
->queue_submit
);
2413 pthread_mutex_unlock(&device
->mutex
);
2418 unreachable("Invalid semaphore type");