2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 * This file implements VkQueue, VkFence, and VkSemaphore
32 #include "anv_private.h"
35 #include "genxml/gen7_pack.h"
37 uint64_t anv_gettime_ns(void)
39 struct timespec current
;
40 clock_gettime(CLOCK_MONOTONIC
, ¤t
);
41 return (uint64_t)current
.tv_sec
* NSEC_PER_SEC
+ current
.tv_nsec
;
44 uint64_t anv_get_absolute_timeout(uint64_t timeout
)
48 uint64_t current_time
= anv_gettime_ns();
49 uint64_t max_timeout
= (uint64_t) INT64_MAX
- current_time
;
51 timeout
= MIN2(max_timeout
, timeout
);
53 return (current_time
+ timeout
);
56 static int64_t anv_get_relative_timeout(uint64_t abs_timeout
)
58 uint64_t now
= anv_gettime_ns();
60 /* We don't want negative timeouts.
62 * DRM_IOCTL_I915_GEM_WAIT uses a signed 64 bit timeout and is
63 * supposed to block indefinitely timeouts < 0. Unfortunately,
64 * this was broken for a couple of kernel releases. Since there's
65 * no way to know whether or not the kernel we're using is one of
66 * the broken ones, the best we can do is to clamp the timeout to
67 * INT64_MAX. This limits the maximum timeout from 584 years to
68 * 292 years - likely not a big deal.
70 if (abs_timeout
< now
)
73 uint64_t rel_timeout
= abs_timeout
- now
;
74 if (rel_timeout
> (uint64_t) INT64_MAX
)
75 rel_timeout
= INT64_MAX
;
80 static struct anv_semaphore
*anv_semaphore_ref(struct anv_semaphore
*semaphore
);
81 static void anv_semaphore_unref(struct anv_device
*device
, struct anv_semaphore
*semaphore
);
82 static void anv_semaphore_impl_cleanup(struct anv_device
*device
,
83 struct anv_semaphore_impl
*impl
);
86 anv_queue_submit_free(struct anv_device
*device
,
87 struct anv_queue_submit
*submit
)
89 const VkAllocationCallbacks
*alloc
= submit
->alloc
;
91 for (uint32_t i
= 0; i
< submit
->temporary_semaphore_count
; i
++)
92 anv_semaphore_impl_cleanup(device
, &submit
->temporary_semaphores
[i
]);
93 for (uint32_t i
= 0; i
< submit
->sync_fd_semaphore_count
; i
++)
94 anv_semaphore_unref(device
, submit
->sync_fd_semaphores
[i
]);
95 /* Execbuf does not consume the in_fence. It's our job to close it. */
96 if (submit
->in_fence
!= -1)
97 close(submit
->in_fence
);
98 if (submit
->out_fence
!= -1)
99 close(submit
->out_fence
);
100 vk_free(alloc
, submit
->fences
);
101 vk_free(alloc
, submit
->temporary_semaphores
);
102 vk_free(alloc
, submit
->wait_timelines
);
103 vk_free(alloc
, submit
->wait_timeline_values
);
104 vk_free(alloc
, submit
->signal_timelines
);
105 vk_free(alloc
, submit
->signal_timeline_values
);
106 vk_free(alloc
, submit
->fence_bos
);
107 vk_free(alloc
, submit
);
111 anv_queue_submit_ready_locked(struct anv_queue_submit
*submit
)
113 for (uint32_t i
= 0; i
< submit
->wait_timeline_count
; i
++) {
114 if (submit
->wait_timeline_values
[i
] > submit
->wait_timelines
[i
]->highest_pending
)
122 anv_timeline_init(struct anv_device
*device
,
123 struct anv_timeline
*timeline
,
124 uint64_t initial_value
)
126 timeline
->highest_past
=
127 timeline
->highest_pending
= initial_value
;
128 list_inithead(&timeline
->points
);
129 list_inithead(&timeline
->free_points
);
135 anv_timeline_finish(struct anv_device
*device
,
136 struct anv_timeline
*timeline
)
138 list_for_each_entry_safe(struct anv_timeline_point
, point
,
139 &timeline
->free_points
, link
) {
140 list_del(&point
->link
);
141 anv_device_release_bo(device
, point
->bo
);
142 vk_free(&device
->alloc
, point
);
144 list_for_each_entry_safe(struct anv_timeline_point
, point
,
145 &timeline
->points
, link
) {
146 list_del(&point
->link
);
147 anv_device_release_bo(device
, point
->bo
);
148 vk_free(&device
->alloc
, point
);
153 anv_timeline_add_point_locked(struct anv_device
*device
,
154 struct anv_timeline
*timeline
,
156 struct anv_timeline_point
**point
)
158 VkResult result
= VK_SUCCESS
;
160 if (list_is_empty(&timeline
->free_points
)) {
162 vk_zalloc(&device
->alloc
, sizeof(**point
),
163 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE
);
165 result
= vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
166 if (result
== VK_SUCCESS
) {
167 result
= anv_device_alloc_bo(device
, 4096,
168 ANV_BO_ALLOC_EXTERNAL
|
169 ANV_BO_ALLOC_IMPLICIT_SYNC
,
171 if (result
!= VK_SUCCESS
)
172 vk_free(&device
->alloc
, *point
);
175 *point
= list_first_entry(&timeline
->free_points
,
176 struct anv_timeline_point
, link
);
177 list_del(&(*point
)->link
);
180 if (result
== VK_SUCCESS
) {
181 (*point
)->serial
= value
;
182 list_addtail(&(*point
)->link
, &timeline
->points
);
189 anv_timeline_gc_locked(struct anv_device
*device
,
190 struct anv_timeline
*timeline
)
192 list_for_each_entry_safe(struct anv_timeline_point
, point
,
193 &timeline
->points
, link
) {
194 /* timeline->higest_pending is only incremented once submission has
195 * happened. If this point has a greater serial, it means the point
196 * hasn't been submitted yet.
198 if (point
->serial
> timeline
->highest_pending
)
201 /* If someone is waiting on this time point, consider it busy and don't
202 * try to recycle it. There's a slim possibility that it's no longer
203 * busy by the time we look at it but we would be recycling it out from
204 * under a waiter and that can lead to weird races.
206 * We walk the list in-order so if this time point is still busy so is
207 * every following time point
209 assert(point
->waiting
>= 0);
213 /* Garbage collect any signaled point. */
214 VkResult result
= anv_device_bo_busy(device
, point
->bo
);
215 if (result
== VK_NOT_READY
) {
216 /* We walk the list in-order so if this time point is still busy so
217 * is every following time point
220 } else if (result
!= VK_SUCCESS
) {
224 assert(timeline
->highest_past
< point
->serial
);
225 timeline
->highest_past
= point
->serial
;
227 list_del(&point
->link
);
228 list_add(&point
->link
, &timeline
->free_points
);
234 static VkResult
anv_queue_submit_add_fence_bo(struct anv_queue_submit
*submit
,
239 anv_queue_submit_timeline_locked(struct anv_queue
*queue
,
240 struct anv_queue_submit
*submit
)
244 for (uint32_t i
= 0; i
< submit
->wait_timeline_count
; i
++) {
245 struct anv_timeline
*timeline
= submit
->wait_timelines
[i
];
246 uint64_t wait_value
= submit
->wait_timeline_values
[i
];
248 if (timeline
->highest_past
>= wait_value
)
251 list_for_each_entry(struct anv_timeline_point
, point
, &timeline
->points
, link
) {
252 if (point
->serial
< wait_value
)
254 result
= anv_queue_submit_add_fence_bo(submit
, point
->bo
, false);
255 if (result
!= VK_SUCCESS
)
260 for (uint32_t i
= 0; i
< submit
->signal_timeline_count
; i
++) {
261 struct anv_timeline
*timeline
= submit
->signal_timelines
[i
];
262 uint64_t signal_value
= submit
->signal_timeline_values
[i
];
263 struct anv_timeline_point
*point
;
265 result
= anv_timeline_add_point_locked(queue
->device
, timeline
,
266 signal_value
, &point
);
267 if (result
!= VK_SUCCESS
)
270 result
= anv_queue_submit_add_fence_bo(submit
, point
->bo
, true);
271 if (result
!= VK_SUCCESS
)
275 result
= anv_queue_execbuf_locked(queue
, submit
);
277 if (result
== VK_SUCCESS
) {
278 /* Update the pending values in the timeline objects. */
279 for (uint32_t i
= 0; i
< submit
->signal_timeline_count
; i
++) {
280 struct anv_timeline
*timeline
= submit
->signal_timelines
[i
];
281 uint64_t signal_value
= submit
->signal_timeline_values
[i
];
283 assert(signal_value
> timeline
->highest_pending
);
284 timeline
->highest_pending
= signal_value
;
287 /* Update signaled semaphores backed by syncfd. */
288 for (uint32_t i
= 0; i
< submit
->sync_fd_semaphore_count
; i
++) {
289 struct anv_semaphore
*semaphore
= submit
->sync_fd_semaphores
[i
];
290 /* Out fences can't have temporary state because that would imply
291 * that we imported a sync file and are trying to signal it.
293 assert(semaphore
->temporary
.type
== ANV_SEMAPHORE_TYPE_NONE
);
294 struct anv_semaphore_impl
*impl
= &semaphore
->permanent
;
296 assert(impl
->type
== ANV_SEMAPHORE_TYPE_SYNC_FILE
);
297 impl
->fd
= dup(submit
->out_fence
);
300 /* Unblock any waiter by signaling the points, the application will get
301 * a device lost error code.
303 for (uint32_t i
= 0; i
< submit
->signal_timeline_count
; i
++) {
304 struct anv_timeline
*timeline
= submit
->signal_timelines
[i
];
305 uint64_t signal_value
= submit
->signal_timeline_values
[i
];
307 assert(signal_value
> timeline
->highest_pending
);
308 timeline
->highest_past
= timeline
->highest_pending
= signal_value
;
316 anv_queue_submit_deferred_locked(struct anv_queue
*queue
, uint32_t *advance
)
318 VkResult result
= VK_SUCCESS
;
320 /* Go through all the queued submissions and submit then until we find one
321 * that's waiting on a point that hasn't materialized yet.
323 list_for_each_entry_safe(struct anv_queue_submit
, submit
,
324 &queue
->queued_submits
, link
) {
325 if (!anv_queue_submit_ready_locked(submit
))
329 list_del(&submit
->link
);
331 result
= anv_queue_submit_timeline_locked(queue
, submit
);
333 anv_queue_submit_free(queue
->device
, submit
);
335 if (result
!= VK_SUCCESS
)
343 anv_device_submit_deferred_locked(struct anv_device
*device
)
345 uint32_t advance
= 0;
346 return anv_queue_submit_deferred_locked(&device
->queue
, &advance
);
350 _anv_queue_submit(struct anv_queue
*queue
, struct anv_queue_submit
**_submit
)
352 struct anv_queue_submit
*submit
= *_submit
;
354 /* Wait before signal behavior means we might keep alive the
355 * anv_queue_submit object a bit longer, so transfer the ownership to the
360 pthread_mutex_lock(&queue
->device
->mutex
);
361 list_addtail(&submit
->link
, &queue
->queued_submits
);
362 VkResult result
= anv_device_submit_deferred_locked(queue
->device
);
363 pthread_mutex_unlock(&queue
->device
->mutex
);
368 anv_queue_init(struct anv_device
*device
, struct anv_queue
*queue
)
370 queue
->_loader_data
.loaderMagic
= ICD_LOADER_MAGIC
;
371 queue
->device
= device
;
374 list_inithead(&queue
->queued_submits
);
380 anv_queue_finish(struct anv_queue
*queue
)
385 anv_queue_submit_add_fence_bo(struct anv_queue_submit
*submit
,
389 if (submit
->fence_bo_count
>= submit
->fence_bo_array_length
) {
390 uint32_t new_len
= MAX2(submit
->fence_bo_array_length
* 2, 64);
393 vk_realloc(submit
->alloc
,
394 submit
->fence_bos
, new_len
* sizeof(*submit
->fence_bos
),
395 8, submit
->alloc_scope
);
396 if (submit
->fence_bos
== NULL
)
397 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
399 submit
->fence_bo_array_length
= new_len
;
402 /* Take advantage that anv_bo are allocated at 8 byte alignement so we can
403 * use the lowest bit to store whether this is a BO we need to signal.
405 submit
->fence_bos
[submit
->fence_bo_count
++] = anv_pack_ptr(bo
, 1, signal
);
411 anv_queue_submit_add_syncobj(struct anv_queue_submit
* submit
,
412 struct anv_device
*device
,
413 uint32_t handle
, uint32_t flags
)
417 if (submit
->fence_count
>= submit
->fence_array_length
) {
418 uint32_t new_len
= MAX2(submit
->fence_array_length
* 2, 64);
421 vk_realloc(submit
->alloc
,
422 submit
->fences
, new_len
* sizeof(*submit
->fences
),
423 8, submit
->alloc_scope
);
424 if (submit
->fences
== NULL
)
425 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
427 submit
->fence_array_length
= new_len
;
430 submit
->fences
[submit
->fence_count
++] = (struct drm_i915_gem_exec_fence
) {
439 anv_queue_submit_add_sync_fd_fence(struct anv_queue_submit
*submit
,
440 struct anv_semaphore
*semaphore
)
442 if (submit
->sync_fd_semaphore_count
>= submit
->sync_fd_semaphore_array_length
) {
443 uint32_t new_len
= MAX2(submit
->sync_fd_semaphore_array_length
* 2, 64);
444 struct anv_semaphore
**new_semaphores
=
445 vk_realloc(submit
->alloc
, submit
->sync_fd_semaphores
,
446 new_len
* sizeof(*submit
->sync_fd_semaphores
), 8,
447 submit
->alloc_scope
);
448 if (new_semaphores
== NULL
)
449 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
451 submit
->sync_fd_semaphores
= new_semaphores
;
454 submit
->sync_fd_semaphores
[submit
->sync_fd_semaphore_count
++] =
455 anv_semaphore_ref(semaphore
);
456 submit
->need_out_fence
= true;
462 anv_queue_submit_add_timeline_wait(struct anv_queue_submit
* submit
,
463 struct anv_device
*device
,
464 struct anv_timeline
*timeline
,
467 if (submit
->wait_timeline_count
>= submit
->wait_timeline_array_length
) {
468 uint32_t new_len
= MAX2(submit
->wait_timeline_array_length
* 2, 64);
470 submit
->wait_timelines
=
471 vk_realloc(submit
->alloc
,
472 submit
->wait_timelines
, new_len
* sizeof(*submit
->wait_timelines
),
473 8, submit
->alloc_scope
);
474 if (submit
->wait_timelines
== NULL
)
475 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
477 submit
->wait_timeline_values
=
478 vk_realloc(submit
->alloc
,
479 submit
->wait_timeline_values
, new_len
* sizeof(*submit
->wait_timeline_values
),
480 8, submit
->alloc_scope
);
481 if (submit
->wait_timeline_values
== NULL
)
482 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
484 submit
->wait_timeline_array_length
= new_len
;
487 submit
->wait_timelines
[submit
->wait_timeline_count
] = timeline
;
488 submit
->wait_timeline_values
[submit
->wait_timeline_count
] = value
;
490 submit
->wait_timeline_count
++;
496 anv_queue_submit_add_timeline_signal(struct anv_queue_submit
* submit
,
497 struct anv_device
*device
,
498 struct anv_timeline
*timeline
,
501 assert(timeline
->highest_pending
< value
);
503 if (submit
->signal_timeline_count
>= submit
->signal_timeline_array_length
) {
504 uint32_t new_len
= MAX2(submit
->signal_timeline_array_length
* 2, 64);
506 submit
->signal_timelines
=
507 vk_realloc(submit
->alloc
,
508 submit
->signal_timelines
, new_len
* sizeof(*submit
->signal_timelines
),
509 8, submit
->alloc_scope
);
510 if (submit
->signal_timelines
== NULL
)
511 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
513 submit
->signal_timeline_values
=
514 vk_realloc(submit
->alloc
,
515 submit
->signal_timeline_values
, new_len
* sizeof(*submit
->signal_timeline_values
),
516 8, submit
->alloc_scope
);
517 if (submit
->signal_timeline_values
== NULL
)
518 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
520 submit
->signal_timeline_array_length
= new_len
;
523 submit
->signal_timelines
[submit
->signal_timeline_count
] = timeline
;
524 submit
->signal_timeline_values
[submit
->signal_timeline_count
] = value
;
526 submit
->signal_timeline_count
++;
531 static struct anv_queue_submit
*
532 anv_queue_submit_alloc(struct anv_device
*device
)
534 const VkAllocationCallbacks
*alloc
= &device
->alloc
;
535 VkSystemAllocationScope alloc_scope
= VK_SYSTEM_ALLOCATION_SCOPE_DEVICE
;
537 struct anv_queue_submit
*submit
= vk_zalloc(alloc
, sizeof(*submit
), 8, alloc_scope
);
541 submit
->alloc
= alloc
;
542 submit
->alloc_scope
= alloc_scope
;
543 submit
->in_fence
= -1;
544 submit
->out_fence
= -1;
550 anv_queue_submit_simple_batch(struct anv_queue
*queue
,
551 struct anv_batch
*batch
)
553 struct anv_device
*device
= queue
->device
;
554 struct anv_queue_submit
*submit
= anv_queue_submit_alloc(device
);
556 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
558 bool has_syncobj_wait
= device
->instance
->physicalDevice
.has_syncobj_wait
;
561 struct anv_bo
*batch_bo
, *sync_bo
;
563 if (has_syncobj_wait
) {
564 syncobj
= anv_gem_syncobj_create(device
, 0);
566 result
= vk_error(VK_ERROR_OUT_OF_DEVICE_MEMORY
);
567 goto err_free_submit
;
570 result
= anv_queue_submit_add_syncobj(submit
, device
, syncobj
,
571 I915_EXEC_FENCE_SIGNAL
);
573 result
= anv_device_alloc_bo(device
, 4096,
574 ANV_BO_ALLOC_EXTERNAL
|
575 ANV_BO_ALLOC_IMPLICIT_SYNC
,
577 if (result
!= VK_SUCCESS
)
578 goto err_free_submit
;
580 result
= anv_queue_submit_add_fence_bo(submit
, sync_bo
, true /* signal */);
583 if (result
!= VK_SUCCESS
)
584 goto err_destroy_sync_primitive
;
587 uint32_t size
= align_u32(batch
->next
- batch
->start
, 8);
588 result
= anv_bo_pool_alloc(&device
->batch_bo_pool
, size
, &batch_bo
);
589 if (result
!= VK_SUCCESS
)
590 goto err_destroy_sync_primitive
;
592 memcpy(batch_bo
->map
, batch
->start
, size
);
593 if (!device
->info
.has_llc
)
594 gen_flush_range(batch_bo
->map
, size
);
596 submit
->simple_bo
= batch_bo
;
597 submit
->simple_bo_size
= size
;
600 result
= _anv_queue_submit(queue
, &submit
);
602 if (result
== VK_SUCCESS
) {
603 if (has_syncobj_wait
) {
604 if (anv_gem_syncobj_wait(device
, &syncobj
, 1,
605 anv_get_absolute_timeout(INT64_MAX
), true))
606 result
= anv_device_set_lost(device
, "anv_gem_syncobj_wait failed: %m");
607 anv_gem_syncobj_destroy(device
, syncobj
);
609 result
= anv_device_wait(device
, sync_bo
,
610 anv_get_relative_timeout(INT64_MAX
));
611 anv_device_release_bo(device
, sync_bo
);
616 anv_bo_pool_free(&device
->batch_bo_pool
, batch_bo
);
619 anv_queue_submit_free(device
, submit
);
623 err_destroy_sync_primitive
:
624 if (has_syncobj_wait
)
625 anv_gem_syncobj_destroy(device
, syncobj
);
627 anv_device_release_bo(device
, sync_bo
);
630 anv_queue_submit_free(device
, submit
);
635 /* Transfer ownership of temporary semaphores from the VkSemaphore object to
636 * the anv_queue_submit object. Those temporary semaphores are then freed in
637 * anv_queue_submit_free() once the driver is finished with them.
640 maybe_transfer_temporary_semaphore(struct anv_queue_submit
*submit
,
641 struct anv_semaphore
*semaphore
,
642 struct anv_semaphore_impl
**out_impl
)
644 struct anv_semaphore_impl
*impl
= &semaphore
->temporary
;
646 if (impl
->type
== ANV_SEMAPHORE_TYPE_NONE
) {
647 *out_impl
= &semaphore
->permanent
;
651 /* BO backed timeline semaphores cannot be temporary. */
652 assert(impl
->type
!= ANV_SEMAPHORE_TYPE_TIMELINE
);
655 * There is a requirement to reset semaphore to their permanent state after
656 * submission. From the Vulkan 1.0.53 spec:
658 * "If the import is temporary, the implementation must restore the
659 * semaphore to its prior permanent state after submitting the next
660 * semaphore wait operation."
662 * In the case we defer the actual submission to a thread because of the
663 * wait-before-submit behavior required for timeline semaphores, we need to
664 * make copies of the temporary syncobj to ensure they stay alive until we
665 * do the actual execbuffer ioctl.
667 if (submit
->temporary_semaphore_count
>= submit
->temporary_semaphore_array_length
) {
668 uint32_t new_len
= MAX2(submit
->temporary_semaphore_array_length
* 2, 8);
669 /* Make sure that if the realloc fails, we still have the old semaphore
670 * array around to properly clean things up on failure.
672 struct anv_semaphore_impl
*new_array
=
673 vk_realloc(submit
->alloc
,
674 submit
->temporary_semaphores
,
675 new_len
* sizeof(*submit
->temporary_semaphores
),
676 8, submit
->alloc_scope
);
677 if (new_array
== NULL
)
678 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
680 submit
->temporary_semaphores
= new_array
;
681 submit
->temporary_semaphore_array_length
= new_len
;
684 /* Copy anv_semaphore_impl into anv_queue_submit. */
685 submit
->temporary_semaphores
[submit
->temporary_semaphore_count
++] = *impl
;
686 *out_impl
= &submit
->temporary_semaphores
[submit
->temporary_semaphore_count
- 1];
688 /* Clear the incoming semaphore */
689 impl
->type
= ANV_SEMAPHORE_TYPE_NONE
;
695 anv_queue_submit(struct anv_queue
*queue
,
696 struct anv_cmd_buffer
*cmd_buffer
,
697 const VkSemaphore
*in_semaphores
,
698 const uint64_t *in_values
,
699 uint32_t num_in_semaphores
,
700 const VkSemaphore
*out_semaphores
,
701 const uint64_t *out_values
,
702 uint32_t num_out_semaphores
,
705 ANV_FROM_HANDLE(anv_fence
, fence
, _fence
);
706 struct anv_device
*device
= queue
->device
;
707 UNUSED
struct anv_physical_device
*pdevice
= &device
->instance
->physicalDevice
;
708 struct anv_queue_submit
*submit
= anv_queue_submit_alloc(device
);
710 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
712 submit
->cmd_buffer
= cmd_buffer
;
714 VkResult result
= VK_SUCCESS
;
716 for (uint32_t i
= 0; i
< num_in_semaphores
; i
++) {
717 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, in_semaphores
[i
]);
718 struct anv_semaphore_impl
*impl
;
720 result
= maybe_transfer_temporary_semaphore(submit
, semaphore
, &impl
);
721 if (result
!= VK_SUCCESS
)
724 switch (impl
->type
) {
725 case ANV_SEMAPHORE_TYPE_BO
:
726 assert(!pdevice
->has_syncobj
);
727 result
= anv_queue_submit_add_fence_bo(submit
, impl
->bo
, false /* signal */);
728 if (result
!= VK_SUCCESS
)
732 case ANV_SEMAPHORE_TYPE_SYNC_FILE
:
733 assert(!pdevice
->has_syncobj
);
734 if (submit
->in_fence
== -1) {
735 submit
->in_fence
= impl
->fd
;
736 if (submit
->in_fence
== -1) {
737 result
= vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
742 int merge
= anv_gem_sync_file_merge(device
, submit
->in_fence
, impl
->fd
);
744 result
= vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
748 close(submit
->in_fence
);
750 submit
->in_fence
= merge
;
754 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
: {
755 result
= anv_queue_submit_add_syncobj(submit
, device
,
757 I915_EXEC_FENCE_WAIT
);
758 if (result
!= VK_SUCCESS
)
763 case ANV_SEMAPHORE_TYPE_TIMELINE
:
764 result
= anv_queue_submit_add_timeline_wait(submit
, device
,
766 in_values
? in_values
[i
] : 0);
767 if (result
!= VK_SUCCESS
)
776 for (uint32_t i
= 0; i
< num_out_semaphores
; i
++) {
777 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, out_semaphores
[i
]);
779 /* Under most circumstances, out fences won't be temporary. However,
780 * the spec does allow it for opaque_fd. From the Vulkan 1.0.53 spec:
782 * "If the import is temporary, the implementation must restore the
783 * semaphore to its prior permanent state after submitting the next
784 * semaphore wait operation."
786 * The spec says nothing whatsoever about signal operations on
787 * temporarily imported semaphores so it appears they are allowed.
788 * There are also CTS tests that require this to work.
790 struct anv_semaphore_impl
*impl
=
791 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
792 &semaphore
->temporary
: &semaphore
->permanent
;
794 switch (impl
->type
) {
795 case ANV_SEMAPHORE_TYPE_BO
:
796 assert(!pdevice
->has_syncobj
);
797 result
= anv_queue_submit_add_fence_bo(submit
, impl
->bo
, true /* signal */);
798 if (result
!= VK_SUCCESS
)
802 case ANV_SEMAPHORE_TYPE_SYNC_FILE
:
803 assert(!pdevice
->has_syncobj
);
804 result
= anv_queue_submit_add_sync_fd_fence(submit
, semaphore
);
805 if (result
!= VK_SUCCESS
)
809 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
: {
810 result
= anv_queue_submit_add_syncobj(submit
, device
, impl
->syncobj
,
811 I915_EXEC_FENCE_SIGNAL
);
812 if (result
!= VK_SUCCESS
)
817 case ANV_SEMAPHORE_TYPE_TIMELINE
:
818 result
= anv_queue_submit_add_timeline_signal(submit
, device
,
820 out_values
? out_values
[i
] : 0);
821 if (result
!= VK_SUCCESS
)
831 /* Under most circumstances, out fences won't be temporary. However,
832 * the spec does allow it for opaque_fd. From the Vulkan 1.0.53 spec:
834 * "If the import is temporary, the implementation must restore the
835 * semaphore to its prior permanent state after submitting the next
836 * semaphore wait operation."
838 * The spec says nothing whatsoever about signal operations on
839 * temporarily imported semaphores so it appears they are allowed.
840 * There are also CTS tests that require this to work.
842 struct anv_fence_impl
*impl
=
843 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
844 &fence
->temporary
: &fence
->permanent
;
846 switch (impl
->type
) {
847 case ANV_FENCE_TYPE_BO
:
848 result
= anv_queue_submit_add_fence_bo(submit
, impl
->bo
.bo
, true /* signal */);
849 if (result
!= VK_SUCCESS
)
853 case ANV_FENCE_TYPE_SYNCOBJ
: {
855 * For the same reason we reset the signaled binary syncobj above,
856 * also reset the fence's syncobj so that they don't contain a
857 * signaled dma-fence.
859 result
= anv_queue_submit_add_syncobj(submit
, device
, impl
->syncobj
,
860 I915_EXEC_FENCE_SIGNAL
);
861 if (result
!= VK_SUCCESS
)
867 unreachable("Invalid fence type");
871 result
= _anv_queue_submit(queue
, &submit
);
872 if (result
!= VK_SUCCESS
)
875 if (fence
&& fence
->permanent
.type
== ANV_FENCE_TYPE_BO
) {
876 /* BO fences can't be shared, so they can't be temporary. */
877 assert(fence
->temporary
.type
== ANV_FENCE_TYPE_NONE
);
879 /* Once the execbuf has returned, we need to set the fence state to
880 * SUBMITTED. We can't do this before calling execbuf because
881 * anv_GetFenceStatus does take the global device lock before checking
884 * We set the fence state to SUBMITTED regardless of whether or not the
885 * execbuf succeeds because we need to ensure that vkWaitForFences() and
886 * vkGetFenceStatus() return a valid result (VK_ERROR_DEVICE_LOST or
887 * VK_SUCCESS) in a finite amount of time even if execbuf fails.
889 fence
->permanent
.bo
.state
= ANV_BO_FENCE_STATE_SUBMITTED
;
894 anv_queue_submit_free(device
, submit
);
899 VkResult
anv_QueueSubmit(
901 uint32_t submitCount
,
902 const VkSubmitInfo
* pSubmits
,
905 ANV_FROM_HANDLE(anv_queue
, queue
, _queue
);
907 /* Query for device status prior to submitting. Technically, we don't need
908 * to do this. However, if we have a client that's submitting piles of
909 * garbage, we would rather break as early as possible to keep the GPU
910 * hanging contained. If we don't check here, we'll either be waiting for
911 * the kernel to kick us or we'll have to wait until the client waits on a
912 * fence before we actually know whether or not we've hung.
914 VkResult result
= anv_device_query_status(queue
->device
);
915 if (result
!= VK_SUCCESS
)
918 if (fence
&& submitCount
== 0) {
919 /* If we don't have any command buffers, we need to submit a dummy
920 * batch to give GEM something to wait on. We could, potentially,
921 * come up with something more efficient but this shouldn't be a
924 result
= anv_queue_submit(queue
, NULL
, NULL
, NULL
, 0, NULL
, NULL
, 0, fence
);
928 for (uint32_t i
= 0; i
< submitCount
; i
++) {
929 /* Fence for this submit. NULL for all but the last one */
930 VkFence submit_fence
= (i
== submitCount
- 1) ? fence
: VK_NULL_HANDLE
;
932 const VkTimelineSemaphoreSubmitInfoKHR
*timeline_info
=
933 vk_find_struct_const(pSubmits
[i
].pNext
,
934 TIMELINE_SEMAPHORE_SUBMIT_INFO_KHR
);
935 const uint64_t *wait_values
=
936 timeline_info
&& timeline_info
->waitSemaphoreValueCount
?
937 timeline_info
->pWaitSemaphoreValues
: NULL
;
938 const uint64_t *signal_values
=
939 timeline_info
&& timeline_info
->signalSemaphoreValueCount
?
940 timeline_info
->pSignalSemaphoreValues
: NULL
;
942 if (pSubmits
[i
].commandBufferCount
== 0) {
943 /* If we don't have any command buffers, we need to submit a dummy
944 * batch to give GEM something to wait on. We could, potentially,
945 * come up with something more efficient but this shouldn't be a
948 result
= anv_queue_submit(queue
, NULL
,
949 pSubmits
[i
].pWaitSemaphores
,
951 pSubmits
[i
].waitSemaphoreCount
,
952 pSubmits
[i
].pSignalSemaphores
,
954 pSubmits
[i
].signalSemaphoreCount
,
956 if (result
!= VK_SUCCESS
)
962 for (uint32_t j
= 0; j
< pSubmits
[i
].commandBufferCount
; j
++) {
963 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
,
964 pSubmits
[i
].pCommandBuffers
[j
]);
965 assert(cmd_buffer
->level
== VK_COMMAND_BUFFER_LEVEL_PRIMARY
);
966 assert(!anv_batch_has_error(&cmd_buffer
->batch
));
968 /* Fence for this execbuf. NULL for all but the last one */
969 VkFence execbuf_fence
=
970 (j
== pSubmits
[i
].commandBufferCount
- 1) ?
971 submit_fence
: VK_NULL_HANDLE
;
973 const VkSemaphore
*in_semaphores
= NULL
, *out_semaphores
= NULL
;
974 const uint64_t *in_values
= NULL
, *out_values
= NULL
;
975 uint32_t num_in_semaphores
= 0, num_out_semaphores
= 0;
977 /* Only the first batch gets the in semaphores */
978 in_semaphores
= pSubmits
[i
].pWaitSemaphores
;
979 in_values
= wait_values
;
980 num_in_semaphores
= pSubmits
[i
].waitSemaphoreCount
;
983 if (j
== pSubmits
[i
].commandBufferCount
- 1) {
984 /* Only the last batch gets the out semaphores */
985 out_semaphores
= pSubmits
[i
].pSignalSemaphores
;
986 out_values
= signal_values
;
987 num_out_semaphores
= pSubmits
[i
].signalSemaphoreCount
;
990 result
= anv_queue_submit(queue
, cmd_buffer
,
991 in_semaphores
, in_values
, num_in_semaphores
,
992 out_semaphores
, out_values
, num_out_semaphores
,
994 if (result
!= VK_SUCCESS
)
1000 if (result
!= VK_SUCCESS
&& result
!= VK_ERROR_DEVICE_LOST
) {
1001 /* In the case that something has gone wrong we may end up with an
1002 * inconsistent state from which it may not be trivial to recover.
1003 * For example, we might have computed address relocations and
1004 * any future attempt to re-submit this job will need to know about
1005 * this and avoid computing relocation addresses again.
1007 * To avoid this sort of issues, we assume that if something was
1008 * wrong during submission we must already be in a really bad situation
1009 * anyway (such us being out of memory) and return
1010 * VK_ERROR_DEVICE_LOST to ensure that clients do not attempt to
1011 * submit the same job again to this device.
1013 * We skip doing this on VK_ERROR_DEVICE_LOST because
1014 * anv_device_set_lost() would have been called already by a callee of
1015 * anv_queue_submit().
1017 result
= anv_device_set_lost(queue
->device
, "vkQueueSubmit() failed");
1023 VkResult
anv_QueueWaitIdle(
1026 ANV_FROM_HANDLE(anv_queue
, queue
, _queue
);
1028 if (anv_device_is_lost(queue
->device
))
1029 return VK_ERROR_DEVICE_LOST
;
1031 return anv_queue_submit_simple_batch(queue
, NULL
);
1034 VkResult
anv_CreateFence(
1036 const VkFenceCreateInfo
* pCreateInfo
,
1037 const VkAllocationCallbacks
* pAllocator
,
1040 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1041 struct anv_fence
*fence
;
1043 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_FENCE_CREATE_INFO
);
1045 fence
= vk_zalloc2(&device
->alloc
, pAllocator
, sizeof(*fence
), 8,
1046 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
1048 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1050 if (device
->instance
->physicalDevice
.has_syncobj_wait
) {
1051 fence
->permanent
.type
= ANV_FENCE_TYPE_SYNCOBJ
;
1053 uint32_t create_flags
= 0;
1054 if (pCreateInfo
->flags
& VK_FENCE_CREATE_SIGNALED_BIT
)
1055 create_flags
|= DRM_SYNCOBJ_CREATE_SIGNALED
;
1057 fence
->permanent
.syncobj
= anv_gem_syncobj_create(device
, create_flags
);
1058 if (!fence
->permanent
.syncobj
)
1059 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1061 fence
->permanent
.type
= ANV_FENCE_TYPE_BO
;
1063 VkResult result
= anv_bo_pool_alloc(&device
->batch_bo_pool
, 4096,
1064 &fence
->permanent
.bo
.bo
);
1065 if (result
!= VK_SUCCESS
)
1068 if (pCreateInfo
->flags
& VK_FENCE_CREATE_SIGNALED_BIT
) {
1069 fence
->permanent
.bo
.state
= ANV_BO_FENCE_STATE_SIGNALED
;
1071 fence
->permanent
.bo
.state
= ANV_BO_FENCE_STATE_RESET
;
1075 *pFence
= anv_fence_to_handle(fence
);
1081 anv_fence_impl_cleanup(struct anv_device
*device
,
1082 struct anv_fence_impl
*impl
)
1084 switch (impl
->type
) {
1085 case ANV_FENCE_TYPE_NONE
:
1086 /* Dummy. Nothing to do */
1089 case ANV_FENCE_TYPE_BO
:
1090 anv_bo_pool_free(&device
->batch_bo_pool
, impl
->bo
.bo
);
1093 case ANV_FENCE_TYPE_SYNCOBJ
:
1094 anv_gem_syncobj_destroy(device
, impl
->syncobj
);
1097 case ANV_FENCE_TYPE_WSI
:
1098 impl
->fence_wsi
->destroy(impl
->fence_wsi
);
1102 unreachable("Invalid fence type");
1105 impl
->type
= ANV_FENCE_TYPE_NONE
;
1108 void anv_DestroyFence(
1111 const VkAllocationCallbacks
* pAllocator
)
1113 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1114 ANV_FROM_HANDLE(anv_fence
, fence
, _fence
);
1119 anv_fence_impl_cleanup(device
, &fence
->temporary
);
1120 anv_fence_impl_cleanup(device
, &fence
->permanent
);
1122 vk_free2(&device
->alloc
, pAllocator
, fence
);
1125 VkResult
anv_ResetFences(
1127 uint32_t fenceCount
,
1128 const VkFence
* pFences
)
1130 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1132 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1133 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1135 /* From the Vulkan 1.0.53 spec:
1137 * "If any member of pFences currently has its payload imported with
1138 * temporary permanence, that fence’s prior permanent payload is
1139 * first restored. The remaining operations described therefore
1140 * operate on the restored payload.
1142 if (fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
)
1143 anv_fence_impl_cleanup(device
, &fence
->temporary
);
1145 struct anv_fence_impl
*impl
= &fence
->permanent
;
1147 switch (impl
->type
) {
1148 case ANV_FENCE_TYPE_BO
:
1149 impl
->bo
.state
= ANV_BO_FENCE_STATE_RESET
;
1152 case ANV_FENCE_TYPE_SYNCOBJ
:
1153 anv_gem_syncobj_reset(device
, impl
->syncobj
);
1157 unreachable("Invalid fence type");
1164 VkResult
anv_GetFenceStatus(
1168 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1169 ANV_FROM_HANDLE(anv_fence
, fence
, _fence
);
1171 if (anv_device_is_lost(device
))
1172 return VK_ERROR_DEVICE_LOST
;
1174 struct anv_fence_impl
*impl
=
1175 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1176 &fence
->temporary
: &fence
->permanent
;
1178 switch (impl
->type
) {
1179 case ANV_FENCE_TYPE_BO
:
1180 /* BO fences don't support import/export */
1181 assert(fence
->temporary
.type
== ANV_FENCE_TYPE_NONE
);
1182 switch (impl
->bo
.state
) {
1183 case ANV_BO_FENCE_STATE_RESET
:
1184 /* If it hasn't even been sent off to the GPU yet, it's not ready */
1185 return VK_NOT_READY
;
1187 case ANV_BO_FENCE_STATE_SIGNALED
:
1188 /* It's been signaled, return success */
1191 case ANV_BO_FENCE_STATE_SUBMITTED
: {
1192 VkResult result
= anv_device_bo_busy(device
, impl
->bo
.bo
);
1193 if (result
== VK_SUCCESS
) {
1194 impl
->bo
.state
= ANV_BO_FENCE_STATE_SIGNALED
;
1201 unreachable("Invalid fence status");
1204 case ANV_FENCE_TYPE_SYNCOBJ
: {
1205 int ret
= anv_gem_syncobj_wait(device
, &impl
->syncobj
, 1, 0, true);
1207 if (errno
== ETIME
) {
1208 return VK_NOT_READY
;
1210 /* We don't know the real error. */
1211 return anv_device_set_lost(device
, "drm_syncobj_wait failed: %m");
1219 unreachable("Invalid fence type");
1224 anv_wait_for_syncobj_fences(struct anv_device
*device
,
1225 uint32_t fenceCount
,
1226 const VkFence
*pFences
,
1228 uint64_t abs_timeout_ns
)
1230 uint32_t *syncobjs
= vk_zalloc(&device
->alloc
,
1231 sizeof(*syncobjs
) * fenceCount
, 8,
1232 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND
);
1234 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1236 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1237 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1238 assert(fence
->permanent
.type
== ANV_FENCE_TYPE_SYNCOBJ
);
1240 struct anv_fence_impl
*impl
=
1241 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1242 &fence
->temporary
: &fence
->permanent
;
1244 assert(impl
->type
== ANV_FENCE_TYPE_SYNCOBJ
);
1245 syncobjs
[i
] = impl
->syncobj
;
1248 /* The gem_syncobj_wait ioctl may return early due to an inherent
1249 * limitation in the way it computes timeouts. Loop until we've actually
1250 * passed the timeout.
1254 ret
= anv_gem_syncobj_wait(device
, syncobjs
, fenceCount
,
1255 abs_timeout_ns
, waitAll
);
1256 } while (ret
== -1 && errno
== ETIME
&& anv_gettime_ns() < abs_timeout_ns
);
1258 vk_free(&device
->alloc
, syncobjs
);
1261 if (errno
== ETIME
) {
1264 /* We don't know the real error. */
1265 return anv_device_set_lost(device
, "drm_syncobj_wait failed: %m");
1273 anv_wait_for_bo_fences(struct anv_device
*device
,
1274 uint32_t fenceCount
,
1275 const VkFence
*pFences
,
1277 uint64_t abs_timeout_ns
)
1279 VkResult result
= VK_SUCCESS
;
1280 uint32_t pending_fences
= fenceCount
;
1281 while (pending_fences
) {
1283 bool signaled_fences
= false;
1284 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1285 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1287 /* This function assumes that all fences are BO fences and that they
1288 * have no temporary state. Since BO fences will never be exported,
1289 * this should be a safe assumption.
1291 assert(fence
->permanent
.type
== ANV_FENCE_TYPE_BO
);
1292 assert(fence
->temporary
.type
== ANV_FENCE_TYPE_NONE
);
1293 struct anv_fence_impl
*impl
= &fence
->permanent
;
1295 switch (impl
->bo
.state
) {
1296 case ANV_BO_FENCE_STATE_RESET
:
1297 /* This fence hasn't been submitted yet, we'll catch it the next
1298 * time around. Yes, this may mean we dead-loop but, short of
1299 * lots of locking and a condition variable, there's not much that
1300 * we can do about that.
1305 case ANV_BO_FENCE_STATE_SIGNALED
:
1306 /* This fence is not pending. If waitAll isn't set, we can return
1307 * early. Otherwise, we have to keep going.
1310 result
= VK_SUCCESS
;
1315 case ANV_BO_FENCE_STATE_SUBMITTED
:
1316 /* These are the fences we really care about. Go ahead and wait
1317 * on it until we hit a timeout.
1319 result
= anv_device_wait(device
, impl
->bo
.bo
,
1320 anv_get_relative_timeout(abs_timeout_ns
));
1323 impl
->bo
.state
= ANV_BO_FENCE_STATE_SIGNALED
;
1324 signaled_fences
= true;
1338 if (pending_fences
&& !signaled_fences
) {
1339 /* If we've hit this then someone decided to vkWaitForFences before
1340 * they've actually submitted any of them to a queue. This is a
1341 * fairly pessimal case, so it's ok to lock here and use a standard
1342 * pthreads condition variable.
1344 pthread_mutex_lock(&device
->mutex
);
1346 /* It's possible that some of the fences have changed state since the
1347 * last time we checked. Now that we have the lock, check for
1348 * pending fences again and don't wait if it's changed.
1350 uint32_t now_pending_fences
= 0;
1351 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1352 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1353 if (fence
->permanent
.bo
.state
== ANV_BO_FENCE_STATE_RESET
)
1354 now_pending_fences
++;
1356 assert(now_pending_fences
<= pending_fences
);
1358 if (now_pending_fences
== pending_fences
) {
1359 struct timespec abstime
= {
1360 .tv_sec
= abs_timeout_ns
/ NSEC_PER_SEC
,
1361 .tv_nsec
= abs_timeout_ns
% NSEC_PER_SEC
,
1365 ret
= pthread_cond_timedwait(&device
->queue_submit
,
1366 &device
->mutex
, &abstime
);
1367 assert(ret
!= EINVAL
);
1368 if (anv_gettime_ns() >= abs_timeout_ns
) {
1369 pthread_mutex_unlock(&device
->mutex
);
1370 result
= VK_TIMEOUT
;
1375 pthread_mutex_unlock(&device
->mutex
);
1380 if (anv_device_is_lost(device
))
1381 return VK_ERROR_DEVICE_LOST
;
1387 anv_wait_for_wsi_fence(struct anv_device
*device
,
1388 const VkFence _fence
,
1389 uint64_t abs_timeout
)
1391 ANV_FROM_HANDLE(anv_fence
, fence
, _fence
);
1392 struct anv_fence_impl
*impl
= &fence
->permanent
;
1394 return impl
->fence_wsi
->wait(impl
->fence_wsi
, abs_timeout
);
1398 anv_wait_for_fences(struct anv_device
*device
,
1399 uint32_t fenceCount
,
1400 const VkFence
*pFences
,
1402 uint64_t abs_timeout
)
1404 VkResult result
= VK_SUCCESS
;
1406 if (fenceCount
<= 1 || waitAll
) {
1407 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1408 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1409 switch (fence
->permanent
.type
) {
1410 case ANV_FENCE_TYPE_BO
:
1411 result
= anv_wait_for_bo_fences(device
, 1, &pFences
[i
],
1414 case ANV_FENCE_TYPE_SYNCOBJ
:
1415 result
= anv_wait_for_syncobj_fences(device
, 1, &pFences
[i
],
1418 case ANV_FENCE_TYPE_WSI
:
1419 result
= anv_wait_for_wsi_fence(device
, pFences
[i
], abs_timeout
);
1421 case ANV_FENCE_TYPE_NONE
:
1422 result
= VK_SUCCESS
;
1425 if (result
!= VK_SUCCESS
)
1430 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1431 if (anv_wait_for_fences(device
, 1, &pFences
[i
], true, 0) == VK_SUCCESS
)
1434 } while (anv_gettime_ns() < abs_timeout
);
1435 result
= VK_TIMEOUT
;
1440 static bool anv_all_fences_syncobj(uint32_t fenceCount
, const VkFence
*pFences
)
1442 for (uint32_t i
= 0; i
< fenceCount
; ++i
) {
1443 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1444 if (fence
->permanent
.type
!= ANV_FENCE_TYPE_SYNCOBJ
)
1450 static bool anv_all_fences_bo(uint32_t fenceCount
, const VkFence
*pFences
)
1452 for (uint32_t i
= 0; i
< fenceCount
; ++i
) {
1453 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1454 if (fence
->permanent
.type
!= ANV_FENCE_TYPE_BO
)
1460 VkResult
anv_WaitForFences(
1462 uint32_t fenceCount
,
1463 const VkFence
* pFences
,
1467 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1469 if (anv_device_is_lost(device
))
1470 return VK_ERROR_DEVICE_LOST
;
1472 uint64_t abs_timeout
= anv_get_absolute_timeout(timeout
);
1473 if (anv_all_fences_syncobj(fenceCount
, pFences
)) {
1474 return anv_wait_for_syncobj_fences(device
, fenceCount
, pFences
,
1475 waitAll
, abs_timeout
);
1476 } else if (anv_all_fences_bo(fenceCount
, pFences
)) {
1477 return anv_wait_for_bo_fences(device
, fenceCount
, pFences
,
1478 waitAll
, abs_timeout
);
1480 return anv_wait_for_fences(device
, fenceCount
, pFences
,
1481 waitAll
, abs_timeout
);
1485 void anv_GetPhysicalDeviceExternalFenceProperties(
1486 VkPhysicalDevice physicalDevice
,
1487 const VkPhysicalDeviceExternalFenceInfo
* pExternalFenceInfo
,
1488 VkExternalFenceProperties
* pExternalFenceProperties
)
1490 ANV_FROM_HANDLE(anv_physical_device
, device
, physicalDevice
);
1492 switch (pExternalFenceInfo
->handleType
) {
1493 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1494 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
:
1495 if (device
->has_syncobj_wait
) {
1496 pExternalFenceProperties
->exportFromImportedHandleTypes
=
1497 VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
|
1498 VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
;
1499 pExternalFenceProperties
->compatibleHandleTypes
=
1500 VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
|
1501 VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
;
1502 pExternalFenceProperties
->externalFenceFeatures
=
1503 VK_EXTERNAL_FENCE_FEATURE_EXPORTABLE_BIT
|
1504 VK_EXTERNAL_FENCE_FEATURE_IMPORTABLE_BIT
;
1513 pExternalFenceProperties
->exportFromImportedHandleTypes
= 0;
1514 pExternalFenceProperties
->compatibleHandleTypes
= 0;
1515 pExternalFenceProperties
->externalFenceFeatures
= 0;
1518 VkResult
anv_ImportFenceFdKHR(
1520 const VkImportFenceFdInfoKHR
* pImportFenceFdInfo
)
1522 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1523 ANV_FROM_HANDLE(anv_fence
, fence
, pImportFenceFdInfo
->fence
);
1524 int fd
= pImportFenceFdInfo
->fd
;
1526 assert(pImportFenceFdInfo
->sType
==
1527 VK_STRUCTURE_TYPE_IMPORT_FENCE_FD_INFO_KHR
);
1529 struct anv_fence_impl new_impl
= {
1530 .type
= ANV_FENCE_TYPE_NONE
,
1533 switch (pImportFenceFdInfo
->handleType
) {
1534 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1535 new_impl
.type
= ANV_FENCE_TYPE_SYNCOBJ
;
1537 new_impl
.syncobj
= anv_gem_syncobj_fd_to_handle(device
, fd
);
1538 if (!new_impl
.syncobj
)
1539 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1543 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
:
1544 /* Sync files are a bit tricky. Because we want to continue using the
1545 * syncobj implementation of WaitForFences, we don't use the sync file
1546 * directly but instead import it into a syncobj.
1548 new_impl
.type
= ANV_FENCE_TYPE_SYNCOBJ
;
1550 new_impl
.syncobj
= anv_gem_syncobj_create(device
, 0);
1551 if (!new_impl
.syncobj
)
1552 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1554 if (anv_gem_syncobj_import_sync_file(device
, new_impl
.syncobj
, fd
)) {
1555 anv_gem_syncobj_destroy(device
, new_impl
.syncobj
);
1556 return vk_errorf(device
->instance
, NULL
,
1557 VK_ERROR_INVALID_EXTERNAL_HANDLE
,
1558 "syncobj sync file import failed: %m");
1563 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1566 /* From the Vulkan 1.0.53 spec:
1568 * "Importing a fence payload from a file descriptor transfers
1569 * ownership of the file descriptor from the application to the
1570 * Vulkan implementation. The application must not perform any
1571 * operations on the file descriptor after a successful import."
1573 * If the import fails, we leave the file descriptor open.
1577 if (pImportFenceFdInfo
->flags
& VK_FENCE_IMPORT_TEMPORARY_BIT
) {
1578 anv_fence_impl_cleanup(device
, &fence
->temporary
);
1579 fence
->temporary
= new_impl
;
1581 anv_fence_impl_cleanup(device
, &fence
->permanent
);
1582 fence
->permanent
= new_impl
;
1588 VkResult
anv_GetFenceFdKHR(
1590 const VkFenceGetFdInfoKHR
* pGetFdInfo
,
1593 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1594 ANV_FROM_HANDLE(anv_fence
, fence
, pGetFdInfo
->fence
);
1596 assert(pGetFdInfo
->sType
== VK_STRUCTURE_TYPE_FENCE_GET_FD_INFO_KHR
);
1598 struct anv_fence_impl
*impl
=
1599 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1600 &fence
->temporary
: &fence
->permanent
;
1602 assert(impl
->type
== ANV_FENCE_TYPE_SYNCOBJ
);
1603 switch (pGetFdInfo
->handleType
) {
1604 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
: {
1605 int fd
= anv_gem_syncobj_handle_to_fd(device
, impl
->syncobj
);
1607 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
1613 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
: {
1614 int fd
= anv_gem_syncobj_export_sync_file(device
, impl
->syncobj
);
1616 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
1623 unreachable("Invalid fence export handle type");
1626 /* From the Vulkan 1.0.53 spec:
1628 * "Export operations have the same transference as the specified handle
1629 * type’s import operations. [...] If the fence was using a
1630 * temporarily imported payload, the fence’s prior permanent payload
1633 if (impl
== &fence
->temporary
)
1634 anv_fence_impl_cleanup(device
, impl
);
1639 // Queue semaphore functions
1641 static VkSemaphoreTypeKHR
1642 get_semaphore_type(const void *pNext
, uint64_t *initial_value
)
1644 const VkSemaphoreTypeCreateInfoKHR
*type_info
=
1645 vk_find_struct_const(pNext
, SEMAPHORE_TYPE_CREATE_INFO_KHR
);
1648 return VK_SEMAPHORE_TYPE_BINARY_KHR
;
1651 *initial_value
= type_info
->initialValue
;
1652 return type_info
->semaphoreType
;
1656 binary_semaphore_create(struct anv_device
*device
,
1657 struct anv_semaphore_impl
*impl
,
1660 if (device
->instance
->physicalDevice
.has_syncobj
) {
1661 impl
->type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
;
1662 impl
->syncobj
= anv_gem_syncobj_create(device
, 0);
1664 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1667 impl
->type
= ANV_SEMAPHORE_TYPE_BO
;
1669 anv_device_alloc_bo(device
, 4096,
1670 ANV_BO_ALLOC_EXTERNAL
|
1671 ANV_BO_ALLOC_IMPLICIT_SYNC
,
1673 /* If we're going to use this as a fence, we need to *not* have the
1674 * EXEC_OBJECT_ASYNC bit set.
1676 assert(!(impl
->bo
->flags
& EXEC_OBJECT_ASYNC
));
1682 timeline_semaphore_create(struct anv_device
*device
,
1683 struct anv_semaphore_impl
*impl
,
1684 uint64_t initial_value
)
1686 impl
->type
= ANV_SEMAPHORE_TYPE_TIMELINE
;
1687 anv_timeline_init(device
, &impl
->timeline
, initial_value
);
1691 VkResult
anv_CreateSemaphore(
1693 const VkSemaphoreCreateInfo
* pCreateInfo
,
1694 const VkAllocationCallbacks
* pAllocator
,
1695 VkSemaphore
* pSemaphore
)
1697 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1698 struct anv_semaphore
*semaphore
;
1700 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO
);
1702 uint64_t timeline_value
= 0;
1703 VkSemaphoreTypeKHR sem_type
= get_semaphore_type(pCreateInfo
->pNext
, &timeline_value
);
1705 semaphore
= vk_alloc(&device
->alloc
, sizeof(*semaphore
), 8,
1706 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE
);
1707 if (semaphore
== NULL
)
1708 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1710 p_atomic_set(&semaphore
->refcount
, 1);
1712 const VkExportSemaphoreCreateInfo
*export
=
1713 vk_find_struct_const(pCreateInfo
->pNext
, EXPORT_SEMAPHORE_CREATE_INFO
);
1714 VkExternalSemaphoreHandleTypeFlags handleTypes
=
1715 export
? export
->handleTypes
: 0;
1718 if (handleTypes
== 0) {
1719 if (sem_type
== VK_SEMAPHORE_TYPE_BINARY_KHR
)
1720 result
= binary_semaphore_create(device
, &semaphore
->permanent
, false);
1722 result
= timeline_semaphore_create(device
, &semaphore
->permanent
, timeline_value
);
1723 if (result
!= VK_SUCCESS
) {
1724 vk_free2(&device
->alloc
, pAllocator
, semaphore
);
1727 } else if (handleTypes
& VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
) {
1728 assert(handleTypes
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
);
1729 assert(sem_type
== VK_SEMAPHORE_TYPE_BINARY_KHR
);
1730 result
= binary_semaphore_create(device
, &semaphore
->permanent
, true);
1731 if (result
!= VK_SUCCESS
) {
1732 vk_free2(&device
->alloc
, pAllocator
, semaphore
);
1735 } else if (handleTypes
& VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
) {
1736 assert(handleTypes
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
);
1737 assert(sem_type
== VK_SEMAPHORE_TYPE_BINARY_KHR
);
1738 if (device
->instance
->physicalDevice
.has_syncobj
) {
1739 semaphore
->permanent
.type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
;
1740 semaphore
->permanent
.syncobj
= anv_gem_syncobj_create(device
, 0);
1741 if (!semaphore
->permanent
.syncobj
) {
1742 vk_free2(&device
->alloc
, pAllocator
, semaphore
);
1743 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1746 semaphore
->permanent
.type
= ANV_SEMAPHORE_TYPE_SYNC_FILE
;
1747 semaphore
->permanent
.fd
= -1;
1750 assert(!"Unknown handle type");
1751 vk_free2(&device
->alloc
, pAllocator
, semaphore
);
1752 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1755 semaphore
->temporary
.type
= ANV_SEMAPHORE_TYPE_NONE
;
1757 *pSemaphore
= anv_semaphore_to_handle(semaphore
);
1763 anv_semaphore_impl_cleanup(struct anv_device
*device
,
1764 struct anv_semaphore_impl
*impl
)
1766 switch (impl
->type
) {
1767 case ANV_SEMAPHORE_TYPE_NONE
:
1768 case ANV_SEMAPHORE_TYPE_DUMMY
:
1769 /* Dummy. Nothing to do */
1772 case ANV_SEMAPHORE_TYPE_BO
:
1773 anv_device_release_bo(device
, impl
->bo
);
1776 case ANV_SEMAPHORE_TYPE_SYNC_FILE
:
1780 case ANV_SEMAPHORE_TYPE_TIMELINE
:
1781 anv_timeline_finish(device
, &impl
->timeline
);
1784 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
:
1785 anv_gem_syncobj_destroy(device
, impl
->syncobj
);
1789 unreachable("Invalid semaphore type");
1792 impl
->type
= ANV_SEMAPHORE_TYPE_NONE
;
1796 anv_semaphore_reset_temporary(struct anv_device
*device
,
1797 struct anv_semaphore
*semaphore
)
1799 if (semaphore
->temporary
.type
== ANV_SEMAPHORE_TYPE_NONE
)
1802 anv_semaphore_impl_cleanup(device
, &semaphore
->temporary
);
1805 static struct anv_semaphore
*
1806 anv_semaphore_ref(struct anv_semaphore
*semaphore
)
1808 assert(semaphore
->refcount
);
1809 p_atomic_inc(&semaphore
->refcount
);
1814 anv_semaphore_unref(struct anv_device
*device
, struct anv_semaphore
*semaphore
)
1816 if (!p_atomic_dec_zero(&semaphore
->refcount
))
1819 anv_semaphore_impl_cleanup(device
, &semaphore
->temporary
);
1820 anv_semaphore_impl_cleanup(device
, &semaphore
->permanent
);
1821 vk_free(&device
->alloc
, semaphore
);
1824 void anv_DestroySemaphore(
1826 VkSemaphore _semaphore
,
1827 const VkAllocationCallbacks
* pAllocator
)
1829 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1830 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, _semaphore
);
1832 if (semaphore
== NULL
)
1835 anv_semaphore_unref(device
, semaphore
);
1838 void anv_GetPhysicalDeviceExternalSemaphoreProperties(
1839 VkPhysicalDevice physicalDevice
,
1840 const VkPhysicalDeviceExternalSemaphoreInfo
* pExternalSemaphoreInfo
,
1841 VkExternalSemaphoreProperties
* pExternalSemaphoreProperties
)
1843 ANV_FROM_HANDLE(anv_physical_device
, device
, physicalDevice
);
1845 VkSemaphoreTypeKHR sem_type
=
1846 get_semaphore_type(pExternalSemaphoreInfo
->pNext
, NULL
);
1848 switch (pExternalSemaphoreInfo
->handleType
) {
1849 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1850 /* Timeline semaphores are not exportable. */
1851 if (sem_type
== VK_SEMAPHORE_TYPE_TIMELINE_KHR
)
1853 pExternalSemaphoreProperties
->exportFromImportedHandleTypes
=
1854 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
;
1855 pExternalSemaphoreProperties
->compatibleHandleTypes
=
1856 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
;
1857 pExternalSemaphoreProperties
->externalSemaphoreFeatures
=
1858 VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT
|
1859 VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT
;
1862 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
:
1863 if (sem_type
== VK_SEMAPHORE_TYPE_TIMELINE_KHR
)
1865 if (!device
->has_exec_fence
)
1867 pExternalSemaphoreProperties
->exportFromImportedHandleTypes
=
1868 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
;
1869 pExternalSemaphoreProperties
->compatibleHandleTypes
=
1870 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
;
1871 pExternalSemaphoreProperties
->externalSemaphoreFeatures
=
1872 VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT
|
1873 VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT
;
1880 pExternalSemaphoreProperties
->exportFromImportedHandleTypes
= 0;
1881 pExternalSemaphoreProperties
->compatibleHandleTypes
= 0;
1882 pExternalSemaphoreProperties
->externalSemaphoreFeatures
= 0;
1885 VkResult
anv_ImportSemaphoreFdKHR(
1887 const VkImportSemaphoreFdInfoKHR
* pImportSemaphoreFdInfo
)
1889 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1890 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pImportSemaphoreFdInfo
->semaphore
);
1891 int fd
= pImportSemaphoreFdInfo
->fd
;
1893 struct anv_semaphore_impl new_impl
= {
1894 .type
= ANV_SEMAPHORE_TYPE_NONE
,
1897 switch (pImportSemaphoreFdInfo
->handleType
) {
1898 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1899 if (device
->instance
->physicalDevice
.has_syncobj
) {
1900 new_impl
.type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
;
1902 new_impl
.syncobj
= anv_gem_syncobj_fd_to_handle(device
, fd
);
1903 if (!new_impl
.syncobj
)
1904 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1906 new_impl
.type
= ANV_SEMAPHORE_TYPE_BO
;
1908 VkResult result
= anv_device_import_bo(device
, fd
,
1909 ANV_BO_ALLOC_EXTERNAL
|
1910 ANV_BO_ALLOC_IMPLICIT_SYNC
,
1912 if (result
!= VK_SUCCESS
)
1915 if (new_impl
.bo
->size
< 4096) {
1916 anv_device_release_bo(device
, new_impl
.bo
);
1917 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1920 /* If we're going to use this as a fence, we need to *not* have the
1921 * EXEC_OBJECT_ASYNC bit set.
1923 assert(!(new_impl
.bo
->flags
& EXEC_OBJECT_ASYNC
));
1926 /* From the Vulkan spec:
1928 * "Importing semaphore state from a file descriptor transfers
1929 * ownership of the file descriptor from the application to the
1930 * Vulkan implementation. The application must not perform any
1931 * operations on the file descriptor after a successful import."
1933 * If the import fails, we leave the file descriptor open.
1938 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
:
1939 if (device
->instance
->physicalDevice
.has_syncobj
) {
1940 new_impl
= (struct anv_semaphore_impl
) {
1941 .type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
,
1942 .syncobj
= anv_gem_syncobj_create(device
, 0),
1944 if (!new_impl
.syncobj
)
1945 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1946 if (anv_gem_syncobj_import_sync_file(device
, new_impl
.syncobj
, fd
)) {
1947 anv_gem_syncobj_destroy(device
, new_impl
.syncobj
);
1948 return vk_errorf(device
->instance
, NULL
,
1949 VK_ERROR_INVALID_EXTERNAL_HANDLE
,
1950 "syncobj sync file import failed: %m");
1952 /* Ownership of the FD is transfered to Anv. Since we don't need it
1953 * anymore because the associated fence has been put into a syncobj,
1954 * we must close the FD.
1958 new_impl
= (struct anv_semaphore_impl
) {
1959 .type
= ANV_SEMAPHORE_TYPE_SYNC_FILE
,
1966 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1969 if (pImportSemaphoreFdInfo
->flags
& VK_SEMAPHORE_IMPORT_TEMPORARY_BIT
) {
1970 anv_semaphore_impl_cleanup(device
, &semaphore
->temporary
);
1971 semaphore
->temporary
= new_impl
;
1973 anv_semaphore_impl_cleanup(device
, &semaphore
->permanent
);
1974 semaphore
->permanent
= new_impl
;
1980 VkResult
anv_GetSemaphoreFdKHR(
1982 const VkSemaphoreGetFdInfoKHR
* pGetFdInfo
,
1985 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1986 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pGetFdInfo
->semaphore
);
1990 assert(pGetFdInfo
->sType
== VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR
);
1992 struct anv_semaphore_impl
*impl
=
1993 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
1994 &semaphore
->temporary
: &semaphore
->permanent
;
1996 switch (impl
->type
) {
1997 case ANV_SEMAPHORE_TYPE_BO
:
1998 result
= anv_device_export_bo(device
, impl
->bo
, pFd
);
1999 if (result
!= VK_SUCCESS
)
2003 case ANV_SEMAPHORE_TYPE_SYNC_FILE
: {
2004 /* There's a potential race here with vkQueueSubmit if you are trying
2005 * to export a semaphore Fd while the queue submit is still happening.
2006 * This can happen if we see all dependencies get resolved via timeline
2007 * semaphore waits completing before the execbuf completes and we
2008 * process the resulting out fence. To work around this, take a lock
2009 * around grabbing the fd.
2011 pthread_mutex_lock(&device
->mutex
);
2013 /* From the Vulkan 1.0.53 spec:
2015 * "...exporting a semaphore payload to a handle with copy
2016 * transference has the same side effects on the source
2017 * semaphore’s payload as executing a semaphore wait operation."
2019 * In other words, it may still be a SYNC_FD semaphore, but it's now
2020 * considered to have been waited on and no longer has a sync file
2026 pthread_mutex_unlock(&device
->mutex
);
2028 /* There are two reasons why this could happen:
2030 * 1) The user is trying to export without submitting something that
2031 * signals the semaphore. If this is the case, it's their bug so
2032 * what we return here doesn't matter.
2034 * 2) The kernel didn't give us a file descriptor. The most likely
2035 * reason for this is running out of file descriptors.
2038 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
2044 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
:
2045 if (pGetFdInfo
->handleType
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
)
2046 fd
= anv_gem_syncobj_export_sync_file(device
, impl
->syncobj
);
2048 assert(pGetFdInfo
->handleType
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
);
2049 fd
= anv_gem_syncobj_handle_to_fd(device
, impl
->syncobj
);
2052 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
2057 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
2060 /* From the Vulkan 1.0.53 spec:
2062 * "Export operations have the same transference as the specified handle
2063 * type’s import operations. [...] If the semaphore was using a
2064 * temporarily imported payload, the semaphore’s prior permanent payload
2067 if (impl
== &semaphore
->temporary
)
2068 anv_semaphore_impl_cleanup(device
, impl
);
2073 VkResult
anv_GetSemaphoreCounterValueKHR(
2075 VkSemaphore _semaphore
,
2078 ANV_FROM_HANDLE(anv_device
, device
, _device
);
2079 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, _semaphore
);
2081 struct anv_semaphore_impl
*impl
=
2082 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
2083 &semaphore
->temporary
: &semaphore
->permanent
;
2085 switch (impl
->type
) {
2086 case ANV_SEMAPHORE_TYPE_TIMELINE
: {
2087 pthread_mutex_lock(&device
->mutex
);
2088 *pValue
= impl
->timeline
.highest_past
;
2089 pthread_mutex_unlock(&device
->mutex
);
2094 unreachable("Invalid semaphore type");
2099 anv_timeline_wait_locked(struct anv_device
*device
,
2100 struct anv_timeline
*timeline
,
2101 uint64_t serial
, uint64_t abs_timeout_ns
)
2103 /* Wait on the queue_submit condition variable until the timeline has a
2104 * time point pending that's at least as high as serial.
2106 while (timeline
->highest_pending
< serial
) {
2107 struct timespec abstime
= {
2108 .tv_sec
= abs_timeout_ns
/ NSEC_PER_SEC
,
2109 .tv_nsec
= abs_timeout_ns
% NSEC_PER_SEC
,
2112 int ret
= pthread_cond_timedwait(&device
->queue_submit
,
2113 &device
->mutex
, &abstime
);
2114 assert(ret
!= EINVAL
);
2115 if (anv_gettime_ns() >= abs_timeout_ns
&&
2116 timeline
->highest_pending
< serial
)
2121 VkResult result
= anv_timeline_gc_locked(device
, timeline
);
2122 if (result
!= VK_SUCCESS
)
2125 if (timeline
->highest_past
>= serial
)
2128 /* If we got here, our earliest time point has a busy BO */
2129 struct anv_timeline_point
*point
=
2130 list_first_entry(&timeline
->points
,
2131 struct anv_timeline_point
, link
);
2133 /* Drop the lock while we wait. */
2135 pthread_mutex_unlock(&device
->mutex
);
2137 result
= anv_device_wait(device
, point
->bo
,
2138 anv_get_relative_timeout(abs_timeout_ns
));
2140 /* Pick the mutex back up */
2141 pthread_mutex_lock(&device
->mutex
);
2144 /* This covers both VK_TIMEOUT and VK_ERROR_DEVICE_LOST */
2145 if (result
!= VK_SUCCESS
)
2151 anv_timelines_wait(struct anv_device
*device
,
2152 struct anv_timeline
**timelines
,
2153 const uint64_t *serials
,
2154 uint32_t n_timelines
,
2156 uint64_t abs_timeout_ns
)
2158 if (!wait_all
&& n_timelines
> 1) {
2161 pthread_mutex_lock(&device
->mutex
);
2162 for (uint32_t i
= 0; i
< n_timelines
; i
++) {
2164 anv_timeline_wait_locked(device
, timelines
[i
], serials
[i
], 0);
2165 if (result
!= VK_TIMEOUT
)
2169 if (result
!= VK_TIMEOUT
||
2170 anv_gettime_ns() >= abs_timeout_ns
) {
2171 pthread_mutex_unlock(&device
->mutex
);
2175 /* If none of them are ready do a short wait so we don't completely
2176 * spin while holding the lock. The 10us is completely arbitrary.
2178 uint64_t abs_short_wait_ns
=
2179 anv_get_absolute_timeout(
2180 MIN2((anv_gettime_ns() - abs_timeout_ns
) / 10, 10 * 1000));
2181 struct timespec abstime
= {
2182 .tv_sec
= abs_short_wait_ns
/ NSEC_PER_SEC
,
2183 .tv_nsec
= abs_short_wait_ns
% NSEC_PER_SEC
,
2186 ret
= pthread_cond_timedwait(&device
->queue_submit
,
2187 &device
->mutex
, &abstime
);
2188 assert(ret
!= EINVAL
);
2191 VkResult result
= VK_SUCCESS
;
2192 pthread_mutex_lock(&device
->mutex
);
2193 for (uint32_t i
= 0; i
< n_timelines
; i
++) {
2195 anv_timeline_wait_locked(device
, timelines
[i
],
2196 serials
[i
], abs_timeout_ns
);
2197 if (result
!= VK_SUCCESS
)
2200 pthread_mutex_unlock(&device
->mutex
);
2205 VkResult
anv_WaitSemaphoresKHR(
2207 const VkSemaphoreWaitInfoKHR
* pWaitInfo
,
2210 ANV_FROM_HANDLE(anv_device
, device
, _device
);
2212 struct anv_timeline
**timelines
=
2213 vk_alloc(&device
->alloc
,
2214 pWaitInfo
->semaphoreCount
* sizeof(*timelines
),
2215 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND
);
2217 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
2219 uint64_t *values
= vk_alloc(&device
->alloc
,
2220 pWaitInfo
->semaphoreCount
* sizeof(*values
),
2221 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND
);
2223 vk_free(&device
->alloc
, timelines
);
2224 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
2227 uint32_t handle_count
= 0;
2228 for (uint32_t i
= 0; i
< pWaitInfo
->semaphoreCount
; i
++) {
2229 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pWaitInfo
->pSemaphores
[i
]);
2230 struct anv_semaphore_impl
*impl
=
2231 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
2232 &semaphore
->temporary
: &semaphore
->permanent
;
2234 assert(impl
->type
== ANV_SEMAPHORE_TYPE_TIMELINE
);
2236 if (pWaitInfo
->pValues
[i
] == 0)
2239 timelines
[handle_count
] = &impl
->timeline
;
2240 values
[handle_count
] = pWaitInfo
->pValues
[i
];
2244 VkResult result
= VK_SUCCESS
;
2245 if (handle_count
> 0) {
2246 result
= anv_timelines_wait(device
, timelines
, values
, handle_count
,
2247 !(pWaitInfo
->flags
& VK_SEMAPHORE_WAIT_ANY_BIT_KHR
),
2251 vk_free(&device
->alloc
, timelines
);
2252 vk_free(&device
->alloc
, values
);
2257 VkResult
anv_SignalSemaphoreKHR(
2259 const VkSemaphoreSignalInfoKHR
* pSignalInfo
)
2261 ANV_FROM_HANDLE(anv_device
, device
, _device
);
2262 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pSignalInfo
->semaphore
);
2264 struct anv_semaphore_impl
*impl
=
2265 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
2266 &semaphore
->temporary
: &semaphore
->permanent
;
2268 switch (impl
->type
) {
2269 case ANV_SEMAPHORE_TYPE_TIMELINE
: {
2270 pthread_mutex_lock(&device
->mutex
);
2272 VkResult result
= anv_timeline_gc_locked(device
, &impl
->timeline
);
2274 assert(pSignalInfo
->value
> impl
->timeline
.highest_pending
);
2276 impl
->timeline
.highest_pending
= impl
->timeline
.highest_past
= pSignalInfo
->value
;
2278 if (result
== VK_SUCCESS
)
2279 result
= anv_device_submit_deferred_locked(device
);
2281 pthread_cond_broadcast(&device
->queue_submit
);
2282 pthread_mutex_unlock(&device
->mutex
);
2287 unreachable("Invalid semaphore type");