2 * Copyright © 2015 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
25 * This file implements VkQueue, VkFence, and VkSemaphore
32 #include "anv_private.h"
35 #include "genxml/gen7_pack.h"
37 uint64_t anv_gettime_ns(void)
39 struct timespec current
;
40 clock_gettime(CLOCK_MONOTONIC
, ¤t
);
41 return (uint64_t)current
.tv_sec
* NSEC_PER_SEC
+ current
.tv_nsec
;
44 uint64_t anv_get_absolute_timeout(uint64_t timeout
)
48 uint64_t current_time
= anv_gettime_ns();
49 uint64_t max_timeout
= (uint64_t) INT64_MAX
- current_time
;
51 timeout
= MIN2(max_timeout
, timeout
);
53 return (current_time
+ timeout
);
56 static int64_t anv_get_relative_timeout(uint64_t abs_timeout
)
58 uint64_t now
= anv_gettime_ns();
60 /* We don't want negative timeouts.
62 * DRM_IOCTL_I915_GEM_WAIT uses a signed 64 bit timeout and is
63 * supposed to block indefinitely timeouts < 0. Unfortunately,
64 * this was broken for a couple of kernel releases. Since there's
65 * no way to know whether or not the kernel we're using is one of
66 * the broken ones, the best we can do is to clamp the timeout to
67 * INT64_MAX. This limits the maximum timeout from 584 years to
68 * 292 years - likely not a big deal.
70 if (abs_timeout
< now
)
73 uint64_t rel_timeout
= abs_timeout
- now
;
74 if (rel_timeout
> (uint64_t) INT64_MAX
)
75 rel_timeout
= INT64_MAX
;
80 static struct anv_semaphore
*anv_semaphore_ref(struct anv_semaphore
*semaphore
);
81 static void anv_semaphore_unref(struct anv_device
*device
, struct anv_semaphore
*semaphore
);
82 static void anv_semaphore_impl_cleanup(struct anv_device
*device
,
83 struct anv_semaphore_impl
*impl
);
86 anv_queue_submit_free(struct anv_device
*device
,
87 struct anv_queue_submit
*submit
)
89 const VkAllocationCallbacks
*alloc
= submit
->alloc
;
91 for (uint32_t i
= 0; i
< submit
->temporary_semaphore_count
; i
++)
92 anv_semaphore_impl_cleanup(device
, &submit
->temporary_semaphores
[i
]);
93 for (uint32_t i
= 0; i
< submit
->sync_fd_semaphore_count
; i
++)
94 anv_semaphore_unref(device
, submit
->sync_fd_semaphores
[i
]);
95 /* Execbuf does not consume the in_fence. It's our job to close it. */
96 if (submit
->in_fence
!= -1)
97 close(submit
->in_fence
);
98 if (submit
->out_fence
!= -1)
99 close(submit
->out_fence
);
100 vk_free(alloc
, submit
->fences
);
101 vk_free(alloc
, submit
->temporary_semaphores
);
102 vk_free(alloc
, submit
->wait_timelines
);
103 vk_free(alloc
, submit
->wait_timeline_values
);
104 vk_free(alloc
, submit
->signal_timelines
);
105 vk_free(alloc
, submit
->signal_timeline_values
);
106 vk_free(alloc
, submit
->fence_bos
);
107 vk_free(alloc
, submit
);
111 anv_queue_submit_ready_locked(struct anv_queue_submit
*submit
)
113 for (uint32_t i
= 0; i
< submit
->wait_timeline_count
; i
++) {
114 if (submit
->wait_timeline_values
[i
] > submit
->wait_timelines
[i
]->highest_pending
)
122 anv_timeline_init(struct anv_device
*device
,
123 struct anv_timeline
*timeline
,
124 uint64_t initial_value
)
126 timeline
->highest_past
=
127 timeline
->highest_pending
= initial_value
;
128 list_inithead(&timeline
->points
);
129 list_inithead(&timeline
->free_points
);
135 anv_timeline_finish(struct anv_device
*device
,
136 struct anv_timeline
*timeline
)
138 list_for_each_entry_safe(struct anv_timeline_point
, point
,
139 &timeline
->free_points
, link
) {
140 list_del(&point
->link
);
141 anv_device_release_bo(device
, point
->bo
);
142 vk_free(&device
->vk
.alloc
, point
);
144 list_for_each_entry_safe(struct anv_timeline_point
, point
,
145 &timeline
->points
, link
) {
146 list_del(&point
->link
);
147 anv_device_release_bo(device
, point
->bo
);
148 vk_free(&device
->vk
.alloc
, point
);
153 anv_timeline_add_point_locked(struct anv_device
*device
,
154 struct anv_timeline
*timeline
,
156 struct anv_timeline_point
**point
)
158 VkResult result
= VK_SUCCESS
;
160 if (list_is_empty(&timeline
->free_points
)) {
162 vk_zalloc(&device
->vk
.alloc
, sizeof(**point
),
163 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE
);
165 result
= vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
166 if (result
== VK_SUCCESS
) {
167 result
= anv_device_alloc_bo(device
, 4096,
168 ANV_BO_ALLOC_EXTERNAL
|
169 ANV_BO_ALLOC_IMPLICIT_SYNC
,
170 0 /* explicit_address */,
172 if (result
!= VK_SUCCESS
)
173 vk_free(&device
->vk
.alloc
, *point
);
176 *point
= list_first_entry(&timeline
->free_points
,
177 struct anv_timeline_point
, link
);
178 list_del(&(*point
)->link
);
181 if (result
== VK_SUCCESS
) {
182 (*point
)->serial
= value
;
183 list_addtail(&(*point
)->link
, &timeline
->points
);
190 anv_timeline_gc_locked(struct anv_device
*device
,
191 struct anv_timeline
*timeline
)
193 list_for_each_entry_safe(struct anv_timeline_point
, point
,
194 &timeline
->points
, link
) {
195 /* timeline->higest_pending is only incremented once submission has
196 * happened. If this point has a greater serial, it means the point
197 * hasn't been submitted yet.
199 if (point
->serial
> timeline
->highest_pending
)
202 /* If someone is waiting on this time point, consider it busy and don't
203 * try to recycle it. There's a slim possibility that it's no longer
204 * busy by the time we look at it but we would be recycling it out from
205 * under a waiter and that can lead to weird races.
207 * We walk the list in-order so if this time point is still busy so is
208 * every following time point
210 assert(point
->waiting
>= 0);
214 /* Garbage collect any signaled point. */
215 VkResult result
= anv_device_bo_busy(device
, point
->bo
);
216 if (result
== VK_NOT_READY
) {
217 /* We walk the list in-order so if this time point is still busy so
218 * is every following time point
221 } else if (result
!= VK_SUCCESS
) {
225 assert(timeline
->highest_past
< point
->serial
);
226 timeline
->highest_past
= point
->serial
;
228 list_del(&point
->link
);
229 list_add(&point
->link
, &timeline
->free_points
);
235 static VkResult
anv_queue_submit_add_fence_bo(struct anv_queue_submit
*submit
,
240 anv_queue_submit_timeline_locked(struct anv_queue
*queue
,
241 struct anv_queue_submit
*submit
)
245 for (uint32_t i
= 0; i
< submit
->wait_timeline_count
; i
++) {
246 struct anv_timeline
*timeline
= submit
->wait_timelines
[i
];
247 uint64_t wait_value
= submit
->wait_timeline_values
[i
];
249 if (timeline
->highest_past
>= wait_value
)
252 list_for_each_entry(struct anv_timeline_point
, point
, &timeline
->points
, link
) {
253 if (point
->serial
< wait_value
)
255 result
= anv_queue_submit_add_fence_bo(submit
, point
->bo
, false);
256 if (result
!= VK_SUCCESS
)
261 for (uint32_t i
= 0; i
< submit
->signal_timeline_count
; i
++) {
262 struct anv_timeline
*timeline
= submit
->signal_timelines
[i
];
263 uint64_t signal_value
= submit
->signal_timeline_values
[i
];
264 struct anv_timeline_point
*point
;
266 result
= anv_timeline_add_point_locked(queue
->device
, timeline
,
267 signal_value
, &point
);
268 if (result
!= VK_SUCCESS
)
271 result
= anv_queue_submit_add_fence_bo(submit
, point
->bo
, true);
272 if (result
!= VK_SUCCESS
)
276 result
= anv_queue_execbuf_locked(queue
, submit
);
278 if (result
== VK_SUCCESS
) {
279 /* Update the pending values in the timeline objects. */
280 for (uint32_t i
= 0; i
< submit
->signal_timeline_count
; i
++) {
281 struct anv_timeline
*timeline
= submit
->signal_timelines
[i
];
282 uint64_t signal_value
= submit
->signal_timeline_values
[i
];
284 assert(signal_value
> timeline
->highest_pending
);
285 timeline
->highest_pending
= signal_value
;
288 /* Update signaled semaphores backed by syncfd. */
289 for (uint32_t i
= 0; i
< submit
->sync_fd_semaphore_count
; i
++) {
290 struct anv_semaphore
*semaphore
= submit
->sync_fd_semaphores
[i
];
291 /* Out fences can't have temporary state because that would imply
292 * that we imported a sync file and are trying to signal it.
294 assert(semaphore
->temporary
.type
== ANV_SEMAPHORE_TYPE_NONE
);
295 struct anv_semaphore_impl
*impl
= &semaphore
->permanent
;
297 assert(impl
->type
== ANV_SEMAPHORE_TYPE_SYNC_FILE
);
298 impl
->fd
= dup(submit
->out_fence
);
301 /* Unblock any waiter by signaling the points, the application will get
302 * a device lost error code.
304 for (uint32_t i
= 0; i
< submit
->signal_timeline_count
; i
++) {
305 struct anv_timeline
*timeline
= submit
->signal_timelines
[i
];
306 uint64_t signal_value
= submit
->signal_timeline_values
[i
];
308 assert(signal_value
> timeline
->highest_pending
);
309 timeline
->highest_past
= timeline
->highest_pending
= signal_value
;
317 anv_queue_submit_deferred_locked(struct anv_queue
*queue
, uint32_t *advance
)
319 VkResult result
= VK_SUCCESS
;
321 /* Go through all the queued submissions and submit then until we find one
322 * that's waiting on a point that hasn't materialized yet.
324 list_for_each_entry_safe(struct anv_queue_submit
, submit
,
325 &queue
->queued_submits
, link
) {
326 if (!anv_queue_submit_ready_locked(submit
))
330 list_del(&submit
->link
);
332 result
= anv_queue_submit_timeline_locked(queue
, submit
);
334 anv_queue_submit_free(queue
->device
, submit
);
336 if (result
!= VK_SUCCESS
)
344 anv_device_submit_deferred_locked(struct anv_device
*device
)
346 uint32_t advance
= 0;
347 return anv_queue_submit_deferred_locked(&device
->queue
, &advance
);
351 _anv_queue_submit(struct anv_queue
*queue
, struct anv_queue_submit
**_submit
,
354 struct anv_queue_submit
*submit
= *_submit
;
356 /* Wait before signal behavior means we might keep alive the
357 * anv_queue_submit object a bit longer, so transfer the ownership to the
362 pthread_mutex_lock(&queue
->device
->mutex
);
363 list_addtail(&submit
->link
, &queue
->queued_submits
);
364 VkResult result
= anv_device_submit_deferred_locked(queue
->device
);
366 while (result
== VK_SUCCESS
&& !list_is_empty(&queue
->queued_submits
)) {
367 int ret
= pthread_cond_wait(&queue
->device
->queue_submit
,
368 &queue
->device
->mutex
);
370 result
= anv_device_set_lost(queue
->device
, "wait timeout");
374 result
= anv_device_submit_deferred_locked(queue
->device
);
377 pthread_mutex_unlock(&queue
->device
->mutex
);
382 anv_queue_init(struct anv_device
*device
, struct anv_queue
*queue
)
384 vk_object_base_init(&device
->vk
, &queue
->base
, VK_OBJECT_TYPE_QUEUE
);
385 queue
->device
= device
;
388 list_inithead(&queue
->queued_submits
);
394 anv_queue_finish(struct anv_queue
*queue
)
396 vk_object_base_finish(&queue
->base
);
400 anv_queue_submit_add_fence_bo(struct anv_queue_submit
*submit
,
404 if (submit
->fence_bo_count
>= submit
->fence_bo_array_length
) {
405 uint32_t new_len
= MAX2(submit
->fence_bo_array_length
* 2, 64);
408 vk_realloc(submit
->alloc
,
409 submit
->fence_bos
, new_len
* sizeof(*submit
->fence_bos
),
410 8, submit
->alloc_scope
);
411 if (submit
->fence_bos
== NULL
)
412 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
414 submit
->fence_bo_array_length
= new_len
;
417 /* Take advantage that anv_bo are allocated at 8 byte alignement so we can
418 * use the lowest bit to store whether this is a BO we need to signal.
420 submit
->fence_bos
[submit
->fence_bo_count
++] = anv_pack_ptr(bo
, 1, signal
);
426 anv_queue_submit_add_syncobj(struct anv_queue_submit
* submit
,
427 struct anv_device
*device
,
428 uint32_t handle
, uint32_t flags
)
432 if (submit
->fence_count
>= submit
->fence_array_length
) {
433 uint32_t new_len
= MAX2(submit
->fence_array_length
* 2, 64);
436 vk_realloc(submit
->alloc
,
437 submit
->fences
, new_len
* sizeof(*submit
->fences
),
438 8, submit
->alloc_scope
);
439 if (submit
->fences
== NULL
)
440 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
442 submit
->fence_array_length
= new_len
;
445 submit
->fences
[submit
->fence_count
++] = (struct drm_i915_gem_exec_fence
) {
454 anv_queue_submit_add_sync_fd_fence(struct anv_queue_submit
*submit
,
455 struct anv_semaphore
*semaphore
)
457 if (submit
->sync_fd_semaphore_count
>= submit
->sync_fd_semaphore_array_length
) {
458 uint32_t new_len
= MAX2(submit
->sync_fd_semaphore_array_length
* 2, 64);
459 struct anv_semaphore
**new_semaphores
=
460 vk_realloc(submit
->alloc
, submit
->sync_fd_semaphores
,
461 new_len
* sizeof(*submit
->sync_fd_semaphores
), 8,
462 submit
->alloc_scope
);
463 if (new_semaphores
== NULL
)
464 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
466 submit
->sync_fd_semaphores
= new_semaphores
;
469 submit
->sync_fd_semaphores
[submit
->sync_fd_semaphore_count
++] =
470 anv_semaphore_ref(semaphore
);
471 submit
->need_out_fence
= true;
477 anv_queue_submit_add_timeline_wait(struct anv_queue_submit
* submit
,
478 struct anv_device
*device
,
479 struct anv_timeline
*timeline
,
482 if (submit
->wait_timeline_count
>= submit
->wait_timeline_array_length
) {
483 uint32_t new_len
= MAX2(submit
->wait_timeline_array_length
* 2, 64);
485 submit
->wait_timelines
=
486 vk_realloc(submit
->alloc
,
487 submit
->wait_timelines
, new_len
* sizeof(*submit
->wait_timelines
),
488 8, submit
->alloc_scope
);
489 if (submit
->wait_timelines
== NULL
)
490 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
492 submit
->wait_timeline_values
=
493 vk_realloc(submit
->alloc
,
494 submit
->wait_timeline_values
, new_len
* sizeof(*submit
->wait_timeline_values
),
495 8, submit
->alloc_scope
);
496 if (submit
->wait_timeline_values
== NULL
)
497 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
499 submit
->wait_timeline_array_length
= new_len
;
502 submit
->wait_timelines
[submit
->wait_timeline_count
] = timeline
;
503 submit
->wait_timeline_values
[submit
->wait_timeline_count
] = value
;
505 submit
->wait_timeline_count
++;
511 anv_queue_submit_add_timeline_signal(struct anv_queue_submit
* submit
,
512 struct anv_device
*device
,
513 struct anv_timeline
*timeline
,
516 assert(timeline
->highest_pending
< value
);
518 if (submit
->signal_timeline_count
>= submit
->signal_timeline_array_length
) {
519 uint32_t new_len
= MAX2(submit
->signal_timeline_array_length
* 2, 64);
521 submit
->signal_timelines
=
522 vk_realloc(submit
->alloc
,
523 submit
->signal_timelines
, new_len
* sizeof(*submit
->signal_timelines
),
524 8, submit
->alloc_scope
);
525 if (submit
->signal_timelines
== NULL
)
526 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
528 submit
->signal_timeline_values
=
529 vk_realloc(submit
->alloc
,
530 submit
->signal_timeline_values
, new_len
* sizeof(*submit
->signal_timeline_values
),
531 8, submit
->alloc_scope
);
532 if (submit
->signal_timeline_values
== NULL
)
533 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
535 submit
->signal_timeline_array_length
= new_len
;
538 submit
->signal_timelines
[submit
->signal_timeline_count
] = timeline
;
539 submit
->signal_timeline_values
[submit
->signal_timeline_count
] = value
;
541 submit
->signal_timeline_count
++;
546 static struct anv_queue_submit
*
547 anv_queue_submit_alloc(struct anv_device
*device
, int perf_query_pass
)
549 const VkAllocationCallbacks
*alloc
= &device
->vk
.alloc
;
550 VkSystemAllocationScope alloc_scope
= VK_SYSTEM_ALLOCATION_SCOPE_DEVICE
;
552 struct anv_queue_submit
*submit
= vk_zalloc(alloc
, sizeof(*submit
), 8, alloc_scope
);
556 submit
->alloc
= alloc
;
557 submit
->alloc_scope
= alloc_scope
;
558 submit
->in_fence
= -1;
559 submit
->out_fence
= -1;
560 submit
->perf_query_pass
= perf_query_pass
;
566 anv_queue_submit_simple_batch(struct anv_queue
*queue
,
567 struct anv_batch
*batch
)
569 if (queue
->device
->no_hw
)
572 struct anv_device
*device
= queue
->device
;
573 struct anv_queue_submit
*submit
= anv_queue_submit_alloc(device
, -1);
575 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
577 bool has_syncobj_wait
= device
->physical
->has_syncobj_wait
;
580 struct anv_bo
*batch_bo
, *sync_bo
;
582 if (has_syncobj_wait
) {
583 syncobj
= anv_gem_syncobj_create(device
, 0);
585 result
= vk_error(VK_ERROR_OUT_OF_DEVICE_MEMORY
);
586 goto err_free_submit
;
589 result
= anv_queue_submit_add_syncobj(submit
, device
, syncobj
,
590 I915_EXEC_FENCE_SIGNAL
);
592 result
= anv_device_alloc_bo(device
, 4096,
593 ANV_BO_ALLOC_EXTERNAL
|
594 ANV_BO_ALLOC_IMPLICIT_SYNC
,
595 0 /* explicit_address */,
597 if (result
!= VK_SUCCESS
)
598 goto err_free_submit
;
600 result
= anv_queue_submit_add_fence_bo(submit
, sync_bo
, true /* signal */);
603 if (result
!= VK_SUCCESS
)
604 goto err_destroy_sync_primitive
;
607 uint32_t size
= align_u32(batch
->next
- batch
->start
, 8);
608 result
= anv_bo_pool_alloc(&device
->batch_bo_pool
, size
, &batch_bo
);
609 if (result
!= VK_SUCCESS
)
610 goto err_destroy_sync_primitive
;
612 memcpy(batch_bo
->map
, batch
->start
, size
);
613 if (!device
->info
.has_llc
)
614 gen_flush_range(batch_bo
->map
, size
);
616 submit
->simple_bo
= batch_bo
;
617 submit
->simple_bo_size
= size
;
620 result
= _anv_queue_submit(queue
, &submit
, true);
622 if (result
== VK_SUCCESS
) {
623 if (has_syncobj_wait
) {
624 if (anv_gem_syncobj_wait(device
, &syncobj
, 1,
625 anv_get_absolute_timeout(INT64_MAX
), true))
626 result
= anv_device_set_lost(device
, "anv_gem_syncobj_wait failed: %m");
627 anv_gem_syncobj_destroy(device
, syncobj
);
629 result
= anv_device_wait(device
, sync_bo
,
630 anv_get_relative_timeout(INT64_MAX
));
631 anv_device_release_bo(device
, sync_bo
);
636 anv_bo_pool_free(&device
->batch_bo_pool
, batch_bo
);
639 anv_queue_submit_free(device
, submit
);
643 err_destroy_sync_primitive
:
644 if (has_syncobj_wait
)
645 anv_gem_syncobj_destroy(device
, syncobj
);
647 anv_device_release_bo(device
, sync_bo
);
650 anv_queue_submit_free(device
, submit
);
655 /* Transfer ownership of temporary semaphores from the VkSemaphore object to
656 * the anv_queue_submit object. Those temporary semaphores are then freed in
657 * anv_queue_submit_free() once the driver is finished with them.
660 maybe_transfer_temporary_semaphore(struct anv_queue_submit
*submit
,
661 struct anv_semaphore
*semaphore
,
662 struct anv_semaphore_impl
**out_impl
)
664 struct anv_semaphore_impl
*impl
= &semaphore
->temporary
;
666 if (impl
->type
== ANV_SEMAPHORE_TYPE_NONE
) {
667 *out_impl
= &semaphore
->permanent
;
671 /* BO backed timeline semaphores cannot be temporary. */
672 assert(impl
->type
!= ANV_SEMAPHORE_TYPE_TIMELINE
);
675 * There is a requirement to reset semaphore to their permanent state after
676 * submission. From the Vulkan 1.0.53 spec:
678 * "If the import is temporary, the implementation must restore the
679 * semaphore to its prior permanent state after submitting the next
680 * semaphore wait operation."
682 * In the case we defer the actual submission to a thread because of the
683 * wait-before-submit behavior required for timeline semaphores, we need to
684 * make copies of the temporary syncobj to ensure they stay alive until we
685 * do the actual execbuffer ioctl.
687 if (submit
->temporary_semaphore_count
>= submit
->temporary_semaphore_array_length
) {
688 uint32_t new_len
= MAX2(submit
->temporary_semaphore_array_length
* 2, 8);
689 /* Make sure that if the realloc fails, we still have the old semaphore
690 * array around to properly clean things up on failure.
692 struct anv_semaphore_impl
*new_array
=
693 vk_realloc(submit
->alloc
,
694 submit
->temporary_semaphores
,
695 new_len
* sizeof(*submit
->temporary_semaphores
),
696 8, submit
->alloc_scope
);
697 if (new_array
== NULL
)
698 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
700 submit
->temporary_semaphores
= new_array
;
701 submit
->temporary_semaphore_array_length
= new_len
;
704 /* Copy anv_semaphore_impl into anv_queue_submit. */
705 submit
->temporary_semaphores
[submit
->temporary_semaphore_count
++] = *impl
;
706 *out_impl
= &submit
->temporary_semaphores
[submit
->temporary_semaphore_count
- 1];
708 /* Clear the incoming semaphore */
709 impl
->type
= ANV_SEMAPHORE_TYPE_NONE
;
715 anv_queue_submit(struct anv_queue
*queue
,
716 struct anv_cmd_buffer
*cmd_buffer
,
717 const VkSemaphore
*in_semaphores
,
718 const uint64_t *in_values
,
719 uint32_t num_in_semaphores
,
720 const VkSemaphore
*out_semaphores
,
721 const uint64_t *out_values
,
722 uint32_t num_out_semaphores
,
723 struct anv_bo
*wsi_signal_bo
,
727 ANV_FROM_HANDLE(anv_fence
, fence
, _fence
);
728 struct anv_device
*device
= queue
->device
;
729 UNUSED
struct anv_physical_device
*pdevice
= device
->physical
;
730 struct anv_queue_submit
*submit
= anv_queue_submit_alloc(device
, perf_query_pass
);
732 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
734 submit
->cmd_buffer
= cmd_buffer
;
736 VkResult result
= VK_SUCCESS
;
738 for (uint32_t i
= 0; i
< num_in_semaphores
; i
++) {
739 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, in_semaphores
[i
]);
740 struct anv_semaphore_impl
*impl
;
742 result
= maybe_transfer_temporary_semaphore(submit
, semaphore
, &impl
);
743 if (result
!= VK_SUCCESS
)
746 switch (impl
->type
) {
747 case ANV_SEMAPHORE_TYPE_BO
:
748 assert(!pdevice
->has_syncobj
);
749 result
= anv_queue_submit_add_fence_bo(submit
, impl
->bo
, false /* signal */);
750 if (result
!= VK_SUCCESS
)
754 case ANV_SEMAPHORE_TYPE_WSI_BO
:
755 /* When using a window-system buffer as a semaphore, always enable
756 * EXEC_OBJECT_WRITE. This gives us a WaR hazard with the display or
757 * compositor's read of the buffer and enforces that we don't start
758 * rendering until they are finished. This is exactly the
759 * synchronization we want with vkAcquireNextImage.
761 result
= anv_queue_submit_add_fence_bo(submit
, impl
->bo
, true /* signal */);
762 if (result
!= VK_SUCCESS
)
766 case ANV_SEMAPHORE_TYPE_SYNC_FILE
:
767 assert(!pdevice
->has_syncobj
);
768 if (submit
->in_fence
== -1) {
769 submit
->in_fence
= impl
->fd
;
770 if (submit
->in_fence
== -1) {
771 result
= vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
776 int merge
= anv_gem_sync_file_merge(device
, submit
->in_fence
, impl
->fd
);
778 result
= vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
782 close(submit
->in_fence
);
784 submit
->in_fence
= merge
;
788 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
: {
789 result
= anv_queue_submit_add_syncobj(submit
, device
,
791 I915_EXEC_FENCE_WAIT
);
792 if (result
!= VK_SUCCESS
)
797 case ANV_SEMAPHORE_TYPE_TIMELINE
:
798 result
= anv_queue_submit_add_timeline_wait(submit
, device
,
800 in_values
? in_values
[i
] : 0);
801 if (result
!= VK_SUCCESS
)
810 for (uint32_t i
= 0; i
< num_out_semaphores
; i
++) {
811 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, out_semaphores
[i
]);
813 /* Under most circumstances, out fences won't be temporary. However,
814 * the spec does allow it for opaque_fd. From the Vulkan 1.0.53 spec:
816 * "If the import is temporary, the implementation must restore the
817 * semaphore to its prior permanent state after submitting the next
818 * semaphore wait operation."
820 * The spec says nothing whatsoever about signal operations on
821 * temporarily imported semaphores so it appears they are allowed.
822 * There are also CTS tests that require this to work.
824 struct anv_semaphore_impl
*impl
=
825 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
826 &semaphore
->temporary
: &semaphore
->permanent
;
828 switch (impl
->type
) {
829 case ANV_SEMAPHORE_TYPE_BO
:
830 assert(!pdevice
->has_syncobj
);
831 result
= anv_queue_submit_add_fence_bo(submit
, impl
->bo
, true /* signal */);
832 if (result
!= VK_SUCCESS
)
836 case ANV_SEMAPHORE_TYPE_SYNC_FILE
:
837 assert(!pdevice
->has_syncobj
);
838 result
= anv_queue_submit_add_sync_fd_fence(submit
, semaphore
);
839 if (result
!= VK_SUCCESS
)
843 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
: {
844 result
= anv_queue_submit_add_syncobj(submit
, device
, impl
->syncobj
,
845 I915_EXEC_FENCE_SIGNAL
);
846 if (result
!= VK_SUCCESS
)
851 case ANV_SEMAPHORE_TYPE_TIMELINE
:
852 result
= anv_queue_submit_add_timeline_signal(submit
, device
,
854 out_values
? out_values
[i
] : 0);
855 if (result
!= VK_SUCCESS
)
865 result
= anv_queue_submit_add_fence_bo(submit
, wsi_signal_bo
, true /* signal */);
866 if (result
!= VK_SUCCESS
)
871 /* Under most circumstances, out fences won't be temporary. However,
872 * the spec does allow it for opaque_fd. From the Vulkan 1.0.53 spec:
874 * "If the import is temporary, the implementation must restore the
875 * semaphore to its prior permanent state after submitting the next
876 * semaphore wait operation."
878 * The spec says nothing whatsoever about signal operations on
879 * temporarily imported semaphores so it appears they are allowed.
880 * There are also CTS tests that require this to work.
882 struct anv_fence_impl
*impl
=
883 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
884 &fence
->temporary
: &fence
->permanent
;
886 switch (impl
->type
) {
887 case ANV_FENCE_TYPE_BO
:
888 result
= anv_queue_submit_add_fence_bo(submit
, impl
->bo
.bo
, true /* signal */);
889 if (result
!= VK_SUCCESS
)
893 case ANV_FENCE_TYPE_SYNCOBJ
: {
895 * For the same reason we reset the signaled binary syncobj above,
896 * also reset the fence's syncobj so that they don't contain a
897 * signaled dma-fence.
899 result
= anv_queue_submit_add_syncobj(submit
, device
, impl
->syncobj
,
900 I915_EXEC_FENCE_SIGNAL
);
901 if (result
!= VK_SUCCESS
)
907 unreachable("Invalid fence type");
911 result
= _anv_queue_submit(queue
, &submit
, false);
912 if (result
!= VK_SUCCESS
)
915 if (fence
&& fence
->permanent
.type
== ANV_FENCE_TYPE_BO
) {
916 /* If we have permanent BO fence, the only type of temporary possible
917 * would be BO_WSI (because BO fences are not shareable). The Vulkan spec
918 * also requires that the fence passed to vkQueueSubmit() be :
921 * * not be associated with any other queue command that has not yet
922 * completed execution on that queue
924 * So the only acceptable type for the temporary is NONE.
926 assert(fence
->temporary
.type
== ANV_FENCE_TYPE_NONE
);
928 /* Once the execbuf has returned, we need to set the fence state to
929 * SUBMITTED. We can't do this before calling execbuf because
930 * anv_GetFenceStatus does take the global device lock before checking
933 * We set the fence state to SUBMITTED regardless of whether or not the
934 * execbuf succeeds because we need to ensure that vkWaitForFences() and
935 * vkGetFenceStatus() return a valid result (VK_ERROR_DEVICE_LOST or
936 * VK_SUCCESS) in a finite amount of time even if execbuf fails.
938 fence
->permanent
.bo
.state
= ANV_BO_FENCE_STATE_SUBMITTED
;
943 anv_queue_submit_free(device
, submit
);
948 VkResult
anv_QueueSubmit(
950 uint32_t submitCount
,
951 const VkSubmitInfo
* pSubmits
,
954 ANV_FROM_HANDLE(anv_queue
, queue
, _queue
);
956 if (queue
->device
->no_hw
)
959 /* Query for device status prior to submitting. Technically, we don't need
960 * to do this. However, if we have a client that's submitting piles of
961 * garbage, we would rather break as early as possible to keep the GPU
962 * hanging contained. If we don't check here, we'll either be waiting for
963 * the kernel to kick us or we'll have to wait until the client waits on a
964 * fence before we actually know whether or not we've hung.
966 VkResult result
= anv_device_query_status(queue
->device
);
967 if (result
!= VK_SUCCESS
)
970 if (fence
&& submitCount
== 0) {
971 /* If we don't have any command buffers, we need to submit a dummy
972 * batch to give GEM something to wait on. We could, potentially,
973 * come up with something more efficient but this shouldn't be a
976 result
= anv_queue_submit(queue
, NULL
, NULL
, NULL
, 0, NULL
, NULL
, 0,
981 for (uint32_t i
= 0; i
< submitCount
; i
++) {
982 /* Fence for this submit. NULL for all but the last one */
983 VkFence submit_fence
= (i
== submitCount
- 1) ? fence
: VK_NULL_HANDLE
;
985 const struct wsi_memory_signal_submit_info
*mem_signal_info
=
986 vk_find_struct_const(pSubmits
[i
].pNext
,
987 WSI_MEMORY_SIGNAL_SUBMIT_INFO_MESA
);
988 struct anv_bo
*wsi_signal_bo
=
989 mem_signal_info
&& mem_signal_info
->memory
!= VK_NULL_HANDLE
?
990 anv_device_memory_from_handle(mem_signal_info
->memory
)->bo
: NULL
;
992 const VkTimelineSemaphoreSubmitInfoKHR
*timeline_info
=
993 vk_find_struct_const(pSubmits
[i
].pNext
,
994 TIMELINE_SEMAPHORE_SUBMIT_INFO_KHR
);
995 const VkPerformanceQuerySubmitInfoKHR
*perf_info
=
996 vk_find_struct_const(pSubmits
[i
].pNext
,
997 PERFORMANCE_QUERY_SUBMIT_INFO_KHR
);
998 const uint64_t *wait_values
=
999 timeline_info
&& timeline_info
->waitSemaphoreValueCount
?
1000 timeline_info
->pWaitSemaphoreValues
: NULL
;
1001 const uint64_t *signal_values
=
1002 timeline_info
&& timeline_info
->signalSemaphoreValueCount
?
1003 timeline_info
->pSignalSemaphoreValues
: NULL
;
1005 if (pSubmits
[i
].commandBufferCount
== 0) {
1006 /* If we don't have any command buffers, we need to submit a dummy
1007 * batch to give GEM something to wait on. We could, potentially,
1008 * come up with something more efficient but this shouldn't be a
1011 result
= anv_queue_submit(queue
, NULL
,
1012 pSubmits
[i
].pWaitSemaphores
,
1014 pSubmits
[i
].waitSemaphoreCount
,
1015 pSubmits
[i
].pSignalSemaphores
,
1017 pSubmits
[i
].signalSemaphoreCount
,
1021 if (result
!= VK_SUCCESS
)
1027 for (uint32_t j
= 0; j
< pSubmits
[i
].commandBufferCount
; j
++) {
1028 ANV_FROM_HANDLE(anv_cmd_buffer
, cmd_buffer
,
1029 pSubmits
[i
].pCommandBuffers
[j
]);
1030 assert(cmd_buffer
->level
== VK_COMMAND_BUFFER_LEVEL_PRIMARY
);
1031 assert(!anv_batch_has_error(&cmd_buffer
->batch
));
1033 /* Fence for this execbuf. NULL for all but the last one */
1034 VkFence execbuf_fence
=
1035 (j
== pSubmits
[i
].commandBufferCount
- 1) ?
1036 submit_fence
: VK_NULL_HANDLE
;
1038 const VkSemaphore
*in_semaphores
= NULL
, *out_semaphores
= NULL
;
1039 const uint64_t *in_values
= NULL
, *out_values
= NULL
;
1040 uint32_t num_in_semaphores
= 0, num_out_semaphores
= 0;
1042 /* Only the first batch gets the in semaphores */
1043 in_semaphores
= pSubmits
[i
].pWaitSemaphores
;
1044 in_values
= wait_values
;
1045 num_in_semaphores
= pSubmits
[i
].waitSemaphoreCount
;
1048 if (j
== pSubmits
[i
].commandBufferCount
- 1) {
1049 /* Only the last batch gets the out semaphores */
1050 out_semaphores
= pSubmits
[i
].pSignalSemaphores
;
1051 out_values
= signal_values
;
1052 num_out_semaphores
= pSubmits
[i
].signalSemaphoreCount
;
1055 result
= anv_queue_submit(queue
, cmd_buffer
,
1056 in_semaphores
, in_values
, num_in_semaphores
,
1057 out_semaphores
, out_values
, num_out_semaphores
,
1058 wsi_signal_bo
, execbuf_fence
,
1059 perf_info
? perf_info
->counterPassIndex
: 0);
1060 if (result
!= VK_SUCCESS
)
1066 if (result
!= VK_SUCCESS
&& result
!= VK_ERROR_DEVICE_LOST
) {
1067 /* In the case that something has gone wrong we may end up with an
1068 * inconsistent state from which it may not be trivial to recover.
1069 * For example, we might have computed address relocations and
1070 * any future attempt to re-submit this job will need to know about
1071 * this and avoid computing relocation addresses again.
1073 * To avoid this sort of issues, we assume that if something was
1074 * wrong during submission we must already be in a really bad situation
1075 * anyway (such us being out of memory) and return
1076 * VK_ERROR_DEVICE_LOST to ensure that clients do not attempt to
1077 * submit the same job again to this device.
1079 * We skip doing this on VK_ERROR_DEVICE_LOST because
1080 * anv_device_set_lost() would have been called already by a callee of
1081 * anv_queue_submit().
1083 result
= anv_device_set_lost(queue
->device
, "vkQueueSubmit() failed");
1089 VkResult
anv_QueueWaitIdle(
1092 ANV_FROM_HANDLE(anv_queue
, queue
, _queue
);
1094 if (anv_device_is_lost(queue
->device
))
1095 return VK_ERROR_DEVICE_LOST
;
1097 return anv_queue_submit_simple_batch(queue
, NULL
);
1100 VkResult
anv_CreateFence(
1102 const VkFenceCreateInfo
* pCreateInfo
,
1103 const VkAllocationCallbacks
* pAllocator
,
1106 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1107 struct anv_fence
*fence
;
1109 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_FENCE_CREATE_INFO
);
1111 fence
= vk_zalloc2(&device
->vk
.alloc
, pAllocator
, sizeof(*fence
), 8,
1112 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT
);
1114 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1116 vk_object_base_init(&device
->vk
, &fence
->base
, VK_OBJECT_TYPE_FENCE
);
1118 if (device
->physical
->has_syncobj_wait
) {
1119 fence
->permanent
.type
= ANV_FENCE_TYPE_SYNCOBJ
;
1121 uint32_t create_flags
= 0;
1122 if (pCreateInfo
->flags
& VK_FENCE_CREATE_SIGNALED_BIT
)
1123 create_flags
|= DRM_SYNCOBJ_CREATE_SIGNALED
;
1125 fence
->permanent
.syncobj
= anv_gem_syncobj_create(device
, create_flags
);
1126 if (!fence
->permanent
.syncobj
)
1127 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1129 fence
->permanent
.type
= ANV_FENCE_TYPE_BO
;
1131 VkResult result
= anv_bo_pool_alloc(&device
->batch_bo_pool
, 4096,
1132 &fence
->permanent
.bo
.bo
);
1133 if (result
!= VK_SUCCESS
)
1136 if (pCreateInfo
->flags
& VK_FENCE_CREATE_SIGNALED_BIT
) {
1137 fence
->permanent
.bo
.state
= ANV_BO_FENCE_STATE_SIGNALED
;
1139 fence
->permanent
.bo
.state
= ANV_BO_FENCE_STATE_RESET
;
1143 *pFence
= anv_fence_to_handle(fence
);
1149 anv_fence_impl_cleanup(struct anv_device
*device
,
1150 struct anv_fence_impl
*impl
)
1152 switch (impl
->type
) {
1153 case ANV_FENCE_TYPE_NONE
:
1154 /* Dummy. Nothing to do */
1157 case ANV_FENCE_TYPE_BO
:
1158 anv_bo_pool_free(&device
->batch_bo_pool
, impl
->bo
.bo
);
1161 case ANV_FENCE_TYPE_WSI_BO
:
1162 anv_device_release_bo(device
, impl
->bo
.bo
);
1165 case ANV_FENCE_TYPE_SYNCOBJ
:
1166 anv_gem_syncobj_destroy(device
, impl
->syncobj
);
1169 case ANV_FENCE_TYPE_WSI
:
1170 impl
->fence_wsi
->destroy(impl
->fence_wsi
);
1174 unreachable("Invalid fence type");
1177 impl
->type
= ANV_FENCE_TYPE_NONE
;
1181 anv_fence_reset_temporary(struct anv_device
*device
,
1182 struct anv_fence
*fence
)
1184 if (fence
->temporary
.type
== ANV_FENCE_TYPE_NONE
)
1187 anv_fence_impl_cleanup(device
, &fence
->temporary
);
1190 void anv_DestroyFence(
1193 const VkAllocationCallbacks
* pAllocator
)
1195 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1196 ANV_FROM_HANDLE(anv_fence
, fence
, _fence
);
1201 anv_fence_impl_cleanup(device
, &fence
->temporary
);
1202 anv_fence_impl_cleanup(device
, &fence
->permanent
);
1204 vk_object_base_finish(&fence
->base
);
1205 vk_free2(&device
->vk
.alloc
, pAllocator
, fence
);
1208 VkResult
anv_ResetFences(
1210 uint32_t fenceCount
,
1211 const VkFence
* pFences
)
1213 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1215 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1216 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1218 /* From the Vulkan 1.0.53 spec:
1220 * "If any member of pFences currently has its payload imported with
1221 * temporary permanence, that fence’s prior permanent payload is
1222 * first restored. The remaining operations described therefore
1223 * operate on the restored payload.
1225 anv_fence_reset_temporary(device
, fence
);
1227 struct anv_fence_impl
*impl
= &fence
->permanent
;
1229 switch (impl
->type
) {
1230 case ANV_FENCE_TYPE_BO
:
1231 impl
->bo
.state
= ANV_BO_FENCE_STATE_RESET
;
1234 case ANV_FENCE_TYPE_SYNCOBJ
:
1235 anv_gem_syncobj_reset(device
, impl
->syncobj
);
1239 unreachable("Invalid fence type");
1246 VkResult
anv_GetFenceStatus(
1250 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1251 ANV_FROM_HANDLE(anv_fence
, fence
, _fence
);
1253 if (anv_device_is_lost(device
))
1254 return VK_ERROR_DEVICE_LOST
;
1256 struct anv_fence_impl
*impl
=
1257 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1258 &fence
->temporary
: &fence
->permanent
;
1260 switch (impl
->type
) {
1261 case ANV_FENCE_TYPE_BO
:
1262 case ANV_FENCE_TYPE_WSI_BO
:
1263 switch (impl
->bo
.state
) {
1264 case ANV_BO_FENCE_STATE_RESET
:
1265 /* If it hasn't even been sent off to the GPU yet, it's not ready */
1266 return VK_NOT_READY
;
1268 case ANV_BO_FENCE_STATE_SIGNALED
:
1269 /* It's been signaled, return success */
1272 case ANV_BO_FENCE_STATE_SUBMITTED
: {
1273 VkResult result
= anv_device_bo_busy(device
, impl
->bo
.bo
);
1274 if (result
== VK_SUCCESS
) {
1275 impl
->bo
.state
= ANV_BO_FENCE_STATE_SIGNALED
;
1282 unreachable("Invalid fence status");
1285 case ANV_FENCE_TYPE_SYNCOBJ
: {
1286 int ret
= anv_gem_syncobj_wait(device
, &impl
->syncobj
, 1, 0, true);
1288 if (errno
== ETIME
) {
1289 return VK_NOT_READY
;
1291 /* We don't know the real error. */
1292 return anv_device_set_lost(device
, "drm_syncobj_wait failed: %m");
1300 unreachable("Invalid fence type");
1305 anv_wait_for_syncobj_fences(struct anv_device
*device
,
1306 uint32_t fenceCount
,
1307 const VkFence
*pFences
,
1309 uint64_t abs_timeout_ns
)
1311 uint32_t *syncobjs
= vk_zalloc(&device
->vk
.alloc
,
1312 sizeof(*syncobjs
) * fenceCount
, 8,
1313 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND
);
1315 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1317 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1318 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1319 assert(fence
->permanent
.type
== ANV_FENCE_TYPE_SYNCOBJ
);
1321 struct anv_fence_impl
*impl
=
1322 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1323 &fence
->temporary
: &fence
->permanent
;
1325 assert(impl
->type
== ANV_FENCE_TYPE_SYNCOBJ
);
1326 syncobjs
[i
] = impl
->syncobj
;
1329 /* The gem_syncobj_wait ioctl may return early due to an inherent
1330 * limitation in the way it computes timeouts. Loop until we've actually
1331 * passed the timeout.
1335 ret
= anv_gem_syncobj_wait(device
, syncobjs
, fenceCount
,
1336 abs_timeout_ns
, waitAll
);
1337 } while (ret
== -1 && errno
== ETIME
&& anv_gettime_ns() < abs_timeout_ns
);
1339 vk_free(&device
->vk
.alloc
, syncobjs
);
1342 if (errno
== ETIME
) {
1345 /* We don't know the real error. */
1346 return anv_device_set_lost(device
, "drm_syncobj_wait failed: %m");
1354 anv_wait_for_bo_fences(struct anv_device
*device
,
1355 uint32_t fenceCount
,
1356 const VkFence
*pFences
,
1358 uint64_t abs_timeout_ns
)
1360 VkResult result
= VK_SUCCESS
;
1361 uint32_t pending_fences
= fenceCount
;
1362 while (pending_fences
) {
1364 bool signaled_fences
= false;
1365 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1366 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1368 struct anv_fence_impl
*impl
=
1369 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1370 &fence
->temporary
: &fence
->permanent
;
1371 assert(impl
->type
== ANV_FENCE_TYPE_BO
||
1372 impl
->type
== ANV_FENCE_TYPE_WSI_BO
);
1374 switch (impl
->bo
.state
) {
1375 case ANV_BO_FENCE_STATE_RESET
:
1376 /* This fence hasn't been submitted yet, we'll catch it the next
1377 * time around. Yes, this may mean we dead-loop but, short of
1378 * lots of locking and a condition variable, there's not much that
1379 * we can do about that.
1384 case ANV_BO_FENCE_STATE_SIGNALED
:
1385 /* This fence is not pending. If waitAll isn't set, we can return
1386 * early. Otherwise, we have to keep going.
1389 result
= VK_SUCCESS
;
1394 case ANV_BO_FENCE_STATE_SUBMITTED
:
1395 /* These are the fences we really care about. Go ahead and wait
1396 * on it until we hit a timeout.
1398 result
= anv_device_wait(device
, impl
->bo
.bo
,
1399 anv_get_relative_timeout(abs_timeout_ns
));
1402 impl
->bo
.state
= ANV_BO_FENCE_STATE_SIGNALED
;
1403 signaled_fences
= true;
1417 if (pending_fences
&& !signaled_fences
) {
1418 /* If we've hit this then someone decided to vkWaitForFences before
1419 * they've actually submitted any of them to a queue. This is a
1420 * fairly pessimal case, so it's ok to lock here and use a standard
1421 * pthreads condition variable.
1423 pthread_mutex_lock(&device
->mutex
);
1425 /* It's possible that some of the fences have changed state since the
1426 * last time we checked. Now that we have the lock, check for
1427 * pending fences again and don't wait if it's changed.
1429 uint32_t now_pending_fences
= 0;
1430 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1431 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1432 if (fence
->permanent
.bo
.state
== ANV_BO_FENCE_STATE_RESET
)
1433 now_pending_fences
++;
1435 assert(now_pending_fences
<= pending_fences
);
1437 if (now_pending_fences
== pending_fences
) {
1438 struct timespec abstime
= {
1439 .tv_sec
= abs_timeout_ns
/ NSEC_PER_SEC
,
1440 .tv_nsec
= abs_timeout_ns
% NSEC_PER_SEC
,
1444 ret
= pthread_cond_timedwait(&device
->queue_submit
,
1445 &device
->mutex
, &abstime
);
1446 assert(ret
!= EINVAL
);
1447 if (anv_gettime_ns() >= abs_timeout_ns
) {
1448 pthread_mutex_unlock(&device
->mutex
);
1449 result
= VK_TIMEOUT
;
1454 pthread_mutex_unlock(&device
->mutex
);
1459 if (anv_device_is_lost(device
))
1460 return VK_ERROR_DEVICE_LOST
;
1466 anv_wait_for_wsi_fence(struct anv_device
*device
,
1467 struct anv_fence_impl
*impl
,
1468 uint64_t abs_timeout
)
1470 return impl
->fence_wsi
->wait(impl
->fence_wsi
, abs_timeout
);
1474 anv_wait_for_fences(struct anv_device
*device
,
1475 uint32_t fenceCount
,
1476 const VkFence
*pFences
,
1478 uint64_t abs_timeout
)
1480 VkResult result
= VK_SUCCESS
;
1482 if (fenceCount
<= 1 || waitAll
) {
1483 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1484 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1485 struct anv_fence_impl
*impl
=
1486 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1487 &fence
->temporary
: &fence
->permanent
;
1489 switch (impl
->type
) {
1490 case ANV_FENCE_TYPE_BO
:
1491 case ANV_FENCE_TYPE_WSI_BO
:
1492 result
= anv_wait_for_bo_fences(device
, 1, &pFences
[i
],
1495 case ANV_FENCE_TYPE_SYNCOBJ
:
1496 result
= anv_wait_for_syncobj_fences(device
, 1, &pFences
[i
],
1499 case ANV_FENCE_TYPE_WSI
:
1500 result
= anv_wait_for_wsi_fence(device
, impl
, abs_timeout
);
1502 case ANV_FENCE_TYPE_NONE
:
1503 result
= VK_SUCCESS
;
1506 if (result
!= VK_SUCCESS
)
1511 for (uint32_t i
= 0; i
< fenceCount
; i
++) {
1512 if (anv_wait_for_fences(device
, 1, &pFences
[i
], true, 0) == VK_SUCCESS
)
1515 } while (anv_gettime_ns() < abs_timeout
);
1516 result
= VK_TIMEOUT
;
1521 static bool anv_all_fences_syncobj(uint32_t fenceCount
, const VkFence
*pFences
)
1523 for (uint32_t i
= 0; i
< fenceCount
; ++i
) {
1524 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1525 struct anv_fence_impl
*impl
=
1526 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1527 &fence
->temporary
: &fence
->permanent
;
1528 if (impl
->type
!= ANV_FENCE_TYPE_SYNCOBJ
)
1534 static bool anv_all_fences_bo(uint32_t fenceCount
, const VkFence
*pFences
)
1536 for (uint32_t i
= 0; i
< fenceCount
; ++i
) {
1537 ANV_FROM_HANDLE(anv_fence
, fence
, pFences
[i
]);
1538 struct anv_fence_impl
*impl
=
1539 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1540 &fence
->temporary
: &fence
->permanent
;
1541 if (impl
->type
!= ANV_FENCE_TYPE_BO
&&
1542 impl
->type
!= ANV_FENCE_TYPE_WSI_BO
)
1548 VkResult
anv_WaitForFences(
1550 uint32_t fenceCount
,
1551 const VkFence
* pFences
,
1555 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1560 if (anv_device_is_lost(device
))
1561 return VK_ERROR_DEVICE_LOST
;
1563 uint64_t abs_timeout
= anv_get_absolute_timeout(timeout
);
1564 if (anv_all_fences_syncobj(fenceCount
, pFences
)) {
1565 return anv_wait_for_syncobj_fences(device
, fenceCount
, pFences
,
1566 waitAll
, abs_timeout
);
1567 } else if (anv_all_fences_bo(fenceCount
, pFences
)) {
1568 return anv_wait_for_bo_fences(device
, fenceCount
, pFences
,
1569 waitAll
, abs_timeout
);
1571 return anv_wait_for_fences(device
, fenceCount
, pFences
,
1572 waitAll
, abs_timeout
);
1576 void anv_GetPhysicalDeviceExternalFenceProperties(
1577 VkPhysicalDevice physicalDevice
,
1578 const VkPhysicalDeviceExternalFenceInfo
* pExternalFenceInfo
,
1579 VkExternalFenceProperties
* pExternalFenceProperties
)
1581 ANV_FROM_HANDLE(anv_physical_device
, device
, physicalDevice
);
1583 switch (pExternalFenceInfo
->handleType
) {
1584 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1585 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
:
1586 if (device
->has_syncobj_wait
) {
1587 pExternalFenceProperties
->exportFromImportedHandleTypes
=
1588 VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
|
1589 VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
;
1590 pExternalFenceProperties
->compatibleHandleTypes
=
1591 VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
|
1592 VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
;
1593 pExternalFenceProperties
->externalFenceFeatures
=
1594 VK_EXTERNAL_FENCE_FEATURE_EXPORTABLE_BIT
|
1595 VK_EXTERNAL_FENCE_FEATURE_IMPORTABLE_BIT
;
1604 pExternalFenceProperties
->exportFromImportedHandleTypes
= 0;
1605 pExternalFenceProperties
->compatibleHandleTypes
= 0;
1606 pExternalFenceProperties
->externalFenceFeatures
= 0;
1609 VkResult
anv_ImportFenceFdKHR(
1611 const VkImportFenceFdInfoKHR
* pImportFenceFdInfo
)
1613 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1614 ANV_FROM_HANDLE(anv_fence
, fence
, pImportFenceFdInfo
->fence
);
1615 int fd
= pImportFenceFdInfo
->fd
;
1617 assert(pImportFenceFdInfo
->sType
==
1618 VK_STRUCTURE_TYPE_IMPORT_FENCE_FD_INFO_KHR
);
1620 struct anv_fence_impl new_impl
= {
1621 .type
= ANV_FENCE_TYPE_NONE
,
1624 switch (pImportFenceFdInfo
->handleType
) {
1625 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1626 new_impl
.type
= ANV_FENCE_TYPE_SYNCOBJ
;
1628 new_impl
.syncobj
= anv_gem_syncobj_fd_to_handle(device
, fd
);
1629 if (!new_impl
.syncobj
)
1630 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1634 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
:
1635 /* Sync files are a bit tricky. Because we want to continue using the
1636 * syncobj implementation of WaitForFences, we don't use the sync file
1637 * directly but instead import it into a syncobj.
1639 new_impl
.type
= ANV_FENCE_TYPE_SYNCOBJ
;
1641 new_impl
.syncobj
= anv_gem_syncobj_create(device
, 0);
1642 if (!new_impl
.syncobj
)
1643 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1645 if (anv_gem_syncobj_import_sync_file(device
, new_impl
.syncobj
, fd
)) {
1646 anv_gem_syncobj_destroy(device
, new_impl
.syncobj
);
1647 return vk_errorf(device
, NULL
, VK_ERROR_INVALID_EXTERNAL_HANDLE
,
1648 "syncobj sync file import failed: %m");
1653 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1656 /* From the Vulkan 1.0.53 spec:
1658 * "Importing a fence payload from a file descriptor transfers
1659 * ownership of the file descriptor from the application to the
1660 * Vulkan implementation. The application must not perform any
1661 * operations on the file descriptor after a successful import."
1663 * If the import fails, we leave the file descriptor open.
1667 if (pImportFenceFdInfo
->flags
& VK_FENCE_IMPORT_TEMPORARY_BIT
) {
1668 anv_fence_impl_cleanup(device
, &fence
->temporary
);
1669 fence
->temporary
= new_impl
;
1671 anv_fence_impl_cleanup(device
, &fence
->permanent
);
1672 fence
->permanent
= new_impl
;
1678 VkResult
anv_GetFenceFdKHR(
1680 const VkFenceGetFdInfoKHR
* pGetFdInfo
,
1683 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1684 ANV_FROM_HANDLE(anv_fence
, fence
, pGetFdInfo
->fence
);
1686 assert(pGetFdInfo
->sType
== VK_STRUCTURE_TYPE_FENCE_GET_FD_INFO_KHR
);
1688 struct anv_fence_impl
*impl
=
1689 fence
->temporary
.type
!= ANV_FENCE_TYPE_NONE
?
1690 &fence
->temporary
: &fence
->permanent
;
1692 assert(impl
->type
== ANV_FENCE_TYPE_SYNCOBJ
);
1693 switch (pGetFdInfo
->handleType
) {
1694 case VK_EXTERNAL_FENCE_HANDLE_TYPE_OPAQUE_FD_BIT
: {
1695 int fd
= anv_gem_syncobj_handle_to_fd(device
, impl
->syncobj
);
1697 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
1703 case VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT
: {
1704 int fd
= anv_gem_syncobj_export_sync_file(device
, impl
->syncobj
);
1706 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
1713 unreachable("Invalid fence export handle type");
1716 /* From the Vulkan 1.0.53 spec:
1718 * "Export operations have the same transference as the specified handle
1719 * type’s import operations. [...] If the fence was using a
1720 * temporarily imported payload, the fence’s prior permanent payload
1723 if (impl
== &fence
->temporary
)
1724 anv_fence_impl_cleanup(device
, impl
);
1729 // Queue semaphore functions
1731 static VkSemaphoreTypeKHR
1732 get_semaphore_type(const void *pNext
, uint64_t *initial_value
)
1734 const VkSemaphoreTypeCreateInfoKHR
*type_info
=
1735 vk_find_struct_const(pNext
, SEMAPHORE_TYPE_CREATE_INFO_KHR
);
1738 return VK_SEMAPHORE_TYPE_BINARY_KHR
;
1741 *initial_value
= type_info
->initialValue
;
1742 return type_info
->semaphoreType
;
1746 binary_semaphore_create(struct anv_device
*device
,
1747 struct anv_semaphore_impl
*impl
,
1750 if (device
->physical
->has_syncobj
) {
1751 impl
->type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
;
1752 impl
->syncobj
= anv_gem_syncobj_create(device
, 0);
1754 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1757 impl
->type
= ANV_SEMAPHORE_TYPE_BO
;
1759 anv_device_alloc_bo(device
, 4096,
1760 ANV_BO_ALLOC_EXTERNAL
|
1761 ANV_BO_ALLOC_IMPLICIT_SYNC
,
1762 0 /* explicit_address */,
1764 /* If we're going to use this as a fence, we need to *not* have the
1765 * EXEC_OBJECT_ASYNC bit set.
1767 assert(!(impl
->bo
->flags
& EXEC_OBJECT_ASYNC
));
1773 timeline_semaphore_create(struct anv_device
*device
,
1774 struct anv_semaphore_impl
*impl
,
1775 uint64_t initial_value
)
1777 impl
->type
= ANV_SEMAPHORE_TYPE_TIMELINE
;
1778 anv_timeline_init(device
, &impl
->timeline
, initial_value
);
1782 VkResult
anv_CreateSemaphore(
1784 const VkSemaphoreCreateInfo
* pCreateInfo
,
1785 const VkAllocationCallbacks
* pAllocator
,
1786 VkSemaphore
* pSemaphore
)
1788 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1789 struct anv_semaphore
*semaphore
;
1791 assert(pCreateInfo
->sType
== VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO
);
1793 uint64_t timeline_value
= 0;
1794 VkSemaphoreTypeKHR sem_type
= get_semaphore_type(pCreateInfo
->pNext
, &timeline_value
);
1796 semaphore
= vk_alloc(&device
->vk
.alloc
, sizeof(*semaphore
), 8,
1797 VK_SYSTEM_ALLOCATION_SCOPE_DEVICE
);
1798 if (semaphore
== NULL
)
1799 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1801 vk_object_base_init(&device
->vk
, &semaphore
->base
, VK_OBJECT_TYPE_SEMAPHORE
);
1803 p_atomic_set(&semaphore
->refcount
, 1);
1805 const VkExportSemaphoreCreateInfo
*export
=
1806 vk_find_struct_const(pCreateInfo
->pNext
, EXPORT_SEMAPHORE_CREATE_INFO
);
1807 VkExternalSemaphoreHandleTypeFlags handleTypes
=
1808 export
? export
->handleTypes
: 0;
1811 if (handleTypes
== 0) {
1812 if (sem_type
== VK_SEMAPHORE_TYPE_BINARY_KHR
)
1813 result
= binary_semaphore_create(device
, &semaphore
->permanent
, false);
1815 result
= timeline_semaphore_create(device
, &semaphore
->permanent
, timeline_value
);
1816 if (result
!= VK_SUCCESS
) {
1817 vk_free2(&device
->vk
.alloc
, pAllocator
, semaphore
);
1820 } else if (handleTypes
& VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
) {
1821 assert(handleTypes
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
);
1822 assert(sem_type
== VK_SEMAPHORE_TYPE_BINARY_KHR
);
1823 result
= binary_semaphore_create(device
, &semaphore
->permanent
, true);
1824 if (result
!= VK_SUCCESS
) {
1825 vk_free2(&device
->vk
.alloc
, pAllocator
, semaphore
);
1828 } else if (handleTypes
& VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
) {
1829 assert(handleTypes
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
);
1830 assert(sem_type
== VK_SEMAPHORE_TYPE_BINARY_KHR
);
1831 if (device
->physical
->has_syncobj
) {
1832 semaphore
->permanent
.type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
;
1833 semaphore
->permanent
.syncobj
= anv_gem_syncobj_create(device
, 0);
1834 if (!semaphore
->permanent
.syncobj
) {
1835 vk_free2(&device
->vk
.alloc
, pAllocator
, semaphore
);
1836 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
1839 semaphore
->permanent
.type
= ANV_SEMAPHORE_TYPE_SYNC_FILE
;
1840 semaphore
->permanent
.fd
= -1;
1843 assert(!"Unknown handle type");
1844 vk_free2(&device
->vk
.alloc
, pAllocator
, semaphore
);
1845 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
1848 semaphore
->temporary
.type
= ANV_SEMAPHORE_TYPE_NONE
;
1850 *pSemaphore
= anv_semaphore_to_handle(semaphore
);
1856 anv_semaphore_impl_cleanup(struct anv_device
*device
,
1857 struct anv_semaphore_impl
*impl
)
1859 switch (impl
->type
) {
1860 case ANV_SEMAPHORE_TYPE_NONE
:
1861 case ANV_SEMAPHORE_TYPE_DUMMY
:
1862 /* Dummy. Nothing to do */
1865 case ANV_SEMAPHORE_TYPE_BO
:
1866 case ANV_SEMAPHORE_TYPE_WSI_BO
:
1867 anv_device_release_bo(device
, impl
->bo
);
1870 case ANV_SEMAPHORE_TYPE_SYNC_FILE
:
1875 case ANV_SEMAPHORE_TYPE_TIMELINE
:
1876 anv_timeline_finish(device
, &impl
->timeline
);
1879 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
:
1880 anv_gem_syncobj_destroy(device
, impl
->syncobj
);
1884 unreachable("Invalid semaphore type");
1887 impl
->type
= ANV_SEMAPHORE_TYPE_NONE
;
1891 anv_semaphore_reset_temporary(struct anv_device
*device
,
1892 struct anv_semaphore
*semaphore
)
1894 if (semaphore
->temporary
.type
== ANV_SEMAPHORE_TYPE_NONE
)
1897 anv_semaphore_impl_cleanup(device
, &semaphore
->temporary
);
1900 static struct anv_semaphore
*
1901 anv_semaphore_ref(struct anv_semaphore
*semaphore
)
1903 assert(semaphore
->refcount
);
1904 p_atomic_inc(&semaphore
->refcount
);
1909 anv_semaphore_unref(struct anv_device
*device
, struct anv_semaphore
*semaphore
)
1911 if (!p_atomic_dec_zero(&semaphore
->refcount
))
1914 anv_semaphore_impl_cleanup(device
, &semaphore
->temporary
);
1915 anv_semaphore_impl_cleanup(device
, &semaphore
->permanent
);
1917 vk_object_base_finish(&semaphore
->base
);
1918 vk_free(&device
->vk
.alloc
, semaphore
);
1921 void anv_DestroySemaphore(
1923 VkSemaphore _semaphore
,
1924 const VkAllocationCallbacks
* pAllocator
)
1926 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1927 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, _semaphore
);
1929 if (semaphore
== NULL
)
1932 anv_semaphore_unref(device
, semaphore
);
1935 void anv_GetPhysicalDeviceExternalSemaphoreProperties(
1936 VkPhysicalDevice physicalDevice
,
1937 const VkPhysicalDeviceExternalSemaphoreInfo
* pExternalSemaphoreInfo
,
1938 VkExternalSemaphoreProperties
* pExternalSemaphoreProperties
)
1940 ANV_FROM_HANDLE(anv_physical_device
, device
, physicalDevice
);
1942 VkSemaphoreTypeKHR sem_type
=
1943 get_semaphore_type(pExternalSemaphoreInfo
->pNext
, NULL
);
1945 switch (pExternalSemaphoreInfo
->handleType
) {
1946 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1947 /* Timeline semaphores are not exportable. */
1948 if (sem_type
== VK_SEMAPHORE_TYPE_TIMELINE_KHR
)
1950 pExternalSemaphoreProperties
->exportFromImportedHandleTypes
=
1951 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
;
1952 pExternalSemaphoreProperties
->compatibleHandleTypes
=
1953 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
;
1954 pExternalSemaphoreProperties
->externalSemaphoreFeatures
=
1955 VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT
|
1956 VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT
;
1959 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
:
1960 if (sem_type
== VK_SEMAPHORE_TYPE_TIMELINE_KHR
)
1962 if (!device
->has_exec_fence
)
1964 pExternalSemaphoreProperties
->exportFromImportedHandleTypes
=
1965 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
;
1966 pExternalSemaphoreProperties
->compatibleHandleTypes
=
1967 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
;
1968 pExternalSemaphoreProperties
->externalSemaphoreFeatures
=
1969 VK_EXTERNAL_SEMAPHORE_FEATURE_EXPORTABLE_BIT
|
1970 VK_EXTERNAL_SEMAPHORE_FEATURE_IMPORTABLE_BIT
;
1977 pExternalSemaphoreProperties
->exportFromImportedHandleTypes
= 0;
1978 pExternalSemaphoreProperties
->compatibleHandleTypes
= 0;
1979 pExternalSemaphoreProperties
->externalSemaphoreFeatures
= 0;
1982 VkResult
anv_ImportSemaphoreFdKHR(
1984 const VkImportSemaphoreFdInfoKHR
* pImportSemaphoreFdInfo
)
1986 ANV_FROM_HANDLE(anv_device
, device
, _device
);
1987 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pImportSemaphoreFdInfo
->semaphore
);
1988 int fd
= pImportSemaphoreFdInfo
->fd
;
1990 struct anv_semaphore_impl new_impl
= {
1991 .type
= ANV_SEMAPHORE_TYPE_NONE
,
1994 switch (pImportSemaphoreFdInfo
->handleType
) {
1995 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
:
1996 if (device
->physical
->has_syncobj
) {
1997 new_impl
.type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
;
1999 new_impl
.syncobj
= anv_gem_syncobj_fd_to_handle(device
, fd
);
2000 if (!new_impl
.syncobj
)
2001 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
2003 new_impl
.type
= ANV_SEMAPHORE_TYPE_BO
;
2005 VkResult result
= anv_device_import_bo(device
, fd
,
2006 ANV_BO_ALLOC_EXTERNAL
|
2007 ANV_BO_ALLOC_IMPLICIT_SYNC
,
2008 0 /* client_address */,
2010 if (result
!= VK_SUCCESS
)
2013 if (new_impl
.bo
->size
< 4096) {
2014 anv_device_release_bo(device
, new_impl
.bo
);
2015 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
2018 /* If we're going to use this as a fence, we need to *not* have the
2019 * EXEC_OBJECT_ASYNC bit set.
2021 assert(!(new_impl
.bo
->flags
& EXEC_OBJECT_ASYNC
));
2024 /* From the Vulkan spec:
2026 * "Importing semaphore state from a file descriptor transfers
2027 * ownership of the file descriptor from the application to the
2028 * Vulkan implementation. The application must not perform any
2029 * operations on the file descriptor after a successful import."
2031 * If the import fails, we leave the file descriptor open.
2036 case VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
:
2037 if (device
->physical
->has_syncobj
) {
2038 new_impl
= (struct anv_semaphore_impl
) {
2039 .type
= ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
,
2040 .syncobj
= anv_gem_syncobj_create(device
, 0),
2042 if (!new_impl
.syncobj
)
2043 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
2044 if (anv_gem_syncobj_import_sync_file(device
, new_impl
.syncobj
, fd
)) {
2045 anv_gem_syncobj_destroy(device
, new_impl
.syncobj
);
2046 return vk_errorf(device
, NULL
, VK_ERROR_INVALID_EXTERNAL_HANDLE
,
2047 "syncobj sync file import failed: %m");
2049 /* Ownership of the FD is transfered to Anv. Since we don't need it
2050 * anymore because the associated fence has been put into a syncobj,
2051 * we must close the FD.
2055 new_impl
= (struct anv_semaphore_impl
) {
2056 .type
= ANV_SEMAPHORE_TYPE_SYNC_FILE
,
2063 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
2066 if (pImportSemaphoreFdInfo
->flags
& VK_SEMAPHORE_IMPORT_TEMPORARY_BIT
) {
2067 anv_semaphore_impl_cleanup(device
, &semaphore
->temporary
);
2068 semaphore
->temporary
= new_impl
;
2070 anv_semaphore_impl_cleanup(device
, &semaphore
->permanent
);
2071 semaphore
->permanent
= new_impl
;
2077 VkResult
anv_GetSemaphoreFdKHR(
2079 const VkSemaphoreGetFdInfoKHR
* pGetFdInfo
,
2082 ANV_FROM_HANDLE(anv_device
, device
, _device
);
2083 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pGetFdInfo
->semaphore
);
2087 assert(pGetFdInfo
->sType
== VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR
);
2089 struct anv_semaphore_impl
*impl
=
2090 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
2091 &semaphore
->temporary
: &semaphore
->permanent
;
2093 switch (impl
->type
) {
2094 case ANV_SEMAPHORE_TYPE_BO
:
2095 result
= anv_device_export_bo(device
, impl
->bo
, pFd
);
2096 if (result
!= VK_SUCCESS
)
2100 case ANV_SEMAPHORE_TYPE_SYNC_FILE
: {
2101 /* There's a potential race here with vkQueueSubmit if you are trying
2102 * to export a semaphore Fd while the queue submit is still happening.
2103 * This can happen if we see all dependencies get resolved via timeline
2104 * semaphore waits completing before the execbuf completes and we
2105 * process the resulting out fence. To work around this, take a lock
2106 * around grabbing the fd.
2108 pthread_mutex_lock(&device
->mutex
);
2110 /* From the Vulkan 1.0.53 spec:
2112 * "...exporting a semaphore payload to a handle with copy
2113 * transference has the same side effects on the source
2114 * semaphore’s payload as executing a semaphore wait operation."
2116 * In other words, it may still be a SYNC_FD semaphore, but it's now
2117 * considered to have been waited on and no longer has a sync file
2123 pthread_mutex_unlock(&device
->mutex
);
2125 /* There are two reasons why this could happen:
2127 * 1) The user is trying to export without submitting something that
2128 * signals the semaphore. If this is the case, it's their bug so
2129 * what we return here doesn't matter.
2131 * 2) The kernel didn't give us a file descriptor. The most likely
2132 * reason for this is running out of file descriptors.
2135 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
2141 case ANV_SEMAPHORE_TYPE_DRM_SYNCOBJ
:
2142 if (pGetFdInfo
->handleType
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT
)
2143 fd
= anv_gem_syncobj_export_sync_file(device
, impl
->syncobj
);
2145 assert(pGetFdInfo
->handleType
== VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT
);
2146 fd
= anv_gem_syncobj_handle_to_fd(device
, impl
->syncobj
);
2149 return vk_error(VK_ERROR_TOO_MANY_OBJECTS
);
2154 return vk_error(VK_ERROR_INVALID_EXTERNAL_HANDLE
);
2157 /* From the Vulkan 1.0.53 spec:
2159 * "Export operations have the same transference as the specified handle
2160 * type’s import operations. [...] If the semaphore was using a
2161 * temporarily imported payload, the semaphore’s prior permanent payload
2164 if (impl
== &semaphore
->temporary
)
2165 anv_semaphore_impl_cleanup(device
, impl
);
2170 VkResult
anv_GetSemaphoreCounterValue(
2172 VkSemaphore _semaphore
,
2175 ANV_FROM_HANDLE(anv_device
, device
, _device
);
2176 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, _semaphore
);
2178 struct anv_semaphore_impl
*impl
=
2179 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
2180 &semaphore
->temporary
: &semaphore
->permanent
;
2182 switch (impl
->type
) {
2183 case ANV_SEMAPHORE_TYPE_TIMELINE
: {
2184 pthread_mutex_lock(&device
->mutex
);
2185 *pValue
= impl
->timeline
.highest_past
;
2186 pthread_mutex_unlock(&device
->mutex
);
2191 unreachable("Invalid semaphore type");
2196 anv_timeline_wait_locked(struct anv_device
*device
,
2197 struct anv_timeline
*timeline
,
2198 uint64_t serial
, uint64_t abs_timeout_ns
)
2200 /* Wait on the queue_submit condition variable until the timeline has a
2201 * time point pending that's at least as high as serial.
2203 while (timeline
->highest_pending
< serial
) {
2204 struct timespec abstime
= {
2205 .tv_sec
= abs_timeout_ns
/ NSEC_PER_SEC
,
2206 .tv_nsec
= abs_timeout_ns
% NSEC_PER_SEC
,
2209 int ret
= pthread_cond_timedwait(&device
->queue_submit
,
2210 &device
->mutex
, &abstime
);
2211 assert(ret
!= EINVAL
);
2212 if (anv_gettime_ns() >= abs_timeout_ns
&&
2213 timeline
->highest_pending
< serial
)
2218 VkResult result
= anv_timeline_gc_locked(device
, timeline
);
2219 if (result
!= VK_SUCCESS
)
2222 if (timeline
->highest_past
>= serial
)
2225 /* If we got here, our earliest time point has a busy BO */
2226 struct anv_timeline_point
*point
=
2227 list_first_entry(&timeline
->points
,
2228 struct anv_timeline_point
, link
);
2230 /* Drop the lock while we wait. */
2232 pthread_mutex_unlock(&device
->mutex
);
2234 result
= anv_device_wait(device
, point
->bo
,
2235 anv_get_relative_timeout(abs_timeout_ns
));
2237 /* Pick the mutex back up */
2238 pthread_mutex_lock(&device
->mutex
);
2241 /* This covers both VK_TIMEOUT and VK_ERROR_DEVICE_LOST */
2242 if (result
!= VK_SUCCESS
)
2248 anv_timelines_wait(struct anv_device
*device
,
2249 struct anv_timeline
**timelines
,
2250 const uint64_t *serials
,
2251 uint32_t n_timelines
,
2253 uint64_t abs_timeout_ns
)
2255 if (!wait_all
&& n_timelines
> 1) {
2256 pthread_mutex_lock(&device
->mutex
);
2260 for (uint32_t i
= 0; i
< n_timelines
; i
++) {
2262 anv_timeline_wait_locked(device
, timelines
[i
], serials
[i
], 0);
2263 if (result
!= VK_TIMEOUT
)
2267 if (result
!= VK_TIMEOUT
||
2268 anv_gettime_ns() >= abs_timeout_ns
) {
2269 pthread_mutex_unlock(&device
->mutex
);
2273 /* If none of them are ready do a short wait so we don't completely
2274 * spin while holding the lock. The 10us is completely arbitrary.
2276 uint64_t abs_short_wait_ns
=
2277 anv_get_absolute_timeout(
2278 MIN2((anv_gettime_ns() - abs_timeout_ns
) / 10, 10 * 1000));
2279 struct timespec abstime
= {
2280 .tv_sec
= abs_short_wait_ns
/ NSEC_PER_SEC
,
2281 .tv_nsec
= abs_short_wait_ns
% NSEC_PER_SEC
,
2284 ret
= pthread_cond_timedwait(&device
->queue_submit
,
2285 &device
->mutex
, &abstime
);
2286 assert(ret
!= EINVAL
);
2289 VkResult result
= VK_SUCCESS
;
2290 pthread_mutex_lock(&device
->mutex
);
2291 for (uint32_t i
= 0; i
< n_timelines
; i
++) {
2293 anv_timeline_wait_locked(device
, timelines
[i
],
2294 serials
[i
], abs_timeout_ns
);
2295 if (result
!= VK_SUCCESS
)
2298 pthread_mutex_unlock(&device
->mutex
);
2303 VkResult
anv_WaitSemaphores(
2305 const VkSemaphoreWaitInfoKHR
* pWaitInfo
,
2308 ANV_FROM_HANDLE(anv_device
, device
, _device
);
2313 struct anv_timeline
**timelines
=
2314 vk_alloc(&device
->vk
.alloc
,
2315 pWaitInfo
->semaphoreCount
* sizeof(*timelines
),
2316 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND
);
2318 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
2320 uint64_t *values
= vk_alloc(&device
->vk
.alloc
,
2321 pWaitInfo
->semaphoreCount
* sizeof(*values
),
2322 8, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND
);
2324 vk_free(&device
->vk
.alloc
, timelines
);
2325 return vk_error(VK_ERROR_OUT_OF_HOST_MEMORY
);
2328 uint32_t handle_count
= 0;
2329 for (uint32_t i
= 0; i
< pWaitInfo
->semaphoreCount
; i
++) {
2330 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pWaitInfo
->pSemaphores
[i
]);
2331 struct anv_semaphore_impl
*impl
=
2332 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
2333 &semaphore
->temporary
: &semaphore
->permanent
;
2335 assert(impl
->type
== ANV_SEMAPHORE_TYPE_TIMELINE
);
2337 if (pWaitInfo
->pValues
[i
] == 0)
2340 timelines
[handle_count
] = &impl
->timeline
;
2341 values
[handle_count
] = pWaitInfo
->pValues
[i
];
2345 VkResult result
= VK_SUCCESS
;
2346 if (handle_count
> 0) {
2347 result
= anv_timelines_wait(device
, timelines
, values
, handle_count
,
2348 !(pWaitInfo
->flags
& VK_SEMAPHORE_WAIT_ANY_BIT_KHR
),
2349 anv_get_absolute_timeout(timeout
));
2352 vk_free(&device
->vk
.alloc
, timelines
);
2353 vk_free(&device
->vk
.alloc
, values
);
2358 VkResult
anv_SignalSemaphore(
2360 const VkSemaphoreSignalInfoKHR
* pSignalInfo
)
2362 ANV_FROM_HANDLE(anv_device
, device
, _device
);
2363 ANV_FROM_HANDLE(anv_semaphore
, semaphore
, pSignalInfo
->semaphore
);
2365 struct anv_semaphore_impl
*impl
=
2366 semaphore
->temporary
.type
!= ANV_SEMAPHORE_TYPE_NONE
?
2367 &semaphore
->temporary
: &semaphore
->permanent
;
2369 switch (impl
->type
) {
2370 case ANV_SEMAPHORE_TYPE_TIMELINE
: {
2371 pthread_mutex_lock(&device
->mutex
);
2373 VkResult result
= anv_timeline_gc_locked(device
, &impl
->timeline
);
2375 assert(pSignalInfo
->value
> impl
->timeline
.highest_pending
);
2377 impl
->timeline
.highest_pending
= impl
->timeline
.highest_past
= pSignalInfo
->value
;
2379 if (result
== VK_SUCCESS
)
2380 result
= anv_device_submit_deferred_locked(device
);
2382 pthread_cond_broadcast(&device
->queue_submit
);
2383 pthread_mutex_unlock(&device
->mutex
);
2388 unreachable("Invalid semaphore type");