+uint64_t anv_gettime_ns(void)
+{
+ struct timespec current;
+ clock_gettime(CLOCK_MONOTONIC, ¤t);
+ return (uint64_t)current.tv_sec * NSEC_PER_SEC + current.tv_nsec;
+}
+
+uint64_t anv_get_absolute_timeout(uint64_t timeout)
+{
+ if (timeout == 0)
+ return 0;
+ uint64_t current_time = anv_gettime_ns();
+ uint64_t max_timeout = (uint64_t) INT64_MAX - current_time;
+
+ timeout = MIN2(max_timeout, timeout);
+
+ return (current_time + timeout);
+}
+
+static int64_t anv_get_relative_timeout(uint64_t abs_timeout)
+{
+ uint64_t now = anv_gettime_ns();
+
+ /* We don't want negative timeouts.
+ *
+ * DRM_IOCTL_I915_GEM_WAIT uses a signed 64 bit timeout and is
+ * supposed to block indefinitely timeouts < 0. Unfortunately,
+ * this was broken for a couple of kernel releases. Since there's
+ * no way to know whether or not the kernel we're using is one of
+ * the broken ones, the best we can do is to clamp the timeout to
+ * INT64_MAX. This limits the maximum timeout from 584 years to
+ * 292 years - likely not a big deal.
+ */
+ if (abs_timeout < now)
+ return 0;
+
+ uint64_t rel_timeout = abs_timeout - now;
+ if (rel_timeout > (uint64_t) INT64_MAX)
+ rel_timeout = INT64_MAX;
+
+ return rel_timeout;
+}
+
+static struct anv_semaphore *anv_semaphore_ref(struct anv_semaphore *semaphore);
+static void anv_semaphore_unref(struct anv_device *device, struct anv_semaphore *semaphore);
+static void anv_semaphore_impl_cleanup(struct anv_device *device,
+ struct anv_semaphore_impl *impl);
+
+static void
+anv_queue_submit_free(struct anv_device *device,
+ struct anv_queue_submit *submit)
+{
+ const VkAllocationCallbacks *alloc = submit->alloc;
+
+ for (uint32_t i = 0; i < submit->temporary_semaphore_count; i++)
+ anv_semaphore_impl_cleanup(device, &submit->temporary_semaphores[i]);
+ for (uint32_t i = 0; i < submit->sync_fd_semaphore_count; i++)
+ anv_semaphore_unref(device, submit->sync_fd_semaphores[i]);
+ /* Execbuf does not consume the in_fence. It's our job to close it. */
+ if (submit->in_fence != -1)
+ close(submit->in_fence);
+ if (submit->out_fence != -1)
+ close(submit->out_fence);
+ vk_free(alloc, submit->fences);
+ vk_free(alloc, submit->temporary_semaphores);
+ vk_free(alloc, submit->wait_timelines);
+ vk_free(alloc, submit->wait_timeline_values);
+ vk_free(alloc, submit->signal_timelines);
+ vk_free(alloc, submit->signal_timeline_values);
+ vk_free(alloc, submit->fence_bos);
+ vk_free(alloc, submit);
+}
+
+static bool
+anv_queue_submit_ready_locked(struct anv_queue_submit *submit)
+{
+ for (uint32_t i = 0; i < submit->wait_timeline_count; i++) {
+ if (submit->wait_timeline_values[i] > submit->wait_timelines[i]->highest_pending)
+ return false;
+ }
+
+ return true;
+}
+
+static VkResult
+anv_timeline_init(struct anv_device *device,
+ struct anv_timeline *timeline,
+ uint64_t initial_value)
+{
+ timeline->highest_past =
+ timeline->highest_pending = initial_value;
+ list_inithead(&timeline->points);
+ list_inithead(&timeline->free_points);
+
+ return VK_SUCCESS;
+}
+
+static void
+anv_timeline_finish(struct anv_device *device,
+ struct anv_timeline *timeline)
+{
+ list_for_each_entry_safe(struct anv_timeline_point, point,
+ &timeline->free_points, link) {
+ list_del(&point->link);
+ anv_device_release_bo(device, point->bo);
+ vk_free(&device->vk.alloc, point);
+ }
+ list_for_each_entry_safe(struct anv_timeline_point, point,
+ &timeline->points, link) {
+ list_del(&point->link);
+ anv_device_release_bo(device, point->bo);
+ vk_free(&device->vk.alloc, point);
+ }
+}
+
+static VkResult
+anv_timeline_add_point_locked(struct anv_device *device,
+ struct anv_timeline *timeline,
+ uint64_t value,
+ struct anv_timeline_point **point)
+{
+ VkResult result = VK_SUCCESS;
+
+ if (list_is_empty(&timeline->free_points)) {
+ *point =
+ vk_zalloc(&device->vk.alloc, sizeof(**point),
+ 8, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
+ if (!(*point))
+ result = vk_error(VK_ERROR_OUT_OF_HOST_MEMORY);
+ if (result == VK_SUCCESS) {
+ result = anv_device_alloc_bo(device, 4096,
+ ANV_BO_ALLOC_EXTERNAL |
+ ANV_BO_ALLOC_IMPLICIT_SYNC,
+ 0 /* explicit_address */,
+ &(*point)->bo);
+ if (result != VK_SUCCESS)
+ vk_free(&device->vk.alloc, *point);
+ }
+ } else {
+ *point = list_first_entry(&timeline->free_points,
+ struct anv_timeline_point, link);
+ list_del(&(*point)->link);
+ }
+
+ if (result == VK_SUCCESS) {
+ (*point)->serial = value;
+ list_addtail(&(*point)->link, &timeline->points);
+ }
+
+ return result;
+}
+
+static VkResult
+anv_timeline_gc_locked(struct anv_device *device,
+ struct anv_timeline *timeline)
+{
+ list_for_each_entry_safe(struct anv_timeline_point, point,
+ &timeline->points, link) {
+ /* timeline->higest_pending is only incremented once submission has
+ * happened. If this point has a greater serial, it means the point
+ * hasn't been submitted yet.
+ */
+ if (point->serial > timeline->highest_pending)
+ return VK_SUCCESS;
+
+ /* If someone is waiting on this time point, consider it busy and don't
+ * try to recycle it. There's a slim possibility that it's no longer
+ * busy by the time we look at it but we would be recycling it out from
+ * under a waiter and that can lead to weird races.
+ *
+ * We walk the list in-order so if this time point is still busy so is
+ * every following time point
+ */
+ assert(point->waiting >= 0);
+ if (point->waiting)
+ return VK_SUCCESS;
+
+ /* Garbage collect any signaled point. */
+ VkResult result = anv_device_bo_busy(device, point->bo);
+ if (result == VK_NOT_READY) {
+ /* We walk the list in-order so if this time point is still busy so
+ * is every following time point
+ */
+ return VK_SUCCESS;
+ } else if (result != VK_SUCCESS) {
+ return result;
+ }
+
+ assert(timeline->highest_past < point->serial);
+ timeline->highest_past = point->serial;
+
+ list_del(&point->link);
+ list_add(&point->link, &timeline->free_points);
+ }
+
+ return VK_SUCCESS;
+}
+
+static VkResult anv_queue_submit_add_fence_bo(struct anv_queue_submit *submit,
+ struct anv_bo *bo,
+ bool signal);
+
+static VkResult
+anv_queue_submit_timeline_locked(struct anv_queue *queue,
+ struct anv_queue_submit *submit)
+{
+ VkResult result;
+
+ for (uint32_t i = 0; i < submit->wait_timeline_count; i++) {
+ struct anv_timeline *timeline = submit->wait_timelines[i];
+ uint64_t wait_value = submit->wait_timeline_values[i];
+
+ if (timeline->highest_past >= wait_value)
+ continue;
+
+ list_for_each_entry(struct anv_timeline_point, point, &timeline->points, link) {
+ if (point->serial < wait_value)
+ continue;
+ result = anv_queue_submit_add_fence_bo(submit, point->bo, false);
+ if (result != VK_SUCCESS)
+ return result;
+ break;
+ }
+ }
+ for (uint32_t i = 0; i < submit->signal_timeline_count; i++) {
+ struct anv_timeline *timeline = submit->signal_timelines[i];
+ uint64_t signal_value = submit->signal_timeline_values[i];
+ struct anv_timeline_point *point;
+
+ result = anv_timeline_add_point_locked(queue->device, timeline,
+ signal_value, &point);
+ if (result != VK_SUCCESS)
+ return result;
+
+ result = anv_queue_submit_add_fence_bo(submit, point->bo, true);
+ if (result != VK_SUCCESS)
+ return result;
+ }
+
+ result = anv_queue_execbuf_locked(queue, submit);
+
+ if (result == VK_SUCCESS) {
+ /* Update the pending values in the timeline objects. */
+ for (uint32_t i = 0; i < submit->signal_timeline_count; i++) {
+ struct anv_timeline *timeline = submit->signal_timelines[i];
+ uint64_t signal_value = submit->signal_timeline_values[i];
+
+ assert(signal_value > timeline->highest_pending);
+ timeline->highest_pending = signal_value;
+ }
+
+ /* Update signaled semaphores backed by syncfd. */
+ for (uint32_t i = 0; i < submit->sync_fd_semaphore_count; i++) {
+ struct anv_semaphore *semaphore = submit->sync_fd_semaphores[i];
+ /* Out fences can't have temporary state because that would imply
+ * that we imported a sync file and are trying to signal it.
+ */
+ assert(semaphore->temporary.type == ANV_SEMAPHORE_TYPE_NONE);
+ struct anv_semaphore_impl *impl = &semaphore->permanent;
+
+ assert(impl->type == ANV_SEMAPHORE_TYPE_SYNC_FILE);
+ impl->fd = dup(submit->out_fence);
+ }
+ } else {
+ /* Unblock any waiter by signaling the points, the application will get
+ * a device lost error code.
+ */
+ for (uint32_t i = 0; i < submit->signal_timeline_count; i++) {
+ struct anv_timeline *timeline = submit->signal_timelines[i];
+ uint64_t signal_value = submit->signal_timeline_values[i];
+
+ assert(signal_value > timeline->highest_pending);
+ timeline->highest_past = timeline->highest_pending = signal_value;
+ }
+ }
+
+ return result;
+}
+
+static VkResult
+anv_queue_submit_deferred_locked(struct anv_queue *queue, uint32_t *advance)
+{
+ VkResult result = VK_SUCCESS;
+
+ /* Go through all the queued submissions and submit then until we find one
+ * that's waiting on a point that hasn't materialized yet.
+ */
+ list_for_each_entry_safe(struct anv_queue_submit, submit,
+ &queue->queued_submits, link) {
+ if (!anv_queue_submit_ready_locked(submit))
+ break;
+
+ (*advance)++;
+ list_del(&submit->link);
+
+ result = anv_queue_submit_timeline_locked(queue, submit);
+
+ anv_queue_submit_free(queue->device, submit);
+
+ if (result != VK_SUCCESS)
+ break;
+ }
+
+ return result;
+}
+
+static VkResult
+anv_device_submit_deferred_locked(struct anv_device *device)
+{
+ uint32_t advance = 0;
+ return anv_queue_submit_deferred_locked(&device->queue, &advance);
+}
+
+static VkResult
+_anv_queue_submit(struct anv_queue *queue, struct anv_queue_submit **_submit,
+ bool flush_queue)
+{
+ struct anv_queue_submit *submit = *_submit;
+
+ /* Wait before signal behavior means we might keep alive the
+ * anv_queue_submit object a bit longer, so transfer the ownership to the
+ * anv_queue.
+ */
+ *_submit = NULL;
+
+ pthread_mutex_lock(&queue->device->mutex);
+ list_addtail(&submit->link, &queue->queued_submits);
+ VkResult result = anv_device_submit_deferred_locked(queue->device);
+ if (flush_queue) {
+ while (result == VK_SUCCESS && !list_is_empty(&queue->queued_submits)) {
+ int ret = pthread_cond_wait(&queue->device->queue_submit,
+ &queue->device->mutex);
+ if (ret != 0) {
+ result = anv_device_set_lost(queue->device, "wait timeout");
+ break;
+ }
+
+ result = anv_device_submit_deferred_locked(queue->device);
+ }
+ }
+ pthread_mutex_unlock(&queue->device->mutex);
+ return result;
+}
+