22acc317c0c0b072ea7a22c73f6cac4eb2917978
[mesa.git] / src / gallium / drivers / iris / iris_fence.c
1 /*
2 * Copyright © 2018 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
21 */
22
23 /**
24 * @file iris_fence.c
25 *
26 * Fences for driver and IPC serialisation, scheduling and synchronisation.
27 */
28
29 #include "drm-uapi/sync_file.h"
30 #include "util/u_debug.h"
31 #include "util/u_inlines.h"
32 #include "intel/common/gen_gem.h"
33
34 #include "iris_batch.h"
35 #include "iris_bufmgr.h"
36 #include "iris_context.h"
37 #include "iris_fence.h"
38 #include "iris_screen.h"
39
40 static uint32_t
41 gem_syncobj_create(int fd, uint32_t flags)
42 {
43 struct drm_syncobj_create args = {
44 .flags = flags,
45 };
46
47 gen_ioctl(fd, DRM_IOCTL_SYNCOBJ_CREATE, &args);
48
49 return args.handle;
50 }
51
52 static void
53 gem_syncobj_destroy(int fd, uint32_t handle)
54 {
55 struct drm_syncobj_destroy args = {
56 .handle = handle,
57 };
58
59 gen_ioctl(fd, DRM_IOCTL_SYNCOBJ_DESTROY, &args);
60 }
61
62 /**
63 * Make a new sync-point.
64 */
65 struct iris_syncobj *
66 iris_create_syncobj(struct iris_screen *screen)
67 {
68 struct iris_syncobj *syncobj = malloc(sizeof(*syncobj));
69
70 if (!syncobj)
71 return NULL;
72
73 syncobj->handle = gem_syncobj_create(screen->fd, 0);
74 assert(syncobj->handle);
75
76 pipe_reference_init(&syncobj->ref, 1);
77
78 return syncobj;
79 }
80
81 void
82 iris_syncobj_destroy(struct iris_screen *screen, struct iris_syncobj *syncobj)
83 {
84 gem_syncobj_destroy(screen->fd, syncobj->handle);
85 free(syncobj);
86 }
87
88 /**
89 * Add a sync-point to the batch, with the given flags.
90 *
91 * \p flags One of I915_EXEC_FENCE_WAIT or I915_EXEC_FENCE_SIGNAL.
92 */
93 void
94 iris_batch_add_syncobj(struct iris_batch *batch,
95 struct iris_syncobj *syncobj,
96 unsigned flags)
97 {
98 struct drm_i915_gem_exec_fence *fence =
99 util_dynarray_grow(&batch->exec_fences, struct drm_i915_gem_exec_fence, 1);
100
101 *fence = (struct drm_i915_gem_exec_fence) {
102 .handle = syncobj->handle,
103 .flags = flags,
104 };
105
106 struct iris_syncobj **store =
107 util_dynarray_grow(&batch->syncobjs, struct iris_syncobj *, 1);
108
109 *store = NULL;
110 iris_syncobj_reference(batch->screen, store, syncobj);
111 }
112
113 /* ------------------------------------------------------------------- */
114
115 struct pipe_fence_handle {
116 struct pipe_reference ref;
117
118 struct pipe_context *unflushed_ctx;
119
120 struct iris_fine_fence *fine[IRIS_BATCH_COUNT];
121 };
122
123 static void
124 iris_fence_destroy(struct pipe_screen *p_screen,
125 struct pipe_fence_handle *fence)
126 {
127 struct iris_screen *screen = (struct iris_screen *)p_screen;
128
129 for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++)
130 iris_fine_fence_reference(screen, &fence->fine[i], NULL);
131
132 free(fence);
133 }
134
135 static void
136 iris_fence_reference(struct pipe_screen *p_screen,
137 struct pipe_fence_handle **dst,
138 struct pipe_fence_handle *src)
139 {
140 if (pipe_reference(*dst ? &(*dst)->ref : NULL,
141 src ? &src->ref : NULL))
142 iris_fence_destroy(p_screen, *dst);
143
144 *dst = src;
145 }
146
147 bool
148 iris_wait_syncobj(struct pipe_screen *p_screen,
149 struct iris_syncobj *syncobj,
150 int64_t timeout_nsec)
151 {
152 if (!syncobj)
153 return false;
154
155 struct iris_screen *screen = (struct iris_screen *)p_screen;
156 struct drm_syncobj_wait args = {
157 .handles = (uintptr_t)&syncobj->handle,
158 .count_handles = 1,
159 .timeout_nsec = timeout_nsec,
160 };
161 return gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_WAIT, &args);
162 }
163
164 #define CSI "\e["
165 #define BLUE_HEADER CSI "0;97;44m"
166 #define NORMAL CSI "0m"
167
168 static void
169 iris_fence_flush(struct pipe_context *ctx,
170 struct pipe_fence_handle **out_fence,
171 unsigned flags)
172 {
173 struct iris_screen *screen = (void *) ctx->screen;
174 struct iris_context *ice = (struct iris_context *)ctx;
175
176 /* We require DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT (kernel 5.2+) for
177 * deferred flushes. Just ignore the request to defer on older kernels.
178 */
179 if (!(screen->kernel_features & KERNEL_HAS_WAIT_FOR_SUBMIT))
180 flags &= ~PIPE_FLUSH_DEFERRED;
181
182 const bool deferred = flags & PIPE_FLUSH_DEFERRED;
183
184 if (flags & PIPE_FLUSH_END_OF_FRAME) {
185 ice->frame++;
186
187 if (INTEL_DEBUG & DEBUG_SUBMIT) {
188 fprintf(stderr, "%s ::: FRAME %-10u (ctx %p)%-35c%s\n",
189 (INTEL_DEBUG & DEBUG_COLOR) ? BLUE_HEADER : "",
190 ice->frame, ctx, ' ',
191 (INTEL_DEBUG & DEBUG_COLOR) ? NORMAL : "");
192 }
193 }
194
195 if (!deferred) {
196 for (unsigned i = 0; i < IRIS_BATCH_COUNT; i++)
197 iris_batch_flush(&ice->batches[i]);
198 }
199
200 if (!out_fence)
201 return;
202
203 struct pipe_fence_handle *fence = calloc(1, sizeof(*fence));
204 if (!fence)
205 return;
206
207 pipe_reference_init(&fence->ref, 1);
208
209 if (deferred)
210 fence->unflushed_ctx = ctx;
211
212 for (unsigned b = 0; b < IRIS_BATCH_COUNT; b++) {
213 struct iris_batch *batch = &ice->batches[b];
214
215 if (deferred && iris_batch_bytes_used(batch) > 0) {
216 struct iris_fine_fence *fine =
217 iris_fine_fence_new(batch, IRIS_FENCE_BOTTOM_OF_PIPE);
218 iris_fine_fence_reference(screen, &fence->fine[b], fine);
219 iris_fine_fence_reference(screen, &fine, NULL);
220 } else {
221 /* This batch has no commands queued up (perhaps we just flushed,
222 * or all the commands are on the other batch). Wait for the last
223 * syncobj on this engine - unless it's already finished by now.
224 */
225 if (iris_fine_fence_signaled(batch->last_fence))
226 continue;
227
228 iris_fine_fence_reference(screen, &fence->fine[b], batch->last_fence);
229 }
230 }
231
232 iris_fence_reference(ctx->screen, out_fence, NULL);
233 *out_fence = fence;
234 }
235
236 static void
237 iris_fence_await(struct pipe_context *ctx,
238 struct pipe_fence_handle *fence)
239 {
240 struct iris_context *ice = (struct iris_context *)ctx;
241
242 /* Unflushed fences from the same context are no-ops. */
243 if (ctx && ctx == fence->unflushed_ctx)
244 return;
245
246 /* XXX: We can't safely flush the other context, because it might be
247 * bound to another thread, and poking at its internals wouldn't
248 * be safe. In the future we should use MI_SEMAPHORE_WAIT and
249 * block until the other job has been submitted, relying on
250 * kernel timeslicing to preempt us until the other job is
251 * actually flushed and the seqno finally passes.
252 */
253 if (fence->unflushed_ctx) {
254 pipe_debug_message(&ice->dbg, CONFORMANCE, "%s",
255 "glWaitSync on unflushed fence from another context "
256 "is unlikely to work without kernel 5.8+\n");
257 }
258
259 for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++) {
260 struct iris_fine_fence *fine = fence->fine[i];
261
262 if (iris_fine_fence_signaled(fine))
263 continue;
264
265 for (unsigned b = 0; b < IRIS_BATCH_COUNT; b++) {
266 struct iris_batch *batch = &ice->batches[b];
267
268 /* We're going to make any future work in this batch wait for our
269 * fence to have gone by. But any currently queued work doesn't
270 * need to wait. Flush the batch now, so it can happen sooner.
271 */
272 iris_batch_flush(batch);
273
274 iris_batch_add_syncobj(batch, fine->syncobj, I915_EXEC_FENCE_WAIT);
275 }
276 }
277 }
278
279 #define NSEC_PER_SEC (1000 * USEC_PER_SEC)
280 #define USEC_PER_SEC (1000 * MSEC_PER_SEC)
281 #define MSEC_PER_SEC (1000)
282
283 static uint64_t
284 gettime_ns(void)
285 {
286 struct timespec current;
287 clock_gettime(CLOCK_MONOTONIC, &current);
288 return (uint64_t)current.tv_sec * NSEC_PER_SEC + current.tv_nsec;
289 }
290
291 static uint64_t
292 rel2abs(uint64_t timeout)
293 {
294 if (timeout == 0)
295 return 0;
296
297 uint64_t current_time = gettime_ns();
298 uint64_t max_timeout = (uint64_t) INT64_MAX - current_time;
299
300 timeout = MIN2(max_timeout, timeout);
301
302 return current_time + timeout;
303 }
304
305 static bool
306 iris_fence_finish(struct pipe_screen *p_screen,
307 struct pipe_context *ctx,
308 struct pipe_fence_handle *fence,
309 uint64_t timeout)
310 {
311 struct iris_context *ice = (struct iris_context *)ctx;
312 struct iris_screen *screen = (struct iris_screen *)p_screen;
313
314 /* If we created the fence with PIPE_FLUSH_DEFERRED, we may not have
315 * flushed yet. Check if our syncobj is the current batch's signalling
316 * syncobj - if so, we haven't flushed and need to now.
317 *
318 * The Gallium docs mention that a flush will occur if \p ctx matches
319 * the context the fence was created with. It may be NULL, so we check
320 * that it matches first.
321 */
322 if (ctx && ctx == fence->unflushed_ctx) {
323 for (unsigned i = 0; i < IRIS_BATCH_COUNT; i++) {
324 struct iris_fine_fence *fine = fence->fine[i];
325
326 if (iris_fine_fence_signaled(fine))
327 continue;
328
329 if (fine->syncobj == iris_batch_get_signal_syncobj(&ice->batches[i]))
330 iris_batch_flush(&ice->batches[i]);
331 }
332
333 /* The fence is no longer deferred. */
334 fence->unflushed_ctx = NULL;
335 }
336
337 unsigned int handle_count = 0;
338 uint32_t handles[ARRAY_SIZE(fence->fine)];
339 for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++) {
340 struct iris_fine_fence *fine = fence->fine[i];
341
342 if (iris_fine_fence_signaled(fine))
343 continue;
344
345 handles[handle_count++] = fine->syncobj->handle;
346 }
347
348 if (handle_count == 0)
349 return true;
350
351 struct drm_syncobj_wait args = {
352 .handles = (uintptr_t)handles,
353 .count_handles = handle_count,
354 .timeout_nsec = rel2abs(timeout),
355 .flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL
356 };
357
358 if (fence->unflushed_ctx) {
359 /* This fence had a deferred flush from another context. We can't
360 * safely flush it here, because the context might be bound to a
361 * different thread, and poking at its internals wouldn't be safe.
362 *
363 * Instead, use the WAIT_FOR_SUBMIT flag to block and hope that
364 * another thread submits the work.
365 */
366 args.flags |= DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT;
367 }
368
369 return gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_WAIT, &args) == 0;
370 }
371
372 static int
373 sync_merge_fd(int sync_fd, int new_fd)
374 {
375 if (sync_fd == -1)
376 return new_fd;
377
378 if (new_fd == -1)
379 return sync_fd;
380
381 struct sync_merge_data args = {
382 .name = "iris fence",
383 .fd2 = new_fd,
384 .fence = -1,
385 };
386
387 gen_ioctl(sync_fd, SYNC_IOC_MERGE, &args);
388 close(new_fd);
389 close(sync_fd);
390
391 return args.fence;
392 }
393
394 static int
395 iris_fence_get_fd(struct pipe_screen *p_screen,
396 struct pipe_fence_handle *fence)
397 {
398 struct iris_screen *screen = (struct iris_screen *)p_screen;
399 int fd = -1;
400
401 /* Deferred fences aren't supported. */
402 if (fence->unflushed_ctx)
403 return -1;
404
405 for (unsigned i = 0; i < ARRAY_SIZE(fence->fine); i++) {
406 struct iris_fine_fence *fine = fence->fine[i];
407
408 if (iris_fine_fence_signaled(fine))
409 continue;
410
411 struct drm_syncobj_handle args = {
412 .handle = fine->syncobj->handle,
413 .flags = DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE,
414 .fd = -1,
415 };
416
417 gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD, &args);
418 fd = sync_merge_fd(fd, args.fd);
419 }
420
421 if (fd == -1) {
422 /* Our fence has no syncobj's recorded. This means that all of the
423 * batches had already completed, their syncobj's had been signalled,
424 * and so we didn't bother to record them. But we're being asked to
425 * export such a fence. So export a dummy already-signalled syncobj.
426 */
427 struct drm_syncobj_handle args = {
428 .flags = DRM_SYNCOBJ_HANDLE_TO_FD_FLAGS_EXPORT_SYNC_FILE, .fd = -1,
429 };
430
431 args.handle = gem_syncobj_create(screen->fd, DRM_SYNCOBJ_CREATE_SIGNALED);
432 gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_HANDLE_TO_FD, &args);
433 gem_syncobj_destroy(screen->fd, args.handle);
434 return args.fd;
435 }
436
437 return fd;
438 }
439
440 static void
441 iris_fence_create_fd(struct pipe_context *ctx,
442 struct pipe_fence_handle **out,
443 int fd,
444 enum pipe_fd_type type)
445 {
446 assert(type == PIPE_FD_TYPE_NATIVE_SYNC);
447
448 struct iris_screen *screen = (struct iris_screen *)ctx->screen;
449 struct drm_syncobj_handle args = {
450 .handle = gem_syncobj_create(screen->fd, DRM_SYNCOBJ_CREATE_SIGNALED),
451 .flags = DRM_SYNCOBJ_FD_TO_HANDLE_FLAGS_IMPORT_SYNC_FILE,
452 .fd = fd,
453 };
454 if (gen_ioctl(screen->fd, DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE, &args) == -1) {
455 fprintf(stderr, "DRM_IOCTL_SYNCOBJ_FD_TO_HANDLE failed: %s\n",
456 strerror(errno));
457 gem_syncobj_destroy(screen->fd, args.handle);
458 *out = NULL;
459 return;
460 }
461
462 struct iris_syncobj *syncobj = malloc(sizeof(*syncobj));
463 if (!syncobj) {
464 *out = NULL;
465 return;
466 }
467 syncobj->handle = args.handle;
468 pipe_reference_init(&syncobj->ref, 1);
469
470 struct iris_fine_fence *fine = calloc(1, sizeof(*fine));
471 if (!fine) {
472 free(syncobj);
473 *out = NULL;
474 return;
475 }
476
477 static const uint32_t zero = 0;
478
479 /* Fences work in terms of iris_fine_fence, but we don't actually have a
480 * seqno for an imported fence. So, create a fake one which always
481 * returns as 'not signaled' so we fall back to using the sync object.
482 */
483 fine->seqno = UINT32_MAX;
484 fine->map = &zero;
485 fine->syncobj = syncobj;
486 fine->flags = IRIS_FENCE_END;
487 pipe_reference_init(&fine->reference, 1);
488
489 struct pipe_fence_handle *fence = calloc(1, sizeof(*fence));
490 if (!fence) {
491 free(fine);
492 free(syncobj);
493 *out = NULL;
494 return;
495 }
496 pipe_reference_init(&fence->ref, 1);
497 fence->fine[0] = fine;
498
499 *out = fence;
500 }
501
502 void
503 iris_init_screen_fence_functions(struct pipe_screen *screen)
504 {
505 screen->fence_reference = iris_fence_reference;
506 screen->fence_finish = iris_fence_finish;
507 screen->fence_get_fd = iris_fence_get_fd;
508 }
509
510 void
511 iris_init_context_fence_functions(struct pipe_context *ctx)
512 {
513 ctx->flush = iris_fence_flush;
514 ctx->create_fence_fd = iris_fence_create_fd;
515 ctx->fence_server_sync = iris_fence_await;
516 }