2 * Copyright 2013-2017 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27 #include "util/os_time.h"
28 #include "util/u_memory.h"
29 #include "util/u_queue.h"
30 #include "util/u_upload_mgr.h"
33 #include "radeon/r600_cs.h"
35 struct si_fine_fence
{
36 struct r600_resource
*buf
;
40 struct si_multi_fence
{
41 struct pipe_reference reference
;
42 struct pipe_fence_handle
*gfx
;
43 struct pipe_fence_handle
*sdma
;
44 struct tc_unflushed_batch_token
*tc_token
;
45 struct util_queue_fence ready
;
47 /* If the context wasn't flushed at fence creation, this is non-NULL. */
49 struct r600_common_context
*ctx
;
53 struct si_fine_fence fine
;
59 * \param event EVENT_TYPE_*
60 * \param event_flags Optional cache flush flags (TC)
61 * \param data_sel 1 = fence, 3 = timestamp
63 * \param va GPU address
64 * \param old_value Previous fence value (for a bug workaround)
65 * \param new_value Fence value to write for this event.
67 void si_gfx_write_event_eop(struct r600_common_context
*ctx
,
68 unsigned event
, unsigned event_flags
,
70 struct r600_resource
*buf
, uint64_t va
,
71 uint32_t new_fence
, unsigned query_type
)
73 struct radeon_winsys_cs
*cs
= ctx
->gfx
.cs
;
74 unsigned op
= EVENT_TYPE(event
) |
77 unsigned sel
= EOP_DATA_SEL(data_sel
);
79 /* Wait for write confirmation before writing data, but don't send
81 if (data_sel
!= EOP_DATA_SEL_DISCARD
)
82 sel
|= EOP_INT_SEL(EOP_INT_SEL_SEND_DATA_AFTER_WR_CONFIRM
);
84 if (ctx
->chip_class
>= GFX9
) {
85 /* A ZPASS_DONE or PIXEL_STAT_DUMP_EVENT (of the DB occlusion
86 * counters) must immediately precede every timestamp event to
87 * prevent a GPU hang on GFX9.
89 * Occlusion queries don't need to do it here, because they
90 * always do ZPASS_DONE before the timestamp.
92 if (ctx
->chip_class
== GFX9
&&
93 query_type
!= PIPE_QUERY_OCCLUSION_COUNTER
&&
94 query_type
!= PIPE_QUERY_OCCLUSION_PREDICATE
&&
95 query_type
!= PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE
) {
96 struct r600_resource
*scratch
= ctx
->eop_bug_scratch
;
98 assert(16 * ctx
->screen
->info
.num_render_backends
<=
100 radeon_emit(cs
, PKT3(PKT3_EVENT_WRITE
, 2, 0));
101 radeon_emit(cs
, EVENT_TYPE(EVENT_TYPE_ZPASS_DONE
) | EVENT_INDEX(1));
102 radeon_emit(cs
, scratch
->gpu_address
);
103 radeon_emit(cs
, scratch
->gpu_address
>> 32);
105 radeon_add_to_buffer_list(ctx
, &ctx
->gfx
, scratch
,
106 RADEON_USAGE_WRITE
, RADEON_PRIO_QUERY
);
109 radeon_emit(cs
, PKT3(PKT3_RELEASE_MEM
, 6, 0));
111 radeon_emit(cs
, sel
);
112 radeon_emit(cs
, va
); /* address lo */
113 radeon_emit(cs
, va
>> 32); /* address hi */
114 radeon_emit(cs
, new_fence
); /* immediate data lo */
115 radeon_emit(cs
, 0); /* immediate data hi */
116 radeon_emit(cs
, 0); /* unused */
118 if (ctx
->chip_class
== CIK
||
119 ctx
->chip_class
== VI
) {
120 struct r600_resource
*scratch
= ctx
->eop_bug_scratch
;
121 uint64_t va
= scratch
->gpu_address
;
123 /* Two EOP events are required to make all engines go idle
124 * (and optional cache flushes executed) before the timestamp
127 radeon_emit(cs
, PKT3(PKT3_EVENT_WRITE_EOP
, 4, 0));
130 radeon_emit(cs
, ((va
>> 32) & 0xffff) | sel
);
131 radeon_emit(cs
, 0); /* immediate data */
132 radeon_emit(cs
, 0); /* unused */
134 radeon_add_to_buffer_list(ctx
, &ctx
->gfx
, scratch
,
135 RADEON_USAGE_WRITE
, RADEON_PRIO_QUERY
);
138 radeon_emit(cs
, PKT3(PKT3_EVENT_WRITE_EOP
, 4, 0));
141 radeon_emit(cs
, ((va
>> 32) & 0xffff) | sel
);
142 radeon_emit(cs
, new_fence
); /* immediate data */
143 radeon_emit(cs
, 0); /* unused */
147 radeon_add_to_buffer_list(ctx
, &ctx
->gfx
, buf
, RADEON_USAGE_WRITE
,
152 unsigned si_gfx_write_fence_dwords(struct si_screen
*screen
)
156 if (screen
->info
.chip_class
== CIK
||
157 screen
->info
.chip_class
== VI
)
163 void si_gfx_wait_fence(struct r600_common_context
*ctx
,
164 uint64_t va
, uint32_t ref
, uint32_t mask
)
166 struct radeon_winsys_cs
*cs
= ctx
->gfx
.cs
;
168 radeon_emit(cs
, PKT3(PKT3_WAIT_REG_MEM
, 5, 0));
169 radeon_emit(cs
, WAIT_REG_MEM_EQUAL
| WAIT_REG_MEM_MEM_SPACE(1));
171 radeon_emit(cs
, va
>> 32);
172 radeon_emit(cs
, ref
); /* reference value */
173 radeon_emit(cs
, mask
); /* mask */
174 radeon_emit(cs
, 4); /* poll interval */
177 static void si_add_fence_dependency(struct r600_common_context
*rctx
,
178 struct pipe_fence_handle
*fence
)
180 struct radeon_winsys
*ws
= rctx
->ws
;
183 ws
->cs_add_fence_dependency(rctx
->dma
.cs
, fence
);
184 ws
->cs_add_fence_dependency(rctx
->gfx
.cs
, fence
);
187 static void si_add_syncobj_signal(struct r600_common_context
*rctx
,
188 struct pipe_fence_handle
*fence
)
190 struct radeon_winsys
*ws
= rctx
->ws
;
192 ws
->cs_add_syncobj_signal(rctx
->gfx
.cs
, fence
);
195 static void si_fence_reference(struct pipe_screen
*screen
,
196 struct pipe_fence_handle
**dst
,
197 struct pipe_fence_handle
*src
)
199 struct radeon_winsys
*ws
= ((struct si_screen
*)screen
)->ws
;
200 struct si_multi_fence
**rdst
= (struct si_multi_fence
**)dst
;
201 struct si_multi_fence
*rsrc
= (struct si_multi_fence
*)src
;
203 if (pipe_reference(&(*rdst
)->reference
, &rsrc
->reference
)) {
204 ws
->fence_reference(&(*rdst
)->gfx
, NULL
);
205 ws
->fence_reference(&(*rdst
)->sdma
, NULL
);
206 tc_unflushed_batch_token_reference(&(*rdst
)->tc_token
, NULL
);
207 r600_resource_reference(&(*rdst
)->fine
.buf
, NULL
);
213 static struct si_multi_fence
*si_create_multi_fence()
215 struct si_multi_fence
*fence
= CALLOC_STRUCT(si_multi_fence
);
219 pipe_reference_init(&fence
->reference
, 1);
220 util_queue_fence_init(&fence
->ready
);
225 struct pipe_fence_handle
*si_create_fence(struct pipe_context
*ctx
,
226 struct tc_unflushed_batch_token
*tc_token
)
228 struct si_multi_fence
*fence
= si_create_multi_fence();
232 util_queue_fence_reset(&fence
->ready
);
233 tc_unflushed_batch_token_reference(&fence
->tc_token
, tc_token
);
235 return (struct pipe_fence_handle
*)fence
;
238 static bool si_fine_fence_signaled(struct radeon_winsys
*rws
,
239 const struct si_fine_fence
*fine
)
241 char *map
= rws
->buffer_map(fine
->buf
->buf
, NULL
, PIPE_TRANSFER_READ
|
242 PIPE_TRANSFER_UNSYNCHRONIZED
);
246 uint32_t *fence
= (uint32_t*)(map
+ fine
->offset
);
250 static void si_fine_fence_set(struct si_context
*ctx
,
251 struct si_fine_fence
*fine
,
256 assert(util_bitcount(flags
& (PIPE_FLUSH_TOP_OF_PIPE
| PIPE_FLUSH_BOTTOM_OF_PIPE
)) == 1);
258 /* Use uncached system memory for the fence. */
259 u_upload_alloc(ctx
->b
.cached_gtt_allocator
, 0, 4, 4,
260 &fine
->offset
, (struct pipe_resource
**)&fine
->buf
, (void **)&fence_ptr
);
266 uint64_t fence_va
= fine
->buf
->gpu_address
+ fine
->offset
;
268 radeon_add_to_buffer_list(&ctx
->b
, &ctx
->b
.gfx
, fine
->buf
,
269 RADEON_USAGE_WRITE
, RADEON_PRIO_QUERY
);
270 if (flags
& PIPE_FLUSH_TOP_OF_PIPE
) {
271 struct radeon_winsys_cs
*cs
= ctx
->b
.gfx
.cs
;
272 radeon_emit(cs
, PKT3(PKT3_WRITE_DATA
, 3, 0));
273 radeon_emit(cs
, S_370_DST_SEL(V_370_MEM_ASYNC
) |
274 S_370_WR_CONFIRM(1) |
275 S_370_ENGINE_SEL(V_370_PFP
));
276 radeon_emit(cs
, fence_va
);
277 radeon_emit(cs
, fence_va
>> 32);
278 radeon_emit(cs
, 0x80000000);
279 } else if (flags
& PIPE_FLUSH_BOTTOM_OF_PIPE
) {
280 si_gfx_write_event_eop(&ctx
->b
, V_028A90_BOTTOM_OF_PIPE_TS
, 0,
281 EOP_DATA_SEL_VALUE_32BIT
,
282 NULL
, fence_va
, 0x80000000,
283 PIPE_QUERY_GPU_FINISHED
);
289 static boolean
si_fence_finish(struct pipe_screen
*screen
,
290 struct pipe_context
*ctx
,
291 struct pipe_fence_handle
*fence
,
294 struct radeon_winsys
*rws
= ((struct si_screen
*)screen
)->ws
;
295 struct si_multi_fence
*rfence
= (struct si_multi_fence
*)fence
;
296 int64_t abs_timeout
= os_time_get_absolute_timeout(timeout
);
298 if (!util_queue_fence_is_signalled(&rfence
->ready
)) {
299 if (rfence
->tc_token
) {
300 /* Ensure that si_flush_from_st will be called for
301 * this fence, but only if we're in the API thread
302 * where the context is current.
304 * Note that the batch containing the flush may already
305 * be in flight in the driver thread, so the fence
306 * may not be ready yet when this call returns.
308 threaded_context_flush(ctx
, rfence
->tc_token
,
315 if (timeout
== PIPE_TIMEOUT_INFINITE
) {
316 util_queue_fence_wait(&rfence
->ready
);
318 if (!util_queue_fence_wait_timeout(&rfence
->ready
, abs_timeout
))
322 if (timeout
&& timeout
!= PIPE_TIMEOUT_INFINITE
) {
323 int64_t time
= os_time_get_nano();
324 timeout
= abs_timeout
> time
? abs_timeout
- time
: 0;
329 if (!rws
->fence_wait(rws
, rfence
->sdma
, timeout
))
332 /* Recompute the timeout after waiting. */
333 if (timeout
&& timeout
!= PIPE_TIMEOUT_INFINITE
) {
334 int64_t time
= os_time_get_nano();
335 timeout
= abs_timeout
> time
? abs_timeout
- time
: 0;
342 if (rfence
->fine
.buf
&&
343 si_fine_fence_signaled(rws
, &rfence
->fine
)) {
344 rws
->fence_reference(&rfence
->gfx
, NULL
);
345 r600_resource_reference(&rfence
->fine
.buf
, NULL
);
349 /* Flush the gfx IB if it hasn't been flushed yet. */
350 if (ctx
&& rfence
->gfx_unflushed
.ctx
) {
351 struct si_context
*sctx
;
353 sctx
= (struct si_context
*)threaded_context_unwrap_unsync(ctx
);
354 if (rfence
->gfx_unflushed
.ctx
== &sctx
->b
&&
355 rfence
->gfx_unflushed
.ib_index
== sctx
->b
.num_gfx_cs_flushes
) {
356 /* Section 4.1.2 (Signaling) of the OpenGL 4.6 (Core profile)
359 * "If the sync object being blocked upon will not be
360 * signaled in finite time (for example, by an associated
361 * fence command issued previously, but not yet flushed to
362 * the graphics pipeline), then ClientWaitSync may hang
363 * forever. To help prevent this behavior, if
364 * ClientWaitSync is called and all of the following are
367 * * the SYNC_FLUSH_COMMANDS_BIT bit is set in flags,
368 * * sync is unsignaled when ClientWaitSync is called,
369 * * and the calls to ClientWaitSync and FenceSync were
370 * issued from the same context,
372 * then the GL will behave as if the equivalent of Flush
373 * were inserted immediately after the creation of sync."
375 * This means we need to flush for such fences even when we're
378 threaded_context_unwrap_sync(ctx
);
379 sctx
->b
.gfx
.flush(&sctx
->b
, timeout
? 0 : PIPE_FLUSH_ASYNC
, NULL
);
380 rfence
->gfx_unflushed
.ctx
= NULL
;
385 /* Recompute the timeout after all that. */
386 if (timeout
&& timeout
!= PIPE_TIMEOUT_INFINITE
) {
387 int64_t time
= os_time_get_nano();
388 timeout
= abs_timeout
> time
? abs_timeout
- time
: 0;
393 if (rws
->fence_wait(rws
, rfence
->gfx
, timeout
))
396 /* Re-check in case the GPU is slow or hangs, but the commands before
397 * the fine-grained fence have completed. */
398 if (rfence
->fine
.buf
&&
399 si_fine_fence_signaled(rws
, &rfence
->fine
))
405 static void si_create_fence_fd(struct pipe_context
*ctx
,
406 struct pipe_fence_handle
**pfence
, int fd
,
407 enum pipe_fd_type type
)
409 struct si_screen
*sscreen
= (struct si_screen
*)ctx
->screen
;
410 struct radeon_winsys
*ws
= sscreen
->ws
;
411 struct si_multi_fence
*rfence
;
415 rfence
= si_create_multi_fence();
420 case PIPE_FD_TYPE_NATIVE_SYNC
:
421 if (!sscreen
->info
.has_fence_to_handle
)
424 rfence
->gfx
= ws
->fence_import_sync_file(ws
, fd
);
427 case PIPE_FD_TYPE_SYNCOBJ
:
428 if (!sscreen
->info
.has_syncobj
)
431 rfence
->gfx
= ws
->fence_import_syncobj(ws
, fd
);
435 unreachable("bad fence fd type when importing");
444 *pfence
= (struct pipe_fence_handle
*)rfence
;
447 static int si_fence_get_fd(struct pipe_screen
*screen
,
448 struct pipe_fence_handle
*fence
)
450 struct si_screen
*sscreen
= (struct si_screen
*)screen
;
451 struct radeon_winsys
*ws
= sscreen
->ws
;
452 struct si_multi_fence
*rfence
= (struct si_multi_fence
*)fence
;
453 int gfx_fd
= -1, sdma_fd
= -1;
455 if (!sscreen
->info
.has_fence_to_handle
)
458 util_queue_fence_wait(&rfence
->ready
);
460 /* Deferred fences aren't supported. */
461 assert(!rfence
->gfx_unflushed
.ctx
);
462 if (rfence
->gfx_unflushed
.ctx
)
466 sdma_fd
= ws
->fence_export_sync_file(ws
, rfence
->sdma
);
471 gfx_fd
= ws
->fence_export_sync_file(ws
, rfence
->gfx
);
479 /* If we don't have FDs at this point, it means we don't have fences
481 if (sdma_fd
== -1 && gfx_fd
== -1)
482 return ws
->export_signalled_sync_file(ws
);
488 /* Get a fence that will be a combination of both fences. */
489 sync_accumulate("radeonsi", &gfx_fd
, sdma_fd
);
494 static void si_flush_from_st(struct pipe_context
*ctx
,
495 struct pipe_fence_handle
**fence
,
498 struct pipe_screen
*screen
= ctx
->screen
;
499 struct r600_common_context
*rctx
= (struct r600_common_context
*)ctx
;
500 struct radeon_winsys
*ws
= rctx
->ws
;
501 struct pipe_fence_handle
*gfx_fence
= NULL
;
502 struct pipe_fence_handle
*sdma_fence
= NULL
;
503 bool deferred_fence
= false;
504 struct si_fine_fence fine
= {};
505 unsigned rflags
= PIPE_FLUSH_ASYNC
;
507 if (flags
& PIPE_FLUSH_END_OF_FRAME
)
508 rflags
|= PIPE_FLUSH_END_OF_FRAME
;
510 if (flags
& (PIPE_FLUSH_TOP_OF_PIPE
| PIPE_FLUSH_BOTTOM_OF_PIPE
)) {
511 assert(flags
& PIPE_FLUSH_DEFERRED
);
514 si_fine_fence_set((struct si_context
*)rctx
, &fine
, flags
);
517 /* DMA IBs are preambles to gfx IBs, therefore must be flushed first. */
519 rctx
->dma
.flush(rctx
, rflags
, fence
? &sdma_fence
: NULL
);
521 if (!radeon_emitted(rctx
->gfx
.cs
, rctx
->initial_gfx_cs_size
)) {
523 ws
->fence_reference(&gfx_fence
, rctx
->last_gfx_fence
);
524 if (!(flags
& PIPE_FLUSH_DEFERRED
))
525 ws
->cs_sync_flush(rctx
->gfx
.cs
);
527 /* Instead of flushing, create a deferred fence. Constraints:
528 * - The state tracker must allow a deferred flush.
529 * - The state tracker must request a fence.
530 * - fence_get_fd is not allowed.
531 * Thread safety in fence_finish must be ensured by the state tracker.
533 if (flags
& PIPE_FLUSH_DEFERRED
&&
534 !(flags
& PIPE_FLUSH_FENCE_FD
) &&
536 gfx_fence
= rctx
->ws
->cs_get_next_fence(rctx
->gfx
.cs
);
537 deferred_fence
= true;
539 rctx
->gfx
.flush(rctx
, rflags
, fence
? &gfx_fence
: NULL
);
543 /* Both engines can signal out of order, so we need to keep both fences. */
545 struct si_multi_fence
*multi_fence
;
547 if (flags
& TC_FLUSH_ASYNC
) {
548 multi_fence
= (struct si_multi_fence
*)*fence
;
551 multi_fence
= si_create_multi_fence();
553 ws
->fence_reference(&sdma_fence
, NULL
);
554 ws
->fence_reference(&gfx_fence
, NULL
);
558 screen
->fence_reference(screen
, fence
, NULL
);
559 *fence
= (struct pipe_fence_handle
*)multi_fence
;
562 /* If both fences are NULL, fence_finish will always return true. */
563 multi_fence
->gfx
= gfx_fence
;
564 multi_fence
->sdma
= sdma_fence
;
566 if (deferred_fence
) {
567 multi_fence
->gfx_unflushed
.ctx
= rctx
;
568 multi_fence
->gfx_unflushed
.ib_index
= rctx
->num_gfx_cs_flushes
;
571 multi_fence
->fine
= fine
;
574 if (flags
& TC_FLUSH_ASYNC
) {
575 util_queue_fence_signal(&multi_fence
->ready
);
576 tc_unflushed_batch_token_reference(&multi_fence
->tc_token
, NULL
);
581 if (!(flags
& PIPE_FLUSH_DEFERRED
)) {
583 ws
->cs_sync_flush(rctx
->dma
.cs
);
584 ws
->cs_sync_flush(rctx
->gfx
.cs
);
588 static void si_fence_server_signal(struct pipe_context
*ctx
,
589 struct pipe_fence_handle
*fence
)
591 struct r600_common_context
*rctx
= (struct r600_common_context
*)ctx
;
592 struct si_multi_fence
*rfence
= (struct si_multi_fence
*)fence
;
594 /* We should have at least one syncobj to signal */
595 assert(rfence
->sdma
|| rfence
->gfx
);
598 si_add_syncobj_signal(rctx
, rfence
->sdma
);
600 si_add_syncobj_signal(rctx
, rfence
->gfx
);
603 * The spec does not require a flush here. We insert a flush
604 * because syncobj based signals are not directly placed into
605 * the command stream. Instead the signal happens when the
606 * submission associated with the syncobj finishes execution.
608 * Therefore, we must make sure that we flush the pipe to avoid
609 * new work being emitted and getting executed before the signal
612 si_flush_from_st(ctx
, NULL
, PIPE_FLUSH_ASYNC
);
615 static void si_fence_server_sync(struct pipe_context
*ctx
,
616 struct pipe_fence_handle
*fence
)
618 struct r600_common_context
*rctx
= (struct r600_common_context
*)ctx
;
619 struct si_multi_fence
*rfence
= (struct si_multi_fence
*)fence
;
621 util_queue_fence_wait(&rfence
->ready
);
623 /* Unflushed fences from the same context are no-ops. */
624 if (rfence
->gfx_unflushed
.ctx
&&
625 rfence
->gfx_unflushed
.ctx
== rctx
)
628 /* All unflushed commands will not start execution before
629 * this fence dependency is signalled.
631 * Therefore we must flush before inserting the dependency
633 si_flush_from_st(ctx
, NULL
, PIPE_FLUSH_ASYNC
);
636 si_add_fence_dependency(rctx
, rfence
->sdma
);
638 si_add_fence_dependency(rctx
, rfence
->gfx
);
641 void si_init_fence_functions(struct si_context
*ctx
)
643 ctx
->b
.b
.flush
= si_flush_from_st
;
644 ctx
->b
.b
.create_fence_fd
= si_create_fence_fd
;
645 ctx
->b
.b
.fence_server_sync
= si_fence_server_sync
;
646 ctx
->b
.b
.fence_server_signal
= si_fence_server_signal
;
649 void si_init_screen_fence_functions(struct si_screen
*screen
)
651 screen
->b
.fence_finish
= si_fence_finish
;
652 screen
->b
.fence_reference
= si_fence_reference
;
653 screen
->b
.fence_get_fd
= si_fence_get_fd
;