2 * Copyright 2013-2017 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
27 #include "util/os_time.h"
28 #include "util/u_memory.h"
29 #include "util/u_queue.h"
33 struct si_multi_fence
{
34 struct pipe_reference reference
;
35 struct pipe_fence_handle
*gfx
;
36 struct pipe_fence_handle
*sdma
;
37 struct tc_unflushed_batch_token
*tc_token
;
38 struct util_queue_fence ready
;
40 /* If the context wasn't flushed at fence creation, this is non-NULL. */
42 struct r600_common_context
*ctx
;
47 static void si_add_fence_dependency(struct r600_common_context
*rctx
,
48 struct pipe_fence_handle
*fence
)
50 struct radeon_winsys
*ws
= rctx
->ws
;
53 ws
->cs_add_fence_dependency(rctx
->dma
.cs
, fence
);
54 ws
->cs_add_fence_dependency(rctx
->gfx
.cs
, fence
);
57 static void si_fence_reference(struct pipe_screen
*screen
,
58 struct pipe_fence_handle
**dst
,
59 struct pipe_fence_handle
*src
)
61 struct radeon_winsys
*ws
= ((struct r600_common_screen
*)screen
)->ws
;
62 struct si_multi_fence
**rdst
= (struct si_multi_fence
**)dst
;
63 struct si_multi_fence
*rsrc
= (struct si_multi_fence
*)src
;
65 if (pipe_reference(&(*rdst
)->reference
, &rsrc
->reference
)) {
66 ws
->fence_reference(&(*rdst
)->gfx
, NULL
);
67 ws
->fence_reference(&(*rdst
)->sdma
, NULL
);
68 tc_unflushed_batch_token_reference(&(*rdst
)->tc_token
, NULL
);
74 static struct si_multi_fence
*si_create_multi_fence()
76 struct si_multi_fence
*fence
= CALLOC_STRUCT(si_multi_fence
);
80 pipe_reference_init(&fence
->reference
, 1);
81 util_queue_fence_init(&fence
->ready
);
86 struct pipe_fence_handle
*si_create_fence(struct pipe_context
*ctx
,
87 struct tc_unflushed_batch_token
*tc_token
)
89 struct si_multi_fence
*fence
= si_create_multi_fence();
93 util_queue_fence_reset(&fence
->ready
);
94 tc_unflushed_batch_token_reference(&fence
->tc_token
, tc_token
);
96 return (struct pipe_fence_handle
*)fence
;
99 static void si_fence_server_sync(struct pipe_context
*ctx
,
100 struct pipe_fence_handle
*fence
)
102 struct r600_common_context
*rctx
= (struct r600_common_context
*)ctx
;
103 struct si_multi_fence
*rfence
= (struct si_multi_fence
*)fence
;
105 util_queue_fence_wait(&rfence
->ready
);
107 /* Unflushed fences from the same context are no-ops. */
108 if (rfence
->gfx_unflushed
.ctx
&&
109 rfence
->gfx_unflushed
.ctx
== rctx
)
112 /* All unflushed commands will not start execution before
113 * this fence dependency is signalled.
115 * Should we flush the context to allow more GPU parallelism?
118 si_add_fence_dependency(rctx
, rfence
->sdma
);
120 si_add_fence_dependency(rctx
, rfence
->gfx
);
123 static boolean
si_fence_finish(struct pipe_screen
*screen
,
124 struct pipe_context
*ctx
,
125 struct pipe_fence_handle
*fence
,
128 struct radeon_winsys
*rws
= ((struct r600_common_screen
*)screen
)->ws
;
129 struct si_multi_fence
*rfence
= (struct si_multi_fence
*)fence
;
130 struct r600_common_context
*rctx
;
131 int64_t abs_timeout
= os_time_get_absolute_timeout(timeout
);
133 ctx
= threaded_context_unwrap_sync(ctx
);
134 rctx
= ctx
? (struct r600_common_context
*)ctx
: NULL
;
136 if (!util_queue_fence_is_signalled(&rfence
->ready
)) {
140 if (rfence
->tc_token
) {
141 /* Ensure that si_flush_from_st will be called for
142 * this fence, but only if we're in the API thread
143 * where the context is current.
145 * Note that the batch containing the flush may already
146 * be in flight in the driver thread, so the fence
147 * may not be ready yet when this call returns.
149 threaded_context_flush(ctx
, rfence
->tc_token
);
152 if (timeout
== PIPE_TIMEOUT_INFINITE
) {
153 util_queue_fence_wait(&rfence
->ready
);
155 if (!util_queue_fence_wait_timeout(&rfence
->ready
, abs_timeout
))
161 if (!rws
->fence_wait(rws
, rfence
->sdma
, timeout
))
164 /* Recompute the timeout after waiting. */
165 if (timeout
&& timeout
!= PIPE_TIMEOUT_INFINITE
) {
166 int64_t time
= os_time_get_nano();
167 timeout
= abs_timeout
> time
? abs_timeout
- time
: 0;
174 /* Flush the gfx IB if it hasn't been flushed yet. */
176 rfence
->gfx_unflushed
.ctx
== rctx
&&
177 rfence
->gfx_unflushed
.ib_index
== rctx
->num_gfx_cs_flushes
) {
178 /* Section 4.1.2 (Signaling) of the OpenGL 4.6 (Core profile)
181 * "If the sync object being blocked upon will not be
182 * signaled in finite time (for example, by an associated
183 * fence command issued previously, but not yet flushed to
184 * the graphics pipeline), then ClientWaitSync may hang
185 * forever. To help prevent this behavior, if
186 * ClientWaitSync is called and all of the following are
189 * * the SYNC_FLUSH_COMMANDS_BIT bit is set in flags,
190 * * sync is unsignaled when ClientWaitSync is called,
191 * * and the calls to ClientWaitSync and FenceSync were
192 * issued from the same context,
194 * then the GL will behave as if the equivalent of Flush
195 * were inserted immediately after the creation of sync."
197 * This means we need to flush for such fences even when we're
200 rctx
->gfx
.flush(rctx
, timeout
? 0 : RADEON_FLUSH_ASYNC
, NULL
);
201 rfence
->gfx_unflushed
.ctx
= NULL
;
206 /* Recompute the timeout after all that. */
207 if (timeout
&& timeout
!= PIPE_TIMEOUT_INFINITE
) {
208 int64_t time
= os_time_get_nano();
209 timeout
= abs_timeout
> time
? abs_timeout
- time
: 0;
213 return rws
->fence_wait(rws
, rfence
->gfx
, timeout
);
216 static void si_create_fence_fd(struct pipe_context
*ctx
,
217 struct pipe_fence_handle
**pfence
, int fd
)
219 struct r600_common_screen
*rscreen
= (struct r600_common_screen
*)ctx
->screen
;
220 struct radeon_winsys
*ws
= rscreen
->ws
;
221 struct si_multi_fence
*rfence
;
225 if (!rscreen
->info
.has_sync_file
)
228 rfence
= si_create_multi_fence();
232 rfence
->gfx
= ws
->fence_import_sync_file(ws
, fd
);
238 *pfence
= (struct pipe_fence_handle
*)rfence
;
241 static int si_fence_get_fd(struct pipe_screen
*screen
,
242 struct pipe_fence_handle
*fence
)
244 struct r600_common_screen
*rscreen
= (struct r600_common_screen
*)screen
;
245 struct radeon_winsys
*ws
= rscreen
->ws
;
246 struct si_multi_fence
*rfence
= (struct si_multi_fence
*)fence
;
247 int gfx_fd
= -1, sdma_fd
= -1;
249 if (!rscreen
->info
.has_sync_file
)
252 util_queue_fence_wait(&rfence
->ready
);
254 /* Deferred fences aren't supported. */
255 assert(!rfence
->gfx_unflushed
.ctx
);
256 if (rfence
->gfx_unflushed
.ctx
)
260 sdma_fd
= ws
->fence_export_sync_file(ws
, rfence
->sdma
);
265 gfx_fd
= ws
->fence_export_sync_file(ws
, rfence
->gfx
);
273 /* If we don't have FDs at this point, it means we don't have fences
280 /* Get a fence that will be a combination of both fences. */
281 sync_accumulate("radeonsi", &gfx_fd
, sdma_fd
);
286 static void si_flush_from_st(struct pipe_context
*ctx
,
287 struct pipe_fence_handle
**fence
,
290 struct pipe_screen
*screen
= ctx
->screen
;
291 struct r600_common_context
*rctx
= (struct r600_common_context
*)ctx
;
292 struct radeon_winsys
*ws
= rctx
->ws
;
293 struct pipe_fence_handle
*gfx_fence
= NULL
;
294 struct pipe_fence_handle
*sdma_fence
= NULL
;
295 bool deferred_fence
= false;
296 unsigned rflags
= RADEON_FLUSH_ASYNC
;
298 if (flags
& PIPE_FLUSH_END_OF_FRAME
)
299 rflags
|= RADEON_FLUSH_END_OF_FRAME
;
301 /* DMA IBs are preambles to gfx IBs, therefore must be flushed first. */
303 rctx
->dma
.flush(rctx
, rflags
, fence
? &sdma_fence
: NULL
);
305 if (!radeon_emitted(rctx
->gfx
.cs
, rctx
->initial_gfx_cs_size
)) {
307 ws
->fence_reference(&gfx_fence
, rctx
->last_gfx_fence
);
308 if (!(flags
& PIPE_FLUSH_DEFERRED
))
309 ws
->cs_sync_flush(rctx
->gfx
.cs
);
311 /* Instead of flushing, create a deferred fence. Constraints:
312 * - The state tracker must allow a deferred flush.
313 * - The state tracker must request a fence.
314 * - fence_get_fd is not allowed.
315 * Thread safety in fence_finish must be ensured by the state tracker.
317 if (flags
& PIPE_FLUSH_DEFERRED
&&
318 !(flags
& PIPE_FLUSH_FENCE_FD
) &&
320 gfx_fence
= rctx
->ws
->cs_get_next_fence(rctx
->gfx
.cs
);
321 deferred_fence
= true;
323 rctx
->gfx
.flush(rctx
, rflags
, fence
? &gfx_fence
: NULL
);
327 /* Both engines can signal out of order, so we need to keep both fences. */
329 struct si_multi_fence
*multi_fence
;
331 if (flags
& TC_FLUSH_ASYNC
) {
332 multi_fence
= (struct si_multi_fence
*)*fence
;
335 multi_fence
= si_create_multi_fence();
337 ws
->fence_reference(&sdma_fence
, NULL
);
338 ws
->fence_reference(&gfx_fence
, NULL
);
342 screen
->fence_reference(screen
, fence
, NULL
);
343 *fence
= (struct pipe_fence_handle
*)multi_fence
;
346 /* If both fences are NULL, fence_finish will always return true. */
347 multi_fence
->gfx
= gfx_fence
;
348 multi_fence
->sdma
= sdma_fence
;
350 if (deferred_fence
) {
351 multi_fence
->gfx_unflushed
.ctx
= rctx
;
352 multi_fence
->gfx_unflushed
.ib_index
= rctx
->num_gfx_cs_flushes
;
355 if (flags
& TC_FLUSH_ASYNC
) {
356 util_queue_fence_signal(&multi_fence
->ready
);
357 tc_unflushed_batch_token_reference(&multi_fence
->tc_token
, NULL
);
361 if (!(flags
& PIPE_FLUSH_DEFERRED
)) {
363 ws
->cs_sync_flush(rctx
->dma
.cs
);
364 ws
->cs_sync_flush(rctx
->gfx
.cs
);
368 void si_init_fence_functions(struct si_context
*ctx
)
370 ctx
->b
.b
.flush
= si_flush_from_st
;
371 ctx
->b
.b
.create_fence_fd
= si_create_fence_fd
;
372 ctx
->b
.b
.fence_server_sync
= si_fence_server_sync
;
375 void si_init_screen_fence_functions(struct si_screen
*screen
)
377 screen
->b
.b
.fence_finish
= si_fence_finish
;
378 screen
->b
.b
.fence_reference
= si_fence_reference
;
379 screen
->b
.b
.fence_get_fd
= si_fence_get_fd
;