1 /**************************************************************************
3 * Copyright 2007 Tungsten Graphics, Inc., Cedar Park, Texas.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
30 * Implementation of fenced buffers.
32 * \author Jose Fonseca <jrfonseca-at-tungstengraphics-dot-com>
33 * \author Thomas Hellström <thomas-at-tungstengraphics-dot-com>
37 #include "pipe/p_config.h"
39 #if defined(PIPE_OS_LINUX)
44 #include "pipe/p_compiler.h"
45 #include "pipe/p_error.h"
46 #include "util/u_debug.h"
47 #include "pipe/p_thread.h"
48 #include "util/u_memory.h"
49 #include "util/u_double_list.h"
51 #include "pb_buffer.h"
52 #include "pb_buffer_fenced.h"
57 * Convenience macro (type safe).
59 #define SUPER(__derived) (&(__derived)->base)
62 struct fenced_buffer_list
66 struct pb_fence_ops
*ops
;
70 struct list_head delayed
;
75 * Wrapper around a pipe buffer which adds fencing and reference counting.
79 struct pb_buffer base
;
81 struct pb_buffer
*buffer
;
83 /* FIXME: protect access with mutex */
86 * A bitmask of PIPE_BUFFER_USAGE_CPU/GPU_READ/WRITE describing the current
92 struct pb_validate
*vl
;
93 unsigned validation_flags
;
94 struct pipe_fence_handle
*fence
;
96 struct list_head head
;
97 struct fenced_buffer_list
*list
;
101 static INLINE
struct fenced_buffer
*
102 fenced_buffer(struct pb_buffer
*buf
)
105 return (struct fenced_buffer
*)buf
;
110 _fenced_buffer_add(struct fenced_buffer
*fenced_buf
)
112 struct fenced_buffer_list
*fenced_list
= fenced_buf
->list
;
114 assert(fenced_buf
->base
.base
.refcount
);
115 assert(fenced_buf
->flags
& PIPE_BUFFER_USAGE_GPU_READ_WRITE
);
116 assert(fenced_buf
->fence
);
118 assert(!fenced_buf
->head
.prev
);
119 assert(!fenced_buf
->head
.next
);
120 LIST_ADDTAIL(&fenced_buf
->head
, &fenced_list
->delayed
);
121 ++fenced_list
->numDelayed
;
126 * Actually destroy the buffer.
129 _fenced_buffer_destroy(struct fenced_buffer
*fenced_buf
)
131 assert(!fenced_buf
->base
.base
.refcount
);
132 assert(!fenced_buf
->fence
);
133 pb_reference(&fenced_buf
->buffer
, NULL
);
139 _fenced_buffer_remove(struct fenced_buffer_list
*fenced_list
,
140 struct fenced_buffer
*fenced_buf
)
142 struct pb_fence_ops
*ops
= fenced_list
->ops
;
144 assert(fenced_buf
->fence
);
145 assert(fenced_buf
->list
== fenced_list
);
147 ops
->fence_reference(ops
, &fenced_buf
->fence
, NULL
);
148 fenced_buf
->flags
&= ~PIPE_BUFFER_USAGE_GPU_READ_WRITE
;
150 assert(fenced_buf
->head
.prev
);
151 assert(fenced_buf
->head
.next
);
152 LIST_DEL(&fenced_buf
->head
);
154 fenced_buf
->head
.prev
= NULL
;
155 fenced_buf
->head
.next
= NULL
;
158 assert(fenced_list
->numDelayed
);
159 --fenced_list
->numDelayed
;
161 if(!fenced_buf
->base
.base
.refcount
)
162 _fenced_buffer_destroy(fenced_buf
);
166 static INLINE
enum pipe_error
167 _fenced_buffer_finish(struct fenced_buffer
*fenced_buf
)
169 struct fenced_buffer_list
*fenced_list
= fenced_buf
->list
;
170 struct pb_fence_ops
*ops
= fenced_list
->ops
;
173 debug_warning("waiting for GPU");
176 assert(fenced_buf
->fence
);
177 if(fenced_buf
->fence
) {
178 if(ops
->fence_finish(ops
, fenced_buf
->fence
, 0) != 0) {
181 /* Remove from the fenced list */
182 /* TODO: remove consequents */
183 _fenced_buffer_remove(fenced_list
, fenced_buf
);
186 fenced_buf
->flags
&= ~PIPE_BUFFER_USAGE_GPU_READ_WRITE
;
192 * Free as many fenced buffers from the list head as possible.
195 _fenced_buffer_list_check_free(struct fenced_buffer_list
*fenced_list
,
198 struct pb_fence_ops
*ops
= fenced_list
->ops
;
199 struct list_head
*curr
, *next
;
200 struct fenced_buffer
*fenced_buf
;
201 struct pipe_fence_handle
*prev_fence
= NULL
;
203 curr
= fenced_list
->delayed
.next
;
205 while(curr
!= &fenced_list
->delayed
) {
206 fenced_buf
= LIST_ENTRY(struct fenced_buffer
, curr
, head
);
208 if(fenced_buf
->fence
!= prev_fence
) {
211 signaled
= ops
->fence_finish(ops
, fenced_buf
->fence
, 0);
213 signaled
= ops
->fence_signalled(ops
, fenced_buf
->fence
, 0);
216 prev_fence
= fenced_buf
->fence
;
219 assert(ops
->fence_signalled(ops
, fenced_buf
->fence
, 0) == 0);
222 _fenced_buffer_remove(fenced_list
, fenced_buf
);
231 fenced_buffer_destroy(struct pb_buffer
*buf
)
233 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
234 struct fenced_buffer_list
*fenced_list
= fenced_buf
->list
;
236 pipe_mutex_lock(fenced_list
->mutex
);
237 assert(fenced_buf
->base
.base
.refcount
== 0);
238 if (fenced_buf
->fence
) {
239 struct pb_fence_ops
*ops
= fenced_list
->ops
;
240 if(ops
->fence_signalled(ops
, fenced_buf
->fence
, 0) == 0) {
241 struct list_head
*curr
, *prev
;
242 curr
= &fenced_buf
->head
;
245 fenced_buf
= LIST_ENTRY(struct fenced_buffer
, curr
, head
);
246 assert(ops
->fence_signalled(ops
, fenced_buf
->fence
, 0) == 0);
247 _fenced_buffer_remove(fenced_list
, fenced_buf
);
250 } while (curr
!= &fenced_list
->delayed
);
253 /* delay destruction */
257 _fenced_buffer_destroy(fenced_buf
);
259 pipe_mutex_unlock(fenced_list
->mutex
);
264 fenced_buffer_map(struct pb_buffer
*buf
,
267 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
270 assert(flags
& PIPE_BUFFER_USAGE_CPU_READ_WRITE
);
271 assert(!(flags
& ~PIPE_BUFFER_USAGE_CPU_READ_WRITE
));
272 flags
&= PIPE_BUFFER_USAGE_CPU_READ_WRITE
;
274 /* Check for GPU read/write access */
275 if(fenced_buf
->flags
& PIPE_BUFFER_USAGE_GPU_WRITE
) {
276 /* Wait for the GPU to finish writing */
277 _fenced_buffer_finish(fenced_buf
);
281 /* Check for CPU write access (read is OK) */
282 if(fenced_buf
->flags
& PIPE_BUFFER_USAGE_CPU_READ_WRITE
) {
283 /* this is legal -- just for debugging */
284 debug_warning("concurrent CPU writes");
288 map
= pb_map(fenced_buf
->buffer
, flags
);
290 ++fenced_buf
->mapcount
;
291 fenced_buf
->flags
|= flags
;
299 fenced_buffer_unmap(struct pb_buffer
*buf
)
301 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
302 assert(fenced_buf
->mapcount
);
303 if(fenced_buf
->mapcount
) {
304 pb_unmap(fenced_buf
->buffer
);
305 --fenced_buf
->mapcount
;
306 if(!fenced_buf
->mapcount
)
307 fenced_buf
->flags
&= ~PIPE_BUFFER_USAGE_CPU_READ_WRITE
;
312 static enum pipe_error
313 fenced_buffer_validate(struct pb_buffer
*buf
,
314 struct pb_validate
*vl
,
317 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
322 fenced_buf
->vl
= NULL
;
323 fenced_buf
->validation_flags
= 0;
327 assert(flags
& PIPE_BUFFER_USAGE_GPU_READ_WRITE
);
328 assert(!(flags
& ~PIPE_BUFFER_USAGE_GPU_READ_WRITE
));
329 flags
&= PIPE_BUFFER_USAGE_GPU_READ_WRITE
;
331 /* Buffer cannot be validated in two different lists */
332 if(fenced_buf
->vl
&& fenced_buf
->vl
!= vl
)
333 return PIPE_ERROR_RETRY
;
335 /* Do not validate if buffer is still mapped */
336 if(fenced_buf
->flags
& PIPE_BUFFER_USAGE_CPU_READ_WRITE
) {
337 /* TODO: wait for the thread that mapped the buffer to unmap it */
338 return PIPE_ERROR_RETRY
;
341 if(fenced_buf
->vl
== vl
&&
342 (fenced_buf
->validation_flags
& flags
) == flags
) {
343 /* Nothing to do -- buffer already validated */
347 /* Final sanity checking */
348 assert(!(fenced_buf
->flags
& PIPE_BUFFER_USAGE_CPU_READ_WRITE
));
349 assert(!fenced_buf
->mapcount
);
351 ret
= pb_validate(fenced_buf
->buffer
, vl
, flags
);
356 fenced_buf
->validation_flags
|= flags
;
363 fenced_buffer_fence(struct pb_buffer
*buf
,
364 struct pipe_fence_handle
*fence
)
366 struct fenced_buffer
*fenced_buf
;
367 struct fenced_buffer_list
*fenced_list
;
368 struct pb_fence_ops
*ops
;
370 fenced_buf
= fenced_buffer(buf
);
371 fenced_list
= fenced_buf
->list
;
372 ops
= fenced_list
->ops
;
374 if(fence
== fenced_buf
->fence
) {
379 assert(fenced_buf
->vl
);
380 assert(fenced_buf
->validation_flags
);
382 pipe_mutex_lock(fenced_list
->mutex
);
383 if (fenced_buf
->fence
)
384 _fenced_buffer_remove(fenced_list
, fenced_buf
);
386 ops
->fence_reference(ops
, &fenced_buf
->fence
, fence
);
387 fenced_buf
->flags
|= fenced_buf
->validation_flags
;
388 _fenced_buffer_add(fenced_buf
);
390 pipe_mutex_unlock(fenced_list
->mutex
);
392 pb_fence(fenced_buf
->buffer
, fence
);
394 fenced_buf
->vl
= NULL
;
395 fenced_buf
->validation_flags
= 0;
400 fenced_buffer_get_base_buffer(struct pb_buffer
*buf
,
401 struct pb_buffer
**base_buf
,
404 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
405 pb_get_base_buffer(fenced_buf
->buffer
, base_buf
, offset
);
409 static const struct pb_vtbl
410 fenced_buffer_vtbl
= {
411 fenced_buffer_destroy
,
414 fenced_buffer_validate
,
416 fenced_buffer_get_base_buffer
421 fenced_buffer_create(struct fenced_buffer_list
*fenced_list
,
422 struct pb_buffer
*buffer
)
424 struct fenced_buffer
*buf
;
429 buf
= CALLOC_STRUCT(fenced_buffer
);
431 pb_reference(&buffer
, NULL
);
435 buf
->base
.base
.refcount
= 1;
436 buf
->base
.base
.alignment
= buffer
->base
.alignment
;
437 buf
->base
.base
.usage
= buffer
->base
.usage
;
438 buf
->base
.base
.size
= buffer
->base
.size
;
440 buf
->base
.vtbl
= &fenced_buffer_vtbl
;
441 buf
->buffer
= buffer
;
442 buf
->list
= fenced_list
;
448 struct fenced_buffer_list
*
449 fenced_buffer_list_create(struct pb_fence_ops
*ops
)
451 struct fenced_buffer_list
*fenced_list
;
453 fenced_list
= CALLOC_STRUCT(fenced_buffer_list
);
457 fenced_list
->ops
= ops
;
459 LIST_INITHEAD(&fenced_list
->delayed
);
461 fenced_list
->numDelayed
= 0;
463 pipe_mutex_init(fenced_list
->mutex
);
470 fenced_buffer_list_check_free(struct fenced_buffer_list
*fenced_list
,
473 pipe_mutex_lock(fenced_list
->mutex
);
474 _fenced_buffer_list_check_free(fenced_list
, wait
);
475 pipe_mutex_unlock(fenced_list
->mutex
);
480 fenced_buffer_list_destroy(struct fenced_buffer_list
*fenced_list
)
482 pipe_mutex_lock(fenced_list
->mutex
);
484 /* Wait on outstanding fences */
485 while (fenced_list
->numDelayed
) {
486 pipe_mutex_unlock(fenced_list
->mutex
);
487 #if defined(PIPE_OS_LINUX)
490 _fenced_buffer_list_check_free(fenced_list
, 1);
491 pipe_mutex_lock(fenced_list
->mutex
);
494 pipe_mutex_unlock(fenced_list
->mutex
);
496 fenced_list
->ops
->destroy(fenced_list
->ops
);