1 /**************************************************************************
3 * Copyright 2007-2009 VMware, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
30 * Implementation of fenced buffers.
32 * \author Jose Fonseca <jrfonseca-at-tungstengraphics-dot-com>
33 * \author Thomas Hellström <thomas-at-tungstengraphics-dot-com>
37 #include "pipe/p_config.h"
39 #if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) || defined(PIPE_OS_SOLARIS)
44 #include "pipe/p_compiler.h"
45 #include "pipe/p_defines.h"
46 #include "util/u_debug.h"
47 #include "pipe/p_thread.h"
48 #include "util/u_memory.h"
49 #include "util/u_double_list.h"
51 #include "pb_buffer.h"
52 #include "pb_buffer_fenced.h"
57 * Convenience macro (type safe).
59 #define SUPER(__derived) (&(__derived)->base)
62 struct fenced_buffer_list
66 struct pb_fence_ops
*ops
;
69 struct list_head delayed
;
73 struct list_head unfenced
;
79 * Wrapper around a pipe buffer which adds fencing and reference counting.
83 struct pb_buffer base
;
85 struct pb_buffer
*buffer
;
87 /* FIXME: protect access with mutex */
90 * A bitmask of PIPE_BUFFER_USAGE_CPU/GPU_READ/WRITE describing the current
96 struct pb_validate
*vl
;
97 unsigned validation_flags
;
98 struct pipe_fence_handle
*fence
;
100 struct list_head head
;
101 struct fenced_buffer_list
*list
;
105 static INLINE
struct fenced_buffer
*
106 fenced_buffer(struct pb_buffer
*buf
)
109 return (struct fenced_buffer
*)buf
;
114 _fenced_buffer_add(struct fenced_buffer
*fenced_buf
)
116 struct fenced_buffer_list
*fenced_list
= fenced_buf
->list
;
118 assert(pipe_is_referenced(&fenced_buf
->base
.base
.reference
));
119 assert(fenced_buf
->flags
& PIPE_BUFFER_USAGE_GPU_READ_WRITE
);
120 assert(fenced_buf
->fence
);
123 LIST_DEL(&fenced_buf
->head
);
124 assert(fenced_list
->numUnfenced
);
125 --fenced_list
->numUnfenced
;
127 LIST_ADDTAIL(&fenced_buf
->head
, &fenced_list
->delayed
);
128 ++fenced_list
->numDelayed
;
133 * Actually destroy the buffer.
136 _fenced_buffer_destroy(struct fenced_buffer
*fenced_buf
)
138 struct fenced_buffer_list
*fenced_list
= fenced_buf
->list
;
140 assert(!pipe_is_referenced(&fenced_buf
->base
.base
.reference
));
141 assert(!fenced_buf
->fence
);
143 assert(fenced_buf
->head
.prev
);
144 assert(fenced_buf
->head
.next
);
145 LIST_DEL(&fenced_buf
->head
);
146 assert(fenced_list
->numUnfenced
);
147 --fenced_list
->numUnfenced
;
151 pb_reference(&fenced_buf
->buffer
, NULL
);
157 _fenced_buffer_remove(struct fenced_buffer_list
*fenced_list
,
158 struct fenced_buffer
*fenced_buf
)
160 struct pb_fence_ops
*ops
= fenced_list
->ops
;
162 assert(fenced_buf
->fence
);
163 assert(fenced_buf
->list
== fenced_list
);
165 ops
->fence_reference(ops
, &fenced_buf
->fence
, NULL
);
166 fenced_buf
->flags
&= ~PIPE_BUFFER_USAGE_GPU_READ_WRITE
;
168 assert(fenced_buf
->head
.prev
);
169 assert(fenced_buf
->head
.next
);
171 LIST_DEL(&fenced_buf
->head
);
172 assert(fenced_list
->numDelayed
);
173 --fenced_list
->numDelayed
;
176 LIST_ADDTAIL(&fenced_buf
->head
, &fenced_list
->unfenced
);
177 ++fenced_list
->numUnfenced
;
184 if(!pipe_is_referenced(&fenced_buf
->base
.base
.reference
))
185 _fenced_buffer_destroy(fenced_buf
);
189 static INLINE
enum pipe_error
190 _fenced_buffer_finish(struct fenced_buffer
*fenced_buf
)
192 struct fenced_buffer_list
*fenced_list
= fenced_buf
->list
;
193 struct pb_fence_ops
*ops
= fenced_list
->ops
;
196 debug_warning("waiting for GPU");
199 assert(fenced_buf
->fence
);
200 if(fenced_buf
->fence
) {
201 if(ops
->fence_finish(ops
, fenced_buf
->fence
, 0) != 0) {
204 /* Remove from the fenced list */
205 /* TODO: remove consequents */
206 _fenced_buffer_remove(fenced_list
, fenced_buf
);
209 fenced_buf
->flags
&= ~PIPE_BUFFER_USAGE_GPU_READ_WRITE
;
215 * Free as many fenced buffers from the list head as possible.
218 _fenced_buffer_list_check_free(struct fenced_buffer_list
*fenced_list
,
221 struct pb_fence_ops
*ops
= fenced_list
->ops
;
222 struct list_head
*curr
, *next
;
223 struct fenced_buffer
*fenced_buf
;
224 struct pb_buffer
*pb_buf
;
225 struct pipe_fence_handle
*prev_fence
= NULL
;
227 curr
= fenced_list
->delayed
.next
;
229 while(curr
!= &fenced_list
->delayed
) {
230 fenced_buf
= LIST_ENTRY(struct fenced_buffer
, curr
, head
);
232 if(fenced_buf
->fence
!= prev_fence
) {
235 signaled
= ops
->fence_finish(ops
, fenced_buf
->fence
, 0);
237 signaled
= ops
->fence_signalled(ops
, fenced_buf
->fence
, 0);
240 prev_fence
= fenced_buf
->fence
;
243 assert(ops
->fence_signalled(ops
, fenced_buf
->fence
, 0) == 0);
246 _fenced_buffer_remove(fenced_list
, fenced_buf
);
255 fenced_buffer_destroy(struct pb_buffer
*buf
)
257 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
258 struct fenced_buffer_list
*fenced_list
= fenced_buf
->list
;
260 pipe_mutex_lock(fenced_list
->mutex
);
261 assert(!pipe_is_referenced(&fenced_buf
->base
.base
.reference
));
262 if (fenced_buf
->fence
) {
263 struct pb_fence_ops
*ops
= fenced_list
->ops
;
264 if(ops
->fence_signalled(ops
, fenced_buf
->fence
, 0) == 0) {
265 struct list_head
*curr
, *prev
;
266 curr
= &fenced_buf
->head
;
269 fenced_buf
= LIST_ENTRY(struct fenced_buffer
, curr
, head
);
270 assert(ops
->fence_signalled(ops
, fenced_buf
->fence
, 0) == 0);
271 _fenced_buffer_remove(fenced_list
, fenced_buf
);
274 } while (curr
!= &fenced_list
->delayed
);
277 /* delay destruction */
281 _fenced_buffer_destroy(fenced_buf
);
283 pipe_mutex_unlock(fenced_list
->mutex
);
288 fenced_buffer_map(struct pb_buffer
*buf
,
291 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
292 struct fenced_buffer_list
*fenced_list
= fenced_buf
->list
;
293 struct pb_fence_ops
*ops
= fenced_list
->ops
;
296 assert(!(flags
& PIPE_BUFFER_USAGE_GPU_READ_WRITE
));
298 /* Serialize writes */
299 if((fenced_buf
->flags
& PIPE_BUFFER_USAGE_GPU_WRITE
) ||
300 ((fenced_buf
->flags
& PIPE_BUFFER_USAGE_GPU_READ
) && (flags
& PIPE_BUFFER_USAGE_CPU_WRITE
))) {
301 if(flags
& PIPE_BUFFER_USAGE_DONTBLOCK
) {
302 /* Don't wait for the GPU to finish writing */
303 if(ops
->fence_signalled(ops
, fenced_buf
->fence
, 0) == 0)
304 _fenced_buffer_remove(fenced_list
, fenced_buf
);
309 /* Wait for the GPU to finish writing */
310 _fenced_buffer_finish(fenced_buf
);
315 /* Check for CPU write access (read is OK) */
316 if(fenced_buf
->flags
& PIPE_BUFFER_USAGE_CPU_READ_WRITE
) {
317 /* this is legal -- just for debugging */
318 debug_warning("concurrent CPU writes");
322 map
= pb_map(fenced_buf
->buffer
, flags
);
324 ++fenced_buf
->mapcount
;
325 fenced_buf
->flags
|= flags
& PIPE_BUFFER_USAGE_CPU_READ_WRITE
;
333 fenced_buffer_unmap(struct pb_buffer
*buf
)
335 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
336 assert(fenced_buf
->mapcount
);
337 if(fenced_buf
->mapcount
) {
338 pb_unmap(fenced_buf
->buffer
);
339 --fenced_buf
->mapcount
;
340 if(!fenced_buf
->mapcount
)
341 fenced_buf
->flags
&= ~PIPE_BUFFER_USAGE_CPU_READ_WRITE
;
346 static enum pipe_error
347 fenced_buffer_validate(struct pb_buffer
*buf
,
348 struct pb_validate
*vl
,
351 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
356 fenced_buf
->vl
= NULL
;
357 fenced_buf
->validation_flags
= 0;
361 assert(flags
& PIPE_BUFFER_USAGE_GPU_READ_WRITE
);
362 assert(!(flags
& ~PIPE_BUFFER_USAGE_GPU_READ_WRITE
));
363 flags
&= PIPE_BUFFER_USAGE_GPU_READ_WRITE
;
365 /* Buffer cannot be validated in two different lists */
366 if(fenced_buf
->vl
&& fenced_buf
->vl
!= vl
)
367 return PIPE_ERROR_RETRY
;
370 /* Do not validate if buffer is still mapped */
371 if(fenced_buf
->flags
& PIPE_BUFFER_USAGE_CPU_READ_WRITE
) {
372 /* TODO: wait for the thread that mapped the buffer to unmap it */
373 return PIPE_ERROR_RETRY
;
375 /* Final sanity checking */
376 assert(!(fenced_buf
->flags
& PIPE_BUFFER_USAGE_CPU_READ_WRITE
));
377 assert(!fenced_buf
->mapcount
);
380 if(fenced_buf
->vl
== vl
&&
381 (fenced_buf
->validation_flags
& flags
) == flags
) {
382 /* Nothing to do -- buffer already validated */
386 ret
= pb_validate(fenced_buf
->buffer
, vl
, flags
);
391 fenced_buf
->validation_flags
|= flags
;
398 fenced_buffer_fence(struct pb_buffer
*buf
,
399 struct pipe_fence_handle
*fence
)
401 struct fenced_buffer
*fenced_buf
;
402 struct fenced_buffer_list
*fenced_list
;
403 struct pb_fence_ops
*ops
;
405 fenced_buf
= fenced_buffer(buf
);
406 fenced_list
= fenced_buf
->list
;
407 ops
= fenced_list
->ops
;
409 if(fence
== fenced_buf
->fence
) {
414 assert(fenced_buf
->vl
);
415 assert(fenced_buf
->validation_flags
);
417 pipe_mutex_lock(fenced_list
->mutex
);
418 if (fenced_buf
->fence
)
419 _fenced_buffer_remove(fenced_list
, fenced_buf
);
421 ops
->fence_reference(ops
, &fenced_buf
->fence
, fence
);
422 fenced_buf
->flags
|= fenced_buf
->validation_flags
;
423 _fenced_buffer_add(fenced_buf
);
425 pipe_mutex_unlock(fenced_list
->mutex
);
427 pb_fence(fenced_buf
->buffer
, fence
);
429 fenced_buf
->vl
= NULL
;
430 fenced_buf
->validation_flags
= 0;
435 fenced_buffer_get_base_buffer(struct pb_buffer
*buf
,
436 struct pb_buffer
**base_buf
,
439 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
440 pb_get_base_buffer(fenced_buf
->buffer
, base_buf
, offset
);
444 static const struct pb_vtbl
445 fenced_buffer_vtbl
= {
446 fenced_buffer_destroy
,
449 fenced_buffer_validate
,
451 fenced_buffer_get_base_buffer
456 fenced_buffer_create(struct fenced_buffer_list
*fenced_list
,
457 struct pb_buffer
*buffer
)
459 struct fenced_buffer
*buf
;
464 buf
= CALLOC_STRUCT(fenced_buffer
);
466 pb_reference(&buffer
, NULL
);
470 pipe_reference_init(&buf
->base
.base
.reference
, 1);
471 buf
->base
.base
.alignment
= buffer
->base
.alignment
;
472 buf
->base
.base
.usage
= buffer
->base
.usage
;
473 buf
->base
.base
.size
= buffer
->base
.size
;
475 buf
->base
.vtbl
= &fenced_buffer_vtbl
;
476 buf
->buffer
= buffer
;
477 buf
->list
= fenced_list
;
480 pipe_mutex_lock(fenced_list
->mutex
);
481 LIST_ADDTAIL(&buf
->head
, &fenced_list
->unfenced
);
482 ++fenced_list
->numUnfenced
;
483 pipe_mutex_unlock(fenced_list
->mutex
);
490 struct fenced_buffer_list
*
491 fenced_buffer_list_create(struct pb_fence_ops
*ops
)
493 struct fenced_buffer_list
*fenced_list
;
495 fenced_list
= CALLOC_STRUCT(fenced_buffer_list
);
499 fenced_list
->ops
= ops
;
501 LIST_INITHEAD(&fenced_list
->delayed
);
502 fenced_list
->numDelayed
= 0;
505 LIST_INITHEAD(&fenced_list
->unfenced
);
506 fenced_list
->numUnfenced
= 0;
509 pipe_mutex_init(fenced_list
->mutex
);
516 fenced_buffer_list_check_free(struct fenced_buffer_list
*fenced_list
,
519 pipe_mutex_lock(fenced_list
->mutex
);
520 _fenced_buffer_list_check_free(fenced_list
, wait
);
521 pipe_mutex_unlock(fenced_list
->mutex
);
527 fenced_buffer_list_dump(struct fenced_buffer_list
*fenced_list
)
529 struct pb_fence_ops
*ops
= fenced_list
->ops
;
530 struct list_head
*curr
, *next
;
531 struct fenced_buffer
*fenced_buf
;
533 pipe_mutex_lock(fenced_list
->mutex
);
535 debug_printf("%10s %7s %7s %10s %s\n",
536 "buffer", "size", "refcount", "fence", "signalled");
538 curr
= fenced_list
->unfenced
.next
;
540 while(curr
!= &fenced_list
->unfenced
) {
541 fenced_buf
= LIST_ENTRY(struct fenced_buffer
, curr
, head
);
542 assert(!fenced_buf
->fence
);
543 debug_printf("%10p %7u %7u\n",
545 fenced_buf
->base
.base
.size
,
546 p_atomic_read(&fenced_buf
->base
.base
.reference
.count
));
551 curr
= fenced_list
->delayed
.next
;
553 while(curr
!= &fenced_list
->delayed
) {
555 fenced_buf
= LIST_ENTRY(struct fenced_buffer
, curr
, head
);
556 signaled
= ops
->fence_signalled(ops
, fenced_buf
->fence
, 0);
557 debug_printf("%10p %7u %7u %10p %s\n",
559 fenced_buf
->base
.base
.size
,
560 p_atomic_read(&fenced_buf
->base
.base
.reference
.count
),
561 (void *) fenced_buf
->fence
,
562 signaled
== 0 ? "y" : "n");
567 pipe_mutex_unlock(fenced_list
->mutex
);
573 fenced_buffer_list_destroy(struct fenced_buffer_list
*fenced_list
)
575 pipe_mutex_lock(fenced_list
->mutex
);
577 /* Wait on outstanding fences */
578 while (fenced_list
->numDelayed
) {
579 pipe_mutex_unlock(fenced_list
->mutex
);
580 #if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) || defined(PIPE_OS_SOLARIS)
583 _fenced_buffer_list_check_free(fenced_list
, 1);
584 pipe_mutex_lock(fenced_list
->mutex
);
588 /*assert(!fenced_list->numUnfenced);*/
591 pipe_mutex_unlock(fenced_list
->mutex
);
593 fenced_list
->ops
->destroy(fenced_list
->ops
);