1 /**************************************************************************
3 * Copyright 2007-2010 VMware, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
30 * Implementation of fenced buffers.
32 * \author Jose Fonseca <jfonseca-at-vmware-dot-com>
33 * \author Thomas Hellström <thellstrom-at-vmware-dot-com>
37 #include "pipe/p_config.h"
39 #if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) || defined(PIPE_OS_SOLARIS)
44 #include "pipe/p_compiler.h"
45 #include "pipe/p_defines.h"
46 #include "util/u_debug.h"
47 #include "os/os_thread.h"
48 #include "util/u_memory.h"
49 #include "util/list.h"
51 #include "pipebuffer/pb_buffer.h"
52 #include "pipebuffer/pb_bufmgr.h"
53 #include "pipebuffer/pb_buffer_fenced.h"
54 #include "vmw_screen.h"
58 * Convenience macro (type safe).
60 #define SUPER(__derived) (&(__derived)->base)
65 struct pb_manager base
;
66 struct pb_manager
*provider
;
67 struct pb_fence_ops
*ops
;
70 * Following members are mutable and protected by this mutex.
77 * All fenced buffers are placed in this listed, ordered from the oldest
78 * fence to the newest fence.
80 struct list_head fenced
;
83 struct list_head unfenced
;
92 * Wrapper around a pipe buffer which adds fencing and reference counting.
100 struct pb_buffer base
;
101 struct fenced_manager
*mgr
;
104 * Following members are mutable and protected by fenced_manager::mutex.
107 struct list_head head
;
110 * Buffer with storage.
112 struct pb_buffer
*buffer
;
116 * A bitmask of PB_USAGE_CPU/GPU_READ/WRITE describing the current
123 struct pb_validate
*vl
;
124 unsigned validation_flags
;
126 struct pipe_fence_handle
*fence
;
130 static INLINE
struct fenced_manager
*
131 fenced_manager(struct pb_manager
*mgr
)
134 return (struct fenced_manager
*)mgr
;
138 static INLINE
struct fenced_buffer
*
139 fenced_buffer(struct pb_buffer
*buf
)
142 return (struct fenced_buffer
*)buf
;
147 fenced_buffer_destroy_gpu_storage_locked(struct fenced_buffer
*fenced_buf
);
149 static enum pipe_error
150 fenced_buffer_create_gpu_storage_locked(struct fenced_manager
*fenced_mgr
,
151 struct fenced_buffer
*fenced_buf
,
152 const struct pb_desc
*desc
,
155 * Dump the fenced buffer list.
157 * Useful to understand failures to allocate buffers.
160 fenced_manager_dump_locked(struct fenced_manager
*fenced_mgr
)
163 struct pb_fence_ops
*ops
= fenced_mgr
->ops
;
164 struct list_head
*curr
, *next
;
165 struct fenced_buffer
*fenced_buf
;
167 debug_printf("%10s %7s %8s %7s %10s %s\n",
168 "buffer", "size", "refcount", "storage", "fence", "signalled");
170 curr
= fenced_mgr
->unfenced
.next
;
172 while(curr
!= &fenced_mgr
->unfenced
) {
173 fenced_buf
= LIST_ENTRY(struct fenced_buffer
, curr
, head
);
174 assert(!fenced_buf
->fence
);
175 debug_printf("%10p %7u %8u %7s\n",
177 fenced_buf
->base
.size
,
178 p_atomic_read(&fenced_buf
->base
.reference
.count
),
179 fenced_buf
->buffer
? "gpu" : "none");
184 curr
= fenced_mgr
->fenced
.next
;
186 while(curr
!= &fenced_mgr
->fenced
) {
188 fenced_buf
= LIST_ENTRY(struct fenced_buffer
, curr
, head
);
189 assert(fenced_buf
->buffer
);
190 signaled
= ops
->fence_signalled(ops
, fenced_buf
->fence
, 0);
191 debug_printf("%10p %7u %8u %7s %10p %s\n",
193 fenced_buf
->base
.size
,
194 p_atomic_read(&fenced_buf
->base
.reference
.count
),
196 (void *) fenced_buf
->fence
,
197 signaled
== 0 ? "y" : "n");
208 fenced_buffer_destroy_locked(struct fenced_manager
*fenced_mgr
,
209 struct fenced_buffer
*fenced_buf
)
211 assert(!pipe_is_referenced(&fenced_buf
->base
.reference
));
213 assert(!fenced_buf
->fence
);
214 assert(fenced_buf
->head
.prev
);
215 assert(fenced_buf
->head
.next
);
216 LIST_DEL(&fenced_buf
->head
);
217 assert(fenced_mgr
->num_unfenced
);
218 --fenced_mgr
->num_unfenced
;
220 fenced_buffer_destroy_gpu_storage_locked(fenced_buf
);
227 * Add the buffer to the fenced list.
229 * Reference count should be incremented before calling this function.
232 fenced_buffer_add_locked(struct fenced_manager
*fenced_mgr
,
233 struct fenced_buffer
*fenced_buf
)
235 assert(pipe_is_referenced(&fenced_buf
->base
.reference
));
236 assert(fenced_buf
->flags
& PB_USAGE_GPU_READ_WRITE
);
237 assert(fenced_buf
->fence
);
239 p_atomic_inc(&fenced_buf
->base
.reference
.count
);
241 LIST_DEL(&fenced_buf
->head
);
242 assert(fenced_mgr
->num_unfenced
);
243 --fenced_mgr
->num_unfenced
;
244 LIST_ADDTAIL(&fenced_buf
->head
, &fenced_mgr
->fenced
);
245 ++fenced_mgr
->num_fenced
;
250 * Remove the buffer from the fenced list, and potentially destroy the buffer
251 * if the reference count reaches zero.
253 * Returns TRUE if the buffer was detroyed.
255 static INLINE boolean
256 fenced_buffer_remove_locked(struct fenced_manager
*fenced_mgr
,
257 struct fenced_buffer
*fenced_buf
)
259 struct pb_fence_ops
*ops
= fenced_mgr
->ops
;
261 assert(fenced_buf
->fence
);
262 assert(fenced_buf
->mgr
== fenced_mgr
);
264 ops
->fence_reference(ops
, &fenced_buf
->fence
, NULL
);
265 fenced_buf
->flags
&= ~PB_USAGE_GPU_READ_WRITE
;
267 assert(fenced_buf
->head
.prev
);
268 assert(fenced_buf
->head
.next
);
270 LIST_DEL(&fenced_buf
->head
);
271 assert(fenced_mgr
->num_fenced
);
272 --fenced_mgr
->num_fenced
;
274 LIST_ADDTAIL(&fenced_buf
->head
, &fenced_mgr
->unfenced
);
275 ++fenced_mgr
->num_unfenced
;
277 if (p_atomic_dec_zero(&fenced_buf
->base
.reference
.count
)) {
278 fenced_buffer_destroy_locked(fenced_mgr
, fenced_buf
);
287 * Wait for the fence to expire, and remove it from the fenced list.
289 * This function will release and re-acquire the mutex, so any copy of mutable
290 * state must be discarded after calling it.
292 static INLINE
enum pipe_error
293 fenced_buffer_finish_locked(struct fenced_manager
*fenced_mgr
,
294 struct fenced_buffer
*fenced_buf
)
296 struct pb_fence_ops
*ops
= fenced_mgr
->ops
;
297 enum pipe_error ret
= PIPE_ERROR
;
300 debug_warning("waiting for GPU");
303 assert(pipe_is_referenced(&fenced_buf
->base
.reference
));
304 assert(fenced_buf
->fence
);
306 if(fenced_buf
->fence
) {
307 struct pipe_fence_handle
*fence
= NULL
;
311 ops
->fence_reference(ops
, &fence
, fenced_buf
->fence
);
313 pipe_mutex_unlock(fenced_mgr
->mutex
);
315 finished
= ops
->fence_finish(ops
, fenced_buf
->fence
, 0);
317 pipe_mutex_lock(fenced_mgr
->mutex
);
319 assert(pipe_is_referenced(&fenced_buf
->base
.reference
));
322 * Only proceed if the fence object didn't change in the meanwhile.
323 * Otherwise assume the work has been already carried out by another
324 * thread that re-acquired the lock before us.
326 proceed
= fence
== fenced_buf
->fence
? TRUE
: FALSE
;
328 ops
->fence_reference(ops
, &fence
, NULL
);
330 if(proceed
&& finished
== 0) {
332 * Remove from the fenced list
337 destroyed
= fenced_buffer_remove_locked(fenced_mgr
, fenced_buf
);
339 /* TODO: remove consequents buffers with the same fence? */
343 fenced_buf
->flags
&= ~PB_USAGE_GPU_READ_WRITE
;
354 * Remove as many fenced buffers from the fenced list as possible.
356 * Returns TRUE if at least one buffer was removed.
359 fenced_manager_check_signalled_locked(struct fenced_manager
*fenced_mgr
,
362 struct pb_fence_ops
*ops
= fenced_mgr
->ops
;
363 struct list_head
*curr
, *next
;
364 struct fenced_buffer
*fenced_buf
;
365 struct pipe_fence_handle
*prev_fence
= NULL
;
368 curr
= fenced_mgr
->fenced
.next
;
370 while(curr
!= &fenced_mgr
->fenced
) {
371 fenced_buf
= LIST_ENTRY(struct fenced_buffer
, curr
, head
);
373 if(fenced_buf
->fence
!= prev_fence
) {
377 signaled
= ops
->fence_finish(ops
, fenced_buf
->fence
, 0);
380 * Don't return just now. Instead preemptively check if the
381 * following buffers' fences already expired,
382 * without further waits.
387 signaled
= ops
->fence_signalled(ops
, fenced_buf
->fence
, 0);
394 prev_fence
= fenced_buf
->fence
;
397 /* This buffer's fence object is identical to the previous buffer's
398 * fence object, so no need to check the fence again.
400 assert(ops
->fence_signalled(ops
, fenced_buf
->fence
, 0) == 0);
403 fenced_buffer_remove_locked(fenced_mgr
, fenced_buf
);
416 * Destroy the GPU storage.
419 fenced_buffer_destroy_gpu_storage_locked(struct fenced_buffer
*fenced_buf
)
421 if(fenced_buf
->buffer
) {
422 pb_reference(&fenced_buf
->buffer
, NULL
);
428 * Try to create GPU storage for this buffer.
430 * This function is a shorthand around pb_manager::create_buffer for
431 * fenced_buffer_create_gpu_storage_locked()'s benefit.
433 static INLINE boolean
434 fenced_buffer_try_create_gpu_storage_locked(struct fenced_manager
*fenced_mgr
,
435 struct fenced_buffer
*fenced_buf
,
436 const struct pb_desc
*desc
)
438 struct pb_manager
*provider
= fenced_mgr
->provider
;
440 assert(!fenced_buf
->buffer
);
442 fenced_buf
->buffer
= provider
->create_buffer(fenced_mgr
->provider
,
443 fenced_buf
->size
, desc
);
444 return fenced_buf
->buffer
? TRUE
: FALSE
;
449 * Create GPU storage for this buffer.
451 static enum pipe_error
452 fenced_buffer_create_gpu_storage_locked(struct fenced_manager
*fenced_mgr
,
453 struct fenced_buffer
*fenced_buf
,
454 const struct pb_desc
*desc
,
457 assert(!fenced_buf
->buffer
);
460 * Check for signaled buffers before trying to allocate.
462 fenced_manager_check_signalled_locked(fenced_mgr
, FALSE
);
464 fenced_buffer_try_create_gpu_storage_locked(fenced_mgr
, fenced_buf
, desc
);
467 * Keep trying while there is some sort of progress:
468 * - fences are expiring,
469 * - or buffers are being being swapped out from GPU memory into CPU memory.
471 while(!fenced_buf
->buffer
&&
472 (fenced_manager_check_signalled_locked(fenced_mgr
, FALSE
))) {
473 fenced_buffer_try_create_gpu_storage_locked(fenced_mgr
, fenced_buf
,
477 if(!fenced_buf
->buffer
&& wait
) {
479 * Same as before, but this time around, wait to free buffers if
482 while(!fenced_buf
->buffer
&&
483 (fenced_manager_check_signalled_locked(fenced_mgr
, TRUE
))) {
484 fenced_buffer_try_create_gpu_storage_locked(fenced_mgr
, fenced_buf
,
489 if(!fenced_buf
->buffer
) {
491 fenced_manager_dump_locked(fenced_mgr
);
494 return PIPE_ERROR_OUT_OF_MEMORY
;
502 fenced_buffer_destroy(struct pb_buffer
*buf
)
504 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
505 struct fenced_manager
*fenced_mgr
= fenced_buf
->mgr
;
507 assert(!pipe_is_referenced(&fenced_buf
->base
.reference
));
509 pipe_mutex_lock(fenced_mgr
->mutex
);
511 fenced_buffer_destroy_locked(fenced_mgr
, fenced_buf
);
513 pipe_mutex_unlock(fenced_mgr
->mutex
);
518 fenced_buffer_map(struct pb_buffer
*buf
,
519 unsigned flags
, void *flush_ctx
)
521 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
522 struct fenced_manager
*fenced_mgr
= fenced_buf
->mgr
;
523 struct pb_fence_ops
*ops
= fenced_mgr
->ops
;
526 pipe_mutex_lock(fenced_mgr
->mutex
);
528 assert(!(flags
& PB_USAGE_GPU_READ_WRITE
));
533 while((fenced_buf
->flags
& PB_USAGE_GPU_WRITE
) ||
534 ((fenced_buf
->flags
& PB_USAGE_GPU_READ
) &&
535 (flags
& PB_USAGE_CPU_WRITE
))) {
538 * Don't wait for the GPU to finish accessing it,
539 * if blocking is forbidden.
541 if((flags
& PB_USAGE_DONTBLOCK
) &&
542 ops
->fence_signalled(ops
, fenced_buf
->fence
, 0) != 0) {
546 if (flags
& PB_USAGE_UNSYNCHRONIZED
) {
551 * Wait for the GPU to finish accessing. This will release and re-acquire
552 * the mutex, so all copies of mutable state must be discarded.
554 fenced_buffer_finish_locked(fenced_mgr
, fenced_buf
);
557 map
= pb_map(fenced_buf
->buffer
, flags
, flush_ctx
);
560 ++fenced_buf
->mapcount
;
561 fenced_buf
->flags
|= flags
& PB_USAGE_CPU_READ_WRITE
;
565 pipe_mutex_unlock(fenced_mgr
->mutex
);
572 fenced_buffer_unmap(struct pb_buffer
*buf
)
574 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
575 struct fenced_manager
*fenced_mgr
= fenced_buf
->mgr
;
577 pipe_mutex_lock(fenced_mgr
->mutex
);
579 assert(fenced_buf
->mapcount
);
580 if(fenced_buf
->mapcount
) {
581 if (fenced_buf
->buffer
)
582 pb_unmap(fenced_buf
->buffer
);
583 --fenced_buf
->mapcount
;
584 if(!fenced_buf
->mapcount
)
585 fenced_buf
->flags
&= ~PB_USAGE_CPU_READ_WRITE
;
588 pipe_mutex_unlock(fenced_mgr
->mutex
);
592 static enum pipe_error
593 fenced_buffer_validate(struct pb_buffer
*buf
,
594 struct pb_validate
*vl
,
597 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
598 struct fenced_manager
*fenced_mgr
= fenced_buf
->mgr
;
601 pipe_mutex_lock(fenced_mgr
->mutex
);
605 fenced_buf
->vl
= NULL
;
606 fenced_buf
->validation_flags
= 0;
611 assert(flags
& PB_USAGE_GPU_READ_WRITE
);
612 assert(!(flags
& ~PB_USAGE_GPU_READ_WRITE
));
613 flags
&= PB_USAGE_GPU_READ_WRITE
;
615 /* Buffer cannot be validated in two different lists */
616 if(fenced_buf
->vl
&& fenced_buf
->vl
!= vl
) {
617 ret
= PIPE_ERROR_RETRY
;
621 if(fenced_buf
->vl
== vl
&&
622 (fenced_buf
->validation_flags
& flags
) == flags
) {
623 /* Nothing to do -- buffer already validated */
628 ret
= pb_validate(fenced_buf
->buffer
, vl
, flags
);
633 fenced_buf
->validation_flags
|= flags
;
636 pipe_mutex_unlock(fenced_mgr
->mutex
);
643 fenced_buffer_fence(struct pb_buffer
*buf
,
644 struct pipe_fence_handle
*fence
)
646 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
647 struct fenced_manager
*fenced_mgr
= fenced_buf
->mgr
;
648 struct pb_fence_ops
*ops
= fenced_mgr
->ops
;
650 pipe_mutex_lock(fenced_mgr
->mutex
);
652 assert(pipe_is_referenced(&fenced_buf
->base
.reference
));
653 assert(fenced_buf
->buffer
);
655 if(fence
!= fenced_buf
->fence
) {
656 assert(fenced_buf
->vl
);
657 assert(fenced_buf
->validation_flags
);
659 if (fenced_buf
->fence
) {
661 destroyed
= fenced_buffer_remove_locked(fenced_mgr
, fenced_buf
);
665 ops
->fence_reference(ops
, &fenced_buf
->fence
, fence
);
666 fenced_buf
->flags
|= fenced_buf
->validation_flags
;
667 fenced_buffer_add_locked(fenced_mgr
, fenced_buf
);
670 pb_fence(fenced_buf
->buffer
, fence
);
672 fenced_buf
->vl
= NULL
;
673 fenced_buf
->validation_flags
= 0;
676 pipe_mutex_unlock(fenced_mgr
->mutex
);
681 fenced_buffer_get_base_buffer(struct pb_buffer
*buf
,
682 struct pb_buffer
**base_buf
,
685 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
686 struct fenced_manager
*fenced_mgr
= fenced_buf
->mgr
;
688 pipe_mutex_lock(fenced_mgr
->mutex
);
690 assert(fenced_buf
->buffer
);
692 if(fenced_buf
->buffer
)
693 pb_get_base_buffer(fenced_buf
->buffer
, base_buf
, offset
);
699 pipe_mutex_unlock(fenced_mgr
->mutex
);
703 static const struct pb_vtbl
704 fenced_buffer_vtbl
= {
705 fenced_buffer_destroy
,
708 fenced_buffer_validate
,
710 fenced_buffer_get_base_buffer
715 * Wrap a buffer in a fenced buffer.
717 static struct pb_buffer
*
718 fenced_bufmgr_create_buffer(struct pb_manager
*mgr
,
720 const struct pb_desc
*desc
)
722 struct fenced_manager
*fenced_mgr
= fenced_manager(mgr
);
723 struct fenced_buffer
*fenced_buf
;
726 fenced_buf
= CALLOC_STRUCT(fenced_buffer
);
730 pipe_reference_init(&fenced_buf
->base
.reference
, 1);
731 fenced_buf
->base
.alignment
= desc
->alignment
;
732 fenced_buf
->base
.usage
= desc
->usage
;
733 fenced_buf
->base
.size
= size
;
734 fenced_buf
->size
= size
;
736 fenced_buf
->base
.vtbl
= &fenced_buffer_vtbl
;
737 fenced_buf
->mgr
= fenced_mgr
;
739 pipe_mutex_lock(fenced_mgr
->mutex
);
742 * Try to create GPU storage without stalling,
744 ret
= fenced_buffer_create_gpu_storage_locked(fenced_mgr
, fenced_buf
,
754 assert(fenced_buf
->buffer
);
756 LIST_ADDTAIL(&fenced_buf
->head
, &fenced_mgr
->unfenced
);
757 ++fenced_mgr
->num_unfenced
;
758 pipe_mutex_unlock(fenced_mgr
->mutex
);
760 return &fenced_buf
->base
;
763 pipe_mutex_unlock(fenced_mgr
->mutex
);
771 fenced_bufmgr_flush(struct pb_manager
*mgr
)
773 struct fenced_manager
*fenced_mgr
= fenced_manager(mgr
);
775 pipe_mutex_lock(fenced_mgr
->mutex
);
776 while(fenced_manager_check_signalled_locked(fenced_mgr
, TRUE
))
778 pipe_mutex_unlock(fenced_mgr
->mutex
);
780 assert(fenced_mgr
->provider
->flush
);
781 if(fenced_mgr
->provider
->flush
)
782 fenced_mgr
->provider
->flush(fenced_mgr
->provider
);
787 fenced_bufmgr_destroy(struct pb_manager
*mgr
)
789 struct fenced_manager
*fenced_mgr
= fenced_manager(mgr
);
791 pipe_mutex_lock(fenced_mgr
->mutex
);
793 /* Wait on outstanding fences */
794 while (fenced_mgr
->num_fenced
) {
795 pipe_mutex_unlock(fenced_mgr
->mutex
);
796 #if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) || defined(PIPE_OS_SOLARIS)
799 pipe_mutex_lock(fenced_mgr
->mutex
);
800 while(fenced_manager_check_signalled_locked(fenced_mgr
, TRUE
))
805 /*assert(!fenced_mgr->num_unfenced);*/
808 pipe_mutex_unlock(fenced_mgr
->mutex
);
809 pipe_mutex_destroy(fenced_mgr
->mutex
);
816 simple_fenced_bufmgr_create(struct pb_manager
*provider
,
817 struct pb_fence_ops
*ops
)
819 struct fenced_manager
*fenced_mgr
;
824 fenced_mgr
= CALLOC_STRUCT(fenced_manager
);
828 fenced_mgr
->base
.destroy
= fenced_bufmgr_destroy
;
829 fenced_mgr
->base
.create_buffer
= fenced_bufmgr_create_buffer
;
830 fenced_mgr
->base
.flush
= fenced_bufmgr_flush
;
832 fenced_mgr
->provider
= provider
;
833 fenced_mgr
->ops
= ops
;
835 LIST_INITHEAD(&fenced_mgr
->fenced
);
836 fenced_mgr
->num_fenced
= 0;
838 LIST_INITHEAD(&fenced_mgr
->unfenced
);
839 fenced_mgr
->num_unfenced
= 0;
841 pipe_mutex_init(fenced_mgr
->mutex
);
843 return &fenced_mgr
->base
;