1 /**************************************************************************
3 * Copyright 2007-2015 VMware, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
30 * Implementation of fenced buffers.
32 * \author Jose Fonseca <jfonseca-at-vmware-dot-com>
33 * \author Thomas Hellström <thellstrom-at-vmware-dot-com>
37 #include "pipe/p_config.h"
39 #if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) || defined(PIPE_OS_SOLARIS)
45 #include "pipe/p_compiler.h"
46 #include "pipe/p_defines.h"
47 #include "util/u_debug.h"
48 #include "os/os_thread.h"
49 #include "util/u_memory.h"
50 #include "util/list.h"
52 #include "pipebuffer/pb_buffer.h"
53 #include "pipebuffer/pb_bufmgr.h"
54 #include "pipebuffer/pb_buffer_fenced.h"
55 #include "vmw_screen.h"
59 * Convenience macro (type safe).
61 #define SUPER(__derived) (&(__derived)->base)
66 struct pb_manager base
;
67 struct pb_manager
*provider
;
68 struct pb_fence_ops
*ops
;
71 * Following members are mutable and protected by this mutex.
78 * All fenced buffers are placed in this listed, ordered from the oldest
79 * fence to the newest fence.
81 struct list_head fenced
;
84 struct list_head unfenced
;
93 * Wrapper around a pipe buffer which adds fencing and reference counting.
101 struct pb_buffer base
;
102 struct fenced_manager
*mgr
;
105 * Following members are mutable and protected by fenced_manager::mutex.
108 struct list_head head
;
111 * Buffer with storage.
113 struct pb_buffer
*buffer
;
117 * A bitmask of PB_USAGE_CPU/GPU_READ/WRITE describing the current
124 struct pb_validate
*vl
;
125 unsigned validation_flags
;
127 struct pipe_fence_handle
*fence
;
131 static inline struct fenced_manager
*
132 fenced_manager(struct pb_manager
*mgr
)
135 return (struct fenced_manager
*)mgr
;
139 static inline struct fenced_buffer
*
140 fenced_buffer(struct pb_buffer
*buf
)
143 return (struct fenced_buffer
*)buf
;
148 fenced_buffer_destroy_gpu_storage_locked(struct fenced_buffer
*fenced_buf
);
150 static enum pipe_error
151 fenced_buffer_create_gpu_storage_locked(struct fenced_manager
*fenced_mgr
,
152 struct fenced_buffer
*fenced_buf
,
153 const struct pb_desc
*desc
,
156 * Dump the fenced buffer list.
158 * Useful to understand failures to allocate buffers.
161 fenced_manager_dump_locked(struct fenced_manager
*fenced_mgr
)
164 struct pb_fence_ops
*ops
= fenced_mgr
->ops
;
165 struct list_head
*curr
, *next
;
166 struct fenced_buffer
*fenced_buf
;
168 debug_printf("%10s %7s %8s %7s %10s %s\n",
169 "buffer", "size", "refcount", "storage", "fence", "signalled");
171 curr
= fenced_mgr
->unfenced
.next
;
173 while(curr
!= &fenced_mgr
->unfenced
) {
174 fenced_buf
= LIST_ENTRY(struct fenced_buffer
, curr
, head
);
175 assert(!fenced_buf
->fence
);
176 debug_printf("%10p %"PRIu64
" %8u %7s\n",
178 fenced_buf
->base
.size
,
179 p_atomic_read(&fenced_buf
->base
.reference
.count
),
180 fenced_buf
->buffer
? "gpu" : "none");
185 curr
= fenced_mgr
->fenced
.next
;
187 while(curr
!= &fenced_mgr
->fenced
) {
189 fenced_buf
= LIST_ENTRY(struct fenced_buffer
, curr
, head
);
190 assert(fenced_buf
->buffer
);
191 signaled
= ops
->fence_signalled(ops
, fenced_buf
->fence
, 0);
192 debug_printf("%10p %"PRIu64
" %8u %7s %10p %s\n",
194 fenced_buf
->base
.size
,
195 p_atomic_read(&fenced_buf
->base
.reference
.count
),
197 (void *) fenced_buf
->fence
,
198 signaled
== 0 ? "y" : "n");
209 fenced_buffer_destroy_locked(struct fenced_manager
*fenced_mgr
,
210 struct fenced_buffer
*fenced_buf
)
212 assert(!pipe_is_referenced(&fenced_buf
->base
.reference
));
214 assert(!fenced_buf
->fence
);
215 assert(fenced_buf
->head
.prev
);
216 assert(fenced_buf
->head
.next
);
217 list_del(&fenced_buf
->head
);
218 assert(fenced_mgr
->num_unfenced
);
219 --fenced_mgr
->num_unfenced
;
221 fenced_buffer_destroy_gpu_storage_locked(fenced_buf
);
228 * Add the buffer to the fenced list.
230 * Reference count should be incremented before calling this function.
233 fenced_buffer_add_locked(struct fenced_manager
*fenced_mgr
,
234 struct fenced_buffer
*fenced_buf
)
236 assert(pipe_is_referenced(&fenced_buf
->base
.reference
));
237 assert(fenced_buf
->flags
& PB_USAGE_GPU_READ_WRITE
);
238 assert(fenced_buf
->fence
);
240 p_atomic_inc(&fenced_buf
->base
.reference
.count
);
242 list_del(&fenced_buf
->head
);
243 assert(fenced_mgr
->num_unfenced
);
244 --fenced_mgr
->num_unfenced
;
245 list_addtail(&fenced_buf
->head
, &fenced_mgr
->fenced
);
246 ++fenced_mgr
->num_fenced
;
251 * Remove the buffer from the fenced list, and potentially destroy the buffer
252 * if the reference count reaches zero.
254 * Returns TRUE if the buffer was detroyed.
256 static inline boolean
257 fenced_buffer_remove_locked(struct fenced_manager
*fenced_mgr
,
258 struct fenced_buffer
*fenced_buf
)
260 struct pb_fence_ops
*ops
= fenced_mgr
->ops
;
262 assert(fenced_buf
->fence
);
263 assert(fenced_buf
->mgr
== fenced_mgr
);
265 ops
->fence_reference(ops
, &fenced_buf
->fence
, NULL
);
266 fenced_buf
->flags
&= ~PB_USAGE_GPU_READ_WRITE
;
268 assert(fenced_buf
->head
.prev
);
269 assert(fenced_buf
->head
.next
);
271 list_del(&fenced_buf
->head
);
272 assert(fenced_mgr
->num_fenced
);
273 --fenced_mgr
->num_fenced
;
275 list_addtail(&fenced_buf
->head
, &fenced_mgr
->unfenced
);
276 ++fenced_mgr
->num_unfenced
;
278 if (p_atomic_dec_zero(&fenced_buf
->base
.reference
.count
)) {
279 fenced_buffer_destroy_locked(fenced_mgr
, fenced_buf
);
288 * Wait for the fence to expire, and remove it from the fenced list.
290 * This function will release and re-acquire the mutex, so any copy of mutable
291 * state must be discarded after calling it.
293 static inline enum pipe_error
294 fenced_buffer_finish_locked(struct fenced_manager
*fenced_mgr
,
295 struct fenced_buffer
*fenced_buf
)
297 struct pb_fence_ops
*ops
= fenced_mgr
->ops
;
298 enum pipe_error ret
= PIPE_ERROR
;
301 debug_warning("waiting for GPU");
304 assert(pipe_is_referenced(&fenced_buf
->base
.reference
));
305 assert(fenced_buf
->fence
);
307 if(fenced_buf
->fence
) {
308 struct pipe_fence_handle
*fence
= NULL
;
312 ops
->fence_reference(ops
, &fence
, fenced_buf
->fence
);
314 mtx_unlock(&fenced_mgr
->mutex
);
316 finished
= ops
->fence_finish(ops
, fenced_buf
->fence
, 0);
318 mtx_lock(&fenced_mgr
->mutex
);
320 assert(pipe_is_referenced(&fenced_buf
->base
.reference
));
323 * Only proceed if the fence object didn't change in the meanwhile.
324 * Otherwise assume the work has been already carried out by another
325 * thread that re-acquired the lock before us.
327 proceed
= fence
== fenced_buf
->fence
? TRUE
: FALSE
;
329 ops
->fence_reference(ops
, &fence
, NULL
);
331 if(proceed
&& finished
== 0) {
333 * Remove from the fenced list
338 destroyed
= fenced_buffer_remove_locked(fenced_mgr
, fenced_buf
);
340 /* TODO: remove consequents buffers with the same fence? */
345 fenced_buf
->flags
&= ~PB_USAGE_GPU_READ_WRITE
;
356 * Remove as many fenced buffers from the fenced list as possible.
358 * Returns TRUE if at least one buffer was removed.
361 fenced_manager_check_signalled_locked(struct fenced_manager
*fenced_mgr
,
364 struct pb_fence_ops
*ops
= fenced_mgr
->ops
;
365 struct list_head
*curr
, *next
;
366 struct fenced_buffer
*fenced_buf
;
367 struct pipe_fence_handle
*prev_fence
= NULL
;
370 curr
= fenced_mgr
->fenced
.next
;
372 while(curr
!= &fenced_mgr
->fenced
) {
373 fenced_buf
= LIST_ENTRY(struct fenced_buffer
, curr
, head
);
375 if(fenced_buf
->fence
!= prev_fence
) {
379 signaled
= ops
->fence_finish(ops
, fenced_buf
->fence
, 0);
382 * Don't return just now. Instead preemptively check if the
383 * following buffers' fences already expired,
384 * without further waits.
389 signaled
= ops
->fence_signalled(ops
, fenced_buf
->fence
, 0);
396 prev_fence
= fenced_buf
->fence
;
399 /* This buffer's fence object is identical to the previous buffer's
400 * fence object, so no need to check the fence again.
402 assert(ops
->fence_signalled(ops
, fenced_buf
->fence
, 0) == 0);
405 fenced_buffer_remove_locked(fenced_mgr
, fenced_buf
);
418 * Destroy the GPU storage.
421 fenced_buffer_destroy_gpu_storage_locked(struct fenced_buffer
*fenced_buf
)
423 if(fenced_buf
->buffer
) {
424 pb_reference(&fenced_buf
->buffer
, NULL
);
430 * Try to create GPU storage for this buffer.
432 * This function is a shorthand around pb_manager::create_buffer for
433 * fenced_buffer_create_gpu_storage_locked()'s benefit.
435 static inline boolean
436 fenced_buffer_try_create_gpu_storage_locked(struct fenced_manager
*fenced_mgr
,
437 struct fenced_buffer
*fenced_buf
,
438 const struct pb_desc
*desc
)
440 struct pb_manager
*provider
= fenced_mgr
->provider
;
442 assert(!fenced_buf
->buffer
);
444 fenced_buf
->buffer
= provider
->create_buffer(fenced_mgr
->provider
,
445 fenced_buf
->size
, desc
);
446 return fenced_buf
->buffer
? TRUE
: FALSE
;
451 * Create GPU storage for this buffer.
453 static enum pipe_error
454 fenced_buffer_create_gpu_storage_locked(struct fenced_manager
*fenced_mgr
,
455 struct fenced_buffer
*fenced_buf
,
456 const struct pb_desc
*desc
,
459 assert(!fenced_buf
->buffer
);
462 * Check for signaled buffers before trying to allocate.
464 fenced_manager_check_signalled_locked(fenced_mgr
, FALSE
);
466 fenced_buffer_try_create_gpu_storage_locked(fenced_mgr
, fenced_buf
, desc
);
469 * Keep trying while there is some sort of progress:
470 * - fences are expiring,
471 * - or buffers are being being swapped out from GPU memory into CPU memory.
473 while(!fenced_buf
->buffer
&&
474 (fenced_manager_check_signalled_locked(fenced_mgr
, FALSE
))) {
475 fenced_buffer_try_create_gpu_storage_locked(fenced_mgr
, fenced_buf
,
479 if(!fenced_buf
->buffer
&& wait
) {
481 * Same as before, but this time around, wait to free buffers if
484 while(!fenced_buf
->buffer
&&
485 (fenced_manager_check_signalled_locked(fenced_mgr
, TRUE
))) {
486 fenced_buffer_try_create_gpu_storage_locked(fenced_mgr
, fenced_buf
,
491 if(!fenced_buf
->buffer
) {
493 fenced_manager_dump_locked(fenced_mgr
);
496 return PIPE_ERROR_OUT_OF_MEMORY
;
504 fenced_buffer_destroy(struct pb_buffer
*buf
)
506 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
507 struct fenced_manager
*fenced_mgr
= fenced_buf
->mgr
;
509 assert(!pipe_is_referenced(&fenced_buf
->base
.reference
));
511 mtx_lock(&fenced_mgr
->mutex
);
513 fenced_buffer_destroy_locked(fenced_mgr
, fenced_buf
);
515 mtx_unlock(&fenced_mgr
->mutex
);
520 fenced_buffer_map(struct pb_buffer
*buf
,
521 unsigned flags
, void *flush_ctx
)
523 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
524 struct fenced_manager
*fenced_mgr
= fenced_buf
->mgr
;
525 struct pb_fence_ops
*ops
= fenced_mgr
->ops
;
528 mtx_lock(&fenced_mgr
->mutex
);
530 assert(!(flags
& PB_USAGE_GPU_READ_WRITE
));
535 while((fenced_buf
->flags
& PB_USAGE_GPU_WRITE
) ||
536 ((fenced_buf
->flags
& PB_USAGE_GPU_READ
) &&
537 (flags
& PB_USAGE_CPU_WRITE
))) {
540 * Don't wait for the GPU to finish accessing it,
541 * if blocking is forbidden.
543 if((flags
& PB_USAGE_DONTBLOCK
) &&
544 ops
->fence_signalled(ops
, fenced_buf
->fence
, 0) != 0) {
548 if (flags
& PB_USAGE_UNSYNCHRONIZED
) {
553 * Wait for the GPU to finish accessing. This will release and re-acquire
554 * the mutex, so all copies of mutable state must be discarded.
556 fenced_buffer_finish_locked(fenced_mgr
, fenced_buf
);
559 map
= pb_map(fenced_buf
->buffer
, flags
, flush_ctx
);
562 ++fenced_buf
->mapcount
;
563 fenced_buf
->flags
|= flags
& PB_USAGE_CPU_READ_WRITE
;
567 mtx_unlock(&fenced_mgr
->mutex
);
574 fenced_buffer_unmap(struct pb_buffer
*buf
)
576 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
577 struct fenced_manager
*fenced_mgr
= fenced_buf
->mgr
;
579 mtx_lock(&fenced_mgr
->mutex
);
581 assert(fenced_buf
->mapcount
);
582 if(fenced_buf
->mapcount
) {
583 if (fenced_buf
->buffer
)
584 pb_unmap(fenced_buf
->buffer
);
585 --fenced_buf
->mapcount
;
586 if(!fenced_buf
->mapcount
)
587 fenced_buf
->flags
&= ~PB_USAGE_CPU_READ_WRITE
;
590 mtx_unlock(&fenced_mgr
->mutex
);
594 static enum pipe_error
595 fenced_buffer_validate(struct pb_buffer
*buf
,
596 struct pb_validate
*vl
,
599 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
600 struct fenced_manager
*fenced_mgr
= fenced_buf
->mgr
;
603 mtx_lock(&fenced_mgr
->mutex
);
607 fenced_buf
->vl
= NULL
;
608 fenced_buf
->validation_flags
= 0;
613 assert(flags
& PB_USAGE_GPU_READ_WRITE
);
614 assert(!(flags
& ~PB_USAGE_GPU_READ_WRITE
));
615 flags
&= PB_USAGE_GPU_READ_WRITE
;
617 /* Buffer cannot be validated in two different lists */
618 if(fenced_buf
->vl
&& fenced_buf
->vl
!= vl
) {
619 ret
= PIPE_ERROR_RETRY
;
623 if(fenced_buf
->vl
== vl
&&
624 (fenced_buf
->validation_flags
& flags
) == flags
) {
625 /* Nothing to do -- buffer already validated */
630 ret
= pb_validate(fenced_buf
->buffer
, vl
, flags
);
635 fenced_buf
->validation_flags
|= flags
;
638 mtx_unlock(&fenced_mgr
->mutex
);
645 fenced_buffer_fence(struct pb_buffer
*buf
,
646 struct pipe_fence_handle
*fence
)
648 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
649 struct fenced_manager
*fenced_mgr
= fenced_buf
->mgr
;
650 struct pb_fence_ops
*ops
= fenced_mgr
->ops
;
652 mtx_lock(&fenced_mgr
->mutex
);
654 assert(pipe_is_referenced(&fenced_buf
->base
.reference
));
655 assert(fenced_buf
->buffer
);
657 if(fence
!= fenced_buf
->fence
) {
658 assert(fenced_buf
->vl
);
659 assert(fenced_buf
->validation_flags
);
661 if (fenced_buf
->fence
) {
663 destroyed
= fenced_buffer_remove_locked(fenced_mgr
, fenced_buf
);
668 ops
->fence_reference(ops
, &fenced_buf
->fence
, fence
);
669 fenced_buf
->flags
|= fenced_buf
->validation_flags
;
670 fenced_buffer_add_locked(fenced_mgr
, fenced_buf
);
673 pb_fence(fenced_buf
->buffer
, fence
);
675 fenced_buf
->vl
= NULL
;
676 fenced_buf
->validation_flags
= 0;
679 mtx_unlock(&fenced_mgr
->mutex
);
684 fenced_buffer_get_base_buffer(struct pb_buffer
*buf
,
685 struct pb_buffer
**base_buf
,
688 struct fenced_buffer
*fenced_buf
= fenced_buffer(buf
);
689 struct fenced_manager
*fenced_mgr
= fenced_buf
->mgr
;
691 mtx_lock(&fenced_mgr
->mutex
);
693 assert(fenced_buf
->buffer
);
695 if(fenced_buf
->buffer
)
696 pb_get_base_buffer(fenced_buf
->buffer
, base_buf
, offset
);
702 mtx_unlock(&fenced_mgr
->mutex
);
706 static const struct pb_vtbl
707 fenced_buffer_vtbl
= {
708 fenced_buffer_destroy
,
711 fenced_buffer_validate
,
713 fenced_buffer_get_base_buffer
718 * Wrap a buffer in a fenced buffer.
720 static struct pb_buffer
*
721 fenced_bufmgr_create_buffer(struct pb_manager
*mgr
,
723 const struct pb_desc
*desc
)
725 struct fenced_manager
*fenced_mgr
= fenced_manager(mgr
);
726 struct fenced_buffer
*fenced_buf
;
729 fenced_buf
= CALLOC_STRUCT(fenced_buffer
);
733 pipe_reference_init(&fenced_buf
->base
.reference
, 1);
734 fenced_buf
->base
.alignment
= desc
->alignment
;
735 fenced_buf
->base
.usage
= desc
->usage
;
736 fenced_buf
->base
.size
= size
;
737 fenced_buf
->size
= size
;
739 fenced_buf
->base
.vtbl
= &fenced_buffer_vtbl
;
740 fenced_buf
->mgr
= fenced_mgr
;
742 mtx_lock(&fenced_mgr
->mutex
);
745 * Try to create GPU storage without stalling,
747 ret
= fenced_buffer_create_gpu_storage_locked(fenced_mgr
, fenced_buf
,
757 assert(fenced_buf
->buffer
);
759 list_addtail(&fenced_buf
->head
, &fenced_mgr
->unfenced
);
760 ++fenced_mgr
->num_unfenced
;
761 mtx_unlock(&fenced_mgr
->mutex
);
763 return &fenced_buf
->base
;
766 mtx_unlock(&fenced_mgr
->mutex
);
774 fenced_bufmgr_flush(struct pb_manager
*mgr
)
776 struct fenced_manager
*fenced_mgr
= fenced_manager(mgr
);
778 mtx_lock(&fenced_mgr
->mutex
);
779 while(fenced_manager_check_signalled_locked(fenced_mgr
, TRUE
))
781 mtx_unlock(&fenced_mgr
->mutex
);
783 assert(fenced_mgr
->provider
->flush
);
784 if(fenced_mgr
->provider
->flush
)
785 fenced_mgr
->provider
->flush(fenced_mgr
->provider
);
790 fenced_bufmgr_destroy(struct pb_manager
*mgr
)
792 struct fenced_manager
*fenced_mgr
= fenced_manager(mgr
);
794 mtx_lock(&fenced_mgr
->mutex
);
796 /* Wait on outstanding fences */
797 while (fenced_mgr
->num_fenced
) {
798 mtx_unlock(&fenced_mgr
->mutex
);
799 #if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) || defined(PIPE_OS_SOLARIS)
802 mtx_lock(&fenced_mgr
->mutex
);
803 while(fenced_manager_check_signalled_locked(fenced_mgr
, TRUE
))
808 /*assert(!fenced_mgr->num_unfenced);*/
811 mtx_unlock(&fenced_mgr
->mutex
);
812 mtx_destroy(&fenced_mgr
->mutex
);
819 simple_fenced_bufmgr_create(struct pb_manager
*provider
,
820 struct pb_fence_ops
*ops
)
822 struct fenced_manager
*fenced_mgr
;
827 fenced_mgr
= CALLOC_STRUCT(fenced_manager
);
831 fenced_mgr
->base
.destroy
= fenced_bufmgr_destroy
;
832 fenced_mgr
->base
.create_buffer
= fenced_bufmgr_create_buffer
;
833 fenced_mgr
->base
.flush
= fenced_bufmgr_flush
;
835 fenced_mgr
->provider
= provider
;
836 fenced_mgr
->ops
= ops
;
838 list_inithead(&fenced_mgr
->fenced
);
839 fenced_mgr
->num_fenced
= 0;
841 list_inithead(&fenced_mgr
->unfenced
);
842 fenced_mgr
->num_unfenced
= 0;
844 (void) mtx_init(&fenced_mgr
->mutex
, mtx_plain
);
846 return &fenced_mgr
->base
;