Merge branch 'mesa_7_6_branch' into mesa_7_7_branch
[mesa.git] / src / gallium / auxiliary / pipebuffer / pb_buffer_fenced.c
1 /**************************************************************************
2 *
3 * Copyright 2007 Tungsten Graphics, Inc., Cedar Park, Texas.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 /**
29 * \file
30 * Implementation of fenced buffers.
31 *
32 * \author Jose Fonseca <jrfonseca-at-tungstengraphics-dot-com>
33 * \author Thomas Hellström <thomas-at-tungstengraphics-dot-com>
34 */
35
36
37 #include "pipe/p_config.h"
38
39 #if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) || defined(PIPE_OS_SOLARIS)
40 #include <unistd.h>
41 #include <sched.h>
42 #endif
43
44 #include "pipe/p_compiler.h"
45 #include "pipe/p_defines.h"
46 #include "util/u_debug.h"
47 #include "pipe/p_thread.h"
48 #include "util/u_memory.h"
49 #include "util/u_double_list.h"
50
51 #include "pb_buffer.h"
52 #include "pb_buffer_fenced.h"
53
54
55
56 /**
57 * Convenience macro (type safe).
58 */
59 #define SUPER(__derived) (&(__derived)->base)
60
61
62 struct fenced_buffer_list
63 {
64 pipe_mutex mutex;
65
66 struct pb_fence_ops *ops;
67
68 pb_size numDelayed;
69 struct list_head delayed;
70
71 #ifdef DEBUG
72 pb_size numUnfenced;
73 struct list_head unfenced;
74 #endif
75 };
76
77
78 /**
79 * Wrapper around a pipe buffer which adds fencing and reference counting.
80 */
81 struct fenced_buffer
82 {
83 struct pb_buffer base;
84
85 struct pb_buffer *buffer;
86
87 /* FIXME: protect access with mutex */
88
89 /**
90 * A bitmask of PIPE_BUFFER_USAGE_CPU/GPU_READ/WRITE describing the current
91 * buffer usage.
92 */
93 unsigned flags;
94
95 unsigned mapcount;
96 struct pb_validate *vl;
97 unsigned validation_flags;
98 struct pipe_fence_handle *fence;
99
100 struct list_head head;
101 struct fenced_buffer_list *list;
102 };
103
104
105 static INLINE struct fenced_buffer *
106 fenced_buffer(struct pb_buffer *buf)
107 {
108 assert(buf);
109 return (struct fenced_buffer *)buf;
110 }
111
112
113 static INLINE void
114 _fenced_buffer_add(struct fenced_buffer *fenced_buf)
115 {
116 struct fenced_buffer_list *fenced_list = fenced_buf->list;
117
118 assert(pipe_is_referenced(&fenced_buf->base.base.reference));
119 assert(fenced_buf->flags & PIPE_BUFFER_USAGE_GPU_READ_WRITE);
120 assert(fenced_buf->fence);
121
122 #ifdef DEBUG
123 LIST_DEL(&fenced_buf->head);
124 assert(fenced_list->numUnfenced);
125 --fenced_list->numUnfenced;
126 #endif
127 LIST_ADDTAIL(&fenced_buf->head, &fenced_list->delayed);
128 ++fenced_list->numDelayed;
129 }
130
131
132 /**
133 * Actually destroy the buffer.
134 */
135 static INLINE void
136 _fenced_buffer_destroy(struct fenced_buffer *fenced_buf)
137 {
138 struct fenced_buffer_list *fenced_list = fenced_buf->list;
139
140 assert(!pipe_is_referenced(&fenced_buf->base.base.reference));
141 assert(!fenced_buf->fence);
142 #ifdef DEBUG
143 assert(fenced_buf->head.prev);
144 assert(fenced_buf->head.next);
145 LIST_DEL(&fenced_buf->head);
146 assert(fenced_list->numUnfenced);
147 --fenced_list->numUnfenced;
148 #else
149 (void)fenced_list;
150 #endif
151 pb_reference(&fenced_buf->buffer, NULL);
152 FREE(fenced_buf);
153 }
154
155
156 static INLINE void
157 _fenced_buffer_remove(struct fenced_buffer_list *fenced_list,
158 struct fenced_buffer *fenced_buf)
159 {
160 struct pb_fence_ops *ops = fenced_list->ops;
161
162 assert(fenced_buf->fence);
163 assert(fenced_buf->list == fenced_list);
164
165 ops->fence_reference(ops, &fenced_buf->fence, NULL);
166 fenced_buf->flags &= ~PIPE_BUFFER_USAGE_GPU_READ_WRITE;
167
168 assert(fenced_buf->head.prev);
169 assert(fenced_buf->head.next);
170
171 LIST_DEL(&fenced_buf->head);
172 assert(fenced_list->numDelayed);
173 --fenced_list->numDelayed;
174
175 #ifdef DEBUG
176 LIST_ADDTAIL(&fenced_buf->head, &fenced_list->unfenced);
177 ++fenced_list->numUnfenced;
178 #endif
179
180 /**
181 * FIXME!!!
182 */
183
184 if(!pipe_is_referenced(&fenced_buf->base.base.reference))
185 _fenced_buffer_destroy(fenced_buf);
186 }
187
188
189 static INLINE enum pipe_error
190 _fenced_buffer_finish(struct fenced_buffer *fenced_buf)
191 {
192 struct fenced_buffer_list *fenced_list = fenced_buf->list;
193 struct pb_fence_ops *ops = fenced_list->ops;
194
195 #if 0
196 debug_warning("waiting for GPU");
197 #endif
198
199 assert(fenced_buf->fence);
200 if(fenced_buf->fence) {
201 if(ops->fence_finish(ops, fenced_buf->fence, 0) != 0) {
202 return PIPE_ERROR;
203 }
204 /* Remove from the fenced list */
205 /* TODO: remove consequents */
206 _fenced_buffer_remove(fenced_list, fenced_buf);
207 }
208
209 fenced_buf->flags &= ~PIPE_BUFFER_USAGE_GPU_READ_WRITE;
210 return PIPE_OK;
211 }
212
213
214 /**
215 * Free as many fenced buffers from the list head as possible.
216 */
217 static void
218 _fenced_buffer_list_check_free(struct fenced_buffer_list *fenced_list,
219 int wait)
220 {
221 struct pb_fence_ops *ops = fenced_list->ops;
222 struct list_head *curr, *next;
223 struct fenced_buffer *fenced_buf;
224 struct pipe_fence_handle *prev_fence = NULL;
225
226 curr = fenced_list->delayed.next;
227 next = curr->next;
228 while(curr != &fenced_list->delayed) {
229 fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head);
230
231 if(fenced_buf->fence != prev_fence) {
232 int signaled;
233 if (wait)
234 signaled = ops->fence_finish(ops, fenced_buf->fence, 0);
235 else
236 signaled = ops->fence_signalled(ops, fenced_buf->fence, 0);
237 if (signaled != 0)
238 break;
239 prev_fence = fenced_buf->fence;
240 }
241 else {
242 assert(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0);
243 }
244
245 _fenced_buffer_remove(fenced_list, fenced_buf);
246
247 curr = next;
248 next = curr->next;
249 }
250 }
251
252
253 static void
254 fenced_buffer_destroy(struct pb_buffer *buf)
255 {
256 struct fenced_buffer *fenced_buf = fenced_buffer(buf);
257 struct fenced_buffer_list *fenced_list = fenced_buf->list;
258
259 pipe_mutex_lock(fenced_list->mutex);
260 assert(!pipe_is_referenced(&fenced_buf->base.base.reference));
261 if (fenced_buf->fence) {
262 struct pb_fence_ops *ops = fenced_list->ops;
263 if(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0) {
264 struct list_head *curr, *prev;
265 curr = &fenced_buf->head;
266 prev = curr->prev;
267 do {
268 fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head);
269 assert(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0);
270 _fenced_buffer_remove(fenced_list, fenced_buf);
271 curr = prev;
272 prev = curr->prev;
273 } while (curr != &fenced_list->delayed);
274 }
275 else {
276 /* delay destruction */
277 }
278 }
279 else {
280 _fenced_buffer_destroy(fenced_buf);
281 }
282 pipe_mutex_unlock(fenced_list->mutex);
283 }
284
285
286 static void *
287 fenced_buffer_map(struct pb_buffer *buf,
288 unsigned flags)
289 {
290 struct fenced_buffer *fenced_buf = fenced_buffer(buf);
291 struct fenced_buffer_list *fenced_list = fenced_buf->list;
292 struct pb_fence_ops *ops = fenced_list->ops;
293 void *map;
294
295 assert(!(flags & PIPE_BUFFER_USAGE_GPU_READ_WRITE));
296
297 /* Serialize writes */
298 if((fenced_buf->flags & PIPE_BUFFER_USAGE_GPU_WRITE) ||
299 ((fenced_buf->flags & PIPE_BUFFER_USAGE_GPU_READ) && (flags & PIPE_BUFFER_USAGE_CPU_WRITE))) {
300 if(flags & PIPE_BUFFER_USAGE_DONTBLOCK) {
301 /* Don't wait for the GPU to finish writing */
302 if(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0)
303 _fenced_buffer_remove(fenced_list, fenced_buf);
304 else
305 return NULL;
306 }
307 else {
308 /* Wait for the GPU to finish writing */
309 _fenced_buffer_finish(fenced_buf);
310 }
311 }
312
313 #if 0
314 /* Check for CPU write access (read is OK) */
315 if(fenced_buf->flags & PIPE_BUFFER_USAGE_CPU_READ_WRITE) {
316 /* this is legal -- just for debugging */
317 debug_warning("concurrent CPU writes");
318 }
319 #endif
320
321 map = pb_map(fenced_buf->buffer, flags);
322 if(map) {
323 ++fenced_buf->mapcount;
324 fenced_buf->flags |= flags & PIPE_BUFFER_USAGE_CPU_READ_WRITE;
325 }
326
327 return map;
328 }
329
330
331 static void
332 fenced_buffer_unmap(struct pb_buffer *buf)
333 {
334 struct fenced_buffer *fenced_buf = fenced_buffer(buf);
335 assert(fenced_buf->mapcount);
336 if(fenced_buf->mapcount) {
337 pb_unmap(fenced_buf->buffer);
338 --fenced_buf->mapcount;
339 if(!fenced_buf->mapcount)
340 fenced_buf->flags &= ~PIPE_BUFFER_USAGE_CPU_READ_WRITE;
341 }
342 }
343
344
345 static enum pipe_error
346 fenced_buffer_validate(struct pb_buffer *buf,
347 struct pb_validate *vl,
348 unsigned flags)
349 {
350 struct fenced_buffer *fenced_buf = fenced_buffer(buf);
351 enum pipe_error ret;
352
353 if(!vl) {
354 /* invalidate */
355 fenced_buf->vl = NULL;
356 fenced_buf->validation_flags = 0;
357 return PIPE_OK;
358 }
359
360 assert(flags & PIPE_BUFFER_USAGE_GPU_READ_WRITE);
361 assert(!(flags & ~PIPE_BUFFER_USAGE_GPU_READ_WRITE));
362 flags &= PIPE_BUFFER_USAGE_GPU_READ_WRITE;
363
364 /* Buffer cannot be validated in two different lists */
365 if(fenced_buf->vl && fenced_buf->vl != vl)
366 return PIPE_ERROR_RETRY;
367
368 #if 0
369 /* Do not validate if buffer is still mapped */
370 if(fenced_buf->flags & PIPE_BUFFER_USAGE_CPU_READ_WRITE) {
371 /* TODO: wait for the thread that mapped the buffer to unmap it */
372 return PIPE_ERROR_RETRY;
373 }
374 /* Final sanity checking */
375 assert(!(fenced_buf->flags & PIPE_BUFFER_USAGE_CPU_READ_WRITE));
376 assert(!fenced_buf->mapcount);
377 #endif
378
379 if(fenced_buf->vl == vl &&
380 (fenced_buf->validation_flags & flags) == flags) {
381 /* Nothing to do -- buffer already validated */
382 return PIPE_OK;
383 }
384
385 ret = pb_validate(fenced_buf->buffer, vl, flags);
386 if (ret != PIPE_OK)
387 return ret;
388
389 fenced_buf->vl = vl;
390 fenced_buf->validation_flags |= flags;
391
392 return PIPE_OK;
393 }
394
395
396 static void
397 fenced_buffer_fence(struct pb_buffer *buf,
398 struct pipe_fence_handle *fence)
399 {
400 struct fenced_buffer *fenced_buf;
401 struct fenced_buffer_list *fenced_list;
402 struct pb_fence_ops *ops;
403
404 fenced_buf = fenced_buffer(buf);
405 fenced_list = fenced_buf->list;
406 ops = fenced_list->ops;
407
408 if(fence == fenced_buf->fence) {
409 /* Nothing to do */
410 return;
411 }
412
413 assert(fenced_buf->vl);
414 assert(fenced_buf->validation_flags);
415
416 pipe_mutex_lock(fenced_list->mutex);
417 if (fenced_buf->fence)
418 _fenced_buffer_remove(fenced_list, fenced_buf);
419 if (fence) {
420 ops->fence_reference(ops, &fenced_buf->fence, fence);
421 fenced_buf->flags |= fenced_buf->validation_flags;
422 _fenced_buffer_add(fenced_buf);
423 }
424 pipe_mutex_unlock(fenced_list->mutex);
425
426 pb_fence(fenced_buf->buffer, fence);
427
428 fenced_buf->vl = NULL;
429 fenced_buf->validation_flags = 0;
430 }
431
432
433 static void
434 fenced_buffer_get_base_buffer(struct pb_buffer *buf,
435 struct pb_buffer **base_buf,
436 pb_size *offset)
437 {
438 struct fenced_buffer *fenced_buf = fenced_buffer(buf);
439 pb_get_base_buffer(fenced_buf->buffer, base_buf, offset);
440 }
441
442
443 static const struct pb_vtbl
444 fenced_buffer_vtbl = {
445 fenced_buffer_destroy,
446 fenced_buffer_map,
447 fenced_buffer_unmap,
448 fenced_buffer_validate,
449 fenced_buffer_fence,
450 fenced_buffer_get_base_buffer
451 };
452
453
454 struct pb_buffer *
455 fenced_buffer_create(struct fenced_buffer_list *fenced_list,
456 struct pb_buffer *buffer)
457 {
458 struct fenced_buffer *buf;
459
460 if(!buffer)
461 return NULL;
462
463 buf = CALLOC_STRUCT(fenced_buffer);
464 if(!buf) {
465 pb_reference(&buffer, NULL);
466 return NULL;
467 }
468
469 pipe_reference_init(&buf->base.base.reference, 1);
470 buf->base.base.alignment = buffer->base.alignment;
471 buf->base.base.usage = buffer->base.usage;
472 buf->base.base.size = buffer->base.size;
473
474 buf->base.vtbl = &fenced_buffer_vtbl;
475 buf->buffer = buffer;
476 buf->list = fenced_list;
477
478 #ifdef DEBUG
479 pipe_mutex_lock(fenced_list->mutex);
480 LIST_ADDTAIL(&buf->head, &fenced_list->unfenced);
481 ++fenced_list->numUnfenced;
482 pipe_mutex_unlock(fenced_list->mutex);
483 #endif
484
485 return &buf->base;
486 }
487
488
489 struct fenced_buffer_list *
490 fenced_buffer_list_create(struct pb_fence_ops *ops)
491 {
492 struct fenced_buffer_list *fenced_list;
493
494 fenced_list = CALLOC_STRUCT(fenced_buffer_list);
495 if (!fenced_list)
496 return NULL;
497
498 fenced_list->ops = ops;
499
500 LIST_INITHEAD(&fenced_list->delayed);
501 fenced_list->numDelayed = 0;
502
503 #ifdef DEBUG
504 LIST_INITHEAD(&fenced_list->unfenced);
505 fenced_list->numUnfenced = 0;
506 #endif
507
508 pipe_mutex_init(fenced_list->mutex);
509
510 return fenced_list;
511 }
512
513
514 void
515 fenced_buffer_list_check_free(struct fenced_buffer_list *fenced_list,
516 int wait)
517 {
518 pipe_mutex_lock(fenced_list->mutex);
519 _fenced_buffer_list_check_free(fenced_list, wait);
520 pipe_mutex_unlock(fenced_list->mutex);
521 }
522
523
524 #ifdef DEBUG
525 void
526 fenced_buffer_list_dump(struct fenced_buffer_list *fenced_list)
527 {
528 struct pb_fence_ops *ops = fenced_list->ops;
529 struct list_head *curr, *next;
530 struct fenced_buffer *fenced_buf;
531
532 pipe_mutex_lock(fenced_list->mutex);
533
534 debug_printf("%10s %7s %7s %10s %s\n",
535 "buffer", "size", "refcount", "fence", "signalled");
536
537 curr = fenced_list->unfenced.next;
538 next = curr->next;
539 while(curr != &fenced_list->unfenced) {
540 fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head);
541 assert(!fenced_buf->fence);
542 debug_printf("%10p %7u %7u\n",
543 (void *) fenced_buf,
544 fenced_buf->base.base.size,
545 p_atomic_read(&fenced_buf->base.base.reference.count));
546 curr = next;
547 next = curr->next;
548 }
549
550 curr = fenced_list->delayed.next;
551 next = curr->next;
552 while(curr != &fenced_list->delayed) {
553 int signaled;
554 fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head);
555 signaled = ops->fence_signalled(ops, fenced_buf->fence, 0);
556 debug_printf("%10p %7u %7u %10p %s\n",
557 (void *) fenced_buf,
558 fenced_buf->base.base.size,
559 p_atomic_read(&fenced_buf->base.base.reference.count),
560 (void *) fenced_buf->fence,
561 signaled == 0 ? "y" : "n");
562 curr = next;
563 next = curr->next;
564 }
565
566 pipe_mutex_unlock(fenced_list->mutex);
567 }
568 #endif
569
570
571 void
572 fenced_buffer_list_destroy(struct fenced_buffer_list *fenced_list)
573 {
574 pipe_mutex_lock(fenced_list->mutex);
575
576 /* Wait on outstanding fences */
577 while (fenced_list->numDelayed) {
578 pipe_mutex_unlock(fenced_list->mutex);
579 #if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) || defined(PIPE_OS_SOLARIS)
580 sched_yield();
581 #endif
582 _fenced_buffer_list_check_free(fenced_list, 1);
583 pipe_mutex_lock(fenced_list->mutex);
584 }
585
586 #ifdef DEBUG
587 /*assert(!fenced_list->numUnfenced);*/
588 #endif
589
590 pipe_mutex_unlock(fenced_list->mutex);
591
592 fenced_list->ops->destroy(fenced_list->ops);
593
594 FREE(fenced_list);
595 }
596
597