src/gallium/auxiliary/pipebuffer/pb_buffer_fenced.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2007-2009 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 /**
  29  * \file
  30  * Implementation of fenced buffers.
  31  *
  32  * \author Jose Fonseca <jrfonseca-at-tungstengraphics-dot-com>
  33  * \author Thomas Hellström <thomas-at-tungstengraphics-dot-com>
  34  */
  35
  36
  37 #include "pipe/p_config.h"
  38
  39 #if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) || defined(PIPE_OS_SOLARIS)
  40 #include <unistd.h>
  41 #include <sched.h>
  42 #endif
  43
  44 #include "pipe/p_compiler.h"
  45 #include "pipe/p_defines.h"
  46 #include "util/u_debug.h"
  47 #include "pipe/p_thread.h"
  48 #include "util/u_memory.h"
  49 #include "util/u_double_list.h"
  50
  51 #include "pb_buffer.h"
  52 #include "pb_buffer_fenced.h"
  53
  54
  55
  56 /**
  57  * Convenience macro (type safe).
  58  */
  59 #define SUPER(__derived) (&(__derived)->base)
  60
  61
  62 struct fenced_buffer_list
  63 {
  64    pipe_mutex mutex;
  65
  66    struct pb_fence_ops *ops;
  67
  68    pb_size numDelayed;
  69    struct list_head delayed;
  70
  71 #ifdef DEBUG
  72    pb_size numUnfenced;
  73    struct list_head unfenced;
  74 #endif
  75 };
  76
  77
  78 /**
  79  * Wrapper around a pipe buffer which adds fencing and reference counting.
  80  */
  81 struct fenced_buffer
  82 {
  83    struct pb_buffer base;
  84
  85    struct pb_buffer *buffer;
  86
  87    /* FIXME: protect access with mutex */
  88
  89    /**
  90     * A bitmask of PIPE_BUFFER_USAGE_CPU/GPU_READ/WRITE describing the current
  91     * buffer usage.
  92     */
  93    unsigned flags;
  94
  95    unsigned mapcount;
  96    struct pb_validate *vl;
  97    unsigned validation_flags;
  98    struct pipe_fence_handle *fence;
  99
 100    struct list_head head;
 101    struct fenced_buffer_list *list;
 102 };
 103
 104
 105 static INLINE struct fenced_buffer *
 106 fenced_buffer(struct pb_buffer *buf)
 107 {
 108    assert(buf);
 109    return (struct fenced_buffer *)buf;
 110 }
 111
 112
 113 static INLINE void
 114 _fenced_buffer_add(struct fenced_buffer *fenced_buf)
 115 {
 116    struct fenced_buffer_list *fenced_list = fenced_buf->list;
 117
 118    assert(pipe_is_referenced(&fenced_buf->base.base.reference));
 119    assert(fenced_buf->flags & PIPE_BUFFER_USAGE_GPU_READ_WRITE);
 120    assert(fenced_buf->fence);
 121
 122 #ifdef DEBUG
 123    LIST_DEL(&fenced_buf->head);
 124    assert(fenced_list->numUnfenced);
 125    --fenced_list->numUnfenced;
 126 #endif
 127    LIST_ADDTAIL(&fenced_buf->head, &fenced_list->delayed);
 128    ++fenced_list->numDelayed;
 129 }
 130
 131
 132 /**
 133  * Actually destroy the buffer.
 134  */
 135 static INLINE void
 136 _fenced_buffer_destroy(struct fenced_buffer *fenced_buf)
 137 {
 138    struct fenced_buffer_list *fenced_list = fenced_buf->list;
 139
 140    assert(!pipe_is_referenced(&fenced_buf->base.base.reference));
 141    assert(!fenced_buf->fence);
 142 #ifdef DEBUG
 143    assert(fenced_buf->head.prev);
 144    assert(fenced_buf->head.next);
 145    LIST_DEL(&fenced_buf->head);
 146    assert(fenced_list->numUnfenced);
 147    --fenced_list->numUnfenced;
 148 #else
 149    (void)fenced_list;
 150 #endif
 151    pb_reference(&fenced_buf->buffer, NULL);
 152    FREE(fenced_buf);
 153 }
 154
 155
 156 static INLINE void
 157 _fenced_buffer_remove(struct fenced_buffer_list *fenced_list,
 158                       struct fenced_buffer *fenced_buf)
 159 {
 160    struct pb_fence_ops *ops = fenced_list->ops;
 161
 162    assert(fenced_buf->fence);
 163    assert(fenced_buf->list == fenced_list);
 164
 165    ops->fence_reference(ops, &fenced_buf->fence, NULL);
 166    fenced_buf->flags &= ~PIPE_BUFFER_USAGE_GPU_READ_WRITE;
 167
 168    assert(fenced_buf->head.prev);
 169    assert(fenced_buf->head.next);
 170
 171    LIST_DEL(&fenced_buf->head);
 172    assert(fenced_list->numDelayed);
 173    --fenced_list->numDelayed;
 174
 175 #ifdef DEBUG
 176    LIST_ADDTAIL(&fenced_buf->head, &fenced_list->unfenced);
 177    ++fenced_list->numUnfenced;
 178 #endif
 179
 180    /**
 181     * FIXME!!!
 182     */
 183
 184    if(!pipe_is_referenced(&fenced_buf->base.base.reference))
 185       _fenced_buffer_destroy(fenced_buf);
 186 }
 187
 188
 189 static INLINE enum pipe_error
 190 _fenced_buffer_finish(struct fenced_buffer *fenced_buf)
 191 {
 192    struct fenced_buffer_list *fenced_list = fenced_buf->list;
 193    struct pb_fence_ops *ops = fenced_list->ops;
 194
 195 #if 0
 196    debug_warning("waiting for GPU");
 197 #endif
 198
 199    assert(fenced_buf->fence);
 200    if(fenced_buf->fence) {
 201       if(ops->fence_finish(ops, fenced_buf->fence, 0) != 0) {
 202          return PIPE_ERROR;
 203       }
 204       /* Remove from the fenced list */
 205       /* TODO: remove consequents */
 206       _fenced_buffer_remove(fenced_list, fenced_buf);
 207    }
 208
 209    fenced_buf->flags &= ~PIPE_BUFFER_USAGE_GPU_READ_WRITE;
 210    return PIPE_OK;
 211 }
 212
 213
 214 /**
 215  * Free as many fenced buffers from the list head as possible.
 216  */
 217 static void
 218 _fenced_buffer_list_check_free(struct fenced_buffer_list *fenced_list,
 219                                int wait)
 220 {
 221    struct pb_fence_ops *ops = fenced_list->ops;
 222    struct list_head *curr, *next;
 223    struct fenced_buffer *fenced_buf;
 224    struct pb_buffer *pb_buf;
 225    struct pipe_fence_handle *prev_fence = NULL;
 226
 227    curr = fenced_list->delayed.next;
 228    next = curr->next;
 229    while(curr != &fenced_list->delayed) {
 230       fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head);
 231
 232       if(fenced_buf->fence != prev_fence) {
 233          int signaled;
 234          if (wait)
 235             signaled = ops->fence_finish(ops, fenced_buf->fence, 0);
 236          else
 237             signaled = ops->fence_signalled(ops, fenced_buf->fence, 0);
 238          if (signaled != 0)
 239             break;
 240          prev_fence = fenced_buf->fence;
 241       }
 242       else {
 243          assert(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0);
 244       }
 245
 246       _fenced_buffer_remove(fenced_list, fenced_buf);
 247
 248       curr = next;
 249       next = curr->next;
 250    }
 251 }
 252
 253
 254 static void
 255 fenced_buffer_destroy(struct pb_buffer *buf)
 256 {
 257    struct fenced_buffer *fenced_buf = fenced_buffer(buf);
 258    struct fenced_buffer_list *fenced_list = fenced_buf->list;
 259
 260    pipe_mutex_lock(fenced_list->mutex);
 261    assert(!pipe_is_referenced(&fenced_buf->base.base.reference));
 262    if (fenced_buf->fence) {
 263       struct pb_fence_ops *ops = fenced_list->ops;
 264       if(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0) {
 265          struct list_head *curr, *prev;
 266          curr = &fenced_buf->head;
 267          prev = curr->prev;
 268          do {
 269             fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head);
 270             assert(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0);
 271             _fenced_buffer_remove(fenced_list, fenced_buf);
 272             curr = prev;
 273             prev = curr->prev;
 274          } while (curr != &fenced_list->delayed);
 275       }
 276       else {
 277          /* delay destruction */
 278       }
 279    }
 280    else {
 281       _fenced_buffer_destroy(fenced_buf);
 282    }
 283    pipe_mutex_unlock(fenced_list->mutex);
 284 }
 285
 286
 287 static void *
 288 fenced_buffer_map(struct pb_buffer *buf,
 289                   unsigned flags)
 290 {
 291    struct fenced_buffer *fenced_buf = fenced_buffer(buf);
 292    struct fenced_buffer_list *fenced_list = fenced_buf->list;
 293    struct pb_fence_ops *ops = fenced_list->ops;
 294    void *map;
 295
 296    assert(!(flags & PIPE_BUFFER_USAGE_GPU_READ_WRITE));
 297
 298    /* Serialize writes */
 299    if((fenced_buf->flags & PIPE_BUFFER_USAGE_GPU_WRITE) ||
 300       ((fenced_buf->flags & PIPE_BUFFER_USAGE_GPU_READ) && (flags & PIPE_BUFFER_USAGE_CPU_WRITE))) {
 301       if(flags & PIPE_BUFFER_USAGE_DONTBLOCK) {
 302          /* Don't wait for the GPU to finish writing */
 303          if(ops->fence_signalled(ops, fenced_buf->fence, 0) == 0)
 304             _fenced_buffer_remove(fenced_list, fenced_buf);
 305          else
 306             return NULL;
 307       }
 308       else {
 309          /* Wait for the GPU to finish writing */
 310          _fenced_buffer_finish(fenced_buf);
 311       }
 312    }
 313
 314 #if 0
 315    /* Check for CPU write access (read is OK) */
 316    if(fenced_buf->flags & PIPE_BUFFER_USAGE_CPU_READ_WRITE) {
 317       /* this is legal -- just for debugging */
 318       debug_warning("concurrent CPU writes");
 319    }
 320 #endif
 321
 322    map = pb_map(fenced_buf->buffer, flags);
 323    if(map) {
 324       ++fenced_buf->mapcount;
 325       fenced_buf->flags |= flags & PIPE_BUFFER_USAGE_CPU_READ_WRITE;
 326    }
 327
 328    return map;
 329 }
 330
 331
 332 static void
 333 fenced_buffer_unmap(struct pb_buffer *buf)
 334 {
 335    struct fenced_buffer *fenced_buf = fenced_buffer(buf);
 336    assert(fenced_buf->mapcount);
 337    if(fenced_buf->mapcount) {
 338       pb_unmap(fenced_buf->buffer);
 339       --fenced_buf->mapcount;
 340       if(!fenced_buf->mapcount)
 341          fenced_buf->flags &= ~PIPE_BUFFER_USAGE_CPU_READ_WRITE;
 342    }
 343 }
 344
 345
 346 static enum pipe_error
 347 fenced_buffer_validate(struct pb_buffer *buf,
 348                        struct pb_validate *vl,
 349                        unsigned flags)
 350 {
 351    struct fenced_buffer *fenced_buf = fenced_buffer(buf);
 352    enum pipe_error ret;
 353
 354    if(!vl) {
 355       /* invalidate */
 356       fenced_buf->vl = NULL;
 357       fenced_buf->validation_flags = 0;
 358       return PIPE_OK;
 359    }
 360
 361    assert(flags & PIPE_BUFFER_USAGE_GPU_READ_WRITE);
 362    assert(!(flags & ~PIPE_BUFFER_USAGE_GPU_READ_WRITE));
 363    flags &= PIPE_BUFFER_USAGE_GPU_READ_WRITE;
 364
 365    /* Buffer cannot be validated in two different lists */
 366    if(fenced_buf->vl && fenced_buf->vl != vl)
 367       return PIPE_ERROR_RETRY;
 368
 369 #if 0
 370    /* Do not validate if buffer is still mapped */
 371    if(fenced_buf->flags & PIPE_BUFFER_USAGE_CPU_READ_WRITE) {
 372       /* TODO: wait for the thread that mapped the buffer to unmap it */
 373       return PIPE_ERROR_RETRY;
 374    }
 375    /* Final sanity checking */
 376    assert(!(fenced_buf->flags & PIPE_BUFFER_USAGE_CPU_READ_WRITE));
 377    assert(!fenced_buf->mapcount);
 378 #endif
 379
 380    if(fenced_buf->vl == vl &&
 381       (fenced_buf->validation_flags & flags) == flags) {
 382       /* Nothing to do -- buffer already validated */
 383       return PIPE_OK;
 384    }
 385
 386    ret = pb_validate(fenced_buf->buffer, vl, flags);
 387    if (ret != PIPE_OK)
 388       return ret;
 389
 390    fenced_buf->vl = vl;
 391    fenced_buf->validation_flags |= flags;
 392
 393    return PIPE_OK;
 394 }
 395
 396
 397 static void
 398 fenced_buffer_fence(struct pb_buffer *buf,
 399                     struct pipe_fence_handle *fence)
 400 {
 401    struct fenced_buffer *fenced_buf;
 402    struct fenced_buffer_list *fenced_list;
 403    struct pb_fence_ops *ops;
 404
 405    fenced_buf = fenced_buffer(buf);
 406    fenced_list = fenced_buf->list;
 407    ops = fenced_list->ops;
 408
 409    if(fence == fenced_buf->fence) {
 410       /* Nothing to do */
 411       return;
 412    }
 413
 414    assert(fenced_buf->vl);
 415    assert(fenced_buf->validation_flags);
 416
 417    pipe_mutex_lock(fenced_list->mutex);
 418    if (fenced_buf->fence)
 419       _fenced_buffer_remove(fenced_list, fenced_buf);
 420    if (fence) {
 421       ops->fence_reference(ops, &fenced_buf->fence, fence);
 422       fenced_buf->flags |= fenced_buf->validation_flags;
 423       _fenced_buffer_add(fenced_buf);
 424    }
 425    pipe_mutex_unlock(fenced_list->mutex);
 426
 427    pb_fence(fenced_buf->buffer, fence);
 428
 429    fenced_buf->vl = NULL;
 430    fenced_buf->validation_flags = 0;
 431 }
 432
 433
 434 static void
 435 fenced_buffer_get_base_buffer(struct pb_buffer *buf,
 436                               struct pb_buffer **base_buf,
 437                               pb_size *offset)
 438 {
 439    struct fenced_buffer *fenced_buf = fenced_buffer(buf);
 440    pb_get_base_buffer(fenced_buf->buffer, base_buf, offset);
 441 }
 442
 443
 444 static const struct pb_vtbl
 445 fenced_buffer_vtbl = {
 446       fenced_buffer_destroy,
 447       fenced_buffer_map,
 448       fenced_buffer_unmap,
 449       fenced_buffer_validate,
 450       fenced_buffer_fence,
 451       fenced_buffer_get_base_buffer
 452 };
 453
 454
 455 struct pb_buffer *
 456 fenced_buffer_create(struct fenced_buffer_list *fenced_list,
 457                      struct pb_buffer *buffer)
 458 {
 459    struct fenced_buffer *buf;
 460
 461    if(!buffer)
 462       return NULL;
 463
 464    buf = CALLOC_STRUCT(fenced_buffer);
 465    if(!buf) {
 466       pb_reference(&buffer, NULL);
 467       return NULL;
 468    }
 469
 470    pipe_reference_init(&buf->base.base.reference, 1);
 471    buf->base.base.alignment = buffer->base.alignment;
 472    buf->base.base.usage = buffer->base.usage;
 473    buf->base.base.size = buffer->base.size;
 474
 475    buf->base.vtbl = &fenced_buffer_vtbl;
 476    buf->buffer = buffer;
 477    buf->list = fenced_list;
 478
 479 #ifdef DEBUG
 480    pipe_mutex_lock(fenced_list->mutex);
 481    LIST_ADDTAIL(&buf->head, &fenced_list->unfenced);
 482    ++fenced_list->numUnfenced;
 483    pipe_mutex_unlock(fenced_list->mutex);
 484 #endif
 485
 486    return &buf->base;
 487 }
 488
 489
 490 struct fenced_buffer_list *
 491 fenced_buffer_list_create(struct pb_fence_ops *ops)
 492 {
 493    struct fenced_buffer_list *fenced_list;
 494
 495    fenced_list = CALLOC_STRUCT(fenced_buffer_list);
 496    if (!fenced_list)
 497       return NULL;
 498
 499    fenced_list->ops = ops;
 500
 501    LIST_INITHEAD(&fenced_list->delayed);
 502    fenced_list->numDelayed = 0;
 503
 504 #ifdef DEBUG
 505    LIST_INITHEAD(&fenced_list->unfenced);
 506    fenced_list->numUnfenced = 0;
 507 #endif
 508
 509    pipe_mutex_init(fenced_list->mutex);
 510
 511    return fenced_list;
 512 }
 513
 514
 515 void
 516 fenced_buffer_list_check_free(struct fenced_buffer_list *fenced_list,
 517                               int wait)
 518 {
 519    pipe_mutex_lock(fenced_list->mutex);
 520    _fenced_buffer_list_check_free(fenced_list, wait);
 521    pipe_mutex_unlock(fenced_list->mutex);
 522 }
 523
 524
 525 #ifdef DEBUG
 526 void
 527 fenced_buffer_list_dump(struct fenced_buffer_list *fenced_list)
 528 {
 529    struct pb_fence_ops *ops = fenced_list->ops;
 530    struct list_head *curr, *next;
 531    struct fenced_buffer *fenced_buf;
 532
 533    pipe_mutex_lock(fenced_list->mutex);
 534
 535    debug_printf("%10s %7s %7s %10s %s\n",
 536                 "buffer", "size", "refcount", "fence", "signalled");
 537
 538    curr = fenced_list->unfenced.next;
 539    next = curr->next;
 540    while(curr != &fenced_list->unfenced) {
 541       fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head);
 542       assert(!fenced_buf->fence);
 543       debug_printf("%10p %7u %7u\n",
 544                    (void *) fenced_buf,
 545                    fenced_buf->base.base.size,
 546                    p_atomic_read(&fenced_buf->base.base.reference.count));
 547       curr = next;
 548       next = curr->next;
 549    }
 550
 551    curr = fenced_list->delayed.next;
 552    next = curr->next;
 553    while(curr != &fenced_list->delayed) {
 554       int signaled;
 555       fenced_buf = LIST_ENTRY(struct fenced_buffer, curr, head);
 556       signaled = ops->fence_signalled(ops, fenced_buf->fence, 0);
 557       debug_printf("%10p %7u %7u %10p %s\n",
 558                    (void *) fenced_buf,
 559                    fenced_buf->base.base.size,
 560                    p_atomic_read(&fenced_buf->base.base.reference.count),
 561                    (void *) fenced_buf->fence,
 562                    signaled == 0 ? "y" : "n");
 563       curr = next;
 564       next = curr->next;
 565    }
 566
 567    pipe_mutex_unlock(fenced_list->mutex);
 568 }
 569 #endif
 570
 571
 572 void
 573 fenced_buffer_list_destroy(struct fenced_buffer_list *fenced_list)
 574 {
 575    pipe_mutex_lock(fenced_list->mutex);
 576
 577    /* Wait on outstanding fences */
 578    while (fenced_list->numDelayed) {
 579       pipe_mutex_unlock(fenced_list->mutex);
 580 #if defined(PIPE_OS_LINUX) || defined(PIPE_OS_BSD) || defined(PIPE_OS_SOLARIS)
 581       sched_yield();
 582 #endif
 583       _fenced_buffer_list_check_free(fenced_list, 1);
 584       pipe_mutex_lock(fenced_list->mutex);
 585    }
 586
 587 #ifdef DEBUG
 588    /*assert(!fenced_list->numUnfenced);*/
 589 #endif
 590
 591    pipe_mutex_unlock(fenced_list->mutex);
 592
 593    fenced_list->ops->destroy(fenced_list->ops);
 594
 595    FREE(fenced_list);
 596 }
 597
 598