src/mesa/drivers/dri/i965/bufmgr_fake.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 /* Originally a fake version of the buffer manager so that we can
  29  * prototype the changes in a driver fairly quickly, has been fleshed
  30  * out to a fully functional interim solution.
  31  *
  32  * Basically wraps the old style memory management in the new
  33  * programming interface, but is more expressive and avoids many of
  34  * the bugs in the old texture manager.
  35  */
  36 #include "bufmgr.h"
  37
  38 #include "intel_context.h"
  39 #include "intel_ioctl.h"
  40 #include "intel_batchbuffer.h"
  41
  42 #include "simple_list.h"
  43 #include "mm.h"
  44 #include "imports.h"
  45
  46 #define BM_POOL_MAX 8
  47
  48 /* Internal flags:
  49  */
  50 #define BM_NO_BACKING_STORE   0x2000
  51 #define BM_NO_FENCE_SUBDATA   0x4000
  52
  53
  54 static int check_fenced( struct intel_context *intel );
  55
  56 static int nr_attach = 0;
  57
  58 /* Wrapper around mm.c's mem_block, which understands that you must
  59  * wait for fences to expire before memory can be freed.  This is
  60  * specific to our use of memcpy for uploads - an upload that was
  61  * processed through the command queue wouldn't need to care about
  62  * fences.
  63  */
  64 struct block {
  65    struct block *next, *prev;
  66    struct pool *pool;           /* BM_MEM_AGP */
  67    struct mem_block *mem;       /* BM_MEM_AGP */
  68
  69    unsigned referenced:1;
  70    unsigned on_hardware:1;
  71    unsigned fenced:1;
  72
  73
  74    unsigned fence;              /* BM_MEM_AGP, Split to read_fence, write_fence */
  75
  76    struct buffer *buf;
  77    void *virtual;
  78 };
  79
  80
  81 struct buffer {
  82    unsigned id;                 /* debug only */
  83    const char *name;
  84    unsigned size;
  85
  86    unsigned mapped:1;
  87    unsigned dirty:1;
  88    unsigned aub_dirty:1;
  89    unsigned alignment:13;
  90    unsigned flags:16;
  91
  92    struct block *block;
  93    void *backing_store;
  94    void (*invalidate_cb)( struct intel_context *, void * );
  95    void *invalidate_ptr;
  96 };
  97
  98 struct pool {
  99    unsigned size;
 100    unsigned low_offset;
 101    struct buffer *static_buffer;
 102    unsigned flags;
 103    struct mem_block *heap;
 104    void *virtual;
 105    struct block lru;            /* only allocated, non-fence-pending blocks here */
 106 };
 107
 108 struct bufmgr {
 109    _glthread_Mutex mutex;       /**< for thread safety */
 110    struct pool pool[BM_POOL_MAX];
 111    unsigned nr_pools;
 112
 113    unsigned buf_nr;             /* for generating ids */
 114
 115    struct block referenced;     /* after bmBufferOffset */
 116    struct block on_hardware;    /* after bmValidateBuffers */
 117    struct block fenced;         /* after bmFenceBuffers (mi_flush, emit irq, write dword) */
 118                                 /* then to pool->lru or free() */
 119
 120    unsigned ctxId;
 121    unsigned last_fence;
 122    unsigned free_on_hardware;
 123
 124    unsigned fail:1;
 125    unsigned need_fence:1;
 126 };
 127
 128 #define MAXFENCE 0x7fffffff
 129
 130 static GLboolean FENCE_LTE( unsigned a, unsigned b )
 131 {
 132    if (a == b)
 133       return GL_TRUE;
 134
 135    if (a < b && b - a < (1<<24))
 136       return GL_TRUE;
 137
 138    if (a > b && MAXFENCE - a + b < (1<<24))
 139       return GL_TRUE;
 140
 141    return GL_FALSE;
 142 }
 143
 144 int bmTestFence( struct intel_context *intel, unsigned fence )
 145 {
 146    /* Slight problem with wrap-around:
 147     */
 148    return fence == 0 || FENCE_LTE(fence, intel->sarea->last_dispatch);
 149 }
 150
 151 #define LOCK(bm) \
 152   int dolock = nr_attach > 1; \
 153   if (dolock) _glthread_LOCK_MUTEX(bm->mutex)
 154
 155 #define UNLOCK(bm) \
 156   if (dolock) _glthread_UNLOCK_MUTEX(bm->mutex)
 157
 158
 159
 160 static GLboolean alloc_from_pool( struct intel_context *intel,
 161                                   unsigned pool_nr,
 162                                   struct buffer *buf )
 163 {
 164    struct bufmgr *bm = intel->bm;
 165    struct pool *pool = &bm->pool[pool_nr];
 166    struct block *block = (struct block *)calloc(sizeof *block, 1);
 167    GLuint sz, align = (1<<buf->alignment);
 168
 169    if (!block)
 170       return GL_FALSE;
 171
 172    sz = (buf->size + align-1) & ~(align-1);
 173
 174    block->mem = mmAllocMem(pool->heap,
 175                            sz,
 176                            buf->alignment, 0);
 177    if (!block->mem) {
 178       free(block);
 179       return GL_FALSE;
 180    }
 181
 182    make_empty_list(block);
 183
 184    /* Insert at head or at tail???
 185     */
 186    insert_at_tail(&pool->lru, block);
 187
 188    block->pool = pool;
 189    block->virtual = pool->virtual + block->mem->ofs;
 190    block->buf = buf;
 191
 192    buf->block = block;
 193
 194    return GL_TRUE;
 195 }
 196
 197
 198
 199
 200
 201
 202
 203
 204 /* Release the card storage associated with buf:
 205  */
 206 static void free_block( struct intel_context *intel, struct block *block )
 207 {
 208    DBG("free block %p\n", block);
 209
 210    if (!block)
 211       return;
 212
 213    check_fenced(intel);
 214
 215    if (block->referenced) {
 216       _mesa_printf("tried to free block on referenced list\n");
 217       assert(0);
 218    }
 219    else if (block->on_hardware) {
 220       block->buf = NULL;
 221       intel->bm->free_on_hardware += block->mem->size;
 222    }
 223    else if (block->fenced) {
 224       block->buf = NULL;
 225    }
 226    else {
 227       DBG("    - free immediately\n");
 228       remove_from_list(block);
 229
 230       mmFreeMem(block->mem);
 231       free(block);
 232    }
 233 }
 234
 235
 236 static void alloc_backing_store( struct intel_context *intel, struct buffer *buf )
 237 {
 238    assert(!buf->backing_store);
 239    assert(!(buf->flags & (BM_NO_EVICT|BM_NO_BACKING_STORE)));
 240
 241    buf->backing_store = ALIGN_MALLOC(buf->size, 64);
 242 }
 243
 244 static void free_backing_store( struct intel_context *intel, struct buffer *buf )
 245 {
 246    assert(!(buf->flags & (BM_NO_EVICT|BM_NO_BACKING_STORE)));
 247
 248    if (buf->backing_store) {
 249       ALIGN_FREE(buf->backing_store);
 250       buf->backing_store = NULL;
 251    }
 252 }
 253
 254
 255
 256
 257
 258
 259 static void set_dirty( struct intel_context *intel,
 260                               struct buffer *buf )
 261 {
 262    if (buf->flags & BM_NO_BACKING_STORE)
 263       buf->invalidate_cb(intel, buf->invalidate_ptr);
 264
 265    assert(!(buf->flags & BM_NO_EVICT));
 266
 267    DBG("set_dirty - buf %d\n", buf->id);
 268    buf->dirty = 1;
 269 }
 270
 271
 272 static int evict_lru( struct intel_context *intel, GLuint max_fence, GLuint *pool )
 273 {
 274    struct bufmgr *bm = intel->bm;
 275    struct block *block, *tmp;
 276    int i;
 277
 278    DBG("%s\n", __FUNCTION__);
 279
 280    for (i = 0; i < bm->nr_pools; i++) {
 281       if (!(bm->pool[i].flags & BM_NO_EVICT)) {
 282          foreach_s(block, tmp, &bm->pool[i].lru) {
 283
 284             if (block->buf &&
 285                 (block->buf->flags & BM_NO_FENCE_SUBDATA))
 286                continue;
 287
 288             if (block->fence && max_fence &&
 289                 !FENCE_LTE(block->fence, max_fence))
 290                return 0;
 291
 292             set_dirty(intel, block->buf);
 293             block->buf->block = NULL;
 294
 295             free_block(intel, block);
 296             *pool = i;
 297             return 1;
 298          }
 299       }
 300    }
 301
 302
 303    return 0;
 304 }
 305
 306
 307 #define foreach_s_rev(ptr, t, list)   \
 308         for(ptr=(list)->prev,t=(ptr)->prev; list != ptr; ptr=t, t=(t)->prev)
 309
 310 static int evict_mru( struct intel_context *intel, GLuint *pool )
 311 {
 312    struct bufmgr *bm = intel->bm;
 313    struct block *block, *tmp;
 314    int i;
 315
 316    DBG("%s\n", __FUNCTION__);
 317
 318    for (i = 0; i < bm->nr_pools; i++) {
 319       if (!(bm->pool[i].flags & BM_NO_EVICT)) {
 320          foreach_s_rev(block, tmp, &bm->pool[i].lru) {
 321
 322             if (block->buf &&
 323                 (block->buf->flags & BM_NO_FENCE_SUBDATA))
 324                continue;
 325
 326             set_dirty(intel, block->buf);
 327             block->buf->block = NULL;
 328
 329             free_block(intel, block);
 330             *pool = i;
 331             return 1;
 332          }
 333       }
 334    }
 335
 336
 337    return 0;
 338 }
 339
 340
 341 static int check_fenced( struct intel_context *intel )
 342 {
 343    struct bufmgr *bm = intel->bm;
 344    struct block *block, *tmp;
 345    int ret = 0;
 346
 347    foreach_s(block, tmp, &bm->fenced ) {
 348       assert(block->fenced);
 349
 350       if (bmTestFence(intel, block->fence)) {
 351
 352          block->fenced = 0;
 353
 354          if (!block->buf) {
 355             DBG("delayed free: offset %x sz %x\n", block->mem->ofs, block->mem->size);
 356             remove_from_list(block);
 357             mmFreeMem(block->mem);
 358             free(block);
 359          }
 360          else {
 361             DBG("return to lru: offset %x sz %x\n", block->mem->ofs, block->mem->size);
 362             move_to_tail(&block->pool->lru, block);
 363          }
 364
 365          ret = 1;
 366       }
 367       else {
 368          /* Blocks are ordered by fence, so if one fails, all from
 369           * here will fail also:
 370           */
 371          break;
 372       }
 373    }
 374
 375    /* Also check the referenced list:
 376     */
 377    foreach_s(block, tmp, &bm->referenced ) {
 378       if (block->fenced &&
 379           bmTestFence(intel, block->fence)) {
 380          block->fenced = 0;
 381       }
 382    }
 383
 384
 385    DBG("%s: %d\n", __FUNCTION__, ret);
 386    return ret;
 387 }
 388
 389
 390
 391 static void fence_blocks( struct intel_context *intel,
 392                           unsigned fence )
 393 {
 394    struct bufmgr *bm = intel->bm;
 395    struct block *block, *tmp;
 396
 397    foreach_s (block, tmp, &bm->on_hardware) {
 398       DBG("Fence block %p (sz 0x%x buf %p) with fence %d\n", block,
 399           block->mem->size, block->buf, fence);
 400       block->fence = fence;
 401
 402       block->on_hardware = 0;
 403       block->fenced = 1;
 404
 405       /* Move to tail of pending list here
 406        */
 407       move_to_tail(&bm->fenced, block);
 408    }
 409
 410    /* Also check the referenced list:
 411     */
 412    foreach_s (block, tmp, &bm->referenced) {
 413       if (block->on_hardware) {
 414          DBG("Fence block %p (sz 0x%x buf %p) with fence %d\n", block,
 415              block->mem->size, block->buf, fence);
 416
 417          block->fence = fence;
 418          block->on_hardware = 0;
 419          block->fenced = 1;
 420       }
 421    }
 422
 423
 424    bm->last_fence = fence;
 425    assert(is_empty_list(&bm->on_hardware));
 426 }
 427
 428
 429
 430
 431 static GLboolean alloc_block( struct intel_context *intel,
 432                               struct buffer *buf )
 433 {
 434    struct bufmgr *bm = intel->bm;
 435    int i;
 436
 437    assert(intel->locked);
 438
 439    DBG("%s 0x%x bytes (%s)\n", __FUNCTION__, buf->size, buf->name);
 440
 441    for (i = 0; i < bm->nr_pools; i++) {
 442       if (!(bm->pool[i].flags & BM_NO_ALLOC) &&
 443           alloc_from_pool(intel, i, buf)) {
 444
 445          DBG("%s --> 0x%x (sz %x)\n", __FUNCTION__,
 446              buf->block->mem->ofs, buf->block->mem->size);
 447
 448          return GL_TRUE;
 449       }
 450    }
 451
 452    DBG("%s --> fail\n", __FUNCTION__);
 453    return GL_FALSE;
 454 }
 455
 456
 457 static GLboolean evict_and_alloc_block( struct intel_context *intel,
 458                                         struct buffer *buf )
 459 {
 460    GLuint pool;
 461    struct bufmgr *bm = intel->bm;
 462
 463    assert(buf->block == NULL);
 464
 465    /* Put a cap on the amount of free memory we'll allow to accumulate
 466     * before emitting a fence.
 467     */
 468    if (bm->free_on_hardware > 1 * 1024 * 1024) {
 469       DBG("fence for free space: %x\n", bm->free_on_hardware);
 470       bmSetFence(intel);
 471    }
 472
 473    /* Search for already free memory:
 474     */
 475    if (alloc_block(intel, buf))
 476       return GL_TRUE;
 477
 478    /* Look for memory that may have become free:
 479     */
 480    if (check_fenced(intel) &&
 481        alloc_block(intel, buf))
 482       return GL_TRUE;
 483
 484    /* Look for memory blocks not used for >1 frame:
 485     */
 486    while (evict_lru(intel, intel->second_last_swap_fence, &pool))
 487       if (alloc_from_pool(intel, pool, buf))
 488          return GL_TRUE;
 489
 490    /* If we're not thrashing, allow lru eviction to dig deeper into
 491     * recently used textures.  We'll probably be thrashing soon:
 492     */
 493    if (!intel->thrashing) {
 494       while (evict_lru(intel, 0, &pool))
 495          if (alloc_from_pool(intel, pool, buf))
 496             return GL_TRUE;
 497    }
 498
 499    /* Keep thrashing counter alive?
 500     */
 501    if (intel->thrashing)
 502       intel->thrashing = 20;
 503
 504    /* Wait on any already pending fences - here we are waiting for any
 505     * freed memory that has been submitted to hardware and fenced to
 506     * become available:
 507     */
 508    while (!is_empty_list(&bm->fenced)) {
 509       GLuint fence = bm->fenced.next->fence;
 510       bmFinishFence(intel, fence);
 511
 512       if (alloc_block(intel, buf))
 513          return GL_TRUE;
 514    }
 515
 516
 517    /*
 518     */
 519    if (!is_empty_list(&bm->on_hardware)) {
 520       bmSetFence(intel);
 521
 522       while (!is_empty_list(&bm->fenced)) {
 523          GLuint fence = bm->fenced.next->fence;
 524          bmFinishFence(intel, fence);
 525       }
 526
 527       if (!intel->thrashing) {
 528          DBG("thrashing\n");
 529       }
 530       intel->thrashing = 20;
 531
 532       if (alloc_block(intel, buf))
 533          return GL_TRUE;
 534    }
 535
 536    while (evict_mru(intel, &pool))
 537       if (alloc_from_pool(intel, pool, buf))
 538          return GL_TRUE;
 539
 540    DBG("%s 0x%x bytes failed\n", __FUNCTION__, buf->size);
 541
 542    assert(is_empty_list(&bm->on_hardware));
 543    assert(is_empty_list(&bm->fenced));
 544
 545    return GL_FALSE;
 546 }
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557 /***********************************************************************
 558  * Public functions
 559  */
 560
 561
 562 /* The initialization functions are skewed in the fake implementation.
 563  * This call would be to attach to an existing manager, rather than to
 564  * create a local one.
 565  */
 566 struct bufmgr *bm_fake_intel_Attach( struct intel_context *intel )
 567 {
 568    _glthread_DECLARE_STATIC_MUTEX(initMutex);
 569    static struct bufmgr bm;
 570
 571    /* This function needs a mutex of its own...
 572     */
 573    _glthread_LOCK_MUTEX(initMutex);
 574
 575    if (nr_attach == 0) {
 576       _glthread_INIT_MUTEX(bm.mutex);
 577
 578       make_empty_list(&bm.referenced);
 579       make_empty_list(&bm.fenced);
 580       make_empty_list(&bm.on_hardware);
 581
 582       /* The context id of any of the share group.  This won't be used
 583        * in communication with the kernel, so it doesn't matter if
 584        * this context is eventually deleted.
 585        */
 586       bm.ctxId = intel->hHWContext;
 587    }
 588
 589    nr_attach++;
 590
 591    _glthread_UNLOCK_MUTEX(initMutex);
 592
 593    return &bm;
 594 }
 595
 596
 597
 598 /* The virtual pointer would go away in a true implementation.
 599  */
 600 int bmInitPool( struct intel_context *intel,
 601                 unsigned long low_offset,
 602                 void *low_virtual,
 603                 unsigned long size,
 604                 unsigned flags)
 605 {
 606    struct bufmgr *bm = intel->bm;
 607    int retval = 0;
 608
 609    LOCK(bm);
 610    {
 611       GLuint i;
 612
 613       for (i = 0; i < bm->nr_pools; i++) {
 614          if (bm->pool[i].low_offset == low_offset &&
 615              bm->pool[i].size == size) {
 616             retval = i;
 617             goto out;
 618          }
 619       }
 620
 621
 622       if (bm->nr_pools >= BM_POOL_MAX)
 623          retval = -1;
 624       else {
 625          i = bm->nr_pools++;
 626
 627          DBG("bmInitPool %d low_offset %x sz %x\n",
 628              i, low_offset, size);
 629
 630          bm->pool[i].low_offset = low_offset;
 631          bm->pool[i].size = size;
 632          bm->pool[i].heap = mmInit( low_offset, size );
 633          bm->pool[i].virtual = low_virtual - low_offset;
 634          bm->pool[i].flags = flags;
 635
 636          make_empty_list(&bm->pool[i].lru);
 637
 638          retval = i;
 639       }
 640    }
 641  out:
 642    UNLOCK(bm);
 643    return retval;
 644 }
 645
 646 static struct buffer *do_GenBuffer(struct intel_context *intel, const char *name, int align)
 647 {
 648    struct bufmgr *bm = intel->bm;
 649    struct buffer *buf = calloc(sizeof(*buf), 1);
 650
 651    buf->id = ++bm->buf_nr;
 652    buf->name = name;
 653    buf->alignment = align;
 654    buf->flags = BM_MEM_AGP|BM_MEM_VRAM|BM_MEM_LOCAL;
 655
 656    return buf;
 657 }
 658
 659
 660 void *bmFindVirtual( struct intel_context *intel,
 661                      unsigned int offset,
 662                      size_t sz )
 663 {
 664    struct bufmgr *bm = intel->bm;
 665    int i;
 666
 667    for (i = 0; i < bm->nr_pools; i++)
 668       if (offset >= bm->pool[i].low_offset &&
 669           offset + sz <= bm->pool[i].low_offset + bm->pool[i].size)
 670          return bm->pool[i].virtual + offset;
 671
 672    return NULL;
 673 }
 674
 675
 676 void bmGenBuffers(struct intel_context *intel,
 677                   const char *name, unsigned n,
 678                   struct buffer **buffers,
 679                   int align )
 680 {
 681    struct bufmgr *bm = intel->bm;
 682    LOCK(bm);
 683    {
 684       int i;
 685
 686       for (i = 0; i < n; i++)
 687          buffers[i] = do_GenBuffer(intel, name, align);
 688    }
 689    UNLOCK(bm);
 690 }
 691
 692
 693 void bmDeleteBuffers(struct intel_context *intel, unsigned n, struct buffer **buffers)
 694 {
 695    struct bufmgr *bm = intel->bm;
 696
 697    LOCK(bm);
 698    {
 699       unsigned i;
 700
 701       for (i = 0; i < n; i++) {
 702          struct buffer *buf = buffers[i];
 703
 704          if (buf && buf->block)
 705             free_block(intel, buf->block);
 706
 707          if (buf)
 708             free(buf);
 709       }
 710    }
 711    UNLOCK(bm);
 712 }
 713
 714
 715
 716
 717 /* Hook to inform faked buffer manager about fixed-position
 718  * front,depth,back buffers.  These may move to a fully memory-managed
 719  * scheme, or they may continue to be managed as is.  It will probably
 720  * be useful to pass a fixed offset here one day.
 721  */
 722 struct buffer *bmGenBufferStatic(struct intel_context *intel,
 723                                  unsigned pool )
 724 {
 725    struct bufmgr *bm = intel->bm;
 726    struct buffer *buf;
 727    LOCK(bm);
 728    {
 729       assert(bm->pool[pool].flags & BM_NO_EVICT);
 730       assert(bm->pool[pool].flags & BM_NO_MOVE);
 731
 732       if (bm->pool[pool].static_buffer)
 733          buf = bm->pool[pool].static_buffer;
 734       else {
 735          buf = do_GenBuffer(intel, "static", 12);
 736
 737          bm->pool[pool].static_buffer = buf;
 738          assert(!buf->block);
 739
 740          buf->size = bm->pool[pool].size;
 741          buf->flags = bm->pool[pool].flags;
 742          buf->alignment = 12;
 743
 744          if (!alloc_from_pool(intel, pool, buf))
 745             assert(0);
 746       }
 747    }
 748    UNLOCK(bm);
 749    return buf;
 750 }
 751
 752
 753 static void wait_quiescent(struct intel_context *intel,
 754                            struct block *block)
 755 {
 756    if (block->on_hardware) {
 757       assert(intel->bm->need_fence);
 758       bmSetFence(intel);
 759       assert(!block->on_hardware);
 760    }
 761
 762
 763    if (block->fenced) {
 764       bmFinishFence(intel, block->fence);
 765    }
 766
 767    assert(!block->on_hardware);
 768    assert(!block->fenced);
 769 }
 770
 771
 772
 773 /* If buffer size changes, free and reallocate.  Otherwise update in
 774  * place.
 775  */
 776 int bmBufferData(struct intel_context *intel,
 777                  struct buffer *buf,
 778                  unsigned size,
 779                  const void *data,
 780                  unsigned flags )
 781 {
 782    struct bufmgr *bm = intel->bm;
 783    int retval = 0;
 784
 785    LOCK(bm);
 786    {
 787       DBG("bmBufferData %d sz 0x%x data: %p\n", buf->id, size, data);
 788
 789       assert(!buf->mapped);
 790
 791       if (buf->block) {
 792          struct block *block = buf->block;
 793
 794          /* Optimistic check to see if we can reuse the block -- not
 795           * required for correctness:
 796           */
 797          if (block->fenced)
 798             check_fenced(intel);
 799
 800          if (block->on_hardware ||
 801              block->fenced ||
 802              (buf->size && buf->size != size) ||
 803              (data == NULL)) {
 804
 805             assert(!block->referenced);
 806
 807             free_block(intel, block);
 808             buf->block = NULL;
 809             buf->dirty = 1;
 810          }
 811       }
 812
 813       buf->size = size;
 814       if (buf->block) {
 815          assert (buf->block->mem->size >= size);
 816       }
 817
 818       if (buf->flags & (BM_NO_BACKING_STORE|BM_NO_EVICT)) {
 819
 820          assert(intel->locked || data == NULL);
 821
 822          if (data != NULL) {
 823             if (!buf->block && !evict_and_alloc_block(intel, buf)) {
 824                bm->fail = 1;
 825                retval = -1;
 826                goto out;
 827             }
 828
 829             wait_quiescent(intel, buf->block);
 830
 831             DBG("bmBufferData %d offset 0x%x sz 0x%x\n",
 832                 buf->id, buf->block->mem->ofs, size);
 833
 834             assert(buf->block->virtual == buf->block->pool->virtual + buf->block->mem->ofs);
 835
 836             do_memcpy(buf->block->virtual, data, size);
 837          }
 838          buf->dirty = 0;
 839       }
 840       else {
 841                DBG("%s - set buf %d dirty\n", __FUNCTION__, buf->id);
 842          set_dirty(intel, buf);
 843          free_backing_store(intel, buf);
 844
 845          if (data != NULL) {
 846             alloc_backing_store(intel, buf);
 847             do_memcpy(buf->backing_store, data, size);
 848          }
 849       }
 850    }
 851  out:
 852    UNLOCK(bm);
 853    return retval;
 854 }
 855
 856
 857 /* Update the buffer in place, in whatever space it is currently resident:
 858  */
 859 int bmBufferSubData(struct intel_context *intel,
 860                      struct buffer *buf,
 861                      unsigned offset,
 862                      unsigned size,
 863                      const void *data )
 864 {
 865    struct bufmgr *bm = intel->bm;
 866    int retval = 0;
 867
 868    if (size == 0)
 869       return 0;
 870
 871    LOCK(bm);
 872    {
 873       DBG("bmBufferSubdata %d offset 0x%x sz 0x%x\n", buf->id, offset, size);
 874
 875       assert(offset+size <= buf->size);
 876
 877       if (buf->flags & (BM_NO_EVICT|BM_NO_BACKING_STORE)) {
 878
 879          assert(intel->locked);
 880
 881          if (!buf->block && !evict_and_alloc_block(intel, buf)) {
 882             bm->fail = 1;
 883             retval = -1;
 884             goto out;
 885          }
 886
 887          if (!(buf->flags & BM_NO_FENCE_SUBDATA))
 888             wait_quiescent(intel, buf->block);
 889
 890          buf->dirty = 0;
 891
 892          do_memcpy(buf->block->virtual + offset, data, size);
 893       }
 894       else {
 895          DBG("%s - set buf %d dirty\n", __FUNCTION__, buf->id);
 896          set_dirty(intel, buf);
 897
 898          if (buf->backing_store == NULL)
 899             alloc_backing_store(intel, buf);
 900
 901          do_memcpy(buf->backing_store + offset, data, size);
 902       }
 903    }
 904  out:
 905    UNLOCK(bm);
 906    return retval;
 907 }
 908
 909
 910
 911 int bmBufferDataAUB(struct intel_context *intel,
 912                      struct buffer *buf,
 913                      unsigned size,
 914                      const void *data,
 915                      unsigned flags,
 916                      unsigned aubtype,
 917                      unsigned aubsubtype )
 918 {
 919    int retval = bmBufferData(intel, buf, size, data, flags);
 920
 921
 922    /* This only works because in this version of the buffer manager we
 923     * allocate all buffers statically in agp space and so can emit the
 924     * uploads to the aub file with the correct offsets as they happen.
 925     */
 926    if (retval == 0 && data && intel->aub_file) {
 927
 928       if (buf->block && !buf->dirty) {
 929          intel->vtbl.aub_gtt_data(intel,
 930                                       buf->block->mem->ofs,
 931                                       buf->block->virtual,
 932                                       size,
 933                                       aubtype,
 934                                       aubsubtype);
 935          buf->aub_dirty = 0;
 936       }
 937    }
 938
 939    return retval;
 940 }
 941
 942
 943 int bmBufferSubDataAUB(struct intel_context *intel,
 944                         struct buffer *buf,
 945                         unsigned offset,
 946                         unsigned size,
 947                         const void *data,
 948                         unsigned aubtype,
 949                         unsigned aubsubtype )
 950 {
 951    int retval = bmBufferSubData(intel, buf, offset, size, data);
 952
 953
 954    /* This only works because in this version of the buffer manager we
 955     * allocate all buffers statically in agp space and so can emit the
 956     * uploads to the aub file with the correct offsets as they happen.
 957     */
 958    if (intel->aub_file) {
 959       if (retval == 0 && buf->block && !buf->dirty)
 960          intel->vtbl.aub_gtt_data(intel,
 961                                       buf->block->mem->ofs + offset,
 962                                       ((const char *)buf->block->virtual) + offset,
 963                                       size,
 964                                       aubtype,
 965                                       aubsubtype);
 966    }
 967
 968    return retval;
 969 }
 970
 971 void bmUnmapBufferAUB( struct intel_context *intel,
 972                        struct buffer *buf,
 973                        unsigned aubtype,
 974                        unsigned aubsubtype )
 975 {
 976    bmUnmapBuffer(intel, buf);
 977
 978    if (intel->aub_file) {
 979       /* Hack - exclude the framebuffer mappings.  If you removed
 980        * this, you'd get very big aubfiles, but you *would* be able to
 981        * see fallback rendering.
 982        */
 983       if (buf->block  && !buf->dirty && buf->block->pool == &intel->bm->pool[0]) {
 984          buf->aub_dirty = 1;
 985       }
 986    }
 987 }
 988
 989 unsigned bmBufferOffset(struct intel_context *intel,
 990                         struct buffer *buf)
 991 {
 992    struct bufmgr *bm = intel->bm;
 993    unsigned retval = 0;
 994
 995    LOCK(bm);
 996    {
 997       assert(intel->locked);
 998
 999       if (!buf->block &&
1000           !evict_and_alloc_block(intel, buf)) {
1001          bm->fail = 1;
1002          retval = ~0;
1003       }
1004       else {
1005          assert(buf->block);
1006          assert(buf->block->buf == buf);
1007
1008          DBG("Add buf %d (block %p, dirty %d) to referenced list\n", buf->id, buf->block,
1009              buf->dirty);
1010
1011          move_to_tail(&bm->referenced, buf->block);
1012          buf->block->referenced = 1;
1013
1014          retval = buf->block->mem->ofs;
1015       }
1016    }
1017    UNLOCK(bm);
1018
1019    return retval;
1020 }
1021
1022
1023
1024 /* Extract data from the buffer:
1025  */
1026 void bmBufferGetSubData(struct intel_context *intel,
1027                         struct buffer *buf,
1028                         unsigned offset,
1029                         unsigned size,
1030                         void *data )
1031 {
1032    struct bufmgr *bm = intel->bm;
1033
1034    LOCK(bm);
1035    {
1036       DBG("bmBufferSubdata %d offset 0x%x sz 0x%x\n", buf->id, offset, size);
1037
1038       if (buf->flags & (BM_NO_EVICT|BM_NO_BACKING_STORE)) {
1039          if (buf->block && size) {
1040             wait_quiescent(intel, buf->block);
1041             do_memcpy(data, buf->block->virtual + offset, size);
1042          }
1043       }
1044       else {
1045          if (buf->backing_store && size) {
1046             do_memcpy(data, buf->backing_store + offset, size);
1047          }
1048       }
1049    }
1050    UNLOCK(bm);
1051 }
1052
1053
1054 /* Return a pointer to whatever space the buffer is currently resident in:
1055  */
1056 void *bmMapBuffer( struct intel_context *intel,
1057                    struct buffer *buf,
1058                    unsigned flags )
1059 {
1060    struct bufmgr *bm = intel->bm;
1061    void *retval = NULL;
1062
1063    LOCK(bm);
1064    {
1065       DBG("bmMapBuffer %d\n", buf->id);
1066
1067       if (buf->mapped) {
1068          _mesa_printf("%s: already mapped\n", __FUNCTION__);
1069          retval = NULL;
1070       }
1071       else if (buf->flags & (BM_NO_BACKING_STORE|BM_NO_EVICT)) {
1072
1073          assert(intel->locked);
1074
1075          if (!buf->block && !evict_and_alloc_block(intel, buf)) {
1076             DBG("%s: alloc failed\n", __FUNCTION__);
1077             bm->fail = 1;
1078             retval = NULL;
1079          }
1080          else {
1081             assert(buf->block);
1082             buf->dirty = 0;
1083
1084             if (!(buf->flags & BM_NO_FENCE_SUBDATA))
1085                wait_quiescent(intel, buf->block);
1086
1087             buf->mapped = 1;
1088             retval = buf->block->virtual;
1089          }
1090       }
1091       else {
1092          DBG("%s - set buf %d dirty\n", __FUNCTION__, buf->id);
1093          set_dirty(intel, buf);
1094
1095          if (buf->backing_store == 0)
1096             alloc_backing_store(intel, buf);
1097
1098          buf->mapped = 1;
1099          retval = buf->backing_store;
1100       }
1101    }
1102    UNLOCK(bm);
1103    return retval;
1104 }
1105
1106 void bmUnmapBuffer( struct intel_context *intel, struct buffer *buf )
1107 {
1108    struct bufmgr *bm = intel->bm;
1109
1110    LOCK(bm);
1111    {
1112       DBG("bmUnmapBuffer %d\n", buf->id);
1113       buf->mapped = 0;
1114    }
1115    UNLOCK(bm);
1116 }
1117
1118
1119
1120
1121 /* This is the big hack that turns on BM_NO_BACKING_STORE.  Basically
1122  * says that an external party will maintain the backing store, eg
1123  * Mesa's local copy of texture data.
1124  */
1125 void bmBufferSetInvalidateCB(struct intel_context *intel,
1126                              struct buffer *buf,
1127                              void (*invalidate_cb)( struct intel_context *, void *ptr ),
1128                              void *ptr,
1129                              GLboolean dont_fence_subdata)
1130 {
1131    struct bufmgr *bm = intel->bm;
1132
1133    LOCK(bm);
1134    {
1135       if (buf->backing_store)
1136          free_backing_store(intel, buf);
1137
1138       buf->flags |= BM_NO_BACKING_STORE;
1139
1140       if (dont_fence_subdata)
1141          buf->flags |= BM_NO_FENCE_SUBDATA;
1142
1143       DBG("bmBufferSetInvalidateCB set buf %d dirty\n", buf->id);
1144       buf->dirty = 1;
1145       buf->invalidate_cb = invalidate_cb;
1146       buf->invalidate_ptr = ptr;
1147
1148       /* Note that it is invalid right from the start.  Also note
1149        * invalidate_cb is called with the bufmgr locked, so cannot
1150        * itself make bufmgr calls.
1151        */
1152       invalidate_cb( intel, ptr );
1153    }
1154    UNLOCK(bm);
1155 }
1156
1157
1158
1159
1160
1161
1162
1163 /* This is only protected against thread interactions by the DRI lock
1164  * and the policy of ensuring that all dma is flushed prior to
1165  * releasing that lock.  Otherwise you might have two threads building
1166  * up a list of buffers to validate at once.
1167  */
1168 int bmValidateBuffers( struct intel_context *intel )
1169 {
1170    struct bufmgr *bm = intel->bm;
1171    int retval = 0;
1172
1173    LOCK(bm);
1174    {
1175       DBG("%s fail %d\n", __FUNCTION__, bm->fail);
1176       assert(intel->locked);
1177
1178       if (!bm->fail) {
1179          struct block *block, *tmp;
1180
1181          foreach_s(block, tmp, &bm->referenced) {
1182             struct buffer *buf = block->buf;
1183
1184             DBG("Validate buf %d / block %p / dirty %d\n", buf->id, block, buf->dirty);
1185
1186             /* Upload the buffer contents if necessary:
1187              */
1188             if (buf->dirty) {
1189                DBG("Upload dirty buf %d (%s) sz %d offset 0x%x\n", buf->id,
1190                    buf->name, buf->size, block->mem->ofs);
1191
1192                assert(!(buf->flags & (BM_NO_BACKING_STORE|BM_NO_EVICT)));
1193
1194                wait_quiescent(intel, buf->block);
1195
1196                do_memcpy(buf->block->virtual,
1197                          buf->backing_store,
1198                          buf->size);
1199
1200                if (intel->aub_file) {
1201                   intel->vtbl.aub_gtt_data(intel,
1202                                                buf->block->mem->ofs,
1203                                                buf->backing_store,
1204                                                buf->size,
1205                                                0,
1206                                                0);
1207                }
1208
1209                buf->dirty = 0;
1210                buf->aub_dirty = 0;
1211             }
1212             else if (buf->aub_dirty) {
1213                intel->vtbl.aub_gtt_data(intel,
1214                                             buf->block->mem->ofs,
1215                                             buf->block->virtual,
1216                                             buf->size,
1217                                             0,
1218                                             0);
1219                buf->aub_dirty = 0;
1220             }
1221
1222             block->referenced = 0;
1223             block->on_hardware = 1;
1224             move_to_tail(&bm->on_hardware, block);
1225          }
1226
1227          bm->need_fence = 1;
1228       }
1229
1230       retval = bm->fail ? -1 : 0;
1231    }
1232    UNLOCK(bm);
1233
1234
1235    if (retval != 0)
1236       DBG("%s failed\n", __FUNCTION__);
1237
1238    return retval;
1239 }
1240
1241
1242
1243
1244 void bmReleaseBuffers( struct intel_context *intel )
1245 {
1246    struct bufmgr *bm = intel->bm;
1247
1248    LOCK(bm);
1249    {
1250       struct block *block, *tmp;
1251
1252       foreach_s (block, tmp, &bm->referenced) {
1253
1254          DBG("remove block %p from referenced list\n", block);
1255
1256          if (block->on_hardware) {
1257             /* Return to the on-hardware list.
1258              */
1259             move_to_tail(&bm->on_hardware, block);
1260          }
1261          else if (block->fenced) {
1262             struct block *s;
1263
1264             /* Hmm - have to scan the fenced list to insert the
1265              * buffers in order.  This is O(nm), but rare and the
1266              * numbers are low.
1267              */
1268             foreach (s, &bm->fenced) {
1269                if (FENCE_LTE(block->fence, s->fence))
1270                   break;
1271             }
1272
1273             move_to_tail(s, block);
1274          }
1275          else {
1276             /* Return to the lru list:
1277              */
1278             move_to_tail(&block->pool->lru, block);
1279          }
1280
1281          block->referenced = 0;
1282       }
1283    }
1284    UNLOCK(bm);
1285 }
1286
1287
1288 /* This functionality is used by the buffer manager, not really sure
1289  * if we need to be exposing it in this way, probably libdrm will
1290  * offer equivalent calls.
1291  *
1292  * For now they can stay, but will likely change/move before final:
1293  */
1294 unsigned bmSetFence( struct intel_context *intel )
1295 {
1296    assert(intel->locked);
1297
1298    /* Emit MI_FLUSH here:
1299     */
1300    if (intel->bm->need_fence) {
1301
1302       /* Emit a flush without using a batchbuffer.  Can't rely on the
1303        * batchbuffer at this level really.  Would really prefer that
1304        * the IRQ ioctly emitted the flush at the same time.
1305        */
1306       GLuint dword[2];
1307       dword[0] = intel->vtbl.flush_cmd();
1308       dword[1] = 0;
1309       intel_cmd_ioctl(intel, (char *)&dword, sizeof(dword));
1310
1311       intel->bm->last_fence = intelEmitIrqLocked( intel );
1312
1313       fence_blocks(intel, intel->bm->last_fence);
1314
1315       intel->vtbl.note_fence(intel, intel->bm->last_fence);
1316       intel->bm->need_fence = 0;
1317
1318       if (intel->thrashing) {
1319          intel->thrashing--;
1320          if (!intel->thrashing)
1321             DBG("not thrashing\n");
1322       }
1323
1324       intel->bm->free_on_hardware = 0;
1325    }
1326
1327    return intel->bm->last_fence;
1328 }
1329
1330 unsigned bmSetFenceLock( struct intel_context *intel )
1331 {
1332   unsigned last;
1333   LOCK(intel->bm);
1334   last = bmSetFence(intel);
1335   UNLOCK(intel->bm);
1336   return last;
1337 }
1338 unsigned bmLockAndFence( struct intel_context *intel )
1339 {
1340    if (intel->bm->need_fence) {
1341       LOCK_HARDWARE(intel);
1342       LOCK(intel->bm);
1343       bmSetFence(intel);
1344       UNLOCK(intel->bm);
1345       UNLOCK_HARDWARE(intel);
1346    }
1347
1348    return intel->bm->last_fence;
1349 }
1350
1351
1352 void bmFinishFence( struct intel_context *intel, unsigned fence )
1353 {
1354    if (!bmTestFence(intel, fence)) {
1355       DBG("...wait on fence %d\n", fence);
1356       intelWaitIrq( intel, fence );
1357    }
1358    assert(bmTestFence(intel, fence));
1359    check_fenced(intel);
1360 }
1361
1362 void bmFinishFenceLock( struct intel_context *intel, unsigned fence )
1363 {
1364    LOCK(intel->bm);
1365    bmFinishFence(intel, fence);
1366    UNLOCK(intel->bm);
1367 }
1368
1369
1370 /* Specifically ignore texture memory sharing.
1371  *  -- just evict everything
1372  *  -- and wait for idle
1373  */
1374 void bm_fake_NotifyContendedLockTake( struct intel_context *intel )
1375 {
1376    struct bufmgr *bm = intel->bm;
1377
1378    LOCK(bm);
1379    {
1380       struct block *block, *tmp;
1381       GLuint i;
1382
1383       assert(is_empty_list(&bm->referenced));
1384
1385       bm->need_fence = 1;
1386       bm->fail = 0;
1387       bmFinishFence(intel, bmSetFence(intel));
1388
1389       assert(is_empty_list(&bm->fenced));
1390       assert(is_empty_list(&bm->on_hardware));
1391
1392       for (i = 0; i < bm->nr_pools; i++) {
1393          if (!(bm->pool[i].flags & BM_NO_EVICT)) {
1394             foreach_s(block, tmp, &bm->pool[i].lru) {
1395                assert(bmTestFence(intel, block->fence));
1396                set_dirty(intel, block->buf);
1397             }
1398          }
1399       }
1400    }
1401    UNLOCK(bm);
1402 }
1403
1404
1405
1406 void bmEvictAll( struct intel_context *intel )
1407 {
1408    struct bufmgr *bm = intel->bm;
1409
1410    LOCK(bm);
1411    {
1412       struct block *block, *tmp;
1413       GLuint i;
1414
1415       DBG("%s\n", __FUNCTION__);
1416
1417       assert(is_empty_list(&bm->referenced));
1418
1419       bm->need_fence = 1;
1420       bm->fail = 0;
1421       bmFinishFence(intel, bmSetFence(intel));
1422
1423       assert(is_empty_list(&bm->fenced));
1424       assert(is_empty_list(&bm->on_hardware));
1425
1426       for (i = 0; i < bm->nr_pools; i++) {
1427          if (!(bm->pool[i].flags & BM_NO_EVICT)) {
1428             foreach_s(block, tmp, &bm->pool[i].lru) {
1429                assert(bmTestFence(intel, block->fence));
1430                set_dirty(intel, block->buf);
1431                block->buf->block = NULL;
1432
1433                free_block(intel, block);
1434             }
1435          }
1436       }
1437    }
1438    UNLOCK(bm);
1439 }
1440
1441
1442 GLboolean bmError( struct intel_context *intel )
1443 {
1444    struct bufmgr *bm = intel->bm;
1445    GLboolean retval;
1446
1447    LOCK(bm);
1448    {
1449       retval = bm->fail;
1450    }
1451    UNLOCK(bm);
1452
1453    return retval;
1454 }
1455
1456
1457 GLuint bmCtxId( struct intel_context *intel )
1458 {
1459    return intel->bm->ctxId;
1460 }