src/mesa/drivers/dri/i965/bufmgr_fake.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 /* Originally a fake version of the buffer manager so that we can
  29  * prototype the changes in a driver fairly quickly, has been fleshed
  30  * out to a fully functional interim solution.
  31  *
  32  * Basically wraps the old style memory management in the new
  33  * programming interface, but is more expressive and avoids many of
  34  * the bugs in the old texture manager.
  35  */
  36 #include "bufmgr.h"
  37
  38 #include "intel_context.h"
  39 #include "intel_ioctl.h"
  40 #include "intel_batchbuffer.h"
  41
  42 #include "simple_list.h"
  43 #include "mm.h"
  44 #include "imports.h"
  45
  46 #define BM_POOL_MAX 8
  47
  48 /* Internal flags:
  49  */
  50 #define BM_NO_BACKING_STORE   0x2000
  51 #define BM_NO_FENCE_SUBDATA   0x4000
  52
  53
  54 static int check_fenced( struct intel_context *intel );
  55
  56 static int nr_attach = 0;
  57
  58 /* Wrapper around mm.c's mem_block, which understands that you must
  59  * wait for fences to expire before memory can be freed.  This is
  60  * specific to our use of memcpy for uploads - an upload that was
  61  * processed through the command queue wouldn't need to care about
  62  * fences.
  63  */
  64 struct block {
  65    struct block *next, *prev;
  66    struct pool *pool;           /* BM_MEM_AGP */
  67    struct mem_block *mem;       /* BM_MEM_AGP */
  68
  69    unsigned referenced:1;
  70    unsigned on_hardware:1;
  71    unsigned fenced:1;
  72
  73
  74    unsigned fence;              /* BM_MEM_AGP, Split to read_fence, write_fence */
  75
  76    struct buffer *buf;
  77    void *virtual;
  78 };
  79
  80
  81 struct buffer {
  82    unsigned id;                 /* debug only */
  83    const char *name;
  84    unsigned size;
  85
  86    unsigned mapped:1;
  87    unsigned dirty:1;
  88    unsigned aub_dirty:1;
  89    unsigned alignment:13;
  90    unsigned flags:16;
  91
  92    struct block *block;
  93    void *backing_store;
  94    void (*invalidate_cb)( struct intel_context *, void * );
  95    void *invalidate_ptr;
  96 };
  97
  98 struct pool {
  99    unsigned size;
 100    unsigned low_offset;
 101    struct buffer *static_buffer;
 102    unsigned flags;
 103    struct mem_block *heap;
 104    void *virtual;
 105    struct block lru;            /* only allocated, non-fence-pending blocks here */
 106 };
 107
 108 struct bufmgr {
 109    _glthread_Mutex mutex;       /**< for thread safety */
 110    struct pool pool[BM_POOL_MAX];
 111    unsigned nr_pools;
 112
 113    unsigned buf_nr;             /* for generating ids */
 114
 115    struct block referenced;     /* after bmBufferOffset */
 116    struct block on_hardware;    /* after bmValidateBuffers */
 117    struct block fenced;         /* after bmFenceBuffers (mi_flush, emit irq, write dword) */
 118                                 /* then to pool->lru or free() */
 119
 120    unsigned last_fence;
 121    unsigned free_on_hardware;
 122
 123    unsigned fail:1;
 124    unsigned need_fence:1;
 125 };
 126
 127 #define MAXFENCE 0x7fffffff
 128
 129 static GLboolean FENCE_LTE( unsigned a, unsigned b )
 130 {
 131    if (a == b)
 132       return GL_TRUE;
 133
 134    if (a < b && b - a < (1<<24))
 135       return GL_TRUE;
 136
 137    if (a > b && MAXFENCE - a + b < (1<<24))
 138       return GL_TRUE;
 139
 140    return GL_FALSE;
 141 }
 142
 143 int bmTestFence( struct intel_context *intel, unsigned fence )
 144 {
 145    /* Slight problem with wrap-around:
 146     */
 147    return fence == 0 || FENCE_LTE(fence, intel->sarea->last_dispatch);
 148 }
 149
 150 #define LOCK(bm) \
 151   int dolock = nr_attach > 1; \
 152   if (dolock) _glthread_LOCK_MUTEX(bm->mutex)
 153
 154 #define UNLOCK(bm) \
 155   if (dolock) _glthread_UNLOCK_MUTEX(bm->mutex)
 156
 157
 158
 159 static GLboolean alloc_from_pool( struct intel_context *intel,
 160                                   unsigned pool_nr,
 161                                   struct buffer *buf )
 162 {
 163    struct bufmgr *bm = intel->bm;
 164    struct pool *pool = &bm->pool[pool_nr];
 165    struct block *block = (struct block *)calloc(sizeof *block, 1);
 166    GLuint sz, align = (1<<buf->alignment);
 167
 168    if (!block)
 169       return GL_FALSE;
 170
 171    sz = (buf->size + align-1) & ~(align-1);
 172
 173    block->mem = mmAllocMem(pool->heap,
 174                            sz,
 175                            buf->alignment, 0);
 176    if (!block->mem) {
 177       free(block);
 178       return GL_FALSE;
 179    }
 180
 181    make_empty_list(block);
 182
 183    /* Insert at head or at tail???
 184     */
 185    insert_at_tail(&pool->lru, block);
 186
 187    block->pool = pool;
 188    block->virtual = pool->virtual + block->mem->ofs;
 189    block->buf = buf;
 190
 191    buf->block = block;
 192
 193    return GL_TRUE;
 194 }
 195
 196
 197
 198
 199
 200
 201
 202
 203 /* Release the card storage associated with buf:
 204  */
 205 static void free_block( struct intel_context *intel, struct block *block )
 206 {
 207    DBG("free block %p\n", block);
 208
 209    if (!block)
 210       return;
 211
 212    check_fenced(intel);
 213
 214    if (block->referenced) {
 215       _mesa_printf("tried to free block on referenced list\n");
 216       assert(0);
 217    }
 218    else if (block->on_hardware) {
 219       block->buf = NULL;
 220       intel->bm->free_on_hardware += block->mem->size;
 221    }
 222    else if (block->fenced) {
 223       block->buf = NULL;
 224    }
 225    else {
 226       DBG("    - free immediately\n");
 227       remove_from_list(block);
 228
 229       mmFreeMem(block->mem);
 230       free(block);
 231    }
 232 }
 233
 234
 235 static void alloc_backing_store( struct intel_context *intel, struct buffer *buf )
 236 {
 237    assert(!buf->backing_store);
 238    assert(!(buf->flags & (BM_NO_EVICT|BM_NO_BACKING_STORE)));
 239
 240    buf->backing_store = ALIGN_MALLOC(buf->size, 64);
 241 }
 242
 243 static void free_backing_store( struct intel_context *intel, struct buffer *buf )
 244 {
 245    assert(!(buf->flags & (BM_NO_EVICT|BM_NO_BACKING_STORE)));
 246
 247    if (buf->backing_store) {
 248       ALIGN_FREE(buf->backing_store);
 249       buf->backing_store = NULL;
 250    }
 251 }
 252
 253
 254
 255
 256
 257
 258 static void set_dirty( struct intel_context *intel,
 259                               struct buffer *buf )
 260 {
 261    if (buf->flags & BM_NO_BACKING_STORE)
 262       buf->invalidate_cb(intel, buf->invalidate_ptr);
 263
 264    assert(!(buf->flags & BM_NO_EVICT));
 265
 266    DBG("set_dirty - buf %d\n", buf->id);
 267    buf->dirty = 1;
 268 }
 269
 270
 271 static int evict_lru( struct intel_context *intel, GLuint max_fence, GLuint *pool )
 272 {
 273    struct bufmgr *bm = intel->bm;
 274    struct block *block, *tmp;
 275    int i;
 276
 277    DBG("%s\n", __FUNCTION__);
 278
 279    for (i = 0; i < bm->nr_pools; i++) {
 280       if (!(bm->pool[i].flags & BM_NO_EVICT)) {
 281          foreach_s(block, tmp, &bm->pool[i].lru) {
 282
 283             if (block->buf &&
 284                 (block->buf->flags & BM_NO_FENCE_SUBDATA))
 285                continue;
 286
 287             if (block->fence && max_fence &&
 288                 !FENCE_LTE(block->fence, max_fence))
 289                return 0;
 290
 291             set_dirty(intel, block->buf);
 292             block->buf->block = NULL;
 293
 294             free_block(intel, block);
 295             *pool = i;
 296             return 1;
 297          }
 298       }
 299    }
 300
 301
 302    return 0;
 303 }
 304
 305
 306 #define foreach_s_rev(ptr, t, list)   \
 307         for(ptr=(list)->prev,t=(ptr)->prev; list != ptr; ptr=t, t=(t)->prev)
 308
 309 static int evict_mru( struct intel_context *intel, GLuint *pool )
 310 {
 311    struct bufmgr *bm = intel->bm;
 312    struct block *block, *tmp;
 313    int i;
 314
 315    DBG("%s\n", __FUNCTION__);
 316
 317    for (i = 0; i < bm->nr_pools; i++) {
 318       if (!(bm->pool[i].flags & BM_NO_EVICT)) {
 319          foreach_s_rev(block, tmp, &bm->pool[i].lru) {
 320
 321             if (block->buf &&
 322                 (block->buf->flags & BM_NO_FENCE_SUBDATA))
 323                continue;
 324
 325             set_dirty(intel, block->buf);
 326             block->buf->block = NULL;
 327
 328             free_block(intel, block);
 329             *pool = i;
 330             return 1;
 331          }
 332       }
 333    }
 334
 335
 336    return 0;
 337 }
 338
 339
 340
 341 static int check_fenced( struct intel_context *intel )
 342 {
 343    struct bufmgr *bm = intel->bm;
 344    struct block *block, *tmp;
 345    int ret = 0;
 346
 347    foreach_s(block, tmp, &bm->fenced ) {
 348       assert(block->fenced);
 349
 350       if (bmTestFence(intel, block->fence)) {
 351
 352          block->fenced = 0;
 353
 354          if (!block->buf) {
 355             DBG("delayed free: offset %x sz %x\n", block->mem->ofs, block->mem->size);
 356             remove_from_list(block);
 357             mmFreeMem(block->mem);
 358             free(block);
 359          }
 360          else {
 361             DBG("return to lru: offset %x sz %x\n", block->mem->ofs, block->mem->size);
 362             move_to_tail(&block->pool->lru, block);
 363          }
 364
 365          ret = 1;
 366       }
 367       else {
 368          /* Blocks are ordered by fence, so if one fails, all from
 369           * here will fail also:
 370           */
 371          break;
 372       }
 373    }
 374
 375    /* Also check the referenced list:
 376     */
 377    foreach_s(block, tmp, &bm->referenced ) {
 378       if (block->fenced &&
 379           bmTestFence(intel, block->fence)) {
 380          block->fenced = 0;
 381       }
 382    }
 383
 384
 385    DBG("%s: %d\n", __FUNCTION__, ret);
 386    return ret;
 387 }
 388
 389
 390
 391 static void fence_blocks( struct intel_context *intel,
 392                           unsigned fence )
 393 {
 394    struct bufmgr *bm = intel->bm;
 395    struct block *block, *tmp;
 396
 397    foreach_s (block, tmp, &bm->on_hardware) {
 398       DBG("Fence block %p (sz 0x%x buf %p) with fence %d\n", block,
 399           block->mem->size, block->buf, fence);
 400       block->fence = fence;
 401
 402       block->on_hardware = 0;
 403       block->fenced = 1;
 404
 405       /* Move to tail of pending list here
 406        */
 407       move_to_tail(&bm->fenced, block);
 408    }
 409
 410    /* Also check the referenced list:
 411     */
 412    foreach_s (block, tmp, &bm->referenced) {
 413       if (block->on_hardware) {
 414          DBG("Fence block %p (sz 0x%x buf %p) with fence %d\n", block,
 415              block->mem->size, block->buf, fence);
 416
 417          block->fence = fence;
 418          block->on_hardware = 0;
 419          block->fenced = 1;
 420       }
 421    }
 422
 423
 424    bm->last_fence = fence;
 425    assert(is_empty_list(&bm->on_hardware));
 426 }
 427
 428
 429
 430
 431 static GLboolean alloc_block( struct intel_context *intel,
 432                               struct buffer *buf )
 433 {
 434    struct bufmgr *bm = intel->bm;
 435    int i;
 436
 437    assert(intel->locked);
 438
 439    DBG("%s 0x%x bytes (%s)\n", __FUNCTION__, buf->size, buf->name);
 440
 441    for (i = 0; i < bm->nr_pools; i++) {
 442       if (!(bm->pool[i].flags & BM_NO_ALLOC) &&
 443           alloc_from_pool(intel, i, buf)) {
 444
 445          DBG("%s --> 0x%x (sz %x)\n", __FUNCTION__,
 446              buf->block->mem->ofs, buf->block->mem->size);
 447
 448          return GL_TRUE;
 449       }
 450    }
 451
 452    DBG("%s --> fail\n", __FUNCTION__);
 453    return GL_FALSE;
 454 }
 455
 456
 457 static GLboolean evict_and_alloc_block( struct intel_context *intel,
 458                                         struct buffer *buf )
 459 {
 460    GLuint pool;
 461    struct bufmgr *bm = intel->bm;
 462
 463    assert(buf->block == NULL);
 464
 465    /* Put a cap on the amount of free memory we'll allow to accumulate
 466     * before emitting a fence.
 467     */
 468    if (bm->free_on_hardware > 1 * 1024 * 1024) {
 469       DBG("fence for free space: %x\n", bm->free_on_hardware);
 470       bmSetFence(intel);
 471    }
 472
 473    /* Search for already free memory:
 474     */
 475    if (alloc_block(intel, buf))
 476       return GL_TRUE;
 477
 478    /* Look for memory that may have become free:
 479     */
 480    if (check_fenced(intel) &&
 481        alloc_block(intel, buf))
 482       return GL_TRUE;
 483
 484    /* Look for memory blocks not used for >1 frame:
 485     */
 486    while (evict_lru(intel, intel->second_last_swap_fence, &pool))
 487       if (alloc_from_pool(intel, pool, buf))
 488          return GL_TRUE;
 489
 490    /* If we're not thrashing, allow lru eviction to dig deeper into
 491     * recently used textures.  We'll probably be thrashing soon:
 492     */
 493    if (!intel->thrashing) {
 494       while (evict_lru(intel, 0, &pool))
 495          if (alloc_from_pool(intel, pool, buf))
 496             return GL_TRUE;
 497    }
 498
 499    /* Keep thrashing counter alive?
 500     */
 501    if (intel->thrashing)
 502       intel->thrashing = 20;
 503
 504    /* Wait on any already pending fences - here we are waiting for any
 505     * freed memory that has been submitted to hardware and fenced to
 506     * become available:
 507     */
 508    while (!is_empty_list(&bm->fenced)) {
 509       GLuint fence = bm->fenced.next->fence;
 510       bmFinishFence(intel, fence);
 511
 512       if (alloc_block(intel, buf))
 513          return GL_TRUE;
 514    }
 515
 516
 517    /*
 518     */
 519    if (!is_empty_list(&bm->on_hardware)) {
 520       bmSetFence(intel);
 521
 522       while (!is_empty_list(&bm->fenced)) {
 523          GLuint fence = bm->fenced.next->fence;
 524          bmFinishFence(intel, fence);
 525       }
 526
 527       if (!intel->thrashing) {
 528          DBG("thrashing\n");
 529       }
 530       intel->thrashing = 20;
 531
 532       if (alloc_block(intel, buf))
 533          return GL_TRUE;
 534    }
 535
 536    while (evict_mru(intel, &pool))
 537       if (alloc_from_pool(intel, pool, buf))
 538          return GL_TRUE;
 539
 540    DBG("%s 0x%x bytes failed\n", __FUNCTION__, buf->size);
 541
 542    assert(is_empty_list(&bm->on_hardware));
 543    assert(is_empty_list(&bm->fenced));
 544
 545    return GL_FALSE;
 546 }
 547
 548
 549
 550
 551
 552
 553
 554
 555
 556
 557 /***********************************************************************
 558  * Public functions
 559  */
 560
 561
 562 /* The initialization functions are skewed in the fake implementation.
 563  * This call would be to attach to an existing manager, rather than to
 564  * create a local one.
 565  */
 566 struct bufmgr *bm_fake_intel_Attach( struct intel_context *intel )
 567 {
 568    _glthread_DECLARE_STATIC_MUTEX(initMutex);
 569    static struct bufmgr bm;
 570
 571    /* This function needs a mutex of its own...
 572     */
 573    _glthread_LOCK_MUTEX(initMutex);
 574
 575    if (nr_attach == 0) {
 576       _glthread_INIT_MUTEX(bm.mutex);
 577
 578       make_empty_list(&bm.referenced);
 579       make_empty_list(&bm.fenced);
 580       make_empty_list(&bm.on_hardware);
 581    }
 582
 583    nr_attach++;
 584
 585    _glthread_UNLOCK_MUTEX(initMutex);
 586
 587    return &bm;
 588 }
 589
 590
 591
 592 /* The virtual pointer would go away in a true implementation.
 593  */
 594 int bmInitPool( struct intel_context *intel,
 595                 unsigned long low_offset,
 596                 void *low_virtual,
 597                 unsigned long size,
 598                 unsigned flags)
 599 {
 600    struct bufmgr *bm = intel->bm;
 601    int retval = 0;
 602
 603    LOCK(bm);
 604    {
 605       GLuint i;
 606
 607       for (i = 0; i < bm->nr_pools; i++) {
 608          if (bm->pool[i].low_offset == low_offset &&
 609              bm->pool[i].size == size) {
 610             retval = i;
 611             goto out;
 612          }
 613       }
 614
 615
 616       if (bm->nr_pools >= BM_POOL_MAX)
 617          retval = -1;
 618       else {
 619          i = bm->nr_pools++;
 620
 621          DBG("bmInitPool %d low_offset %x sz %x\n",
 622              i, low_offset, size);
 623
 624          bm->pool[i].low_offset = low_offset;
 625          bm->pool[i].size = size;
 626          bm->pool[i].heap = mmInit( low_offset, size );
 627          bm->pool[i].virtual = low_virtual - low_offset;
 628          bm->pool[i].flags = flags;
 629
 630          make_empty_list(&bm->pool[i].lru);
 631
 632          retval = i;
 633       }
 634    }
 635  out:
 636    UNLOCK(bm);
 637    return retval;
 638 }
 639
 640 static struct buffer *do_GenBuffer(struct intel_context *intel, const char *name, int align)
 641 {
 642    struct bufmgr *bm = intel->bm;
 643    struct buffer *buf = calloc(sizeof(*buf), 1);
 644
 645    buf->id = ++bm->buf_nr;
 646    buf->name = name;
 647    buf->alignment = align;
 648    buf->flags = BM_MEM_AGP|BM_MEM_VRAM|BM_MEM_LOCAL;
 649
 650    return buf;
 651 }
 652
 653
 654 void *bmFindVirtual( struct intel_context *intel,
 655                      unsigned int offset,
 656                      size_t sz )
 657 {
 658    struct bufmgr *bm = intel->bm;
 659    int i;
 660
 661    for (i = 0; i < bm->nr_pools; i++)
 662       if (offset >= bm->pool[i].low_offset &&
 663           offset + sz <= bm->pool[i].low_offset + bm->pool[i].size)
 664          return bm->pool[i].virtual + offset;
 665
 666    return NULL;
 667 }
 668
 669
 670 void bmGenBuffers(struct intel_context *intel,
 671                   const char *name, unsigned n,
 672                   struct buffer **buffers,
 673                   int align )
 674 {
 675    struct bufmgr *bm = intel->bm;
 676    LOCK(bm);
 677    {
 678       int i;
 679
 680       for (i = 0; i < n; i++)
 681          buffers[i] = do_GenBuffer(intel, name, align);
 682    }
 683    UNLOCK(bm);
 684 }
 685
 686
 687 void bmDeleteBuffers(struct intel_context *intel, unsigned n, struct buffer **buffers)
 688 {
 689    struct bufmgr *bm = intel->bm;
 690
 691    LOCK(bm);
 692    {
 693       unsigned i;
 694
 695       for (i = 0; i < n; i++) {
 696          struct buffer *buf = buffers[i];
 697
 698          if (buf && buf->block)
 699             free_block(intel, buf->block);
 700
 701          if (buf)
 702             free(buf);
 703       }
 704    }
 705    UNLOCK(bm);
 706 }
 707
 708
 709
 710
 711 /* Hook to inform faked buffer manager about fixed-position
 712  * front,depth,back buffers.  These may move to a fully memory-managed
 713  * scheme, or they may continue to be managed as is.  It will probably
 714  * be useful to pass a fixed offset here one day.
 715  */
 716 struct buffer *bmGenBufferStatic(struct intel_context *intel,
 717                                  unsigned pool )
 718 {
 719    struct bufmgr *bm = intel->bm;
 720    struct buffer *buf;
 721    LOCK(bm);
 722    {
 723       assert(bm->pool[pool].flags & BM_NO_EVICT);
 724       assert(bm->pool[pool].flags & BM_NO_MOVE);
 725
 726       if (bm->pool[pool].static_buffer)
 727          buf = bm->pool[pool].static_buffer;
 728       else {
 729          buf = do_GenBuffer(intel, "static", 12);
 730
 731          bm->pool[pool].static_buffer = buf;
 732          assert(!buf->block);
 733
 734          buf->size = bm->pool[pool].size;
 735          buf->flags = bm->pool[pool].flags;
 736          buf->alignment = 12;
 737
 738          if (!alloc_from_pool(intel, pool, buf))
 739             assert(0);
 740       }
 741    }
 742    UNLOCK(bm);
 743    return buf;
 744 }
 745
 746
 747 static void wait_quiescent(struct intel_context *intel,
 748                            struct block *block)
 749 {
 750    if (block->on_hardware) {
 751       assert(intel->bm->need_fence);
 752       bmSetFence(intel);
 753       assert(!block->on_hardware);
 754    }
 755
 756
 757    if (block->fenced) {
 758       bmFinishFence(intel, block->fence);
 759    }
 760
 761    assert(!block->on_hardware);
 762    assert(!block->fenced);
 763 }
 764
 765
 766
 767 /* If buffer size changes, free and reallocate.  Otherwise update in
 768  * place.
 769  */
 770 int bmBufferData(struct intel_context *intel,
 771                  struct buffer *buf,
 772                  unsigned size,
 773                  const void *data,
 774                  unsigned flags )
 775 {
 776    struct bufmgr *bm = intel->bm;
 777    int retval = 0;
 778
 779    LOCK(bm);
 780    {
 781       DBG("bmBufferData %d sz 0x%x data: %p\n", buf->id, size, data);
 782
 783       assert(!buf->mapped);
 784
 785       if (buf->block) {
 786          struct block *block = buf->block;
 787
 788          /* Optimistic check to see if we can reuse the block -- not
 789           * required for correctness:
 790           */
 791          if (block->fenced)
 792             check_fenced(intel);
 793
 794          if (block->on_hardware ||
 795              block->fenced ||
 796              (buf->size && buf->size != size) ||
 797              (data == NULL)) {
 798
 799             assert(!block->referenced);
 800
 801             free_block(intel, block);
 802             buf->block = NULL;
 803             buf->dirty = 1;
 804          }
 805       }
 806
 807       buf->size = size;
 808       if (buf->block) {
 809          assert (buf->block->mem->size >= size);
 810       }
 811
 812       if (buf->flags & (BM_NO_BACKING_STORE|BM_NO_EVICT)) {
 813
 814          assert(intel->locked || data == NULL);
 815
 816          if (data != NULL) {
 817             if (!buf->block && !evict_and_alloc_block(intel, buf)) {
 818                bm->fail = 1;
 819                retval = -1;
 820                goto out;
 821             }
 822
 823             wait_quiescent(intel, buf->block);
 824
 825             DBG("bmBufferData %d offset 0x%x sz 0x%x\n",
 826                 buf->id, buf->block->mem->ofs, size);
 827
 828             assert(buf->block->virtual == buf->block->pool->virtual + buf->block->mem->ofs);
 829
 830             do_memcpy(buf->block->virtual, data, size);
 831          }
 832          buf->dirty = 0;
 833       }
 834       else {
 835                DBG("%s - set buf %d dirty\n", __FUNCTION__, buf->id);
 836          set_dirty(intel, buf);
 837          free_backing_store(intel, buf);
 838
 839          if (data != NULL) {
 840             alloc_backing_store(intel, buf);
 841             do_memcpy(buf->backing_store, data, size);
 842          }
 843       }
 844    }
 845  out:
 846    UNLOCK(bm);
 847    return retval;
 848 }
 849
 850
 851 /* Update the buffer in place, in whatever space it is currently resident:
 852  */
 853 int bmBufferSubData(struct intel_context *intel,
 854                      struct buffer *buf,
 855                      unsigned offset,
 856                      unsigned size,
 857                      const void *data )
 858 {
 859    struct bufmgr *bm = intel->bm;
 860    int retval = 0;
 861
 862    if (size == 0)
 863       return 0;
 864
 865    LOCK(bm);
 866    {
 867       DBG("bmBufferSubdata %d offset 0x%x sz 0x%x\n", buf->id, offset, size);
 868
 869       assert(offset+size <= buf->size);
 870
 871       if (buf->flags & (BM_NO_EVICT|BM_NO_BACKING_STORE)) {
 872
 873          assert(intel->locked);
 874
 875          if (!buf->block && !evict_and_alloc_block(intel, buf)) {
 876             bm->fail = 1;
 877             retval = -1;
 878             goto out;
 879          }
 880
 881          if (!(buf->flags & BM_NO_FENCE_SUBDATA))
 882             wait_quiescent(intel, buf->block);
 883
 884          buf->dirty = 0;
 885
 886          do_memcpy(buf->block->virtual + offset, data, size);
 887       }
 888       else {
 889          DBG("%s - set buf %d dirty\n", __FUNCTION__, buf->id);
 890          set_dirty(intel, buf);
 891
 892          if (buf->backing_store == NULL)
 893             alloc_backing_store(intel, buf);
 894
 895          do_memcpy(buf->backing_store + offset, data, size);
 896       }
 897    }
 898  out:
 899    UNLOCK(bm);
 900    return retval;
 901 }
 902
 903
 904
 905 int bmBufferDataAUB(struct intel_context *intel,
 906                      struct buffer *buf,
 907                      unsigned size,
 908                      const void *data,
 909                      unsigned flags,
 910                      unsigned aubtype,
 911                      unsigned aubsubtype )
 912 {
 913    int retval = bmBufferData(intel, buf, size, data, flags);
 914
 915
 916    /* This only works because in this version of the buffer manager we
 917     * allocate all buffers statically in agp space and so can emit the
 918     * uploads to the aub file with the correct offsets as they happen.
 919     */
 920    if (retval == 0 && data && intel->aub_file) {
 921
 922       if (buf->block && !buf->dirty) {
 923          intel->vtbl.aub_gtt_data(intel,
 924                                       buf->block->mem->ofs,
 925                                       buf->block->virtual,
 926                                       size,
 927                                       aubtype,
 928                                       aubsubtype);
 929          buf->aub_dirty = 0;
 930       }
 931    }
 932
 933    return retval;
 934 }
 935
 936
 937 int bmBufferSubDataAUB(struct intel_context *intel,
 938                         struct buffer *buf,
 939                         unsigned offset,
 940                         unsigned size,
 941                         const void *data,
 942                         unsigned aubtype,
 943                         unsigned aubsubtype )
 944 {
 945    int retval = bmBufferSubData(intel, buf, offset, size, data);
 946
 947
 948    /* This only works because in this version of the buffer manager we
 949     * allocate all buffers statically in agp space and so can emit the
 950     * uploads to the aub file with the correct offsets as they happen.
 951     */
 952    if (intel->aub_file) {
 953       if (retval == 0 && buf->block && !buf->dirty)
 954          intel->vtbl.aub_gtt_data(intel,
 955                                       buf->block->mem->ofs + offset,
 956                                       ((const char *)buf->block->virtual) + offset,
 957                                       size,
 958                                       aubtype,
 959                                       aubsubtype);
 960    }
 961
 962    return retval;
 963 }
 964
 965 void bmUnmapBufferAUB( struct intel_context *intel,
 966                        struct buffer *buf,
 967                        unsigned aubtype,
 968                        unsigned aubsubtype )
 969 {
 970    bmUnmapBuffer(intel, buf);
 971
 972    if (intel->aub_file) {
 973       /* Hack - exclude the framebuffer mappings.  If you removed
 974        * this, you'd get very big aubfiles, but you *would* be able to
 975        * see fallback rendering.
 976        */
 977       if (buf->block  && !buf->dirty && buf->block->pool == &intel->bm->pool[0]) {
 978          buf->aub_dirty = 1;
 979       }
 980    }
 981 }
 982
 983 unsigned bmBufferOffset(struct intel_context *intel,
 984                         struct buffer *buf)
 985 {
 986    struct bufmgr *bm = intel->bm;
 987    unsigned retval = 0;
 988
 989    LOCK(bm);
 990    {
 991       assert(intel->locked);
 992
 993       if (!buf->block &&
 994           !evict_and_alloc_block(intel, buf)) {
 995          bm->fail = 1;
 996          retval = ~0;
 997       }
 998       else {
 999          assert(buf->block);
1000          assert(buf->block->buf == buf);
1001
1002          DBG("Add buf %d (block %p, dirty %d) to referenced list\n", buf->id, buf->block,
1003              buf->dirty);
1004
1005          move_to_tail(&bm->referenced, buf->block);
1006          buf->block->referenced = 1;
1007
1008          retval = buf->block->mem->ofs;
1009       }
1010    }
1011    UNLOCK(bm);
1012
1013    return retval;
1014 }
1015
1016
1017
1018 /* Extract data from the buffer:
1019  */
1020 void bmBufferGetSubData(struct intel_context *intel,
1021                         struct buffer *buf,
1022                         unsigned offset,
1023                         unsigned size,
1024                         void *data )
1025 {
1026    struct bufmgr *bm = intel->bm;
1027
1028    LOCK(bm);
1029    {
1030       DBG("bmBufferSubdata %d offset 0x%x sz 0x%x\n", buf->id, offset, size);
1031
1032       if (buf->flags & (BM_NO_EVICT|BM_NO_BACKING_STORE)) {
1033          if (buf->block && size) {
1034             wait_quiescent(intel, buf->block);
1035             do_memcpy(data, buf->block->virtual + offset, size);
1036          }
1037       }
1038       else {
1039          if (buf->backing_store && size) {
1040             do_memcpy(data, buf->backing_store + offset, size);
1041          }
1042       }
1043    }
1044    UNLOCK(bm);
1045 }
1046
1047
1048 /* Return a pointer to whatever space the buffer is currently resident in:
1049  */
1050 void *bmMapBuffer( struct intel_context *intel,
1051                    struct buffer *buf,
1052                    unsigned flags )
1053 {
1054    struct bufmgr *bm = intel->bm;
1055    void *retval = NULL;
1056
1057    LOCK(bm);
1058    {
1059       DBG("bmMapBuffer %d\n", buf->id);
1060
1061       if (buf->mapped) {
1062          _mesa_printf("%s: already mapped\n", __FUNCTION__);
1063          retval = NULL;
1064       }
1065       else if (buf->flags & (BM_NO_BACKING_STORE|BM_NO_EVICT)) {
1066
1067          assert(intel->locked);
1068
1069          if (!buf->block && !evict_and_alloc_block(intel, buf)) {
1070             DBG("%s: alloc failed\n", __FUNCTION__);
1071             bm->fail = 1;
1072             retval = NULL;
1073          }
1074          else {
1075             assert(buf->block);
1076             buf->dirty = 0;
1077
1078             if (!(buf->flags & BM_NO_FENCE_SUBDATA))
1079                wait_quiescent(intel, buf->block);
1080
1081             buf->mapped = 1;
1082             retval = buf->block->virtual;
1083          }
1084       }
1085       else {
1086          DBG("%s - set buf %d dirty\n", __FUNCTION__, buf->id);
1087          set_dirty(intel, buf);
1088
1089          if (buf->backing_store == 0)
1090             alloc_backing_store(intel, buf);
1091
1092          buf->mapped = 1;
1093          retval = buf->backing_store;
1094       }
1095    }
1096    UNLOCK(bm);
1097    return retval;
1098 }
1099
1100 void bmUnmapBuffer( struct intel_context *intel, struct buffer *buf )
1101 {
1102    struct bufmgr *bm = intel->bm;
1103
1104    LOCK(bm);
1105    {
1106       DBG("bmUnmapBuffer %d\n", buf->id);
1107       buf->mapped = 0;
1108    }
1109    UNLOCK(bm);
1110 }
1111
1112
1113
1114
1115 /* This is the big hack that turns on BM_NO_BACKING_STORE.  Basically
1116  * says that an external party will maintain the backing store, eg
1117  * Mesa's local copy of texture data.
1118  */
1119 void bmBufferSetInvalidateCB(struct intel_context *intel,
1120                              struct buffer *buf,
1121                              void (*invalidate_cb)( struct intel_context *, void *ptr ),
1122                              void *ptr,
1123                              GLboolean dont_fence_subdata)
1124 {
1125    struct bufmgr *bm = intel->bm;
1126
1127    LOCK(bm);
1128    {
1129       if (buf->backing_store)
1130          free_backing_store(intel, buf);
1131
1132       buf->flags |= BM_NO_BACKING_STORE;
1133
1134       if (dont_fence_subdata)
1135          buf->flags |= BM_NO_FENCE_SUBDATA;
1136
1137       DBG("bmBufferSetInvalidateCB set buf %d dirty\n", buf->id);
1138       buf->dirty = 1;
1139       buf->invalidate_cb = invalidate_cb;
1140       buf->invalidate_ptr = ptr;
1141
1142       /* Note that it is invalid right from the start.  Also note
1143        * invalidate_cb is called with the bufmgr locked, so cannot
1144        * itself make bufmgr calls.
1145        */
1146       invalidate_cb( intel, ptr );
1147    }
1148    UNLOCK(bm);
1149 }
1150
1151
1152
1153
1154
1155
1156
1157 /* This is only protected against thread interactions by the DRI lock
1158  * and the policy of ensuring that all dma is flushed prior to
1159  * releasing that lock.  Otherwise you might have two threads building
1160  * up a list of buffers to validate at once.
1161  */
1162 int bmValidateBuffers( struct intel_context *intel )
1163 {
1164    struct bufmgr *bm = intel->bm;
1165    int retval = 0;
1166
1167    LOCK(bm);
1168    {
1169       DBG("%s fail %d\n", __FUNCTION__, bm->fail);
1170       assert(intel->locked);
1171
1172       if (!bm->fail) {
1173          struct block *block, *tmp;
1174
1175          foreach_s(block, tmp, &bm->referenced) {
1176             struct buffer *buf = block->buf;
1177
1178             DBG("Validate buf %d / block %p / dirty %d\n", buf->id, block, buf->dirty);
1179
1180             /* Upload the buffer contents if necessary:
1181              */
1182             if (buf->dirty) {
1183                DBG("Upload dirty buf %d (%s) sz %d offset 0x%x\n", buf->id,
1184                    buf->name, buf->size, block->mem->ofs);
1185
1186                assert(!(buf->flags & (BM_NO_BACKING_STORE|BM_NO_EVICT)));
1187
1188                wait_quiescent(intel, buf->block);
1189
1190                do_memcpy(buf->block->virtual,
1191                          buf->backing_store,
1192                          buf->size);
1193
1194                if (intel->aub_file) {
1195                   intel->vtbl.aub_gtt_data(intel,
1196                                                buf->block->mem->ofs,
1197                                                buf->backing_store,
1198                                                buf->size,
1199                                                0,
1200                                                0);
1201                }
1202
1203                buf->dirty = 0;
1204                buf->aub_dirty = 0;
1205             }
1206             else if (buf->aub_dirty) {
1207                intel->vtbl.aub_gtt_data(intel,
1208                                             buf->block->mem->ofs,
1209                                             buf->block->virtual,
1210                                             buf->size,
1211                                             0,
1212                                             0);
1213                buf->aub_dirty = 0;
1214             }
1215
1216             block->referenced = 0;
1217             block->on_hardware = 1;
1218             move_to_tail(&bm->on_hardware, block);
1219          }
1220
1221          bm->need_fence = 1;
1222       }
1223
1224       retval = bm->fail ? -1 : 0;
1225    }
1226    UNLOCK(bm);
1227
1228
1229    if (retval != 0)
1230       DBG("%s failed\n", __FUNCTION__);
1231
1232    return retval;
1233 }
1234
1235
1236
1237
1238 void bmReleaseBuffers( struct intel_context *intel )
1239 {
1240    struct bufmgr *bm = intel->bm;
1241
1242    LOCK(bm);
1243    {
1244       struct block *block, *tmp;
1245       assert(intel->locked);
1246
1247       foreach_s (block, tmp, &bm->referenced) {
1248
1249          DBG("remove block %p from referenced list\n", block);
1250
1251          if (block->on_hardware) {
1252             /* Return to the on-hardware list.
1253              */
1254             move_to_tail(&bm->on_hardware, block);
1255          }
1256          else if (block->fenced) {
1257             struct block *s;
1258
1259             /* Hmm - have to scan the fenced list to insert the
1260              * buffers in order.  This is O(nm), but rare and the
1261              * numbers are low.
1262              */
1263             foreach (s, &bm->fenced) {
1264                if (FENCE_LTE(block->fence, s->fence))
1265                   break;
1266             }
1267
1268             move_to_tail(s, block);
1269          }
1270          else {
1271             /* Return to the lru list:
1272              */
1273             move_to_tail(&block->pool->lru, block);
1274          }
1275
1276          block->referenced = 0;
1277       }
1278    }
1279    UNLOCK(bm);
1280 }
1281
1282
1283 /* This functionality is used by the buffer manager, not really sure
1284  * if we need to be exposing it in this way, probably libdrm will
1285  * offer equivalent calls.
1286  *
1287  * For now they can stay, but will likely change/move before final:
1288  */
1289 unsigned bmSetFence( struct intel_context *intel )
1290 {
1291    assert(intel->locked);
1292
1293    /* Emit MI_FLUSH here:
1294     */
1295    if (intel->bm->need_fence) {
1296
1297       /* Emit a flush without using a batchbuffer.  Can't rely on the
1298        * batchbuffer at this level really.  Would really prefer that
1299        * the IRQ ioctly emitted the flush at the same time.
1300        */
1301       GLuint dword[2];
1302       dword[0] = intel->vtbl.flush_cmd();
1303       dword[1] = 0;
1304       intel_cmd_ioctl(intel, (char *)&dword, sizeof(dword), GL_TRUE);
1305
1306       intel->bm->last_fence = intelEmitIrqLocked( intel );
1307
1308       fence_blocks(intel, intel->bm->last_fence);
1309
1310       intel->vtbl.note_fence(intel, intel->bm->last_fence);
1311       intel->bm->need_fence = 0;
1312
1313       if (intel->thrashing) {
1314          intel->thrashing--;
1315          if (!intel->thrashing)
1316             DBG("not thrashing\n");
1317       }
1318
1319       intel->bm->free_on_hardware = 0;
1320    }
1321
1322    return intel->bm->last_fence;
1323 }
1324
1325 unsigned bmLockAndFence( struct intel_context *intel )
1326 {
1327    if (intel->bm->need_fence) {
1328       LOCK_HARDWARE(intel);
1329       bmSetFence(intel);
1330       UNLOCK_HARDWARE(intel);
1331    }
1332
1333    return intel->bm->last_fence;
1334 }
1335
1336
1337 void bmFinishFence( struct intel_context *intel, unsigned fence )
1338 {
1339    if (!bmTestFence(intel, fence)) {
1340       DBG("...wait on fence %d\n", fence);
1341       intelWaitIrq( intel, fence );
1342    }
1343    assert(bmTestFence(intel, fence));
1344    check_fenced(intel);
1345 }
1346
1347
1348
1349
1350 /* Specifically ignore texture memory sharing.
1351  *  -- just evict everything
1352  *  -- and wait for idle
1353  */
1354 void bm_fake_NotifyContendedLockTake( struct intel_context *intel )
1355 {
1356    struct bufmgr *bm = intel->bm;
1357
1358    LOCK(bm);
1359    {
1360       struct block *block, *tmp;
1361       GLuint i;
1362
1363       assert(is_empty_list(&bm->referenced));
1364
1365       bm->need_fence = 1;
1366       bm->fail = 0;
1367       bmFinishFence(intel, bmSetFence(intel));
1368
1369       assert(is_empty_list(&bm->fenced));
1370       assert(is_empty_list(&bm->on_hardware));
1371
1372       for (i = 0; i < bm->nr_pools; i++) {
1373          if (!(bm->pool[i].flags & BM_NO_EVICT)) {
1374             foreach_s(block, tmp, &bm->pool[i].lru) {
1375                assert(bmTestFence(intel, block->fence));
1376                set_dirty(intel, block->buf);
1377             }
1378          }
1379       }
1380    }
1381    UNLOCK(bm);
1382 }
1383
1384
1385
1386 void bmEvictAll( struct intel_context *intel )
1387 {
1388    struct bufmgr *bm = intel->bm;
1389
1390    LOCK(bm);
1391    {
1392       struct block *block, *tmp;
1393       GLuint i;
1394
1395       DBG("%s\n", __FUNCTION__);
1396
1397       assert(is_empty_list(&bm->referenced));
1398
1399       bm->need_fence = 1;
1400       bm->fail = 0;
1401       bmFinishFence(intel, bmSetFence(intel));
1402
1403       assert(is_empty_list(&bm->fenced));
1404       assert(is_empty_list(&bm->on_hardware));
1405
1406       for (i = 0; i < bm->nr_pools; i++) {
1407          if (!(bm->pool[i].flags & BM_NO_EVICT)) {
1408             foreach_s(block, tmp, &bm->pool[i].lru) {
1409                assert(bmTestFence(intel, block->fence));
1410                set_dirty(intel, block->buf);
1411                block->buf->block = NULL;
1412
1413                free_block(intel, block);
1414             }
1415          }
1416       }
1417    }
1418    UNLOCK(bm);
1419 }
1420
1421
1422 GLboolean bmError( struct intel_context *intel )
1423 {
1424    struct bufmgr *bm = intel->bm;
1425    GLboolean retval;
1426
1427    LOCK(bm);
1428    {
1429       retval = bm->fail;
1430    }
1431    UNLOCK(bm);
1432
1433    return retval;
1434 }