src/mesa/drivers/dri/i965/bufmgr_fake.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 /* Originally a fake version of the buffer manager so that we can
  29  * prototype the changes in a driver fairly quickly, has been fleshed
  30  * out to a fully functional interim solution.
  31  *
  32  * Basically wraps the old style memory management in the new
  33  * programming interface, but is more expressive and avoids many of
  34  * the bugs in the old texture manager.
  35  */
  36 #include "bufmgr.h"
  37
  38 #include "intel_context.h"
  39 #include "intel_ioctl.h"
  40 #include "intel_batchbuffer.h"
  41
  42 #include "simple_list.h"
  43 #include "mm.h"
  44 #include "imports.h"
  45
  46 #define BM_POOL_MAX 8
  47
  48 /* Internal flags:
  49  */
  50 #define BM_NO_BACKING_STORE   0x2000
  51 #define BM_NO_FENCE_SUBDATA   0x4000
  52
  53
  54 static int check_fenced( struct intel_context *intel );
  55
  56 static int nr_attach = 0;
  57
  58 /* Wrapper around mm.c's mem_block, which understands that you must
  59  * wait for fences to expire before memory can be freed.  This is
  60  * specific to our use of memcpy for uploads - an upload that was
  61  * processed through the command queue wouldn't need to care about
  62  * fences.
  63  */
  64 struct block {
  65    struct block *next, *prev;
  66    struct pool *pool;           /* BM_MEM_AGP */
  67    struct mem_block *mem;       /* BM_MEM_AGP */
  68
  69    unsigned referenced:1;
  70    unsigned on_hardware:1;
  71    unsigned fenced:1;
  72
  73
  74    unsigned fence;              /* BM_MEM_AGP, Split to read_fence, write_fence */
  75
  76    struct buffer *buf;
  77    void *virtual;
  78 };
  79
  80
  81 struct buffer {
  82    unsigned id;                 /* debug only */
  83    const char *name;
  84    unsigned size;
  85
  86    unsigned mapped:1;
  87    unsigned dirty:1;
  88    unsigned aub_dirty:1;
  89    unsigned alignment:13;
  90    unsigned flags:16;
  91
  92    struct block *block;
  93    void *backing_store;
  94    void (*invalidate_cb)( struct intel_context *, void * );
  95    void *invalidate_ptr;
  96 };
  97
  98 struct pool {
  99    unsigned size;
 100    unsigned low_offset;
 101    struct buffer *static_buffer;
 102    unsigned flags;
 103    struct mem_block *heap;
 104    void *virtual;
 105    struct block lru;            /* only allocated, non-fence-pending blocks here */
 106 };
 107
 108 struct bufmgr {
 109    _glthread_Mutex mutex;       /**< for thread safety */
 110    struct pool pool[BM_POOL_MAX];
 111    unsigned nr_pools;
 112
 113    unsigned buf_nr;             /* for generating ids */
 114
 115    struct block referenced;     /* after bmBufferOffset */
 116    struct block on_hardware;    /* after bmValidateBuffers */
 117    struct block fenced;         /* after bmFenceBuffers (mi_flush, emit irq, write dword) */
 118                                 /* then to pool->lru or free() */
 119
 120    unsigned last_fence;
 121    unsigned free_on_hardware;
 122
 123    unsigned fail:1;
 124    unsigned need_fence:1;
 125 };
 126
 127 #define MAXFENCE 0x7fffffff
 128
 129 static GLboolean FENCE_LTE( unsigned a, unsigned b )
 130 {
 131    if (a == b)
 132       return GL_TRUE;
 133
 134    if (a < b && b - a < (1<<24))
 135       return GL_TRUE;
 136
 137    if (a > b && MAXFENCE - a + b < (1<<24))
 138       return GL_TRUE;
 139
 140    return GL_FALSE;
 141 }
 142
 143 int bmTestFence( struct intel_context *intel, unsigned fence )
 144 {
 145    /* Slight problem with wrap-around:
 146     */
 147    return fence == 0 || FENCE_LTE(fence, intel->sarea->last_dispatch);
 148 }
 149
 150 #define LOCK(bm) \
 151   int dolock = nr_attach > 1; \
 152   if (dolock) _glthread_LOCK_MUTEX(bm->mutex)
 153
 154 #define UNLOCK(bm) \
 155   if (dolock) _glthread_UNLOCK_MUTEX(bm->mutex)
 156
 157
 158
 159 static GLboolean alloc_from_pool( struct intel_context *intel,
 160                                   unsigned pool_nr,
 161                                   struct buffer *buf )
 162 {
 163    struct bufmgr *bm = intel->bm;
 164    struct pool *pool = &bm->pool[pool_nr];
 165    struct block *block = (struct block *)calloc(sizeof *block, 1);
 166    GLuint sz, align = (1<<buf->alignment);
 167
 168    if (!block)
 169       return GL_FALSE;
 170
 171    sz = (buf->size + align-1) & ~(align-1);
 172
 173    block->mem = mmAllocMem(pool->heap,
 174                            sz,
 175                            buf->alignment, 0);
 176    if (!block->mem) {
 177       free(block);
 178       return GL_FALSE;
 179    }
 180
 181    make_empty_list(block);
 182
 183    /* Insert at head or at tail???
 184     */
 185    insert_at_tail(&pool->lru, block);
 186
 187    block->pool = pool;
 188    block->virtual = pool->virtual + block->mem->ofs;
 189    block->buf = buf;
 190
 191    buf->block = block;
 192
 193    return GL_TRUE;
 194 }
 195
 196
 197
 198
 199
 200
 201
 202
 203 /* Release the card storage associated with buf:
 204  */
 205 static void free_block( struct intel_context *intel, struct block *block )
 206 {
 207    DBG("free block %p\n", block);
 208
 209    if (!block)
 210       return;
 211
 212    check_fenced(intel);
 213
 214    if (block->referenced) {
 215       _mesa_printf("tried to free block on referenced list\n");
 216       assert(0);
 217    }
 218    else if (block->on_hardware) {
 219       block->buf = NULL;
 220       intel->bm->free_on_hardware += block->mem->size;
 221    }
 222    else if (block->fenced) {
 223       block->buf = NULL;
 224    }
 225    else {
 226       DBG("    - free immediately\n");
 227       remove_from_list(block);
 228
 229       mmFreeMem(block->mem);
 230       free(block);
 231    }
 232 }
 233
 234
 235 static void alloc_backing_store( struct intel_context *intel, struct buffer *buf )
 236 {
 237    assert(!buf->backing_store);
 238    assert(!(buf->flags & (BM_NO_EVICT|BM_NO_BACKING_STORE)));
 239
 240    buf->backing_store = ALIGN_MALLOC(buf->size, 64);
 241 }
 242
 243 static void free_backing_store( struct intel_context *intel, struct buffer *buf )
 244 {
 245    assert(!(buf->flags & (BM_NO_EVICT|BM_NO_BACKING_STORE)));
 246
 247    if (buf->backing_store) {
 248       ALIGN_FREE(buf->backing_store);
 249       buf->backing_store = NULL;
 250    }
 251 }
 252
 253
 254
 255
 256
 257
 258 static void set_dirty( struct intel_context *intel,
 259                               struct buffer *buf )
 260 {
 261    if (buf->flags & BM_NO_BACKING_STORE)
 262       buf->invalidate_cb(intel, buf->invalidate_ptr);
 263
 264    assert(!(buf->flags & BM_NO_EVICT));
 265
 266    DBG("set_dirty - buf %d\n", buf->id);
 267    buf->dirty = 1;
 268 }
 269
 270
 271 static int evict_lru( struct intel_context *intel, GLuint max_fence )
 272 {
 273    struct bufmgr *bm = intel->bm;
 274    struct block *block, *tmp;
 275    int i;
 276
 277    DBG("%s\n", __FUNCTION__);
 278
 279    for (i = 0; i < bm->nr_pools; i++) {
 280       if (!(bm->pool[i].flags & BM_NO_EVICT)) {
 281          foreach_s(block, tmp, &bm->pool[i].lru) {
 282
 283             if (block->buf &&
 284                 (block->buf->flags & BM_NO_FENCE_SUBDATA))
 285                continue;
 286
 287             if (block->fence && max_fence &&
 288                 !FENCE_LTE(block->fence, max_fence))
 289                return 0;
 290
 291             set_dirty(intel, block->buf);
 292             block->buf->block = NULL;
 293
 294             free_block(intel, block);
 295             return 1;
 296          }
 297       }
 298    }
 299
 300
 301    return 0;
 302 }
 303
 304
 305 #define foreach_s_rev(ptr, t, list)   \
 306         for(ptr=(list)->prev,t=(ptr)->prev; list != ptr; ptr=t, t=(t)->prev)
 307
 308 static int evict_mru( struct intel_context *intel)
 309 {
 310    struct bufmgr *bm = intel->bm;
 311    struct block *block, *tmp;
 312    int i;
 313
 314    DBG("%s\n", __FUNCTION__);
 315
 316    for (i = 0; i < bm->nr_pools; i++) {
 317       if (!(bm->pool[i].flags & BM_NO_EVICT)) {
 318          foreach_s_rev(block, tmp, &bm->pool[i].lru) {
 319
 320             if (block->buf &&
 321                 (block->buf->flags & BM_NO_FENCE_SUBDATA))
 322                continue;
 323
 324             set_dirty(intel, block->buf);
 325             block->buf->block = NULL;
 326
 327             free_block(intel, block);
 328             return 1;
 329          }
 330       }
 331    }
 332
 333
 334    return 0;
 335 }
 336
 337
 338
 339 static int check_fenced( struct intel_context *intel )
 340 {
 341    struct bufmgr *bm = intel->bm;
 342    struct block *block, *tmp;
 343    int ret = 0;
 344
 345    foreach_s(block, tmp, &bm->fenced ) {
 346       assert(block->fenced);
 347
 348       if (bmTestFence(intel, block->fence)) {
 349
 350          block->fenced = 0;
 351
 352          if (!block->buf) {
 353             DBG("delayed free: offset %x sz %x\n", block->mem->ofs, block->mem->size);
 354             remove_from_list(block);
 355             mmFreeMem(block->mem);
 356             free(block);
 357          }
 358          else {
 359             DBG("return to lru: offset %x sz %x\n", block->mem->ofs, block->mem->size);
 360             move_to_tail(&block->pool->lru, block);
 361          }
 362
 363          ret = 1;
 364       }
 365       else {
 366          /* Blocks are ordered by fence, so if one fails, all from
 367           * here will fail also:
 368           */
 369          break;
 370       }
 371    }
 372
 373    /* Also check the referenced list:
 374     */
 375    foreach_s(block, tmp, &bm->referenced ) {
 376       if (block->fenced &&
 377           bmTestFence(intel, block->fence)) {
 378          block->fenced = 0;
 379       }
 380    }
 381
 382
 383    DBG("%s: %d\n", __FUNCTION__, ret);
 384    return ret;
 385 }
 386
 387
 388
 389 static void fence_blocks( struct intel_context *intel,
 390                           unsigned fence )
 391 {
 392    struct bufmgr *bm = intel->bm;
 393    struct block *block, *tmp;
 394
 395    foreach_s (block, tmp, &bm->on_hardware) {
 396       DBG("Fence block %p (sz 0x%x buf %p) with fence %d\n", block,
 397           block->mem->size, block->buf, fence);
 398       block->fence = fence;
 399
 400       block->on_hardware = 0;
 401       block->fenced = 1;
 402
 403       /* Move to tail of pending list here
 404        */
 405       move_to_tail(&bm->fenced, block);
 406    }
 407
 408    /* Also check the referenced list:
 409     */
 410    foreach_s (block, tmp, &bm->referenced) {
 411       if (block->on_hardware) {
 412          DBG("Fence block %p (sz 0x%x buf %p) with fence %d\n", block,
 413              block->mem->size, block->buf, fence);
 414
 415          block->fence = fence;
 416          block->on_hardware = 0;
 417          block->fenced = 1;
 418       }
 419    }
 420
 421
 422    bm->last_fence = fence;
 423    assert(is_empty_list(&bm->on_hardware));
 424 }
 425
 426
 427
 428
 429 static GLboolean alloc_block( struct intel_context *intel,
 430                               struct buffer *buf )
 431 {
 432    struct bufmgr *bm = intel->bm;
 433    int i;
 434
 435    DBG("%s 0x%x bytes (%s)\n", __FUNCTION__, buf->size, buf->name);
 436
 437    for (i = 0; i < bm->nr_pools; i++) {
 438       if (!(bm->pool[i].flags & BM_NO_ALLOC) &&
 439           alloc_from_pool(intel, i, buf)) {
 440
 441          DBG("%s --> 0x%x (sz %x)\n", __FUNCTION__,
 442              buf->block->mem->ofs, buf->block->mem->size);
 443
 444          return GL_TRUE;
 445       }
 446    }
 447
 448    DBG("%s --> fail\n", __FUNCTION__);
 449    return GL_FALSE;
 450 }
 451
 452
 453 static GLboolean evict_and_alloc_block( struct intel_context *intel,
 454                                         struct buffer *buf )
 455 {
 456    struct bufmgr *bm = intel->bm;
 457
 458    assert(buf->block == NULL);
 459
 460    /* Put a cap on the amount of free memory we'll allow to accumulate
 461     * before emitting a fence.
 462     */
 463    if (bm->free_on_hardware > 1 * 1024 * 1024) {
 464       DBG("fence for free space: %x\n", bm->free_on_hardware);
 465       bmSetFence(intel);
 466    }
 467
 468    /* Search for already free memory:
 469     */
 470    if (alloc_block(intel, buf))
 471       return GL_TRUE;
 472
 473    /* Look for memory that may have become free:
 474     */
 475    if (check_fenced(intel) &&
 476        alloc_block(intel, buf))
 477       return GL_TRUE;
 478
 479    /* Look for memory blocks not used for >1 frame:
 480     */
 481    while (evict_lru(intel, intel->second_last_swap_fence))
 482       if (alloc_block(intel, buf))
 483          return GL_TRUE;
 484
 485    /* If we're not thrashing, allow lru eviction to dig deeper into
 486     * recently used textures.  We'll probably be thrashing soon:
 487     */
 488    if (!intel->thrashing) {
 489       while (evict_lru(intel, 0))
 490          if (alloc_block(intel, buf))
 491             return GL_TRUE;
 492    }
 493
 494    /* Keep thrashing counter alive?
 495     */
 496    if (intel->thrashing)
 497       intel->thrashing = 20;
 498
 499    /* Wait on any already pending fences - here we are waiting for any
 500     * freed memory that has been submitted to hardware and fenced to
 501     * become available:
 502     */
 503    while (!is_empty_list(&bm->fenced)) {
 504       GLuint fence = bm->fenced.next->fence;
 505       bmFinishFence(intel, fence);
 506
 507       if (alloc_block(intel, buf))
 508          return GL_TRUE;
 509    }
 510
 511
 512    /*
 513     */
 514    if (!is_empty_list(&bm->on_hardware)) {
 515       bmSetFence(intel);
 516
 517       if (!is_empty_list(&bm->fenced)) {
 518          GLuint fence = bm->fenced.next->fence;
 519          bmFinishFence(intel, fence);
 520       }
 521
 522       if (!intel->thrashing) {
 523          DBG("thrashing\n");
 524       }
 525       intel->thrashing = 20;
 526
 527       if (alloc_block(intel, buf))
 528          return GL_TRUE;
 529    }
 530
 531    while (evict_mru(intel))
 532       if (alloc_block(intel, buf))
 533          return GL_TRUE;
 534
 535    return GL_FALSE;
 536 }
 537
 538
 539
 540
 541
 542
 543
 544
 545
 546
 547 /***********************************************************************
 548  * Public functions
 549  */
 550
 551
 552 /* The initialization functions are skewed in the fake implementation.
 553  * This call would be to attach to an existing manager, rather than to
 554  * create a local one.
 555  */
 556 struct bufmgr *bm_fake_intel_Attach( struct intel_context *intel )
 557 {
 558    _glthread_DECLARE_STATIC_MUTEX(initMutex);
 559    static struct bufmgr bm;
 560
 561    /* This function needs a mutex of its own...
 562     */
 563    _glthread_LOCK_MUTEX(initMutex);
 564
 565    if (nr_attach == 0) {
 566       _glthread_INIT_MUTEX(bm.mutex);
 567
 568       make_empty_list(&bm.referenced);
 569       make_empty_list(&bm.fenced);
 570       make_empty_list(&bm.on_hardware);
 571    }
 572
 573    nr_attach++;
 574
 575    _glthread_UNLOCK_MUTEX(initMutex);
 576
 577    return &bm;
 578 }
 579
 580
 581
 582 /* The virtual pointer would go away in a true implementation.
 583  */
 584 int bmInitPool( struct intel_context *intel,
 585                 unsigned long low_offset,
 586                 void *low_virtual,
 587                 unsigned long size,
 588                 unsigned flags)
 589 {
 590    struct bufmgr *bm = intel->bm;
 591    int retval;
 592
 593    LOCK(bm);
 594    {
 595       GLuint i;
 596
 597       for (i = 0; i < bm->nr_pools; i++) {
 598          if (bm->pool[i].low_offset == low_offset &&
 599              bm->pool[i].size == size) {
 600             retval = i;
 601             goto out;
 602          }
 603       }
 604
 605
 606       if (bm->nr_pools >= BM_POOL_MAX)
 607          retval = -1;
 608       else {
 609          i = bm->nr_pools++;
 610
 611          DBG("bmInitPool %d low_offset %x sz %x\n",
 612              i, low_offset, size);
 613
 614          bm->pool[i].low_offset = low_offset;
 615          bm->pool[i].size = size;
 616          bm->pool[i].heap = mmInit( low_offset, size );
 617          bm->pool[i].virtual = low_virtual - low_offset;
 618          bm->pool[i].flags = flags;
 619
 620          make_empty_list(&bm->pool[i].lru);
 621
 622          retval = i;
 623       }
 624    }
 625  out:
 626    UNLOCK(bm);
 627    return retval;
 628 }
 629
 630 static struct buffer *do_GenBuffer(struct intel_context *intel, const char *name, int align)
 631 {
 632    struct bufmgr *bm = intel->bm;
 633    struct buffer *buf = calloc(sizeof(*buf), 1);
 634
 635    buf->id = ++bm->buf_nr;
 636    buf->name = name;
 637    buf->alignment = align ? align : 6;
 638    buf->flags = BM_MEM_AGP|BM_MEM_VRAM|BM_MEM_LOCAL;
 639
 640    return buf;
 641 }
 642
 643
 644
 645 void bmGenBuffers(struct intel_context *intel,
 646                   const char *name, unsigned n,
 647                   struct buffer **buffers,
 648                   int align )
 649 {
 650    struct bufmgr *bm = intel->bm;
 651    LOCK(bm);
 652    {
 653       int i;
 654
 655       for (i = 0; i < n; i++)
 656          buffers[i] = do_GenBuffer(intel, name, align);
 657    }
 658    UNLOCK(bm);
 659 }
 660
 661
 662 void bmDeleteBuffers(struct intel_context *intel, unsigned n, struct buffer **buffers)
 663 {
 664    struct bufmgr *bm = intel->bm;
 665
 666    LOCK(bm);
 667    {
 668       unsigned i;
 669
 670       for (i = 0; i < n; i++) {
 671          struct buffer *buf = buffers[i];
 672
 673          if (buf && buf->block)
 674             free_block(intel, buf->block);
 675
 676          if (buf)
 677             free(buf);
 678       }
 679    }
 680    UNLOCK(bm);
 681 }
 682
 683
 684
 685
 686 /* Hook to inform faked buffer manager about fixed-position
 687  * front,depth,back buffers.  These may move to a fully memory-managed
 688  * scheme, or they may continue to be managed as is.  It will probably
 689  * be useful to pass a fixed offset here one day.
 690  */
 691 struct buffer *bmGenBufferStatic(struct intel_context *intel,
 692                                  unsigned pool )
 693 {
 694    struct bufmgr *bm = intel->bm;
 695    struct buffer *buf;
 696    LOCK(bm);
 697    {
 698       assert(bm->pool[pool].flags & BM_NO_EVICT);
 699       assert(bm->pool[pool].flags & BM_NO_MOVE);
 700
 701       if (bm->pool[pool].static_buffer)
 702          buf = bm->pool[pool].static_buffer;
 703       else {
 704          buf = do_GenBuffer(intel, "static", 12);
 705
 706          bm->pool[pool].static_buffer = buf;
 707          assert(!buf->block);
 708
 709          buf->size = bm->pool[pool].size;
 710          buf->flags = bm->pool[pool].flags;
 711          buf->alignment = 12;
 712
 713          if (!alloc_from_pool(intel, pool, buf))
 714             assert(0);
 715       }
 716    }
 717    UNLOCK(bm);
 718    return buf;
 719 }
 720
 721
 722 static void wait_quiescent(struct intel_context *intel,
 723                            struct block *block)
 724 {
 725    if (block->on_hardware) {
 726       assert(intel->bm->need_fence);
 727       bmSetFence(intel);
 728       assert(!block->on_hardware);
 729    }
 730
 731
 732    if (block->fenced) {
 733       bmFinishFence(intel, block->fence);
 734    }
 735
 736    assert(!block->on_hardware);
 737    assert(!block->fenced);
 738 }
 739
 740
 741
 742 /* If buffer size changes, free and reallocate.  Otherwise update in
 743  * place.
 744  */
 745 void bmBufferData(struct intel_context *intel,
 746                   struct buffer *buf,
 747                   unsigned size,
 748                   const void *data,
 749                   unsigned flags )
 750 {
 751    struct bufmgr *bm = intel->bm;
 752
 753    LOCK(bm);
 754    {
 755       DBG("bmBufferData %d sz 0x%x data: %p\n", buf->id, size, data);
 756
 757       assert(!buf->mapped);
 758
 759       if (buf->block) {
 760          struct block *block = buf->block;
 761
 762          /* Optimistic check to see if we can reuse the block -- not
 763           * required for correctness:
 764           */
 765          if (block->fenced)
 766             check_fenced(intel);
 767
 768          if (block->on_hardware ||
 769              block->fenced ||
 770              (buf->size && buf->size != size) ||
 771              (data == NULL)) {
 772
 773             assert(!block->referenced);
 774
 775             free_block(intel, block);
 776             buf->block = NULL;
 777             buf->dirty = 1;
 778          }
 779       }
 780
 781       buf->size = size;
 782       if (buf->block) {
 783          assert (buf->block->mem->size == size);
 784       }
 785
 786       if (buf->flags & (BM_NO_BACKING_STORE|BM_NO_EVICT)) {
 787          if (data != NULL) {
 788             if (!buf->block && !evict_and_alloc_block(intel, buf))
 789                assert(0);
 790
 791             wait_quiescent(intel, buf->block);
 792
 793             DBG("bmBufferData %d offset 0x%x sz 0x%x\n",
 794                 buf->id, buf->block->mem->ofs, size);
 795
 796             assert(buf->block->virtual == buf->block->pool->virtual + buf->block->mem->ofs);
 797
 798             do_memcpy(buf->block->virtual, data, size);
 799          }
 800          buf->dirty = 0;
 801       }
 802       else {
 803                DBG("%s - set buf %d dirty\n", __FUNCTION__, buf->id);
 804          set_dirty(intel, buf);
 805          free_backing_store(intel, buf);
 806
 807          if (data != NULL) {
 808             alloc_backing_store(intel, buf);
 809             do_memcpy(buf->backing_store, data, size);
 810          }
 811       }
 812    }
 813    UNLOCK(bm);
 814 }
 815
 816
 817 /* Update the buffer in place, in whatever space it is currently resident:
 818  */
 819 void bmBufferSubData(struct intel_context *intel,
 820                      struct buffer *buf,
 821                      unsigned offset,
 822                      unsigned size,
 823                      const void *data )
 824 {
 825    struct bufmgr *bm = intel->bm;
 826
 827    if (size == 0)
 828       return;
 829
 830    LOCK(bm);
 831    {
 832       DBG("bmBufferSubdata %d offset 0x%x sz 0x%x\n", buf->id, offset, size);
 833
 834       assert(offset+size <= buf->size);
 835
 836       if (buf->flags & (BM_NO_EVICT|BM_NO_BACKING_STORE)) {
 837          if (!buf->block && !evict_and_alloc_block(intel, buf))
 838             assert(0);
 839
 840          if (!(buf->flags & BM_NO_FENCE_SUBDATA))
 841             wait_quiescent(intel, buf->block);
 842
 843          buf->dirty = 0;
 844
 845          do_memcpy(buf->block->virtual + offset, data, size);
 846       }
 847       else {
 848          DBG("%s - set buf %d dirty\n", __FUNCTION__, buf->id);
 849          set_dirty(intel, buf);
 850
 851          if (buf->backing_store == NULL)
 852             alloc_backing_store(intel, buf);
 853
 854          do_memcpy(buf->backing_store + offset, data, size);
 855       }
 856    }
 857    UNLOCK(bm);
 858 }
 859
 860
 861
 862 void bmBufferDataAUB(struct intel_context *intel,
 863                      struct buffer *buf,
 864                      unsigned size,
 865                      const void *data,
 866                      unsigned flags,
 867                      unsigned aubtype,
 868                      unsigned aubsubtype )
 869 {
 870    bmBufferData(intel, buf, size, data, flags);
 871
 872
 873    /* This only works because in this version of the buffer manager we
 874     * allocate all buffers statically in agp space and so can emit the
 875     * uploads to the aub file with the correct offsets as they happen.
 876     */
 877    if (data && intel->aub_file) {
 878
 879       if (buf->block && !buf->dirty) {
 880          intel->vtbl.aub_gtt_data(intel,
 881                                       buf->block->mem->ofs,
 882                                       buf->block->virtual,
 883                                       size,
 884                                       aubtype,
 885                                       aubsubtype);
 886          buf->aub_dirty = 0;
 887       }
 888    }
 889 }
 890
 891
 892 void bmBufferSubDataAUB(struct intel_context *intel,
 893                         struct buffer *buf,
 894                         unsigned offset,
 895                         unsigned size,
 896                         const void *data,
 897                         unsigned aubtype,
 898                         unsigned aubsubtype )
 899 {
 900    bmBufferSubData(intel, buf, offset, size, data);
 901
 902
 903    /* This only works because in this version of the buffer manager we
 904     * allocate all buffers statically in agp space and so can emit the
 905     * uploads to the aub file with the correct offsets as they happen.
 906     */
 907    if (intel->aub_file) {
 908       if (buf->block && !buf->dirty)
 909          intel->vtbl.aub_gtt_data(intel,
 910                                       buf->block->mem->ofs + offset,
 911                                       ((const char *)buf->block->virtual) + offset,
 912                                       size,
 913                                       aubtype,
 914                                       aubsubtype);
 915    }
 916 }
 917
 918 void bmUnmapBufferAUB( struct intel_context *intel,
 919                        struct buffer *buf,
 920                        unsigned aubtype,
 921                        unsigned aubsubtype )
 922 {
 923    bmUnmapBuffer(intel, buf);
 924
 925    if (intel->aub_file) {
 926       /* Hack - exclude the framebuffer mappings.  If you removed
 927        * this, you'd get very big aubfiles, but you *would* be able to
 928        * see fallback rendering.
 929        */
 930       if (buf->block  && !buf->dirty && buf->block->pool == &intel->bm->pool[0]) {
 931          buf->aub_dirty = 1;
 932       }
 933    }
 934 }
 935
 936 unsigned bmBufferOffset(struct intel_context *intel,
 937                         struct buffer *buf)
 938 {
 939    struct bufmgr *bm = intel->bm;
 940    unsigned retval;
 941
 942    LOCK(bm);
 943    {
 944       assert(intel->locked);
 945
 946       if (!buf->block &&
 947           !evict_and_alloc_block(intel, buf)) {
 948          bm->fail = 1;
 949          retval = ~0;
 950       }
 951       else {
 952          assert(buf->block);
 953          assert(buf->block->buf == buf);
 954
 955          DBG("Add buf %d (block %p, dirty %d) to referenced list\n", buf->id, buf->block,
 956              buf->dirty);
 957
 958          move_to_tail(&bm->referenced, buf->block);
 959          buf->block->referenced = 1;
 960
 961          retval = buf->block->mem->ofs;
 962       }
 963    }
 964    UNLOCK(bm);
 965
 966    return retval;
 967 }
 968
 969
 970
 971 /* Extract data from the buffer:
 972  */
 973 void bmBufferGetSubData(struct intel_context *intel,
 974                         struct buffer *buf,
 975                         unsigned offset,
 976                         unsigned size,
 977                         void *data )
 978 {
 979    struct bufmgr *bm = intel->bm;
 980
 981    LOCK(bm);
 982    {
 983       DBG("bmBufferSubdata %d offset 0x%x sz 0x%x\n", buf->id, offset, size);
 984
 985       if (buf->flags & (BM_NO_EVICT|BM_NO_BACKING_STORE)) {
 986          if (buf->block && size) {
 987             wait_quiescent(intel, buf->block);
 988             do_memcpy(data, buf->block->virtual + offset, size);
 989          }
 990       }
 991       else {
 992          if (buf->backing_store && size) {
 993             do_memcpy(data, buf->backing_store + offset, size);
 994          }
 995       }
 996    }
 997    UNLOCK(bm);
 998 }
 999
1000
1001 /* Return a pointer to whatever space the buffer is currently resident in:
1002  */
1003 void *bmMapBuffer( struct intel_context *intel,
1004                    struct buffer *buf,
1005                    unsigned flags )
1006 {
1007    struct bufmgr *bm = intel->bm;
1008    void *retval;
1009
1010    LOCK(bm);
1011    {
1012       DBG("bmMapBuffer %d\n", buf->id);
1013
1014       if (buf->mapped) {
1015          _mesa_printf("%s: already mapped\n", __FUNCTION__);
1016          retval = NULL;
1017       }
1018       else if (buf->flags & (BM_NO_BACKING_STORE|BM_NO_EVICT)) {
1019          if (!buf->block && !evict_and_alloc_block(intel, buf)) {
1020             _mesa_printf("%s: alloc failed\n", __FUNCTION__);
1021             retval = NULL;
1022          }
1023          else {
1024             assert(buf->block);
1025             buf->dirty = 0;
1026
1027             if (!(buf->flags & BM_NO_FENCE_SUBDATA))
1028                wait_quiescent(intel, buf->block);
1029
1030             buf->mapped = 1;
1031             retval = buf->block->virtual;
1032          }
1033       }
1034       else {
1035                DBG("%s - set buf %d dirty\n", __FUNCTION__, buf->id);
1036          set_dirty(intel, buf);
1037
1038          if (buf->backing_store == 0)
1039             alloc_backing_store(intel, buf);
1040
1041          buf->mapped = 1;
1042          retval = buf->backing_store;
1043       }
1044    }
1045    UNLOCK(bm);
1046    return retval;
1047 }
1048
1049 void bmUnmapBuffer( struct intel_context *intel, struct buffer *buf )
1050 {
1051    struct bufmgr *bm = intel->bm;
1052
1053    LOCK(bm);
1054    {
1055       DBG("bmUnmapBuffer %d\n", buf->id);
1056       buf->mapped = 0;
1057    }
1058    UNLOCK(bm);
1059 }
1060
1061
1062
1063
1064 /* This is the big hack that turns on BM_NO_BACKING_STORE.  Basically
1065  * says that an external party will maintain the backing store, eg
1066  * Mesa's local copy of texture data.
1067  */
1068 void bmBufferSetInvalidateCB(struct intel_context *intel,
1069                              struct buffer *buf,
1070                              void (*invalidate_cb)( struct intel_context *, void *ptr ),
1071                              void *ptr,
1072                              GLboolean dont_fence_subdata)
1073 {
1074    struct bufmgr *bm = intel->bm;
1075
1076    LOCK(bm);
1077    {
1078       if (buf->backing_store)
1079          free_backing_store(intel, buf);
1080
1081       buf->flags |= BM_NO_BACKING_STORE;
1082
1083       if (dont_fence_subdata)
1084          buf->flags |= BM_NO_FENCE_SUBDATA;
1085
1086       DBG("bmBufferSetInvalidateCB set buf %d dirty\n", buf->id);
1087       buf->dirty = 1;
1088       buf->invalidate_cb = invalidate_cb;
1089       buf->invalidate_ptr = ptr;
1090
1091       /* Note that it is invalid right from the start.  Also note
1092        * invalidate_cb is called with the bufmgr locked, so cannot
1093        * itself make bufmgr calls.
1094        */
1095       invalidate_cb( intel, ptr );
1096    }
1097    UNLOCK(bm);
1098 }
1099
1100
1101
1102
1103
1104
1105
1106 /* This is only protected against thread interactions by the DRI lock
1107  * and the policy of ensuring that all dma is flushed prior to
1108  * releasing that lock.  Otherwise you might have two threads building
1109  * up a list of buffers to validate at once.
1110  */
1111 int bmValidateBuffers( struct intel_context *intel )
1112 {
1113    struct bufmgr *bm = intel->bm;
1114    int retval;
1115
1116    LOCK(bm);
1117    {
1118       DBG("%s fail %d\n", __FUNCTION__, bm->fail);
1119
1120       if (!bm->fail) {
1121          struct block *block, *tmp;
1122
1123          foreach_s(block, tmp, &bm->referenced) {
1124             struct buffer *buf = block->buf;
1125
1126             DBG("Validate buf %d / block %p / dirty %d\n", buf->id, block, buf->dirty);
1127
1128             /* Upload the buffer contents if necessary:
1129              */
1130             if (buf->dirty) {
1131                DBG("Upload dirty buf %d (%s) sz %d offset 0x%x\n", buf->id,
1132                    buf->name, buf->size, block->mem->ofs);
1133
1134                assert(!(buf->flags & (BM_NO_BACKING_STORE|BM_NO_EVICT)));
1135
1136                wait_quiescent(intel, buf->block);
1137
1138                do_memcpy(buf->block->virtual,
1139                          buf->backing_store,
1140                          buf->size);
1141
1142                if (intel->aub_file) {
1143                   intel->vtbl.aub_gtt_data(intel,
1144                                                buf->block->mem->ofs,
1145                                                buf->backing_store,
1146                                                buf->size,
1147                                                0,
1148                                                0);
1149                }
1150
1151                buf->dirty = 0;
1152                buf->aub_dirty = 0;
1153             }
1154             else if (buf->aub_dirty) {
1155                intel->vtbl.aub_gtt_data(intel,
1156                                             buf->block->mem->ofs,
1157                                             buf->block->virtual,
1158                                             buf->size,
1159                                             0,
1160                                             0);
1161                buf->aub_dirty = 0;
1162             }
1163
1164             block->referenced = 0;
1165             block->on_hardware = 1;
1166             move_to_tail(&bm->on_hardware, block);
1167          }
1168
1169          bm->need_fence = 1;
1170       }
1171
1172       retval = !bm->fail;
1173       bm->fail = 0;
1174       assert(is_empty_list(&bm->referenced));
1175    }
1176    UNLOCK(bm);
1177
1178    return retval;
1179 }
1180
1181
1182
1183
1184 void bmReleaseBuffers( struct intel_context *intel )
1185 {
1186    struct bufmgr *bm = intel->bm;
1187
1188    LOCK(bm);
1189    {
1190       struct block *block, *tmp;
1191
1192       foreach_s (block, tmp, &bm->referenced) {
1193
1194          DBG("remove block %p from referenced list\n", block);
1195
1196          if (block->on_hardware) {
1197             /* Return to the on-hardware list.
1198              */
1199             move_to_tail(&bm->on_hardware, block);
1200          }
1201          else if (block->fenced) {
1202             struct block *s;
1203
1204             /* Hmm - have to scan the fenced list to insert the
1205              * buffers in order.  This is O(nm), but rare and the
1206              * numbers are low.
1207              */
1208             foreach (s, &bm->fenced) {
1209                if (FENCE_LTE(block->fence, s->fence))
1210                   break;
1211             }
1212
1213             move_to_tail(s, block);
1214          }
1215          else {
1216             /* Return to the lru list:
1217              */
1218             move_to_tail(&block->pool->lru, block);
1219          }
1220
1221          block->referenced = 0;
1222       }
1223
1224       bm->fail = 0;
1225    }
1226    UNLOCK(bm);
1227 }
1228
1229
1230 /* This functionality is used by the buffer manager, not really sure
1231  * if we need to be exposing it in this way, probably libdrm will
1232  * offer equivalent calls.
1233  *
1234  * For now they can stay, but will likely change/move before final:
1235  */
1236 unsigned bmSetFence( struct intel_context *intel )
1237 {
1238    assert(intel->locked);
1239
1240    /* Emit MI_FLUSH here:
1241     */
1242    if (intel->bm->need_fence) {
1243
1244       /* Emit a flush without using a batchbuffer.  Can't rely on the
1245        * batchbuffer at this level really.  Would really prefer that
1246        * the IRQ ioctly emitted the flush at the same time.
1247        */
1248       GLuint dword[2];
1249       dword[0] = intel->vtbl.flush_cmd();
1250       dword[1] = 0;
1251       intel_cmd_ioctl(intel, (char *)&dword, sizeof(dword), GL_TRUE);
1252
1253       intel->bm->last_fence = intelEmitIrqLocked( intel );
1254
1255       fence_blocks(intel, intel->bm->last_fence);
1256
1257       intel->vtbl.note_fence(intel, intel->bm->last_fence);
1258       intel->bm->need_fence = 0;
1259
1260       if (intel->thrashing) {
1261          intel->thrashing--;
1262          if (!intel->thrashing)
1263             DBG("not thrashing\n");
1264       }
1265
1266       intel->bm->free_on_hardware = 0;
1267    }
1268
1269    return intel->bm->last_fence;
1270 }
1271
1272 unsigned bmLockAndFence( struct intel_context *intel )
1273 {
1274    if (intel->bm->need_fence) {
1275       LOCK_HARDWARE(intel);
1276       bmSetFence(intel);
1277       UNLOCK_HARDWARE(intel);
1278    }
1279
1280    return intel->bm->last_fence;
1281 }
1282
1283
1284 void bmFinishFence( struct intel_context *intel, unsigned fence )
1285 {
1286    if (!bmTestFence(intel, fence)) {
1287       DBG("...wait on fence %d\n", fence);
1288       intelWaitIrq( intel, fence );
1289    }
1290    assert(bmTestFence(intel, fence));
1291    check_fenced(intel);
1292 }
1293
1294
1295
1296
1297 /* Specifically ignore texture memory sharing.
1298  *  -- just evict everything
1299  *  -- and wait for idle
1300  */
1301 void bm_fake_NotifyContendedLockTake( struct intel_context *intel )
1302 {
1303    struct bufmgr *bm = intel->bm;
1304
1305    LOCK(bm);
1306    {
1307       struct block *block, *tmp;
1308       GLuint i;
1309
1310       assert(is_empty_list(&bm->referenced));
1311
1312       bm->need_fence = 1;
1313       bmFinishFence(intel, bmSetFence(intel));
1314
1315       for (i = 0; i < bm->nr_pools; i++) {
1316          if (!(bm->pool[i].flags & BM_NO_EVICT)) {
1317             foreach_s(block, tmp, &bm->pool[i].lru) {
1318                assert(bmTestFence(intel, block->fence));
1319                set_dirty(intel, block->buf);
1320             }
1321          }
1322       }
1323    }
1324    UNLOCK(bm);
1325 }
1326
1327