src/gallium/drivers/svga/svga_resource_buffer_upload.c

   1 /**********************************************************
   2  * Copyright 2008-2009 VMware, Inc.  All rights reserved.
   3  *
   4  * Permission is hereby granted, free of charge, to any person
   5  * obtaining a copy of this software and associated documentation
   6  * files (the "Software"), to deal in the Software without
   7  * restriction, including without limitation the rights to use, copy,
   8  * modify, merge, publish, distribute, sublicense, and/or sell copies
   9  * of the Software, and to permit persons to whom the Software is
  10  * furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice shall be
  13  * included in all copies or substantial portions of the Software.
  14  *
  15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  16  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  17  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  18  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  19  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  20  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  21  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22  * SOFTWARE.
  23  *
  24  **********************************************************/
  25
  26 #include "svga_cmd.h"
  27
  28 #include "pipe/p_state.h"
  29 #include "pipe/p_defines.h"
  30 #include "util/u_inlines.h"
  31 #include "os/os_thread.h"
  32 #include "util/u_math.h"
  33 #include "util/u_memory.h"
  34
  35 #include "svga_context.h"
  36 #include "svga_screen.h"
  37 #include "svga_resource_buffer.h"
  38 #include "svga_resource_buffer_upload.h"
  39 #include "svga_winsys.h"
  40 #include "svga_debug.h"
  41
  42
  43 #define MAX_DMA_SIZE (4 * 1024 * 1024)
  44
  45
  46 /**
  47  * Allocate a winsys_buffer (ie. DMA, aka GMR memory).
  48  *
  49  * It will flush and retry in case the first attempt to create a DMA buffer
  50  * fails, so it should not be called from any function involved in flushing
  51  * to avoid recursion.
  52  */
  53 struct svga_winsys_buffer *
  54 svga_winsys_buffer_create( struct svga_context *svga,
  55                            unsigned alignment,
  56                            unsigned usage,
  57                            unsigned size )
  58 {
  59    struct svga_screen *svgascreen = svga_screen(svga->pipe.screen);
  60    struct svga_winsys_screen *sws = svgascreen->sws;
  61    struct svga_winsys_buffer *buf;
  62
  63    /* XXX this shouldn't be a hard-coded number; it should be queried
  64     * somehow.
  65     */
  66    if (size > MAX_DMA_SIZE) {
  67       return NULL;
  68    }
  69
  70    /* Just try */
  71    buf = sws->buffer_create(sws, alignment, usage, size);
  72    if(!buf) {
  73
  74       SVGA_DBG(DEBUG_DMA|DEBUG_PERF, "flushing screen to find %d bytes GMR\n",
  75                size);
  76
  77       /* Try flushing all pending DMAs */
  78       svga_context_flush(svga, NULL);
  79       buf = sws->buffer_create(sws, alignment, usage, size);
  80    }
  81
  82    return buf;
  83 }
  84
  85
  86 void
  87 svga_buffer_destroy_hw_storage(struct svga_screen *ss, struct svga_buffer *sbuf)
  88 {
  89    struct svga_winsys_screen *sws = ss->sws;
  90
  91    assert(!sbuf->map.count);
  92    assert(sbuf->hwbuf);
  93    if(sbuf->hwbuf) {
  94       sws->buffer_destroy(sws, sbuf->hwbuf);
  95       sbuf->hwbuf = NULL;
  96    }
  97 }
  98
  99
 100
 101 /**
 102  * Allocate DMA'ble storage for the buffer.
 103  *
 104  * Called before mapping a buffer.
 105  */
 106 enum pipe_error
 107 svga_buffer_create_hw_storage(struct svga_screen *ss,
 108                               struct svga_buffer *sbuf)
 109 {
 110    assert(!sbuf->user);
 111
 112    if(!sbuf->hwbuf) {
 113       struct svga_winsys_screen *sws = ss->sws;
 114       unsigned alignment = 16;
 115       unsigned usage = 0;
 116       unsigned size = sbuf->b.b.width0;
 117
 118       sbuf->hwbuf = sws->buffer_create(sws, alignment, usage, size);
 119       if(!sbuf->hwbuf)
 120          return PIPE_ERROR_OUT_OF_MEMORY;
 121
 122       assert(!sbuf->dma.pending);
 123    }
 124
 125    return PIPE_OK;
 126 }
 127
 128
 129
 130 enum pipe_error
 131 svga_buffer_create_host_surface(struct svga_screen *ss,
 132                                 struct svga_buffer *sbuf)
 133 {
 134    if(!sbuf->handle) {
 135       sbuf->key.flags = 0;
 136
 137       sbuf->key.format = SVGA3D_BUFFER;
 138       if(sbuf->b.b.bind & PIPE_BIND_VERTEX_BUFFER)
 139          sbuf->key.flags |= SVGA3D_SURFACE_HINT_VERTEXBUFFER;
 140       if(sbuf->b.b.bind & PIPE_BIND_INDEX_BUFFER)
 141          sbuf->key.flags |= SVGA3D_SURFACE_HINT_INDEXBUFFER;
 142
 143       sbuf->key.size.width = sbuf->b.b.width0;
 144       sbuf->key.size.height = 1;
 145       sbuf->key.size.depth = 1;
 146
 147       sbuf->key.numFaces = 1;
 148       sbuf->key.numMipLevels = 1;
 149       sbuf->key.cachable = 1;
 150
 151       SVGA_DBG(DEBUG_DMA, "surface_create for buffer sz %d\n", sbuf->b.b.width0);
 152
 153       sbuf->handle = svga_screen_surface_create(ss, &sbuf->key);
 154       if(!sbuf->handle)
 155          return PIPE_ERROR_OUT_OF_MEMORY;
 156
 157       /* Always set the discard flag on the first time the buffer is written
 158        * as svga_screen_surface_create might have passed a recycled host
 159        * buffer.
 160        */
 161       sbuf->dma.flags.discard = TRUE;
 162
 163       SVGA_DBG(DEBUG_DMA, "   --> got sid %p sz %d (buffer)\n", sbuf->handle, sbuf->b.b.width0);
 164    }
 165
 166    return PIPE_OK;
 167 }
 168
 169
 170 void
 171 svga_buffer_destroy_host_surface(struct svga_screen *ss,
 172                                  struct svga_buffer *sbuf)
 173 {
 174    if(sbuf->handle) {
 175       SVGA_DBG(DEBUG_DMA, " ungrab sid %p sz %d\n", sbuf->handle, sbuf->b.b.width0);
 176       svga_screen_surface_destroy(ss, &sbuf->key, &sbuf->handle);
 177    }
 178 }
 179
 180
 181 /**
 182  * Variant of SVGA3D_BufferDMA which leaves the copy box temporarily in blank.
 183  */
 184 static enum pipe_error
 185 svga_buffer_upload_command(struct svga_context *svga,
 186                            struct svga_buffer *sbuf)
 187 {
 188    struct svga_winsys_context *swc = svga->swc;
 189    struct svga_winsys_buffer *guest = sbuf->hwbuf;
 190    struct svga_winsys_surface *host = sbuf->handle;
 191    SVGA3dTransferType transfer = SVGA3D_WRITE_HOST_VRAM;
 192    SVGA3dCmdSurfaceDMA *cmd;
 193    uint32 numBoxes = sbuf->map.num_ranges;
 194    SVGA3dCopyBox *boxes;
 195    SVGA3dCmdSurfaceDMASuffix *pSuffix;
 196    unsigned region_flags;
 197    unsigned surface_flags;
 198    struct pipe_resource *dummy;
 199
 200    if(transfer == SVGA3D_WRITE_HOST_VRAM) {
 201       region_flags = SVGA_RELOC_READ;
 202       surface_flags = SVGA_RELOC_WRITE;
 203    }
 204    else if(transfer == SVGA3D_READ_HOST_VRAM) {
 205       region_flags = SVGA_RELOC_WRITE;
 206       surface_flags = SVGA_RELOC_READ;
 207    }
 208    else {
 209       assert(0);
 210       return PIPE_ERROR_BAD_INPUT;
 211    }
 212
 213    assert(numBoxes);
 214
 215    cmd = SVGA3D_FIFOReserve(swc,
 216                             SVGA_3D_CMD_SURFACE_DMA,
 217                             sizeof *cmd + numBoxes * sizeof *boxes + sizeof *pSuffix,
 218                             2);
 219    if(!cmd)
 220       return PIPE_ERROR_OUT_OF_MEMORY;
 221
 222    swc->region_relocation(swc, &cmd->guest.ptr, guest, 0, region_flags);
 223    cmd->guest.pitch = 0;
 224
 225    swc->surface_relocation(swc, &cmd->host.sid, host, surface_flags);
 226    cmd->host.face = 0;
 227    cmd->host.mipmap = 0;
 228
 229    cmd->transfer = transfer;
 230
 231    sbuf->dma.boxes = (SVGA3dCopyBox *)&cmd[1];
 232    sbuf->dma.svga = svga;
 233
 234    /* Increment reference count */
 235    dummy = NULL;
 236    pipe_resource_reference(&dummy, &sbuf->b.b);
 237
 238    pSuffix = (SVGA3dCmdSurfaceDMASuffix *)((uint8_t*)cmd + sizeof *cmd + numBoxes * sizeof *boxes);
 239    pSuffix->suffixSize = sizeof *pSuffix;
 240    pSuffix->maximumOffset = sbuf->b.b.width0;
 241    pSuffix->flags = sbuf->dma.flags;
 242
 243    SVGA_FIFOCommitAll(swc);
 244
 245    sbuf->dma.flags.discard = FALSE;
 246
 247    return PIPE_OK;
 248 }
 249
 250
 251 /**
 252  * Patch up the upload DMA command reserved by svga_buffer_upload_command
 253  * with the final ranges.
 254  */
 255 static void
 256 svga_buffer_upload_flush(struct svga_context *svga,
 257                          struct svga_buffer *sbuf)
 258 {
 259    SVGA3dCopyBox *boxes;
 260    unsigned i;
 261    struct pipe_resource *dummy;
 262
 263    assert(sbuf->handle);
 264    assert(sbuf->hwbuf);
 265    assert(sbuf->map.num_ranges);
 266    assert(sbuf->dma.svga == svga);
 267    assert(sbuf->dma.boxes);
 268
 269    /*
 270     * Patch the DMA command with the final copy box.
 271     */
 272
 273    SVGA_DBG(DEBUG_DMA, "dma to sid %p\n", sbuf->handle);
 274
 275    boxes = sbuf->dma.boxes;
 276    for(i = 0; i < sbuf->map.num_ranges; ++i) {
 277       SVGA_DBG(DEBUG_DMA, "  bytes %u - %u\n",
 278                sbuf->map.ranges[i].start, sbuf->map.ranges[i].end);
 279
 280       boxes[i].x = sbuf->map.ranges[i].start;
 281       boxes[i].y = 0;
 282       boxes[i].z = 0;
 283       boxes[i].w = sbuf->map.ranges[i].end - sbuf->map.ranges[i].start;
 284       boxes[i].h = 1;
 285       boxes[i].d = 1;
 286       boxes[i].srcx = sbuf->map.ranges[i].start;
 287       boxes[i].srcy = 0;
 288       boxes[i].srcz = 0;
 289    }
 290
 291    sbuf->map.num_ranges = 0;
 292
 293    assert(sbuf->head.prev && sbuf->head.next);
 294    LIST_DEL(&sbuf->head);
 295 #ifdef DEBUG
 296    sbuf->head.next = sbuf->head.prev = NULL;
 297 #endif
 298    sbuf->dma.pending = FALSE;
 299
 300    sbuf->dma.svga = NULL;
 301    sbuf->dma.boxes = NULL;
 302
 303    /* Decrement reference count (and potentially destroy) */
 304    dummy = &sbuf->b.b;
 305    pipe_resource_reference(&dummy, NULL);
 306 }
 307
 308
 309
 310 /**
 311  * Note a dirty range.
 312  *
 313  * This function only notes the range down. It doesn't actually emit a DMA
 314  * upload command. That only happens when a context tries to refer to this
 315  * buffer, and the DMA upload command is added to that context's command buffer.
 316  *
 317  * We try to lump as many contiguous DMA transfers together as possible.
 318  */
 319 void
 320 svga_buffer_add_range(struct svga_buffer *sbuf,
 321                       unsigned start,
 322                       unsigned end)
 323 {
 324    unsigned i;
 325    unsigned nearest_range;
 326    unsigned nearest_dist;
 327
 328    assert(end > start);
 329
 330    if (sbuf->map.num_ranges < SVGA_BUFFER_MAX_RANGES) {
 331       nearest_range = sbuf->map.num_ranges;
 332       nearest_dist = ~0;
 333    } else {
 334       nearest_range = SVGA_BUFFER_MAX_RANGES - 1;
 335       nearest_dist = 0;
 336    }
 337
 338    /*
 339     * Try to grow one of the ranges.
 340     *
 341     * Note that it is not this function task to care about overlapping ranges,
 342     * as the GMR was already given so it is too late to do anything. Situations
 343     * where overlapping ranges may pose a problem should be detected via
 344     * pipe_context::is_resource_referenced and the context that refers to the
 345     * buffer should be flushed.
 346     */
 347
 348    for(i = 0; i < sbuf->map.num_ranges; ++i) {
 349       int left_dist;
 350       int right_dist;
 351       int dist;
 352
 353       left_dist = start - sbuf->map.ranges[i].end;
 354       right_dist = sbuf->map.ranges[i].start - end;
 355       dist = MAX2(left_dist, right_dist);
 356
 357       if (dist <= 0) {
 358          /*
 359           * Ranges are contiguous or overlapping -- extend this one and return.
 360           */
 361
 362          sbuf->map.ranges[i].start = MIN2(sbuf->map.ranges[i].start, start);
 363          sbuf->map.ranges[i].end   = MAX2(sbuf->map.ranges[i].end,   end);
 364          return;
 365       }
 366       else {
 367          /*
 368           * Discontiguous ranges -- keep track of the nearest range.
 369           */
 370
 371          if (dist < nearest_dist) {
 372             nearest_range = i;
 373             nearest_dist = dist;
 374          }
 375       }
 376    }
 377
 378    /*
 379     * We cannot add a new range to an existing DMA command, so patch-up the
 380     * pending DMA upload and start clean.
 381     */
 382
 383    if(sbuf->dma.pending)
 384       svga_buffer_upload_flush(sbuf->dma.svga, sbuf);
 385
 386    assert(!sbuf->dma.pending);
 387    assert(!sbuf->dma.svga);
 388    assert(!sbuf->dma.boxes);
 389
 390    if (sbuf->map.num_ranges < SVGA_BUFFER_MAX_RANGES) {
 391       /*
 392        * Add a new range.
 393        */
 394
 395       sbuf->map.ranges[sbuf->map.num_ranges].start = start;
 396       sbuf->map.ranges[sbuf->map.num_ranges].end = end;
 397       ++sbuf->map.num_ranges;
 398    } else {
 399       /*
 400        * Everything else failed, so just extend the nearest range.
 401        *
 402        * It is OK to do this because we always keep a local copy of the
 403        * host buffer data, for SW TNL, and the host never modifies the buffer.
 404        */
 405
 406       assert(nearest_range < SVGA_BUFFER_MAX_RANGES);
 407       assert(nearest_range < sbuf->map.num_ranges);
 408       sbuf->map.ranges[nearest_range].start = MIN2(sbuf->map.ranges[nearest_range].start, start);
 409       sbuf->map.ranges[nearest_range].end   = MAX2(sbuf->map.ranges[nearest_range].end,   end);
 410    }
 411 }
 412
 413
 414
 415 /**
 416  * Copy the contents of the malloc buffer to a hardware buffer.
 417  */
 418 static INLINE enum pipe_error
 419 svga_buffer_update_hw(struct svga_screen *ss, struct svga_buffer *sbuf)
 420 {
 421    assert(!sbuf->user);
 422    if(!sbuf->hwbuf) {
 423       enum pipe_error ret;
 424       void *map;
 425
 426       assert(sbuf->swbuf);
 427       if(!sbuf->swbuf)
 428          return PIPE_ERROR;
 429
 430       ret = svga_buffer_create_hw_storage(ss, sbuf);
 431       if(ret != PIPE_OK)
 432          return ret;
 433
 434       pipe_mutex_lock(ss->swc_mutex);
 435       map = ss->sws->buffer_map(ss->sws, sbuf->hwbuf, PIPE_TRANSFER_WRITE);
 436       assert(map);
 437       if(!map) {
 438          pipe_mutex_unlock(ss->swc_mutex);
 439          svga_buffer_destroy_hw_storage(ss, sbuf);
 440          return PIPE_ERROR;
 441       }
 442
 443       memcpy(map, sbuf->swbuf, sbuf->b.b.width0);
 444       ss->sws->buffer_unmap(ss->sws, sbuf->hwbuf);
 445
 446       /* This user/malloc buffer is now indistinguishable from a gpu buffer */
 447       assert(!sbuf->map.count);
 448       if(!sbuf->map.count) {
 449          if(sbuf->user)
 450             sbuf->user = FALSE;
 451          else
 452             align_free(sbuf->swbuf);
 453          sbuf->swbuf = NULL;
 454       }
 455
 456       pipe_mutex_unlock(ss->swc_mutex);
 457    }
 458
 459    return PIPE_OK;
 460 }
 461
 462
 463 /**
 464  * Upload the buffer to the host in a piecewise fashion.
 465  *
 466  * Used when the buffer is too big to fit in the GMR aperture.
 467  */
 468 static INLINE enum pipe_error
 469 svga_buffer_upload_piecewise(struct svga_screen *ss,
 470                              struct svga_context *svga,
 471                              struct svga_buffer *sbuf)
 472 {
 473    struct svga_winsys_screen *sws = ss->sws;
 474    const unsigned alignment = sizeof(void *);
 475    const unsigned usage = 0;
 476    unsigned i;
 477
 478    assert(sbuf->map.num_ranges);
 479    assert(!sbuf->dma.pending);
 480
 481    SVGA_DBG(DEBUG_DMA, "dma to sid %p\n", sbuf->handle);
 482
 483    for (i = 0; i < sbuf->map.num_ranges; ++i) {
 484       struct svga_buffer_range *range = &sbuf->map.ranges[i];
 485       unsigned offset = range->start;
 486       unsigned size = range->end - range->start;
 487
 488       while (offset < range->end) {
 489          struct svga_winsys_buffer *hwbuf;
 490          uint8_t *map;
 491          enum pipe_error ret;
 492
 493          if (offset + size > range->end)
 494             size = range->end - offset;
 495
 496          hwbuf = sws->buffer_create(sws, alignment, usage, size);
 497          while (!hwbuf) {
 498             size /= 2;
 499             if (!size)
 500                return PIPE_ERROR_OUT_OF_MEMORY;
 501             hwbuf = sws->buffer_create(sws, alignment, usage, size);
 502          }
 503
 504          SVGA_DBG(DEBUG_DMA, "  bytes %u - %u\n",
 505                   offset, offset + size);
 506
 507          map = sws->buffer_map(sws, hwbuf,
 508                                PIPE_TRANSFER_WRITE |
 509                                PIPE_TRANSFER_DISCARD);
 510          assert(map);
 511          if (map) {
 512             memcpy(map, sbuf->swbuf, size);
 513             sws->buffer_unmap(sws, hwbuf);
 514          }
 515
 516          ret = SVGA3D_BufferDMA(svga->swc,
 517                                 hwbuf, sbuf->handle,
 518                                 SVGA3D_WRITE_HOST_VRAM,
 519                                 size, 0, offset, sbuf->dma.flags);
 520          if(ret != PIPE_OK) {
 521             svga_context_flush(svga, NULL);
 522             ret =  SVGA3D_BufferDMA(svga->swc,
 523                                     hwbuf, sbuf->handle,
 524                                     SVGA3D_WRITE_HOST_VRAM,
 525                                     size, 0, offset, sbuf->dma.flags);
 526             assert(ret == PIPE_OK);
 527          }
 528
 529          sbuf->dma.flags.discard = FALSE;
 530
 531          sws->buffer_destroy(sws, hwbuf);
 532
 533          offset += size;
 534       }
 535    }
 536
 537    sbuf->map.num_ranges = 0;
 538
 539    return PIPE_OK;
 540 }
 541
 542
 543
 544
 545 /* Get (or create/upload) the winsys surface handle so that we can
 546  * refer to this buffer in fifo commands.
 547  */
 548 struct svga_winsys_surface *
 549 svga_buffer_handle(struct svga_context *svga,
 550                    struct pipe_resource *buf)
 551 {
 552    struct pipe_screen *screen = svga->pipe.screen;
 553    struct svga_screen *ss = svga_screen(screen);
 554    struct svga_buffer *sbuf;
 555    enum pipe_error ret;
 556
 557    if(!buf)
 558       return NULL;
 559
 560    sbuf = svga_buffer(buf);
 561
 562    assert(!sbuf->map.count);
 563    assert(!sbuf->user);
 564
 565    if(!sbuf->handle) {
 566       ret = svga_buffer_create_host_surface(ss, sbuf);
 567       if(ret != PIPE_OK)
 568          return NULL;
 569    }
 570
 571    assert(sbuf->handle);
 572
 573    if (sbuf->map.num_ranges) {
 574       if (!sbuf->dma.pending) {
 575          /*
 576           * No pending DMA upload yet, so insert a DMA upload command now.
 577           */
 578
 579          /*
 580           * Migrate the data from swbuf -> hwbuf if necessary.
 581           */
 582          ret = svga_buffer_update_hw(ss, sbuf);
 583          if (ret == PIPE_OK) {
 584             /*
 585              * Queue a dma command.
 586              */
 587
 588             ret = svga_buffer_upload_command(svga, sbuf);
 589             if (ret == PIPE_ERROR_OUT_OF_MEMORY) {
 590                svga_context_flush(svga, NULL);
 591                ret = svga_buffer_upload_command(svga, sbuf);
 592                assert(ret == PIPE_OK);
 593             }
 594             if (ret == PIPE_OK) {
 595                sbuf->dma.pending = TRUE;
 596                assert(!sbuf->head.prev && !sbuf->head.next);
 597                LIST_ADDTAIL(&sbuf->head, &svga->dirty_buffers);
 598             }
 599          }
 600          else if (ret == PIPE_ERROR_OUT_OF_MEMORY) {
 601             /*
 602              * The buffer is too big to fit in the GMR aperture, so break it in
 603              * smaller pieces.
 604              */
 605             ret = svga_buffer_upload_piecewise(ss, svga, sbuf);
 606          }
 607
 608          if (ret != PIPE_OK) {
 609             /*
 610              * Something unexpected happened above. There is very little that
 611              * we can do other than proceeding while ignoring the dirty ranges.
 612              */
 613             assert(0);
 614             sbuf->map.num_ranges = 0;
 615          }
 616       }
 617       else {
 618          /*
 619           * There a pending dma already. Make sure it is from this context.
 620           */
 621          assert(sbuf->dma.svga == svga);
 622       }
 623    }
 624
 625    assert(!sbuf->map.num_ranges || sbuf->dma.pending);
 626
 627    return sbuf->handle;
 628 }
 629
 630
 631
 632 void
 633 svga_context_flush_buffers(struct svga_context *svga)
 634 {
 635    struct list_head *curr, *next;
 636    struct svga_buffer *sbuf;
 637
 638    curr = svga->dirty_buffers.next;
 639    next = curr->next;
 640    while(curr != &svga->dirty_buffers) {
 641       sbuf = LIST_ENTRY(struct svga_buffer, curr, head);
 642
 643       assert(p_atomic_read(&sbuf->b.b.reference.count) != 0);
 644       assert(sbuf->dma.pending);
 645
 646       svga_buffer_upload_flush(svga, sbuf);
 647
 648       curr = next;
 649       next = curr->next;
 650    }
 651 }
 652
 653
 654 void
 655 svga_redefine_user_buffer(struct pipe_context *pipe,
 656                           struct pipe_resource *resource,
 657                           unsigned offset,
 658                           unsigned size)
 659 {
 660    struct svga_screen *ss = svga_screen(pipe->screen);
 661    struct svga_context *svga = svga_context(pipe);
 662    struct svga_buffer *sbuf = svga_buffer(resource);
 663
 664    assert(sbuf->user);
 665
 666    /*
 667     * Release any uploaded user buffer.
 668     *
 669     * TODO: As an optimization, we could try to update the uploaded buffer
 670     * instead.
 671     */
 672
 673    pipe_resource_reference(&sbuf->uploaded.buffer, NULL);
 674
 675    pipe_mutex_lock(ss->swc_mutex);
 676
 677    if (offset + size > resource->width0) {
 678       /*
 679        * User buffers shouldn't have DMA directly, unless
 680        * SVGA_COMBINE_USERBUFFERS is not set.
 681        */
 682
 683       if (sbuf->dma.pending) {
 684          svga_buffer_upload_flush(svga, sbuf);
 685       }
 686
 687       if (sbuf->handle) {
 688          svga_buffer_destroy_host_surface(ss, sbuf);
 689       }
 690
 691       if (sbuf->hwbuf) {
 692          svga_buffer_destroy_hw_storage(ss, sbuf);
 693       }
 694
 695       sbuf->key.size.width = sbuf->b.b.width0 = offset + size;
 696    }
 697
 698    sbuf->source_offset = offset;
 699    pipe_mutex_unlock(ss->swc_mutex);
 700
 701    svga->curr.any_user_vertex_buffers = TRUE;
 702    svga->dirty |= SVGA_NEW_VBUFFER | SVGA_NEW_VELEMENT;
 703 }