src/gallium/drivers/nouveau/nouveau_buffer.c

   1
   2 #include "util/u_inlines.h"
   3 #include "util/u_memory.h"
   4 #include "util/u_math.h"
   5
   6 #include "nouveau_screen.h"
   7 #include "nouveau_context.h"
   8 #include "nouveau_winsys.h"
   9 #include "nouveau_fence.h"
  10 #include "nouveau_buffer.h"
  11 #include "nouveau_mm.h"
  12
  13 #define NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD 192
  14
  15 struct nouveau_transfer {
  16    struct pipe_transfer base;
  17
  18    uint8_t *map;
  19    struct nouveau_bo *bo;
  20    struct nouveau_mm_allocation *mm;
  21    uint32_t offset;
  22 };
  23
  24 static INLINE struct nouveau_transfer *
  25 nouveau_transfer(struct pipe_transfer *transfer)
  26 {
  27    return (struct nouveau_transfer *)transfer;
  28 }
  29
  30 static INLINE boolean
  31 nouveau_buffer_malloc(struct nv04_resource *buf)
  32 {
  33    if (!buf->data)
  34       buf->data = align_malloc(buf->base.width0, NOUVEAU_MIN_BUFFER_MAP_ALIGN);
  35    return !!buf->data;
  36 }
  37
  38 static INLINE boolean
  39 nouveau_buffer_allocate(struct nouveau_screen *screen,
  40                         struct nv04_resource *buf, unsigned domain)
  41 {
  42    uint32_t size = buf->base.width0;
  43
  44    if (buf->base.bind & PIPE_BIND_CONSTANT_BUFFER)
  45       size = align(size, 0x100);
  46
  47    if (domain == NOUVEAU_BO_VRAM) {
  48       buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size,
  49                                     &buf->bo, &buf->offset);
  50       if (!buf->bo)
  51          return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
  52    } else
  53    if (domain == NOUVEAU_BO_GART) {
  54       buf->mm = nouveau_mm_allocate(screen->mm_GART, size,
  55                                     &buf->bo, &buf->offset);
  56       if (!buf->bo)
  57          return FALSE;
  58    } else {
  59       assert(domain == 0);
  60       if (!nouveau_buffer_malloc(buf))
  61          return FALSE;
  62    }
  63    buf->domain = domain;
  64    if (buf->bo)
  65       buf->address = buf->bo->offset + buf->offset;
  66
  67    return TRUE;
  68 }
  69
  70 static INLINE void
  71 release_allocation(struct nouveau_mm_allocation **mm,
  72                    struct nouveau_fence *fence)
  73 {
  74    nouveau_fence_work(fence, nouveau_mm_free_work, *mm);
  75    (*mm) = NULL;
  76 }
  77
  78 INLINE void
  79 nouveau_buffer_release_gpu_storage(struct nv04_resource *buf)
  80 {
  81    nouveau_bo_ref(NULL, &buf->bo);
  82
  83    if (buf->mm)
  84       release_allocation(&buf->mm, buf->fence);
  85
  86    buf->domain = 0;
  87 }
  88
  89 static INLINE boolean
  90 nouveau_buffer_reallocate(struct nouveau_screen *screen,
  91                           struct nv04_resource *buf, unsigned domain)
  92 {
  93    nouveau_buffer_release_gpu_storage(buf);
  94
  95    nouveau_fence_ref(NULL, &buf->fence);
  96    nouveau_fence_ref(NULL, &buf->fence_wr);
  97
  98    buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK;
  99
 100    return nouveau_buffer_allocate(screen, buf, domain);
 101 }
 102
 103 static void
 104 nouveau_buffer_destroy(struct pipe_screen *pscreen,
 105                        struct pipe_resource *presource)
 106 {
 107    struct nv04_resource *res = nv04_resource(presource);
 108
 109    nouveau_buffer_release_gpu_storage(res);
 110
 111    if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
 112       align_free(res->data);
 113
 114    nouveau_fence_ref(NULL, &res->fence);
 115    nouveau_fence_ref(NULL, &res->fence_wr);
 116
 117    FREE(res);
 118 }
 119
 120 static uint8_t *
 121 nouveau_transfer_staging(struct nouveau_context *nv,
 122                          struct nouveau_transfer *tx, boolean permit_pb)
 123 {
 124    const unsigned adj = tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK;
 125    const unsigned size = align(tx->base.box.width, 4) + adj;
 126
 127    if ((size <= NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD) && permit_pb) {
 128       tx->map = align_malloc(size, NOUVEAU_MIN_BUFFER_MAP_ALIGN);
 129       if (tx->map)
 130          tx->map += adj;
 131    } else {
 132       tx->mm =
 133          nouveau_mm_allocate(nv->screen->mm_GART, size, &tx->bo, &tx->offset);
 134       if (tx->bo) {
 135          tx->offset += adj;
 136          if (!nouveau_bo_map(tx->bo, 0, NULL))
 137             tx->map = (uint8_t *)tx->bo->map + tx->offset;
 138       }
 139    }
 140    return tx->map;
 141 }
 142
 143 /* Maybe just migrate to GART right away if we actually need to do this. */
 144 static boolean
 145 nouveau_transfer_read(struct nouveau_context *nv, struct nouveau_transfer *tx)
 146 {
 147    struct nv04_resource *buf = nv04_resource(tx->base.resource);
 148    const unsigned base = tx->base.box.x;
 149    const unsigned size = tx->base.box.width;
 150
 151    nv->copy_data(nv, tx->bo, tx->offset, NOUVEAU_BO_GART,
 152                  buf->bo, buf->offset + base, buf->domain, size);
 153
 154    if (nouveau_bo_wait(tx->bo, NOUVEAU_BO_RD, nv->client))
 155       return FALSE;
 156
 157    if (buf->data)
 158       memcpy(buf->data + base, tx->map, size);
 159
 160    return TRUE;
 161 }
 162
 163 static void
 164 nouveau_transfer_write(struct nouveau_context *nv, struct nouveau_transfer *tx,
 165                        unsigned offset, unsigned size)
 166 {
 167    struct nv04_resource *buf = nv04_resource(tx->base.resource);
 168    uint8_t *data = tx->map + offset;
 169    const unsigned base = tx->base.box.x + offset;
 170    const boolean can_cb = !((base | size) & 3);
 171
 172    if (buf->data)
 173       memcpy(data, buf->data + base, size);
 174    else
 175       buf->status |= NOUVEAU_BUFFER_STATUS_DIRTY;
 176
 177    if (tx->bo)
 178       nv->copy_data(nv, buf->bo, buf->offset + base, buf->domain,
 179                     tx->bo, tx->offset + offset, NOUVEAU_BO_GART, size);
 180    else
 181    if ((buf->base.bind & PIPE_BIND_CONSTANT_BUFFER) && nv->push_cb && can_cb)
 182       nv->push_cb(nv, buf->bo, buf->domain, buf->offset, buf->base.width0,
 183                   base, size / 4, (const uint32_t *)data);
 184    else
 185       nv->push_data(nv, buf->bo, buf->offset + base, buf->domain, size, data);
 186 }
 187
 188
 189 static INLINE boolean
 190 nouveau_buffer_sync(struct nv04_resource *buf, unsigned rw)
 191 {
 192    if (rw == PIPE_TRANSFER_READ) {
 193       if (!buf->fence_wr)
 194          return TRUE;
 195       if (!nouveau_fence_wait(buf->fence_wr))
 196          return FALSE;
 197    } else {
 198       if (!buf->fence)
 199          return TRUE;
 200       if (!nouveau_fence_wait(buf->fence))
 201          return FALSE;
 202
 203       nouveau_fence_ref(NULL, &buf->fence);
 204    }
 205    nouveau_fence_ref(NULL, &buf->fence_wr);
 206
 207    return TRUE;
 208 }
 209
 210 static INLINE boolean
 211 nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw)
 212 {
 213    if (rw == PIPE_TRANSFER_READ)
 214       return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr));
 215    else
 216       return (buf->fence && !nouveau_fence_signalled(buf->fence));
 217 }
 218
 219 static INLINE void
 220 nouveau_buffer_transfer_init(struct nouveau_transfer *tx,
 221                              struct pipe_resource *resource,
 222                              const struct pipe_box *box,
 223                              unsigned usage)
 224 {
 225    tx->base.resource = resource;
 226    tx->base.level = 0;
 227    tx->base.usage = usage;
 228    tx->base.box.x = box->x;
 229    tx->base.box.y = 0;
 230    tx->base.box.z = 0;
 231    tx->base.box.width = box->width;
 232    tx->base.box.height = 1;
 233    tx->base.box.depth = 1;
 234    tx->base.stride = 0;
 235    tx->base.layer_stride = 0;
 236
 237    tx->bo = NULL;
 238    tx->map = NULL;
 239 }
 240
 241 static INLINE void
 242 nouveau_buffer_transfer_del(struct nouveau_context *nv,
 243                             struct nouveau_transfer *tx)
 244 {
 245    if (tx->map) {
 246       if (likely(tx->bo)) {
 247          nouveau_bo_ref(NULL, &tx->bo);
 248          if (tx->mm)
 249             release_allocation(&tx->mm, nv->screen->fence.current);
 250       } else {
 251          align_free(tx->map -
 252                     (tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK));
 253       }
 254    }
 255 }
 256
 257 static boolean
 258 nouveau_buffer_cache(struct nouveau_context *nv, struct nv04_resource *buf)
 259 {
 260    struct nouveau_transfer tx;
 261    boolean ret;
 262    tx.base.resource = &buf->base;
 263    tx.base.box.x = 0;
 264    tx.base.box.width = buf->base.width0;
 265    tx.bo = NULL;
 266
 267    if (!buf->data)
 268       if (!nouveau_buffer_malloc(buf))
 269          return FALSE;
 270    if (!(buf->status & NOUVEAU_BUFFER_STATUS_DIRTY))
 271       return TRUE;
 272    nv->stats.buf_cache_count++;
 273
 274    if (!nouveau_transfer_staging(nv, &tx, FALSE))
 275       return FALSE;
 276
 277    ret = nouveau_transfer_read(nv, &tx);
 278    if (ret) {
 279       buf->status &= ~NOUVEAU_BUFFER_STATUS_DIRTY;
 280       memcpy(buf->data, tx.map, buf->base.width0);
 281    }
 282    nouveau_buffer_transfer_del(nv, &tx);
 283    return ret;
 284 }
 285
 286
 287 #define NOUVEAU_TRANSFER_DISCARD \
 288    (PIPE_TRANSFER_DISCARD_RANGE | PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)
 289
 290 static INLINE boolean
 291 nouveau_buffer_should_discard(struct nv04_resource *buf, unsigned usage)
 292 {
 293    if (!(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE))
 294       return FALSE;
 295    if (unlikely(buf->base.bind & PIPE_BIND_SHARED))
 296       return FALSE;
 297    return buf->mm && nouveau_buffer_busy(buf, PIPE_TRANSFER_WRITE);
 298 }
 299
 300 static void *
 301 nouveau_buffer_transfer_map(struct pipe_context *pipe,
 302                             struct pipe_resource *resource,
 303                             unsigned level, unsigned usage,
 304                             const struct pipe_box *box,
 305                             struct pipe_transfer **ptransfer)
 306 {
 307    struct nouveau_context *nv = nouveau_context(pipe);
 308    struct nv04_resource *buf = nv04_resource(resource);
 309    struct nouveau_transfer *tx = MALLOC_STRUCT(nouveau_transfer);
 310    uint8_t *map;
 311    int ret;
 312
 313    if (!tx)
 314       return NULL;
 315    nouveau_buffer_transfer_init(tx, resource, box, usage);
 316    *ptransfer = &tx->base;
 317
 318    if (buf->domain == NOUVEAU_BO_VRAM) {
 319       if (usage & NOUVEAU_TRANSFER_DISCARD) {
 320          if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)
 321             buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK;
 322          nouveau_transfer_staging(nv, tx, TRUE);
 323       } else {
 324          if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING) {
 325             if (buf->data) {
 326                align_free(buf->data);
 327                buf->data = NULL;
 328             }
 329             nouveau_transfer_staging(nv, tx, FALSE);
 330             nouveau_transfer_read(nv, tx);
 331          } else {
 332             if (usage & PIPE_TRANSFER_WRITE)
 333                nouveau_transfer_staging(nv, tx, TRUE);
 334             if (!buf->data)
 335                nouveau_buffer_cache(nv, buf);
 336          }
 337       }
 338       return buf->data ? (buf->data + box->x) : tx->map;
 339    } else
 340    if (unlikely(buf->domain == 0)) {
 341       return buf->data + box->x;
 342    }
 343
 344    if (nouveau_buffer_should_discard(buf, usage)) {
 345       int ref = buf->base.reference.count - 1;
 346       nouveau_buffer_reallocate(nv->screen, buf, buf->domain);
 347       if (ref > 0) /* any references inside context possible ? */
 348          nv->invalidate_resource_storage(nv, &buf->base, ref);
 349    }
 350
 351    ret = nouveau_bo_map(buf->bo,
 352                         buf->mm ? 0 : nouveau_screen_transfer_flags(usage),
 353                         nv->client);
 354    if (ret) {
 355       FREE(tx);
 356       return NULL;
 357    }
 358    map = (uint8_t *)buf->bo->map + buf->offset + box->x;
 359
 360    /* using kernel fences only if !buf->mm */
 361    if ((usage & PIPE_TRANSFER_UNSYNCHRONIZED) || !buf->mm)
 362       return map;
 363
 364    if (nouveau_buffer_busy(buf, usage & PIPE_TRANSFER_READ_WRITE)) {
 365       if (unlikely(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)) {
 366          /* Discarding was not possible, must sync because
 367           * subsequent transfers might use UNSYNCHRONIZED. */
 368          nouveau_buffer_sync(buf, usage & PIPE_TRANSFER_READ_WRITE);
 369       } else
 370       if (usage & PIPE_TRANSFER_DISCARD_RANGE) {
 371          nouveau_transfer_staging(nv, tx, TRUE);
 372          map = tx->map;
 373       } else
 374       if (nouveau_buffer_busy(buf, PIPE_TRANSFER_READ)) {
 375          if (usage & PIPE_TRANSFER_DONTBLOCK)
 376             map = NULL;
 377          else
 378             nouveau_buffer_sync(buf, usage & PIPE_TRANSFER_READ_WRITE);
 379       } else {
 380          nouveau_transfer_staging(nv, tx, TRUE);
 381          if (tx->map)
 382             memcpy(tx->map, map, box->width);
 383          map = tx->map;
 384       }
 385    }
 386    if (!map)
 387       FREE(tx);
 388    return map;
 389 }
 390
 391
 392
 393 static void
 394 nouveau_buffer_transfer_flush_region(struct pipe_context *pipe,
 395                                      struct pipe_transfer *transfer,
 396                                      const struct pipe_box *box)
 397 {
 398    struct nouveau_transfer *tx = nouveau_transfer(transfer);
 399    if (tx->map)
 400       nouveau_transfer_write(nouveau_context(pipe), tx, box->x, box->width);
 401 }
 402
 403 static void
 404 nouveau_buffer_transfer_unmap(struct pipe_context *pipe,
 405                               struct pipe_transfer *transfer)
 406 {
 407    struct nouveau_context *nv = nouveau_context(pipe);
 408    struct nouveau_transfer *tx = nouveau_transfer(transfer);
 409    struct nv04_resource *buf = nv04_resource(transfer->resource);
 410
 411    if (tx->base.usage & PIPE_TRANSFER_WRITE) {
 412       if (!(tx->base.usage & PIPE_TRANSFER_FLUSH_EXPLICIT) && tx->map)
 413          nouveau_transfer_write(nv, tx, 0, tx->base.box.width);
 414
 415       if (likely(buf->domain)) {
 416          const uint8_t bind = buf->base.bind;
 417          /* make sure we invalidate dedicated caches */
 418          if (bind & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER))
 419             nv->vbo_dirty = TRUE;
 420          if (bind & (PIPE_BIND_CONSTANT_BUFFER))
 421             nv->cb_dirty = TRUE;
 422       }
 423    }
 424
 425    nouveau_buffer_transfer_del(nv, tx);
 426    FREE(tx);
 427 }
 428
 429
 430 void *
 431 nouveau_resource_map_offset(struct nouveau_context *nv,
 432                             struct nv04_resource *res, uint32_t offset,
 433                             uint32_t flags)
 434 {
 435    if (unlikely(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
 436       return res->data + offset;
 437
 438    if (res->domain == NOUVEAU_BO_VRAM) {
 439       if (!res->data || (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING))
 440          nouveau_buffer_cache(nv, res);
 441    }
 442    if (res->domain != NOUVEAU_BO_GART)
 443       return res->data + offset;
 444
 445    if (res->mm) {
 446       unsigned rw;
 447       rw = (flags & NOUVEAU_BO_WR) ? PIPE_TRANSFER_WRITE : PIPE_TRANSFER_READ;
 448       nouveau_buffer_sync(res, rw);
 449       if (nouveau_bo_map(res->bo, 0, NULL))
 450          return NULL;
 451    } else {
 452       if (nouveau_bo_map(res->bo, flags, nv->client))
 453          return NULL;
 454    }
 455    return (uint8_t *)res->bo->map + res->offset + offset;
 456 }
 457
 458
 459 const struct u_resource_vtbl nouveau_buffer_vtbl =
 460 {
 461    u_default_resource_get_handle,     /* get_handle */
 462    nouveau_buffer_destroy,               /* resource_destroy */
 463    nouveau_buffer_transfer_map,          /* transfer_map */
 464    nouveau_buffer_transfer_flush_region, /* transfer_flush_region */
 465    nouveau_buffer_transfer_unmap,        /* transfer_unmap */
 466    u_default_transfer_inline_write    /* transfer_inline_write */
 467 };
 468
 469 struct pipe_resource *
 470 nouveau_buffer_create(struct pipe_screen *pscreen,
 471                       const struct pipe_resource *templ)
 472 {
 473    struct nouveau_screen *screen = nouveau_screen(pscreen);
 474    struct nv04_resource *buffer;
 475    boolean ret;
 476
 477    buffer = CALLOC_STRUCT(nv04_resource);
 478    if (!buffer)
 479       return NULL;
 480
 481    buffer->base = *templ;
 482    buffer->vtbl = &nouveau_buffer_vtbl;
 483    pipe_reference_init(&buffer->base.reference, 1);
 484    buffer->base.screen = pscreen;
 485
 486    if (buffer->base.bind &
 487        (screen->vidmem_bindings & screen->sysmem_bindings)) {
 488       switch (buffer->base.usage) {
 489       case PIPE_USAGE_DEFAULT:
 490       case PIPE_USAGE_IMMUTABLE:
 491       case PIPE_USAGE_STATIC:
 492          buffer->domain = NOUVEAU_BO_VRAM;
 493          break;
 494       case PIPE_USAGE_DYNAMIC:
 495          /* For most apps, we'd have to do staging transfers to avoid sync
 496           * with this usage, and GART -> GART copies would be suboptimal.
 497           */
 498          buffer->domain = NOUVEAU_BO_VRAM;
 499          break;
 500       case PIPE_USAGE_STAGING:
 501       case PIPE_USAGE_STREAM:
 502          buffer->domain = NOUVEAU_BO_GART;
 503          break;
 504       default:
 505          assert(0);
 506          break;
 507       }
 508    } else {
 509       if (buffer->base.bind & screen->vidmem_bindings)
 510          buffer->domain = NOUVEAU_BO_VRAM;
 511       else
 512       if (buffer->base.bind & screen->sysmem_bindings)
 513          buffer->domain = NOUVEAU_BO_GART;
 514    }
 515    ret = nouveau_buffer_allocate(screen, buffer, buffer->domain);
 516
 517    if (ret == FALSE)
 518       goto fail;
 519
 520    if (buffer->domain == NOUVEAU_BO_VRAM && screen->hint_buf_keep_sysmem_copy)
 521       nouveau_buffer_cache(NULL, buffer);
 522
 523    return &buffer->base;
 524
 525 fail:
 526    FREE(buffer);
 527    return NULL;
 528 }
 529
 530
 531 struct pipe_resource *
 532 nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr,
 533                            unsigned bytes, unsigned bind)
 534 {
 535    struct nv04_resource *buffer;
 536
 537    buffer = CALLOC_STRUCT(nv04_resource);
 538    if (!buffer)
 539       return NULL;
 540
 541    pipe_reference_init(&buffer->base.reference, 1);
 542    buffer->vtbl = &nouveau_buffer_vtbl;
 543    buffer->base.screen = pscreen;
 544    buffer->base.format = PIPE_FORMAT_R8_UNORM;
 545    buffer->base.usage = PIPE_USAGE_IMMUTABLE;
 546    buffer->base.bind = bind;
 547    buffer->base.width0 = bytes;
 548    buffer->base.height0 = 1;
 549    buffer->base.depth0 = 1;
 550
 551    buffer->data = ptr;
 552    buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY;
 553
 554    return &buffer->base;
 555 }
 556
 557 static INLINE boolean
 558 nouveau_buffer_data_fetch(struct nouveau_context *nv, struct nv04_resource *buf,
 559                           struct nouveau_bo *bo, unsigned offset, unsigned size)
 560 {
 561    if (!nouveau_buffer_malloc(buf))
 562       return FALSE;
 563    if (nouveau_bo_map(bo, NOUVEAU_BO_RD, nv->client))
 564       return FALSE;
 565    memcpy(buf->data, (uint8_t *)bo->map + offset, size);
 566    return TRUE;
 567 }
 568
 569 /* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
 570 boolean
 571 nouveau_buffer_migrate(struct nouveau_context *nv,
 572                        struct nv04_resource *buf, const unsigned new_domain)
 573 {
 574    struct nouveau_screen *screen = nv->screen;
 575    struct nouveau_bo *bo;
 576    const unsigned old_domain = buf->domain;
 577    unsigned size = buf->base.width0;
 578    unsigned offset;
 579    int ret;
 580
 581    assert(new_domain != old_domain);
 582
 583    if (new_domain == NOUVEAU_BO_GART && old_domain == 0) {
 584       if (!nouveau_buffer_allocate(screen, buf, new_domain))
 585          return FALSE;
 586       ret = nouveau_bo_map(buf->bo, 0, nv->client);
 587       if (ret)
 588          return ret;
 589       memcpy((uint8_t *)buf->bo->map + buf->offset, buf->data, size);
 590       align_free(buf->data);
 591    } else
 592    if (old_domain != 0 && new_domain != 0) {
 593       struct nouveau_mm_allocation *mm = buf->mm;
 594
 595       if (new_domain == NOUVEAU_BO_VRAM) {
 596          /* keep a system memory copy of our data in case we hit a fallback */
 597          if (!nouveau_buffer_data_fetch(nv, buf, buf->bo, buf->offset, size))
 598             return FALSE;
 599          if (nouveau_mesa_debug)
 600             debug_printf("migrating %u KiB to VRAM\n", size / 1024);
 601       }
 602
 603       offset = buf->offset;
 604       bo = buf->bo;
 605       buf->bo = NULL;
 606       buf->mm = NULL;
 607       nouveau_buffer_allocate(screen, buf, new_domain);
 608
 609       nv->copy_data(nv, buf->bo, buf->offset, new_domain,
 610                     bo, offset, old_domain, buf->base.width0);
 611
 612       nouveau_bo_ref(NULL, &bo);
 613       if (mm)
 614          release_allocation(&mm, screen->fence.current);
 615    } else
 616    if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) {
 617       struct nouveau_transfer tx;
 618       if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
 619          return FALSE;
 620       tx.base.resource = &buf->base;
 621       tx.base.box.x = 0;
 622       tx.base.box.width = buf->base.width0;
 623       tx.bo = NULL;
 624       if (!nouveau_transfer_staging(nv, &tx, FALSE))
 625          return FALSE;
 626       nouveau_transfer_write(nv, &tx, 0, tx.base.box.width);
 627       nouveau_buffer_transfer_del(nv, &tx);
 628    } else
 629       return FALSE;
 630
 631    assert(buf->domain == new_domain);
 632    return TRUE;
 633 }
 634
 635 /* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
 636  * We'd like to only allocate @size bytes here, but then we'd have to rebase
 637  * the vertex indices ...
 638  */
 639 boolean
 640 nouveau_user_buffer_upload(struct nouveau_context *nv,
 641                            struct nv04_resource *buf,
 642                            unsigned base, unsigned size)
 643 {
 644    struct nouveau_screen *screen = nouveau_screen(buf->base.screen);
 645    int ret;
 646
 647    assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY);
 648
 649    buf->base.width0 = base + size;
 650    if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART))
 651       return FALSE;
 652
 653    ret = nouveau_bo_map(buf->bo, 0, nv->client);
 654    if (ret)
 655       return FALSE;
 656    memcpy((uint8_t *)buf->bo->map + buf->offset + base, buf->data + base, size);
 657
 658    return TRUE;
 659 }
 660
 661
 662 /* Scratch data allocation. */
 663
 664 static INLINE int
 665 nouveau_scratch_bo_alloc(struct nouveau_context *nv, struct nouveau_bo **pbo,
 666                          unsigned size)
 667 {
 668    return nouveau_bo_new(nv->screen->device, NOUVEAU_BO_GART | NOUVEAU_BO_MAP,
 669                          4096, size, NULL, pbo);
 670 }
 671
 672 void
 673 nouveau_scratch_runout_release(struct nouveau_context *nv)
 674 {
 675    if (!nv->scratch.nr_runout)
 676       return;
 677    do {
 678       --nv->scratch.nr_runout;
 679       nouveau_bo_ref(NULL, &nv->scratch.runout[nv->scratch.nr_runout]);
 680    } while (nv->scratch.nr_runout);
 681
 682    FREE(nv->scratch.runout);
 683    nv->scratch.end = 0;
 684    nv->scratch.runout = NULL;
 685 }
 686
 687 /* Allocate an extra bo if we can't fit everything we need simultaneously.
 688  * (Could happen for very large user arrays.)
 689  */
 690 static INLINE boolean
 691 nouveau_scratch_runout(struct nouveau_context *nv, unsigned size)
 692 {
 693    int ret;
 694    const unsigned n = nv->scratch.nr_runout++;
 695
 696    nv->scratch.runout = REALLOC(nv->scratch.runout,
 697                                 (n + 0) * sizeof(*nv->scratch.runout),
 698                                 (n + 1) * sizeof(*nv->scratch.runout));
 699    nv->scratch.runout[n] = NULL;
 700
 701    ret = nouveau_scratch_bo_alloc(nv, &nv->scratch.runout[n], size);
 702    if (!ret) {
 703       ret = nouveau_bo_map(nv->scratch.runout[n], 0, NULL);
 704       if (ret)
 705          nouveau_bo_ref(NULL, &nv->scratch.runout[--nv->scratch.nr_runout]);
 706    }
 707    if (!ret) {
 708       nv->scratch.current = nv->scratch.runout[n];
 709       nv->scratch.offset = 0;
 710       nv->scratch.end = size;
 711       nv->scratch.map = nv->scratch.current->map;
 712    }
 713    return !ret;
 714 }
 715
 716 /* Continue to next scratch buffer, if available (no wrapping, large enough).
 717  * Allocate it if it has not yet been created.
 718  */
 719 static INLINE boolean
 720 nouveau_scratch_next(struct nouveau_context *nv, unsigned size)
 721 {
 722    struct nouveau_bo *bo;
 723    int ret;
 724    const unsigned i = (nv->scratch.id + 1) % NOUVEAU_MAX_SCRATCH_BUFS;
 725
 726    if ((size > nv->scratch.bo_size) || (i == nv->scratch.wrap))
 727       return FALSE;
 728    nv->scratch.id = i;
 729
 730    bo = nv->scratch.bo[i];
 731    if (!bo) {
 732       ret = nouveau_scratch_bo_alloc(nv, &bo, nv->scratch.bo_size);
 733       if (ret)
 734          return FALSE;
 735       nv->scratch.bo[i] = bo;
 736    }
 737    nv->scratch.current = bo;
 738    nv->scratch.offset = 0;
 739    nv->scratch.end = nv->scratch.bo_size;
 740
 741    ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, nv->client);
 742    if (!ret)
 743       nv->scratch.map = bo->map;
 744    return !ret;
 745 }
 746
 747 static boolean
 748 nouveau_scratch_more(struct nouveau_context *nv, unsigned min_size)
 749 {
 750    boolean ret;
 751
 752    ret = nouveau_scratch_next(nv, min_size);
 753    if (!ret)
 754       ret = nouveau_scratch_runout(nv, min_size);
 755    return ret;
 756 }
 757
 758
 759 /* Copy data to a scratch buffer and return address & bo the data resides in. */
 760 uint64_t
 761 nouveau_scratch_data(struct nouveau_context *nv,
 762                      const void *data, unsigned base, unsigned size,
 763                      struct nouveau_bo **bo)
 764 {
 765    unsigned bgn = MAX2(base, nv->scratch.offset);
 766    unsigned end = bgn + size;
 767
 768    if (end >= nv->scratch.end) {
 769       end = base + size;
 770       if (!nouveau_scratch_more(nv, end))
 771          return 0;
 772       bgn = base;
 773    }
 774    nv->scratch.offset = align(end, 4);
 775
 776    memcpy(nv->scratch.map + bgn, (const uint8_t *)data + base, size);
 777
 778    *bo = nv->scratch.current;
 779    return (*bo)->offset + (bgn - base);
 780 }
 781
 782 void *
 783 nouveau_scratch_get(struct nouveau_context *nv,
 784                     unsigned size, uint64_t *gpu_addr, struct nouveau_bo **pbo)
 785 {
 786    unsigned bgn = nv->scratch.offset;
 787    unsigned end = nv->scratch.offset + size;
 788
 789    if (end >= nv->scratch.end) {
 790       end = size;
 791       if (!nouveau_scratch_more(nv, end))
 792          return NULL;
 793       bgn = 0;
 794    }
 795    nv->scratch.offset = align(end, 4);
 796
 797    *pbo = nv->scratch.current;
 798    *gpu_addr = nv->scratch.current->offset + bgn;
 799    return nv->scratch.map + bgn;
 800 }