src/gallium/drivers/nouveau/nouveau_buffer.c

   1
   2 #include "util/u_inlines.h"
   3 #include "util/u_memory.h"
   4 #include "util/u_math.h"
   5
   6 #include "nouveau_screen.h"
   7 #include "nouveau_context.h"
   8 #include "nouveau_winsys.h"
   9 #include "nouveau_fence.h"
  10 #include "nouveau_buffer.h"
  11 #include "nouveau_mm.h"
  12
  13 struct nouveau_transfer {
  14    struct pipe_transfer base;
  15 };
  16
  17 static INLINE struct nouveau_transfer *
  18 nouveau_transfer(struct pipe_transfer *transfer)
  19 {
  20    return (struct nouveau_transfer *)transfer;
  21 }
  22
  23 static INLINE boolean
  24 nouveau_buffer_allocate(struct nouveau_screen *screen,
  25                         struct nv04_resource *buf, unsigned domain)
  26 {
  27    uint32_t size = buf->base.width0;
  28
  29    if (buf->base.bind & PIPE_BIND_CONSTANT_BUFFER)
  30       size = align(size, 0x100);
  31
  32    if (domain == NOUVEAU_BO_VRAM) {
  33       buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size,
  34                                     &buf->bo, &buf->offset);
  35       if (!buf->bo)
  36          return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
  37    } else
  38    if (domain == NOUVEAU_BO_GART) {
  39       buf->mm = nouveau_mm_allocate(screen->mm_GART, size,
  40                                     &buf->bo, &buf->offset);
  41       if (!buf->bo)
  42          return FALSE;
  43    }
  44    if (domain != NOUVEAU_BO_GART) {
  45       if (!buf->data) {
  46          buf->data = MALLOC(buf->base.width0);
  47          if (!buf->data)
  48             return FALSE;
  49       }
  50    }
  51    buf->domain = domain;
  52    if (buf->bo)
  53       buf->address = buf->bo->offset + buf->offset;
  54
  55    return TRUE;
  56 }
  57
  58 static INLINE void
  59 release_allocation(struct nouveau_mm_allocation **mm,
  60                    struct nouveau_fence *fence)
  61 {
  62    nouveau_fence_work(fence, nouveau_mm_free_work, *mm);
  63    (*mm) = NULL;
  64 }
  65
  66 INLINE void
  67 nouveau_buffer_release_gpu_storage(struct nv04_resource *buf)
  68 {
  69    nouveau_bo_ref(NULL, &buf->bo);
  70
  71    if (buf->mm)
  72       release_allocation(&buf->mm, buf->fence);
  73
  74    buf->domain = 0;
  75 }
  76
  77 static INLINE boolean
  78 nouveau_buffer_reallocate(struct nouveau_screen *screen,
  79                           struct nv04_resource *buf, unsigned domain)
  80 {
  81    nouveau_buffer_release_gpu_storage(buf);
  82
  83    return nouveau_buffer_allocate(screen, buf, domain);
  84 }
  85
  86 static void
  87 nouveau_buffer_destroy(struct pipe_screen *pscreen,
  88                        struct pipe_resource *presource)
  89 {
  90    struct nv04_resource *res = nv04_resource(presource);
  91
  92    nouveau_buffer_release_gpu_storage(res);
  93
  94    if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
  95       FREE(res->data);
  96
  97    FREE(res);
  98 }
  99
 100 /* Maybe just migrate to GART right away if we actually need to do this. */
 101 boolean
 102 nouveau_buffer_download(struct nouveau_context *nv, struct nv04_resource *buf,
 103                         unsigned start, unsigned size)
 104 {
 105    struct nouveau_mm_allocation *mm;
 106    struct nouveau_bo *bounce = NULL;
 107    uint32_t offset;
 108
 109    assert(buf->domain == NOUVEAU_BO_VRAM);
 110
 111    mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset);
 112    if (!bounce)
 113       return FALSE;
 114
 115    nv->copy_data(nv, bounce, offset, NOUVEAU_BO_GART,
 116                  buf->bo, buf->offset + start, NOUVEAU_BO_VRAM, size);
 117
 118    if (nouveau_bo_map(bounce, NOUVEAU_BO_RD, nv->screen->client))
 119       return FALSE;
 120    memcpy(buf->data + start, (uint8_t *)bounce->map + offset, size);
 121
 122    buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
 123
 124    nouveau_bo_ref(NULL, &bounce);
 125    if (mm)
 126       nouveau_mm_free(mm);
 127    return TRUE;
 128 }
 129
 130 static boolean
 131 nouveau_buffer_upload(struct nouveau_context *nv, struct nv04_resource *buf,
 132                       unsigned start, unsigned size)
 133 {
 134    struct nouveau_mm_allocation *mm;
 135    struct nouveau_bo *bounce = NULL;
 136    uint32_t offset;
 137
 138    if (size <= 192 && (nv->push_data || nv->push_cb)) {
 139       if (buf->base.bind & PIPE_BIND_CONSTANT_BUFFER)
 140          nv->push_cb(nv, buf->bo, buf->domain, buf->offset, buf->base.width0,
 141                      start, size / 4, (const uint32_t *)(buf->data + start));
 142       else
 143          nv->push_data(nv, buf->bo, buf->offset + start, buf->domain,
 144                        size, buf->data + start);
 145       return TRUE;
 146    }
 147
 148    mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset);
 149    if (!bounce)
 150       return FALSE;
 151
 152    nouveau_bo_map(bounce, 0, nv->screen->client);
 153    memcpy((uint8_t *)bounce->map + offset, buf->data + start, size);
 154
 155    nv->copy_data(nv, buf->bo, buf->offset + start, NOUVEAU_BO_VRAM,
 156                  bounce, offset, NOUVEAU_BO_GART, size);
 157
 158    nouveau_bo_ref(NULL, &bounce);
 159    if (mm)
 160       release_allocation(&mm, nv->screen->fence.current);
 161
 162    if (start == 0 && size == buf->base.width0)
 163       buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
 164    return TRUE;
 165 }
 166
 167 static struct pipe_transfer *
 168 nouveau_buffer_transfer_get(struct pipe_context *pipe,
 169                             struct pipe_resource *resource,
 170                             unsigned level, unsigned usage,
 171                             const struct pipe_box *box)
 172 {
 173    struct nv04_resource *buf = nv04_resource(resource);
 174    struct nouveau_context *nv = nouveau_context(pipe);
 175    struct nouveau_transfer *xfr = CALLOC_STRUCT(nouveau_transfer);
 176    if (!xfr)
 177       return NULL;
 178
 179    xfr->base.resource = resource;
 180    xfr->base.box.x = box->x;
 181    xfr->base.box.width = box->width;
 182    xfr->base.usage = usage;
 183
 184    if (buf->domain == NOUVEAU_BO_VRAM) {
 185       if (usage & PIPE_TRANSFER_READ) {
 186          if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING)
 187             nouveau_buffer_download(nv, buf, 0, buf->base.width0);
 188       }
 189    }
 190
 191    return &xfr->base;
 192 }
 193
 194 static void
 195 nouveau_buffer_transfer_destroy(struct pipe_context *pipe,
 196                                 struct pipe_transfer *transfer)
 197 {
 198    struct nv04_resource *buf = nv04_resource(transfer->resource);
 199    struct nouveau_transfer *xfr = nouveau_transfer(transfer);
 200    struct nouveau_context *nv = nouveau_context(pipe);
 201
 202    if (xfr->base.usage & PIPE_TRANSFER_WRITE) {
 203       if (buf->domain == NOUVEAU_BO_VRAM) {
 204          nouveau_buffer_upload(nv, buf, transfer->box.x, transfer->box.width);
 205       }
 206
 207       if (buf->domain != 0 && (buf->base.bind & (PIPE_BIND_VERTEX_BUFFER |
 208                                                  PIPE_BIND_INDEX_BUFFER)))
 209          nouveau_context(pipe)->vbo_dirty = TRUE;
 210    }
 211
 212    FREE(xfr);
 213 }
 214
 215 static INLINE boolean
 216 nouveau_buffer_sync(struct nv04_resource *buf, unsigned rw)
 217 {
 218    if (rw == PIPE_TRANSFER_READ) {
 219       if (!buf->fence_wr)
 220          return TRUE;
 221       if (!nouveau_fence_wait(buf->fence_wr))
 222          return FALSE;
 223    } else {
 224       if (!buf->fence)
 225          return TRUE;
 226       if (!nouveau_fence_wait(buf->fence))
 227          return FALSE;
 228
 229       nouveau_fence_ref(NULL, &buf->fence);
 230    }
 231    nouveau_fence_ref(NULL, &buf->fence_wr);
 232
 233    return TRUE;
 234 }
 235
 236 static INLINE boolean
 237 nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw)
 238 {
 239    if (rw == PIPE_TRANSFER_READ)
 240       return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr));
 241    else
 242       return (buf->fence && !nouveau_fence_signalled(buf->fence));
 243 }
 244
 245 static void *
 246 nouveau_buffer_transfer_map(struct pipe_context *pipe,
 247                             struct pipe_transfer *transfer)
 248 {
 249    struct nouveau_context *nv = nouveau_context(pipe);
 250    struct nouveau_transfer *xfr = nouveau_transfer(transfer);
 251    struct nv04_resource *buf = nv04_resource(transfer->resource);
 252    struct nouveau_bo *bo = buf->bo;
 253    uint8_t *map;
 254    int ret;
 255    uint32_t offset = xfr->base.box.x;
 256    uint32_t flags = 0;
 257
 258    if (buf->domain != NOUVEAU_BO_GART)
 259       return buf->data + offset;
 260
 261    if (!buf->mm)
 262       flags = nouveau_screen_transfer_flags(xfr->base.usage);
 263
 264    offset += buf->offset;
 265
 266    ret = nouveau_bo_map(buf->bo, flags, nv->screen->client);
 267    if (ret)
 268       return NULL;
 269    map = (uint8_t *)bo->map + offset;
 270
 271    if (buf->mm) {
 272       if (xfr->base.usage & PIPE_TRANSFER_DONTBLOCK) {
 273          if (nouveau_buffer_busy(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE))
 274             return NULL;
 275       } else
 276       if (!(xfr->base.usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
 277          nouveau_buffer_sync(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE);
 278       }
 279    }
 280    return map;
 281 }
 282
 283
 284
 285 static void
 286 nouveau_buffer_transfer_flush_region(struct pipe_context *pipe,
 287                                      struct pipe_transfer *transfer,
 288                                      const struct pipe_box *box)
 289 {
 290 #if 0
 291    struct nv04_resource *res = nv04_resource(transfer->resource);
 292    struct nouveau_bo *bo = res->bo;
 293    unsigned offset = res->offset + transfer->box.x + box->x;
 294
 295    /* not using non-snoop system memory yet, no need for cflush */
 296    if (1)
 297       return;
 298
 299    /* XXX: maybe need to upload for VRAM buffers here */
 300 #endif
 301 }
 302
 303 static void
 304 nouveau_buffer_transfer_unmap(struct pipe_context *pipe,
 305                               struct pipe_transfer *transfer)
 306 {
 307 }
 308
 309
 310 void *
 311 nouveau_resource_map_offset(struct nouveau_context *nv,
 312                             struct nv04_resource *res, uint32_t offset,
 313                             uint32_t flags)
 314 {
 315    if ((res->domain == NOUVEAU_BO_VRAM) &&
 316        (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING))
 317       nouveau_buffer_download(nv, res, 0, res->base.width0);
 318
 319    if ((res->domain != NOUVEAU_BO_GART) ||
 320        (res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
 321       return res->data + offset;
 322
 323    if (res->mm) {
 324       unsigned rw;
 325       rw = (flags & NOUVEAU_BO_WR) ? PIPE_TRANSFER_WRITE : PIPE_TRANSFER_READ;
 326       nouveau_buffer_sync(res, rw);
 327       if (nouveau_bo_map(res->bo, 0, NULL))
 328          return NULL;
 329    } else {
 330       if (nouveau_bo_map(res->bo, flags, nv->screen->client))
 331          return NULL;
 332    }
 333    return (uint8_t *)res->bo->map + res->offset + offset;
 334 }
 335
 336
 337 const struct u_resource_vtbl nouveau_buffer_vtbl =
 338 {
 339    u_default_resource_get_handle,     /* get_handle */
 340    nouveau_buffer_destroy,               /* resource_destroy */
 341    nouveau_buffer_transfer_get,          /* get_transfer */
 342    nouveau_buffer_transfer_destroy,      /* transfer_destroy */
 343    nouveau_buffer_transfer_map,          /* transfer_map */
 344    nouveau_buffer_transfer_flush_region, /* transfer_flush_region */
 345    nouveau_buffer_transfer_unmap,        /* transfer_unmap */
 346    u_default_transfer_inline_write    /* transfer_inline_write */
 347 };
 348
 349 struct pipe_resource *
 350 nouveau_buffer_create(struct pipe_screen *pscreen,
 351                       const struct pipe_resource *templ)
 352 {
 353    struct nouveau_screen *screen = nouveau_screen(pscreen);
 354    struct nv04_resource *buffer;
 355    boolean ret;
 356
 357    buffer = CALLOC_STRUCT(nv04_resource);
 358    if (!buffer)
 359       return NULL;
 360
 361    buffer->base = *templ;
 362    buffer->vtbl = &nouveau_buffer_vtbl;
 363    pipe_reference_init(&buffer->base.reference, 1);
 364    buffer->base.screen = pscreen;
 365
 366    if ((buffer->base.bind & screen->sysmem_bindings) == screen->sysmem_bindings)
 367       ret = nouveau_buffer_allocate(screen, buffer, 0);
 368    else
 369       ret = nouveau_buffer_allocate(screen, buffer, NOUVEAU_BO_GART);
 370
 371    if (ret == FALSE)
 372       goto fail;
 373
 374    return &buffer->base;
 375
 376 fail:
 377    FREE(buffer);
 378    return NULL;
 379 }
 380
 381
 382 struct pipe_resource *
 383 nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr,
 384                            unsigned bytes, unsigned bind)
 385 {
 386    struct nv04_resource *buffer;
 387
 388    buffer = CALLOC_STRUCT(nv04_resource);
 389    if (!buffer)
 390       return NULL;
 391
 392    pipe_reference_init(&buffer->base.reference, 1);
 393    buffer->vtbl = &nouveau_buffer_vtbl;
 394    buffer->base.screen = pscreen;
 395    buffer->base.format = PIPE_FORMAT_R8_UNORM;
 396    buffer->base.usage = PIPE_USAGE_IMMUTABLE;
 397    buffer->base.bind = bind;
 398    buffer->base.width0 = bytes;
 399    buffer->base.height0 = 1;
 400    buffer->base.depth0 = 1;
 401    buffer->base.user_ptr = ptr;
 402
 403    buffer->data = ptr;
 404    buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY;
 405
 406    return &buffer->base;
 407 }
 408
 409 /* Like download, but for GART buffers. Merge ? */
 410 static INLINE boolean
 411 nouveau_buffer_data_fetch(struct nouveau_context *nv, struct nv04_resource *buf,
 412                           struct nouveau_bo *bo, unsigned offset, unsigned size)
 413 {
 414    if (!buf->data) {
 415       buf->data = MALLOC(size);
 416       if (!buf->data)
 417          return FALSE;
 418    }
 419    if (nouveau_bo_map(bo, NOUVEAU_BO_RD, nv->screen->client))
 420       return FALSE;
 421    memcpy(buf->data, (uint8_t *)bo->map + offset, size);
 422
 423    return TRUE;
 424 }
 425
 426 /* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
 427 boolean
 428 nouveau_buffer_migrate(struct nouveau_context *nv,
 429                        struct nv04_resource *buf, const unsigned new_domain)
 430 {
 431    struct nouveau_screen *screen = nv->screen;
 432    struct nouveau_bo *bo;
 433    const unsigned old_domain = buf->domain;
 434    unsigned size = buf->base.width0;
 435    unsigned offset;
 436    int ret;
 437
 438    assert(new_domain != old_domain);
 439
 440    if (new_domain == NOUVEAU_BO_GART && old_domain == 0) {
 441       if (!nouveau_buffer_allocate(screen, buf, new_domain))
 442          return FALSE;
 443       ret = nouveau_bo_map(buf->bo, 0, nv->screen->client);
 444       if (ret)
 445          return ret;
 446       memcpy((uint8_t *)buf->bo->map + buf->offset, buf->data, size);
 447       FREE(buf->data);
 448    } else
 449    if (old_domain != 0 && new_domain != 0) {
 450       struct nouveau_mm_allocation *mm = buf->mm;
 451
 452       if (new_domain == NOUVEAU_BO_VRAM) {
 453          /* keep a system memory copy of our data in case we hit a fallback */
 454          if (!nouveau_buffer_data_fetch(nv, buf, buf->bo, buf->offset, size))
 455             return FALSE;
 456          if (nouveau_mesa_debug)
 457             debug_printf("migrating %u KiB to VRAM\n", size / 1024);
 458       }
 459
 460       offset = buf->offset;
 461       bo = buf->bo;
 462       buf->bo = NULL;
 463       buf->mm = NULL;
 464       nouveau_buffer_allocate(screen, buf, new_domain);
 465
 466       nv->copy_data(nv, buf->bo, buf->offset, new_domain,
 467                     bo, offset, old_domain, buf->base.width0);
 468
 469       nouveau_bo_ref(NULL, &bo);
 470       if (mm)
 471          release_allocation(&mm, screen->fence.current);
 472    } else
 473    if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) {
 474       if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
 475          return FALSE;
 476       if (!nouveau_buffer_upload(nv, buf, 0, buf->base.width0))
 477          return FALSE;
 478    } else
 479       return FALSE;
 480
 481    assert(buf->domain == new_domain);
 482    return TRUE;
 483 }
 484
 485 /* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
 486  * We'd like to only allocate @size bytes here, but then we'd have to rebase
 487  * the vertex indices ...
 488  */
 489 boolean
 490 nouveau_user_buffer_upload(struct nouveau_context *nv,
 491                            struct nv04_resource *buf,
 492                            unsigned base, unsigned size)
 493 {
 494    struct nouveau_screen *screen = nouveau_screen(buf->base.screen);
 495    int ret;
 496
 497    assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY);
 498
 499    buf->base.width0 = base + size;
 500    if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART))
 501       return FALSE;
 502
 503    ret = nouveau_bo_map(buf->bo, 0, nv->screen->client);
 504    if (ret)
 505       return FALSE;
 506    memcpy((uint8_t *)buf->bo->map + buf->offset + base, buf->data + base, size);
 507
 508    return TRUE;
 509 }
 510
 511
 512 /* Scratch data allocation. */
 513
 514 static INLINE int
 515 nouveau_scratch_bo_alloc(struct nouveau_context *nv, struct nouveau_bo **pbo,
 516                          unsigned size)
 517 {
 518    return nouveau_bo_new(nv->screen->device, NOUVEAU_BO_GART | NOUVEAU_BO_MAP,
 519                          4096, size, NULL, pbo);
 520 }
 521
 522 void
 523 nouveau_scratch_runout_release(struct nouveau_context *nv)
 524 {
 525    if (!nv->scratch.nr_runout)
 526       return;
 527    while (nv->scratch.nr_runout--)
 528       nouveau_bo_ref(NULL, &nv->scratch.runout[nv->scratch.nr_runout]);
 529
 530    FREE(nv->scratch.runout);
 531    nv->scratch.end = 0;
 532    nv->scratch.runout = NULL;
 533 }
 534
 535 /* Allocate an extra bo if we can't fit everything we need simultaneously.
 536  * (Could happen for very large user arrays.)
 537  */
 538 static INLINE boolean
 539 nouveau_scratch_runout(struct nouveau_context *nv, unsigned size)
 540 {
 541    int ret;
 542    const unsigned n = nv->scratch.nr_runout++;
 543
 544    nv->scratch.runout = REALLOC(nv->scratch.runout,
 545                                 (n + 0) * sizeof(*nv->scratch.runout),
 546                                 (n + 1) * sizeof(*nv->scratch.runout));
 547    nv->scratch.runout[n] = NULL;
 548
 549    ret = nouveau_scratch_bo_alloc(nv, &nv->scratch.runout[n], size);
 550    if (!ret) {
 551       ret = nouveau_bo_map(nv->scratch.runout[n], 0, NULL);
 552       if (ret)
 553          nouveau_bo_ref(NULL, &nv->scratch.runout[--nv->scratch.nr_runout]);
 554    }
 555    if (!ret) {
 556       nv->scratch.current = nv->scratch.runout[n];
 557       nv->scratch.offset = 0;
 558       nv->scratch.end = size;
 559       nv->scratch.map = nv->scratch.current->map;
 560    }
 561    return !ret;
 562 }
 563
 564 /* Continue to next scratch buffer, if available (no wrapping, large enough).
 565  * Allocate it if it has not yet been created.
 566  */
 567 static INLINE boolean
 568 nouveau_scratch_next(struct nouveau_context *nv, unsigned size)
 569 {
 570    struct nouveau_bo *bo;
 571    int ret;
 572    const unsigned i = (nv->scratch.id + 1) % NOUVEAU_MAX_SCRATCH_BUFS;
 573
 574    if ((size > nv->scratch.bo_size) || (i == nv->scratch.wrap))
 575       return FALSE;
 576    nv->scratch.id = i;
 577
 578    bo = nv->scratch.bo[i];
 579    if (!bo) {
 580       ret = nouveau_scratch_bo_alloc(nv, &bo, nv->scratch.bo_size);
 581       if (ret)
 582          return FALSE;
 583       nv->scratch.bo[i] = bo;
 584    }
 585    nv->scratch.current = bo;
 586    nv->scratch.offset = 0;
 587    nv->scratch.end = nv->scratch.bo_size;
 588
 589    ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, nv->screen->client);
 590    if (!ret)
 591       nv->scratch.map = bo->map;
 592    return !ret;
 593 }
 594
 595 static boolean
 596 nouveau_scratch_more(struct nouveau_context *nv, unsigned min_size)
 597 {
 598    boolean ret;
 599
 600    ret = nouveau_scratch_next(nv, min_size);
 601    if (!ret)
 602       ret = nouveau_scratch_runout(nv, min_size);
 603    return ret;
 604 }
 605
 606 /* Upload data to scratch memory and update buffer address.
 607  * Returns the bo the data resides in, if successful.
 608  */
 609 struct nouveau_bo *
 610 nouveau_scratch_data(struct nouveau_context *nv,
 611                      struct nv04_resource *buf, unsigned base, unsigned size)
 612 {
 613    struct nouveau_bo *bo;
 614    unsigned bgn = MAX2(base, nv->scratch.offset);
 615    unsigned end = bgn + size;
 616
 617    if (end >= nv->scratch.end) {
 618       end = base + size;
 619       if (!nouveau_scratch_more(nv, end))
 620          return NULL;
 621       bgn = base;
 622    }
 623    nv->scratch.offset = align(end, 4);
 624
 625    memcpy(nv->scratch.map + bgn, buf->data + base, size);
 626
 627    bo = nv->scratch.current;
 628    buf->address = bo->offset + (bgn - base);
 629    return bo;
 630 }
 631
 632 void *
 633 nouveau_scratch_get(struct nouveau_context *nv,
 634                     unsigned size, uint64_t *gpu_addr, struct nouveau_bo **pbo)
 635 {
 636    unsigned bgn = nv->scratch.offset;
 637    unsigned end = nv->scratch.offset + size;
 638
 639    if (end >= nv->scratch.end) {
 640       end = size;
 641       if (!nouveau_scratch_more(nv, end))
 642          return NULL;
 643       bgn = 0;
 644    }
 645    nv->scratch.offset = align(end, 4);
 646
 647    *pbo = nv->scratch.current;
 648    *gpu_addr = nv->scratch.current->offset + bgn;
 649    return nv->scratch.map + bgn;
 650 }