src/gallium/drivers/nouveau/nouveau_buffer.c

   1
   2 #include "util/u_inlines.h"
   3 #include "util/u_memory.h"
   4 #include "util/u_math.h"
   5
   6 #include "nouveau_screen.h"
   7 #include "nouveau_context.h"
   8 #include "nouveau_winsys.h"
   9 #include "nouveau_fence.h"
  10 #include "nouveau_buffer.h"
  11 #include "nouveau_mm.h"
  12
  13 struct nouveau_transfer {
  14    struct pipe_transfer base;
  15 };
  16
  17 static INLINE struct nouveau_transfer *
  18 nouveau_transfer(struct pipe_transfer *transfer)
  19 {
  20    return (struct nouveau_transfer *)transfer;
  21 }
  22
  23 static INLINE boolean
  24 nouveau_buffer_allocate(struct nouveau_screen *screen,
  25                         struct nv04_resource *buf, unsigned domain)
  26 {
  27    uint32_t size = buf->base.width0;
  28
  29    if (buf->base.bind & PIPE_BIND_CONSTANT_BUFFER)
  30       size = align(size, 0x100);
  31
  32    if (domain == NOUVEAU_BO_VRAM) {
  33       buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size,
  34                                     &buf->bo, &buf->offset);
  35       if (!buf->bo)
  36          return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
  37    } else
  38    if (domain == NOUVEAU_BO_GART) {
  39       buf->mm = nouveau_mm_allocate(screen->mm_GART, size,
  40                                     &buf->bo, &buf->offset);
  41       if (!buf->bo)
  42          return FALSE;
  43    }
  44    if (domain != NOUVEAU_BO_GART) {
  45       if (!buf->data) {
  46          buf->data = MALLOC(buf->base.width0);
  47          if (!buf->data)
  48             return FALSE;
  49       }
  50    }
  51    buf->domain = domain;
  52    if (buf->bo)
  53       buf->address = buf->bo->offset + buf->offset;
  54
  55    return TRUE;
  56 }
  57
  58 static INLINE void
  59 release_allocation(struct nouveau_mm_allocation **mm,
  60                    struct nouveau_fence *fence)
  61 {
  62    nouveau_fence_work(fence, nouveau_mm_free_work, *mm);
  63    (*mm) = NULL;
  64 }
  65
  66 INLINE void
  67 nouveau_buffer_release_gpu_storage(struct nv04_resource *buf)
  68 {
  69    nouveau_bo_ref(NULL, &buf->bo);
  70
  71    if (buf->mm)
  72       release_allocation(&buf->mm, buf->fence);
  73
  74    buf->domain = 0;
  75 }
  76
  77 static INLINE boolean
  78 nouveau_buffer_reallocate(struct nouveau_screen *screen,
  79                           struct nv04_resource *buf, unsigned domain)
  80 {
  81    nouveau_buffer_release_gpu_storage(buf);
  82
  83    return nouveau_buffer_allocate(screen, buf, domain);
  84 }
  85
  86 static void
  87 nouveau_buffer_destroy(struct pipe_screen *pscreen,
  88                        struct pipe_resource *presource)
  89 {
  90    struct nv04_resource *res = nv04_resource(presource);
  91
  92    nouveau_buffer_release_gpu_storage(res);
  93
  94    if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
  95       FREE(res->data);
  96
  97    FREE(res);
  98 }
  99
 100 /* Maybe just migrate to GART right away if we actually need to do this. */
 101 boolean
 102 nouveau_buffer_download(struct nouveau_context *nv, struct nv04_resource *buf,
 103                         unsigned start, unsigned size)
 104 {
 105    struct nouveau_mm_allocation *mm;
 106    struct nouveau_bo *bounce = NULL;
 107    uint32_t offset;
 108
 109    assert(buf->domain == NOUVEAU_BO_VRAM);
 110
 111    mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset);
 112    if (!bounce)
 113       return FALSE;
 114
 115    nv->copy_data(nv, bounce, offset, NOUVEAU_BO_GART,
 116                  buf->bo, buf->offset + start, NOUVEAU_BO_VRAM, size);
 117
 118    if (nouveau_bo_map(bounce, NOUVEAU_BO_RD, nv->screen->client))
 119       return FALSE;
 120    memcpy(buf->data + start, (uint8_t *)bounce->map + offset, size);
 121
 122    buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
 123
 124    nouveau_bo_ref(NULL, &bounce);
 125    if (mm)
 126       nouveau_mm_free(mm);
 127    return TRUE;
 128 }
 129
 130 static boolean
 131 nouveau_buffer_upload(struct nouveau_context *nv, struct nv04_resource *buf,
 132                       unsigned start, unsigned size)
 133 {
 134    struct nouveau_mm_allocation *mm;
 135    struct nouveau_bo *bounce = NULL;
 136    uint32_t offset;
 137
 138    if (size <= 192 && (nv->push_data || nv->push_cb)) {
 139       if (buf->base.bind & PIPE_BIND_CONSTANT_BUFFER)
 140          nv->push_cb(nv, buf->bo, buf->domain, buf->offset, buf->base.width0,
 141                      start, size / 4, (const uint32_t *)(buf->data + start));
 142       else
 143          nv->push_data(nv, buf->bo, buf->offset + start, buf->domain,
 144                        size, buf->data + start);
 145       return TRUE;
 146    }
 147
 148    mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset);
 149    if (!bounce)
 150       return FALSE;
 151
 152    nouveau_bo_map(bounce, 0, nv->screen->client);
 153    memcpy((uint8_t *)bounce->map + offset, buf->data + start, size);
 154
 155    nv->copy_data(nv, buf->bo, buf->offset + start, NOUVEAU_BO_VRAM,
 156                  bounce, offset, NOUVEAU_BO_GART, size);
 157
 158    nouveau_bo_ref(NULL, &bounce);
 159    if (mm)
 160       release_allocation(&mm, nv->screen->fence.current);
 161
 162    if (start == 0 && size == buf->base.width0)
 163       buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
 164    return TRUE;
 165 }
 166
 167 static struct pipe_transfer *
 168 nouveau_buffer_transfer_get(struct pipe_context *pipe,
 169                             struct pipe_resource *resource,
 170                             unsigned level, unsigned usage,
 171                             const struct pipe_box *box)
 172 {
 173    struct nv04_resource *buf = nv04_resource(resource);
 174    struct nouveau_context *nv = nouveau_context(pipe);
 175    struct nouveau_transfer *xfr = CALLOC_STRUCT(nouveau_transfer);
 176    if (!xfr)
 177       return NULL;
 178
 179    xfr->base.resource = resource;
 180    xfr->base.box.x = box->x;
 181    xfr->base.box.width = box->width;
 182    xfr->base.usage = usage;
 183
 184    if (buf->domain == NOUVEAU_BO_VRAM) {
 185       if (usage & PIPE_TRANSFER_READ) {
 186          if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING)
 187             nouveau_buffer_download(nv, buf, 0, buf->base.width0);
 188       }
 189    }
 190
 191    return &xfr->base;
 192 }
 193
 194 static void
 195 nouveau_buffer_transfer_destroy(struct pipe_context *pipe,
 196                                 struct pipe_transfer *transfer)
 197 {
 198    struct nv04_resource *buf = nv04_resource(transfer->resource);
 199    struct nouveau_transfer *xfr = nouveau_transfer(transfer);
 200    struct nouveau_context *nv = nouveau_context(pipe);
 201
 202    if (xfr->base.usage & PIPE_TRANSFER_WRITE) {
 203       if (buf->domain == NOUVEAU_BO_VRAM) {
 204          nouveau_buffer_upload(nv, buf, transfer->box.x, transfer->box.width);
 205       }
 206
 207       if (buf->domain != 0 && (buf->base.bind & (PIPE_BIND_VERTEX_BUFFER |
 208                                                  PIPE_BIND_INDEX_BUFFER)))
 209          nouveau_context(pipe)->vbo_dirty = TRUE;
 210    }
 211
 212    FREE(xfr);
 213 }
 214
 215 static INLINE boolean
 216 nouveau_buffer_sync(struct nv04_resource *buf, unsigned rw)
 217 {
 218    if (rw == PIPE_TRANSFER_READ) {
 219       if (!buf->fence_wr)
 220          return TRUE;
 221       if (!nouveau_fence_wait(buf->fence_wr))
 222          return FALSE;
 223    } else {
 224       if (!buf->fence)
 225          return TRUE;
 226       if (!nouveau_fence_wait(buf->fence))
 227          return FALSE;
 228
 229       nouveau_fence_ref(NULL, &buf->fence);
 230    }
 231    nouveau_fence_ref(NULL, &buf->fence_wr);
 232
 233    return TRUE;
 234 }
 235
 236 static INLINE boolean
 237 nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw)
 238 {
 239    if (rw == PIPE_TRANSFER_READ)
 240       return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr));
 241    else
 242       return (buf->fence && !nouveau_fence_signalled(buf->fence));
 243 }
 244
 245 static void *
 246 nouveau_buffer_transfer_map(struct pipe_context *pipe,
 247                             struct pipe_transfer *transfer)
 248 {
 249    struct nouveau_context *nv = nouveau_context(pipe);
 250    struct nouveau_transfer *xfr = nouveau_transfer(transfer);
 251    struct nv04_resource *buf = nv04_resource(transfer->resource);
 252    struct nouveau_bo *bo = buf->bo;
 253    uint8_t *map;
 254    int ret;
 255    uint32_t offset = xfr->base.box.x;
 256    uint32_t flags = 0;
 257
 258    if (buf->domain != NOUVEAU_BO_GART)
 259       return buf->data + offset;
 260
 261    if (!buf->mm)
 262       flags = nouveau_screen_transfer_flags(xfr->base.usage);
 263
 264    offset += buf->offset;
 265
 266    ret = nouveau_bo_map(buf->bo, flags, nv->screen->client);
 267    if (ret)
 268       return NULL;
 269    map = (uint8_t *)bo->map + offset;
 270
 271    if (buf->mm) {
 272       if (xfr->base.usage & PIPE_TRANSFER_DONTBLOCK) {
 273          if (nouveau_buffer_busy(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE))
 274             return NULL;
 275       } else
 276       if (!(xfr->base.usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
 277          nouveau_buffer_sync(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE);
 278       }
 279    }
 280    return map;
 281 }
 282
 283
 284
 285 static void
 286 nouveau_buffer_transfer_flush_region(struct pipe_context *pipe,
 287                                      struct pipe_transfer *transfer,
 288                                      const struct pipe_box *box)
 289 {
 290 #if 0
 291    struct nv04_resource *res = nv04_resource(transfer->resource);
 292    struct nouveau_bo *bo = res->bo;
 293    unsigned offset = res->offset + transfer->box.x + box->x;
 294
 295    /* not using non-snoop system memory yet, no need for cflush */
 296    if (1)
 297       return;
 298
 299    /* XXX: maybe need to upload for VRAM buffers here */
 300 #endif
 301 }
 302
 303 static void
 304 nouveau_buffer_transfer_unmap(struct pipe_context *pipe,
 305                               struct pipe_transfer *transfer)
 306 {
 307 }
 308
 309
 310 void *
 311 nouveau_resource_map_offset(struct nouveau_context *nv,
 312                             struct nv04_resource *res, uint32_t offset,
 313                             uint32_t flags)
 314 {
 315    if ((res->domain == NOUVEAU_BO_VRAM) &&
 316        (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING))
 317       nouveau_buffer_download(nv, res, 0, res->base.width0);
 318
 319    if ((res->domain != NOUVEAU_BO_GART) ||
 320        (res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
 321       return res->data + offset;
 322
 323    if (res->mm) {
 324       unsigned rw;
 325       rw = (flags & NOUVEAU_BO_WR) ? PIPE_TRANSFER_WRITE : PIPE_TRANSFER_READ;
 326       nouveau_buffer_sync(res, rw);
 327       if (nouveau_bo_map(res->bo, 0, NULL))
 328          return NULL;
 329    } else {
 330       if (nouveau_bo_map(res->bo, flags, nv->screen->client))
 331          return NULL;
 332    }
 333    return (uint8_t *)res->bo->map + res->offset + offset;
 334 }
 335
 336
 337 const struct u_resource_vtbl nouveau_buffer_vtbl =
 338 {
 339    u_default_resource_get_handle,     /* get_handle */
 340    nouveau_buffer_destroy,               /* resource_destroy */
 341    nouveau_buffer_transfer_get,          /* get_transfer */
 342    nouveau_buffer_transfer_destroy,      /* transfer_destroy */
 343    nouveau_buffer_transfer_map,          /* transfer_map */
 344    nouveau_buffer_transfer_flush_region, /* transfer_flush_region */
 345    nouveau_buffer_transfer_unmap,        /* transfer_unmap */
 346    u_default_transfer_inline_write    /* transfer_inline_write */
 347 };
 348
 349 struct pipe_resource *
 350 nouveau_buffer_create(struct pipe_screen *pscreen,
 351                       const struct pipe_resource *templ)
 352 {
 353    struct nouveau_screen *screen = nouveau_screen(pscreen);
 354    struct nv04_resource *buffer;
 355    boolean ret;
 356
 357    buffer = CALLOC_STRUCT(nv04_resource);
 358    if (!buffer)
 359       return NULL;
 360
 361    buffer->base = *templ;
 362    buffer->vtbl = &nouveau_buffer_vtbl;
 363    pipe_reference_init(&buffer->base.reference, 1);
 364    buffer->base.screen = pscreen;
 365
 366    if ((buffer->base.bind & screen->sysmem_bindings) == screen->sysmem_bindings)
 367       ret = nouveau_buffer_allocate(screen, buffer, 0);
 368    else
 369       ret = nouveau_buffer_allocate(screen, buffer, NOUVEAU_BO_GART);
 370
 371    if (ret == FALSE)
 372       goto fail;
 373
 374    return &buffer->base;
 375
 376 fail:
 377    FREE(buffer);
 378    return NULL;
 379 }
 380
 381
 382 struct pipe_resource *
 383 nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr,
 384                            unsigned bytes, unsigned bind)
 385 {
 386    struct nv04_resource *buffer;
 387
 388    buffer = CALLOC_STRUCT(nv04_resource);
 389    if (!buffer)
 390       return NULL;
 391
 392    pipe_reference_init(&buffer->base.reference, 1);
 393    buffer->vtbl = &nouveau_buffer_vtbl;
 394    buffer->base.screen = pscreen;
 395    buffer->base.format = PIPE_FORMAT_R8_UNORM;
 396    buffer->base.usage = PIPE_USAGE_IMMUTABLE;
 397    buffer->base.bind = bind;
 398    buffer->base.width0 = bytes;
 399    buffer->base.height0 = 1;
 400    buffer->base.depth0 = 1;
 401
 402    buffer->data = ptr;
 403    buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY;
 404
 405    return &buffer->base;
 406 }
 407
 408 /* Like download, but for GART buffers. Merge ? */
 409 static INLINE boolean
 410 nouveau_buffer_data_fetch(struct nouveau_context *nv, struct nv04_resource *buf,
 411                           struct nouveau_bo *bo, unsigned offset, unsigned size)
 412 {
 413    if (!buf->data) {
 414       buf->data = MALLOC(size);
 415       if (!buf->data)
 416          return FALSE;
 417    }
 418    if (nouveau_bo_map(bo, NOUVEAU_BO_RD, nv->screen->client))
 419       return FALSE;
 420    memcpy(buf->data, (uint8_t *)bo->map + offset, size);
 421
 422    return TRUE;
 423 }
 424
 425 /* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
 426 boolean
 427 nouveau_buffer_migrate(struct nouveau_context *nv,
 428                        struct nv04_resource *buf, const unsigned new_domain)
 429 {
 430    struct nouveau_screen *screen = nv->screen;
 431    struct nouveau_bo *bo;
 432    const unsigned old_domain = buf->domain;
 433    unsigned size = buf->base.width0;
 434    unsigned offset;
 435    int ret;
 436
 437    assert(new_domain != old_domain);
 438
 439    if (new_domain == NOUVEAU_BO_GART && old_domain == 0) {
 440       if (!nouveau_buffer_allocate(screen, buf, new_domain))
 441          return FALSE;
 442       ret = nouveau_bo_map(buf->bo, 0, nv->screen->client);
 443       if (ret)
 444          return ret;
 445       memcpy((uint8_t *)buf->bo->map + buf->offset, buf->data, size);
 446       FREE(buf->data);
 447    } else
 448    if (old_domain != 0 && new_domain != 0) {
 449       struct nouveau_mm_allocation *mm = buf->mm;
 450
 451       if (new_domain == NOUVEAU_BO_VRAM) {
 452          /* keep a system memory copy of our data in case we hit a fallback */
 453          if (!nouveau_buffer_data_fetch(nv, buf, buf->bo, buf->offset, size))
 454             return FALSE;
 455          if (nouveau_mesa_debug)
 456             debug_printf("migrating %u KiB to VRAM\n", size / 1024);
 457       }
 458
 459       offset = buf->offset;
 460       bo = buf->bo;
 461       buf->bo = NULL;
 462       buf->mm = NULL;
 463       nouveau_buffer_allocate(screen, buf, new_domain);
 464
 465       nv->copy_data(nv, buf->bo, buf->offset, new_domain,
 466                     bo, offset, old_domain, buf->base.width0);
 467
 468       nouveau_bo_ref(NULL, &bo);
 469       if (mm)
 470          release_allocation(&mm, screen->fence.current);
 471    } else
 472    if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) {
 473       if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
 474          return FALSE;
 475       if (!nouveau_buffer_upload(nv, buf, 0, buf->base.width0))
 476          return FALSE;
 477    } else
 478       return FALSE;
 479
 480    assert(buf->domain == new_domain);
 481    return TRUE;
 482 }
 483
 484 /* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
 485  * We'd like to only allocate @size bytes here, but then we'd have to rebase
 486  * the vertex indices ...
 487  */
 488 boolean
 489 nouveau_user_buffer_upload(struct nouveau_context *nv,
 490                            struct nv04_resource *buf,
 491                            unsigned base, unsigned size)
 492 {
 493    struct nouveau_screen *screen = nouveau_screen(buf->base.screen);
 494    int ret;
 495
 496    assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY);
 497
 498    buf->base.width0 = base + size;
 499    if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART))
 500       return FALSE;
 501
 502    ret = nouveau_bo_map(buf->bo, 0, nv->screen->client);
 503    if (ret)
 504       return FALSE;
 505    memcpy((uint8_t *)buf->bo->map + buf->offset + base, buf->data + base, size);
 506
 507    return TRUE;
 508 }
 509
 510
 511 /* Scratch data allocation. */
 512
 513 static INLINE int
 514 nouveau_scratch_bo_alloc(struct nouveau_context *nv, struct nouveau_bo **pbo,
 515                          unsigned size)
 516 {
 517    return nouveau_bo_new(nv->screen->device, NOUVEAU_BO_GART | NOUVEAU_BO_MAP,
 518                          4096, size, NULL, pbo);
 519 }
 520
 521 void
 522 nouveau_scratch_runout_release(struct nouveau_context *nv)
 523 {
 524    if (!nv->scratch.nr_runout)
 525       return;
 526    do {
 527       --nv->scratch.nr_runout;
 528       nouveau_bo_ref(NULL, &nv->scratch.runout[nv->scratch.nr_runout]);
 529    } while (nv->scratch.nr_runout);
 530
 531    FREE(nv->scratch.runout);
 532    nv->scratch.end = 0;
 533    nv->scratch.runout = NULL;
 534 }
 535
 536 /* Allocate an extra bo if we can't fit everything we need simultaneously.
 537  * (Could happen for very large user arrays.)
 538  */
 539 static INLINE boolean
 540 nouveau_scratch_runout(struct nouveau_context *nv, unsigned size)
 541 {
 542    int ret;
 543    const unsigned n = nv->scratch.nr_runout++;
 544
 545    nv->scratch.runout = REALLOC(nv->scratch.runout,
 546                                 (n + 0) * sizeof(*nv->scratch.runout),
 547                                 (n + 1) * sizeof(*nv->scratch.runout));
 548    nv->scratch.runout[n] = NULL;
 549
 550    ret = nouveau_scratch_bo_alloc(nv, &nv->scratch.runout[n], size);
 551    if (!ret) {
 552       ret = nouveau_bo_map(nv->scratch.runout[n], 0, NULL);
 553       if (ret)
 554          nouveau_bo_ref(NULL, &nv->scratch.runout[--nv->scratch.nr_runout]);
 555    }
 556    if (!ret) {
 557       nv->scratch.current = nv->scratch.runout[n];
 558       nv->scratch.offset = 0;
 559       nv->scratch.end = size;
 560       nv->scratch.map = nv->scratch.current->map;
 561    }
 562    return !ret;
 563 }
 564
 565 /* Continue to next scratch buffer, if available (no wrapping, large enough).
 566  * Allocate it if it has not yet been created.
 567  */
 568 static INLINE boolean
 569 nouveau_scratch_next(struct nouveau_context *nv, unsigned size)
 570 {
 571    struct nouveau_bo *bo;
 572    int ret;
 573    const unsigned i = (nv->scratch.id + 1) % NOUVEAU_MAX_SCRATCH_BUFS;
 574
 575    if ((size > nv->scratch.bo_size) || (i == nv->scratch.wrap))
 576       return FALSE;
 577    nv->scratch.id = i;
 578
 579    bo = nv->scratch.bo[i];
 580    if (!bo) {
 581       ret = nouveau_scratch_bo_alloc(nv, &bo, nv->scratch.bo_size);
 582       if (ret)
 583          return FALSE;
 584       nv->scratch.bo[i] = bo;
 585    }
 586    nv->scratch.current = bo;
 587    nv->scratch.offset = 0;
 588    nv->scratch.end = nv->scratch.bo_size;
 589
 590    ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, nv->screen->client);
 591    if (!ret)
 592       nv->scratch.map = bo->map;
 593    return !ret;
 594 }
 595
 596 static boolean
 597 nouveau_scratch_more(struct nouveau_context *nv, unsigned min_size)
 598 {
 599    boolean ret;
 600
 601    ret = nouveau_scratch_next(nv, min_size);
 602    if (!ret)
 603       ret = nouveau_scratch_runout(nv, min_size);
 604    return ret;
 605 }
 606
 607 /* Upload data to scratch memory and update buffer address.
 608  * Returns the bo the data resides in, if successful.
 609  */
 610 struct nouveau_bo *
 611 nouveau_scratch_data(struct nouveau_context *nv,
 612                      struct nv04_resource *buf, unsigned base, unsigned size)
 613 {
 614    struct nouveau_bo *bo;
 615    unsigned bgn = MAX2(base, nv->scratch.offset);
 616    unsigned end = bgn + size;
 617
 618    if (end >= nv->scratch.end) {
 619       end = base + size;
 620       if (!nouveau_scratch_more(nv, end))
 621          return NULL;
 622       bgn = base;
 623    }
 624    nv->scratch.offset = align(end, 4);
 625
 626    memcpy(nv->scratch.map + bgn, buf->data + base, size);
 627
 628    bo = nv->scratch.current;
 629    buf->address = bo->offset + (bgn - base);
 630    return bo;
 631 }
 632
 633 void *
 634 nouveau_scratch_get(struct nouveau_context *nv,
 635                     unsigned size, uint64_t *gpu_addr, struct nouveau_bo **pbo)
 636 {
 637    unsigned bgn = nv->scratch.offset;
 638    unsigned end = nv->scratch.offset + size;
 639
 640    if (end >= nv->scratch.end) {
 641       end = size;
 642       if (!nouveau_scratch_more(nv, end))
 643          return NULL;
 644       bgn = 0;
 645    }
 646    nv->scratch.offset = align(end, 4);
 647
 648    *pbo = nv->scratch.current;
 649    *gpu_addr = nv->scratch.current->offset + bgn;
 650    return nv->scratch.map + bgn;
 651 }