src/gallium/drivers/nouveau/nouveau_buffer.c

   1
   2 #include "util/u_inlines.h"
   3 #include "util/u_memory.h"
   4 #include "util/u_math.h"
   5 #include "util/u_surface.h"
   6
   7 #include "nouveau_screen.h"
   8 #include "nouveau_context.h"
   9 #include "nouveau_winsys.h"
  10 #include "nouveau_fence.h"
  11 #include "nouveau_buffer.h"
  12 #include "nouveau_mm.h"
  13
  14 #define NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD 192
  15
  16 struct nouveau_transfer {
  17    struct pipe_transfer base;
  18
  19    uint8_t *map;
  20    struct nouveau_bo *bo;
  21    struct nouveau_mm_allocation *mm;
  22    uint32_t offset;
  23 };
  24
  25 static INLINE struct nouveau_transfer *
  26 nouveau_transfer(struct pipe_transfer *transfer)
  27 {
  28    return (struct nouveau_transfer *)transfer;
  29 }
  30
  31 static INLINE boolean
  32 nouveau_buffer_malloc(struct nv04_resource *buf)
  33 {
  34    if (!buf->data)
  35       buf->data = align_malloc(buf->base.width0, NOUVEAU_MIN_BUFFER_MAP_ALIGN);
  36    return !!buf->data;
  37 }
  38
  39 static INLINE boolean
  40 nouveau_buffer_allocate(struct nouveau_screen *screen,
  41                         struct nv04_resource *buf, unsigned domain)
  42 {
  43    uint32_t size = buf->base.width0;
  44
  45    if (buf->base.bind & (PIPE_BIND_CONSTANT_BUFFER |
  46                          PIPE_BIND_COMPUTE_RESOURCE |
  47                          PIPE_BIND_SHADER_RESOURCE))
  48       size = align(size, 0x100);
  49
  50    if (domain == NOUVEAU_BO_VRAM) {
  51       buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size,
  52                                     &buf->bo, &buf->offset);
  53       if (!buf->bo)
  54          return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
  55       NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_vid, buf->base.width0);
  56    } else
  57    if (domain == NOUVEAU_BO_GART) {
  58       buf->mm = nouveau_mm_allocate(screen->mm_GART, size,
  59                                     &buf->bo, &buf->offset);
  60       if (!buf->bo)
  61          return FALSE;
  62       NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_sys, buf->base.width0);
  63    } else {
  64       assert(domain == 0);
  65       if (!nouveau_buffer_malloc(buf))
  66          return FALSE;
  67    }
  68    buf->domain = domain;
  69    if (buf->bo)
  70       buf->address = buf->bo->offset + buf->offset;
  71
  72    return TRUE;
  73 }
  74
  75 static INLINE void
  76 release_allocation(struct nouveau_mm_allocation **mm,
  77                    struct nouveau_fence *fence)
  78 {
  79    nouveau_fence_work(fence, nouveau_mm_free_work, *mm);
  80    (*mm) = NULL;
  81 }
  82
  83 INLINE void
  84 nouveau_buffer_release_gpu_storage(struct nv04_resource *buf)
  85 {
  86    nouveau_bo_ref(NULL, &buf->bo);
  87
  88    if (buf->mm)
  89       release_allocation(&buf->mm, buf->fence);
  90
  91    if (buf->domain == NOUVEAU_BO_VRAM)
  92       NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_vid, -(uint64_t)buf->base.width0);
  93    if (buf->domain == NOUVEAU_BO_GART)
  94       NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_sys, -(uint64_t)buf->base.width0);
  95
  96    buf->domain = 0;
  97 }
  98
  99 static INLINE boolean
 100 nouveau_buffer_reallocate(struct nouveau_screen *screen,
 101                           struct nv04_resource *buf, unsigned domain)
 102 {
 103    nouveau_buffer_release_gpu_storage(buf);
 104
 105    nouveau_fence_ref(NULL, &buf->fence);
 106    nouveau_fence_ref(NULL, &buf->fence_wr);
 107
 108    buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK;
 109
 110    return nouveau_buffer_allocate(screen, buf, domain);
 111 }
 112
 113 static void
 114 nouveau_buffer_destroy(struct pipe_screen *pscreen,
 115                        struct pipe_resource *presource)
 116 {
 117    struct nv04_resource *res = nv04_resource(presource);
 118
 119    nouveau_buffer_release_gpu_storage(res);
 120
 121    if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
 122       align_free(res->data);
 123
 124    nouveau_fence_ref(NULL, &res->fence);
 125    nouveau_fence_ref(NULL, &res->fence_wr);
 126
 127    FREE(res);
 128
 129    NOUVEAU_DRV_STAT(nouveau_screen(pscreen), buf_obj_current_count, -1);
 130 }
 131
 132 static uint8_t *
 133 nouveau_transfer_staging(struct nouveau_context *nv,
 134                          struct nouveau_transfer *tx, boolean permit_pb)
 135 {
 136    const unsigned adj = tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK;
 137    const unsigned size = align(tx->base.box.width, 4) + adj;
 138
 139    if (!nv->push_data)
 140       permit_pb = FALSE;
 141
 142    if ((size <= NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD) && permit_pb) {
 143       tx->map = align_malloc(size, NOUVEAU_MIN_BUFFER_MAP_ALIGN);
 144       if (tx->map)
 145          tx->map += adj;
 146    } else {
 147       tx->mm =
 148          nouveau_mm_allocate(nv->screen->mm_GART, size, &tx->bo, &tx->offset);
 149       if (tx->bo) {
 150          tx->offset += adj;
 151          if (!nouveau_bo_map(tx->bo, 0, NULL))
 152             tx->map = (uint8_t *)tx->bo->map + tx->offset;
 153       }
 154    }
 155    return tx->map;
 156 }
 157
 158 /* Maybe just migrate to GART right away if we actually need to do this. */
 159 static boolean
 160 nouveau_transfer_read(struct nouveau_context *nv, struct nouveau_transfer *tx)
 161 {
 162    struct nv04_resource *buf = nv04_resource(tx->base.resource);
 163    const unsigned base = tx->base.box.x;
 164    const unsigned size = tx->base.box.width;
 165
 166    NOUVEAU_DRV_STAT(nv->screen, buf_read_bytes_staging_vid, size);
 167
 168    nv->copy_data(nv, tx->bo, tx->offset, NOUVEAU_BO_GART,
 169                  buf->bo, buf->offset + base, buf->domain, size);
 170
 171    if (nouveau_bo_wait(tx->bo, NOUVEAU_BO_RD, nv->client))
 172       return FALSE;
 173
 174    if (buf->data)
 175       memcpy(buf->data + base, tx->map, size);
 176
 177    return TRUE;
 178 }
 179
 180 static void
 181 nouveau_transfer_write(struct nouveau_context *nv, struct nouveau_transfer *tx,
 182                        unsigned offset, unsigned size)
 183 {
 184    struct nv04_resource *buf = nv04_resource(tx->base.resource);
 185    uint8_t *data = tx->map + offset;
 186    const unsigned base = tx->base.box.x + offset;
 187    const boolean can_cb = !((base | size) & 3);
 188
 189    if (buf->data)
 190       memcpy(data, buf->data + base, size);
 191    else
 192       buf->status |= NOUVEAU_BUFFER_STATUS_DIRTY;
 193
 194    if (buf->domain == NOUVEAU_BO_VRAM)
 195       NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_staging_vid, size);
 196    if (buf->domain == NOUVEAU_BO_GART)
 197       NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_staging_sys, size);
 198
 199    if (tx->bo)
 200       nv->copy_data(nv, buf->bo, buf->offset + base, buf->domain,
 201                     tx->bo, tx->offset + offset, NOUVEAU_BO_GART, size);
 202    else
 203    if ((buf->base.bind & PIPE_BIND_CONSTANT_BUFFER) && nv->push_cb && can_cb)
 204       nv->push_cb(nv, buf->bo, buf->domain, buf->offset, buf->base.width0,
 205                   base, size / 4, (const uint32_t *)data);
 206    else
 207       nv->push_data(nv, buf->bo, buf->offset + base, buf->domain, size, data);
 208 }
 209
 210
 211 static INLINE boolean
 212 nouveau_buffer_sync(struct nv04_resource *buf, unsigned rw)
 213 {
 214    if (rw == PIPE_TRANSFER_READ) {
 215       if (!buf->fence_wr)
 216          return TRUE;
 217       NOUVEAU_DRV_STAT_RES(buf, buf_non_kernel_fence_sync_count,
 218                            !nouveau_fence_signalled(buf->fence_wr));
 219       if (!nouveau_fence_wait(buf->fence_wr))
 220          return FALSE;
 221    } else {
 222       if (!buf->fence)
 223          return TRUE;
 224       NOUVEAU_DRV_STAT_RES(buf, buf_non_kernel_fence_sync_count,
 225                            !nouveau_fence_signalled(buf->fence));
 226       if (!nouveau_fence_wait(buf->fence))
 227          return FALSE;
 228
 229       nouveau_fence_ref(NULL, &buf->fence);
 230    }
 231    nouveau_fence_ref(NULL, &buf->fence_wr);
 232
 233    return TRUE;
 234 }
 235
 236 static INLINE boolean
 237 nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw)
 238 {
 239    if (rw == PIPE_TRANSFER_READ)
 240       return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr));
 241    else
 242       return (buf->fence && !nouveau_fence_signalled(buf->fence));
 243 }
 244
 245 static INLINE void
 246 nouveau_buffer_transfer_init(struct nouveau_transfer *tx,
 247                              struct pipe_resource *resource,
 248                              const struct pipe_box *box,
 249                              unsigned usage)
 250 {
 251    tx->base.resource = resource;
 252    tx->base.level = 0;
 253    tx->base.usage = usage;
 254    tx->base.box.x = box->x;
 255    tx->base.box.y = 0;
 256    tx->base.box.z = 0;
 257    tx->base.box.width = box->width;
 258    tx->base.box.height = 1;
 259    tx->base.box.depth = 1;
 260    tx->base.stride = 0;
 261    tx->base.layer_stride = 0;
 262
 263    tx->bo = NULL;
 264    tx->map = NULL;
 265 }
 266
 267 static INLINE void
 268 nouveau_buffer_transfer_del(struct nouveau_context *nv,
 269                             struct nouveau_transfer *tx)
 270 {
 271    if (tx->map) {
 272       if (likely(tx->bo)) {
 273          nouveau_bo_ref(NULL, &tx->bo);
 274          if (tx->mm)
 275             release_allocation(&tx->mm, nv->screen->fence.current);
 276       } else {
 277          align_free(tx->map -
 278                     (tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK));
 279       }
 280    }
 281 }
 282
 283 static boolean
 284 nouveau_buffer_cache(struct nouveau_context *nv, struct nv04_resource *buf)
 285 {
 286    struct nouveau_transfer tx;
 287    boolean ret;
 288    tx.base.resource = &buf->base;
 289    tx.base.box.x = 0;
 290    tx.base.box.width = buf->base.width0;
 291    tx.bo = NULL;
 292
 293    if (!buf->data)
 294       if (!nouveau_buffer_malloc(buf))
 295          return FALSE;
 296    if (!(buf->status & NOUVEAU_BUFFER_STATUS_DIRTY))
 297       return TRUE;
 298    nv->stats.buf_cache_count++;
 299
 300    if (!nouveau_transfer_staging(nv, &tx, FALSE))
 301       return FALSE;
 302
 303    ret = nouveau_transfer_read(nv, &tx);
 304    if (ret) {
 305       buf->status &= ~NOUVEAU_BUFFER_STATUS_DIRTY;
 306       memcpy(buf->data, tx.map, buf->base.width0);
 307    }
 308    nouveau_buffer_transfer_del(nv, &tx);
 309    return ret;
 310 }
 311
 312
 313 #define NOUVEAU_TRANSFER_DISCARD \
 314    (PIPE_TRANSFER_DISCARD_RANGE | PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)
 315
 316 static INLINE boolean
 317 nouveau_buffer_should_discard(struct nv04_resource *buf, unsigned usage)
 318 {
 319    if (!(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE))
 320       return FALSE;
 321    if (unlikely(buf->base.bind & PIPE_BIND_SHARED))
 322       return FALSE;
 323    return buf->mm && nouveau_buffer_busy(buf, PIPE_TRANSFER_WRITE);
 324 }
 325
 326 static void *
 327 nouveau_buffer_transfer_map(struct pipe_context *pipe,
 328                             struct pipe_resource *resource,
 329                             unsigned level, unsigned usage,
 330                             const struct pipe_box *box,
 331                             struct pipe_transfer **ptransfer)
 332 {
 333    struct nouveau_context *nv = nouveau_context(pipe);
 334    struct nv04_resource *buf = nv04_resource(resource);
 335    struct nouveau_transfer *tx = MALLOC_STRUCT(nouveau_transfer);
 336    uint8_t *map;
 337    int ret;
 338
 339    if (!tx)
 340       return NULL;
 341    nouveau_buffer_transfer_init(tx, resource, box, usage);
 342    *ptransfer = &tx->base;
 343
 344    if (usage & PIPE_TRANSFER_READ)
 345       NOUVEAU_DRV_STAT(nv->screen, buf_transfers_rd, 1);
 346    if (usage & PIPE_TRANSFER_WRITE)
 347       NOUVEAU_DRV_STAT(nv->screen, buf_transfers_wr, 1);
 348
 349    if (buf->domain == NOUVEAU_BO_VRAM) {
 350       if (usage & NOUVEAU_TRANSFER_DISCARD) {
 351          if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)
 352             buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK;
 353          nouveau_transfer_staging(nv, tx, TRUE);
 354       } else {
 355          if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING) {
 356             if (buf->data) {
 357                align_free(buf->data);
 358                buf->data = NULL;
 359             }
 360             nouveau_transfer_staging(nv, tx, FALSE);
 361             nouveau_transfer_read(nv, tx);
 362          } else {
 363             if (usage & PIPE_TRANSFER_WRITE)
 364                nouveau_transfer_staging(nv, tx, TRUE);
 365             if (!buf->data)
 366                nouveau_buffer_cache(nv, buf);
 367          }
 368       }
 369       return buf->data ? (buf->data + box->x) : tx->map;
 370    } else
 371    if (unlikely(buf->domain == 0)) {
 372       return buf->data + box->x;
 373    }
 374
 375    if (nouveau_buffer_should_discard(buf, usage)) {
 376       int ref = buf->base.reference.count - 1;
 377       nouveau_buffer_reallocate(nv->screen, buf, buf->domain);
 378       if (ref > 0) /* any references inside context possible ? */
 379          nv->invalidate_resource_storage(nv, &buf->base, ref);
 380    }
 381
 382    ret = nouveau_bo_map(buf->bo,
 383                         buf->mm ? 0 : nouveau_screen_transfer_flags(usage),
 384                         nv->client);
 385    if (ret) {
 386       FREE(tx);
 387       return NULL;
 388    }
 389    map = (uint8_t *)buf->bo->map + buf->offset + box->x;
 390
 391    /* using kernel fences only if !buf->mm */
 392    if ((usage & PIPE_TRANSFER_UNSYNCHRONIZED) || !buf->mm)
 393       return map;
 394
 395    if (nouveau_buffer_busy(buf, usage & PIPE_TRANSFER_READ_WRITE)) {
 396       if (unlikely(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)) {
 397          /* Discarding was not possible, must sync because
 398           * subsequent transfers might use UNSYNCHRONIZED. */
 399          nouveau_buffer_sync(buf, usage & PIPE_TRANSFER_READ_WRITE);
 400       } else
 401       if (usage & PIPE_TRANSFER_DISCARD_RANGE) {
 402          nouveau_transfer_staging(nv, tx, TRUE);
 403          map = tx->map;
 404       } else
 405       if (nouveau_buffer_busy(buf, PIPE_TRANSFER_READ)) {
 406          if (usage & PIPE_TRANSFER_DONTBLOCK)
 407             map = NULL;
 408          else
 409             nouveau_buffer_sync(buf, usage & PIPE_TRANSFER_READ_WRITE);
 410       } else {
 411          nouveau_transfer_staging(nv, tx, TRUE);
 412          if (tx->map)
 413             memcpy(tx->map, map, box->width);
 414          map = tx->map;
 415       }
 416    }
 417    if (!map)
 418       FREE(tx);
 419    return map;
 420 }
 421
 422
 423
 424 static void
 425 nouveau_buffer_transfer_flush_region(struct pipe_context *pipe,
 426                                      struct pipe_transfer *transfer,
 427                                      const struct pipe_box *box)
 428 {
 429    struct nouveau_transfer *tx = nouveau_transfer(transfer);
 430    if (tx->map)
 431       nouveau_transfer_write(nouveau_context(pipe), tx, box->x, box->width);
 432 }
 433
 434 static void
 435 nouveau_buffer_transfer_unmap(struct pipe_context *pipe,
 436                               struct pipe_transfer *transfer)
 437 {
 438    struct nouveau_context *nv = nouveau_context(pipe);
 439    struct nouveau_transfer *tx = nouveau_transfer(transfer);
 440    struct nv04_resource *buf = nv04_resource(transfer->resource);
 441
 442    if (tx->base.usage & PIPE_TRANSFER_WRITE) {
 443       if (!(tx->base.usage & PIPE_TRANSFER_FLUSH_EXPLICIT) && tx->map)
 444          nouveau_transfer_write(nv, tx, 0, tx->base.box.width);
 445
 446       if (likely(buf->domain)) {
 447          const uint8_t bind = buf->base.bind;
 448          /* make sure we invalidate dedicated caches */
 449          if (bind & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER))
 450             nv->vbo_dirty = TRUE;
 451          if (bind & (PIPE_BIND_CONSTANT_BUFFER))
 452             nv->cb_dirty = TRUE;
 453       }
 454    }
 455
 456    if (!tx->bo && (tx->base.usage & PIPE_TRANSFER_WRITE))
 457       NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_direct, tx->base.box.width);
 458
 459    nouveau_buffer_transfer_del(nv, tx);
 460    FREE(tx);
 461 }
 462
 463
 464 void
 465 nouveau_copy_buffer(struct nouveau_context *nv,
 466                     struct nv04_resource *dst, unsigned dstx,
 467                     struct nv04_resource *src, unsigned srcx, unsigned size)
 468 {
 469    assert(dst->base.target == PIPE_BUFFER && src->base.target == PIPE_BUFFER);
 470
 471    if (likely(dst->domain) && likely(src->domain)) {
 472       nv->copy_data(nv,
 473                     dst->bo, dst->offset + dstx, dst->domain,
 474                     src->bo, src->offset + srcx, src->domain, size);
 475
 476       dst->status |= NOUVEAU_BUFFER_STATUS_GPU_WRITING;
 477       nouveau_fence_ref(nv->screen->fence.current, &dst->fence);
 478       nouveau_fence_ref(nv->screen->fence.current, &dst->fence_wr);
 479
 480       src->status |= NOUVEAU_BUFFER_STATUS_GPU_READING;
 481       nouveau_fence_ref(nv->screen->fence.current, &src->fence);
 482    } else {
 483       struct pipe_box src_box;
 484       src_box.x = srcx;
 485       src_box.y = 0;
 486       src_box.z = 0;
 487       src_box.width = size;
 488       src_box.height = 1;
 489       src_box.depth = 1;
 490       util_resource_copy_region(&nv->pipe,
 491                                 &dst->base, 0, dstx, 0, 0,
 492                                 &src->base, 0, &src_box);
 493    }
 494 }
 495
 496
 497 void *
 498 nouveau_resource_map_offset(struct nouveau_context *nv,
 499                             struct nv04_resource *res, uint32_t offset,
 500                             uint32_t flags)
 501 {
 502    if (unlikely(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
 503       return res->data + offset;
 504
 505    if (res->domain == NOUVEAU_BO_VRAM) {
 506       if (!res->data || (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING))
 507          nouveau_buffer_cache(nv, res);
 508    }
 509    if (res->domain != NOUVEAU_BO_GART)
 510       return res->data + offset;
 511
 512    if (res->mm) {
 513       unsigned rw;
 514       rw = (flags & NOUVEAU_BO_WR) ? PIPE_TRANSFER_WRITE : PIPE_TRANSFER_READ;
 515       nouveau_buffer_sync(res, rw);
 516       if (nouveau_bo_map(res->bo, 0, NULL))
 517          return NULL;
 518    } else {
 519       if (nouveau_bo_map(res->bo, flags, nv->client))
 520          return NULL;
 521    }
 522    return (uint8_t *)res->bo->map + res->offset + offset;
 523 }
 524
 525
 526 const struct u_resource_vtbl nouveau_buffer_vtbl =
 527 {
 528    u_default_resource_get_handle,     /* get_handle */
 529    nouveau_buffer_destroy,               /* resource_destroy */
 530    nouveau_buffer_transfer_map,          /* transfer_map */
 531    nouveau_buffer_transfer_flush_region, /* transfer_flush_region */
 532    nouveau_buffer_transfer_unmap,        /* transfer_unmap */
 533    u_default_transfer_inline_write    /* transfer_inline_write */
 534 };
 535
 536 struct pipe_resource *
 537 nouveau_buffer_create(struct pipe_screen *pscreen,
 538                       const struct pipe_resource *templ)
 539 {
 540    struct nouveau_screen *screen = nouveau_screen(pscreen);
 541    struct nv04_resource *buffer;
 542    boolean ret;
 543
 544    buffer = CALLOC_STRUCT(nv04_resource);
 545    if (!buffer)
 546       return NULL;
 547
 548    buffer->base = *templ;
 549    buffer->vtbl = &nouveau_buffer_vtbl;
 550    pipe_reference_init(&buffer->base.reference, 1);
 551    buffer->base.screen = pscreen;
 552
 553    if (buffer->base.bind &
 554        (screen->vidmem_bindings & screen->sysmem_bindings)) {
 555       switch (buffer->base.usage) {
 556       case PIPE_USAGE_DEFAULT:
 557       case PIPE_USAGE_IMMUTABLE:
 558       case PIPE_USAGE_STATIC:
 559          buffer->domain = NOUVEAU_BO_VRAM;
 560          break;
 561       case PIPE_USAGE_DYNAMIC:
 562          /* For most apps, we'd have to do staging transfers to avoid sync
 563           * with this usage, and GART -> GART copies would be suboptimal.
 564           */
 565          buffer->domain = NOUVEAU_BO_VRAM;
 566          break;
 567       case PIPE_USAGE_STAGING:
 568       case PIPE_USAGE_STREAM:
 569          buffer->domain = NOUVEAU_BO_GART;
 570          break;
 571       default:
 572          assert(0);
 573          break;
 574       }
 575    } else {
 576       if (buffer->base.bind & screen->vidmem_bindings)
 577          buffer->domain = NOUVEAU_BO_VRAM;
 578       else
 579       if (buffer->base.bind & screen->sysmem_bindings)
 580          buffer->domain = NOUVEAU_BO_GART;
 581    }
 582    ret = nouveau_buffer_allocate(screen, buffer, buffer->domain);
 583
 584    if (ret == FALSE)
 585       goto fail;
 586
 587    if (buffer->domain == NOUVEAU_BO_VRAM && screen->hint_buf_keep_sysmem_copy)
 588       nouveau_buffer_cache(NULL, buffer);
 589
 590    NOUVEAU_DRV_STAT(screen, buf_obj_current_count, 1);
 591
 592    return &buffer->base;
 593
 594 fail:
 595    FREE(buffer);
 596    return NULL;
 597 }
 598
 599
 600 struct pipe_resource *
 601 nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr,
 602                            unsigned bytes, unsigned bind)
 603 {
 604    struct nv04_resource *buffer;
 605
 606    buffer = CALLOC_STRUCT(nv04_resource);
 607    if (!buffer)
 608       return NULL;
 609
 610    pipe_reference_init(&buffer->base.reference, 1);
 611    buffer->vtbl = &nouveau_buffer_vtbl;
 612    buffer->base.screen = pscreen;
 613    buffer->base.format = PIPE_FORMAT_R8_UNORM;
 614    buffer->base.usage = PIPE_USAGE_IMMUTABLE;
 615    buffer->base.bind = bind;
 616    buffer->base.width0 = bytes;
 617    buffer->base.height0 = 1;
 618    buffer->base.depth0 = 1;
 619
 620    buffer->data = ptr;
 621    buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY;
 622
 623    return &buffer->base;
 624 }
 625
 626 static INLINE boolean
 627 nouveau_buffer_data_fetch(struct nouveau_context *nv, struct nv04_resource *buf,
 628                           struct nouveau_bo *bo, unsigned offset, unsigned size)
 629 {
 630    if (!nouveau_buffer_malloc(buf))
 631       return FALSE;
 632    if (nouveau_bo_map(bo, NOUVEAU_BO_RD, nv->client))
 633       return FALSE;
 634    memcpy(buf->data, (uint8_t *)bo->map + offset, size);
 635    return TRUE;
 636 }
 637
 638 /* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
 639 boolean
 640 nouveau_buffer_migrate(struct nouveau_context *nv,
 641                        struct nv04_resource *buf, const unsigned new_domain)
 642 {
 643    struct nouveau_screen *screen = nv->screen;
 644    struct nouveau_bo *bo;
 645    const unsigned old_domain = buf->domain;
 646    unsigned size = buf->base.width0;
 647    unsigned offset;
 648    int ret;
 649
 650    assert(new_domain != old_domain);
 651
 652    if (new_domain == NOUVEAU_BO_GART && old_domain == 0) {
 653       if (!nouveau_buffer_allocate(screen, buf, new_domain))
 654          return FALSE;
 655       ret = nouveau_bo_map(buf->bo, 0, nv->client);
 656       if (ret)
 657          return ret;
 658       memcpy((uint8_t *)buf->bo->map + buf->offset, buf->data, size);
 659       align_free(buf->data);
 660    } else
 661    if (old_domain != 0 && new_domain != 0) {
 662       struct nouveau_mm_allocation *mm = buf->mm;
 663
 664       if (new_domain == NOUVEAU_BO_VRAM) {
 665          /* keep a system memory copy of our data in case we hit a fallback */
 666          if (!nouveau_buffer_data_fetch(nv, buf, buf->bo, buf->offset, size))
 667             return FALSE;
 668          if (nouveau_mesa_debug)
 669             debug_printf("migrating %u KiB to VRAM\n", size / 1024);
 670       }
 671
 672       offset = buf->offset;
 673       bo = buf->bo;
 674       buf->bo = NULL;
 675       buf->mm = NULL;
 676       nouveau_buffer_allocate(screen, buf, new_domain);
 677
 678       nv->copy_data(nv, buf->bo, buf->offset, new_domain,
 679                     bo, offset, old_domain, buf->base.width0);
 680
 681       nouveau_bo_ref(NULL, &bo);
 682       if (mm)
 683          release_allocation(&mm, screen->fence.current);
 684    } else
 685    if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) {
 686       struct nouveau_transfer tx;
 687       if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
 688          return FALSE;
 689       tx.base.resource = &buf->base;
 690       tx.base.box.x = 0;
 691       tx.base.box.width = buf->base.width0;
 692       tx.bo = NULL;
 693       if (!nouveau_transfer_staging(nv, &tx, FALSE))
 694          return FALSE;
 695       nouveau_transfer_write(nv, &tx, 0, tx.base.box.width);
 696       nouveau_buffer_transfer_del(nv, &tx);
 697    } else
 698       return FALSE;
 699
 700    assert(buf->domain == new_domain);
 701    return TRUE;
 702 }
 703
 704 /* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
 705  * We'd like to only allocate @size bytes here, but then we'd have to rebase
 706  * the vertex indices ...
 707  */
 708 boolean
 709 nouveau_user_buffer_upload(struct nouveau_context *nv,
 710                            struct nv04_resource *buf,
 711                            unsigned base, unsigned size)
 712 {
 713    struct nouveau_screen *screen = nouveau_screen(buf->base.screen);
 714    int ret;
 715
 716    assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY);
 717
 718    buf->base.width0 = base + size;
 719    if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART))
 720       return FALSE;
 721
 722    ret = nouveau_bo_map(buf->bo, 0, nv->client);
 723    if (ret)
 724       return FALSE;
 725    memcpy((uint8_t *)buf->bo->map + buf->offset + base, buf->data + base, size);
 726
 727    return TRUE;
 728 }
 729
 730
 731 /* Scratch data allocation. */
 732
 733 static INLINE int
 734 nouveau_scratch_bo_alloc(struct nouveau_context *nv, struct nouveau_bo **pbo,
 735                          unsigned size)
 736 {
 737    return nouveau_bo_new(nv->screen->device, NOUVEAU_BO_GART | NOUVEAU_BO_MAP,
 738                          4096, size, NULL, pbo);
 739 }
 740
 741 void
 742 nouveau_scratch_runout_release(struct nouveau_context *nv)
 743 {
 744    if (!nv->scratch.nr_runout)
 745       return;
 746    do {
 747       --nv->scratch.nr_runout;
 748       nouveau_bo_ref(NULL, &nv->scratch.runout[nv->scratch.nr_runout]);
 749    } while (nv->scratch.nr_runout);
 750
 751    FREE(nv->scratch.runout);
 752    nv->scratch.end = 0;
 753    nv->scratch.runout = NULL;
 754 }
 755
 756 /* Allocate an extra bo if we can't fit everything we need simultaneously.
 757  * (Could happen for very large user arrays.)
 758  */
 759 static INLINE boolean
 760 nouveau_scratch_runout(struct nouveau_context *nv, unsigned size)
 761 {
 762    int ret;
 763    const unsigned n = nv->scratch.nr_runout++;
 764
 765    nv->scratch.runout = REALLOC(nv->scratch.runout,
 766                                 (n + 0) * sizeof(*nv->scratch.runout),
 767                                 (n + 1) * sizeof(*nv->scratch.runout));
 768    nv->scratch.runout[n] = NULL;
 769
 770    ret = nouveau_scratch_bo_alloc(nv, &nv->scratch.runout[n], size);
 771    if (!ret) {
 772       ret = nouveau_bo_map(nv->scratch.runout[n], 0, NULL);
 773       if (ret)
 774          nouveau_bo_ref(NULL, &nv->scratch.runout[--nv->scratch.nr_runout]);
 775    }
 776    if (!ret) {
 777       nv->scratch.current = nv->scratch.runout[n];
 778       nv->scratch.offset = 0;
 779       nv->scratch.end = size;
 780       nv->scratch.map = nv->scratch.current->map;
 781    }
 782    return !ret;
 783 }
 784
 785 /* Continue to next scratch buffer, if available (no wrapping, large enough).
 786  * Allocate it if it has not yet been created.
 787  */
 788 static INLINE boolean
 789 nouveau_scratch_next(struct nouveau_context *nv, unsigned size)
 790 {
 791    struct nouveau_bo *bo;
 792    int ret;
 793    const unsigned i = (nv->scratch.id + 1) % NOUVEAU_MAX_SCRATCH_BUFS;
 794
 795    if ((size > nv->scratch.bo_size) || (i == nv->scratch.wrap))
 796       return FALSE;
 797    nv->scratch.id = i;
 798
 799    bo = nv->scratch.bo[i];
 800    if (!bo) {
 801       ret = nouveau_scratch_bo_alloc(nv, &bo, nv->scratch.bo_size);
 802       if (ret)
 803          return FALSE;
 804       nv->scratch.bo[i] = bo;
 805    }
 806    nv->scratch.current = bo;
 807    nv->scratch.offset = 0;
 808    nv->scratch.end = nv->scratch.bo_size;
 809
 810    ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, nv->client);
 811    if (!ret)
 812       nv->scratch.map = bo->map;
 813    return !ret;
 814 }
 815
 816 static boolean
 817 nouveau_scratch_more(struct nouveau_context *nv, unsigned min_size)
 818 {
 819    boolean ret;
 820
 821    ret = nouveau_scratch_next(nv, min_size);
 822    if (!ret)
 823       ret = nouveau_scratch_runout(nv, min_size);
 824    return ret;
 825 }
 826
 827
 828 /* Copy data to a scratch buffer and return address & bo the data resides in. */
 829 uint64_t
 830 nouveau_scratch_data(struct nouveau_context *nv,
 831                      const void *data, unsigned base, unsigned size,
 832                      struct nouveau_bo **bo)
 833 {
 834    unsigned bgn = MAX2(base, nv->scratch.offset);
 835    unsigned end = bgn + size;
 836
 837    if (end >= nv->scratch.end) {
 838       end = base + size;
 839       if (!nouveau_scratch_more(nv, end))
 840          return 0;
 841       bgn = base;
 842    }
 843    nv->scratch.offset = align(end, 4);
 844
 845    memcpy(nv->scratch.map + bgn, (const uint8_t *)data + base, size);
 846
 847    *bo = nv->scratch.current;
 848    return (*bo)->offset + (bgn - base);
 849 }
 850
 851 void *
 852 nouveau_scratch_get(struct nouveau_context *nv,
 853                     unsigned size, uint64_t *gpu_addr, struct nouveau_bo **pbo)
 854 {
 855    unsigned bgn = nv->scratch.offset;
 856    unsigned end = nv->scratch.offset + size;
 857
 858    if (end >= nv->scratch.end) {
 859       end = size;
 860       if (!nouveau_scratch_more(nv, end))
 861          return NULL;
 862       bgn = 0;
 863    }
 864    nv->scratch.offset = align(end, 4);
 865
 866    *pbo = nv->scratch.current;
 867    *gpu_addr = nv->scratch.current->offset + bgn;
 868    return nv->scratch.map + bgn;
 869 }