src/gallium/drivers/nouveau/nouveau_buffer.c

   1
   2 #include "util/u_inlines.h"
   3 #include "util/u_memory.h"
   4 #include "util/u_math.h"
   5 #include "util/u_surface.h"
   6
   7 #include "nouveau_screen.h"
   8 #include "nouveau_context.h"
   9 #include "nouveau_winsys.h"
  10 #include "nouveau_fence.h"
  11 #include "nouveau_buffer.h"
  12 #include "nouveau_mm.h"
  13
  14 #define NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD 192
  15
  16 struct nouveau_transfer {
  17    struct pipe_transfer base;
  18
  19    uint8_t *map;
  20    struct nouveau_bo *bo;
  21    struct nouveau_mm_allocation *mm;
  22    uint32_t offset;
  23 };
  24
  25 static INLINE struct nouveau_transfer *
  26 nouveau_transfer(struct pipe_transfer *transfer)
  27 {
  28    return (struct nouveau_transfer *)transfer;
  29 }
  30
  31 static INLINE boolean
  32 nouveau_buffer_malloc(struct nv04_resource *buf)
  33 {
  34    if (!buf->data)
  35       buf->data = align_malloc(buf->base.width0, NOUVEAU_MIN_BUFFER_MAP_ALIGN);
  36    return !!buf->data;
  37 }
  38
  39 static INLINE boolean
  40 nouveau_buffer_allocate(struct nouveau_screen *screen,
  41                         struct nv04_resource *buf, unsigned domain)
  42 {
  43    uint32_t size = buf->base.width0;
  44
  45    if (buf->base.bind & (PIPE_BIND_CONSTANT_BUFFER |
  46                          PIPE_BIND_COMPUTE_RESOURCE |
  47                          PIPE_BIND_SHADER_RESOURCE))
  48       size = align(size, 0x100);
  49
  50    if (domain == NOUVEAU_BO_VRAM) {
  51       buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size,
  52                                     &buf->bo, &buf->offset);
  53       if (!buf->bo)
  54          return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
  55       NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_vid, buf->base.width0);
  56    } else
  57    if (domain == NOUVEAU_BO_GART) {
  58       buf->mm = nouveau_mm_allocate(screen->mm_GART, size,
  59                                     &buf->bo, &buf->offset);
  60       if (!buf->bo)
  61          return FALSE;
  62       NOUVEAU_DRV_STAT(screen, buf_obj_current_bytes_sys, buf->base.width0);
  63    } else {
  64       assert(domain == 0);
  65       if (!nouveau_buffer_malloc(buf))
  66          return FALSE;
  67    }
  68    buf->domain = domain;
  69    if (buf->bo)
  70       buf->address = buf->bo->offset + buf->offset;
  71
  72    util_range_set_empty(&buf->valid_buffer_range);
  73
  74    return TRUE;
  75 }
  76
  77 static INLINE void
  78 release_allocation(struct nouveau_mm_allocation **mm,
  79                    struct nouveau_fence *fence)
  80 {
  81    nouveau_fence_work(fence, nouveau_mm_free_work, *mm);
  82    (*mm) = NULL;
  83 }
  84
  85 INLINE void
  86 nouveau_buffer_release_gpu_storage(struct nv04_resource *buf)
  87 {
  88    nouveau_bo_ref(NULL, &buf->bo);
  89
  90    if (buf->mm)
  91       release_allocation(&buf->mm, buf->fence);
  92
  93    if (buf->domain == NOUVEAU_BO_VRAM)
  94       NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_vid, -(uint64_t)buf->base.width0);
  95    if (buf->domain == NOUVEAU_BO_GART)
  96       NOUVEAU_DRV_STAT_RES(buf, buf_obj_current_bytes_sys, -(uint64_t)buf->base.width0);
  97
  98    buf->domain = 0;
  99 }
 100
 101 static INLINE boolean
 102 nouveau_buffer_reallocate(struct nouveau_screen *screen,
 103                           struct nv04_resource *buf, unsigned domain)
 104 {
 105    nouveau_buffer_release_gpu_storage(buf);
 106
 107    nouveau_fence_ref(NULL, &buf->fence);
 108    nouveau_fence_ref(NULL, &buf->fence_wr);
 109
 110    buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK;
 111
 112    return nouveau_buffer_allocate(screen, buf, domain);
 113 }
 114
 115 static void
 116 nouveau_buffer_destroy(struct pipe_screen *pscreen,
 117                        struct pipe_resource *presource)
 118 {
 119    struct nv04_resource *res = nv04_resource(presource);
 120
 121    nouveau_buffer_release_gpu_storage(res);
 122
 123    if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
 124       align_free(res->data);
 125
 126    nouveau_fence_ref(NULL, &res->fence);
 127    nouveau_fence_ref(NULL, &res->fence_wr);
 128
 129    util_range_destroy(&res->valid_buffer_range);
 130
 131    FREE(res);
 132
 133    NOUVEAU_DRV_STAT(nouveau_screen(pscreen), buf_obj_current_count, -1);
 134 }
 135
 136 /* Set up a staging area for the transfer. This is either done in "regular"
 137  * system memory if the driver supports push_data (nv50+) and the data is
 138  * small enough (and permit_pb == true), or in GART memory.
 139  */
 140 static uint8_t *
 141 nouveau_transfer_staging(struct nouveau_context *nv,
 142                          struct nouveau_transfer *tx, boolean permit_pb)
 143 {
 144    const unsigned adj = tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK;
 145    const unsigned size = align(tx->base.box.width, 4) + adj;
 146
 147    if (!nv->push_data)
 148       permit_pb = FALSE;
 149
 150    if ((size <= NOUVEAU_TRANSFER_PUSHBUF_THRESHOLD) && permit_pb) {
 151       tx->map = align_malloc(size, NOUVEAU_MIN_BUFFER_MAP_ALIGN);
 152       if (tx->map)
 153          tx->map += adj;
 154    } else {
 155       tx->mm =
 156          nouveau_mm_allocate(nv->screen->mm_GART, size, &tx->bo, &tx->offset);
 157       if (tx->bo) {
 158          tx->offset += adj;
 159          if (!nouveau_bo_map(tx->bo, 0, NULL))
 160             tx->map = (uint8_t *)tx->bo->map + tx->offset;
 161       }
 162    }
 163    return tx->map;
 164 }
 165
 166 /* Copies data from the resource into the the transfer's temporary GART
 167  * buffer. Also updates buf->data if present.
 168  *
 169  * Maybe just migrate to GART right away if we actually need to do this. */
 170 static boolean
 171 nouveau_transfer_read(struct nouveau_context *nv, struct nouveau_transfer *tx)
 172 {
 173    struct nv04_resource *buf = nv04_resource(tx->base.resource);
 174    const unsigned base = tx->base.box.x;
 175    const unsigned size = tx->base.box.width;
 176
 177    NOUVEAU_DRV_STAT(nv->screen, buf_read_bytes_staging_vid, size);
 178
 179    nv->copy_data(nv, tx->bo, tx->offset, NOUVEAU_BO_GART,
 180                  buf->bo, buf->offset + base, buf->domain, size);
 181
 182    if (nouveau_bo_wait(tx->bo, NOUVEAU_BO_RD, nv->client))
 183       return FALSE;
 184
 185    if (buf->data)
 186       memcpy(buf->data + base, tx->map, size);
 187
 188    return TRUE;
 189 }
 190
 191 static void
 192 nouveau_transfer_write(struct nouveau_context *nv, struct nouveau_transfer *tx,
 193                        unsigned offset, unsigned size)
 194 {
 195    struct nv04_resource *buf = nv04_resource(tx->base.resource);
 196    uint8_t *data = tx->map + offset;
 197    const unsigned base = tx->base.box.x + offset;
 198    const boolean can_cb = !((base | size) & 3);
 199
 200    if (buf->data)
 201       memcpy(data, buf->data + base, size);
 202    else
 203       buf->status |= NOUVEAU_BUFFER_STATUS_DIRTY;
 204
 205    if (buf->domain == NOUVEAU_BO_VRAM)
 206       NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_staging_vid, size);
 207    if (buf->domain == NOUVEAU_BO_GART)
 208       NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_staging_sys, size);
 209
 210    if (tx->bo)
 211       nv->copy_data(nv, buf->bo, buf->offset + base, buf->domain,
 212                     tx->bo, tx->offset + offset, NOUVEAU_BO_GART, size);
 213    else
 214    if ((buf->base.bind & PIPE_BIND_CONSTANT_BUFFER) && nv->push_cb && can_cb)
 215       nv->push_cb(nv, buf->bo, buf->domain, buf->offset, buf->base.width0,
 216                   base, size / 4, (const uint32_t *)data);
 217    else
 218       nv->push_data(nv, buf->bo, buf->offset + base, buf->domain, size, data);
 219
 220    nouveau_fence_ref(nv->screen->fence.current, &buf->fence);
 221    nouveau_fence_ref(nv->screen->fence.current, &buf->fence_wr);
 222 }
 223
 224 /* Does a CPU wait for the buffer's backing data to become reliably accessible
 225  * for write/read by waiting on the buffer's relevant fences.
 226  */
 227 static INLINE boolean
 228 nouveau_buffer_sync(struct nv04_resource *buf, unsigned rw)
 229 {
 230    if (rw == PIPE_TRANSFER_READ) {
 231       if (!buf->fence_wr)
 232          return TRUE;
 233       NOUVEAU_DRV_STAT_RES(buf, buf_non_kernel_fence_sync_count,
 234                            !nouveau_fence_signalled(buf->fence_wr));
 235       if (!nouveau_fence_wait(buf->fence_wr))
 236          return FALSE;
 237    } else {
 238       if (!buf->fence)
 239          return TRUE;
 240       NOUVEAU_DRV_STAT_RES(buf, buf_non_kernel_fence_sync_count,
 241                            !nouveau_fence_signalled(buf->fence));
 242       if (!nouveau_fence_wait(buf->fence))
 243          return FALSE;
 244
 245       nouveau_fence_ref(NULL, &buf->fence);
 246    }
 247    nouveau_fence_ref(NULL, &buf->fence_wr);
 248
 249    return TRUE;
 250 }
 251
 252 static INLINE boolean
 253 nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw)
 254 {
 255    if (rw == PIPE_TRANSFER_READ)
 256       return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr));
 257    else
 258       return (buf->fence && !nouveau_fence_signalled(buf->fence));
 259 }
 260
 261 static INLINE void
 262 nouveau_buffer_transfer_init(struct nouveau_transfer *tx,
 263                              struct pipe_resource *resource,
 264                              const struct pipe_box *box,
 265                              unsigned usage)
 266 {
 267    tx->base.resource = resource;
 268    tx->base.level = 0;
 269    tx->base.usage = usage;
 270    tx->base.box.x = box->x;
 271    tx->base.box.y = 0;
 272    tx->base.box.z = 0;
 273    tx->base.box.width = box->width;
 274    tx->base.box.height = 1;
 275    tx->base.box.depth = 1;
 276    tx->base.stride = 0;
 277    tx->base.layer_stride = 0;
 278
 279    tx->bo = NULL;
 280    tx->map = NULL;
 281 }
 282
 283 static INLINE void
 284 nouveau_buffer_transfer_del(struct nouveau_context *nv,
 285                             struct nouveau_transfer *tx)
 286 {
 287    if (tx->map) {
 288       if (likely(tx->bo)) {
 289          nouveau_bo_ref(NULL, &tx->bo);
 290          if (tx->mm)
 291             release_allocation(&tx->mm, nv->screen->fence.current);
 292       } else {
 293          align_free(tx->map -
 294                     (tx->base.box.x & NOUVEAU_MIN_BUFFER_MAP_ALIGN_MASK));
 295       }
 296    }
 297 }
 298
 299 /* Creates a cache in system memory of the buffer data. */
 300 static boolean
 301 nouveau_buffer_cache(struct nouveau_context *nv, struct nv04_resource *buf)
 302 {
 303    struct nouveau_transfer tx;
 304    boolean ret;
 305    tx.base.resource = &buf->base;
 306    tx.base.box.x = 0;
 307    tx.base.box.width = buf->base.width0;
 308    tx.bo = NULL;
 309    tx.map = NULL;
 310
 311    if (!buf->data)
 312       if (!nouveau_buffer_malloc(buf))
 313          return FALSE;
 314    if (!(buf->status & NOUVEAU_BUFFER_STATUS_DIRTY))
 315       return TRUE;
 316    nv->stats.buf_cache_count++;
 317
 318    if (!nouveau_transfer_staging(nv, &tx, FALSE))
 319       return FALSE;
 320
 321    ret = nouveau_transfer_read(nv, &tx);
 322    if (ret) {
 323       buf->status &= ~NOUVEAU_BUFFER_STATUS_DIRTY;
 324       memcpy(buf->data, tx.map, buf->base.width0);
 325    }
 326    nouveau_buffer_transfer_del(nv, &tx);
 327    return ret;
 328 }
 329
 330
 331 #define NOUVEAU_TRANSFER_DISCARD \
 332    (PIPE_TRANSFER_DISCARD_RANGE | PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)
 333
 334 /* Checks whether it is possible to completely discard the memory backing this
 335  * resource. This can be useful if we would otherwise have to wait for a read
 336  * operation to complete on this data.
 337  */
 338 static INLINE boolean
 339 nouveau_buffer_should_discard(struct nv04_resource *buf, unsigned usage)
 340 {
 341    if (!(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE))
 342       return FALSE;
 343    if (unlikely(buf->base.bind & PIPE_BIND_SHARED))
 344       return FALSE;
 345    if (unlikely(usage & PIPE_TRANSFER_PERSISTENT))
 346       return FALSE;
 347    return buf->mm && nouveau_buffer_busy(buf, PIPE_TRANSFER_WRITE);
 348 }
 349
 350 /* Returns a pointer to a memory area representing a window into the
 351  * resource's data.
 352  *
 353  * This may or may not be the _actual_ memory area of the resource. However
 354  * when calling nouveau_buffer_transfer_unmap, if it wasn't the actual memory
 355  * area, the contents of the returned map are copied over to the resource.
 356  *
 357  * The usage indicates what the caller plans to do with the map:
 358  *
 359  *   WRITE means that the user plans to write to it
 360  *
 361  *   READ means that the user plans on reading from it
 362  *
 363  *   DISCARD_WHOLE_RESOURCE means that the whole resource is going to be
 364  *   potentially overwritten, and even if it isn't, the bits that aren't don't
 365  *   need to be maintained.
 366  *
 367  *   DISCARD_RANGE means that all the data in the specified range is going to
 368  *   be overwritten.
 369  *
 370  * The strategy for determining what kind of memory area to return is complex,
 371  * see comments inside of the function.
 372  */
 373 static void *
 374 nouveau_buffer_transfer_map(struct pipe_context *pipe,
 375                             struct pipe_resource *resource,
 376                             unsigned level, unsigned usage,
 377                             const struct pipe_box *box,
 378                             struct pipe_transfer **ptransfer)
 379 {
 380    struct nouveau_context *nv = nouveau_context(pipe);
 381    struct nv04_resource *buf = nv04_resource(resource);
 382    struct nouveau_transfer *tx = MALLOC_STRUCT(nouveau_transfer);
 383    uint8_t *map;
 384    int ret;
 385
 386    if (!tx)
 387       return NULL;
 388    nouveau_buffer_transfer_init(tx, resource, box, usage);
 389    *ptransfer = &tx->base;
 390
 391    if (usage & PIPE_TRANSFER_READ)
 392       NOUVEAU_DRV_STAT(nv->screen, buf_transfers_rd, 1);
 393    if (usage & PIPE_TRANSFER_WRITE)
 394       NOUVEAU_DRV_STAT(nv->screen, buf_transfers_wr, 1);
 395
 396    /* If we are trying to write to an uninitialized range, the user shouldn't
 397     * care what was there before. So we can treat the write as if the target
 398     * range were being discarded. Furthermore, since we know that even if this
 399     * buffer is busy due to GPU activity, because the contents were
 400     * uninitialized, the GPU can't care what was there, and so we can treat
 401     * the write as being unsynchronized.
 402     */
 403    if ((usage & PIPE_TRANSFER_WRITE) &&
 404        !util_ranges_intersect(&buf->valid_buffer_range, box->x, box->x + box->width))
 405       usage |= PIPE_TRANSFER_DISCARD_RANGE | PIPE_TRANSFER_UNSYNCHRONIZED;
 406
 407    if (usage & PIPE_TRANSFER_PERSISTENT)
 408       usage |= PIPE_TRANSFER_UNSYNCHRONIZED;
 409
 410    if (buf->domain == NOUVEAU_BO_VRAM) {
 411       if (usage & NOUVEAU_TRANSFER_DISCARD) {
 412          /* Set up a staging area for the user to write to. It will be copied
 413           * back into VRAM on unmap. */
 414          if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)
 415             buf->status &= NOUVEAU_BUFFER_STATUS_REALLOC_MASK;
 416          nouveau_transfer_staging(nv, tx, TRUE);
 417       } else {
 418          if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING) {
 419             /* The GPU is currently writing to this buffer. Copy its current
 420              * contents to a staging area in the GART. This is necessary since
 421              * not the whole area being mapped is being discarded.
 422              */
 423             if (buf->data) {
 424                align_free(buf->data);
 425                buf->data = NULL;
 426             }
 427             nouveau_transfer_staging(nv, tx, FALSE);
 428             nouveau_transfer_read(nv, tx);
 429          } else {
 430             /* The buffer is currently idle. Create a staging area for writes,
 431              * and make sure that the cached data is up-to-date. */
 432             if (usage & PIPE_TRANSFER_WRITE)
 433                nouveau_transfer_staging(nv, tx, TRUE);
 434             if (!buf->data)
 435                nouveau_buffer_cache(nv, buf);
 436          }
 437       }
 438       return buf->data ? (buf->data + box->x) : tx->map;
 439    } else
 440    if (unlikely(buf->domain == 0)) {
 441       return buf->data + box->x;
 442    }
 443
 444    /* At this point, buf->domain == GART */
 445
 446    if (nouveau_buffer_should_discard(buf, usage)) {
 447       int ref = buf->base.reference.count - 1;
 448       nouveau_buffer_reallocate(nv->screen, buf, buf->domain);
 449       if (ref > 0) /* any references inside context possible ? */
 450          nv->invalidate_resource_storage(nv, &buf->base, ref);
 451    }
 452
 453    /* Note that nouveau_bo_map ends up doing a nouveau_bo_wait with the
 454     * relevant flags. If buf->mm is set, that means this resource is part of a
 455     * larger slab bo that holds multiple resources. So in that case, don't
 456     * wait on the whole slab and instead use the logic below to return a
 457     * reasonable buffer for that case.
 458     */
 459    ret = nouveau_bo_map(buf->bo,
 460                         buf->mm ? 0 : nouveau_screen_transfer_flags(usage),
 461                         nv->client);
 462    if (ret) {
 463       FREE(tx);
 464       return NULL;
 465    }
 466    map = (uint8_t *)buf->bo->map + buf->offset + box->x;
 467
 468    /* using kernel fences only if !buf->mm */
 469    if ((usage & PIPE_TRANSFER_UNSYNCHRONIZED) || !buf->mm)
 470       return map;
 471
 472    /* If the GPU is currently reading/writing this buffer, we shouldn't
 473     * interfere with its progress. So instead we either wait for the GPU to
 474     * complete its operation, or set up a staging area to perform our work in.
 475     */
 476    if (nouveau_buffer_busy(buf, usage & PIPE_TRANSFER_READ_WRITE)) {
 477       if (unlikely(usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE)) {
 478          /* Discarding was not possible, must sync because
 479           * subsequent transfers might use UNSYNCHRONIZED. */
 480          nouveau_buffer_sync(buf, usage & PIPE_TRANSFER_READ_WRITE);
 481       } else
 482       if (usage & PIPE_TRANSFER_DISCARD_RANGE) {
 483          /* The whole range is being discarded, so it doesn't matter what was
 484           * there before. No need to copy anything over. */
 485          nouveau_transfer_staging(nv, tx, TRUE);
 486          map = tx->map;
 487       } else
 488       if (nouveau_buffer_busy(buf, PIPE_TRANSFER_READ)) {
 489          if (usage & PIPE_TRANSFER_DONTBLOCK)
 490             map = NULL;
 491          else
 492             nouveau_buffer_sync(buf, usage & PIPE_TRANSFER_READ_WRITE);
 493       } else {
 494          /* It is expected that the returned buffer be a representation of the
 495           * data in question, so we must copy it over from the buffer. */
 496          nouveau_transfer_staging(nv, tx, TRUE);
 497          if (tx->map)
 498             memcpy(tx->map, map, box->width);
 499          map = tx->map;
 500       }
 501    }
 502    if (!map)
 503       FREE(tx);
 504    return map;
 505 }
 506
 507
 508
 509 static void
 510 nouveau_buffer_transfer_flush_region(struct pipe_context *pipe,
 511                                      struct pipe_transfer *transfer,
 512                                      const struct pipe_box *box)
 513 {
 514    struct nouveau_transfer *tx = nouveau_transfer(transfer);
 515    struct nv04_resource *buf = nv04_resource(transfer->resource);
 516
 517    if (tx->map)
 518       nouveau_transfer_write(nouveau_context(pipe), tx, box->x, box->width);
 519
 520    util_range_add(&buf->valid_buffer_range,
 521                   tx->base.box.x + box->x,
 522                   tx->base.box.x + box->x + box->width);
 523 }
 524
 525 /* Unmap stage of the transfer. If it was a WRITE transfer and the map that
 526  * was returned was not the real resource's data, this needs to transfer the
 527  * data back to the resource.
 528  *
 529  * Also marks vbo dirty based on the buffer's binding
 530  */
 531 static void
 532 nouveau_buffer_transfer_unmap(struct pipe_context *pipe,
 533                               struct pipe_transfer *transfer)
 534 {
 535    struct nouveau_context *nv = nouveau_context(pipe);
 536    struct nouveau_transfer *tx = nouveau_transfer(transfer);
 537    struct nv04_resource *buf = nv04_resource(transfer->resource);
 538
 539    if (tx->base.usage & PIPE_TRANSFER_WRITE) {
 540       if (!(tx->base.usage & PIPE_TRANSFER_FLUSH_EXPLICIT) && tx->map)
 541          nouveau_transfer_write(nv, tx, 0, tx->base.box.width);
 542
 543       if (likely(buf->domain)) {
 544          const uint8_t bind = buf->base.bind;
 545          /* make sure we invalidate dedicated caches */
 546          if (bind & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER))
 547             nv->vbo_dirty = TRUE;
 548       }
 549
 550       util_range_add(&buf->valid_buffer_range,
 551                      tx->base.box.x, tx->base.box.x + tx->base.box.width);
 552    }
 553
 554    if (!tx->bo && (tx->base.usage & PIPE_TRANSFER_WRITE))
 555       NOUVEAU_DRV_STAT(nv->screen, buf_write_bytes_direct, tx->base.box.width);
 556
 557    nouveau_buffer_transfer_del(nv, tx);
 558    FREE(tx);
 559 }
 560
 561
 562 void
 563 nouveau_copy_buffer(struct nouveau_context *nv,
 564                     struct nv04_resource *dst, unsigned dstx,
 565                     struct nv04_resource *src, unsigned srcx, unsigned size)
 566 {
 567    assert(dst->base.target == PIPE_BUFFER && src->base.target == PIPE_BUFFER);
 568
 569    if (likely(dst->domain) && likely(src->domain)) {
 570       nv->copy_data(nv,
 571                     dst->bo, dst->offset + dstx, dst->domain,
 572                     src->bo, src->offset + srcx, src->domain, size);
 573
 574       dst->status |= NOUVEAU_BUFFER_STATUS_GPU_WRITING;
 575       nouveau_fence_ref(nv->screen->fence.current, &dst->fence);
 576       nouveau_fence_ref(nv->screen->fence.current, &dst->fence_wr);
 577
 578       src->status |= NOUVEAU_BUFFER_STATUS_GPU_READING;
 579       nouveau_fence_ref(nv->screen->fence.current, &src->fence);
 580    } else {
 581       struct pipe_box src_box;
 582       src_box.x = srcx;
 583       src_box.y = 0;
 584       src_box.z = 0;
 585       src_box.width = size;
 586       src_box.height = 1;
 587       src_box.depth = 1;
 588       util_resource_copy_region(&nv->pipe,
 589                                 &dst->base, 0, dstx, 0, 0,
 590                                 &src->base, 0, &src_box);
 591    }
 592
 593    util_range_add(&dst->valid_buffer_range, dstx, dstx + size);
 594 }
 595
 596
 597 void *
 598 nouveau_resource_map_offset(struct nouveau_context *nv,
 599                             struct nv04_resource *res, uint32_t offset,
 600                             uint32_t flags)
 601 {
 602    if (unlikely(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
 603       return res->data + offset;
 604
 605    if (res->domain == NOUVEAU_BO_VRAM) {
 606       if (!res->data || (res->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING))
 607          nouveau_buffer_cache(nv, res);
 608    }
 609    if (res->domain != NOUVEAU_BO_GART)
 610       return res->data + offset;
 611
 612    if (res->mm) {
 613       unsigned rw;
 614       rw = (flags & NOUVEAU_BO_WR) ? PIPE_TRANSFER_WRITE : PIPE_TRANSFER_READ;
 615       nouveau_buffer_sync(res, rw);
 616       if (nouveau_bo_map(res->bo, 0, NULL))
 617          return NULL;
 618    } else {
 619       if (nouveau_bo_map(res->bo, flags, nv->client))
 620          return NULL;
 621    }
 622    return (uint8_t *)res->bo->map + res->offset + offset;
 623 }
 624
 625
 626 const struct u_resource_vtbl nouveau_buffer_vtbl =
 627 {
 628    u_default_resource_get_handle,     /* get_handle */
 629    nouveau_buffer_destroy,               /* resource_destroy */
 630    nouveau_buffer_transfer_map,          /* transfer_map */
 631    nouveau_buffer_transfer_flush_region, /* transfer_flush_region */
 632    nouveau_buffer_transfer_unmap,        /* transfer_unmap */
 633    u_default_transfer_inline_write    /* transfer_inline_write */
 634 };
 635
 636 struct pipe_resource *
 637 nouveau_buffer_create(struct pipe_screen *pscreen,
 638                       const struct pipe_resource *templ)
 639 {
 640    struct nouveau_screen *screen = nouveau_screen(pscreen);
 641    struct nv04_resource *buffer;
 642    boolean ret;
 643
 644    buffer = CALLOC_STRUCT(nv04_resource);
 645    if (!buffer)
 646       return NULL;
 647
 648    buffer->base = *templ;
 649    buffer->vtbl = &nouveau_buffer_vtbl;
 650    pipe_reference_init(&buffer->base.reference, 1);
 651    buffer->base.screen = pscreen;
 652
 653    if (buffer->base.flags & (PIPE_RESOURCE_FLAG_MAP_PERSISTENT |
 654                              PIPE_RESOURCE_FLAG_MAP_COHERENT)) {
 655       buffer->domain = NOUVEAU_BO_GART;
 656    } else if (buffer->base.bind &
 657               (screen->vidmem_bindings & screen->sysmem_bindings)) {
 658       switch (buffer->base.usage) {
 659       case PIPE_USAGE_DEFAULT:
 660       case PIPE_USAGE_IMMUTABLE:
 661          buffer->domain = NOUVEAU_BO_VRAM;
 662          break;
 663       case PIPE_USAGE_DYNAMIC:
 664          /* For most apps, we'd have to do staging transfers to avoid sync
 665           * with this usage, and GART -> GART copies would be suboptimal.
 666           */
 667          buffer->domain = NOUVEAU_BO_VRAM;
 668          break;
 669       case PIPE_USAGE_STAGING:
 670       case PIPE_USAGE_STREAM:
 671          buffer->domain = NOUVEAU_BO_GART;
 672          break;
 673       default:
 674          assert(0);
 675          break;
 676       }
 677    } else {
 678       if (buffer->base.bind & screen->vidmem_bindings)
 679          buffer->domain = NOUVEAU_BO_VRAM;
 680       else
 681       if (buffer->base.bind & screen->sysmem_bindings)
 682          buffer->domain = NOUVEAU_BO_GART;
 683    }
 684    ret = nouveau_buffer_allocate(screen, buffer, buffer->domain);
 685
 686    if (ret == FALSE)
 687       goto fail;
 688
 689    if (buffer->domain == NOUVEAU_BO_VRAM && screen->hint_buf_keep_sysmem_copy)
 690       nouveau_buffer_cache(NULL, buffer);
 691
 692    NOUVEAU_DRV_STAT(screen, buf_obj_current_count, 1);
 693
 694    util_range_init(&buffer->valid_buffer_range);
 695
 696    return &buffer->base;
 697
 698 fail:
 699    FREE(buffer);
 700    return NULL;
 701 }
 702
 703
 704 struct pipe_resource *
 705 nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr,
 706                            unsigned bytes, unsigned bind)
 707 {
 708    struct nv04_resource *buffer;
 709
 710    buffer = CALLOC_STRUCT(nv04_resource);
 711    if (!buffer)
 712       return NULL;
 713
 714    pipe_reference_init(&buffer->base.reference, 1);
 715    buffer->vtbl = &nouveau_buffer_vtbl;
 716    buffer->base.screen = pscreen;
 717    buffer->base.format = PIPE_FORMAT_R8_UNORM;
 718    buffer->base.usage = PIPE_USAGE_IMMUTABLE;
 719    buffer->base.bind = bind;
 720    buffer->base.width0 = bytes;
 721    buffer->base.height0 = 1;
 722    buffer->base.depth0 = 1;
 723
 724    buffer->data = ptr;
 725    buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY;
 726
 727    util_range_init(&buffer->valid_buffer_range);
 728    util_range_add(&buffer->valid_buffer_range, 0, bytes);
 729
 730    return &buffer->base;
 731 }
 732
 733 static INLINE boolean
 734 nouveau_buffer_data_fetch(struct nouveau_context *nv, struct nv04_resource *buf,
 735                           struct nouveau_bo *bo, unsigned offset, unsigned size)
 736 {
 737    if (!nouveau_buffer_malloc(buf))
 738       return FALSE;
 739    if (nouveau_bo_map(bo, NOUVEAU_BO_RD, nv->client))
 740       return FALSE;
 741    memcpy(buf->data, (uint8_t *)bo->map + offset, size);
 742    return TRUE;
 743 }
 744
 745 /* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
 746 boolean
 747 nouveau_buffer_migrate(struct nouveau_context *nv,
 748                        struct nv04_resource *buf, const unsigned new_domain)
 749 {
 750    struct nouveau_screen *screen = nv->screen;
 751    struct nouveau_bo *bo;
 752    const unsigned old_domain = buf->domain;
 753    unsigned size = buf->base.width0;
 754    unsigned offset;
 755    int ret;
 756
 757    assert(new_domain != old_domain);
 758
 759    if (new_domain == NOUVEAU_BO_GART && old_domain == 0) {
 760       if (!nouveau_buffer_allocate(screen, buf, new_domain))
 761          return FALSE;
 762       ret = nouveau_bo_map(buf->bo, 0, nv->client);
 763       if (ret)
 764          return ret;
 765       memcpy((uint8_t *)buf->bo->map + buf->offset, buf->data, size);
 766       align_free(buf->data);
 767    } else
 768    if (old_domain != 0 && new_domain != 0) {
 769       struct nouveau_mm_allocation *mm = buf->mm;
 770
 771       if (new_domain == NOUVEAU_BO_VRAM) {
 772          /* keep a system memory copy of our data in case we hit a fallback */
 773          if (!nouveau_buffer_data_fetch(nv, buf, buf->bo, buf->offset, size))
 774             return FALSE;
 775          if (nouveau_mesa_debug)
 776             debug_printf("migrating %u KiB to VRAM\n", size / 1024);
 777       }
 778
 779       offset = buf->offset;
 780       bo = buf->bo;
 781       buf->bo = NULL;
 782       buf->mm = NULL;
 783       nouveau_buffer_allocate(screen, buf, new_domain);
 784
 785       nv->copy_data(nv, buf->bo, buf->offset, new_domain,
 786                     bo, offset, old_domain, buf->base.width0);
 787
 788       nouveau_bo_ref(NULL, &bo);
 789       if (mm)
 790          release_allocation(&mm, screen->fence.current);
 791    } else
 792    if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) {
 793       struct nouveau_transfer tx;
 794       if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
 795          return FALSE;
 796       tx.base.resource = &buf->base;
 797       tx.base.box.x = 0;
 798       tx.base.box.width = buf->base.width0;
 799       tx.bo = NULL;
 800       tx.map = NULL;
 801       if (!nouveau_transfer_staging(nv, &tx, FALSE))
 802          return FALSE;
 803       nouveau_transfer_write(nv, &tx, 0, tx.base.box.width);
 804       nouveau_buffer_transfer_del(nv, &tx);
 805    } else
 806       return FALSE;
 807
 808    assert(buf->domain == new_domain);
 809    return TRUE;
 810 }
 811
 812 /* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
 813  * We'd like to only allocate @size bytes here, but then we'd have to rebase
 814  * the vertex indices ...
 815  */
 816 boolean
 817 nouveau_user_buffer_upload(struct nouveau_context *nv,
 818                            struct nv04_resource *buf,
 819                            unsigned base, unsigned size)
 820 {
 821    struct nouveau_screen *screen = nouveau_screen(buf->base.screen);
 822    int ret;
 823
 824    assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY);
 825
 826    buf->base.width0 = base + size;
 827    if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART))
 828       return FALSE;
 829
 830    ret = nouveau_bo_map(buf->bo, 0, nv->client);
 831    if (ret)
 832       return FALSE;
 833    memcpy((uint8_t *)buf->bo->map + buf->offset + base, buf->data + base, size);
 834
 835    return TRUE;
 836 }
 837
 838
 839 /* Scratch data allocation. */
 840
 841 static INLINE int
 842 nouveau_scratch_bo_alloc(struct nouveau_context *nv, struct nouveau_bo **pbo,
 843                          unsigned size)
 844 {
 845    return nouveau_bo_new(nv->screen->device, NOUVEAU_BO_GART | NOUVEAU_BO_MAP,
 846                          4096, size, NULL, pbo);
 847 }
 848
 849 static void
 850 nouveau_scratch_unref_bos(void *d)
 851 {
 852    struct runout *b = d;
 853    int i;
 854
 855    for (i = 0; i < b->nr; ++i)
 856       nouveau_bo_ref(NULL, &b->bo[i]);
 857
 858    FREE(b);
 859 }
 860
 861 void
 862 nouveau_scratch_runout_release(struct nouveau_context *nv)
 863 {
 864    if (!nv->scratch.runout)
 865       return;
 866
 867    if (!nouveau_fence_work(nv->screen->fence.current, nouveau_scratch_unref_bos,
 868          nv->scratch.runout))
 869       return;
 870
 871    nv->scratch.end = 0;
 872    nv->scratch.runout = NULL;
 873 }
 874
 875 /* Allocate an extra bo if we can't fit everything we need simultaneously.
 876  * (Could happen for very large user arrays.)
 877  */
 878 static INLINE boolean
 879 nouveau_scratch_runout(struct nouveau_context *nv, unsigned size)
 880 {
 881    int ret;
 882    unsigned n;
 883
 884    if (nv->scratch.runout)
 885       n = nv->scratch.runout->nr;
 886    else
 887       n = 0;
 888    nv->scratch.runout = REALLOC(nv->scratch.runout, n == 0 ? 0 :
 889                                 (sizeof(*nv->scratch.runout) + (n + 0) * sizeof(void *)),
 890                                  sizeof(*nv->scratch.runout) + (n + 1) * sizeof(void *));
 891    nv->scratch.runout->nr = n + 1;
 892    nv->scratch.runout->bo[n] = NULL;
 893
 894    ret = nouveau_scratch_bo_alloc(nv, &nv->scratch.runout->bo[n], size);
 895    if (!ret) {
 896       ret = nouveau_bo_map(nv->scratch.runout->bo[n], 0, NULL);
 897       if (ret)
 898          nouveau_bo_ref(NULL, &nv->scratch.runout->bo[--nv->scratch.runout->nr]);
 899    }
 900    if (!ret) {
 901       nv->scratch.current = nv->scratch.runout->bo[n];
 902       nv->scratch.offset = 0;
 903       nv->scratch.end = size;
 904       nv->scratch.map = nv->scratch.current->map;
 905    }
 906    return !ret;
 907 }
 908
 909 /* Continue to next scratch buffer, if available (no wrapping, large enough).
 910  * Allocate it if it has not yet been created.
 911  */
 912 static INLINE boolean
 913 nouveau_scratch_next(struct nouveau_context *nv, unsigned size)
 914 {
 915    struct nouveau_bo *bo;
 916    int ret;
 917    const unsigned i = (nv->scratch.id + 1) % NOUVEAU_MAX_SCRATCH_BUFS;
 918
 919    if ((size > nv->scratch.bo_size) || (i == nv->scratch.wrap))
 920       return FALSE;
 921    nv->scratch.id = i;
 922
 923    bo = nv->scratch.bo[i];
 924    if (!bo) {
 925       ret = nouveau_scratch_bo_alloc(nv, &bo, nv->scratch.bo_size);
 926       if (ret)
 927          return FALSE;
 928       nv->scratch.bo[i] = bo;
 929    }
 930    nv->scratch.current = bo;
 931    nv->scratch.offset = 0;
 932    nv->scratch.end = nv->scratch.bo_size;
 933
 934    ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, nv->client);
 935    if (!ret)
 936       nv->scratch.map = bo->map;
 937    return !ret;
 938 }
 939
 940 static boolean
 941 nouveau_scratch_more(struct nouveau_context *nv, unsigned min_size)
 942 {
 943    boolean ret;
 944
 945    ret = nouveau_scratch_next(nv, min_size);
 946    if (!ret)
 947       ret = nouveau_scratch_runout(nv, min_size);
 948    return ret;
 949 }
 950
 951
 952 /* Copy data to a scratch buffer and return address & bo the data resides in. */
 953 uint64_t
 954 nouveau_scratch_data(struct nouveau_context *nv,
 955                      const void *data, unsigned base, unsigned size,
 956                      struct nouveau_bo **bo)
 957 {
 958    unsigned bgn = MAX2(base, nv->scratch.offset);
 959    unsigned end = bgn + size;
 960
 961    if (end >= nv->scratch.end) {
 962       end = base + size;
 963       if (!nouveau_scratch_more(nv, end))
 964          return 0;
 965       bgn = base;
 966    }
 967    nv->scratch.offset = align(end, 4);
 968
 969    memcpy(nv->scratch.map + bgn, (const uint8_t *)data + base, size);
 970
 971    *bo = nv->scratch.current;
 972    return (*bo)->offset + (bgn - base);
 973 }
 974
 975 void *
 976 nouveau_scratch_get(struct nouveau_context *nv,
 977                     unsigned size, uint64_t *gpu_addr, struct nouveau_bo **pbo)
 978 {
 979    unsigned bgn = nv->scratch.offset;
 980    unsigned end = nv->scratch.offset + size;
 981
 982    if (end >= nv->scratch.end) {
 983       end = size;
 984       if (!nouveau_scratch_more(nv, end))
 985          return NULL;
 986       bgn = 0;
 987    }
 988    nv->scratch.offset = align(end, 4);
 989
 990    *pbo = nv->scratch.current;
 991    *gpu_addr = nv->scratch.current->offset + bgn;
 992    return nv->scratch.map + bgn;
 993 }