src/gallium/drivers/nouveau/nouveau_buffer.c

   1
   2 #include "util/u_inlines.h"
   3 #include "util/u_memory.h"
   4 #include "util/u_math.h"
   5
   6 #include "nouveau_screen.h"
   7 #include "nouveau_context.h"
   8 #include "nouveau_winsys.h"
   9 #include "nouveau_fence.h"
  10 #include "nouveau_buffer.h"
  11 #include "nouveau_mm.h"
  12
  13 struct nouveau_transfer {
  14    struct pipe_transfer base;
  15 };
  16
  17 static INLINE struct nouveau_transfer *
  18 nouveau_transfer(struct pipe_transfer *transfer)
  19 {
  20    return (struct nouveau_transfer *)transfer;
  21 }
  22
  23 static INLINE boolean
  24 nouveau_buffer_allocate(struct nouveau_screen *screen,
  25                         struct nv04_resource *buf, unsigned domain)
  26 {
  27    uint32_t size = buf->base.width0;
  28
  29    if (buf->base.bind & PIPE_BIND_CONSTANT_BUFFER)
  30       size = align(size, 0x100);
  31
  32    if (domain == NOUVEAU_BO_VRAM) {
  33       buf->mm = nouveau_mm_allocate(screen->mm_VRAM, size,
  34                                     &buf->bo, &buf->offset);
  35       if (!buf->bo)
  36          return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
  37    } else
  38    if (domain == NOUVEAU_BO_GART) {
  39       buf->mm = nouveau_mm_allocate(screen->mm_GART, size,
  40                                     &buf->bo, &buf->offset);
  41       if (!buf->bo)
  42          return FALSE;
  43    }
  44    if (domain != NOUVEAU_BO_GART) {
  45       if (!buf->data) {
  46          buf->data = MALLOC(buf->base.width0);
  47          if (!buf->data)
  48             return FALSE;
  49       }
  50    }
  51    buf->domain = domain;
  52    return TRUE;
  53 }
  54
  55 static INLINE void
  56 release_allocation(struct nouveau_mm_allocation **mm,
  57                    struct nouveau_fence *fence)
  58 {
  59    nouveau_fence_work(fence, nouveau_mm_free_work, *mm);
  60    (*mm) = NULL;
  61 }
  62
  63 INLINE void
  64 nouveau_buffer_release_gpu_storage(struct nv04_resource *buf)
  65 {
  66    nouveau_bo_ref(NULL, &buf->bo);
  67
  68    if (buf->mm)
  69       release_allocation(&buf->mm, buf->fence);
  70
  71    buf->domain = 0;
  72 }
  73
  74 static INLINE boolean
  75 nouveau_buffer_reallocate(struct nouveau_screen *screen,
  76                           struct nv04_resource *buf, unsigned domain)
  77 {
  78    nouveau_buffer_release_gpu_storage(buf);
  79
  80    return nouveau_buffer_allocate(screen, buf, domain);
  81 }
  82
  83 static void
  84 nouveau_buffer_destroy(struct pipe_screen *pscreen,
  85                        struct pipe_resource *presource)
  86 {
  87    struct nv04_resource *res = nv04_resource(presource);
  88
  89    nouveau_buffer_release_gpu_storage(res);
  90
  91    if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
  92       FREE(res->data);
  93
  94    FREE(res);
  95 }
  96
  97 /* Maybe just migrate to GART right away if we actually need to do this. */
  98 boolean
  99 nouveau_buffer_download(struct nouveau_context *nv, struct nv04_resource *buf,
 100                         unsigned start, unsigned size)
 101 {
 102    struct nouveau_mm_allocation *mm;
 103    struct nouveau_bo *bounce = NULL;
 104    uint32_t offset;
 105
 106    assert(buf->domain == NOUVEAU_BO_VRAM);
 107
 108    mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset);
 109    if (!bounce)
 110       return FALSE;
 111
 112    nv->copy_data(nv, bounce, offset, NOUVEAU_BO_GART,
 113                  buf->bo, buf->offset + start, NOUVEAU_BO_VRAM, size);
 114
 115    if (nouveau_bo_map_range(bounce, offset, size, NOUVEAU_BO_RD))
 116       return FALSE;
 117    memcpy(buf->data + start, bounce->map, size);
 118    nouveau_bo_unmap(bounce);
 119
 120    buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
 121
 122    nouveau_bo_ref(NULL, &bounce);
 123    if (mm)
 124       nouveau_mm_free(mm);
 125    return TRUE;
 126 }
 127
 128 static boolean
 129 nouveau_buffer_upload(struct nouveau_context *nv, struct nv04_resource *buf,
 130                       unsigned start, unsigned size)
 131 {
 132    struct nouveau_mm_allocation *mm;
 133    struct nouveau_bo *bounce = NULL;
 134    uint32_t offset;
 135
 136    if (size <= 192) {
 137       if (buf->base.bind & PIPE_BIND_CONSTANT_BUFFER)
 138          nv->push_cb(nv, buf->bo, buf->domain, buf->offset, buf->base.width0,
 139                      start, size / 4, (const uint32_t *)(buf->data + start));
 140       else
 141          nv->push_data(nv, buf->bo, buf->offset + start, buf->domain,
 142                        size, buf->data + start);
 143       return TRUE;
 144    }
 145
 146    mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset);
 147    if (!bounce)
 148       return FALSE;
 149
 150    nouveau_bo_map_range(bounce, offset, size,
 151                         NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC);
 152    memcpy(bounce->map, buf->data + start, size);
 153    nouveau_bo_unmap(bounce);
 154
 155    nv->copy_data(nv, buf->bo, buf->offset + start, NOUVEAU_BO_VRAM,
 156                  bounce, offset, NOUVEAU_BO_GART, size);
 157
 158    nouveau_bo_ref(NULL, &bounce);
 159    if (mm)
 160       release_allocation(&mm, nv->screen->fence.current);
 161
 162    if (start == 0 && size == buf->base.width0)
 163       buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
 164    return TRUE;
 165 }
 166
 167 static struct pipe_transfer *
 168 nouveau_buffer_transfer_get(struct pipe_context *pipe,
 169                             struct pipe_resource *resource,
 170                             unsigned level, unsigned usage,
 171                             const struct pipe_box *box)
 172 {
 173    struct nv04_resource *buf = nv04_resource(resource);
 174    struct nouveau_context *nv = nouveau_context(pipe);
 175    struct nouveau_transfer *xfr = CALLOC_STRUCT(nouveau_transfer);
 176    if (!xfr)
 177       return NULL;
 178
 179    xfr->base.resource = resource;
 180    xfr->base.box.x = box->x;
 181    xfr->base.box.width = box->width;
 182    xfr->base.usage = usage;
 183
 184    if (buf->domain == NOUVEAU_BO_VRAM) {
 185       if (usage & PIPE_TRANSFER_READ) {
 186          if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING)
 187             nouveau_buffer_download(nv, buf, 0, buf->base.width0);
 188       }
 189    }
 190
 191    return &xfr->base;
 192 }
 193
 194 static void
 195 nouveau_buffer_transfer_destroy(struct pipe_context *pipe,
 196                                 struct pipe_transfer *transfer)
 197 {
 198    struct nv04_resource *buf = nv04_resource(transfer->resource);
 199    struct nouveau_transfer *xfr = nouveau_transfer(transfer);
 200    struct nouveau_context *nv = nouveau_context(pipe);
 201
 202    if (xfr->base.usage & PIPE_TRANSFER_WRITE) {
 203       /* writing is worse */
 204       nouveau_buffer_adjust_score(nv, buf, -5000);
 205
 206       if (buf->domain == NOUVEAU_BO_VRAM) {
 207          nouveau_buffer_upload(nv, buf, transfer->box.x, transfer->box.width);
 208       }
 209
 210       if (buf->domain != 0 && (buf->base.bind & (PIPE_BIND_VERTEX_BUFFER |
 211                                                  PIPE_BIND_INDEX_BUFFER)))
 212          nouveau_context(pipe)->vbo_dirty = TRUE;
 213    }
 214
 215    FREE(xfr);
 216 }
 217
 218 static INLINE boolean
 219 nouveau_buffer_sync(struct nv04_resource *buf, unsigned rw)
 220 {
 221    if (rw == PIPE_TRANSFER_READ) {
 222       if (!buf->fence_wr)
 223          return TRUE;
 224       if (!nouveau_fence_wait(buf->fence_wr))
 225          return FALSE;
 226    } else {
 227       if (!buf->fence)
 228          return TRUE;
 229       if (!nouveau_fence_wait(buf->fence))
 230          return FALSE;
 231
 232       nouveau_fence_ref(NULL, &buf->fence);
 233    }
 234    nouveau_fence_ref(NULL, &buf->fence_wr);
 235
 236    return TRUE;
 237 }
 238
 239 static INLINE boolean
 240 nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw)
 241 {
 242    if (rw == PIPE_TRANSFER_READ)
 243       return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr));
 244    else
 245       return (buf->fence && !nouveau_fence_signalled(buf->fence));
 246 }
 247
 248 static void *
 249 nouveau_buffer_transfer_map(struct pipe_context *pipe,
 250                             struct pipe_transfer *transfer)
 251 {
 252    struct nouveau_transfer *xfr = nouveau_transfer(transfer);
 253    struct nv04_resource *buf = nv04_resource(transfer->resource);
 254    struct nouveau_bo *bo = buf->bo;
 255    uint8_t *map;
 256    int ret;
 257    uint32_t offset = xfr->base.box.x;
 258    uint32_t flags;
 259
 260    nouveau_buffer_adjust_score(nouveau_context(pipe), buf, -250);
 261
 262    if (buf->domain != NOUVEAU_BO_GART)
 263       return buf->data + offset;
 264
 265    if (buf->mm)
 266       flags = NOUVEAU_BO_NOSYNC | NOUVEAU_BO_RDWR;
 267    else
 268       flags = nouveau_screen_transfer_flags(xfr->base.usage);
 269
 270    offset += buf->offset;
 271
 272    ret = nouveau_bo_map_range(buf->bo, offset, xfr->base.box.width, flags);
 273    if (ret)
 274       return NULL;
 275    map = bo->map;
 276
 277    /* Unmap right now. Since multiple buffers can share a single nouveau_bo,
 278     * not doing so might make future maps fail or trigger "reloc while mapped"
 279     * errors. For now, mappings to userspace are guaranteed to be persistent.
 280     */
 281    nouveau_bo_unmap(bo);
 282
 283    if (buf->mm) {
 284       if (xfr->base.usage & PIPE_TRANSFER_DONTBLOCK) {
 285          if (nouveau_buffer_busy(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE))
 286             return NULL;
 287       } else
 288       if (!(xfr->base.usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
 289          nouveau_buffer_sync(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE);
 290       }
 291    }
 292    return map;
 293 }
 294
 295
 296
 297 static void
 298 nouveau_buffer_transfer_flush_region(struct pipe_context *pipe,
 299                                      struct pipe_transfer *transfer,
 300                                      const struct pipe_box *box)
 301 {
 302    struct nv04_resource *res = nv04_resource(transfer->resource);
 303    struct nouveau_bo *bo = res->bo;
 304    unsigned offset = res->offset + transfer->box.x + box->x;
 305
 306    /* not using non-snoop system memory yet, no need for cflush */
 307    if (1)
 308       return;
 309
 310    /* XXX: maybe need to upload for VRAM buffers here */
 311
 312    nouveau_screen_bo_map_flush_range(pipe->screen, bo, offset, box->width);
 313 }
 314
 315 static void
 316 nouveau_buffer_transfer_unmap(struct pipe_context *pipe,
 317                               struct pipe_transfer *transfer)
 318 {
 319    /* we've called nouveau_bo_unmap right after map */
 320 }
 321
 322 const struct u_resource_vtbl nouveau_buffer_vtbl =
 323 {
 324    u_default_resource_get_handle,     /* get_handle */
 325    nouveau_buffer_destroy,               /* resource_destroy */
 326    nouveau_buffer_transfer_get,          /* get_transfer */
 327    nouveau_buffer_transfer_destroy,      /* transfer_destroy */
 328    nouveau_buffer_transfer_map,          /* transfer_map */
 329    nouveau_buffer_transfer_flush_region, /* transfer_flush_region */
 330    nouveau_buffer_transfer_unmap,        /* transfer_unmap */
 331    u_default_transfer_inline_write    /* transfer_inline_write */
 332 };
 333
 334 struct pipe_resource *
 335 nouveau_buffer_create(struct pipe_screen *pscreen,
 336                       const struct pipe_resource *templ)
 337 {
 338    struct nouveau_screen *screen = nouveau_screen(pscreen);
 339    struct nv04_resource *buffer;
 340    boolean ret;
 341
 342    buffer = CALLOC_STRUCT(nv04_resource);
 343    if (!buffer)
 344       return NULL;
 345
 346    buffer->base = *templ;
 347    buffer->vtbl = &nouveau_buffer_vtbl;
 348    pipe_reference_init(&buffer->base.reference, 1);
 349    buffer->base.screen = pscreen;
 350
 351    if ((buffer->base.bind & screen->sysmem_bindings) == screen->sysmem_bindings)
 352       ret = nouveau_buffer_allocate(screen, buffer, 0);
 353    else
 354       ret = nouveau_buffer_allocate(screen, buffer, NOUVEAU_BO_GART);
 355
 356    if (ret == FALSE)
 357       goto fail;
 358
 359    return &buffer->base;
 360
 361 fail:
 362    FREE(buffer);
 363    return NULL;
 364 }
 365
 366
 367 struct pipe_resource *
 368 nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr,
 369                            unsigned bytes, unsigned bind)
 370 {
 371    struct nv04_resource *buffer;
 372
 373    buffer = CALLOC_STRUCT(nv04_resource);
 374    if (!buffer)
 375       return NULL;
 376
 377    pipe_reference_init(&buffer->base.reference, 1);
 378    buffer->vtbl = &nouveau_buffer_vtbl;
 379    buffer->base.screen = pscreen;
 380    buffer->base.format = PIPE_FORMAT_R8_UNORM;
 381    buffer->base.usage = PIPE_USAGE_IMMUTABLE;
 382    buffer->base.bind = bind;
 383    buffer->base.width0 = bytes;
 384    buffer->base.height0 = 1;
 385    buffer->base.depth0 = 1;
 386
 387    buffer->data = ptr;
 388    buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY;
 389
 390    return &buffer->base;
 391 }
 392
 393 /* Like download, but for GART buffers. Merge ? */
 394 static INLINE boolean
 395 nouveau_buffer_data_fetch(struct nv04_resource *buf, struct nouveau_bo *bo,
 396                           unsigned offset, unsigned size)
 397 {
 398    if (!buf->data) {
 399       buf->data = MALLOC(size);
 400       if (!buf->data)
 401          return FALSE;
 402    }
 403    if (nouveau_bo_map_range(bo, offset, size, NOUVEAU_BO_RD))
 404       return FALSE;
 405    memcpy(buf->data, bo->map, size);
 406    nouveau_bo_unmap(bo);
 407
 408    return TRUE;
 409 }
 410
 411 /* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
 412 boolean
 413 nouveau_buffer_migrate(struct nouveau_context *nv,
 414                        struct nv04_resource *buf, const unsigned new_domain)
 415 {
 416    struct nouveau_screen *screen = nv->screen;
 417    struct nouveau_bo *bo;
 418    const unsigned old_domain = buf->domain;
 419    unsigned size = buf->base.width0;
 420    unsigned offset;
 421    int ret;
 422
 423    assert(new_domain != old_domain);
 424
 425    if (new_domain == NOUVEAU_BO_GART && old_domain == 0) {
 426       if (!nouveau_buffer_allocate(screen, buf, new_domain))
 427          return FALSE;
 428       ret = nouveau_bo_map_range(buf->bo, buf->offset, size, NOUVEAU_BO_WR |
 429                                  NOUVEAU_BO_NOSYNC);
 430       if (ret)
 431          return ret;
 432       memcpy(buf->bo->map, buf->data, size);
 433       nouveau_bo_unmap(buf->bo);
 434       FREE(buf->data);
 435    } else
 436    if (old_domain != 0 && new_domain != 0) {
 437       struct nouveau_mm_allocation *mm = buf->mm;
 438
 439       if (new_domain == NOUVEAU_BO_VRAM) {
 440          /* keep a system memory copy of our data in case we hit a fallback */
 441          if (!nouveau_buffer_data_fetch(buf, buf->bo, buf->offset, size))
 442             return FALSE;
 443          if (nouveau_mesa_debug)
 444             debug_printf("migrating %u KiB to VRAM\n", size / 1024);
 445       }
 446
 447       offset = buf->offset;
 448       bo = buf->bo;
 449       buf->bo = NULL;
 450       buf->mm = NULL;
 451       nouveau_buffer_allocate(screen, buf, new_domain);
 452
 453       nv->copy_data(nv, buf->bo, buf->offset, new_domain,
 454                     bo, offset, old_domain, buf->base.width0);
 455
 456       nouveau_bo_ref(NULL, &bo);
 457       if (mm)
 458          release_allocation(&mm, screen->fence.current);
 459    } else
 460    if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) {
 461       if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
 462          return FALSE;
 463       if (!nouveau_buffer_upload(nv, buf, 0, buf->base.width0))
 464          return FALSE;
 465    } else
 466       return FALSE;
 467
 468    assert(buf->domain == new_domain);
 469    return TRUE;
 470 }
 471
 472 /* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
 473  * We'd like to only allocate @size bytes here, but then we'd have to rebase
 474  * the vertex indices ...
 475  */
 476 boolean
 477 nouveau_user_buffer_upload(struct nv04_resource *buf,
 478                            unsigned base, unsigned size)
 479 {
 480    struct nouveau_screen *screen = nouveau_screen(buf->base.screen);
 481    int ret;
 482
 483    assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY);
 484
 485    buf->base.width0 = base + size;
 486    if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART))
 487       return FALSE;
 488
 489    ret = nouveau_bo_map_range(buf->bo, buf->offset + base, size,
 490                               NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC);
 491    if (ret)
 492       return FALSE;
 493    memcpy(buf->bo->map, buf->data + base, size);
 494    nouveau_bo_unmap(buf->bo);
 495
 496    return TRUE;
 497 }