src/gallium/drivers/nouveau/nouveau_buffer.c

   1
   2 #include "util/u_inlines.h"
   3 #include "util/u_memory.h"
   4 #include "util/u_math.h"
   5
   6 #include "nouveau_screen.h"
   7 #include "nouveau_context.h"
   8 #include "nouveau_winsys.h"
   9 #include "nouveau_fence.h"
  10 #include "nouveau_buffer.h"
  11 #include "nouveau_mm.h"
  12
  13 struct nouveau_transfer {
  14    struct pipe_transfer base;
  15 };
  16
  17 static INLINE struct nouveau_transfer *
  18 nouveau_transfer(struct pipe_transfer *transfer)
  19 {
  20    return (struct nouveau_transfer *)transfer;
  21 }
  22
  23 static INLINE boolean
  24 nouveau_buffer_allocate(struct nouveau_screen *screen,
  25                         struct nv04_resource *buf, unsigned domain)
  26 {
  27    if (domain == NOUVEAU_BO_VRAM) {
  28       buf->mm = nouveau_mm_allocate(screen->mm_VRAM, buf->base.width0,
  29                                     &buf->bo, &buf->offset);
  30       if (!buf->bo)
  31          return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
  32    } else
  33    if (domain == NOUVEAU_BO_GART) {
  34       buf->mm = nouveau_mm_allocate(screen->mm_GART, buf->base.width0,
  35                                     &buf->bo, &buf->offset);
  36       if (!buf->bo)
  37          return FALSE;
  38    }
  39    if (domain != NOUVEAU_BO_GART) {
  40       if (!buf->data) {
  41          buf->data = MALLOC(buf->base.width0);
  42          if (!buf->data)
  43             return FALSE;
  44       }
  45    }
  46    buf->domain = domain;
  47    return TRUE;
  48 }
  49
  50 static INLINE void
  51 release_allocation(struct nouveau_mm_allocation **mm,
  52                    struct nouveau_fence *fence)
  53 {
  54    nouveau_fence_work(fence, nouveau_mm_free_work, *mm);
  55    (*mm) = NULL;
  56 }
  57
  58 INLINE void
  59 nouveau_buffer_release_gpu_storage(struct nv04_resource *buf)
  60 {
  61    nouveau_bo_ref(NULL, &buf->bo);
  62
  63    if (buf->mm)
  64       release_allocation(&buf->mm, buf->fence);
  65
  66    buf->domain = 0;
  67 }
  68
  69 static INLINE boolean
  70 nouveau_buffer_reallocate(struct nouveau_screen *screen,
  71                           struct nv04_resource *buf, unsigned domain)
  72 {
  73    nouveau_buffer_release_gpu_storage(buf);
  74
  75    return nouveau_buffer_allocate(screen, buf, domain);
  76 }
  77
  78 static void
  79 nouveau_buffer_destroy(struct pipe_screen *pscreen,
  80                        struct pipe_resource *presource)
  81 {
  82    struct nv04_resource *res = nv04_resource(presource);
  83
  84    nouveau_buffer_release_gpu_storage(res);
  85
  86    if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
  87       FREE(res->data);
  88
  89    FREE(res);
  90 }
  91
  92 /* Maybe just migrate to GART right away if we actually need to do this. */
  93 boolean
  94 nouveau_buffer_download(struct nouveau_context *nv, struct nv04_resource *buf,
  95                         unsigned start, unsigned size)
  96 {
  97    struct nouveau_mm_allocation *mm;
  98    struct nouveau_bo *bounce = NULL;
  99    uint32_t offset;
 100
 101    assert(buf->domain == NOUVEAU_BO_VRAM);
 102
 103    mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset);
 104    if (!bounce)
 105       return FALSE;
 106
 107    nv->copy_data(nv, bounce, offset, NOUVEAU_BO_GART,
 108                  buf->bo, buf->offset + start, NOUVEAU_BO_VRAM, size);
 109
 110    if (nouveau_bo_map_range(bounce, offset, size, NOUVEAU_BO_RD))
 111       return FALSE;
 112    memcpy(buf->data + start, bounce->map, size);
 113    nouveau_bo_unmap(bounce);
 114
 115    buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
 116
 117    nouveau_bo_ref(NULL, &bounce);
 118    if (mm)
 119       nouveau_mm_free(mm);
 120    return TRUE;
 121 }
 122
 123 static boolean
 124 nouveau_buffer_upload(struct nouveau_context *nv, struct nv04_resource *buf,
 125                       unsigned start, unsigned size)
 126 {
 127    struct nouveau_mm_allocation *mm;
 128    struct nouveau_bo *bounce = NULL;
 129    uint32_t offset;
 130
 131    if (size <= 192) {
 132       nv->push_data(nv, buf->bo, buf->offset + start, buf->domain,
 133                     size, buf->data + start);
 134       return TRUE;
 135    }
 136
 137    mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset);
 138    if (!bounce)
 139       return FALSE;
 140
 141    nouveau_bo_map_range(bounce, offset, size,
 142                         NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC);
 143    memcpy(bounce->map, buf->data + start, size);
 144    nouveau_bo_unmap(bounce);
 145
 146    nv->copy_data(nv, buf->bo, buf->offset + start, NOUVEAU_BO_VRAM,
 147                  bounce, offset, NOUVEAU_BO_GART, size);
 148
 149    nouveau_bo_ref(NULL, &bounce);
 150    if (mm)
 151       release_allocation(&mm, nv->screen->fence.current);
 152
 153    if (start == 0 && size == buf->base.width0)
 154       buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
 155    return TRUE;
 156 }
 157
 158 static struct pipe_transfer *
 159 nouveau_buffer_transfer_get(struct pipe_context *pipe,
 160                             struct pipe_resource *resource,
 161                             unsigned level, unsigned usage,
 162                             const struct pipe_box *box)
 163 {
 164    struct nv04_resource *buf = nv04_resource(resource);
 165    struct nouveau_context *nv = nouveau_context(pipe);
 166    struct nouveau_transfer *xfr = CALLOC_STRUCT(nouveau_transfer);
 167    if (!xfr)
 168       return NULL;
 169
 170    xfr->base.resource = resource;
 171    xfr->base.box.x = box->x;
 172    xfr->base.box.width = box->width;
 173    xfr->base.usage = usage;
 174
 175    if (buf->domain == NOUVEAU_BO_VRAM) {
 176       if (usage & PIPE_TRANSFER_READ) {
 177          if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING)
 178             nouveau_buffer_download(nv, buf, 0, buf->base.width0);
 179       }
 180    }
 181
 182    return &xfr->base;
 183 }
 184
 185 static void
 186 nouveau_buffer_transfer_destroy(struct pipe_context *pipe,
 187                                 struct pipe_transfer *transfer)
 188 {
 189    struct nv04_resource *buf = nv04_resource(transfer->resource);
 190    struct nouveau_transfer *xfr = nouveau_transfer(transfer);
 191    struct nouveau_context *nv = nouveau_context(pipe);
 192
 193    if (xfr->base.usage & PIPE_TRANSFER_WRITE) {
 194       /* writing is worse */
 195       nouveau_buffer_adjust_score(nv, buf, -5000);
 196
 197       if (buf->domain == NOUVEAU_BO_VRAM) {
 198          nouveau_buffer_upload(nv, buf, transfer->box.x, transfer->box.width);
 199       }
 200
 201       if (buf->domain != 0 && (buf->base.bind & (PIPE_BIND_VERTEX_BUFFER |
 202                                                  PIPE_BIND_INDEX_BUFFER)))
 203          nouveau_context(pipe)->vbo_dirty = TRUE;
 204    }
 205
 206    FREE(xfr);
 207 }
 208
 209 static INLINE boolean
 210 nouveau_buffer_sync(struct nv04_resource *buf, unsigned rw)
 211 {
 212    if (rw == PIPE_TRANSFER_READ) {
 213       if (!buf->fence_wr)
 214          return TRUE;
 215       if (!nouveau_fence_wait(buf->fence_wr))
 216          return FALSE;
 217    } else {
 218       if (!buf->fence)
 219          return TRUE;
 220       if (!nouveau_fence_wait(buf->fence))
 221          return FALSE;
 222
 223       nouveau_fence_ref(NULL, &buf->fence);
 224    }
 225    nouveau_fence_ref(NULL, &buf->fence_wr);
 226
 227    return TRUE;
 228 }
 229
 230 static INLINE boolean
 231 nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw)
 232 {
 233    if (rw == PIPE_TRANSFER_READ)
 234       return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr));
 235    else
 236       return (buf->fence && !nouveau_fence_signalled(buf->fence));
 237 }
 238
 239 static void *
 240 nouveau_buffer_transfer_map(struct pipe_context *pipe,
 241                             struct pipe_transfer *transfer)
 242 {
 243    struct nouveau_transfer *xfr = nouveau_transfer(transfer);
 244    struct nv04_resource *buf = nv04_resource(transfer->resource);
 245    struct nouveau_bo *bo = buf->bo;
 246    uint8_t *map;
 247    int ret;
 248    uint32_t offset = xfr->base.box.x;
 249    uint32_t flags;
 250
 251    nouveau_buffer_adjust_score(nouveau_context(pipe), buf, -250);
 252
 253    if (buf->domain != NOUVEAU_BO_GART)
 254       return buf->data + offset;
 255
 256    if (buf->mm)
 257       flags = NOUVEAU_BO_NOSYNC | NOUVEAU_BO_RDWR;
 258    else
 259       flags = nouveau_screen_transfer_flags(xfr->base.usage);
 260
 261    offset += buf->offset;
 262
 263    ret = nouveau_bo_map_range(buf->bo, offset, xfr->base.box.width, flags);
 264    if (ret)
 265       return NULL;
 266    map = bo->map;
 267
 268    /* Unmap right now. Since multiple buffers can share a single nouveau_bo,
 269     * not doing so might make future maps fail or trigger "reloc while mapped"
 270     * errors. For now, mappings to userspace are guaranteed to be persistent.
 271     */
 272    nouveau_bo_unmap(bo);
 273
 274    if (buf->mm) {
 275       if (xfr->base.usage & PIPE_TRANSFER_DONTBLOCK) {
 276          if (nouveau_buffer_busy(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE))
 277             return NULL;
 278       } else
 279       if (!(xfr->base.usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
 280          nouveau_buffer_sync(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE);
 281       }
 282    }
 283    return map;
 284 }
 285
 286
 287
 288 static void
 289 nouveau_buffer_transfer_flush_region(struct pipe_context *pipe,
 290                                      struct pipe_transfer *transfer,
 291                                      const struct pipe_box *box)
 292 {
 293    struct nv04_resource *res = nv04_resource(transfer->resource);
 294    struct nouveau_bo *bo = res->bo;
 295    unsigned offset = res->offset + transfer->box.x + box->x;
 296
 297    /* not using non-snoop system memory yet, no need for cflush */
 298    if (1)
 299       return;
 300
 301    /* XXX: maybe need to upload for VRAM buffers here */
 302
 303    nouveau_screen_bo_map_flush_range(pipe->screen, bo, offset, box->width);
 304 }
 305
 306 static void
 307 nouveau_buffer_transfer_unmap(struct pipe_context *pipe,
 308                               struct pipe_transfer *transfer)
 309 {
 310    /* we've called nouveau_bo_unmap right after map */
 311 }
 312
 313 const struct u_resource_vtbl nouveau_buffer_vtbl =
 314 {
 315    u_default_resource_get_handle,     /* get_handle */
 316    nouveau_buffer_destroy,               /* resource_destroy */
 317    nouveau_buffer_transfer_get,          /* get_transfer */
 318    nouveau_buffer_transfer_destroy,      /* transfer_destroy */
 319    nouveau_buffer_transfer_map,          /* transfer_map */
 320    nouveau_buffer_transfer_flush_region, /* transfer_flush_region */
 321    nouveau_buffer_transfer_unmap,        /* transfer_unmap */
 322    u_default_transfer_inline_write    /* transfer_inline_write */
 323 };
 324
 325 struct pipe_resource *
 326 nouveau_buffer_create(struct pipe_screen *pscreen,
 327                       const struct pipe_resource *templ)
 328 {
 329    struct nouveau_screen *screen = nouveau_screen(pscreen);
 330    struct nv04_resource *buffer;
 331    boolean ret;
 332
 333    buffer = CALLOC_STRUCT(nv04_resource);
 334    if (!buffer)
 335       return NULL;
 336
 337    buffer->base = *templ;
 338    buffer->vtbl = &nouveau_buffer_vtbl;
 339    pipe_reference_init(&buffer->base.reference, 1);
 340    buffer->base.screen = pscreen;
 341
 342    if ((buffer->base.bind & screen->sysmem_bindings) == screen->sysmem_bindings)
 343       ret = nouveau_buffer_allocate(screen, buffer, 0);
 344    else
 345       ret = nouveau_buffer_allocate(screen, buffer, NOUVEAU_BO_GART);
 346
 347    if (ret == FALSE)
 348       goto fail;
 349
 350    return &buffer->base;
 351
 352 fail:
 353    FREE(buffer);
 354    return NULL;
 355 }
 356
 357
 358 struct pipe_resource *
 359 nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr,
 360                            unsigned bytes, unsigned bind)
 361 {
 362    struct nv04_resource *buffer;
 363
 364    buffer = CALLOC_STRUCT(nv04_resource);
 365    if (!buffer)
 366       return NULL;
 367
 368    pipe_reference_init(&buffer->base.reference, 1);
 369    buffer->vtbl = &nouveau_buffer_vtbl;
 370    buffer->base.screen = pscreen;
 371    buffer->base.format = PIPE_FORMAT_R8_UNORM;
 372    buffer->base.usage = PIPE_USAGE_IMMUTABLE;
 373    buffer->base.bind = bind;
 374    buffer->base.width0 = bytes;
 375    buffer->base.height0 = 1;
 376    buffer->base.depth0 = 1;
 377
 378    buffer->data = ptr;
 379    buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY;
 380
 381    return &buffer->base;
 382 }
 383
 384 /* Like download, but for GART buffers. Merge ? */
 385 static INLINE boolean
 386 nouveau_buffer_data_fetch(struct nv04_resource *buf, struct nouveau_bo *bo,
 387                           unsigned offset, unsigned size)
 388 {
 389    if (!buf->data) {
 390       buf->data = MALLOC(size);
 391       if (!buf->data)
 392          return FALSE;
 393    }
 394    if (nouveau_bo_map_range(bo, offset, size, NOUVEAU_BO_RD))
 395       return FALSE;
 396    memcpy(buf->data, bo->map, size);
 397    nouveau_bo_unmap(bo);
 398
 399    return TRUE;
 400 }
 401
 402 /* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
 403 boolean
 404 nouveau_buffer_migrate(struct nouveau_context *nv,
 405                        struct nv04_resource *buf, const unsigned new_domain)
 406 {
 407    struct nouveau_screen *screen = nv->screen;
 408    struct nouveau_bo *bo;
 409    const unsigned old_domain = buf->domain;
 410    unsigned size = buf->base.width0;
 411    unsigned offset;
 412    int ret;
 413
 414    assert(new_domain != old_domain);
 415
 416    if (new_domain == NOUVEAU_BO_GART && old_domain == 0) {
 417       if (!nouveau_buffer_allocate(screen, buf, new_domain))
 418          return FALSE;
 419       ret = nouveau_bo_map_range(buf->bo, buf->offset, size, NOUVEAU_BO_WR |
 420                                  NOUVEAU_BO_NOSYNC);
 421       if (ret)
 422          return ret;
 423       memcpy(buf->bo->map, buf->data, size);
 424       nouveau_bo_unmap(buf->bo);
 425       FREE(buf->data);
 426    } else
 427    if (old_domain != 0 && new_domain != 0) {
 428       struct nouveau_mm_allocation *mm = buf->mm;
 429
 430       if (new_domain == NOUVEAU_BO_VRAM) {
 431          /* keep a system memory copy of our data in case we hit a fallback */
 432          if (!nouveau_buffer_data_fetch(buf, buf->bo, buf->offset, size))
 433             return FALSE;
 434          debug_printf("migrating %u KiB to VRAM\n", size / 1024);
 435       }
 436
 437       offset = buf->offset;
 438       bo = buf->bo;
 439       buf->bo = NULL;
 440       buf->mm = NULL;
 441       nouveau_buffer_allocate(screen, buf, new_domain);
 442
 443       nv->copy_data(nv, buf->bo, buf->offset, new_domain,
 444                     bo, offset, old_domain, buf->base.width0);
 445
 446       nouveau_bo_ref(NULL, &bo);
 447       if (mm)
 448          release_allocation(&mm, screen->fence.current);
 449    } else
 450    if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) {
 451       if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
 452          return FALSE;
 453       if (!nouveau_buffer_upload(nv, buf, 0, buf->base.width0))
 454          return FALSE;
 455    } else
 456       return FALSE;
 457
 458    assert(buf->domain == new_domain);
 459    return TRUE;
 460 }
 461
 462 /* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
 463  * We'd like to only allocate @size bytes here, but then we'd have to rebase
 464  * the vertex indices ...
 465  */
 466 boolean
 467 nouveau_user_buffer_upload(struct nv04_resource *buf,
 468                            unsigned base, unsigned size)
 469 {
 470    struct nouveau_screen *screen = nouveau_screen(buf->base.screen);
 471    int ret;
 472
 473    assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY);
 474
 475    buf->base.width0 = base + size;
 476    if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART))
 477       return FALSE;
 478
 479    ret = nouveau_bo_map_range(buf->bo, buf->offset + base, size,
 480                               NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC);
 481    if (ret)
 482       return FALSE;
 483    memcpy(buf->bo->map, buf->data + base, size);
 484    nouveau_bo_unmap(buf->bo);
 485
 486    return TRUE;
 487 }