src/gallium/drivers/nouveau/nouveau_buffer.c

   1
   2 #include "util/u_inlines.h"
   3 #include "util/u_memory.h"
   4 #include "util/u_math.h"
   5
   6 #include "nouveau_screen.h"
   7 #include "nouveau_context.h"
   8 #include "nouveau_winsys.h"
   9 #include "nouveau_fence.h"
  10 #include "nouveau_buffer.h"
  11 #include "nouveau_mm.h"
  12
  13 struct nouveau_transfer {
  14    struct pipe_transfer base;
  15 };
  16
  17 static INLINE struct nouveau_transfer *
  18 nouveau_transfer(struct pipe_transfer *transfer)
  19 {
  20    return (struct nouveau_transfer *)transfer;
  21 }
  22
  23 static INLINE boolean
  24 nouveau_buffer_allocate(struct nouveau_screen *screen,
  25                         struct nv04_resource *buf, unsigned domain)
  26 {
  27    if (domain == NOUVEAU_BO_VRAM) {
  28       buf->mm = nouveau_mm_allocate(screen->mm_VRAM, buf->base.width0,
  29                                     &buf->bo, &buf->offset);
  30       if (!buf->bo)
  31          return nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_GART);
  32    } else
  33    if (domain == NOUVEAU_BO_GART) {
  34       buf->mm = nouveau_mm_allocate(screen->mm_GART, buf->base.width0,
  35                                     &buf->bo, &buf->offset);
  36       if (!buf->bo)
  37          return FALSE;
  38    }
  39    if (domain != NOUVEAU_BO_GART) {
  40       if (!buf->data) {
  41          buf->data = MALLOC(buf->base.width0);
  42          if (!buf->data)
  43             return FALSE;
  44       }
  45    }
  46    buf->domain = domain;
  47    return TRUE;
  48 }
  49
  50 static INLINE void
  51 release_allocation(struct nouveau_mm_allocation **mm,
  52                    struct nouveau_fence *fence)
  53 {
  54    nouveau_fence_work(fence, nouveau_mm_free_work, *mm);
  55    (*mm) = NULL;
  56 }
  57
  58 INLINE void
  59 nouveau_buffer_release_gpu_storage(struct nv04_resource *buf)
  60 {
  61    nouveau_bo_ref(NULL, &buf->bo);
  62
  63    if (buf->mm)
  64       release_allocation(&buf->mm, buf->fence);
  65
  66    buf->domain = 0;
  67 }
  68
  69 static INLINE boolean
  70 nouveau_buffer_reallocate(struct nouveau_screen *screen,
  71                           struct nv04_resource *buf, unsigned domain)
  72 {
  73    nouveau_buffer_release_gpu_storage(buf);
  74
  75    return nouveau_buffer_allocate(screen, buf, domain);
  76 }
  77
  78 static void
  79 nouveau_buffer_destroy(struct pipe_screen *pscreen,
  80                        struct pipe_resource *presource)
  81 {
  82    struct nv04_resource *res = nv04_resource(presource);
  83
  84    nouveau_buffer_release_gpu_storage(res);
  85
  86    if (res->data && !(res->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY))
  87       FREE(res->data);
  88
  89    FREE(res);
  90 }
  91
  92 /* Maybe just migrate to GART right away if we actually need to do this. */
  93 boolean
  94 nouveau_buffer_download(struct nouveau_context *nv, struct nv04_resource *buf,
  95                         unsigned start, unsigned size)
  96 {
  97    struct nouveau_mm_allocation *mm;
  98    struct nouveau_bo *bounce = NULL;
  99    uint32_t offset;
 100
 101    assert(buf->domain == NOUVEAU_BO_VRAM);
 102
 103    mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset);
 104    if (!bounce)
 105       return FALSE;
 106
 107    nv->copy_data(nv, bounce, offset, NOUVEAU_BO_GART,
 108                  buf->bo, buf->offset + start, NOUVEAU_BO_VRAM, size);
 109
 110    if (nouveau_bo_map_range(bounce, offset, size, NOUVEAU_BO_RD))
 111       return FALSE;
 112    memcpy(buf->data + start, bounce->map, size);
 113    nouveau_bo_unmap(bounce);
 114
 115    buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
 116
 117    nouveau_bo_ref(NULL, &bounce);
 118    if (mm)
 119       nouveau_mm_free(mm);
 120    return TRUE;
 121 }
 122
 123 static boolean
 124 nouveau_buffer_upload(struct nouveau_context *nv, struct nv04_resource *buf,
 125                       unsigned start, unsigned size)
 126 {
 127    struct nouveau_mm_allocation *mm;
 128    struct nouveau_bo *bounce = NULL;
 129    uint32_t offset;
 130
 131    if (size <= 192) {
 132       nv->push_data(nv, buf->bo, buf->offset + start, buf->domain,
 133                     size, buf->data + start);
 134       return TRUE;
 135    }
 136
 137    mm = nouveau_mm_allocate(nv->screen->mm_GART, size, &bounce, &offset);
 138    if (!bounce)
 139       return FALSE;
 140
 141    nouveau_bo_map_range(bounce, offset, size,
 142                         NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC);
 143    memcpy(bounce->map, buf->data + start, size);
 144    nouveau_bo_unmap(bounce);
 145
 146    nv->copy_data(nv, buf->bo, buf->offset + start, NOUVEAU_BO_VRAM,
 147                  bounce, offset, NOUVEAU_BO_GART, size);
 148
 149    nouveau_bo_ref(NULL, &bounce);
 150    if (mm)
 151       release_allocation(&mm, nv->screen->fence.current);
 152
 153    if (start == 0 && size == buf->base.width0)
 154       buf->status &= ~NOUVEAU_BUFFER_STATUS_GPU_WRITING;
 155    return TRUE;
 156 }
 157
 158 static struct pipe_transfer *
 159 nouveau_buffer_transfer_get(struct pipe_context *pipe,
 160                             struct pipe_resource *resource,
 161                             unsigned level, unsigned usage,
 162                             const struct pipe_box *box)
 163 {
 164    struct nv04_resource *buf = nv04_resource(resource);
 165    struct nouveau_context *nv = nouveau_context(pipe);
 166    struct nouveau_transfer *xfr = CALLOC_STRUCT(nouveau_transfer);
 167    if (!xfr)
 168       return NULL;
 169
 170    xfr->base.resource = resource;
 171    xfr->base.box.x = box->x;
 172    xfr->base.box.width = box->width;
 173    xfr->base.usage = usage;
 174
 175    if (buf->domain == NOUVEAU_BO_VRAM) {
 176       if (usage & PIPE_TRANSFER_READ) {
 177          if (buf->status & NOUVEAU_BUFFER_STATUS_GPU_WRITING)
 178             nouveau_buffer_download(nv, buf, 0, buf->base.width0);
 179       }
 180    }
 181
 182    return &xfr->base;
 183 }
 184
 185 static void
 186 nouveau_buffer_transfer_destroy(struct pipe_context *pipe,
 187                                 struct pipe_transfer *transfer)
 188 {
 189    struct nv04_resource *buf = nv04_resource(transfer->resource);
 190    struct nouveau_transfer *xfr = nouveau_transfer(transfer);
 191    struct nouveau_context *nv = nouveau_context(pipe);
 192
 193    if (xfr->base.usage & PIPE_TRANSFER_WRITE) {
 194       /* writing is worse */
 195       nouveau_buffer_adjust_score(nv, buf, -5000);
 196
 197       if (buf->domain == NOUVEAU_BO_VRAM) {
 198          nouveau_buffer_upload(nv, buf, transfer->box.x, transfer->box.width);
 199       }
 200
 201       if (buf->domain != 0 && (buf->base.bind & (PIPE_BIND_VERTEX_BUFFER |
 202                                                  PIPE_BIND_INDEX_BUFFER)))
 203          nouveau_context(pipe)->vbo_dirty = TRUE;
 204    }
 205
 206    FREE(xfr);
 207 }
 208
 209 static INLINE boolean
 210 nouveau_buffer_sync(struct nv04_resource *buf, unsigned rw)
 211 {
 212    if (rw == PIPE_TRANSFER_READ) {
 213       if (!buf->fence_wr)
 214          return TRUE;
 215       if (!nouveau_fence_wait(buf->fence_wr))
 216          return FALSE;
 217    } else {
 218       if (!buf->fence)
 219          return TRUE;
 220       if (!nouveau_fence_wait(buf->fence))
 221          return FALSE;
 222
 223       nouveau_fence_ref(NULL, &buf->fence);
 224    }
 225    nouveau_fence_ref(NULL, &buf->fence_wr);
 226
 227    return TRUE;
 228 }
 229
 230 static INLINE boolean
 231 nouveau_buffer_busy(struct nv04_resource *buf, unsigned rw)
 232 {
 233    if (rw == PIPE_TRANSFER_READ)
 234       return (buf->fence_wr && !nouveau_fence_signalled(buf->fence_wr));
 235    else
 236       return (buf->fence && !nouveau_fence_signalled(buf->fence));
 237 }
 238
 239 static void *
 240 nouveau_buffer_transfer_map(struct pipe_context *pipe,
 241                             struct pipe_transfer *transfer)
 242 {
 243    struct nouveau_transfer *xfr = nouveau_transfer(transfer);
 244    struct nv04_resource *buf = nv04_resource(transfer->resource);
 245    struct nouveau_bo *bo = buf->bo;
 246    uint8_t *map;
 247    int ret;
 248    uint32_t offset = xfr->base.box.x;
 249    uint32_t flags;
 250
 251    nouveau_buffer_adjust_score(nouveau_context(pipe), buf, -250);
 252
 253    if (buf->domain != NOUVEAU_BO_GART)
 254       return buf->data + offset;
 255
 256    if (buf->mm)
 257       flags = NOUVEAU_BO_NOSYNC | NOUVEAU_BO_RDWR;
 258    else
 259       flags = nouveau_screen_transfer_flags(xfr->base.usage);
 260
 261    offset += buf->offset;
 262
 263    ret = nouveau_bo_map_range(buf->bo, offset, xfr->base.box.width, flags);
 264    if (ret)
 265       return NULL;
 266    map = bo->map;
 267
 268    /* Unmap right now. Since multiple buffers can share a single nouveau_bo,
 269     * not doing so might make future maps fail or trigger "reloc while mapped"
 270     * errors. For now, mappings to userspace are guaranteed to be persistent.
 271     */
 272    nouveau_bo_unmap(bo);
 273
 274    if (buf->mm) {
 275       if (xfr->base.usage & PIPE_TRANSFER_DONTBLOCK) {
 276          if (nouveau_buffer_busy(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE))
 277             return NULL;
 278       } else
 279       if (!(xfr->base.usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
 280          nouveau_buffer_sync(buf, xfr->base.usage & PIPE_TRANSFER_READ_WRITE);
 281       }
 282    }
 283    return map;
 284 }
 285
 286
 287
 288 static void
 289 nouveau_buffer_transfer_flush_region(struct pipe_context *pipe,
 290                                      struct pipe_transfer *transfer,
 291                                      const struct pipe_box *box)
 292 {
 293    struct nv04_resource *res = nv04_resource(transfer->resource);
 294    struct nouveau_bo *bo = res->bo;
 295    unsigned offset = res->offset + transfer->box.x + box->x;
 296
 297    /* not using non-snoop system memory yet, no need for cflush */
 298    if (1)
 299       return;
 300
 301    /* XXX: maybe need to upload for VRAM buffers here */
 302
 303    nouveau_screen_bo_map_flush_range(pipe->screen, bo, offset, box->width);
 304 }
 305
 306 static void
 307 nouveau_buffer_transfer_unmap(struct pipe_context *pipe,
 308                               struct pipe_transfer *transfer)
 309 {
 310    /* we've called nouveau_bo_unmap right after map */
 311 }
 312
 313 const struct u_resource_vtbl nouveau_buffer_vtbl =
 314 {
 315    u_default_resource_get_handle,     /* get_handle */
 316    nouveau_buffer_destroy,               /* resource_destroy */
 317    NULL,                              /* is_resource_referenced */
 318    nouveau_buffer_transfer_get,          /* get_transfer */
 319    nouveau_buffer_transfer_destroy,      /* transfer_destroy */
 320    nouveau_buffer_transfer_map,          /* transfer_map */
 321    nouveau_buffer_transfer_flush_region, /* transfer_flush_region */
 322    nouveau_buffer_transfer_unmap,        /* transfer_unmap */
 323    u_default_transfer_inline_write    /* transfer_inline_write */
 324 };
 325
 326 struct pipe_resource *
 327 nouveau_buffer_create(struct pipe_screen *pscreen,
 328                       const struct pipe_resource *templ)
 329 {
 330    struct nouveau_screen *screen = nouveau_screen(pscreen);
 331    struct nv04_resource *buffer;
 332    boolean ret;
 333
 334    buffer = CALLOC_STRUCT(nv04_resource);
 335    if (!buffer)
 336       return NULL;
 337
 338    buffer->base = *templ;
 339    buffer->vtbl = &nouveau_buffer_vtbl;
 340    pipe_reference_init(&buffer->base.reference, 1);
 341    buffer->base.screen = pscreen;
 342
 343    if ((buffer->base.bind & screen->sysmem_bindings) == screen->sysmem_bindings)
 344       ret = nouveau_buffer_allocate(screen, buffer, 0);
 345    else
 346       ret = nouveau_buffer_allocate(screen, buffer, NOUVEAU_BO_GART);
 347
 348    if (ret == FALSE)
 349       goto fail;
 350
 351    return &buffer->base;
 352
 353 fail:
 354    FREE(buffer);
 355    return NULL;
 356 }
 357
 358
 359 struct pipe_resource *
 360 nouveau_user_buffer_create(struct pipe_screen *pscreen, void *ptr,
 361                            unsigned bytes, unsigned bind)
 362 {
 363    struct nv04_resource *buffer;
 364
 365    buffer = CALLOC_STRUCT(nv04_resource);
 366    if (!buffer)
 367       return NULL;
 368
 369    pipe_reference_init(&buffer->base.reference, 1);
 370    buffer->vtbl = &nouveau_buffer_vtbl;
 371    buffer->base.screen = pscreen;
 372    buffer->base.format = PIPE_FORMAT_R8_UNORM;
 373    buffer->base.usage = PIPE_USAGE_IMMUTABLE;
 374    buffer->base.bind = bind;
 375    buffer->base.width0 = bytes;
 376    buffer->base.height0 = 1;
 377    buffer->base.depth0 = 1;
 378
 379    buffer->data = ptr;
 380    buffer->status = NOUVEAU_BUFFER_STATUS_USER_MEMORY;
 381
 382    return &buffer->base;
 383 }
 384
 385 /* Like download, but for GART buffers. Merge ? */
 386 static INLINE boolean
 387 nouveau_buffer_data_fetch(struct nv04_resource *buf, struct nouveau_bo *bo,
 388                           unsigned offset, unsigned size)
 389 {
 390    if (!buf->data) {
 391       buf->data = MALLOC(size);
 392       if (!buf->data)
 393          return FALSE;
 394    }
 395    if (nouveau_bo_map_range(bo, offset, size, NOUVEAU_BO_RD))
 396       return FALSE;
 397    memcpy(buf->data, bo->map, size);
 398    nouveau_bo_unmap(bo);
 399
 400    return TRUE;
 401 }
 402
 403 /* Migrate a linear buffer (vertex, index, constants) USER -> GART -> VRAM. */
 404 boolean
 405 nouveau_buffer_migrate(struct nouveau_context *nv,
 406                        struct nv04_resource *buf, const unsigned new_domain)
 407 {
 408    struct nouveau_screen *screen = nv->screen;
 409    struct nouveau_bo *bo;
 410    const unsigned old_domain = buf->domain;
 411    unsigned size = buf->base.width0;
 412    unsigned offset;
 413    int ret;
 414
 415    assert(new_domain != old_domain);
 416
 417    if (new_domain == NOUVEAU_BO_GART && old_domain == 0) {
 418       if (!nouveau_buffer_allocate(screen, buf, new_domain))
 419          return FALSE;
 420       ret = nouveau_bo_map_range(buf->bo, buf->offset, size, NOUVEAU_BO_WR |
 421                                  NOUVEAU_BO_NOSYNC);
 422       if (ret)
 423          return ret;
 424       memcpy(buf->bo->map, buf->data, size);
 425       nouveau_bo_unmap(buf->bo);
 426       FREE(buf->data);
 427    } else
 428    if (old_domain != 0 && new_domain != 0) {
 429       struct nouveau_mm_allocation *mm = buf->mm;
 430
 431       if (new_domain == NOUVEAU_BO_VRAM) {
 432          /* keep a system memory copy of our data in case we hit a fallback */
 433          if (!nouveau_buffer_data_fetch(buf, buf->bo, buf->offset, size))
 434             return FALSE;
 435          debug_printf("migrating %u KiB to VRAM\n", size / 1024);
 436       }
 437
 438       offset = buf->offset;
 439       bo = buf->bo;
 440       buf->bo = NULL;
 441       buf->mm = NULL;
 442       nouveau_buffer_allocate(screen, buf, new_domain);
 443
 444       nv->copy_data(nv, buf->bo, buf->offset, new_domain,
 445                     bo, offset, old_domain, buf->base.width0);
 446
 447       nouveau_bo_ref(NULL, &bo);
 448       if (mm)
 449          release_allocation(&mm, screen->fence.current);
 450    } else
 451    if (new_domain == NOUVEAU_BO_VRAM && old_domain == 0) {
 452       if (!nouveau_buffer_allocate(screen, buf, NOUVEAU_BO_VRAM))
 453          return FALSE;
 454       if (!nouveau_buffer_upload(nv, buf, 0, buf->base.width0))
 455          return FALSE;
 456    } else
 457       return FALSE;
 458
 459    assert(buf->domain == new_domain);
 460    return TRUE;
 461 }
 462
 463 /* Migrate data from glVertexAttribPointer(non-VBO) user buffers to GART.
 464  * We'd like to only allocate @size bytes here, but then we'd have to rebase
 465  * the vertex indices ...
 466  */
 467 boolean
 468 nouveau_user_buffer_upload(struct nv04_resource *buf,
 469                            unsigned base, unsigned size)
 470 {
 471    struct nouveau_screen *screen = nouveau_screen(buf->base.screen);
 472    int ret;
 473
 474    assert(buf->status & NOUVEAU_BUFFER_STATUS_USER_MEMORY);
 475
 476    buf->base.width0 = base + size;
 477    if (!nouveau_buffer_reallocate(screen, buf, NOUVEAU_BO_GART))
 478       return FALSE;
 479
 480    ret = nouveau_bo_map_range(buf->bo, buf->offset + base, size,
 481                               NOUVEAU_BO_WR | NOUVEAU_BO_NOSYNC);
 482    if (ret)
 483       return FALSE;
 484    memcpy(buf->bo->map, buf->data + base, size);
 485    nouveau_bo_unmap(buf->bo);
 486
 487    return TRUE;
 488 }