src/gallium/drivers/nvc0/nvc0_transfer.c

   1
   2 #include "util/u_format.h"
   3
   4 #include "nvc0_context.h"
   5
   6 #include "nv50/nv50_defs.xml.h"
   7
   8 struct nvc0_transfer {
   9    struct pipe_transfer base;
  10    struct nv50_m2mf_rect rect[2];
  11    uint32_t nblocksx;
  12    uint16_t nblocksy;
  13    uint16_t nlayers;
  14 };
  15
  16 static void
  17 nvc0_m2mf_transfer_rect(struct nvc0_context *nvc0,
  18                         const struct nv50_m2mf_rect *dst,
  19                         const struct nv50_m2mf_rect *src,
  20                         uint32_t nblocksx, uint32_t nblocksy)
  21 {
  22    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
  23    struct nouveau_bufctx *bctx = nvc0->bufctx;
  24    const int cpp = dst->cpp;
  25    uint32_t src_ofst = src->base;
  26    uint32_t dst_ofst = dst->base;
  27    uint32_t height = nblocksy;
  28    uint32_t sy = src->y;
  29    uint32_t dy = dst->y;
  30    uint32_t exec = (1 << 20);
  31
  32    assert(dst->cpp == src->cpp);
  33
  34    nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD);
  35    nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR);
  36    nouveau_pushbuf_bufctx(push, bctx);
  37    nouveau_pushbuf_validate(push);
  38
  39    if (nouveau_bo_memtype(src->bo)) {
  40       BEGIN_NVC0(push, NVC0_M2MF(TILING_MODE_IN), 5);
  41       PUSH_DATA (push, src->tile_mode);
  42       PUSH_DATA (push, src->width * cpp);
  43       PUSH_DATA (push, src->height);
  44       PUSH_DATA (push, src->depth);
  45       PUSH_DATA (push, src->z);
  46    } else {
  47       src_ofst += src->y * src->pitch + src->x * cpp;
  48
  49       BEGIN_NVC0(push, NVC0_M2MF(PITCH_IN), 1);
  50       PUSH_DATA (push, src->width * cpp);
  51
  52       exec |= NVC0_M2MF_EXEC_LINEAR_IN;
  53    }
  54
  55    if (nouveau_bo_memtype(dst->bo)) {
  56       BEGIN_NVC0(push, NVC0_M2MF(TILING_MODE_OUT), 5);
  57       PUSH_DATA (push, dst->tile_mode);
  58       PUSH_DATA (push, dst->width * cpp);
  59       PUSH_DATA (push, dst->height);
  60       PUSH_DATA (push, dst->depth);
  61       PUSH_DATA (push, dst->z);
  62    } else {
  63       dst_ofst += dst->y * dst->pitch + dst->x * cpp;
  64
  65       BEGIN_NVC0(push, NVC0_M2MF(PITCH_OUT), 1);
  66       PUSH_DATA (push, dst->width * cpp);
  67
  68       exec |= NVC0_M2MF_EXEC_LINEAR_OUT;
  69    }
  70
  71    while (height) {
  72       int line_count = height > 2047 ? 2047 : height;
  73
  74       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_IN_HIGH), 2);
  75       PUSH_DATAh(push, src->bo->offset + src_ofst);
  76       PUSH_DATA (push, src->bo->offset + src_ofst);
  77
  78       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
  79       PUSH_DATAh(push, dst->bo->offset + dst_ofst);
  80       PUSH_DATA (push, dst->bo->offset + dst_ofst);
  81
  82       if (!(exec & NVC0_M2MF_EXEC_LINEAR_IN)) {
  83          BEGIN_NVC0(push, NVC0_M2MF(TILING_POSITION_IN_X), 2);
  84          PUSH_DATA (push, src->x * cpp);
  85          PUSH_DATA (push, sy);
  86       } else {
  87          src_ofst += line_count * src->pitch;
  88       }
  89       if (!(exec & NVC0_M2MF_EXEC_LINEAR_OUT)) {
  90          BEGIN_NVC0(push, NVC0_M2MF(TILING_POSITION_OUT_X), 2);
  91          PUSH_DATA (push, dst->x * cpp);
  92          PUSH_DATA (push, dy);
  93       } else {
  94          dst_ofst += line_count * dst->pitch;
  95       }
  96
  97       BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
  98       PUSH_DATA (push, nblocksx * cpp);
  99       PUSH_DATA (push, line_count);
 100       BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
 101       PUSH_DATA (push, exec);
 102
 103       height -= line_count;
 104       sy += line_count;
 105       dy += line_count;
 106    }
 107
 108    nouveau_bufctx_reset(bctx, 0);
 109 }
 110
 111 static void
 112 nve4_m2mf_transfer_rect(struct nvc0_context *nvc0,
 113                         const struct nv50_m2mf_rect *dst,
 114                         const struct nv50_m2mf_rect *src,
 115                         uint32_t nblocksx, uint32_t nblocksy)
 116 {
 117    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 118    struct nouveau_bufctx *bctx = nvc0->bufctx;
 119    uint32_t exec;
 120    uint32_t src_base = src->base;
 121    uint32_t dst_base = dst->base;
 122    const int cpp = dst->cpp;
 123
 124    assert(dst->cpp == src->cpp);
 125
 126    nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR);
 127    nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD);
 128    nouveau_pushbuf_bufctx(push, bctx);
 129    nouveau_pushbuf_validate(push);
 130
 131    exec = 0x200 /* 2D_ENABLE */ | 0x6 /* UNK */;
 132
 133    if (!nouveau_bo_memtype(dst->bo)) {
 134       assert(!dst->z);
 135       dst_base += dst->y * dst->pitch + dst->x * cpp;
 136       exec |= 0x100; /* DST_MODE_2D_LINEAR */
 137    }
 138    if (!nouveau_bo_memtype(src->bo)) {
 139       assert(!src->z);
 140       src_base += src->y * src->pitch + src->x * cpp;
 141       exec |= 0x080; /* SRC_MODE_2D_LINEAR */
 142    }
 143
 144    BEGIN_NVC0(push, SUBC_COPY(0x070c), 6);
 145    PUSH_DATA (push, 0x1000 | dst->tile_mode);
 146    PUSH_DATA (push, dst->pitch);
 147    PUSH_DATA (push, dst->height);
 148    PUSH_DATA (push, dst->depth);
 149    PUSH_DATA (push, dst->z);
 150    PUSH_DATA (push, (dst->y << 16) | (dst->x * cpp));
 151
 152    BEGIN_NVC0(push, SUBC_COPY(0x0728), 6);
 153    PUSH_DATA (push, 0x1000 | src->tile_mode);
 154    PUSH_DATA (push, src->pitch);
 155    PUSH_DATA (push, src->height);
 156    PUSH_DATA (push, src->depth);
 157    PUSH_DATA (push, src->z);
 158    PUSH_DATA (push, (src->y << 16) | (src->x * cpp));
 159
 160    BEGIN_NVC0(push, SUBC_COPY(0x0400), 8);
 161    PUSH_DATAh(push, src->bo->offset + src_base);
 162    PUSH_DATA (push, src->bo->offset + src_base);
 163    PUSH_DATAh(push, dst->bo->offset + dst_base);
 164    PUSH_DATA (push, dst->bo->offset + dst_base);
 165    PUSH_DATA (push, src->pitch);
 166    PUSH_DATA (push, dst->pitch);
 167    PUSH_DATA (push, nblocksx * cpp);
 168    PUSH_DATA (push, nblocksy);
 169
 170    BEGIN_NVC0(push, SUBC_COPY(0x0300), 1);
 171    PUSH_DATA (push, exec);
 172
 173    nouveau_bufctx_reset(bctx, 0);
 174 }
 175
 176 void
 177 nvc0_m2mf_push_linear(struct nouveau_context *nv,
 178                       struct nouveau_bo *dst, unsigned offset, unsigned domain,
 179                       unsigned size, const void *data)
 180 {
 181    struct nvc0_context *nvc0 = nvc0_context(&nv->pipe);
 182    struct nouveau_pushbuf *push = nv->pushbuf;
 183    uint32_t *src = (uint32_t *)data;
 184    unsigned count = (size + 3) / 4;
 185
 186    nouveau_bufctx_refn(nvc0->bufctx, 0, dst, domain | NOUVEAU_BO_WR);
 187    nouveau_pushbuf_bufctx(push, nvc0->bufctx);
 188    nouveau_pushbuf_validate(push);
 189
 190    while (count) {
 191       unsigned nr;
 192
 193       if (!PUSH_SPACE(push, 16))
 194          break;
 195       nr = PUSH_AVAIL(push);
 196       assert(nr >= 16);
 197       nr = MIN2(count, nr - 9);
 198       nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN);
 199
 200       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
 201       PUSH_DATAh(push, dst->offset + offset);
 202       PUSH_DATA (push, dst->offset + offset);
 203       BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
 204       PUSH_DATA (push, MIN2(size, nr * 4));
 205       PUSH_DATA (push, 1);
 206       BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
 207       PUSH_DATA (push, 0x100111);
 208
 209       /* must not be interrupted (trap on QUERY fence, 0x50 works however) */
 210       BEGIN_NIC0(push, NVC0_M2MF(DATA), nr);
 211       PUSH_DATAp(push, src, nr);
 212
 213       count -= nr;
 214       src += nr;
 215       offset += nr * 4;
 216       size -= nr * 4;
 217    }
 218
 219    nouveau_bufctx_reset(nvc0->bufctx, 0);
 220 }
 221
 222 void
 223 nve4_p2mf_push_linear(struct nouveau_context *nv,
 224                       struct nouveau_bo *dst, unsigned offset, unsigned domain,
 225                       unsigned size, const void *data)
 226 {
 227    struct nvc0_context *nvc0 = nvc0_context(&nv->pipe);
 228    struct nouveau_pushbuf *push = nv->pushbuf;
 229    uint32_t *src = (uint32_t *)data;
 230    unsigned count = (size + 3) / 4;
 231
 232    nouveau_bufctx_refn(nvc0->bufctx, 0, dst, domain | NOUVEAU_BO_WR);
 233    nouveau_pushbuf_bufctx(push, nvc0->bufctx);
 234    nouveau_pushbuf_validate(push);
 235
 236    while (count) {
 237       unsigned nr;
 238
 239       if (!PUSH_SPACE(push, 16))
 240          break;
 241       nr = PUSH_AVAIL(push);
 242       assert(nr >= 16);
 243       nr = MIN2(count, nr - 8);
 244       nr = MIN2(nr, (NV04_PFIFO_MAX_PACKET_LEN - 1));
 245
 246       BEGIN_NVC0(push, NVE4_P2MF(DST_ADDRESS_HIGH), 2);
 247       PUSH_DATAh(push, dst->offset + offset);
 248       PUSH_DATA (push, dst->offset + offset);
 249       BEGIN_NVC0(push, NVE4_P2MF(LINE_LENGTH_IN), 2);
 250       PUSH_DATA (push, MIN2(size, nr * 4));
 251       PUSH_DATA (push, 1);
 252       /* must not be interrupted (trap on QUERY fence, 0x50 works however) */
 253       BEGIN_1IC0(push, NVE4_P2MF(EXEC), nr + 1);
 254       PUSH_DATA (push, 0x1001);
 255       PUSH_DATAp(push, src, nr);
 256
 257       count -= nr;
 258       src += nr;
 259       offset += nr * 4;
 260       size -= nr * 4;
 261    }
 262
 263    nouveau_bufctx_reset(nvc0->bufctx, 0);
 264 }
 265
 266 static void
 267 nvc0_m2mf_copy_linear(struct nouveau_context *nv,
 268                       struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
 269                       struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
 270                       unsigned size)
 271 {
 272    struct nouveau_pushbuf *push = nv->pushbuf;
 273    struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
 274
 275    nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD);
 276    nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR);
 277    nouveau_pushbuf_bufctx(push, bctx);
 278    nouveau_pushbuf_validate(push);
 279
 280    while (size) {
 281       unsigned bytes = MIN2(size, 1 << 17);
 282
 283       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
 284       PUSH_DATAh(push, dst->offset + dstoff);
 285       PUSH_DATA (push, dst->offset + dstoff);
 286       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_IN_HIGH), 2);
 287       PUSH_DATAh(push, src->offset + srcoff);
 288       PUSH_DATA (push, src->offset + srcoff);
 289       BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
 290       PUSH_DATA (push, bytes);
 291       PUSH_DATA (push, 1);
 292       BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
 293       PUSH_DATA (push, (1 << NVC0_M2MF_EXEC_INC__SHIFT) |
 294                  NVC0_M2MF_EXEC_LINEAR_IN | NVC0_M2MF_EXEC_LINEAR_OUT);
 295
 296       srcoff += bytes;
 297       dstoff += bytes;
 298       size -= bytes;
 299    }
 300
 301    nouveau_bufctx_reset(bctx, 0);
 302 }
 303
 304 static void
 305 nve4_m2mf_copy_linear(struct nouveau_context *nv,
 306                       struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
 307                       struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
 308                       unsigned size)
 309 {
 310    struct nouveau_pushbuf *push = nv->pushbuf;
 311    struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
 312
 313    nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD);
 314    nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR);
 315    nouveau_pushbuf_bufctx(push, bctx);
 316    nouveau_pushbuf_validate(push);
 317
 318    BEGIN_NVC0(push, SUBC_COPY(0x0400), 4);
 319    PUSH_DATAh(push, src->offset + srcoff);
 320    PUSH_DATA (push, src->offset + srcoff);
 321    PUSH_DATAh(push, dst->offset + dstoff);
 322    PUSH_DATA (push, dst->offset + dstoff);
 323    BEGIN_NVC0(push, SUBC_COPY(0x0418), 1);
 324    PUSH_DATA (push, size);
 325    BEGIN_NVC0(push, SUBC_COPY(0x0300), 1);
 326    PUSH_DATA (push, 0x186);
 327
 328    nouveau_bufctx_reset(bctx, 0);
 329 }
 330
 331 void *
 332 nvc0_miptree_transfer_map(struct pipe_context *pctx,
 333                           struct pipe_resource *res,
 334                           unsigned level,
 335                           unsigned usage,
 336                           const struct pipe_box *box,
 337                           struct pipe_transfer **ptransfer)
 338 {
 339    struct nvc0_context *nvc0 = nvc0_context(pctx);
 340    struct nouveau_device *dev = nvc0->screen->base.device;
 341    struct nv50_miptree *mt = nv50_miptree(res);
 342    struct nvc0_transfer *tx;
 343    uint32_t size;
 344    int ret;
 345    unsigned flags = 0;
 346
 347    if (usage & PIPE_TRANSFER_MAP_DIRECTLY)
 348       return NULL;
 349
 350    tx = CALLOC_STRUCT(nvc0_transfer);
 351    if (!tx)
 352       return NULL;
 353
 354    pipe_resource_reference(&tx->base.resource, res);
 355
 356    tx->base.level = level;
 357    tx->base.usage = usage;
 358    tx->base.box = *box;
 359
 360    if (util_format_is_plain(res->format)) {
 361       tx->nblocksx = box->width << mt->ms_x;
 362       tx->nblocksy = box->height << mt->ms_y;
 363    } else {
 364       tx->nblocksx = util_format_get_nblocksx(res->format, box->width);
 365       tx->nblocksy = util_format_get_nblocksy(res->format, box->height);
 366    }
 367    tx->nlayers = box->depth;
 368
 369    tx->base.stride = tx->nblocksx * util_format_get_blocksize(res->format);
 370    tx->base.layer_stride = tx->nblocksy * tx->base.stride;
 371
 372    nv50_m2mf_rect_setup(&tx->rect[0], res, level, box->x, box->y, box->z);
 373
 374    size = tx->base.layer_stride;
 375
 376    ret = nouveau_bo_new(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 0,
 377                         size * tx->nlayers, NULL, &tx->rect[1].bo);
 378    if (ret) {
 379       pipe_resource_reference(&tx->base.resource, NULL);
 380       FREE(tx);
 381       return NULL;
 382    }
 383
 384    tx->rect[1].cpp = tx->rect[0].cpp;
 385    tx->rect[1].width = tx->nblocksx;
 386    tx->rect[1].height = tx->nblocksy;
 387    tx->rect[1].depth = 1;
 388    tx->rect[1].pitch = tx->base.stride;
 389    tx->rect[1].domain = NOUVEAU_BO_GART;
 390
 391    if (usage & PIPE_TRANSFER_READ) {
 392       unsigned base = tx->rect[0].base;
 393       unsigned z = tx->rect[0].z;
 394       unsigned i;
 395       for (i = 0; i < tx->nlayers; ++i) {
 396          nvc0->m2mf_copy_rect(nvc0, &tx->rect[1], &tx->rect[0],
 397                               tx->nblocksx, tx->nblocksy);
 398          if (mt->layout_3d)
 399             tx->rect[0].z++;
 400          else
 401             tx->rect[0].base += mt->layer_stride;
 402          tx->rect[1].base += size;
 403       }
 404       tx->rect[0].z = z;
 405       tx->rect[0].base = base;
 406       tx->rect[1].base = 0;
 407    }
 408
 409    if (tx->rect[1].bo->map) {
 410       *ptransfer = &tx->base;
 411       return tx->rect[1].bo->map;
 412    }
 413
 414    if (usage & PIPE_TRANSFER_READ)
 415       flags = NOUVEAU_BO_RD;
 416    if (usage & PIPE_TRANSFER_WRITE)
 417       flags |= NOUVEAU_BO_WR;
 418
 419    ret = nouveau_bo_map(tx->rect[1].bo, flags, nvc0->screen->base.client);
 420    if (ret) {
 421       pipe_resource_reference(&tx->base.resource, NULL);
 422       nouveau_bo_ref(NULL, &tx->rect[1].bo);
 423       FREE(tx);
 424       return NULL;
 425    }
 426
 427    *ptransfer = &tx->base;
 428    return tx->rect[1].bo->map;
 429 }
 430
 431 void
 432 nvc0_miptree_transfer_unmap(struct pipe_context *pctx,
 433                             struct pipe_transfer *transfer)
 434 {
 435    struct nvc0_context *nvc0 = nvc0_context(pctx);
 436    struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer;
 437    struct nv50_miptree *mt = nv50_miptree(tx->base.resource);
 438    unsigned i;
 439
 440    if (tx->base.usage & PIPE_TRANSFER_WRITE) {
 441       for (i = 0; i < tx->nlayers; ++i) {
 442          nvc0->m2mf_copy_rect(nvc0, &tx->rect[0], &tx->rect[1],
 443                               tx->nblocksx, tx->nblocksy);
 444          if (mt->layout_3d)
 445             tx->rect[0].z++;
 446          else
 447             tx->rect[0].base += mt->layer_stride;
 448          tx->rect[1].base += tx->nblocksy * tx->base.stride;
 449       }
 450    }
 451
 452    nouveau_bo_ref(NULL, &tx->rect[1].bo);
 453    pipe_resource_reference(&transfer->resource, NULL);
 454
 455    FREE(tx);
 456 }
 457
 458 void
 459 nvc0_cb_push(struct nouveau_context *nv,
 460              struct nouveau_bo *bo, unsigned domain,
 461              unsigned base, unsigned size,
 462              unsigned offset, unsigned words, const uint32_t *data)
 463 {
 464    struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
 465    struct nouveau_pushbuf *push = nv->pushbuf;
 466
 467    assert(!(offset & 3));
 468    size = align(size, 0x100);
 469
 470    nouveau_bufctx_refn(bctx, 0, bo, NOUVEAU_BO_WR | domain);
 471    nouveau_pushbuf_bufctx(push, bctx);
 472    nouveau_pushbuf_validate(push);
 473
 474    BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
 475    PUSH_DATA (push, size);
 476    PUSH_DATAh(push, bo->offset + base);
 477    PUSH_DATA (push, bo->offset + base);
 478
 479    while (words) {
 480       unsigned nr = PUSH_AVAIL(push);
 481       nr = MIN2(nr, words);
 482       nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN - 1);
 483
 484       BEGIN_1IC0(push, NVC0_3D(CB_POS), nr + 1);
 485       PUSH_DATA (push, offset);
 486       PUSH_DATAp(push, data, nr);
 487
 488       words -= nr;
 489       data += nr;
 490       offset += nr * 4;
 491    }
 492
 493    nouveau_bufctx_reset(bctx, 0);
 494 }
 495
 496 void
 497 nvc0_init_transfer_functions(struct nvc0_context *nvc0)
 498 {
 499    if (nvc0->screen->base.class_3d >= NVE4_3D_CLASS) {
 500       nvc0->m2mf_copy_rect = nve4_m2mf_transfer_rect;
 501       nvc0->base.copy_data = nve4_m2mf_copy_linear;
 502       nvc0->base.push_data = nve4_p2mf_push_linear;
 503    } else {
 504       nvc0->m2mf_copy_rect = nvc0_m2mf_transfer_rect;
 505       nvc0->base.copy_data = nvc0_m2mf_copy_linear;
 506       nvc0->base.push_data = nvc0_m2mf_push_linear;
 507    }
 508    nvc0->base.push_cb = nvc0_cb_push;
 509 }