src/gallium/drivers/nvc0/nvc0_transfer.c

   1
   2 #include "util/u_format.h"
   3
   4 #include "nvc0_context.h"
   5
   6 #include "nv50/nv50_defs.xml.h"
   7
   8 struct nvc0_transfer {
   9    struct pipe_transfer base;
  10    struct nv50_m2mf_rect rect[2];
  11    uint32_t nblocksx;
  12    uint16_t nblocksy;
  13    uint16_t nlayers;
  14 };
  15
  16 static void
  17 nvc0_m2mf_transfer_rect(struct nvc0_context *nvc0,
  18                         const struct nv50_m2mf_rect *dst,
  19                         const struct nv50_m2mf_rect *src,
  20                         uint32_t nblocksx, uint32_t nblocksy)
  21 {
  22    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
  23    struct nouveau_bufctx *bctx = nvc0->bufctx;
  24    const int cpp = dst->cpp;
  25    uint32_t src_ofst = src->base;
  26    uint32_t dst_ofst = dst->base;
  27    uint32_t height = nblocksy;
  28    uint32_t sy = src->y;
  29    uint32_t dy = dst->y;
  30    uint32_t exec = (1 << 20);
  31
  32    assert(dst->cpp == src->cpp);
  33
  34    nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD);
  35    nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR);
  36    nouveau_pushbuf_bufctx(push, bctx);
  37    nouveau_pushbuf_validate(push);
  38
  39    if (nouveau_bo_memtype(src->bo)) {
  40       BEGIN_NVC0(push, NVC0_M2MF(TILING_MODE_IN), 5);
  41       PUSH_DATA (push, src->tile_mode);
  42       PUSH_DATA (push, src->width * cpp);
  43       PUSH_DATA (push, src->height);
  44       PUSH_DATA (push, src->depth);
  45       PUSH_DATA (push, src->z);
  46    } else {
  47       src_ofst += src->y * src->pitch + src->x * cpp;
  48
  49       BEGIN_NVC0(push, NVC0_M2MF(PITCH_IN), 1);
  50       PUSH_DATA (push, src->width * cpp);
  51
  52       exec |= NVC0_M2MF_EXEC_LINEAR_IN;
  53    }
  54
  55    if (nouveau_bo_memtype(dst->bo)) {
  56       BEGIN_NVC0(push, NVC0_M2MF(TILING_MODE_OUT), 5);
  57       PUSH_DATA (push, dst->tile_mode);
  58       PUSH_DATA (push, dst->width * cpp);
  59       PUSH_DATA (push, dst->height);
  60       PUSH_DATA (push, dst->depth);
  61       PUSH_DATA (push, dst->z);
  62    } else {
  63       dst_ofst += dst->y * dst->pitch + dst->x * cpp;
  64
  65       BEGIN_NVC0(push, NVC0_M2MF(PITCH_OUT), 1);
  66       PUSH_DATA (push, dst->width * cpp);
  67
  68       exec |= NVC0_M2MF_EXEC_LINEAR_OUT;
  69    }
  70
  71    while (height) {
  72       int line_count = height > 2047 ? 2047 : height;
  73
  74       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_IN_HIGH), 2);
  75       PUSH_DATAh(push, src->bo->offset + src_ofst);
  76       PUSH_DATA (push, src->bo->offset + src_ofst);
  77
  78       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
  79       PUSH_DATAh(push, dst->bo->offset + dst_ofst);
  80       PUSH_DATA (push, dst->bo->offset + dst_ofst);
  81
  82       if (!(exec & NVC0_M2MF_EXEC_LINEAR_IN)) {
  83          BEGIN_NVC0(push, NVC0_M2MF(TILING_POSITION_IN_X), 2);
  84          PUSH_DATA (push, src->x * cpp);
  85          PUSH_DATA (push, sy);
  86       } else {
  87          src_ofst += line_count * src->pitch;
  88       }
  89       if (!(exec & NVC0_M2MF_EXEC_LINEAR_OUT)) {
  90          BEGIN_NVC0(push, NVC0_M2MF(TILING_POSITION_OUT_X), 2);
  91          PUSH_DATA (push, dst->x * cpp);
  92          PUSH_DATA (push, dy);
  93       } else {
  94          dst_ofst += line_count * dst->pitch;
  95       }
  96
  97       BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
  98       PUSH_DATA (push, nblocksx * cpp);
  99       PUSH_DATA (push, line_count);
 100       BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
 101       PUSH_DATA (push, exec);
 102
 103       height -= line_count;
 104       sy += line_count;
 105       dy += line_count;
 106    }
 107
 108    nouveau_bufctx_reset(bctx, 0);
 109 }
 110
 111 static void
 112 nve4_m2mf_transfer_rect(struct nvc0_context *nvc0,
 113                         const struct nv50_m2mf_rect *dst,
 114                         const struct nv50_m2mf_rect *src,
 115                         uint32_t nblocksx, uint32_t nblocksy)
 116 {
 117    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
 118    struct nouveau_bufctx *bctx = nvc0->bufctx;
 119    uint32_t exec;
 120    uint32_t src_base = src->base;
 121    uint32_t dst_base = dst->base;
 122    const int cpp = dst->cpp;
 123
 124    assert(dst->cpp == src->cpp);
 125
 126    nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR);
 127    nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD);
 128    nouveau_pushbuf_bufctx(push, bctx);
 129    nouveau_pushbuf_validate(push);
 130
 131    exec = 0x200 /* 2D_ENABLE */ | 0x6 /* UNK */;
 132
 133    if (!nouveau_bo_memtype(dst->bo)) {
 134       assert(!dst->z);
 135       dst_base += dst->y * dst->pitch + dst->x * cpp;
 136       exec |= 0x100; /* DST_MODE_2D_LINEAR */
 137    }
 138    if (!nouveau_bo_memtype(src->bo)) {
 139       assert(!src->z);
 140       src_base += src->y * src->pitch + src->x * cpp;
 141       exec |= 0x080; /* SRC_MODE_2D_LINEAR */
 142    }
 143
 144    BEGIN_NVC0(push, SUBC_COPY(0x070c), 6);
 145    PUSH_DATA (push, 0x1000 | dst->tile_mode);
 146    PUSH_DATA (push, dst->pitch);
 147    PUSH_DATA (push, dst->height);
 148    PUSH_DATA (push, dst->depth);
 149    PUSH_DATA (push, dst->z);
 150    PUSH_DATA (push, (dst->y << 16) | (dst->x * cpp));
 151
 152    BEGIN_NVC0(push, SUBC_COPY(0x0728), 6);
 153    PUSH_DATA (push, 0x1000 | src->tile_mode);
 154    PUSH_DATA (push, src->pitch);
 155    PUSH_DATA (push, src->height);
 156    PUSH_DATA (push, src->depth);
 157    PUSH_DATA (push, src->z);
 158    PUSH_DATA (push, (src->y << 16) | (src->x * cpp));
 159
 160    BEGIN_NVC0(push, SUBC_COPY(0x0400), 8);
 161    PUSH_DATAh(push, src->bo->offset + src_base);
 162    PUSH_DATA (push, src->bo->offset + src_base);
 163    PUSH_DATAh(push, dst->bo->offset + dst_base);
 164    PUSH_DATA (push, dst->bo->offset + dst_base);
 165    PUSH_DATA (push, src->pitch);
 166    PUSH_DATA (push, dst->pitch);
 167    PUSH_DATA (push, nblocksx * cpp);
 168    PUSH_DATA (push, nblocksy);
 169
 170    BEGIN_NVC0(push, SUBC_COPY(0x0300), 1);
 171    PUSH_DATA (push, exec);
 172
 173    nouveau_bufctx_reset(bctx, 0);
 174 }
 175
 176 void
 177 nvc0_m2mf_push_linear(struct nouveau_context *nv,
 178                       struct nouveau_bo *dst, unsigned offset, unsigned domain,
 179                       unsigned size, const void *data)
 180 {
 181    struct nvc0_context *nvc0 = nvc0_context(&nv->pipe);
 182    struct nouveau_pushbuf *push = nv->pushbuf;
 183    uint32_t *src = (uint32_t *)data;
 184    unsigned count = (size + 3) / 4;
 185
 186    nouveau_bufctx_refn(nvc0->bufctx, 0, dst, domain | NOUVEAU_BO_WR);
 187    nouveau_pushbuf_bufctx(push, nvc0->bufctx);
 188    nouveau_pushbuf_validate(push);
 189
 190    while (count) {
 191       unsigned nr;
 192
 193       if (!PUSH_SPACE(push, 16))
 194          break;
 195       nr = PUSH_AVAIL(push);
 196       assert(nr >= 16);
 197       nr = MIN2(count, nr - 9);
 198       nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN);
 199
 200       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
 201       PUSH_DATAh(push, dst->offset + offset);
 202       PUSH_DATA (push, dst->offset + offset);
 203       BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
 204       PUSH_DATA (push, nr * 4);
 205       PUSH_DATA (push, 1);
 206       BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
 207       PUSH_DATA (push, 0x100111);
 208
 209       /* must not be interrupted (trap on QUERY fence, 0x50 works however) */
 210       BEGIN_NIC0(push, NVC0_M2MF(DATA), nr);
 211       PUSH_DATAp(push, src, nr);
 212
 213       count -= nr;
 214       src += nr;
 215       offset += nr * 4;
 216    }
 217
 218    nouveau_bufctx_reset(nvc0->bufctx, 0);
 219 }
 220
 221 void
 222 nve4_p2mf_push_linear(struct nouveau_context *nv,
 223                       struct nouveau_bo *dst, unsigned offset, unsigned domain,
 224                       unsigned size, const void *data)
 225 {
 226    struct nvc0_context *nvc0 = nvc0_context(&nv->pipe);
 227    struct nouveau_pushbuf *push = nv->pushbuf;
 228    uint32_t *src = (uint32_t *)data;
 229    unsigned count = (size + 3) / 4;
 230
 231    nouveau_bufctx_refn(nvc0->bufctx, 0, dst, domain | NOUVEAU_BO_WR);
 232    nouveau_pushbuf_bufctx(push, nvc0->bufctx);
 233    nouveau_pushbuf_validate(push);
 234
 235    while (count) {
 236       unsigned nr;
 237
 238       if (!PUSH_SPACE(push, 16))
 239          break;
 240       nr = PUSH_AVAIL(push);
 241       assert(nr >= 16);
 242       nr = MIN2(count, nr - 8);
 243       nr = MIN2(nr, (NV04_PFIFO_MAX_PACKET_LEN - 1));
 244
 245       BEGIN_NVC0(push, NVE4_P2MF(DST_ADDRESS_HIGH), 2);
 246       PUSH_DATAh(push, dst->offset + offset);
 247       PUSH_DATA (push, dst->offset + offset);
 248       BEGIN_NVC0(push, NVE4_P2MF(LINE_LENGTH_IN), 2);
 249       PUSH_DATA (push, nr * 4);
 250       PUSH_DATA (push, 1);
 251       /* must not be interrupted (trap on QUERY fence, 0x50 works however) */
 252       BEGIN_1IC0(push, NVE4_P2MF(EXEC), nr + 1);
 253       PUSH_DATA (push, 0x1001);
 254       PUSH_DATAp(push, src, nr);
 255
 256       count -= nr;
 257       src += nr;
 258       offset += nr * 4;
 259    }
 260
 261    nouveau_bufctx_reset(nvc0->bufctx, 0);
 262 }
 263
 264 static void
 265 nvc0_m2mf_copy_linear(struct nouveau_context *nv,
 266                       struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
 267                       struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
 268                       unsigned size)
 269 {
 270    struct nouveau_pushbuf *push = nv->pushbuf;
 271    struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
 272
 273    nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD);
 274    nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR);
 275    nouveau_pushbuf_bufctx(push, bctx);
 276    nouveau_pushbuf_validate(push);
 277
 278    while (size) {
 279       unsigned bytes = MIN2(size, 1 << 17);
 280
 281       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
 282       PUSH_DATAh(push, dst->offset + dstoff);
 283       PUSH_DATA (push, dst->offset + dstoff);
 284       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_IN_HIGH), 2);
 285       PUSH_DATAh(push, src->offset + srcoff);
 286       PUSH_DATA (push, src->offset + srcoff);
 287       BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
 288       PUSH_DATA (push, bytes);
 289       PUSH_DATA (push, 1);
 290       BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
 291       PUSH_DATA (push, (1 << NVC0_M2MF_EXEC_INC__SHIFT) |
 292                  NVC0_M2MF_EXEC_LINEAR_IN | NVC0_M2MF_EXEC_LINEAR_OUT);
 293
 294       srcoff += bytes;
 295       dstoff += bytes;
 296       size -= bytes;
 297    }
 298
 299    nouveau_bufctx_reset(bctx, 0);
 300 }
 301
 302 static void
 303 nve4_m2mf_copy_linear(struct nouveau_context *nv,
 304                       struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
 305                       struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
 306                       unsigned size)
 307 {
 308    struct nouveau_pushbuf *push = nv->pushbuf;
 309    struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
 310
 311    nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD);
 312    nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR);
 313    nouveau_pushbuf_bufctx(push, bctx);
 314    nouveau_pushbuf_validate(push);
 315
 316    BEGIN_NVC0(push, SUBC_COPY(0x0400), 4);
 317    PUSH_DATAh(push, src->offset + srcoff);
 318    PUSH_DATA (push, src->offset + srcoff);
 319    PUSH_DATAh(push, dst->offset + dstoff);
 320    PUSH_DATA (push, dst->offset + dstoff);
 321    BEGIN_NVC0(push, SUBC_COPY(0x0418), 1);
 322    PUSH_DATA (push, size);
 323    BEGIN_NVC0(push, SUBC_COPY(0x0300), 1);
 324    PUSH_DATA (push, 0x186);
 325
 326    nouveau_bufctx_reset(bctx, 0);
 327 }
 328
 329 struct pipe_transfer *
 330 nvc0_miptree_transfer_new(struct pipe_context *pctx,
 331                           struct pipe_resource *res,
 332                           unsigned level,
 333                           unsigned usage,
 334                           const struct pipe_box *box)
 335 {
 336    struct nvc0_context *nvc0 = nvc0_context(pctx);
 337    struct nouveau_device *dev = nvc0->screen->base.device;
 338    struct nv50_miptree *mt = nv50_miptree(res);
 339    struct nvc0_transfer *tx;
 340    uint32_t size;
 341    int ret;
 342
 343    if (usage & PIPE_TRANSFER_MAP_DIRECTLY)
 344       return NULL;
 345
 346    tx = CALLOC_STRUCT(nvc0_transfer);
 347    if (!tx)
 348       return NULL;
 349
 350    pipe_resource_reference(&tx->base.resource, res);
 351
 352    tx->base.level = level;
 353    tx->base.usage = usage;
 354    tx->base.box = *box;
 355
 356    if (util_format_is_plain(res->format)) {
 357       tx->nblocksx = box->width << mt->ms_x;
 358       tx->nblocksy = box->height << mt->ms_y;
 359    } else {
 360       tx->nblocksx = util_format_get_nblocksx(res->format, box->width);
 361       tx->nblocksy = util_format_get_nblocksy(res->format, box->height);
 362    }
 363    tx->nlayers = box->depth;
 364
 365    tx->base.stride = tx->nblocksx * util_format_get_blocksize(res->format);
 366    tx->base.layer_stride = tx->nblocksy * tx->base.stride;
 367
 368    nv50_m2mf_rect_setup(&tx->rect[0], res, level, box->x, box->y, box->z);
 369
 370    size = tx->base.layer_stride;
 371
 372    ret = nouveau_bo_new(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 0,
 373                         size * tx->nlayers, NULL, &tx->rect[1].bo);
 374    if (ret) {
 375       FREE(tx);
 376       return NULL;
 377    }
 378
 379    tx->rect[1].cpp = tx->rect[0].cpp;
 380    tx->rect[1].width = tx->nblocksx;
 381    tx->rect[1].height = tx->nblocksy;
 382    tx->rect[1].depth = 1;
 383    tx->rect[1].pitch = tx->base.stride;
 384    tx->rect[1].domain = NOUVEAU_BO_GART;
 385
 386    if (usage & PIPE_TRANSFER_READ) {
 387       unsigned base = tx->rect[0].base;
 388       unsigned z = tx->rect[0].z;
 389       unsigned i;
 390       for (i = 0; i < tx->nlayers; ++i) {
 391          nvc0->m2mf_copy_rect(nvc0, &tx->rect[1], &tx->rect[0],
 392                               tx->nblocksx, tx->nblocksy);
 393          if (mt->layout_3d)
 394             tx->rect[0].z++;
 395          else
 396             tx->rect[0].base += mt->layer_stride;
 397          tx->rect[1].base += size;
 398       }
 399       tx->rect[0].z = z;
 400       tx->rect[0].base = base;
 401       tx->rect[1].base = 0;
 402    }
 403
 404    return &tx->base;
 405 }
 406
 407 void
 408 nvc0_miptree_transfer_del(struct pipe_context *pctx,
 409                           struct pipe_transfer *transfer)
 410 {
 411    struct nvc0_context *nvc0 = nvc0_context(pctx);
 412    struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer;
 413    struct nv50_miptree *mt = nv50_miptree(tx->base.resource);
 414    unsigned i;
 415
 416    if (tx->base.usage & PIPE_TRANSFER_WRITE) {
 417       for (i = 0; i < tx->nlayers; ++i) {
 418          nvc0->m2mf_copy_rect(nvc0, &tx->rect[0], &tx->rect[1],
 419                               tx->nblocksx, tx->nblocksy);
 420          if (mt->layout_3d)
 421             tx->rect[0].z++;
 422          else
 423             tx->rect[0].base += mt->layer_stride;
 424          tx->rect[1].base += tx->nblocksy * tx->base.stride;
 425       }
 426    }
 427
 428    nouveau_bo_ref(NULL, &tx->rect[1].bo);
 429    pipe_resource_reference(&transfer->resource, NULL);
 430
 431    FREE(tx);
 432 }
 433
 434 void *
 435 nvc0_miptree_transfer_map(struct pipe_context *pctx,
 436                           struct pipe_transfer *transfer)
 437 {
 438    struct nvc0_context *nvc0 = nvc0_context(pctx);
 439    struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer;
 440    int ret;
 441    unsigned flags = 0;
 442
 443    if (tx->rect[1].bo->map)
 444       return tx->rect[1].bo->map;
 445
 446    if (transfer->usage & PIPE_TRANSFER_READ)
 447       flags = NOUVEAU_BO_RD;
 448    if (transfer->usage & PIPE_TRANSFER_WRITE)
 449       flags |= NOUVEAU_BO_WR;
 450
 451    ret = nouveau_bo_map(tx->rect[1].bo, flags, nvc0->screen->base.client);
 452    if (ret)
 453       return NULL;
 454    return tx->rect[1].bo->map;
 455 }
 456
 457 void
 458 nvc0_miptree_transfer_unmap(struct pipe_context *pctx,
 459                             struct pipe_transfer *transfer)
 460 {
 461 }
 462
 463 void
 464 nvc0_cb_push(struct nouveau_context *nv,
 465              struct nouveau_bo *bo, unsigned domain,
 466              unsigned base, unsigned size,
 467              unsigned offset, unsigned words, const uint32_t *data)
 468 {
 469    struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
 470    struct nouveau_pushbuf *push = nv->pushbuf;
 471
 472    assert(!(offset & 3));
 473    size = align(size, 0x100);
 474
 475    nouveau_bufctx_refn(bctx, 0, bo, NOUVEAU_BO_WR | domain);
 476    nouveau_pushbuf_bufctx(push, bctx);
 477    nouveau_pushbuf_validate(push);
 478
 479    BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
 480    PUSH_DATA (push, size);
 481    PUSH_DATAh(push, bo->offset + base);
 482    PUSH_DATA (push, bo->offset + base);
 483
 484    while (words) {
 485       unsigned nr = PUSH_AVAIL(push);
 486       nr = MIN2(nr, words);
 487       nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN - 1);
 488
 489       BEGIN_1IC0(push, NVC0_3D(CB_POS), nr + 1);
 490       PUSH_DATA (push, offset);
 491       PUSH_DATAp(push, data, nr);
 492
 493       words -= nr;
 494       data += nr;
 495       offset += nr * 4;
 496    }
 497
 498    nouveau_bufctx_reset(bctx, 0);
 499 }
 500
 501 void
 502 nvc0_init_transfer_functions(struct nvc0_context *nvc0)
 503 {
 504    if (nvc0->screen->base.class_3d >= NVE4_3D_CLASS) {
 505       nvc0->m2mf_copy_rect = nve4_m2mf_transfer_rect;
 506       nvc0->base.copy_data = nve4_m2mf_copy_linear;
 507       nvc0->base.push_data = nve4_p2mf_push_linear;
 508    } else {
 509       nvc0->m2mf_copy_rect = nvc0_m2mf_transfer_rect;
 510       nvc0->base.copy_data = nvc0_m2mf_copy_linear;
 511       nvc0->base.push_data = nvc0_m2mf_push_linear;
 512    }
 513    nvc0->base.push_cb = nvc0_cb_push;
 514 }