src/gallium/drivers/nvc0/nvc0_transfer.c

   1
   2 #include "util/u_format.h"
   3
   4 #include "nvc0_context.h"
   5
   6 #include "nv50/nv50_defs.xml.h"
   7
   8 struct nvc0_transfer {
   9    struct pipe_transfer base;
  10    struct nv50_m2mf_rect rect[2];
  11    uint32_t nblocksx;
  12    uint16_t nblocksy;
  13    uint16_t nlayers;
  14 };
  15
  16 void
  17 nvc0_m2mf_transfer_rect(struct nvc0_context *nvc0,
  18                         const struct nv50_m2mf_rect *dst,
  19                         const struct nv50_m2mf_rect *src,
  20                         uint32_t nblocksx, uint32_t nblocksy)
  21 {
  22    struct nouveau_pushbuf *push = nvc0->base.pushbuf;
  23    struct nouveau_bufctx *bctx = nvc0->bufctx;
  24    const int cpp = dst->cpp;
  25    uint32_t src_ofst = src->base;
  26    uint32_t dst_ofst = dst->base;
  27    uint32_t height = nblocksy;
  28    uint32_t sy = src->y;
  29    uint32_t dy = dst->y;
  30    uint32_t exec = (1 << 20);
  31
  32    assert(dst->cpp == src->cpp);
  33
  34    nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD);
  35    nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR);
  36    nouveau_pushbuf_bufctx(push, bctx);
  37    nouveau_pushbuf_validate(push);
  38
  39    if (nouveau_bo_memtype(src->bo)) {
  40       BEGIN_NVC0(push, NVC0_M2MF(TILING_MODE_IN), 5);
  41       PUSH_DATA (push, src->tile_mode);
  42       PUSH_DATA (push, src->width * cpp);
  43       PUSH_DATA (push, src->height);
  44       PUSH_DATA (push, src->depth);
  45       PUSH_DATA (push, src->z);
  46    } else {
  47       src_ofst += src->y * src->pitch + src->x * cpp;
  48
  49       BEGIN_NVC0(push, NVC0_M2MF(PITCH_IN), 1);
  50       PUSH_DATA (push, src->width * cpp);
  51
  52       exec |= NVC0_M2MF_EXEC_LINEAR_IN;
  53    }
  54
  55    if (nouveau_bo_memtype(dst->bo)) {
  56       BEGIN_NVC0(push, NVC0_M2MF(TILING_MODE_OUT), 5);
  57       PUSH_DATA (push, dst->tile_mode);
  58       PUSH_DATA (push, dst->width * cpp);
  59       PUSH_DATA (push, dst->height);
  60       PUSH_DATA (push, dst->depth);
  61       PUSH_DATA (push, dst->z);
  62    } else {
  63       dst_ofst += dst->y * dst->pitch + dst->x * cpp;
  64
  65       BEGIN_NVC0(push, NVC0_M2MF(PITCH_OUT), 1);
  66       PUSH_DATA (push, dst->width * cpp);
  67
  68       exec |= NVC0_M2MF_EXEC_LINEAR_OUT;
  69    }
  70
  71    while (height) {
  72       int line_count = height > 2047 ? 2047 : height;
  73
  74       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_IN_HIGH), 2);
  75       PUSH_DATAh(push, src->bo->offset + src_ofst);
  76       PUSH_DATA (push, src->bo->offset + src_ofst);
  77
  78       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
  79       PUSH_DATAh(push, dst->bo->offset + dst_ofst);
  80       PUSH_DATA (push, dst->bo->offset + dst_ofst);
  81
  82       if (!(exec & NVC0_M2MF_EXEC_LINEAR_IN)) {
  83          BEGIN_NVC0(push, NVC0_M2MF(TILING_POSITION_IN_X), 2);
  84          PUSH_DATA (push, src->x * cpp);
  85          PUSH_DATA (push, sy);
  86       } else {
  87          src_ofst += line_count * src->pitch;
  88       }
  89       if (!(exec & NVC0_M2MF_EXEC_LINEAR_OUT)) {
  90          BEGIN_NVC0(push, NVC0_M2MF(TILING_POSITION_OUT_X), 2);
  91          PUSH_DATA (push, dst->x * cpp);
  92          PUSH_DATA (push, dy);
  93       } else {
  94          dst_ofst += line_count * dst->pitch;
  95       }
  96
  97       BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
  98       PUSH_DATA (push, nblocksx * cpp);
  99       PUSH_DATA (push, line_count);
 100       BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
 101       PUSH_DATA (push, exec);
 102
 103       height -= line_count;
 104       sy += line_count;
 105       dy += line_count;
 106    }
 107
 108    nouveau_bufctx_reset(bctx, 0);
 109 }
 110
 111 void
 112 nvc0_m2mf_push_linear(struct nouveau_context *nv,
 113                       struct nouveau_bo *dst, unsigned offset, unsigned domain,
 114                       unsigned size, const void *data)
 115 {
 116    struct nvc0_context *nvc0 = nvc0_context(&nv->pipe);
 117    struct nouveau_pushbuf *push = nv->pushbuf;
 118    uint32_t *src = (uint32_t *)data;
 119    unsigned count = (size + 3) / 4;
 120
 121    nouveau_bufctx_refn(nvc0->bufctx, 0, dst, domain | NOUVEAU_BO_WR);
 122    nouveau_pushbuf_bufctx(push, nvc0->bufctx);
 123    nouveau_pushbuf_validate(push);
 124
 125    while (count) {
 126       unsigned nr;
 127
 128       if (!PUSH_SPACE(push, 16))
 129          break;
 130       nr = PUSH_AVAIL(push);
 131       assert(nr >= 16);
 132       nr = MIN2(count, nr - 9);
 133       nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN);
 134
 135       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
 136       PUSH_DATAh(push, dst->offset + offset);
 137       PUSH_DATA (push, dst->offset + offset);
 138       BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
 139       PUSH_DATA (push, nr * 4);
 140       PUSH_DATA (push, 1);
 141       BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
 142       PUSH_DATA (push, 0x100111);
 143
 144       /* must not be interrupted (trap on QUERY fence, 0x50 works however) */
 145       BEGIN_NIC0(push, NVC0_M2MF(DATA), nr);
 146       PUSH_DATAp(push, src, nr);
 147
 148       count -= nr;
 149       src += nr;
 150       offset += nr * 4;
 151    }
 152
 153    nouveau_bufctx_reset(nvc0->bufctx, 0);
 154 }
 155
 156 void
 157 nvc0_m2mf_copy_linear(struct nouveau_context *nv,
 158                       struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
 159                       struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
 160                       unsigned size)
 161 {
 162    struct nouveau_pushbuf *push = nv->pushbuf;
 163    struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
 164
 165    nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD);
 166    nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR);
 167    nouveau_pushbuf_bufctx(push, bctx);
 168    nouveau_pushbuf_validate(push);
 169
 170    while (size) {
 171       unsigned bytes = MIN2(size, 1 << 17);
 172
 173       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_OUT_HIGH), 2);
 174       PUSH_DATAh(push, dst->offset + dstoff);
 175       PUSH_DATA (push, dst->offset + dstoff);
 176       BEGIN_NVC0(push, NVC0_M2MF(OFFSET_IN_HIGH), 2);
 177       PUSH_DATAh(push, src->offset + srcoff);
 178       PUSH_DATA (push, src->offset + srcoff);
 179       BEGIN_NVC0(push, NVC0_M2MF(LINE_LENGTH_IN), 2);
 180       PUSH_DATA (push, bytes);
 181       PUSH_DATA (push, 1);
 182       BEGIN_NVC0(push, NVC0_M2MF(EXEC), 1);
 183       PUSH_DATA (push, (1 << NVC0_M2MF_EXEC_INC__SHIFT) |
 184                  NVC0_M2MF_EXEC_LINEAR_IN | NVC0_M2MF_EXEC_LINEAR_OUT);
 185
 186       srcoff += bytes;
 187       dstoff += bytes;
 188       size -= bytes;
 189    }
 190
 191    nouveau_bufctx_reset(bctx, 0);
 192 }
 193
 194 struct pipe_transfer *
 195 nvc0_miptree_transfer_new(struct pipe_context *pctx,
 196                           struct pipe_resource *res,
 197                           unsigned level,
 198                           unsigned usage,
 199                           const struct pipe_box *box)
 200 {
 201    struct nvc0_context *nvc0 = nvc0_context(pctx);
 202    struct nouveau_device *dev = nvc0->screen->base.device;
 203    struct nv50_miptree *mt = nv50_miptree(res);
 204    struct nvc0_transfer *tx;
 205    uint32_t size;
 206    int ret;
 207
 208    if (usage & PIPE_TRANSFER_MAP_DIRECTLY)
 209       return NULL;
 210
 211    tx = CALLOC_STRUCT(nvc0_transfer);
 212    if (!tx)
 213       return NULL;
 214
 215    pipe_resource_reference(&tx->base.resource, res);
 216
 217    tx->base.level = level;
 218    tx->base.usage = usage;
 219    tx->base.box = *box;
 220
 221    if (util_format_is_plain(res->format)) {
 222       tx->nblocksx = box->width << mt->ms_x;
 223       tx->nblocksy = box->height << mt->ms_y;
 224    } else {
 225       tx->nblocksx = util_format_get_nblocksx(res->format, box->width);
 226       tx->nblocksy = util_format_get_nblocksy(res->format, box->height);
 227    }
 228    tx->nlayers = box->depth;
 229
 230    tx->base.stride = tx->nblocksx * util_format_get_blocksize(res->format);
 231    tx->base.layer_stride = tx->nblocksy * tx->base.stride;
 232
 233    nv50_m2mf_rect_setup(&tx->rect[0], res, level, box->x, box->y, box->z);
 234
 235    size = tx->base.layer_stride;
 236
 237    ret = nouveau_bo_new(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 0,
 238                         size * tx->nlayers, NULL, &tx->rect[1].bo);
 239    if (ret) {
 240       FREE(tx);
 241       return NULL;
 242    }
 243
 244    tx->rect[1].cpp = tx->rect[0].cpp;
 245    tx->rect[1].width = tx->nblocksx;
 246    tx->rect[1].height = tx->nblocksy;
 247    tx->rect[1].depth = 1;
 248    tx->rect[1].pitch = tx->base.stride;
 249    tx->rect[1].domain = NOUVEAU_BO_GART;
 250
 251    if (usage & PIPE_TRANSFER_READ) {
 252       unsigned base = tx->rect[0].base;
 253       unsigned z = tx->rect[0].z;
 254       unsigned i;
 255       for (i = 0; i < tx->nlayers; ++i) {
 256          nvc0_m2mf_transfer_rect(nvc0, &tx->rect[1], &tx->rect[0],
 257                                  tx->nblocksx, tx->nblocksy);
 258          if (mt->layout_3d)
 259             tx->rect[0].z++;
 260          else
 261             tx->rect[0].base += mt->layer_stride;
 262          tx->rect[1].base += size;
 263       }
 264       tx->rect[0].z = z;
 265       tx->rect[0].base = base;
 266       tx->rect[1].base = 0;
 267    }
 268
 269    return &tx->base;
 270 }
 271
 272 void
 273 nvc0_miptree_transfer_del(struct pipe_context *pctx,
 274                           struct pipe_transfer *transfer)
 275 {
 276    struct nvc0_context *nvc0 = nvc0_context(pctx);
 277    struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer;
 278    struct nv50_miptree *mt = nv50_miptree(tx->base.resource);
 279    unsigned i;
 280
 281    if (tx->base.usage & PIPE_TRANSFER_WRITE) {
 282       for (i = 0; i < tx->nlayers; ++i) {
 283          nvc0_m2mf_transfer_rect(nvc0, &tx->rect[0], &tx->rect[1],
 284                                  tx->nblocksx, tx->nblocksy);
 285          if (mt->layout_3d)
 286             tx->rect[0].z++;
 287          else
 288             tx->rect[0].base += mt->layer_stride;
 289          tx->rect[1].base += tx->nblocksy * tx->base.stride;
 290       }
 291    }
 292
 293    nouveau_bo_ref(NULL, &tx->rect[1].bo);
 294    pipe_resource_reference(&transfer->resource, NULL);
 295
 296    FREE(tx);
 297 }
 298
 299 void *
 300 nvc0_miptree_transfer_map(struct pipe_context *pctx,
 301                           struct pipe_transfer *transfer)
 302 {
 303    struct nvc0_context *nvc0 = nvc0_context(pctx);
 304    struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer;
 305    int ret;
 306    unsigned flags = 0;
 307
 308    if (tx->rect[1].bo->map)
 309       return tx->rect[1].bo->map;
 310
 311    if (transfer->usage & PIPE_TRANSFER_READ)
 312       flags = NOUVEAU_BO_RD;
 313    if (transfer->usage & PIPE_TRANSFER_WRITE)
 314       flags |= NOUVEAU_BO_WR;
 315
 316    ret = nouveau_bo_map(tx->rect[1].bo, flags, nvc0->screen->base.client);
 317    if (ret)
 318       return NULL;
 319    return tx->rect[1].bo->map;
 320 }
 321
 322 void
 323 nvc0_miptree_transfer_unmap(struct pipe_context *pctx,
 324                             struct pipe_transfer *transfer)
 325 {
 326 }
 327
 328 void
 329 nvc0_cb_push(struct nouveau_context *nv,
 330              struct nouveau_bo *bo, unsigned domain,
 331              unsigned base, unsigned size,
 332              unsigned offset, unsigned words, const uint32_t *data)
 333 {
 334    struct nouveau_bufctx *bctx = nvc0_context(&nv->pipe)->bufctx;
 335    struct nouveau_pushbuf *push = nv->pushbuf;
 336
 337    assert(!(offset & 3));
 338    size = align(size, 0x100);
 339
 340    nouveau_bufctx_refn(bctx, 0, bo, NOUVEAU_BO_WR | domain);
 341    nouveau_pushbuf_bufctx(push, bctx);
 342    nouveau_pushbuf_validate(push);
 343
 344    BEGIN_NVC0(push, NVC0_3D(CB_SIZE), 3);
 345    PUSH_DATA (push, size);
 346    PUSH_DATAh(push, bo->offset + base);
 347    PUSH_DATA (push, bo->offset + base);
 348
 349    while (words) {
 350       unsigned nr = PUSH_AVAIL(push);
 351       nr = MIN2(nr, words);
 352       nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN - 1);
 353
 354       BEGIN_1IC0(push, NVC0_3D(CB_POS), nr + 1);
 355       PUSH_DATA (push, offset);
 356       PUSH_DATAp(push, data, nr);
 357
 358       words -= nr;
 359       data += nr;
 360       offset += nr * 4;
 361    }
 362
 363    nouveau_bufctx_reset(bctx, 0);
 364 }