src/gallium/drivers/nouveau/nv50/nv50_transfer.c

   1
   2 #include "util/u_format.h"
   3
   4 #include "nv50/nv50_context.h"
   5
   6 #include "nv50/nv50_defs.xml.h"
   7
   8 struct nv50_transfer {
   9    struct pipe_transfer base;
  10    struct nv50_m2mf_rect rect[2];
  11    uint32_t nblocksx;
  12    uint32_t nblocksy;
  13 };
  14
  15 void
  16 nv50_m2mf_rect_setup(struct nv50_m2mf_rect *rect,
  17                      struct pipe_resource *restrict res, unsigned l,
  18                      unsigned x, unsigned y, unsigned z)
  19 {
  20    struct nv50_miptree *mt = nv50_miptree(res);
  21    const unsigned w = u_minify(res->width0, l);
  22    const unsigned h = u_minify(res->height0, l);
  23
  24    rect->bo = mt->base.bo;
  25    rect->domain = mt->base.domain;
  26    rect->base = mt->level[l].offset;
  27    if (mt->base.bo->offset != mt->base.address)
  28       rect->base += mt->base.address - mt->base.bo->offset;
  29    rect->pitch = mt->level[l].pitch;
  30    if (util_format_is_plain(res->format)) {
  31       rect->width = w << mt->ms_x;
  32       rect->height = h << mt->ms_y;
  33       rect->x = x << mt->ms_x;
  34       rect->y = y << mt->ms_y;
  35    } else {
  36       rect->width = util_format_get_nblocksx(res->format, w);
  37       rect->height = util_format_get_nblocksy(res->format, h);
  38       rect->x = util_format_get_nblocksx(res->format, x);
  39       rect->y = util_format_get_nblocksy(res->format, y);
  40    }
  41    rect->tile_mode = mt->level[l].tile_mode;
  42    rect->cpp = util_format_get_blocksize(res->format);
  43
  44    if (mt->layout_3d) {
  45       rect->z = z;
  46       rect->depth = u_minify(res->depth0, l);
  47    } else {
  48       rect->base += z * mt->layer_stride;
  49       rect->z = 0;
  50       rect->depth = 1;
  51    }
  52 }
  53
  54 void
  55 nv50_m2mf_transfer_rect(struct nv50_context *nv50,
  56                         const struct nv50_m2mf_rect *dst,
  57                         const struct nv50_m2mf_rect *src,
  58                         uint32_t nblocksx, uint32_t nblocksy)
  59 {
  60    struct nouveau_pushbuf *push = nv50->base.pushbuf;
  61    struct nouveau_bufctx *bctx = nv50->bufctx;
  62    const int cpp = dst->cpp;
  63    uint32_t src_ofst = src->base;
  64    uint32_t dst_ofst = dst->base;
  65    uint32_t height = nblocksy;
  66    uint32_t sy = src->y;
  67    uint32_t dy = dst->y;
  68
  69    assert(dst->cpp == src->cpp);
  70
  71    nouveau_bufctx_refn(bctx, 0, src->bo, src->domain | NOUVEAU_BO_RD);
  72    nouveau_bufctx_refn(bctx, 0, dst->bo, dst->domain | NOUVEAU_BO_WR);
  73    nouveau_pushbuf_bufctx(push, bctx);
  74    nouveau_pushbuf_validate(push);
  75
  76    if (nouveau_bo_memtype(src->bo)) {
  77       BEGIN_NV04(push, NV50_M2MF(LINEAR_IN), 6);
  78       PUSH_DATA (push, 0);
  79       PUSH_DATA (push, src->tile_mode);
  80       PUSH_DATA (push, src->width * cpp);
  81       PUSH_DATA (push, src->height);
  82       PUSH_DATA (push, src->depth);
  83       PUSH_DATA (push, src->z);
  84    } else {
  85       src_ofst += src->y * src->pitch + src->x * cpp;
  86
  87       BEGIN_NV04(push, NV50_M2MF(LINEAR_IN), 1);
  88       PUSH_DATA (push, 1);
  89       BEGIN_NV04(push, SUBC_M2MF(NV03_M2MF_PITCH_IN), 1);
  90       PUSH_DATA (push, src->pitch);
  91    }
  92
  93    if (nouveau_bo_memtype(dst->bo)) {
  94       BEGIN_NV04(push, NV50_M2MF(LINEAR_OUT), 6);
  95       PUSH_DATA (push, 0);
  96       PUSH_DATA (push, dst->tile_mode);
  97       PUSH_DATA (push, dst->width * cpp);
  98       PUSH_DATA (push, dst->height);
  99       PUSH_DATA (push, dst->depth);
 100       PUSH_DATA (push, dst->z);
 101    } else {
 102       dst_ofst += dst->y * dst->pitch + dst->x * cpp;
 103
 104       BEGIN_NV04(push, NV50_M2MF(LINEAR_OUT), 1);
 105       PUSH_DATA (push, 1);
 106       BEGIN_NV04(push, SUBC_M2MF(NV03_M2MF_PITCH_OUT), 1);
 107       PUSH_DATA (push, dst->pitch);
 108    }
 109
 110    while (height) {
 111       int line_count = height > 2047 ? 2047 : height;
 112
 113       BEGIN_NV04(push, NV50_M2MF(OFFSET_IN_HIGH), 2);
 114       PUSH_DATAh(push, src->bo->offset + src_ofst);
 115       PUSH_DATAh(push, dst->bo->offset + dst_ofst);
 116
 117       BEGIN_NV04(push, SUBC_M2MF(NV03_M2MF_OFFSET_IN), 2);
 118       PUSH_DATA (push, src->bo->offset + src_ofst);
 119       PUSH_DATA (push, dst->bo->offset + dst_ofst);
 120
 121       if (nouveau_bo_memtype(src->bo)) {
 122          BEGIN_NV04(push, NV50_M2MF(TILING_POSITION_IN), 1);
 123          PUSH_DATA (push, (sy << 16) | (src->x * cpp));
 124       } else {
 125          src_ofst += line_count * src->pitch;
 126       }
 127       if (nouveau_bo_memtype(dst->bo)) {
 128          BEGIN_NV04(push, NV50_M2MF(TILING_POSITION_OUT), 1);
 129          PUSH_DATA (push, (dy << 16) | (dst->x * cpp));
 130       } else {
 131          dst_ofst += line_count * dst->pitch;
 132       }
 133
 134       BEGIN_NV04(push, SUBC_M2MF(NV03_M2MF_LINE_LENGTH_IN), 4);
 135       PUSH_DATA (push, nblocksx * cpp);
 136       PUSH_DATA (push, line_count);
 137       PUSH_DATA (push, (1 << 8) | (1 << 0));
 138       PUSH_DATA (push, 0);
 139
 140       height -= line_count;
 141       sy += line_count;
 142       dy += line_count;
 143    }
 144
 145    nouveau_bufctx_reset(bctx, 0);
 146 }
 147
 148 void
 149 nv50_sifc_linear_u8(struct nouveau_context *nv,
 150                     struct nouveau_bo *dst, unsigned offset, unsigned domain,
 151                     unsigned size, const void *data)
 152 {
 153    struct nv50_context *nv50 = nv50_context(&nv->pipe);
 154    struct nouveau_pushbuf *push = nv50->base.pushbuf;
 155    uint32_t *src = (uint32_t *)data;
 156    unsigned count = (size + 3) / 4;
 157    unsigned xcoord = offset & 0xff;
 158
 159    nouveau_bufctx_refn(nv50->bufctx, 0, dst, domain | NOUVEAU_BO_WR);
 160    nouveau_pushbuf_bufctx(push, nv50->bufctx);
 161    nouveau_pushbuf_validate(push);
 162
 163    offset &= ~0xff;
 164
 165    BEGIN_NV04(push, NV50_2D(DST_FORMAT), 2);
 166    PUSH_DATA (push, NV50_SURFACE_FORMAT_R8_UNORM);
 167    PUSH_DATA (push, 1);
 168    BEGIN_NV04(push, NV50_2D(DST_PITCH), 5);
 169    PUSH_DATA (push, 262144);
 170    PUSH_DATA (push, 65536);
 171    PUSH_DATA (push, 1);
 172    PUSH_DATAh(push, dst->offset + offset);
 173    PUSH_DATA (push, dst->offset + offset);
 174    BEGIN_NV04(push, NV50_2D(SIFC_BITMAP_ENABLE), 2);
 175    PUSH_DATA (push, 0);
 176    PUSH_DATA (push, NV50_SURFACE_FORMAT_R8_UNORM);
 177    BEGIN_NV04(push, NV50_2D(SIFC_WIDTH), 10);
 178    PUSH_DATA (push, size);
 179    PUSH_DATA (push, 1);
 180    PUSH_DATA (push, 0);
 181    PUSH_DATA (push, 1);
 182    PUSH_DATA (push, 0);
 183    PUSH_DATA (push, 1);
 184    PUSH_DATA (push, 0);
 185    PUSH_DATA (push, xcoord);
 186    PUSH_DATA (push, 0);
 187    PUSH_DATA (push, 0);
 188
 189    while (count) {
 190       unsigned nr;
 191
 192       if (!PUSH_SPACE(push, 16))
 193          break;
 194       nr = PUSH_AVAIL(push);
 195       assert(nr >= 16);
 196       nr = MIN2(count, nr - 1);
 197       nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN);
 198
 199       BEGIN_NI04(push, NV50_2D(SIFC_DATA), nr);
 200       PUSH_DATAp(push, src, nr);
 201
 202       src += nr;
 203       count -= nr;
 204    }
 205
 206    nouveau_bufctx_reset(nv50->bufctx, 0);
 207 }
 208
 209 void
 210 nv50_m2mf_copy_linear(struct nouveau_context *nv,
 211                       struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
 212                       struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
 213                       unsigned size)
 214 {
 215    struct nouveau_pushbuf *push = nv->pushbuf;
 216    struct nouveau_bufctx *bctx = nv50_context(&nv->pipe)->bufctx;
 217
 218    nouveau_bufctx_refn(bctx, 0, src, srcdom | NOUVEAU_BO_RD);
 219    nouveau_bufctx_refn(bctx, 0, dst, dstdom | NOUVEAU_BO_WR);
 220    nouveau_pushbuf_bufctx(push, bctx);
 221    nouveau_pushbuf_validate(push);
 222
 223    BEGIN_NV04(push, NV50_M2MF(LINEAR_IN), 1);
 224    PUSH_DATA (push, 1);
 225    BEGIN_NV04(push, NV50_M2MF(LINEAR_OUT), 1);
 226    PUSH_DATA (push, 1);
 227
 228    while (size) {
 229       unsigned bytes = MIN2(size, 1 << 17);
 230
 231       BEGIN_NV04(push, NV50_M2MF(OFFSET_IN_HIGH), 2);
 232       PUSH_DATAh(push, src->offset + srcoff);
 233       PUSH_DATAh(push, dst->offset + dstoff);
 234       BEGIN_NV04(push, SUBC_M2MF(NV03_M2MF_OFFSET_IN), 2);
 235       PUSH_DATA (push, src->offset + srcoff);
 236       PUSH_DATA (push, dst->offset + dstoff);
 237       BEGIN_NV04(push, SUBC_M2MF(NV03_M2MF_LINE_LENGTH_IN), 4);
 238       PUSH_DATA (push, bytes);
 239       PUSH_DATA (push, 1);
 240       PUSH_DATA (push, (1 << 8) | (1 << 0));
 241       PUSH_DATA (push, 0);
 242
 243       srcoff += bytes;
 244       dstoff += bytes;
 245       size -= bytes;
 246    }
 247
 248    nouveau_bufctx_reset(bctx, 0);
 249 }
 250
 251 void *
 252 nv50_miptree_transfer_map(struct pipe_context *pctx,
 253                           struct pipe_resource *res,
 254                           unsigned level,
 255                           unsigned usage,
 256                           const struct pipe_box *box,
 257                           struct pipe_transfer **ptransfer)
 258 {
 259    struct nv50_screen *screen = nv50_screen(pctx->screen);
 260    struct nv50_context *nv50 = nv50_context(pctx);
 261    struct nouveau_device *dev = nv50->screen->base.device;
 262    const struct nv50_miptree *mt = nv50_miptree(res);
 263    struct nv50_transfer *tx;
 264    uint32_t size;
 265    int ret;
 266    unsigned flags = 0;
 267
 268    if (usage & PIPE_TRANSFER_MAP_DIRECTLY)
 269       return NULL;
 270
 271    tx = CALLOC_STRUCT(nv50_transfer);
 272    if (!tx)
 273       return NULL;
 274
 275    pipe_resource_reference(&tx->base.resource, res);
 276
 277    tx->base.level = level;
 278    tx->base.usage = usage;
 279    tx->base.box = *box;
 280
 281    if (util_format_is_plain(res->format)) {
 282       tx->nblocksx = box->width << mt->ms_x;
 283       tx->nblocksy = box->height << mt->ms_y;
 284    } else {
 285       tx->nblocksx = util_format_get_nblocksx(res->format, box->width);
 286       tx->nblocksy = util_format_get_nblocksy(res->format, box->height);
 287    }
 288
 289    tx->base.stride = tx->nblocksx * util_format_get_blocksize(res->format);
 290    tx->base.layer_stride = tx->nblocksy * tx->base.stride;
 291
 292    nv50_m2mf_rect_setup(&tx->rect[0], res, level, box->x, box->y, box->z);
 293
 294    size = tx->base.layer_stride;
 295
 296    ret = nouveau_bo_new(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 0,
 297                         size * tx->base.box.depth, NULL, &tx->rect[1].bo);
 298    if (ret) {
 299       FREE(tx);
 300       return NULL;
 301    }
 302
 303    tx->rect[1].cpp = tx->rect[0].cpp;
 304    tx->rect[1].width = tx->nblocksx;
 305    tx->rect[1].height = tx->nblocksy;
 306    tx->rect[1].depth = 1;
 307    tx->rect[1].pitch = tx->base.stride;
 308    tx->rect[1].domain = NOUVEAU_BO_GART;
 309
 310    if (usage & PIPE_TRANSFER_READ) {
 311       unsigned base = tx->rect[0].base;
 312       unsigned z = tx->rect[0].z;
 313       unsigned i;
 314       for (i = 0; i < box->depth; ++i) {
 315          nv50_m2mf_transfer_rect(nv50, &tx->rect[1], &tx->rect[0],
 316                                  tx->nblocksx, tx->nblocksy);
 317          if (mt->layout_3d)
 318             tx->rect[0].z++;
 319          else
 320             tx->rect[0].base += mt->layer_stride;
 321          tx->rect[1].base += size;
 322       }
 323       tx->rect[0].z = z;
 324       tx->rect[0].base = base;
 325       tx->rect[1].base = 0;
 326    }
 327
 328    if (tx->rect[1].bo->map) {
 329       *ptransfer = &tx->base;
 330       return tx->rect[1].bo->map;
 331    }
 332
 333    if (usage & PIPE_TRANSFER_READ)
 334       flags = NOUVEAU_BO_RD;
 335    if (usage & PIPE_TRANSFER_WRITE)
 336       flags |= NOUVEAU_BO_WR;
 337
 338    ret = nouveau_bo_map(tx->rect[1].bo, flags, screen->base.client);
 339    if (ret) {
 340       nouveau_bo_ref(NULL, &tx->rect[1].bo);
 341       FREE(tx);
 342       return NULL;
 343    }
 344
 345    *ptransfer = &tx->base;
 346    return tx->rect[1].bo->map;
 347 }
 348
 349 void
 350 nv50_miptree_transfer_unmap(struct pipe_context *pctx,
 351                             struct pipe_transfer *transfer)
 352 {
 353    struct nv50_context *nv50 = nv50_context(pctx);
 354    struct nv50_transfer *tx = (struct nv50_transfer *)transfer;
 355    struct nv50_miptree *mt = nv50_miptree(tx->base.resource);
 356    unsigned i;
 357
 358    if (tx->base.usage & PIPE_TRANSFER_WRITE) {
 359       for (i = 0; i < tx->base.box.depth; ++i) {
 360          nv50_m2mf_transfer_rect(nv50, &tx->rect[0], &tx->rect[1],
 361                                  tx->nblocksx, tx->nblocksy);
 362          if (mt->layout_3d)
 363             tx->rect[0].z++;
 364          else
 365             tx->rect[0].base += mt->layer_stride;
 366          tx->rect[1].base += tx->nblocksy * tx->base.stride;
 367       }
 368
 369       /* Allow the copies above to finish executing before freeing the source */
 370       nouveau_fence_work(nv50->screen->base.fence.current,
 371                          nouveau_fence_unref_bo, tx->rect[1].bo);
 372    } else {
 373       nouveau_bo_ref(NULL, &tx->rect[1].bo);
 374    }
 375
 376    pipe_resource_reference(&transfer->resource, NULL);
 377
 378    FREE(tx);
 379 }
 380
 381 void
 382 nv50_cb_push(struct nouveau_context *nv,
 383              struct nouveau_bo *bo, unsigned domain,
 384              unsigned base, unsigned size,
 385              unsigned offset, unsigned words, const uint32_t *data)
 386 {
 387    struct nouveau_pushbuf *push = nv->pushbuf;
 388    struct nouveau_bufctx *bctx = nv50_context(&nv->pipe)->bufctx;
 389
 390    assert(!(offset & 3));
 391    size = align(size, 0x100);
 392
 393    nouveau_bufctx_refn(bctx, 0, bo, NOUVEAU_BO_WR | domain);
 394    nouveau_pushbuf_bufctx(push, bctx);
 395    nouveau_pushbuf_validate(push);
 396
 397    while (words) {
 398       unsigned nr;
 399
 400       nr = PUSH_AVAIL(push);
 401       nr = MIN2(nr - 7, words);
 402       nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN - 1);
 403
 404       BEGIN_NV04(push, NV50_3D(CB_DEF_ADDRESS_HIGH), 3);
 405       PUSH_DATAh(push, bo->offset + base);
 406       PUSH_DATA (push, bo->offset + base);
 407       PUSH_DATA (push, (NV50_CB_TMP << 16) | (size & 0xffff));
 408       BEGIN_NV04(push, NV50_3D(CB_ADDR), 1);
 409       PUSH_DATA (push, (offset << 6) | NV50_CB_TMP);
 410       BEGIN_NI04(push, NV50_3D(CB_DATA(0)), nr);
 411       PUSH_DATAp(push, data, nr);
 412
 413       words -= nr;
 414       data += nr;
 415       offset += nr * 4;
 416    }
 417
 418    nouveau_bufctx_reset(bctx, 0);
 419 }