src/gallium/drivers/nvc0/nvc0_transfer.c

   1
   2 #include "util/u_format.h"
   3
   4 #include "nvc0_context.h"
   5 #include "nvc0_transfer.h"
   6
   7 #include "nv50_defs.xml.h"
   8
   9 struct nvc0_transfer {
  10    struct pipe_transfer base;
  11    struct nvc0_m2mf_rect rect[2];
  12    uint32_t nblocksx;
  13    uint16_t nblocksy;
  14    uint16_t nlayers;
  15 };
  16
  17 static void
  18 nvc0_m2mf_transfer_rect(struct pipe_screen *pscreen,
  19                         const struct nvc0_m2mf_rect *dst,
  20                         const struct nvc0_m2mf_rect *src,
  21                         uint32_t nblocksx, uint32_t nblocksy)
  22 {
  23    struct nouveau_channel *chan = nouveau_screen(pscreen)->channel;
  24    const int cpp = dst->cpp;
  25    uint32_t src_ofst = src->base;
  26    uint32_t dst_ofst = dst->base;
  27    uint32_t height = nblocksy;
  28    uint32_t sy = src->y;
  29    uint32_t dy = dst->y;
  30    uint32_t exec = (1 << 20);
  31
  32    assert(dst->cpp == src->cpp);
  33
  34    if (nouveau_bo_tile_layout(src->bo)) {
  35       BEGIN_RING(chan, RING_MF(TILING_MODE_IN), 5);
  36       OUT_RING  (chan, src->tile_mode);
  37       OUT_RING  (chan, src->width * cpp);
  38       OUT_RING  (chan, src->height);
  39       OUT_RING  (chan, src->depth);
  40       OUT_RING  (chan, src->z);
  41    } else {
  42       src_ofst += src->y * src->pitch + src->x * cpp;
  43
  44       BEGIN_RING(chan, RING_MF(PITCH_IN), 1);
  45       OUT_RING  (chan, src->width * cpp);
  46
  47       exec |= NVC0_M2MF_EXEC_LINEAR_IN;
  48    }
  49
  50    if (nouveau_bo_tile_layout(dst->bo)) {
  51       BEGIN_RING(chan, RING_MF(TILING_MODE_OUT), 5);
  52       OUT_RING  (chan, dst->tile_mode);
  53       OUT_RING  (chan, dst->width * cpp);
  54       OUT_RING  (chan, dst->height);
  55       OUT_RING  (chan, dst->depth);
  56       OUT_RING  (chan, dst->z);
  57    } else {
  58       dst_ofst += dst->y * dst->pitch + dst->x * cpp;
  59
  60       BEGIN_RING(chan, RING_MF(PITCH_OUT), 1);
  61       OUT_RING  (chan, dst->width * cpp);
  62
  63       exec |= NVC0_M2MF_EXEC_LINEAR_OUT;
  64    }
  65
  66    while (height) {
  67       int line_count = height > 2047 ? 2047 : height;
  68
  69       MARK_RING (chan, 17, 4);
  70
  71       BEGIN_RING(chan, RING_MF(OFFSET_IN_HIGH), 2);
  72       OUT_RELOCh(chan, src->bo, src_ofst, src->domain | NOUVEAU_BO_RD);
  73       OUT_RELOCl(chan, src->bo, src_ofst, src->domain | NOUVEAU_BO_RD);
  74
  75       BEGIN_RING(chan, RING_MF(OFFSET_OUT_HIGH), 2);
  76       OUT_RELOCh(chan, dst->bo, dst_ofst, dst->domain | NOUVEAU_BO_WR);
  77       OUT_RELOCl(chan, dst->bo, dst_ofst, dst->domain | NOUVEAU_BO_WR);
  78
  79       if (!(exec & NVC0_M2MF_EXEC_LINEAR_IN)) {
  80          BEGIN_RING(chan, RING_MF(TILING_POSITION_IN_X), 2);
  81          OUT_RING  (chan, src->x * cpp);
  82          OUT_RING  (chan, sy);
  83       } else {
  84          src_ofst += line_count * src->pitch;
  85       }
  86       if (!(exec & NVC0_M2MF_EXEC_LINEAR_OUT)) {
  87          BEGIN_RING(chan, RING_MF(TILING_POSITION_OUT_X), 2);
  88          OUT_RING  (chan, dst->x * cpp);
  89          OUT_RING  (chan, dy);
  90       } else {
  91          dst_ofst += line_count * dst->pitch;
  92       }
  93
  94       BEGIN_RING(chan, RING_MF(LINE_LENGTH_IN), 2);
  95       OUT_RING  (chan, nblocksx * cpp);
  96       OUT_RING  (chan, line_count);
  97       BEGIN_RING(chan, RING_MF(EXEC), 1);
  98       OUT_RING  (chan, exec);
  99
 100       height -= line_count;
 101       sy += line_count;
 102       dy += line_count;
 103    }
 104 }
 105
 106 void
 107 nvc0_m2mf_push_linear(struct nvc0_context *nvc0,
 108                       struct nouveau_bo *dst, unsigned domain, int offset,
 109                       unsigned size, void *data)
 110 {
 111    struct nouveau_channel *chan = nvc0->screen->base.channel;
 112    uint32_t *src = (uint32_t *)data;
 113    unsigned count = (size + 3) / 4;
 114
 115    MARK_RING (chan, 8, 2);
 116
 117    BEGIN_RING(chan, RING_MF(OFFSET_OUT_HIGH), 2);
 118    OUT_RELOCh(chan, dst, offset, domain | NOUVEAU_BO_WR);
 119    OUT_RELOCl(chan, dst, offset, domain | NOUVEAU_BO_WR);
 120    BEGIN_RING(chan, RING_MF(LINE_LENGTH_IN), 2);
 121    OUT_RING  (chan, size);
 122    OUT_RING  (chan, 1);
 123    BEGIN_RING(chan, RING_MF(EXEC), 1);
 124    OUT_RING  (chan, 0x100111);
 125
 126    while (count) {
 127       unsigned nr = AVAIL_RING(chan);
 128
 129       if (nr < 9) {
 130          FIRE_RING(chan);
 131          nouveau_bo_validate(chan, dst, NOUVEAU_BO_WR);
 132          continue;
 133       }
 134       nr = MIN2(count, nr - 1);
 135       nr = MIN2(nr, NV04_PFIFO_MAX_PACKET_LEN);
 136
 137       BEGIN_RING_NI(chan, RING_MF(DATA), nr);
 138       OUT_RINGp (chan, src, nr);
 139
 140       src += nr;
 141       count -= nr;
 142    }
 143 }
 144
 145 void
 146 nvc0_m2mf_copy_linear(struct nvc0_context *nvc0,
 147                       struct nouveau_bo *dst, unsigned dstoff, unsigned dstdom,
 148                       struct nouveau_bo *src, unsigned srcoff, unsigned srcdom,
 149                       unsigned size)
 150 {
 151    struct nouveau_channel *chan = nvc0->screen->base.channel;
 152
 153    while (size) {
 154       unsigned bytes = MIN2(size, 1 << 17);
 155
 156       MARK_RING (chan, 11, 4);
 157
 158       BEGIN_RING(chan, RING_MF(OFFSET_OUT_HIGH), 2);
 159       OUT_RELOCh(chan, dst, dstoff, dstdom | NOUVEAU_BO_WR);
 160       OUT_RELOCl(chan, dst, dstoff, dstdom | NOUVEAU_BO_WR);
 161       BEGIN_RING(chan, RING_MF(OFFSET_IN_HIGH), 2);
 162       OUT_RELOCh(chan, src, srcoff, srcdom | NOUVEAU_BO_RD);
 163       OUT_RELOCl(chan, src, srcoff, srcdom | NOUVEAU_BO_RD);
 164       BEGIN_RING(chan, RING_MF(LINE_LENGTH_IN), 2);
 165       OUT_RING  (chan, bytes);
 166       OUT_RING  (chan, 1);
 167       BEGIN_RING(chan, RING_MF(EXEC), 1);
 168       OUT_RING  (chan, (1 << NVC0_M2MF_EXEC_INC__SHIFT) |
 169                  NVC0_M2MF_EXEC_LINEAR_IN | NVC0_M2MF_EXEC_LINEAR_OUT);
 170
 171       srcoff += bytes;
 172       dstoff += bytes;
 173       size -= bytes;
 174    }
 175 }
 176
 177 static void
 178 nvc0_m2mf_push_rect(struct pipe_screen *pscreen,
 179                     const struct nvc0_m2mf_rect *dst,
 180                     const void *data,
 181                     unsigned nblocksx, unsigned nblocksy)
 182 {
 183    struct nouveau_channel *chan;
 184    const uint8_t *src = (const uint8_t *)data;
 185    const int cpp = dst->cpp;
 186    const int line_len = nblocksx * cpp;
 187    int dy = dst->y;
 188
 189    assert(nouveau_bo_tile_layout(dst->bo));
 190
 191    BEGIN_RING(chan, RING_MF(TILING_MODE_OUT), 5);
 192    OUT_RING  (chan, dst->tile_mode);
 193    OUT_RING  (chan, dst->width * cpp);
 194    OUT_RING  (chan, dst->height);
 195    OUT_RING  (chan, dst->depth);
 196    OUT_RING  (chan, dst->z);
 197
 198    while (nblocksy) {
 199       int line_count, words;
 200       int size = MIN2(AVAIL_RING(chan), NV04_PFIFO_MAX_PACKET_LEN);
 201
 202       if (size < (12 + words)) {
 203          FIRE_RING(chan);
 204          continue;
 205       }
 206       line_count = (size * 4) / line_len;
 207       words = (line_count * line_len + 3) / 4;
 208
 209       BEGIN_RING(chan, RING_MF(OFFSET_OUT_HIGH), 2);
 210       OUT_RELOCh(chan, dst->bo, dst->base, dst->domain | NOUVEAU_BO_WR);
 211       OUT_RELOCl(chan, dst->bo, dst->base, dst->domain | NOUVEAU_BO_WR);
 212
 213       BEGIN_RING(chan, RING_MF(TILING_POSITION_OUT_X), 2);
 214       OUT_RING  (chan, dst->x * cpp);
 215       OUT_RING  (chan, dy);
 216       BEGIN_RING(chan, RING_MF(LINE_LENGTH_IN), 2);
 217       OUT_RING  (chan, line_len);
 218       OUT_RING  (chan, line_count);
 219       BEGIN_RING(chan, RING_MF(EXEC), 1);
 220       OUT_RING  (chan, (1 << NVC0_M2MF_EXEC_INC__SHIFT) |
 221                  NVC0_M2MF_EXEC_PUSH | NVC0_M2MF_EXEC_LINEAR_IN);
 222
 223       BEGIN_RING_NI(chan, RING_MF(DATA), words);
 224       OUT_RINGp (chan, src, words);
 225
 226       dy += line_count;
 227       src += line_len * line_count;
 228       nblocksy -= line_count;
 229    }
 230 }
 231
 232 struct pipe_transfer *
 233 nvc0_miptree_transfer_new(struct pipe_context *pctx,
 234                           struct pipe_resource *res,
 235                           unsigned level,
 236                           unsigned usage,
 237                           const struct pipe_box *box)
 238 {
 239    struct nvc0_context *nvc0 = nvc0_context(pctx);
 240    struct pipe_screen *pscreen = pctx->screen;
 241    struct nouveau_device *dev = nvc0->screen->base.device;
 242    struct nvc0_miptree *mt = nvc0_miptree(res);
 243    struct nvc0_miptree_level *lvl = &mt->level[level];
 244    struct nvc0_transfer *tx;
 245    uint32_t size;
 246    uint32_t w, h, d, z, layer, box_h, box_y;
 247    int ret;
 248
 249    tx = CALLOC_STRUCT(nvc0_transfer);
 250    if (!tx)
 251       return NULL;
 252
 253    box_y = box->y;
 254    box_h = box->height;
 255
 256    if (mt->layout_3d) {
 257       z = box->z;
 258       d = u_minify(res->depth0, level);
 259       layer = 0;
 260       tx->nlayers = box->depth;
 261    } else {
 262       z = 0;
 263       d = 1;
 264       if (res->target == PIPE_TEXTURE_1D ||
 265           res->target == PIPE_TEXTURE_1D_ARRAY) {
 266          box_y = 0;
 267          box_h = 1;
 268          layer = box->y;
 269          tx->nlayers = box->height;
 270       } else {
 271          layer = box->z;
 272          tx->nlayers = box->depth;
 273       }
 274    }
 275
 276    pipe_resource_reference(&tx->base.resource, res);
 277
 278    tx->base.level = level;
 279    tx->base.usage = usage;
 280    tx->base.box = *box;
 281
 282    tx->nblocksx = util_format_get_nblocksx(res->format, box->width);
 283    tx->nblocksy = util_format_get_nblocksy(res->format, box_h);
 284
 285    tx->base.stride = tx->nblocksx * util_format_get_blocksize(res->format);
 286    tx->base.layer_stride = tx->nblocksy * tx->base.stride;
 287
 288    w = u_minify(res->width0, level);
 289    h = u_minify(res->height0, level);
 290
 291    tx->rect[0].cpp = tx->rect[1].cpp = util_format_get_blocksize(res->format);
 292
 293    tx->rect[0].bo = mt->base.bo;
 294    tx->rect[0].base = lvl->offset + layer * mt->layer_stride;
 295    tx->rect[0].tile_mode = lvl->tile_mode;
 296    tx->rect[0].x = util_format_get_nblocksx(res->format, box->x);
 297    tx->rect[0].y = util_format_get_nblocksy(res->format, box_y);
 298    tx->rect[0].z = z;
 299    tx->rect[0].width = util_format_get_nblocksx(res->format, w);
 300    tx->rect[0].height = util_format_get_nblocksy(res->format, h);
 301    tx->rect[0].depth = d;
 302    tx->rect[0].pitch = lvl->pitch;
 303    tx->rect[0].domain = NOUVEAU_BO_VRAM;
 304
 305    size = tx->base.layer_stride;
 306
 307    ret = nouveau_bo_new(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 0,
 308                         size * tx->nlayers, &tx->rect[1].bo);
 309    if (ret) {
 310       FREE(tx);
 311       return NULL;
 312    }
 313
 314    tx->rect[1].width = tx->nblocksx;
 315    tx->rect[1].height = tx->nblocksy;
 316    tx->rect[1].depth = 1;
 317    tx->rect[1].pitch = tx->base.stride;
 318    tx->rect[1].domain = NOUVEAU_BO_GART;
 319
 320    if (usage & PIPE_TRANSFER_READ) {
 321       unsigned i;
 322       for (i = 0; i < tx->nlayers; ++i) {
 323          nvc0_m2mf_transfer_rect(pscreen, &tx->rect[1], &tx->rect[0],
 324                                  tx->nblocksx, tx->nblocksy);
 325          if (mt->layout_3d)
 326             tx->rect[0].z++;
 327          else
 328             tx->rect[0].base += mt->layer_stride;
 329          tx->rect[1].base += size;
 330       }
 331    }
 332    tx->rect[0].z = z;
 333    tx->rect[1].base = 0;
 334
 335    return &tx->base;
 336 }
 337
 338 void
 339 nvc0_miptree_transfer_del(struct pipe_context *pctx,
 340                           struct pipe_transfer *transfer)
 341 {
 342    struct pipe_screen *pscreen = pctx->screen;
 343    struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer;
 344    struct nvc0_miptree *mt = nvc0_miptree(tx->base.resource);
 345    unsigned i;
 346
 347    if (tx->base.usage & PIPE_TRANSFER_WRITE) {
 348       for (i = 0; i < tx->nlayers; ++i) {
 349          nvc0_m2mf_transfer_rect(pscreen, &tx->rect[0], &tx->rect[1],
 350                                  tx->nblocksx, tx->nblocksy);
 351          if (mt->layout_3d)
 352             tx->rect[0].z++;
 353          else
 354             tx->rect[0].base += mt->layer_stride;
 355          tx->rect[1].base += tx->nblocksy * tx->base.stride;
 356       }
 357    }
 358
 359    nouveau_bo_ref(NULL, &tx->rect[1].bo);
 360    pipe_resource_reference(&transfer->resource, NULL);
 361
 362    FREE(tx);
 363 }
 364
 365 void *
 366 nvc0_miptree_transfer_map(struct pipe_context *pctx,
 367                           struct pipe_transfer *transfer)
 368 {
 369    struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer;
 370    int ret;
 371    unsigned flags = 0;
 372
 373    if (tx->rect[1].bo->map)
 374       return tx->rect[1].bo->map;
 375
 376    if (transfer->usage & PIPE_TRANSFER_READ)
 377       flags = NOUVEAU_BO_RD;
 378    if (transfer->usage & PIPE_TRANSFER_WRITE)
 379       flags |= NOUVEAU_BO_WR;
 380
 381    ret = nouveau_bo_map(tx->rect[1].bo, flags);
 382    if (ret)
 383       return NULL;
 384    return tx->rect[1].bo->map;
 385 }
 386
 387 void
 388 nvc0_miptree_transfer_unmap(struct pipe_context *pctx,
 389                             struct pipe_transfer *transfer)
 390 {
 391    struct nvc0_transfer *tx = (struct nvc0_transfer *)transfer;
 392
 393    nouveau_bo_unmap(tx->rect[1].bo);
 394 }
 395