src/gallium/drivers/nv50/nv50_transfer.c

   1
   2 #include "pipe/p_context.h"
   3 #include "pipe/p_inlines.h"
   4 #include "util/u_math.h"
   5
   6 #include "nv50_context.h"
   7
   8 struct nv50_transfer {
   9         struct pipe_transfer base;
  10         struct nouveau_bo *bo;
  11         unsigned level_offset;
  12         unsigned level_tiling;
  13         int level_pitch;
  14         int level_width;
  15         int level_height;
  16         int level_depth;
  17         int level_x;
  18         int level_y;
  19 };
  20
  21 static void
  22 nv50_transfer_rect_m2mf(struct pipe_screen *pscreen,
  23                         struct nouveau_bo *src_bo, unsigned src_offset,
  24                         int src_pitch, unsigned src_tile_mode,
  25                         int sx, int sy, int sw, int sh, int sd,
  26                         struct nouveau_bo *dst_bo, unsigned dst_offset,
  27                         int dst_pitch, unsigned dst_tile_mode,
  28                         int dx, int dy, int dw, int dh, int dd,
  29                         int cpp, int width, int height,
  30                         unsigned src_reloc, unsigned dst_reloc)
  31 {
  32         struct nv50_screen *screen = nv50_screen(pscreen);
  33         struct nouveau_channel *chan = screen->m2mf->channel;
  34         struct nouveau_grobj *m2mf = screen->m2mf;
  35
  36         src_reloc |= NOUVEAU_BO_RD;
  37         dst_reloc |= NOUVEAU_BO_WR;
  38
  39         WAIT_RING (chan, 14);
  40
  41         if (!src_bo->tile_flags) {
  42                 BEGIN_RING(chan, m2mf,
  43                         NV50_MEMORY_TO_MEMORY_FORMAT_LINEAR_IN, 1);
  44                 OUT_RING  (chan, 1);
  45                 BEGIN_RING(chan, m2mf,
  46                         NV50_MEMORY_TO_MEMORY_FORMAT_PITCH_IN, 1);
  47                 OUT_RING  (chan, src_pitch);
  48                 src_offset += (sy * src_pitch) + (sx * cpp);
  49         } else {
  50                 BEGIN_RING(chan, m2mf,
  51                         NV50_MEMORY_TO_MEMORY_FORMAT_LINEAR_IN, 6);
  52                 OUT_RING  (chan, 0);
  53                 OUT_RING  (chan, src_tile_mode << 4);
  54                 OUT_RING  (chan, sw * cpp);
  55                 OUT_RING  (chan, sh);
  56                 OUT_RING  (chan, sd);
  57                 OUT_RING  (chan, 0);
  58         }
  59
  60         if (!dst_bo->tile_flags) {
  61                 BEGIN_RING(chan, m2mf,
  62                         NV50_MEMORY_TO_MEMORY_FORMAT_LINEAR_OUT, 1);
  63                 OUT_RING  (chan, 1);
  64                 BEGIN_RING(chan, m2mf,
  65                         NV50_MEMORY_TO_MEMORY_FORMAT_PITCH_OUT, 1);
  66                 OUT_RING  (chan, dst_pitch);
  67                 dst_offset += (dy * dst_pitch) + (dx * cpp);
  68         } else {
  69                 BEGIN_RING(chan, m2mf,
  70                         NV50_MEMORY_TO_MEMORY_FORMAT_LINEAR_OUT, 6);
  71                 OUT_RING  (chan, 0);
  72                 OUT_RING  (chan, dst_tile_mode << 4);
  73                 OUT_RING  (chan, dw * cpp);
  74                 OUT_RING  (chan, dh);
  75                 OUT_RING  (chan, dd);
  76                 OUT_RING  (chan, 0);
  77         }
  78
  79         while (height) {
  80                 int line_count = height > 2047 ? 2047 : height;
  81
  82                 WAIT_RING (chan, 15);
  83                 BEGIN_RING(chan, m2mf,
  84                         NV50_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN_HIGH, 2);
  85                 OUT_RELOCh(chan, src_bo, src_offset, src_reloc);
  86                 OUT_RELOCh(chan, dst_bo, dst_offset, dst_reloc);
  87                 BEGIN_RING(chan, m2mf,
  88                         NV50_MEMORY_TO_MEMORY_FORMAT_OFFSET_IN, 2);
  89                 OUT_RELOCl(chan, src_bo, src_offset, src_reloc);
  90                 OUT_RELOCl(chan, dst_bo, dst_offset, dst_reloc);
  91                 if (src_bo->tile_flags) {
  92                         BEGIN_RING(chan, m2mf,
  93                                 NV50_MEMORY_TO_MEMORY_FORMAT_TILING_POSITION_IN, 1);
  94                         OUT_RING  (chan, (sy << 16) | (sx * cpp));
  95                 } else {
  96                         src_offset += (line_count * src_pitch);
  97                 }
  98                 if (dst_bo->tile_flags) {
  99                         BEGIN_RING(chan, m2mf,
 100                                 NV50_MEMORY_TO_MEMORY_FORMAT_TILING_POSITION_OUT, 1);
 101                         OUT_RING  (chan, (dy << 16) | (dx * cpp));
 102                 } else {
 103                         dst_offset += (line_count * dst_pitch);
 104                 }
 105                 BEGIN_RING(chan, m2mf,
 106                         NV50_MEMORY_TO_MEMORY_FORMAT_LINE_LENGTH_IN, 4);
 107                 OUT_RING  (chan, width * cpp);
 108                 OUT_RING  (chan, line_count);
 109                 OUT_RING  (chan, 0x00000101);
 110                 OUT_RING  (chan, 0);
 111                 FIRE_RING (chan);
 112
 113                 height -= line_count;
 114                 sy += line_count;
 115                 dy += line_count;
 116         }
 117 }
 118
 119 static INLINE unsigned
 120 get_zslice_offset(unsigned tile_mode, unsigned z, unsigned pitch, unsigned ny)
 121 {
 122         unsigned tile_h = get_tile_height(tile_mode);
 123         unsigned tile_d = get_tile_depth(tile_mode);
 124
 125         /* pitch_2d == to next slice within this volume-tile */
 126         /* pitch_3d == to next slice in next 2D array of blocks */
 127         unsigned pitch_2d = tile_h * 64;
 128         unsigned pitch_3d = tile_d * align(ny, tile_h) * pitch;
 129
 130         return (z % tile_d) * pitch_2d + (z / tile_d) * pitch_3d;
 131 }
 132
 133 static struct pipe_transfer *
 134 nv50_transfer_new(struct pipe_screen *pscreen, struct pipe_texture *pt,
 135                   unsigned face, unsigned level, unsigned zslice,
 136                   enum pipe_transfer_usage usage,
 137                   unsigned x, unsigned y, unsigned w, unsigned h)
 138 {
 139         struct nouveau_device *dev = nouveau_screen(pscreen)->device;
 140         struct nv50_miptree *mt = nv50_miptree(pt);
 141         struct nv50_miptree_level *lvl = &mt->level[level];
 142         struct nv50_transfer *tx;
 143         unsigned nx, ny, image = 0;
 144         int ret;
 145
 146         if (pt->target == PIPE_TEXTURE_CUBE)
 147                 image = face;
 148
 149         tx = CALLOC_STRUCT(nv50_transfer);
 150         if (!tx)
 151                 return NULL;
 152
 153         pipe_texture_reference(&tx->base.texture, pt);
 154         tx->base.format = pt->format;
 155         tx->base.width = w;
 156         tx->base.height = h;
 157         tx->base.block = pt->block;
 158         if (!pt->nblocksx[level]) {
 159                 tx->base.nblocksx = pf_get_nblocksx(&pt->block,
 160                                                     u_minify(pt->width0, level));
 161                 tx->base.nblocksy = pf_get_nblocksy(&pt->block,
 162                                                     u_minify(pt->height0, level));
 163         } else {
 164                 tx->base.nblocksx = pt->nblocksx[level];
 165                 tx->base.nblocksy = pt->nblocksy[level];
 166         }
 167         tx->base.stride = tx->base.nblocksx * pt->block.size;
 168         tx->base.usage = usage;
 169
 170         tx->level_pitch = lvl->pitch;
 171         tx->level_width = u_minify(mt->base.base.width0, level);
 172         tx->level_height = u_minify(mt->base.base.height0, level);
 173         tx->level_depth = u_minify(mt->base.base.depth0, level);
 174         tx->level_offset = lvl->image_offset[image];
 175         tx->level_tiling = lvl->tile_mode;
 176         tx->level_x = pf_get_nblocksx(&tx->base.block, x);
 177         tx->level_y = pf_get_nblocksy(&tx->base.block, y);
 178         ret = nouveau_bo_new(dev, NOUVEAU_BO_GART | NOUVEAU_BO_MAP, 0,
 179                              tx->base.nblocksy * tx->base.stride, &tx->bo);
 180         if (ret) {
 181                 FREE(tx);
 182                 return NULL;
 183         }
 184
 185         if (pt->target == PIPE_TEXTURE_3D)
 186                 tx->level_offset += get_zslice_offset(lvl->tile_mode, zslice,
 187                                                       lvl->pitch,
 188                                                       tx->base.nblocksy);
 189
 190         if (usage & PIPE_TRANSFER_READ) {
 191                 nx = pf_get_nblocksx(&tx->base.block, tx->base.width);
 192                 ny = pf_get_nblocksy(&tx->base.block, tx->base.height);
 193
 194                 nv50_transfer_rect_m2mf(pscreen, mt->base.bo, tx->level_offset,
 195                                         tx->level_pitch, tx->level_tiling,
 196                                         x, y,
 197                                         tx->base.nblocksx, tx->base.nblocksy,
 198                                         tx->level_depth,
 199                                         tx->bo, 0,
 200                                         tx->base.stride, tx->bo->tile_mode,
 201                                         0, 0,
 202                                         tx->base.nblocksx, tx->base.nblocksy, 1,
 203                                         tx->base.block.size, nx, ny,
 204                                         NOUVEAU_BO_VRAM | NOUVEAU_BO_GART,
 205                                         NOUVEAU_BO_GART);
 206         }
 207
 208         return &tx->base;
 209 }
 210
 211 static void
 212 nv50_transfer_del(struct pipe_transfer *ptx)
 213 {
 214         struct nv50_transfer *tx = (struct nv50_transfer *)ptx;
 215         struct nv50_miptree *mt = nv50_miptree(ptx->texture);
 216
 217         unsigned nx = pf_get_nblocksx(&tx->base.block, tx->base.width);
 218         unsigned ny = pf_get_nblocksy(&tx->base.block, tx->base.height);
 219
 220         if (ptx->usage & PIPE_TRANSFER_WRITE) {
 221                 struct pipe_screen *pscreen = ptx->texture->screen;
 222
 223                 nv50_transfer_rect_m2mf(pscreen, tx->bo, 0,
 224                                         tx->base.stride, tx->bo->tile_mode,
 225                                         0, 0,
 226                                         tx->base.nblocksx, tx->base.nblocksy, 1,
 227                                         mt->base.bo, tx->level_offset,
 228                                         tx->level_pitch, tx->level_tiling,
 229                                         tx->level_x, tx->level_y,
 230                                         tx->base.nblocksx, tx->base.nblocksy,
 231                                         tx->level_depth,
 232                                         tx->base.block.size, nx, ny,
 233                                         NOUVEAU_BO_GART, NOUVEAU_BO_VRAM |
 234                                         NOUVEAU_BO_GART);
 235         }
 236
 237         nouveau_bo_ref(NULL, &tx->bo);
 238         pipe_texture_reference(&ptx->texture, NULL);
 239         FREE(ptx);
 240 }
 241
 242 static void *
 243 nv50_transfer_map(struct pipe_screen *pscreen, struct pipe_transfer *ptx)
 244 {
 245         struct nv50_transfer *tx = (struct nv50_transfer *)ptx;
 246         unsigned flags = 0;
 247         int ret;
 248
 249         if (ptx->usage & PIPE_TRANSFER_WRITE)
 250                 flags |= NOUVEAU_BO_WR;
 251         if (ptx->usage & PIPE_TRANSFER_READ)
 252                 flags |= NOUVEAU_BO_RD;
 253
 254         ret = nouveau_bo_map(tx->bo, flags);
 255         if (ret)
 256                 return NULL;
 257         return tx->bo->map;
 258 }
 259
 260 static void
 261 nv50_transfer_unmap(struct pipe_screen *pscreen, struct pipe_transfer *ptx)
 262 {
 263         struct nv50_transfer *tx = (struct nv50_transfer *)ptx;
 264
 265         nouveau_bo_unmap(tx->bo);
 266 }
 267
 268 void
 269 nv50_transfer_init_screen_functions(struct pipe_screen *pscreen)
 270 {
 271         pscreen->get_tex_transfer = nv50_transfer_new;
 272         pscreen->tex_transfer_destroy = nv50_transfer_del;
 273         pscreen->transfer_map = nv50_transfer_map;
 274         pscreen->transfer_unmap = nv50_transfer_unmap;
 275 }
 276
 277 void
 278 nv50_upload_sifc(struct nv50_context *nv50,
 279                  struct nouveau_bo *bo, unsigned dst_offset, unsigned reloc,
 280                  unsigned dst_format, int dst_w, int dst_h, int dst_pitch,
 281                  void *src, unsigned src_format, int src_pitch,
 282                  int x, int y, int w, int h, int cpp)
 283 {
 284         struct nouveau_channel *chan = nv50->screen->base.channel;
 285         struct nouveau_grobj *eng2d = nv50->screen->eng2d;
 286         struct nouveau_grobj *tesla = nv50->screen->tesla;
 287         unsigned line_dwords = (w * cpp + 3) / 4;
 288
 289         reloc |= NOUVEAU_BO_WR;
 290
 291         WAIT_RING (chan, 32);
 292
 293         if (bo->tile_flags) {
 294                 BEGIN_RING(chan, eng2d, NV50_2D_DST_FORMAT, 5);
 295                 OUT_RING  (chan, dst_format);
 296                 OUT_RING  (chan, 0);
 297                 OUT_RING  (chan, bo->tile_mode << 4);
 298                 OUT_RING  (chan, 1);
 299                 OUT_RING  (chan, 0);
 300         } else {
 301                 BEGIN_RING(chan, eng2d, NV50_2D_DST_FORMAT, 2);
 302                 OUT_RING  (chan, dst_format);
 303                 OUT_RING  (chan, 1);
 304                 BEGIN_RING(chan, eng2d, NV50_2D_DST_PITCH, 1);
 305                 OUT_RING  (chan, dst_pitch);
 306         }
 307
 308         BEGIN_RING(chan, eng2d, NV50_2D_DST_WIDTH, 4);
 309         OUT_RING  (chan, dst_w);
 310         OUT_RING  (chan, dst_h);
 311         OUT_RELOCh(chan, bo, dst_offset, reloc);
 312         OUT_RELOCl(chan, bo, dst_offset, reloc);
 313
 314         /* NV50_2D_OPERATION_SRCCOPY assumed already set */
 315
 316         BEGIN_RING(chan, eng2d, NV50_2D_SIFC_UNK0800, 2);
 317         OUT_RING  (chan, 0);
 318         OUT_RING  (chan, src_format);
 319         BEGIN_RING(chan, eng2d, NV50_2D_SIFC_WIDTH, 10);
 320         OUT_RING  (chan, w);
 321         OUT_RING  (chan, h);
 322         OUT_RING  (chan, 0);
 323         OUT_RING  (chan, 1);
 324         OUT_RING  (chan, 0);
 325         OUT_RING  (chan, 1);
 326         OUT_RING  (chan, 0);
 327         OUT_RING  (chan, x);
 328         OUT_RING  (chan, 0);
 329         OUT_RING  (chan, y);
 330
 331         while (h--) {
 332                 const uint32_t *p = src;
 333                 unsigned count = line_dwords;
 334
 335                 while (count) {
 336                         unsigned nr = MIN2(count, 1792);
 337
 338                         if (chan->pushbuf->remaining <= nr) {
 339                                 FIRE_RING (chan);
 340
 341                                 BEGIN_RING(chan, eng2d,
 342                                            NV50_2D_DST_ADDRESS_HIGH, 2);
 343                                 OUT_RELOCh(chan, bo, dst_offset, reloc);
 344                                 OUT_RELOCl(chan, bo, dst_offset, reloc);
 345                         }
 346                         assert(chan->pushbuf->remaining > nr);
 347
 348                         BEGIN_RING(chan, eng2d,
 349                                    NV50_2D_SIFC_DATA | (2 << 29), nr);
 350                         OUT_RINGp (chan, p, nr);
 351
 352                         p += nr;
 353                         count -= nr;
 354                 }
 355
 356                 src += src_pitch;
 357         }
 358
 359         BEGIN_RING(chan, tesla, 0x1440, 1);
 360         OUT_RING  (chan, 0);
 361 }