src/gallium/drivers/ilo/ilo_transfer.c

   1 /*
   2  * Mesa 3-D graphics library
   3  *
   4  * Copyright (C) 2012-2013 LunarG, Inc.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included
  14  * in all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22  * DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors:
  25  *    Chia-I Wu <olv@lunarg.com>
  26  */
  27
  28 #include "util/u_surface.h"
  29 #include "util/u_transfer.h"
  30 #include "util/u_format_etc.h"
  31
  32 #include "ilo_cp.h"
  33 #include "ilo_context.h"
  34 #include "ilo_resource.h"
  35 #include "ilo_transfer.h"
  36
  37 enum ilo_transfer_map_method {
  38    /* map() / map_gtt() / map_unsynchronized() */
  39    ILO_TRANSFER_MAP_CPU,
  40    ILO_TRANSFER_MAP_GTT,
  41    ILO_TRANSFER_MAP_UNSYNC,
  42
  43    /* use staging system buffer */
  44    ILO_TRANSFER_MAP_SW_CONVERT,
  45    ILO_TRANSFER_MAP_SW_ZS,
  46 };
  47
  48 struct ilo_transfer {
  49    struct pipe_transfer base;
  50
  51    enum ilo_transfer_map_method method;
  52    void *ptr;
  53
  54    void *staging_sys;
  55 };
  56
  57 static inline struct ilo_transfer *
  58 ilo_transfer(struct pipe_transfer *transfer)
  59 {
  60    return (struct ilo_transfer *) transfer;
  61 }
  62
  63 static bool
  64 is_bo_busy(struct ilo_context *ilo, struct intel_bo *bo, bool *need_flush)
  65 {
  66    const bool referenced = ilo->cp->bo->references(ilo->cp->bo, bo);
  67
  68    if (need_flush)
  69       *need_flush = referenced;
  70
  71    if (referenced)
  72       return true;
  73
  74    /*
  75     * XXX With hardware context support, the bo may be needed by GPU
  76     * without being referenced by ilo->cp->bo.  We have to flush
  77     * unconditionally, and that is bad.
  78     */
  79    if (ilo->cp->render_ctx)
  80       ilo_cp_flush(ilo->cp);
  81
  82    return intel_bo_is_busy(bo);
  83 }
  84
  85 static bool
  86 map_bo_for_transfer(struct ilo_context *ilo, struct intel_bo *bo,
  87                     const struct ilo_transfer *xfer)
  88 {
  89    int err;
  90
  91    switch (xfer->method) {
  92    case ILO_TRANSFER_MAP_CPU:
  93       err = bo->map(bo, (xfer->base.usage & PIPE_TRANSFER_WRITE));
  94       break;
  95    case ILO_TRANSFER_MAP_GTT:
  96       err = bo->map_gtt(bo);
  97       break;
  98    case ILO_TRANSFER_MAP_UNSYNC:
  99       err = bo->map_unsynchronized(bo);
 100       break;
 101    default:
 102       assert(!"unknown mapping method");
 103       err = -1;
 104       break;
 105    }
 106
 107    return !err;
 108 }
 109
 110 /**
 111  * Choose the best mapping method, depending on the transfer usage and whether
 112  * the bo is busy.
 113  */
 114 static bool
 115 choose_transfer_method(struct ilo_context *ilo, struct ilo_transfer *xfer)
 116 {
 117    struct pipe_resource *res = xfer->base.resource;
 118    const unsigned usage = xfer->base.usage;
 119    /* prefer map() when there is the last-level cache */
 120    const bool prefer_cpu =
 121       (ilo->dev->has_llc || (usage & PIPE_TRANSFER_READ));
 122    struct ilo_texture *tex;
 123    struct ilo_buffer *buf;
 124    struct intel_bo *bo;
 125    bool tiled, need_flush;
 126
 127    if (res->target == PIPE_BUFFER) {
 128       tex = NULL;
 129
 130       buf = ilo_buffer(res);
 131       bo = buf->bo;
 132       tiled = false;
 133    }
 134    else {
 135       buf = NULL;
 136
 137       tex = ilo_texture(res);
 138       bo = tex->bo;
 139       tiled = (tex->tiling != INTEL_TILING_NONE);
 140    }
 141
 142    /* choose between mapping through CPU or GTT */
 143    if (usage & PIPE_TRANSFER_MAP_DIRECTLY) {
 144       /* we do not want fencing */
 145       if (tiled || prefer_cpu)
 146          xfer->method = ILO_TRANSFER_MAP_CPU;
 147       else
 148          xfer->method = ILO_TRANSFER_MAP_GTT;
 149    }
 150    else {
 151       if (!tiled && prefer_cpu)
 152          xfer->method = ILO_TRANSFER_MAP_CPU;
 153       else
 154          xfer->method = ILO_TRANSFER_MAP_GTT;
 155    }
 156
 157    /* see if we can avoid stalling */
 158    if (is_bo_busy(ilo, bo, &need_flush)) {
 159       bool will_stall = true;
 160
 161       if (usage & PIPE_TRANSFER_MAP_DIRECTLY) {
 162          /* nothing we can do */
 163       }
 164       else if (usage & PIPE_TRANSFER_UNSYNCHRONIZED) {
 165          /* unsynchronized gtt mapping does not stall */
 166          xfer->method = ILO_TRANSFER_MAP_UNSYNC;
 167          will_stall = false;
 168       }
 169       else if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE) {
 170          /* discard old bo and allocate a new one for mapping */
 171          if ((tex && ilo_texture_alloc_bo(tex)) ||
 172              (buf && ilo_buffer_alloc_bo(buf)))
 173             will_stall = false;
 174       }
 175       else if (usage & PIPE_TRANSFER_FLUSH_EXPLICIT) {
 176          /*
 177           * We could allocate and return a system buffer here.  When a region of
 178           * the buffer is explicitly flushed, we pwrite() the region to a
 179           * temporary bo and emit pipelined copy blit.
 180           *
 181           * For now, do nothing.
 182           */
 183       }
 184       else if (usage & PIPE_TRANSFER_DISCARD_RANGE) {
 185          /*
 186           * We could allocate a temporary bo for mapping, and emit pipelined copy
 187           * blit upon unmapping.
 188           *
 189           * For now, do nothing.
 190           */
 191       }
 192
 193       if (will_stall) {
 194          if (usage & PIPE_TRANSFER_DONTBLOCK)
 195             return false;
 196
 197          /* flush to make bo busy (so that map() stalls as it should be) */
 198          if (need_flush)
 199             ilo_cp_flush(ilo->cp);
 200       }
 201    }
 202
 203    if (tex && !(usage & PIPE_TRANSFER_MAP_DIRECTLY)) {
 204       if (tex->separate_s8 || tex->bo_format == PIPE_FORMAT_S8_UINT)
 205          xfer->method = ILO_TRANSFER_MAP_SW_ZS;
 206       /* need to convert on-the-fly */
 207       else if (tex->bo_format != tex->base.format)
 208          xfer->method = ILO_TRANSFER_MAP_SW_CONVERT;
 209    }
 210
 211    return true;
 212 }
 213
 214 static void
 215 tex_get_box_origin(const struct ilo_texture *tex,
 216                    unsigned level, unsigned slice,
 217                    const struct pipe_box *box,
 218                    unsigned *mem_x, unsigned *mem_y)
 219 {
 220    unsigned x, y;
 221
 222    x = tex->slice_offsets[level][slice + box->z].x + box->x;
 223    y = tex->slice_offsets[level][slice + box->z].y + box->y;
 224
 225    assert(x % tex->block_width == 0 && y % tex->block_height == 0);
 226
 227    *mem_x = x / tex->block_width * tex->bo_cpp;
 228    *mem_y = y / tex->block_height;
 229 }
 230
 231 static unsigned
 232 tex_get_box_offset(const struct ilo_texture *tex, unsigned level,
 233                    const struct pipe_box *box)
 234 {
 235    unsigned mem_x, mem_y;
 236
 237    tex_get_box_origin(tex, level, 0, box, &mem_x, &mem_y);
 238
 239    return mem_y * tex->bo_stride + mem_x;
 240 }
 241
 242 static unsigned
 243 tex_get_slice_stride(const struct ilo_texture *tex, unsigned level)
 244 {
 245    unsigned qpitch;
 246
 247    /* there is no 3D array texture */
 248    assert(tex->base.array_size == 1 || tex->base.depth0 == 1);
 249
 250    if (tex->base.array_size == 1) {
 251       /* non-array, non-3D */
 252       if (tex->base.depth0 == 1)
 253          return 0;
 254
 255       /* only the first level has a fixed slice stride */
 256       if (level > 0) {
 257          assert(!"no slice stride for 3D texture with level > 0");
 258          return 0;
 259       }
 260    }
 261
 262    qpitch = tex->slice_offsets[level][1].y - tex->slice_offsets[level][0].y;
 263    assert(qpitch % tex->block_height == 0);
 264
 265    return (qpitch / tex->block_height) * tex->bo_stride;
 266 }
 267
 268 static unsigned
 269 tex_tile_x_swizzle(unsigned addr)
 270 {
 271    /*
 272     * From the Ivy Bridge PRM, volume 1 part 2, page 24:
 273     *
 274     *     "As shown in the tiling algorithm, the new address bit[6] should be:
 275     *
 276     *        Address bit[6] <= TiledAddr bit[6] XOR
 277     *                          TiledAddr bit[9] XOR
 278     *                          TiledAddr bit[10]"
 279     */
 280    return addr ^ (((addr >> 3) ^ (addr >> 4)) & 0x40);
 281 }
 282
 283 static unsigned
 284 tex_tile_y_swizzle(unsigned addr)
 285 {
 286    /*
 287     * From the Ivy Bridge PRM, volume 1 part 2, page 24:
 288     *
 289     *     "As shown in the tiling algorithm, The new address bit[6] becomes:
 290     *
 291     *        Address bit[6] <= TiledAddr bit[6] XOR
 292     *                          TiledAddr bit[9]"
 293     */
 294    return addr ^ ((addr >> 3) & 0x40);
 295 }
 296
 297 static unsigned
 298 tex_tile_x_offset(unsigned mem_x, unsigned mem_y,
 299                   unsigned tiles_per_row, bool swizzle)
 300 {
 301    /*
 302     * From the Sandy Bridge PRM, volume 1 part 2, page 21, we know that a
 303     * X-major tile has 8 rows and 32 OWord columns (512 bytes).  Tiles in the
 304     * tiled region are numbered in row-major order, starting from zero.  The
 305     * tile number can thus be calculated as follows:
 306     *
 307     *    tile = (mem_y / 8) * tiles_per_row + (mem_x / 512)
 308     *
 309     * OWords in that tile are also numbered in row-major order, starting from
 310     * zero.  The OWord number can thus be calculated as follows:
 311     *
 312     *    oword = (mem_y % 8) * 32 + ((mem_x % 512) / 16)
 313     *
 314     * and the tiled offset is
 315     *
 316     *    offset = tile * 4096 + oword * 16 + (mem_x % 16)
 317     *           = tile * 4096 + (mem_y % 8) * 512 + (mem_x % 512)
 318     */
 319    unsigned tile, offset;
 320
 321    tile = (mem_y >> 3) * tiles_per_row + (mem_x >> 9);
 322    offset = tile << 12 | (mem_y & 0x7) << 9 | (mem_x & 0x1ff);
 323
 324    return (swizzle) ? tex_tile_x_swizzle(offset) : offset;
 325 }
 326
 327 static unsigned
 328 tex_tile_y_offset(unsigned mem_x, unsigned mem_y,
 329                   unsigned tiles_per_row, bool swizzle)
 330 {
 331    /*
 332     * From the Sandy Bridge PRM, volume 1 part 2, page 22, we know that a
 333     * Y-major tile has 32 rows and 8 OWord columns (128 bytes).  Tiles in the
 334     * tiled region are numbered in row-major order, starting from zero.  The
 335     * tile number can thus be calculated as follows:
 336     *
 337     *    tile = (mem_y / 32) * tiles_per_row + (mem_x / 128)
 338     *
 339     * OWords in that tile are numbered in column-major order, starting from
 340     * zero.  The OWord number can thus be calculated as follows:
 341     *
 342     *    oword = ((mem_x % 128) / 16) * 32 + (mem_y % 32)
 343     *
 344     * and the tiled offset is
 345     *
 346     *    offset = tile * 4096 + oword * 16 + (mem_x % 16)
 347     */
 348    unsigned tile, oword, offset;
 349
 350    tile = (mem_y >> 5) * tiles_per_row + (mem_x >> 7);
 351    oword = (mem_x & 0x70) << 1 | (mem_y & 0x1f);
 352    offset = tile << 12 | oword << 4 | (mem_x & 0xf);
 353
 354    return (swizzle) ? tex_tile_y_swizzle(offset) : offset;
 355 }
 356
 357 static unsigned
 358 tex_tile_w_offset(unsigned mem_x, unsigned mem_y,
 359                   unsigned tiles_per_row, bool swizzle)
 360 {
 361    /*
 362     * From the Sandy Bridge PRM, volume 1 part 2, page 23, we know that a
 363     * W-major tile has 8 8x8-block rows and 8 8x8-block columns.  Tiles in the
 364     * tiled region are numbered in row-major order, starting from zero.  The
 365     * tile number can thus be calculated as follows:
 366     *
 367     *    tile = (mem_y / 64) * tiles_per_row + (mem_x / 64)
 368     *
 369     * 8x8-blocks in that tile are numbered in column-major order, starting
 370     * from zero.  The 8x8-block number can thus be calculated as follows:
 371     *
 372     *    blk8 = ((mem_x % 64) / 8) * 8 + ((mem_y % 64) / 8)
 373     *
 374     * Each 8x8-block is divided into 4 4x4-blocks, in row-major order.  Each
 375     * 4x4-block is further divided into 4 2x2-blocks, also in row-major order.
 376     * We have
 377     *
 378     *    blk4 = (((mem_y % 64) / 4) & 1) * 2 + (((mem_x % 64) / 4) & 1)
 379     *    blk2 = (((mem_y % 64) / 2) & 1) * 2 + (((mem_x % 64) / 2) & 1)
 380     *    blk1 = (((mem_y % 64)    ) & 1) * 2 + (((mem_x % 64)    ) & 1)
 381     *
 382     * and the tiled offset is
 383     *
 384     *    offset = tile * 4096 + blk8 * 64 + blk4 * 16 + blk2 * 4 + blk1
 385     */
 386    unsigned tile, blk8, blk4, blk2, blk1, offset;
 387
 388    tile = (mem_y >> 6) * tiles_per_row + (mem_x >> 6);
 389    blk8 = ((mem_x >> 3) & 0x7) << 3 | ((mem_y >> 3) & 0x7);
 390    blk4 = ((mem_y >> 2) & 0x1) << 1 | ((mem_x >> 2) & 0x1);
 391    blk2 = ((mem_y >> 1) & 0x1) << 1 | ((mem_x >> 1) & 0x1);
 392    blk1 = ((mem_y     ) & 0x1) << 1 | ((mem_x     ) & 0x1);
 393    offset = tile << 12 | blk8 << 6 | blk4 << 4 | blk2 << 2 | blk1;
 394
 395    return (swizzle) ? tex_tile_y_swizzle(offset) : offset;
 396 }
 397
 398 static unsigned
 399 tex_tile_none_offset(unsigned mem_x, unsigned mem_y,
 400                      unsigned tiles_per_row, bool swizzle)
 401 {
 402    return mem_y * tiles_per_row + mem_x;
 403 }
 404
 405 typedef unsigned (*tex_tile_offset_func)(unsigned mem_x, unsigned mem_y,
 406                                          unsigned tiles_per_row,
 407                                          bool swizzle);
 408
 409 static tex_tile_offset_func
 410 tex_tile_choose_offset_func(const struct ilo_texture *tex,
 411                             unsigned *tiles_per_row)
 412 {
 413    switch (tex->tiling) {
 414    case INTEL_TILING_X:
 415       *tiles_per_row = tex->bo_stride / 512;
 416       return tex_tile_x_offset;
 417    case INTEL_TILING_Y:
 418       *tiles_per_row = tex->bo_stride / 128;
 419       return tex_tile_y_offset;
 420    case INTEL_TILING_NONE:
 421    default:
 422       /* W-tiling */
 423       if (tex->bo_format == PIPE_FORMAT_S8_UINT) {
 424          *tiles_per_row = tex->bo_stride / 64;
 425          return tex_tile_w_offset;
 426       }
 427       else {
 428          *tiles_per_row = tex->bo_stride;
 429          return tex_tile_none_offset;
 430       }
 431    }
 432 }
 433
 434 static void
 435 tex_staging_sys_zs_read(struct ilo_context *ilo,
 436                         struct ilo_texture *tex,
 437                         const struct ilo_transfer *xfer)
 438 {
 439    const bool swizzle = ilo->dev->has_address_swizzling;
 440    const struct pipe_box *box = &xfer->base.box;
 441    const uint8_t *src = tex->bo->get_virtual(tex->bo);
 442    tex_tile_offset_func tile_offset;
 443    unsigned tiles_per_row;
 444    int slice;
 445
 446    tile_offset = tex_tile_choose_offset_func(tex, &tiles_per_row);
 447
 448    assert(tex->block_width == 1 && tex->block_height == 1);
 449
 450    if (tex->separate_s8) {
 451       struct ilo_texture *s8_tex = tex->separate_s8;
 452       const uint8_t *s8_src = s8_tex->bo->get_virtual(s8_tex->bo);
 453       tex_tile_offset_func s8_tile_offset;
 454       unsigned s8_tiles_per_row;
 455       int dst_cpp, dst_s8_pos, src_cpp_used;
 456
 457       s8_tile_offset = tex_tile_choose_offset_func(s8_tex, &s8_tiles_per_row);
 458
 459       if (tex->base.format == PIPE_FORMAT_Z24_UNORM_S8_UINT) {
 460          assert(tex->bo_format == PIPE_FORMAT_Z24X8_UNORM);
 461
 462          dst_cpp = 4;
 463          dst_s8_pos = 3;
 464          src_cpp_used = 3;
 465       }
 466       else {
 467          assert(tex->base.format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT);
 468          assert(tex->bo_format == PIPE_FORMAT_Z32_FLOAT);
 469
 470          dst_cpp = 8;
 471          dst_s8_pos = 4;
 472          src_cpp_used = 4;
 473       }
 474
 475       for (slice = 0; slice < box->depth; slice++) {
 476          unsigned mem_x, mem_y, s8_mem_x, s8_mem_y;
 477          uint8_t *dst;
 478          int i, j;
 479
 480          tex_get_box_origin(tex, xfer->base.level, slice,
 481                             box, &mem_x, &mem_y);
 482          tex_get_box_origin(s8_tex, xfer->base.level, slice,
 483                             box, &s8_mem_x, &s8_mem_y);
 484
 485          dst = xfer->staging_sys + xfer->base.layer_stride * slice;
 486
 487          for (i = 0; i < box->height; i++) {
 488             unsigned x = mem_x, s8_x = s8_mem_x;
 489             uint8_t *d = dst;
 490
 491             for (j = 0; j < box->width; j++) {
 492                const unsigned offset =
 493                   tile_offset(x, mem_y, tiles_per_row, swizzle);
 494                const unsigned s8_offset =
 495                   s8_tile_offset(s8_x, s8_mem_y, s8_tiles_per_row, swizzle);
 496
 497                memcpy(d, src + offset, src_cpp_used);
 498                d[dst_s8_pos] = s8_src[s8_offset];
 499
 500                d += dst_cpp;
 501                x += tex->bo_cpp;
 502                s8_x++;
 503             }
 504
 505             dst += xfer->base.stride;
 506             mem_y++;
 507             s8_mem_y++;
 508          }
 509       }
 510    }
 511    else {
 512       assert(tex->bo_format == PIPE_FORMAT_S8_UINT);
 513
 514       for (slice = 0; slice < box->depth; slice++) {
 515          unsigned mem_x, mem_y;
 516          uint8_t *dst;
 517          int i, j;
 518
 519          tex_get_box_origin(tex, xfer->base.level, slice,
 520                             box, &mem_x, &mem_y);
 521
 522          dst = xfer->staging_sys + xfer->base.layer_stride * slice;
 523
 524          for (i = 0; i < box->height; i++) {
 525             unsigned x = mem_x;
 526             uint8_t *d = dst;
 527
 528             for (j = 0; j < box->width; j++) {
 529                const unsigned offset =
 530                   tile_offset(x, mem_y, tiles_per_row, swizzle);
 531
 532                *d = src[offset];
 533
 534                d++;
 535                x++;
 536             }
 537
 538             dst += xfer->base.stride;
 539             mem_y++;
 540          }
 541       }
 542    }
 543 }
 544
 545 static void
 546 tex_staging_sys_zs_write(struct ilo_context *ilo,
 547                          struct ilo_texture *tex,
 548                          const struct ilo_transfer *xfer)
 549 {
 550    const bool swizzle = ilo->dev->has_address_swizzling;
 551    const struct pipe_box *box = &xfer->base.box;
 552    uint8_t *dst = tex->bo->get_virtual(tex->bo);
 553    tex_tile_offset_func tile_offset;
 554    unsigned tiles_per_row;
 555    int slice;
 556
 557    tile_offset = tex_tile_choose_offset_func(tex, &tiles_per_row);
 558
 559    assert(tex->block_width == 1 && tex->block_height == 1);
 560
 561    if (tex->separate_s8) {
 562       struct ilo_texture *s8_tex = tex->separate_s8;
 563       uint8_t *s8_dst = s8_tex->bo->get_virtual(s8_tex->bo);
 564       tex_tile_offset_func s8_tile_offset;
 565       unsigned s8_tiles_per_row;
 566       int src_cpp, src_s8_pos, dst_cpp_used;
 567
 568       s8_tile_offset = tex_tile_choose_offset_func(s8_tex, &s8_tiles_per_row);
 569
 570       if (tex->base.format == PIPE_FORMAT_Z24_UNORM_S8_UINT) {
 571          assert(tex->bo_format == PIPE_FORMAT_Z24X8_UNORM);
 572
 573          src_cpp = 4;
 574          src_s8_pos = 3;
 575          dst_cpp_used = 3;
 576       }
 577       else {
 578          assert(tex->base.format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT);
 579          assert(tex->bo_format == PIPE_FORMAT_Z32_FLOAT);
 580
 581          src_cpp = 8;
 582          src_s8_pos = 4;
 583          dst_cpp_used = 4;
 584       }
 585
 586       for (slice = 0; slice < box->depth; slice++) {
 587          unsigned mem_x, mem_y, s8_mem_x, s8_mem_y;
 588          const uint8_t *src;
 589          int i, j;
 590
 591          tex_get_box_origin(tex, xfer->base.level, slice,
 592                             box, &mem_x, &mem_y);
 593          tex_get_box_origin(s8_tex, xfer->base.level, slice,
 594                             box, &s8_mem_x, &s8_mem_y);
 595
 596          src = xfer->staging_sys + xfer->base.layer_stride * slice;
 597
 598          for (i = 0; i < box->height; i++) {
 599             unsigned x = mem_x, s8_x = s8_mem_x;
 600             const uint8_t *s = src;
 601
 602             for (j = 0; j < box->width; j++) {
 603                const unsigned offset =
 604                   tile_offset(x, mem_y, tiles_per_row, swizzle);
 605                const unsigned s8_offset =
 606                   s8_tile_offset(s8_x, s8_mem_y, s8_tiles_per_row, swizzle);
 607
 608                memcpy(dst + offset, s, dst_cpp_used);
 609                s8_dst[s8_offset] = s[src_s8_pos];
 610
 611                s += src_cpp;
 612                x += tex->bo_cpp;
 613                s8_x++;
 614             }
 615
 616             src += xfer->base.stride;
 617             mem_y++;
 618             s8_mem_y++;
 619          }
 620       }
 621    }
 622    else {
 623       assert(tex->bo_format == PIPE_FORMAT_S8_UINT);
 624
 625       for (slice = 0; slice < box->depth; slice++) {
 626          unsigned mem_x, mem_y;
 627          const uint8_t *src;
 628          int i, j;
 629
 630          tex_get_box_origin(tex, xfer->base.level, slice,
 631                             box, &mem_x, &mem_y);
 632
 633          src = xfer->staging_sys + xfer->base.layer_stride * slice;
 634
 635          for (i = 0; i < box->height; i++) {
 636             unsigned x = mem_x;
 637             const uint8_t *s = src;
 638
 639             for (j = 0; j < box->width; j++) {
 640                const unsigned offset =
 641                   tile_offset(x, mem_y, tiles_per_row, swizzle);
 642
 643                dst[offset] = *s;
 644
 645                s++;
 646                x++;
 647             }
 648
 649             src += xfer->base.stride;
 650             mem_y++;
 651          }
 652       }
 653    }
 654 }
 655
 656 static void
 657 tex_staging_sys_convert_write(struct ilo_context *ilo,
 658                               struct ilo_texture *tex,
 659                               const struct ilo_transfer *xfer)
 660 {
 661    const struct pipe_box *box = &xfer->base.box;
 662    unsigned dst_slice_stride;
 663    void *dst;
 664    int slice;
 665
 666    dst = tex->bo->get_virtual(tex->bo);
 667    dst += tex_get_box_offset(tex, xfer->base.level, box);
 668
 669    /* slice stride is not always available */
 670    if (box->depth > 1)
 671       dst_slice_stride = tex_get_slice_stride(tex, xfer->base.level);
 672    else
 673       dst_slice_stride = 0;
 674
 675    if (unlikely(tex->bo_format == tex->base.format)) {
 676       util_copy_box(dst, tex->bo_format, tex->bo_stride, dst_slice_stride,
 677             0, 0, 0, box->width, box->height, box->depth,
 678             xfer->staging_sys, xfer->base.stride, xfer->base.layer_stride,
 679             0, 0, 0);
 680       return;
 681    }
 682
 683    switch (tex->base.format) {
 684    case PIPE_FORMAT_ETC1_RGB8:
 685       assert(tex->bo_format == PIPE_FORMAT_R8G8B8X8_UNORM);
 686
 687       for (slice = 0; slice < box->depth; slice++) {
 688          const void *src =
 689             xfer->staging_sys + xfer->base.layer_stride * slice;
 690
 691          util_format_etc1_rgb8_unpack_rgba_8unorm(dst,
 692                tex->bo_stride, src, xfer->base.stride,
 693                box->width, box->height);
 694
 695          dst += dst_slice_stride;
 696       }
 697       break;
 698    default:
 699       assert(!"unable to convert the staging data");
 700       break;
 701    }
 702 }
 703
 704 static bool
 705 tex_staging_sys_map_bo(const struct ilo_context *ilo,
 706                        const struct ilo_texture *tex,
 707                        bool for_read_back, bool linear_view)
 708 {
 709    const bool prefer_cpu = (ilo->dev->has_llc || for_read_back);
 710    int err;
 711
 712    if (prefer_cpu && (tex->tiling == INTEL_TILING_NONE || !linear_view))
 713       err = tex->bo->map(tex->bo, !for_read_back);
 714    else
 715       err = tex->bo->map_gtt(tex->bo);
 716
 717    if (!tex->separate_s8)
 718       return !err;
 719
 720    err = tex->separate_s8->bo->map(tex->separate_s8->bo, !for_read_back);
 721    if (err)
 722       tex->bo->unmap(tex->bo);
 723
 724    return !err;
 725 }
 726
 727 static void
 728 tex_staging_sys_unmap_bo(const struct ilo_context *ilo,
 729                          const struct ilo_texture *tex)
 730 {
 731    if (tex->separate_s8)
 732       tex->separate_s8->bo->unmap(tex->separate_s8->bo);
 733
 734    tex->bo->unmap(tex->bo);
 735 }
 736
 737 static void
 738 tex_staging_sys_unmap(struct ilo_context *ilo,
 739                       struct ilo_texture *tex,
 740                       struct ilo_transfer *xfer)
 741 {
 742    bool success;
 743
 744    if (!(xfer->base.usage & PIPE_TRANSFER_WRITE)) {
 745       FREE(xfer->staging_sys);
 746       return;
 747    }
 748
 749    switch (xfer->method) {
 750    case ILO_TRANSFER_MAP_SW_CONVERT:
 751       success = tex_staging_sys_map_bo(ilo, tex, false, true);
 752       if (success) {
 753          tex_staging_sys_convert_write(ilo, tex, xfer);
 754          tex_staging_sys_unmap_bo(ilo, tex);
 755       }
 756       break;
 757    case ILO_TRANSFER_MAP_SW_ZS:
 758       success = tex_staging_sys_map_bo(ilo, tex, false, false);
 759       if (success) {
 760          tex_staging_sys_zs_write(ilo, tex, xfer);
 761          tex_staging_sys_unmap_bo(ilo, tex);
 762       }
 763       break;
 764    default:
 765       assert(!"unknown mapping method");
 766       success = false;
 767       break;
 768    }
 769
 770    if (!success)
 771       ilo_err("failed to map resource for moving staging data\n");
 772
 773    FREE(xfer->staging_sys);
 774 }
 775
 776 static bool
 777 tex_staging_sys_map(struct ilo_context *ilo,
 778                     struct ilo_texture *tex,
 779                     struct ilo_transfer *xfer)
 780 {
 781    const struct pipe_box *box = &xfer->base.box;
 782    const size_t stride = util_format_get_stride(tex->base.format, box->width);
 783    const size_t size =
 784       util_format_get_2d_size(tex->base.format, stride, box->height);
 785    bool read_back = false, success;
 786
 787    xfer->staging_sys = MALLOC(size * box->depth);
 788    if (!xfer->staging_sys)
 789       return false;
 790
 791    xfer->base.stride = stride;
 792    xfer->base.layer_stride = size;
 793    xfer->ptr = xfer->staging_sys;
 794
 795    /* see if we need to read the resource back */
 796    if (xfer->base.usage & PIPE_TRANSFER_READ) {
 797       read_back = true;
 798    }
 799    else if (xfer->base.usage & PIPE_TRANSFER_WRITE) {
 800       const unsigned discard_flags =
 801          (PIPE_TRANSFER_DISCARD_RANGE | PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE);
 802
 803       if (!(xfer->base.usage & discard_flags))
 804          read_back = true;
 805    }
 806
 807    if (!read_back)
 808       return true;
 809
 810    switch (xfer->method) {
 811    case ILO_TRANSFER_MAP_SW_CONVERT:
 812       assert(!"no on-the-fly format conversion for mapping");
 813       success = false;
 814       break;
 815    case ILO_TRANSFER_MAP_SW_ZS:
 816       success = tex_staging_sys_map_bo(ilo, tex, true, false);
 817       if (success) {
 818          tex_staging_sys_zs_read(ilo, tex, xfer);
 819          tex_staging_sys_unmap_bo(ilo, tex);
 820       }
 821       break;
 822    default:
 823       assert(!"unknown mapping method");
 824       success = false;
 825       break;
 826    }
 827
 828    return success;
 829 }
 830
 831 static void
 832 tex_direct_unmap(struct ilo_context *ilo,
 833                  struct ilo_texture *tex,
 834                  struct ilo_transfer *xfer)
 835 {
 836    tex->bo->unmap(tex->bo);
 837 }
 838
 839 static bool
 840 tex_direct_map(struct ilo_context *ilo,
 841                struct ilo_texture *tex,
 842                struct ilo_transfer *xfer)
 843 {
 844    if (!map_bo_for_transfer(ilo, tex->bo, xfer))
 845       return false;
 846
 847    /* note that stride is for a block row, not a texel row */
 848    xfer->base.stride = tex->bo_stride;
 849
 850    /* slice stride is not always available */
 851    if (xfer->base.box.depth > 1)
 852       xfer->base.layer_stride = tex_get_slice_stride(tex, xfer->base.level);
 853    else
 854       xfer->base.layer_stride = 0;
 855
 856    xfer->ptr = tex->bo->get_virtual(tex->bo);
 857    xfer->ptr += tex_get_box_offset(tex, xfer->base.level, &xfer->base.box);
 858
 859    return true;
 860 }
 861
 862 static bool
 863 tex_map(struct ilo_context *ilo, struct ilo_transfer *xfer)
 864 {
 865    struct ilo_texture *tex = ilo_texture(xfer->base.resource);
 866    bool success;
 867
 868    if (!choose_transfer_method(ilo, xfer))
 869       return false;
 870
 871    switch (xfer->method) {
 872    case ILO_TRANSFER_MAP_CPU:
 873    case ILO_TRANSFER_MAP_GTT:
 874    case ILO_TRANSFER_MAP_UNSYNC:
 875       success = tex_direct_map(ilo, tex, xfer);
 876       break;
 877    case ILO_TRANSFER_MAP_SW_CONVERT:
 878    case ILO_TRANSFER_MAP_SW_ZS:
 879       success = tex_staging_sys_map(ilo, tex, xfer);
 880       break;
 881    default:
 882       assert(!"unknown mapping method");
 883       success = false;
 884       break;
 885    }
 886
 887    return success;
 888 }
 889
 890 static void
 891 tex_unmap(struct ilo_context *ilo, struct ilo_transfer *xfer)
 892 {
 893    struct ilo_texture *tex = ilo_texture(xfer->base.resource);
 894
 895    switch (xfer->method) {
 896    case ILO_TRANSFER_MAP_CPU:
 897    case ILO_TRANSFER_MAP_GTT:
 898    case ILO_TRANSFER_MAP_UNSYNC:
 899       tex_direct_unmap(ilo, tex, xfer);
 900       break;
 901    case ILO_TRANSFER_MAP_SW_CONVERT:
 902    case ILO_TRANSFER_MAP_SW_ZS:
 903       tex_staging_sys_unmap(ilo, tex, xfer);
 904       break;
 905    default:
 906       assert(!"unknown mapping method");
 907       break;
 908    }
 909 }
 910
 911 static bool
 912 buf_map(struct ilo_context *ilo, struct ilo_transfer *xfer)
 913 {
 914    struct ilo_buffer *buf = ilo_buffer(xfer->base.resource);
 915
 916    if (!choose_transfer_method(ilo, xfer))
 917       return false;
 918
 919    if (!map_bo_for_transfer(ilo, buf->bo, xfer))
 920       return false;
 921
 922    assert(xfer->base.level == 0);
 923    assert(xfer->base.box.y == 0);
 924    assert(xfer->base.box.z == 0);
 925    assert(xfer->base.box.height == 1);
 926    assert(xfer->base.box.depth == 1);
 927
 928    xfer->base.stride = 0;
 929    xfer->base.layer_stride = 0;
 930
 931    xfer->ptr = buf->bo->get_virtual(buf->bo);
 932    xfer->ptr += xfer->base.box.x;
 933
 934    return true;
 935 }
 936
 937 static void
 938 buf_unmap(struct ilo_context *ilo, struct ilo_transfer *xfer)
 939 {
 940    struct ilo_buffer *buf = ilo_buffer(xfer->base.resource);
 941
 942    buf->bo->unmap(buf->bo);
 943 }
 944
 945 static void
 946 buf_pwrite(struct ilo_context *ilo, struct ilo_buffer *buf,
 947            unsigned usage, int offset, int size, const void *data)
 948 {
 949    bool need_flush;
 950
 951    /* see if we can avoid stalling */
 952    if (is_bo_busy(ilo, buf->bo, &need_flush)) {
 953       bool will_stall = true;
 954
 955       if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE) {
 956          /* old data not needed so discard the old bo to avoid stalling */
 957          if (ilo_buffer_alloc_bo(buf))
 958             will_stall = false;
 959       }
 960       else {
 961          /*
 962           * We could allocate a temporary bo to hold the data and emit
 963           * pipelined copy blit to move them to buf->bo.  But for now, do
 964           * nothing.
 965           */
 966       }
 967
 968       /* flush to make bo busy (so that pwrite() stalls as it should be) */
 969       if (will_stall && need_flush)
 970          ilo_cp_flush(ilo->cp);
 971    }
 972
 973    buf->bo->pwrite(buf->bo, offset, size, data);
 974 }
 975
 976 static void
 977 ilo_transfer_flush_region(struct pipe_context *pipe,
 978                           struct pipe_transfer *transfer,
 979                           const struct pipe_box *box)
 980 {
 981 }
 982
 983 static void
 984 ilo_transfer_unmap(struct pipe_context *pipe,
 985                    struct pipe_transfer *transfer)
 986 {
 987    struct ilo_context *ilo = ilo_context(pipe);
 988    struct ilo_transfer *xfer = ilo_transfer(transfer);
 989
 990    if (xfer->base.resource->target == PIPE_BUFFER)
 991       buf_unmap(ilo, xfer);
 992    else
 993       tex_unmap(ilo, xfer);
 994
 995    pipe_resource_reference(&xfer->base.resource, NULL);
 996    FREE(xfer);
 997 }
 998
 999 static void *
1000 ilo_transfer_map(struct pipe_context *pipe,
1001                  struct pipe_resource *res,
1002                  unsigned level,
1003                  unsigned usage,
1004                  const struct pipe_box *box,
1005                  struct pipe_transfer **transfer)
1006 {
1007    struct ilo_context *ilo = ilo_context(pipe);
1008    struct ilo_transfer *xfer;
1009    bool success;
1010
1011    xfer = MALLOC_STRUCT(ilo_transfer);
1012    if (!xfer) {
1013       *transfer = NULL;
1014       return NULL;
1015    }
1016
1017    xfer->base.resource = NULL;
1018    pipe_resource_reference(&xfer->base.resource, res);
1019    xfer->base.level = level;
1020    xfer->base.usage = usage;
1021    xfer->base.box = *box;
1022
1023    if (res->target == PIPE_BUFFER)
1024       success = buf_map(ilo, xfer);
1025    else
1026       success = tex_map(ilo, xfer);
1027
1028    if (!success) {
1029       pipe_resource_reference(&xfer->base.resource, NULL);
1030       FREE(xfer);
1031       *transfer = NULL;
1032       return NULL;
1033    }
1034
1035    *transfer = &xfer->base;
1036
1037    return xfer->ptr;
1038 }
1039
1040 static void
1041 ilo_transfer_inline_write(struct pipe_context *pipe,
1042                           struct pipe_resource *res,
1043                           unsigned level,
1044                           unsigned usage,
1045                           const struct pipe_box *box,
1046                           const void *data,
1047                           unsigned stride,
1048                           unsigned layer_stride)
1049 {
1050    if (likely(res->target == PIPE_BUFFER) &&
1051        !(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
1052       /* they should specify just an offset and a size */
1053       assert(level == 0);
1054       assert(box->y == 0);
1055       assert(box->z == 0);
1056       assert(box->height == 1);
1057       assert(box->depth == 1);
1058
1059       buf_pwrite(ilo_context(pipe), ilo_buffer(res),
1060             usage, box->x, box->width, data);
1061    }
1062    else {
1063       u_default_transfer_inline_write(pipe, res,
1064             level, usage, box, data, stride, layer_stride);
1065    }
1066 }
1067
1068 /**
1069  * Initialize transfer-related functions.
1070  */
1071 void
1072 ilo_init_transfer_functions(struct ilo_context *ilo)
1073 {
1074    ilo->base.transfer_map = ilo_transfer_map;
1075    ilo->base.transfer_flush_region = ilo_transfer_flush_region;
1076    ilo->base.transfer_unmap = ilo_transfer_unmap;
1077    ilo->base.transfer_inline_write = ilo_transfer_inline_write;
1078 }