src/gallium/drivers/ilo/ilo_transfer.c

   1 /*
   2  * Mesa 3-D graphics library
   3  *
   4  * Copyright (C) 2012-2013 LunarG, Inc.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included
  14  * in all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  22  * DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors:
  25  *    Chia-I Wu <olv@lunarg.com>
  26  */
  27
  28 #include "util/u_surface.h"
  29 #include "util/u_transfer.h"
  30 #include "util/u_format_etc.h"
  31
  32 #include "ilo_cp.h"
  33 #include "ilo_context.h"
  34 #include "ilo_resource.h"
  35 #include "ilo_transfer.h"
  36
  37 static bool
  38 is_bo_busy(struct ilo_context *ilo, struct intel_bo *bo, bool *need_flush)
  39 {
  40    const bool referenced = intel_bo_references(ilo->cp->bo, bo);
  41
  42    if (need_flush)
  43       *need_flush = referenced;
  44
  45    if (referenced)
  46       return true;
  47
  48    /*
  49     * XXX With hardware context support, the bo may be needed by GPU
  50     * without being referenced by ilo->cp->bo.  We have to flush
  51     * unconditionally, and that is bad.
  52     */
  53    if (ilo->cp->render_ctx)
  54       ilo_cp_flush(ilo->cp);
  55
  56    return intel_bo_is_busy(bo);
  57 }
  58
  59 static bool
  60 map_bo_for_transfer(struct ilo_context *ilo, struct intel_bo *bo,
  61                     const struct ilo_transfer *xfer)
  62 {
  63    int err;
  64
  65    switch (xfer->method) {
  66    case ILO_TRANSFER_MAP_CPU:
  67       err = intel_bo_map(bo, (xfer->base.usage & PIPE_TRANSFER_WRITE));
  68       break;
  69    case ILO_TRANSFER_MAP_GTT:
  70       err = intel_bo_map_gtt(bo);
  71       break;
  72    case ILO_TRANSFER_MAP_UNSYNC:
  73       err = intel_bo_map_unsynchronized(bo);
  74       break;
  75    default:
  76       assert(!"unknown mapping method");
  77       err = -1;
  78       break;
  79    }
  80
  81    return !err;
  82 }
  83
  84 /**
  85  * Choose the best mapping method, depending on the transfer usage and whether
  86  * the bo is busy.
  87  */
  88 static bool
  89 choose_transfer_method(struct ilo_context *ilo, struct ilo_transfer *xfer)
  90 {
  91    struct pipe_resource *res = xfer->base.resource;
  92    const unsigned usage = xfer->base.usage;
  93    /* prefer map() when there is the last-level cache */
  94    const bool prefer_cpu =
  95       (ilo->dev->has_llc || (usage & PIPE_TRANSFER_READ));
  96    struct ilo_texture *tex;
  97    struct ilo_buffer *buf;
  98    struct intel_bo *bo;
  99    bool tiled, need_flush;
 100
 101    if (res->target == PIPE_BUFFER) {
 102       tex = NULL;
 103
 104       buf = ilo_buffer(res);
 105       bo = buf->bo;
 106       tiled = false;
 107    }
 108    else {
 109       buf = NULL;
 110
 111       tex = ilo_texture(res);
 112       bo = tex->bo;
 113       tiled = (tex->tiling != INTEL_TILING_NONE);
 114    }
 115
 116    /* choose between mapping through CPU or GTT */
 117    if (usage & PIPE_TRANSFER_MAP_DIRECTLY) {
 118       /* we do not want fencing */
 119       if (tiled || prefer_cpu)
 120          xfer->method = ILO_TRANSFER_MAP_CPU;
 121       else
 122          xfer->method = ILO_TRANSFER_MAP_GTT;
 123    }
 124    else {
 125       if (!tiled && prefer_cpu)
 126          xfer->method = ILO_TRANSFER_MAP_CPU;
 127       else
 128          xfer->method = ILO_TRANSFER_MAP_GTT;
 129    }
 130
 131    /* see if we can avoid stalling */
 132    if (is_bo_busy(ilo, bo, &need_flush)) {
 133       bool will_stall = true;
 134
 135       if (usage & PIPE_TRANSFER_MAP_DIRECTLY) {
 136          /* nothing we can do */
 137       }
 138       else if (usage & PIPE_TRANSFER_UNSYNCHRONIZED) {
 139          /* unsynchronized gtt mapping does not stall */
 140          xfer->method = ILO_TRANSFER_MAP_UNSYNC;
 141          will_stall = false;
 142       }
 143       else if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE) {
 144          /* discard old bo and allocate a new one for mapping */
 145          if ((tex && ilo_texture_alloc_bo(tex)) ||
 146              (buf && ilo_buffer_alloc_bo(buf)))
 147             will_stall = false;
 148       }
 149       else if (usage & PIPE_TRANSFER_FLUSH_EXPLICIT) {
 150          /*
 151           * We could allocate and return a system buffer here.  When a region of
 152           * the buffer is explicitly flushed, we pwrite() the region to a
 153           * temporary bo and emit pipelined copy blit.
 154           *
 155           * For now, do nothing.
 156           */
 157       }
 158       else if (usage & PIPE_TRANSFER_DISCARD_RANGE) {
 159          /*
 160           * We could allocate a temporary bo for mapping, and emit pipelined copy
 161           * blit upon unmapping.
 162           *
 163           * For now, do nothing.
 164           */
 165       }
 166
 167       if (will_stall) {
 168          if (usage & PIPE_TRANSFER_DONTBLOCK)
 169             return false;
 170
 171          /* flush to make bo busy (so that map() stalls as it should be) */
 172          if (need_flush)
 173             ilo_cp_flush(ilo->cp);
 174       }
 175    }
 176
 177    if (tex && !(usage & PIPE_TRANSFER_MAP_DIRECTLY)) {
 178       if (tex->separate_s8 || tex->bo_format == PIPE_FORMAT_S8_UINT)
 179          xfer->method = ILO_TRANSFER_MAP_SW_ZS;
 180       /* need to convert on-the-fly */
 181       else if (tex->bo_format != tex->base.format)
 182          xfer->method = ILO_TRANSFER_MAP_SW_CONVERT;
 183    }
 184
 185    return true;
 186 }
 187
 188 static void
 189 tex_get_box_origin(const struct ilo_texture *tex,
 190                    unsigned level, unsigned slice,
 191                    const struct pipe_box *box,
 192                    unsigned *mem_x, unsigned *mem_y)
 193 {
 194    unsigned x, y;
 195
 196    x = tex->slice_offsets[level][slice + box->z].x + box->x;
 197    y = tex->slice_offsets[level][slice + box->z].y + box->y;
 198
 199    assert(x % tex->block_width == 0 && y % tex->block_height == 0);
 200
 201    *mem_x = x / tex->block_width * tex->bo_cpp;
 202    *mem_y = y / tex->block_height;
 203 }
 204
 205 static unsigned
 206 tex_get_box_offset(const struct ilo_texture *tex, unsigned level,
 207                    const struct pipe_box *box)
 208 {
 209    unsigned mem_x, mem_y;
 210
 211    tex_get_box_origin(tex, level, 0, box, &mem_x, &mem_y);
 212
 213    return mem_y * tex->bo_stride + mem_x;
 214 }
 215
 216 static unsigned
 217 tex_get_slice_stride(const struct ilo_texture *tex, unsigned level)
 218 {
 219    unsigned qpitch;
 220
 221    /* there is no 3D array texture */
 222    assert(tex->base.array_size == 1 || tex->base.depth0 == 1);
 223
 224    if (tex->base.array_size == 1) {
 225       /* non-array, non-3D */
 226       if (tex->base.depth0 == 1)
 227          return 0;
 228
 229       /* only the first level has a fixed slice stride */
 230       if (level > 0) {
 231          assert(!"no slice stride for 3D texture with level > 0");
 232          return 0;
 233       }
 234    }
 235
 236    qpitch = tex->slice_offsets[level][1].y - tex->slice_offsets[level][0].y;
 237    assert(qpitch % tex->block_height == 0);
 238
 239    return (qpitch / tex->block_height) * tex->bo_stride;
 240 }
 241
 242 static unsigned
 243 tex_tile_x_swizzle(unsigned addr)
 244 {
 245    /*
 246     * From the Ivy Bridge PRM, volume 1 part 2, page 24:
 247     *
 248     *     "As shown in the tiling algorithm, the new address bit[6] should be:
 249     *
 250     *        Address bit[6] <= TiledAddr bit[6] XOR
 251     *                          TiledAddr bit[9] XOR
 252     *                          TiledAddr bit[10]"
 253     */
 254    return addr ^ (((addr >> 3) ^ (addr >> 4)) & 0x40);
 255 }
 256
 257 static unsigned
 258 tex_tile_y_swizzle(unsigned addr)
 259 {
 260    /*
 261     * From the Ivy Bridge PRM, volume 1 part 2, page 24:
 262     *
 263     *     "As shown in the tiling algorithm, The new address bit[6] becomes:
 264     *
 265     *        Address bit[6] <= TiledAddr bit[6] XOR
 266     *                          TiledAddr bit[9]"
 267     */
 268    return addr ^ ((addr >> 3) & 0x40);
 269 }
 270
 271 static unsigned
 272 tex_tile_x_offset(unsigned mem_x, unsigned mem_y,
 273                   unsigned tiles_per_row, bool swizzle)
 274 {
 275    /*
 276     * From the Sandy Bridge PRM, volume 1 part 2, page 21, we know that a
 277     * X-major tile has 8 rows and 32 OWord columns (512 bytes).  Tiles in the
 278     * tiled region are numbered in row-major order, starting from zero.  The
 279     * tile number can thus be calculated as follows:
 280     *
 281     *    tile = (mem_y / 8) * tiles_per_row + (mem_x / 512)
 282     *
 283     * OWords in that tile are also numbered in row-major order, starting from
 284     * zero.  The OWord number can thus be calculated as follows:
 285     *
 286     *    oword = (mem_y % 8) * 32 + ((mem_x % 512) / 16)
 287     *
 288     * and the tiled offset is
 289     *
 290     *    offset = tile * 4096 + oword * 16 + (mem_x % 16)
 291     *           = tile * 4096 + (mem_y % 8) * 512 + (mem_x % 512)
 292     */
 293    unsigned tile, offset;
 294
 295    tile = (mem_y >> 3) * tiles_per_row + (mem_x >> 9);
 296    offset = tile << 12 | (mem_y & 0x7) << 9 | (mem_x & 0x1ff);
 297
 298    return (swizzle) ? tex_tile_x_swizzle(offset) : offset;
 299 }
 300
 301 static unsigned
 302 tex_tile_y_offset(unsigned mem_x, unsigned mem_y,
 303                   unsigned tiles_per_row, bool swizzle)
 304 {
 305    /*
 306     * From the Sandy Bridge PRM, volume 1 part 2, page 22, we know that a
 307     * Y-major tile has 32 rows and 8 OWord columns (128 bytes).  Tiles in the
 308     * tiled region are numbered in row-major order, starting from zero.  The
 309     * tile number can thus be calculated as follows:
 310     *
 311     *    tile = (mem_y / 32) * tiles_per_row + (mem_x / 128)
 312     *
 313     * OWords in that tile are numbered in column-major order, starting from
 314     * zero.  The OWord number can thus be calculated as follows:
 315     *
 316     *    oword = ((mem_x % 128) / 16) * 32 + (mem_y % 32)
 317     *
 318     * and the tiled offset is
 319     *
 320     *    offset = tile * 4096 + oword * 16 + (mem_x % 16)
 321     */
 322    unsigned tile, oword, offset;
 323
 324    tile = (mem_y >> 5) * tiles_per_row + (mem_x >> 7);
 325    oword = (mem_x & 0x70) << 1 | (mem_y & 0x1f);
 326    offset = tile << 12 | oword << 4 | (mem_x & 0xf);
 327
 328    return (swizzle) ? tex_tile_y_swizzle(offset) : offset;
 329 }
 330
 331 static unsigned
 332 tex_tile_w_offset(unsigned mem_x, unsigned mem_y,
 333                   unsigned tiles_per_row, bool swizzle)
 334 {
 335    /*
 336     * From the Sandy Bridge PRM, volume 1 part 2, page 23, we know that a
 337     * W-major tile has 8 8x8-block rows and 8 8x8-block columns.  Tiles in the
 338     * tiled region are numbered in row-major order, starting from zero.  The
 339     * tile number can thus be calculated as follows:
 340     *
 341     *    tile = (mem_y / 64) * tiles_per_row + (mem_x / 64)
 342     *
 343     * 8x8-blocks in that tile are numbered in column-major order, starting
 344     * from zero.  The 8x8-block number can thus be calculated as follows:
 345     *
 346     *    blk8 = ((mem_x % 64) / 8) * 8 + ((mem_y % 64) / 8)
 347     *
 348     * Each 8x8-block is divided into 4 4x4-blocks, in row-major order.  Each
 349     * 4x4-block is further divided into 4 2x2-blocks, also in row-major order.
 350     * We have
 351     *
 352     *    blk4 = (((mem_y % 64) / 4) & 1) * 2 + (((mem_x % 64) / 4) & 1)
 353     *    blk2 = (((mem_y % 64) / 2) & 1) * 2 + (((mem_x % 64) / 2) & 1)
 354     *    blk1 = (((mem_y % 64)    ) & 1) * 2 + (((mem_x % 64)    ) & 1)
 355     *
 356     * and the tiled offset is
 357     *
 358     *    offset = tile * 4096 + blk8 * 64 + blk4 * 16 + blk2 * 4 + blk1
 359     */
 360    unsigned tile, blk8, blk4, blk2, blk1, offset;
 361
 362    tile = (mem_y >> 6) * tiles_per_row + (mem_x >> 6);
 363    blk8 = ((mem_x >> 3) & 0x7) << 3 | ((mem_y >> 3) & 0x7);
 364    blk4 = ((mem_y >> 2) & 0x1) << 1 | ((mem_x >> 2) & 0x1);
 365    blk2 = ((mem_y >> 1) & 0x1) << 1 | ((mem_x >> 1) & 0x1);
 366    blk1 = ((mem_y     ) & 0x1) << 1 | ((mem_x     ) & 0x1);
 367    offset = tile << 12 | blk8 << 6 | blk4 << 4 | blk2 << 2 | blk1;
 368
 369    return (swizzle) ? tex_tile_y_swizzle(offset) : offset;
 370 }
 371
 372 static unsigned
 373 tex_tile_none_offset(unsigned mem_x, unsigned mem_y,
 374                      unsigned tiles_per_row, bool swizzle)
 375 {
 376    return mem_y * tiles_per_row + mem_x;
 377 }
 378
 379 typedef unsigned (*tex_tile_offset_func)(unsigned mem_x, unsigned mem_y,
 380                                          unsigned tiles_per_row,
 381                                          bool swizzle);
 382
 383 static tex_tile_offset_func
 384 tex_tile_choose_offset_func(const struct ilo_texture *tex,
 385                             unsigned *tiles_per_row)
 386 {
 387    switch (tex->tiling) {
 388    case INTEL_TILING_X:
 389       *tiles_per_row = tex->bo_stride / 512;
 390       return tex_tile_x_offset;
 391    case INTEL_TILING_Y:
 392       *tiles_per_row = tex->bo_stride / 128;
 393       return tex_tile_y_offset;
 394    case INTEL_TILING_NONE:
 395    default:
 396       /* W-tiling */
 397       if (tex->bo_format == PIPE_FORMAT_S8_UINT) {
 398          *tiles_per_row = tex->bo_stride / 64;
 399          return tex_tile_w_offset;
 400       }
 401       else {
 402          *tiles_per_row = tex->bo_stride;
 403          return tex_tile_none_offset;
 404       }
 405    }
 406 }
 407
 408 static void
 409 tex_staging_sys_zs_read(struct ilo_context *ilo,
 410                         struct ilo_texture *tex,
 411                         const struct ilo_transfer *xfer)
 412 {
 413    const bool swizzle = ilo->dev->has_address_swizzling;
 414    const struct pipe_box *box = &xfer->base.box;
 415    const uint8_t *src = intel_bo_get_virtual(tex->bo);
 416    tex_tile_offset_func tile_offset;
 417    unsigned tiles_per_row;
 418    int slice;
 419
 420    tile_offset = tex_tile_choose_offset_func(tex, &tiles_per_row);
 421
 422    assert(tex->block_width == 1 && tex->block_height == 1);
 423
 424    if (tex->separate_s8) {
 425       struct ilo_texture *s8_tex = tex->separate_s8;
 426       const uint8_t *s8_src = intel_bo_get_virtual(s8_tex->bo);
 427       tex_tile_offset_func s8_tile_offset;
 428       unsigned s8_tiles_per_row;
 429       int dst_cpp, dst_s8_pos, src_cpp_used;
 430
 431       s8_tile_offset = tex_tile_choose_offset_func(s8_tex, &s8_tiles_per_row);
 432
 433       if (tex->base.format == PIPE_FORMAT_Z24_UNORM_S8_UINT) {
 434          assert(tex->bo_format == PIPE_FORMAT_Z24X8_UNORM);
 435
 436          dst_cpp = 4;
 437          dst_s8_pos = 3;
 438          src_cpp_used = 3;
 439       }
 440       else {
 441          assert(tex->base.format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT);
 442          assert(tex->bo_format == PIPE_FORMAT_Z32_FLOAT);
 443
 444          dst_cpp = 8;
 445          dst_s8_pos = 4;
 446          src_cpp_used = 4;
 447       }
 448
 449       for (slice = 0; slice < box->depth; slice++) {
 450          unsigned mem_x, mem_y, s8_mem_x, s8_mem_y;
 451          uint8_t *dst;
 452          int i, j;
 453
 454          tex_get_box_origin(tex, xfer->base.level, slice,
 455                             box, &mem_x, &mem_y);
 456          tex_get_box_origin(s8_tex, xfer->base.level, slice,
 457                             box, &s8_mem_x, &s8_mem_y);
 458
 459          dst = xfer->staging_sys + xfer->base.layer_stride * slice;
 460
 461          for (i = 0; i < box->height; i++) {
 462             unsigned x = mem_x, s8_x = s8_mem_x;
 463             uint8_t *d = dst;
 464
 465             for (j = 0; j < box->width; j++) {
 466                const unsigned offset =
 467                   tile_offset(x, mem_y, tiles_per_row, swizzle);
 468                const unsigned s8_offset =
 469                   s8_tile_offset(s8_x, s8_mem_y, s8_tiles_per_row, swizzle);
 470
 471                memcpy(d, src + offset, src_cpp_used);
 472                d[dst_s8_pos] = s8_src[s8_offset];
 473
 474                d += dst_cpp;
 475                x += tex->bo_cpp;
 476                s8_x++;
 477             }
 478
 479             dst += xfer->base.stride;
 480             mem_y++;
 481             s8_mem_y++;
 482          }
 483       }
 484    }
 485    else {
 486       assert(tex->bo_format == PIPE_FORMAT_S8_UINT);
 487
 488       for (slice = 0; slice < box->depth; slice++) {
 489          unsigned mem_x, mem_y;
 490          uint8_t *dst;
 491          int i, j;
 492
 493          tex_get_box_origin(tex, xfer->base.level, slice,
 494                             box, &mem_x, &mem_y);
 495
 496          dst = xfer->staging_sys + xfer->base.layer_stride * slice;
 497
 498          for (i = 0; i < box->height; i++) {
 499             unsigned x = mem_x;
 500             uint8_t *d = dst;
 501
 502             for (j = 0; j < box->width; j++) {
 503                const unsigned offset =
 504                   tile_offset(x, mem_y, tiles_per_row, swizzle);
 505
 506                *d = src[offset];
 507
 508                d++;
 509                x++;
 510             }
 511
 512             dst += xfer->base.stride;
 513             mem_y++;
 514          }
 515       }
 516    }
 517 }
 518
 519 static void
 520 tex_staging_sys_zs_write(struct ilo_context *ilo,
 521                          struct ilo_texture *tex,
 522                          const struct ilo_transfer *xfer)
 523 {
 524    const bool swizzle = ilo->dev->has_address_swizzling;
 525    const struct pipe_box *box = &xfer->base.box;
 526    uint8_t *dst = intel_bo_get_virtual(tex->bo);
 527    tex_tile_offset_func tile_offset;
 528    unsigned tiles_per_row;
 529    int slice;
 530
 531    tile_offset = tex_tile_choose_offset_func(tex, &tiles_per_row);
 532
 533    assert(tex->block_width == 1 && tex->block_height == 1);
 534
 535    if (tex->separate_s8) {
 536       struct ilo_texture *s8_tex = tex->separate_s8;
 537       uint8_t *s8_dst = intel_bo_get_virtual(s8_tex->bo);
 538       tex_tile_offset_func s8_tile_offset;
 539       unsigned s8_tiles_per_row;
 540       int src_cpp, src_s8_pos, dst_cpp_used;
 541
 542       s8_tile_offset = tex_tile_choose_offset_func(s8_tex, &s8_tiles_per_row);
 543
 544       if (tex->base.format == PIPE_FORMAT_Z24_UNORM_S8_UINT) {
 545          assert(tex->bo_format == PIPE_FORMAT_Z24X8_UNORM);
 546
 547          src_cpp = 4;
 548          src_s8_pos = 3;
 549          dst_cpp_used = 3;
 550       }
 551       else {
 552          assert(tex->base.format == PIPE_FORMAT_Z32_FLOAT_S8X24_UINT);
 553          assert(tex->bo_format == PIPE_FORMAT_Z32_FLOAT);
 554
 555          src_cpp = 8;
 556          src_s8_pos = 4;
 557          dst_cpp_used = 4;
 558       }
 559
 560       for (slice = 0; slice < box->depth; slice++) {
 561          unsigned mem_x, mem_y, s8_mem_x, s8_mem_y;
 562          const uint8_t *src;
 563          int i, j;
 564
 565          tex_get_box_origin(tex, xfer->base.level, slice,
 566                             box, &mem_x, &mem_y);
 567          tex_get_box_origin(s8_tex, xfer->base.level, slice,
 568                             box, &s8_mem_x, &s8_mem_y);
 569
 570          src = xfer->staging_sys + xfer->base.layer_stride * slice;
 571
 572          for (i = 0; i < box->height; i++) {
 573             unsigned x = mem_x, s8_x = s8_mem_x;
 574             const uint8_t *s = src;
 575
 576             for (j = 0; j < box->width; j++) {
 577                const unsigned offset =
 578                   tile_offset(x, mem_y, tiles_per_row, swizzle);
 579                const unsigned s8_offset =
 580                   s8_tile_offset(s8_x, s8_mem_y, s8_tiles_per_row, swizzle);
 581
 582                memcpy(dst + offset, s, dst_cpp_used);
 583                s8_dst[s8_offset] = s[src_s8_pos];
 584
 585                s += src_cpp;
 586                x += tex->bo_cpp;
 587                s8_x++;
 588             }
 589
 590             src += xfer->base.stride;
 591             mem_y++;
 592             s8_mem_y++;
 593          }
 594       }
 595    }
 596    else {
 597       assert(tex->bo_format == PIPE_FORMAT_S8_UINT);
 598
 599       for (slice = 0; slice < box->depth; slice++) {
 600          unsigned mem_x, mem_y;
 601          const uint8_t *src;
 602          int i, j;
 603
 604          tex_get_box_origin(tex, xfer->base.level, slice,
 605                             box, &mem_x, &mem_y);
 606
 607          src = xfer->staging_sys + xfer->base.layer_stride * slice;
 608
 609          for (i = 0; i < box->height; i++) {
 610             unsigned x = mem_x;
 611             const uint8_t *s = src;
 612
 613             for (j = 0; j < box->width; j++) {
 614                const unsigned offset =
 615                   tile_offset(x, mem_y, tiles_per_row, swizzle);
 616
 617                dst[offset] = *s;
 618
 619                s++;
 620                x++;
 621             }
 622
 623             src += xfer->base.stride;
 624             mem_y++;
 625          }
 626       }
 627    }
 628 }
 629
 630 static void
 631 tex_staging_sys_convert_write(struct ilo_context *ilo,
 632                               struct ilo_texture *tex,
 633                               const struct ilo_transfer *xfer)
 634 {
 635    const struct pipe_box *box = &xfer->base.box;
 636    unsigned dst_slice_stride;
 637    void *dst;
 638    int slice;
 639
 640    dst = intel_bo_get_virtual(tex->bo);
 641    dst += tex_get_box_offset(tex, xfer->base.level, box);
 642
 643    /* slice stride is not always available */
 644    if (box->depth > 1)
 645       dst_slice_stride = tex_get_slice_stride(tex, xfer->base.level);
 646    else
 647       dst_slice_stride = 0;
 648
 649    if (unlikely(tex->bo_format == tex->base.format)) {
 650       util_copy_box(dst, tex->bo_format, tex->bo_stride, dst_slice_stride,
 651             0, 0, 0, box->width, box->height, box->depth,
 652             xfer->staging_sys, xfer->base.stride, xfer->base.layer_stride,
 653             0, 0, 0);
 654       return;
 655    }
 656
 657    switch (tex->base.format) {
 658    case PIPE_FORMAT_ETC1_RGB8:
 659       assert(tex->bo_format == PIPE_FORMAT_R8G8B8X8_UNORM);
 660
 661       for (slice = 0; slice < box->depth; slice++) {
 662          const void *src =
 663             xfer->staging_sys + xfer->base.layer_stride * slice;
 664
 665          util_format_etc1_rgb8_unpack_rgba_8unorm(dst,
 666                tex->bo_stride, src, xfer->base.stride,
 667                box->width, box->height);
 668
 669          dst += dst_slice_stride;
 670       }
 671       break;
 672    default:
 673       assert(!"unable to convert the staging data");
 674       break;
 675    }
 676 }
 677
 678 static bool
 679 tex_staging_sys_map_bo(const struct ilo_context *ilo,
 680                        const struct ilo_texture *tex,
 681                        bool for_read_back, bool linear_view)
 682 {
 683    const bool prefer_cpu = (ilo->dev->has_llc || for_read_back);
 684    int err;
 685
 686    if (prefer_cpu && (tex->tiling == INTEL_TILING_NONE || !linear_view))
 687       err = intel_bo_map(tex->bo, !for_read_back);
 688    else
 689       err = intel_bo_map_gtt(tex->bo);
 690
 691    if (!tex->separate_s8)
 692       return !err;
 693
 694    err = intel_bo_map(tex->separate_s8->bo, !for_read_back);
 695    if (err)
 696       intel_bo_unmap(tex->bo);
 697
 698    return !err;
 699 }
 700
 701 static void
 702 tex_staging_sys_unmap_bo(const struct ilo_context *ilo,
 703                          const struct ilo_texture *tex)
 704 {
 705    if (tex->separate_s8)
 706       intel_bo_unmap(tex->separate_s8->bo);
 707
 708    intel_bo_unmap(tex->bo);
 709 }
 710
 711 static void
 712 tex_staging_sys_unmap(struct ilo_context *ilo,
 713                       struct ilo_texture *tex,
 714                       struct ilo_transfer *xfer)
 715 {
 716    bool success;
 717
 718    if (!(xfer->base.usage & PIPE_TRANSFER_WRITE)) {
 719       FREE(xfer->staging_sys);
 720       return;
 721    }
 722
 723    switch (xfer->method) {
 724    case ILO_TRANSFER_MAP_SW_CONVERT:
 725       success = tex_staging_sys_map_bo(ilo, tex, false, true);
 726       if (success) {
 727          tex_staging_sys_convert_write(ilo, tex, xfer);
 728          tex_staging_sys_unmap_bo(ilo, tex);
 729       }
 730       break;
 731    case ILO_TRANSFER_MAP_SW_ZS:
 732       success = tex_staging_sys_map_bo(ilo, tex, false, false);
 733       if (success) {
 734          tex_staging_sys_zs_write(ilo, tex, xfer);
 735          tex_staging_sys_unmap_bo(ilo, tex);
 736       }
 737       break;
 738    default:
 739       assert(!"unknown mapping method");
 740       success = false;
 741       break;
 742    }
 743
 744    if (!success)
 745       ilo_err("failed to map resource for moving staging data\n");
 746
 747    FREE(xfer->staging_sys);
 748 }
 749
 750 static bool
 751 tex_staging_sys_map(struct ilo_context *ilo,
 752                     struct ilo_texture *tex,
 753                     struct ilo_transfer *xfer)
 754 {
 755    const struct pipe_box *box = &xfer->base.box;
 756    const size_t stride = util_format_get_stride(tex->base.format, box->width);
 757    const size_t size =
 758       util_format_get_2d_size(tex->base.format, stride, box->height);
 759    bool read_back = false, success;
 760
 761    xfer->staging_sys = MALLOC(size * box->depth);
 762    if (!xfer->staging_sys)
 763       return false;
 764
 765    xfer->base.stride = stride;
 766    xfer->base.layer_stride = size;
 767    xfer->ptr = xfer->staging_sys;
 768
 769    /* see if we need to read the resource back */
 770    if (xfer->base.usage & PIPE_TRANSFER_READ) {
 771       read_back = true;
 772    }
 773    else if (xfer->base.usage & PIPE_TRANSFER_WRITE) {
 774       const unsigned discard_flags =
 775          (PIPE_TRANSFER_DISCARD_RANGE | PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE);
 776
 777       if (!(xfer->base.usage & discard_flags))
 778          read_back = true;
 779    }
 780
 781    if (!read_back)
 782       return true;
 783
 784    switch (xfer->method) {
 785    case ILO_TRANSFER_MAP_SW_CONVERT:
 786       assert(!"no on-the-fly format conversion for mapping");
 787       success = false;
 788       break;
 789    case ILO_TRANSFER_MAP_SW_ZS:
 790       success = tex_staging_sys_map_bo(ilo, tex, true, false);
 791       if (success) {
 792          tex_staging_sys_zs_read(ilo, tex, xfer);
 793          tex_staging_sys_unmap_bo(ilo, tex);
 794       }
 795       break;
 796    default:
 797       assert(!"unknown mapping method");
 798       success = false;
 799       break;
 800    }
 801
 802    return success;
 803 }
 804
 805 static void
 806 tex_direct_unmap(struct ilo_context *ilo,
 807                  struct ilo_texture *tex,
 808                  struct ilo_transfer *xfer)
 809 {
 810    intel_bo_unmap(tex->bo);
 811 }
 812
 813 static bool
 814 tex_direct_map(struct ilo_context *ilo,
 815                struct ilo_texture *tex,
 816                struct ilo_transfer *xfer)
 817 {
 818    if (!map_bo_for_transfer(ilo, tex->bo, xfer))
 819       return false;
 820
 821    /* note that stride is for a block row, not a texel row */
 822    xfer->base.stride = tex->bo_stride;
 823
 824    /* slice stride is not always available */
 825    if (xfer->base.box.depth > 1)
 826       xfer->base.layer_stride = tex_get_slice_stride(tex, xfer->base.level);
 827    else
 828       xfer->base.layer_stride = 0;
 829
 830    xfer->ptr = intel_bo_get_virtual(tex->bo);
 831    xfer->ptr += tex_get_box_offset(tex, xfer->base.level, &xfer->base.box);
 832
 833    return true;
 834 }
 835
 836 static bool
 837 tex_map(struct ilo_context *ilo, struct ilo_transfer *xfer)
 838 {
 839    struct ilo_texture *tex = ilo_texture(xfer->base.resource);
 840    bool success;
 841
 842    if (!choose_transfer_method(ilo, xfer))
 843       return false;
 844
 845    switch (xfer->method) {
 846    case ILO_TRANSFER_MAP_CPU:
 847    case ILO_TRANSFER_MAP_GTT:
 848    case ILO_TRANSFER_MAP_UNSYNC:
 849       success = tex_direct_map(ilo, tex, xfer);
 850       break;
 851    case ILO_TRANSFER_MAP_SW_CONVERT:
 852    case ILO_TRANSFER_MAP_SW_ZS:
 853       success = tex_staging_sys_map(ilo, tex, xfer);
 854       break;
 855    default:
 856       assert(!"unknown mapping method");
 857       success = false;
 858       break;
 859    }
 860
 861    return success;
 862 }
 863
 864 static void
 865 tex_unmap(struct ilo_context *ilo, struct ilo_transfer *xfer)
 866 {
 867    struct ilo_texture *tex = ilo_texture(xfer->base.resource);
 868
 869    switch (xfer->method) {
 870    case ILO_TRANSFER_MAP_CPU:
 871    case ILO_TRANSFER_MAP_GTT:
 872    case ILO_TRANSFER_MAP_UNSYNC:
 873       tex_direct_unmap(ilo, tex, xfer);
 874       break;
 875    case ILO_TRANSFER_MAP_SW_CONVERT:
 876    case ILO_TRANSFER_MAP_SW_ZS:
 877       tex_staging_sys_unmap(ilo, tex, xfer);
 878       break;
 879    default:
 880       assert(!"unknown mapping method");
 881       break;
 882    }
 883 }
 884
 885 static bool
 886 buf_map(struct ilo_context *ilo, struct ilo_transfer *xfer)
 887 {
 888    struct ilo_buffer *buf = ilo_buffer(xfer->base.resource);
 889
 890    if (!choose_transfer_method(ilo, xfer))
 891       return false;
 892
 893    if (!map_bo_for_transfer(ilo, buf->bo, xfer))
 894       return false;
 895
 896    assert(xfer->base.level == 0);
 897    assert(xfer->base.box.y == 0);
 898    assert(xfer->base.box.z == 0);
 899    assert(xfer->base.box.height == 1);
 900    assert(xfer->base.box.depth == 1);
 901
 902    xfer->base.stride = 0;
 903    xfer->base.layer_stride = 0;
 904
 905    xfer->ptr = intel_bo_get_virtual(buf->bo);
 906    xfer->ptr += xfer->base.box.x;
 907
 908    return true;
 909 }
 910
 911 static void
 912 buf_unmap(struct ilo_context *ilo, struct ilo_transfer *xfer)
 913 {
 914    struct ilo_buffer *buf = ilo_buffer(xfer->base.resource);
 915
 916    intel_bo_unmap(buf->bo);
 917 }
 918
 919 static void
 920 buf_pwrite(struct ilo_context *ilo, struct ilo_buffer *buf,
 921            unsigned usage, int offset, int size, const void *data)
 922 {
 923    bool need_flush;
 924
 925    /* see if we can avoid stalling */
 926    if (is_bo_busy(ilo, buf->bo, &need_flush)) {
 927       bool will_stall = true;
 928
 929       if (usage & PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE) {
 930          /* old data not needed so discard the old bo to avoid stalling */
 931          if (ilo_buffer_alloc_bo(buf))
 932             will_stall = false;
 933       }
 934       else {
 935          /*
 936           * We could allocate a temporary bo to hold the data and emit
 937           * pipelined copy blit to move them to buf->bo.  But for now, do
 938           * nothing.
 939           */
 940       }
 941
 942       /* flush to make bo busy (so that pwrite() stalls as it should be) */
 943       if (will_stall && need_flush)
 944          ilo_cp_flush(ilo->cp);
 945    }
 946
 947    intel_bo_pwrite(buf->bo, offset, size, data);
 948 }
 949
 950 static void
 951 ilo_transfer_flush_region(struct pipe_context *pipe,
 952                           struct pipe_transfer *transfer,
 953                           const struct pipe_box *box)
 954 {
 955 }
 956
 957 static void
 958 ilo_transfer_unmap(struct pipe_context *pipe,
 959                    struct pipe_transfer *transfer)
 960 {
 961    struct ilo_context *ilo = ilo_context(pipe);
 962    struct ilo_transfer *xfer = ilo_transfer(transfer);
 963
 964    if (xfer->base.resource->target == PIPE_BUFFER)
 965       buf_unmap(ilo, xfer);
 966    else
 967       tex_unmap(ilo, xfer);
 968
 969    pipe_resource_reference(&xfer->base.resource, NULL);
 970
 971    util_slab_free(&ilo->transfer_mempool, xfer);
 972 }
 973
 974 static void *
 975 ilo_transfer_map(struct pipe_context *pipe,
 976                  struct pipe_resource *res,
 977                  unsigned level,
 978                  unsigned usage,
 979                  const struct pipe_box *box,
 980                  struct pipe_transfer **transfer)
 981 {
 982    struct ilo_context *ilo = ilo_context(pipe);
 983    struct ilo_transfer *xfer;
 984    bool success;
 985
 986    xfer = util_slab_alloc(&ilo->transfer_mempool);
 987    if (!xfer) {
 988       *transfer = NULL;
 989       return NULL;
 990    }
 991
 992    xfer->base.resource = NULL;
 993    pipe_resource_reference(&xfer->base.resource, res);
 994    xfer->base.level = level;
 995    xfer->base.usage = usage;
 996    xfer->base.box = *box;
 997
 998    if (res->target == PIPE_BUFFER)
 999       success = buf_map(ilo, xfer);
1000    else
1001       success = tex_map(ilo, xfer);
1002
1003    if (!success) {
1004       pipe_resource_reference(&xfer->base.resource, NULL);
1005       FREE(xfer);
1006       *transfer = NULL;
1007       return NULL;
1008    }
1009
1010    *transfer = &xfer->base;
1011
1012    return xfer->ptr;
1013 }
1014
1015 static void
1016 ilo_transfer_inline_write(struct pipe_context *pipe,
1017                           struct pipe_resource *res,
1018                           unsigned level,
1019                           unsigned usage,
1020                           const struct pipe_box *box,
1021                           const void *data,
1022                           unsigned stride,
1023                           unsigned layer_stride)
1024 {
1025    if (likely(res->target == PIPE_BUFFER) &&
1026        !(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
1027       /* they should specify just an offset and a size */
1028       assert(level == 0);
1029       assert(box->y == 0);
1030       assert(box->z == 0);
1031       assert(box->height == 1);
1032       assert(box->depth == 1);
1033
1034       buf_pwrite(ilo_context(pipe), ilo_buffer(res),
1035             usage, box->x, box->width, data);
1036    }
1037    else {
1038       u_default_transfer_inline_write(pipe, res,
1039             level, usage, box, data, stride, layer_stride);
1040    }
1041 }
1042
1043 /**
1044  * Initialize transfer-related functions.
1045  */
1046 void
1047 ilo_init_transfer_functions(struct ilo_context *ilo)
1048 {
1049    ilo->base.transfer_map = ilo_transfer_map;
1050    ilo->base.transfer_flush_region = ilo_transfer_flush_region;
1051    ilo->base.transfer_unmap = ilo_transfer_unmap;
1052    ilo->base.transfer_inline_write = ilo_transfer_inline_write;
1053 }