src/gallium/drivers/iris/iris_resource.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included
  12  * in all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  15  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  20  * DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 /**
  24  * @file iris_resource.c
  25  *
  26  * Resources are images, buffers, and other objects used by the GPU.
  27  *
  28  * XXX: explain resources
  29  */
  30
  31 #include <stdio.h>
  32 #include <errno.h>
  33 #include "pipe/p_defines.h"
  34 #include "pipe/p_state.h"
  35 #include "pipe/p_context.h"
  36 #include "pipe/p_screen.h"
  37 #include "util/os_memory.h"
  38 #include "util/u_cpu_detect.h"
  39 #include "util/u_inlines.h"
  40 #include "util/u_format.h"
  41 #include "util/u_transfer.h"
  42 #include "util/u_transfer_helper.h"
  43 #include "util/u_upload_mgr.h"
  44 #include "util/ralloc.h"
  45 #include "iris_batch.h"
  46 #include "iris_context.h"
  47 #include "iris_resource.h"
  48 #include "iris_screen.h"
  49 #include "intel/common/gen_debug.h"
  50 #include "isl/isl.h"
  51 #include "drm-uapi/drm_fourcc.h"
  52 #include "drm-uapi/i915_drm.h"
  53
  54 // XXX: u_transfer_helper...for separate stencil...
  55
  56 enum modifier_priority {
  57    MODIFIER_PRIORITY_INVALID = 0,
  58    MODIFIER_PRIORITY_LINEAR,
  59    MODIFIER_PRIORITY_X,
  60    MODIFIER_PRIORITY_Y,
  61    MODIFIER_PRIORITY_Y_CCS,
  62 };
  63
  64 static const uint64_t priority_to_modifier[] = {
  65    [MODIFIER_PRIORITY_INVALID] = DRM_FORMAT_MOD_INVALID,
  66    [MODIFIER_PRIORITY_LINEAR] = DRM_FORMAT_MOD_LINEAR,
  67    [MODIFIER_PRIORITY_X] = I915_FORMAT_MOD_X_TILED,
  68    [MODIFIER_PRIORITY_Y] = I915_FORMAT_MOD_Y_TILED,
  69    [MODIFIER_PRIORITY_Y_CCS] = I915_FORMAT_MOD_Y_TILED_CCS,
  70 };
  71
  72 static bool
  73 modifier_is_supported(const struct gen_device_info *devinfo,
  74                       uint64_t modifier)
  75 {
  76    /* XXX: do something real */
  77    switch (modifier) {
  78    case I915_FORMAT_MOD_Y_TILED:
  79    case I915_FORMAT_MOD_X_TILED:
  80    case DRM_FORMAT_MOD_LINEAR:
  81       return true;
  82    case I915_FORMAT_MOD_Y_TILED_CCS:
  83    case DRM_FORMAT_MOD_INVALID:
  84    default:
  85       return false;
  86    }
  87 }
  88
  89 static uint64_t
  90 select_best_modifier(struct gen_device_info *devinfo,
  91                      const uint64_t *modifiers,
  92                      int count)
  93 {
  94    enum modifier_priority prio = MODIFIER_PRIORITY_INVALID;
  95
  96    for (int i = 0; i < count; i++) {
  97       if (!modifier_is_supported(devinfo, modifiers[i]))
  98          continue;
  99
 100       switch (modifiers[i]) {
 101       case I915_FORMAT_MOD_Y_TILED_CCS:
 102          prio = MAX2(prio, MODIFIER_PRIORITY_Y_CCS);
 103          break;
 104       case I915_FORMAT_MOD_Y_TILED:
 105          prio = MAX2(prio, MODIFIER_PRIORITY_Y);
 106          break;
 107       case I915_FORMAT_MOD_X_TILED:
 108          prio = MAX2(prio, MODIFIER_PRIORITY_X);
 109          break;
 110       case DRM_FORMAT_MOD_LINEAR:
 111          prio = MAX2(prio, MODIFIER_PRIORITY_LINEAR);
 112          break;
 113       case DRM_FORMAT_MOD_INVALID:
 114       default:
 115          break;
 116       }
 117    }
 118
 119    return priority_to_modifier[prio];
 120 }
 121
 122 static enum isl_surf_dim
 123 target_to_isl_surf_dim(enum pipe_texture_target target)
 124 {
 125    switch (target) {
 126    case PIPE_BUFFER:
 127    case PIPE_TEXTURE_1D:
 128    case PIPE_TEXTURE_1D_ARRAY:
 129       return ISL_SURF_DIM_1D;
 130    case PIPE_TEXTURE_2D:
 131    case PIPE_TEXTURE_CUBE:
 132    case PIPE_TEXTURE_RECT:
 133    case PIPE_TEXTURE_2D_ARRAY:
 134    case PIPE_TEXTURE_CUBE_ARRAY:
 135       return ISL_SURF_DIM_2D;
 136    case PIPE_TEXTURE_3D:
 137       return ISL_SURF_DIM_3D;
 138    case PIPE_MAX_TEXTURE_TYPES:
 139       break;
 140    }
 141    unreachable("invalid texture type");
 142 }
 143
 144 static isl_surf_usage_flags_t
 145 pipe_bind_to_isl_usage(unsigned bindings)
 146 {
 147    isl_surf_usage_flags_t usage = 0;
 148
 149    if (bindings & PIPE_BIND_RENDER_TARGET)
 150       usage |= ISL_SURF_USAGE_RENDER_TARGET_BIT;
 151
 152    if (bindings & PIPE_BIND_SAMPLER_VIEW)
 153       usage |= ISL_SURF_USAGE_TEXTURE_BIT;
 154
 155    if (bindings & (PIPE_BIND_SHADER_IMAGE | PIPE_BIND_SHADER_BUFFER))
 156       usage |= ISL_SURF_USAGE_STORAGE_BIT;
 157
 158    if (bindings & PIPE_BIND_DISPLAY_TARGET)
 159       usage |= ISL_SURF_USAGE_DISPLAY_BIT;
 160
 161    return usage;
 162 }
 163
 164 struct pipe_resource *
 165 iris_resource_get_separate_stencil(struct pipe_resource *p_res)
 166 {
 167    /* For packed depth-stencil, we treat depth as the primary resource
 168     * and store S8 as the "second plane" resource.
 169     */
 170    return p_res->next;
 171 }
 172
 173 static void
 174 iris_resource_set_separate_stencil(struct pipe_resource *p_res,
 175                                    struct pipe_resource *stencil)
 176 {
 177    assert(util_format_has_depth(util_format_description(p_res->format)));
 178    pipe_resource_reference(&p_res->next, stencil);
 179 }
 180
 181 void
 182 iris_get_depth_stencil_resources(struct pipe_resource *res,
 183                                  struct iris_resource **out_z,
 184                                  struct iris_resource **out_s)
 185 {
 186    if (!res) {
 187       *out_z = NULL;
 188       *out_s = NULL;
 189       return;
 190    }
 191
 192    const struct util_format_description *desc =
 193       util_format_description(res->format);
 194
 195    if (util_format_has_depth(desc)) {
 196       *out_z = (void *) res;
 197       *out_s = (void *) iris_resource_get_separate_stencil(res);
 198    } else {
 199       assert(util_format_has_stencil(desc));
 200       *out_z = NULL;
 201       *out_s = (void *) res;
 202    }
 203 }
 204
 205 static void
 206 iris_resource_destroy(struct pipe_screen *screen,
 207                       struct pipe_resource *resource)
 208 {
 209    struct iris_resource *res = (struct iris_resource *)resource;
 210
 211    iris_bo_unreference(res->bo);
 212    free(res);
 213 }
 214
 215 static struct iris_resource *
 216 iris_alloc_resource(struct pipe_screen *pscreen,
 217                     const struct pipe_resource *templ)
 218 {
 219    struct iris_resource *res = calloc(1, sizeof(struct iris_resource));
 220    if (!res)
 221       return NULL;
 222
 223    res->base = *templ;
 224    res->base.screen = pscreen;
 225    pipe_reference_init(&res->base.reference, 1);
 226
 227    return res;
 228 }
 229
 230 static struct pipe_resource *
 231 iris_resource_create_with_modifiers(struct pipe_screen *pscreen,
 232                                     const struct pipe_resource *templ,
 233                                     const uint64_t *modifiers,
 234                                     int modifiers_count)
 235 {
 236    struct iris_screen *screen = (struct iris_screen *)pscreen;
 237    struct gen_device_info *devinfo = &screen->devinfo;
 238    struct iris_resource *res = iris_alloc_resource(pscreen, templ);
 239    const struct util_format_description *format_desc =
 240       util_format_description(templ->format);
 241
 242    if (!res)
 243       return NULL;
 244
 245    const bool has_depth = util_format_has_depth(format_desc);
 246    uint64_t modifier = DRM_FORMAT_MOD_INVALID;
 247
 248    if (modifiers_count == 0 || !modifiers) {
 249       if (has_depth) {
 250          modifier = I915_FORMAT_MOD_Y_TILED;
 251       } else if (templ->target == PIPE_TEXTURE_1D ||
 252                  templ->target == PIPE_TEXTURE_1D_ARRAY) {
 253          modifier = DRM_FORMAT_MOD_LINEAR;
 254       } else if (templ->bind & PIPE_BIND_DISPLAY_TARGET) {
 255          /* Display is X-tiled for historical reasons. */
 256          modifier = I915_FORMAT_MOD_X_TILED;
 257       } else {
 258          modifier = I915_FORMAT_MOD_Y_TILED;
 259       }
 260       /* XXX: make sure this doesn't do stupid things for internal textures */
 261    }
 262
 263    if (templ->target == PIPE_BUFFER || templ->usage == PIPE_USAGE_STAGING)
 264       modifier = DRM_FORMAT_MOD_LINEAR;
 265
 266    if (templ->bind & (PIPE_BIND_LINEAR | PIPE_BIND_CURSOR))
 267       modifier = DRM_FORMAT_MOD_LINEAR;
 268
 269    if (modifier == DRM_FORMAT_MOD_INVALID) {
 270       /* User requested specific modifiers */
 271       modifier = select_best_modifier(devinfo, modifiers, modifiers_count);
 272       if (modifier == DRM_FORMAT_MOD_INVALID)
 273          return NULL;
 274    }
 275
 276    const struct isl_drm_modifier_info *mod_info =
 277       isl_drm_modifier_get_info(modifier);
 278
 279    enum isl_tiling tiling = templ->format == PIPE_FORMAT_S8_UINT ?
 280       ISL_TILING_W : mod_info->tiling;
 281
 282    isl_surf_usage_flags_t usage = pipe_bind_to_isl_usage(templ->bind);
 283
 284    if (templ->target == PIPE_TEXTURE_CUBE ||
 285        templ->target == PIPE_TEXTURE_CUBE_ARRAY)
 286       usage |= ISL_SURF_USAGE_CUBE_BIT;
 287
 288    if (templ->usage != PIPE_USAGE_STAGING) {
 289       if (templ->format == PIPE_FORMAT_S8_UINT)
 290          usage |= ISL_SURF_USAGE_STENCIL_BIT;
 291       else if (has_depth)
 292          usage |= ISL_SURF_USAGE_DEPTH_BIT;
 293    }
 294
 295    enum pipe_format pfmt = templ->format;
 296    res->internal_format = pfmt;
 297
 298    /* Should be handled by u_transfer_helper */
 299    assert(!util_format_is_depth_and_stencil(pfmt));
 300
 301    struct iris_format_info fmt = iris_format_for_usage(devinfo, pfmt, usage);
 302    assert(fmt.fmt != ISL_FORMAT_UNSUPPORTED);
 303
 304    UNUSED const bool isl_surf_created_successfully =
 305       isl_surf_init(&screen->isl_dev, &res->surf,
 306                     .dim = target_to_isl_surf_dim(templ->target),
 307                     .format = fmt.fmt,
 308                     .width = templ->width0,
 309                     .height = templ->height0,
 310                     .depth = templ->depth0,
 311                     .levels = templ->last_level + 1,
 312                     .array_len = templ->array_size,
 313                     .samples = MAX2(templ->nr_samples, 1),
 314                     .min_alignment_B = 0,
 315                     .row_pitch_B = 0,
 316                     .usage = usage,
 317                     .tiling_flags = 1 << tiling);
 318    assert(isl_surf_created_successfully);
 319
 320    enum iris_memory_zone memzone = IRIS_MEMZONE_OTHER;
 321    const char *name = templ->target == PIPE_BUFFER ? "buffer" : "miptree";
 322    if (templ->flags & IRIS_RESOURCE_FLAG_SHADER_MEMZONE) {
 323       memzone = IRIS_MEMZONE_SHADER;
 324       name = "shader kernels";
 325    } else if (templ->flags & IRIS_RESOURCE_FLAG_SURFACE_MEMZONE) {
 326       memzone = IRIS_MEMZONE_SURFACE;
 327       name = "surface state";
 328    } else if (templ->flags & IRIS_RESOURCE_FLAG_DYNAMIC_MEMZONE) {
 329       memzone = IRIS_MEMZONE_DYNAMIC;
 330       name = "dynamic state";
 331    }
 332
 333    res->bo = iris_bo_alloc_tiled(screen->bufmgr, name, res->surf.size_B,
 334                                  memzone,
 335                                  isl_tiling_to_i915_tiling(res->surf.tiling),
 336                                  res->surf.row_pitch_B, 0);
 337    if (!res->bo)
 338       goto fail;
 339
 340    return &res->base;
 341
 342 fail:
 343    iris_resource_destroy(pscreen, &res->base);
 344    return NULL;
 345 }
 346
 347 static struct pipe_resource *
 348 iris_resource_create(struct pipe_screen *pscreen,
 349                      const struct pipe_resource *templ)
 350 {
 351    return iris_resource_create_with_modifiers(pscreen, templ, NULL, 0);
 352 }
 353
 354 static uint64_t
 355 tiling_to_modifier(uint32_t tiling)
 356 {
 357    static const uint64_t map[] = {
 358       [I915_TILING_NONE]   = DRM_FORMAT_MOD_LINEAR,
 359       [I915_TILING_X]      = I915_FORMAT_MOD_X_TILED,
 360       [I915_TILING_Y]      = I915_FORMAT_MOD_Y_TILED,
 361    };
 362
 363    assert(tiling < ARRAY_SIZE(map));
 364
 365    return map[tiling];
 366 }
 367
 368 static struct pipe_resource *
 369 iris_resource_from_user_memory(struct pipe_screen *pscreen,
 370                                const struct pipe_resource *templ,
 371                                void *user_memory)
 372 {
 373    struct iris_screen *screen = (struct iris_screen *)pscreen;
 374    struct gen_device_info *devinfo = &screen->devinfo;
 375    struct iris_bufmgr *bufmgr = screen->bufmgr;
 376    struct iris_resource *res = iris_alloc_resource(pscreen, templ);
 377    if (!res)
 378       return NULL;
 379
 380    res->bo = iris_bo_create_userptr(bufmgr, "user",
 381                                     user_memory, templ->width0,
 382                                     IRIS_MEMZONE_OTHER);
 383    if (!res->bo) {
 384       free(res);
 385       return NULL;
 386    }
 387
 388    res->internal_format = templ->format;
 389
 390    // XXX: usage...
 391    isl_surf_usage_flags_t isl_usage = 0;
 392
 393    const struct iris_format_info fmt =
 394       iris_format_for_usage(devinfo, templ->format, isl_usage);
 395
 396    isl_surf_init(&screen->isl_dev, &res->surf,
 397                  .dim = target_to_isl_surf_dim(templ->target),
 398                  .format = fmt.fmt,
 399                  .width = templ->width0,
 400                  .height = templ->height0,
 401                  .depth = templ->depth0,
 402                  .levels = templ->last_level + 1,
 403                  .array_len = templ->array_size,
 404                  .samples = MAX2(templ->nr_samples, 1),
 405                  .min_alignment_B = 0,
 406                  .row_pitch_B = 0,
 407                  .usage = isl_usage,
 408                  .tiling_flags = 1 << ISL_TILING_LINEAR);
 409
 410    assert(res->bo->tiling_mode == isl_tiling_to_i915_tiling(res->surf.tiling));
 411
 412    return &res->base;
 413 }
 414
 415 static struct pipe_resource *
 416 iris_resource_from_handle(struct pipe_screen *pscreen,
 417                           const struct pipe_resource *templ,
 418                           struct winsys_handle *whandle,
 419                           unsigned usage)
 420 {
 421    struct iris_screen *screen = (struct iris_screen *)pscreen;
 422    struct gen_device_info *devinfo = &screen->devinfo;
 423    struct iris_bufmgr *bufmgr = screen->bufmgr;
 424    struct iris_resource *res = iris_alloc_resource(pscreen, templ);
 425    if (!res)
 426       return NULL;
 427
 428    if (whandle->offset != 0) {
 429       dbg_printf("Attempt to import unsupported winsys offset %u\n",
 430                  whandle->offset);
 431       goto fail;
 432    }
 433
 434    switch (whandle->type) {
 435    case WINSYS_HANDLE_TYPE_FD:
 436       res->bo = iris_bo_import_dmabuf(bufmgr, whandle->handle);
 437       break;
 438    case WINSYS_HANDLE_TYPE_SHARED:
 439       res->bo = iris_bo_gem_create_from_name(bufmgr, "winsys image",
 440                                              whandle->handle);
 441       break;
 442    default:
 443       unreachable("invalid winsys handle type");
 444    }
 445    if (!res->bo)
 446       return NULL;
 447
 448    uint64_t modifier = whandle->modifier;
 449    if (modifier == DRM_FORMAT_MOD_INVALID) {
 450           modifier = tiling_to_modifier(res->bo->tiling_mode);
 451    }
 452    const struct isl_drm_modifier_info *mod_info =
 453       isl_drm_modifier_get_info(modifier);
 454    assert(mod_info);
 455
 456    // XXX: usage...
 457    isl_surf_usage_flags_t isl_usage = ISL_SURF_USAGE_DISPLAY_BIT;
 458
 459    const struct iris_format_info fmt =
 460       iris_format_for_usage(devinfo, templ->format, isl_usage);
 461
 462    isl_surf_init(&screen->isl_dev, &res->surf,
 463                  .dim = target_to_isl_surf_dim(templ->target),
 464                  .format = fmt.fmt,
 465                  .width = templ->width0,
 466                  .height = templ->height0,
 467                  .depth = templ->depth0,
 468                  .levels = templ->last_level + 1,
 469                  .array_len = templ->array_size,
 470                  .samples = MAX2(templ->nr_samples, 1),
 471                  .min_alignment_B = 0,
 472                  .row_pitch_B = 0,
 473                  .usage = isl_usage,
 474                  .tiling_flags = 1 << mod_info->tiling);
 475
 476    assert(res->bo->tiling_mode == isl_tiling_to_i915_tiling(res->surf.tiling));
 477
 478    return &res->base;
 479
 480 fail:
 481    iris_resource_destroy(pscreen, &res->base);
 482    return NULL;
 483 }
 484
 485 static boolean
 486 iris_resource_get_handle(struct pipe_screen *pscreen,
 487                          struct pipe_context *ctx,
 488                          struct pipe_resource *resource,
 489                          struct winsys_handle *whandle,
 490                          unsigned usage)
 491 {
 492    struct iris_resource *res = (struct iris_resource *)resource;
 493
 494    whandle->stride = res->surf.row_pitch_B;
 495    whandle->modifier = tiling_to_modifier(res->bo->tiling_mode);
 496
 497    switch (whandle->type) {
 498    case WINSYS_HANDLE_TYPE_SHARED:
 499       return iris_bo_flink(res->bo, &whandle->handle) == 0;
 500    case WINSYS_HANDLE_TYPE_KMS:
 501       whandle->handle = iris_bo_export_gem_handle(res->bo);
 502       return true;
 503    case WINSYS_HANDLE_TYPE_FD:
 504       return iris_bo_export_dmabuf(res->bo, (int *) &whandle->handle) == 0;
 505    }
 506
 507    return false;
 508 }
 509
 510 static void
 511 get_image_offset_el(struct isl_surf *surf, unsigned level, unsigned z,
 512                     unsigned *out_x0_el, unsigned *out_y0_el)
 513 {
 514    if (surf->dim == ISL_SURF_DIM_3D) {
 515       isl_surf_get_image_offset_el(surf, level, 0, z, out_x0_el, out_y0_el);
 516    } else {
 517       isl_surf_get_image_offset_el(surf, level, z, 0, out_x0_el, out_y0_el);
 518    }
 519 }
 520
 521 /**
 522  * Get pointer offset into stencil buffer.
 523  *
 524  * The stencil buffer is W tiled. Since the GTT is incapable of W fencing, we
 525  * must decode the tile's layout in software.
 526  *
 527  * See
 528  *   - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.2.1 W-Major Tile
 529  *     Format.
 530  *   - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.3 Tiling Algorithm
 531  *
 532  * Even though the returned offset is always positive, the return type is
 533  * signed due to
 534  *    commit e8b1c6d6f55f5be3bef25084fdd8b6127517e137
 535  *    mesa: Fix return type of  _mesa_get_format_bytes() (#37351)
 536  */
 537 static intptr_t
 538 s8_offset(uint32_t stride, uint32_t x, uint32_t y, bool swizzled)
 539 {
 540    uint32_t tile_size = 4096;
 541    uint32_t tile_width = 64;
 542    uint32_t tile_height = 64;
 543    uint32_t row_size = 64 * stride / 2; /* Two rows are interleaved. */
 544
 545    uint32_t tile_x = x / tile_width;
 546    uint32_t tile_y = y / tile_height;
 547
 548    /* The byte's address relative to the tile's base addres. */
 549    uint32_t byte_x = x % tile_width;
 550    uint32_t byte_y = y % tile_height;
 551
 552    uintptr_t u = tile_y * row_size
 553                + tile_x * tile_size
 554                + 512 * (byte_x / 8)
 555                +  64 * (byte_y / 8)
 556                +  32 * ((byte_y / 4) % 2)
 557                +  16 * ((byte_x / 4) % 2)
 558                +   8 * ((byte_y / 2) % 2)
 559                +   4 * ((byte_x / 2) % 2)
 560                +   2 * (byte_y % 2)
 561                +   1 * (byte_x % 2);
 562
 563    if (swizzled) {
 564       /* adjust for bit6 swizzling */
 565       if (((byte_x / 8) % 2) == 1) {
 566          if (((byte_y / 8) % 2) == 0) {
 567             u += 64;
 568          } else {
 569             u -= 64;
 570          }
 571       }
 572    }
 573
 574    return u;
 575 }
 576
 577 static void
 578 iris_unmap_s8(struct iris_transfer *map)
 579 {
 580    struct pipe_transfer *xfer = &map->base;
 581    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 582    struct isl_surf *surf = &res->surf;
 583    const bool has_swizzling = false;
 584
 585    if (xfer->usage & PIPE_TRANSFER_WRITE) {
 586       uint8_t *untiled_s8_map = map->ptr;
 587       uint8_t *tiled_s8_map =
 588          iris_bo_map(map->dbg, res->bo, xfer->usage | MAP_RAW);
 589
 590       struct pipe_box box = xfer->box;
 591
 592       for (int s = 0; s < box.depth; s++) {
 593          unsigned x0_el, y0_el;
 594          get_image_offset_el(surf, xfer->level, box.z, &x0_el, &y0_el);
 595
 596          for (uint32_t y = 0; y < box.height; y++) {
 597             for (uint32_t x = 0; x < box.width; x++) {
 598                ptrdiff_t offset = s8_offset(surf->row_pitch_B,
 599                                             x0_el + box.x + x,
 600                                             y0_el + box.y + y,
 601                                             has_swizzling);
 602                tiled_s8_map[offset] =
 603                   untiled_s8_map[s * xfer->layer_stride + y * xfer->stride + x];
 604             }
 605          }
 606
 607          box.z++;
 608       }
 609    }
 610
 611    free(map->buffer);
 612 }
 613
 614 static void
 615 iris_map_s8(struct iris_transfer *map)
 616 {
 617    struct pipe_transfer *xfer = &map->base;
 618    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 619    struct isl_surf *surf = &res->surf;
 620
 621    xfer->stride = surf->row_pitch_B;
 622    xfer->layer_stride = xfer->stride * xfer->box.height;
 623
 624    /* The tiling and detiling functions require that the linear buffer has
 625     * a 16-byte alignment (that is, its `x0` is 16-byte aligned).  Here we
 626     * over-allocate the linear buffer to get the proper alignment.
 627     */
 628    map->buffer = map->ptr = malloc(xfer->layer_stride * xfer->box.depth);
 629    assert(map->buffer);
 630
 631    const bool has_swizzling = false;
 632
 633    /* One of either READ_BIT or WRITE_BIT or both is set.  READ_BIT implies no
 634     * INVALIDATE_RANGE_BIT.  WRITE_BIT needs the original values read in unless
 635     * invalidate is set, since we'll be writing the whole rectangle from our
 636     * temporary buffer back out.
 637     */
 638    if (!(xfer->usage & PIPE_TRANSFER_DISCARD_RANGE)) {
 639       uint8_t *untiled_s8_map = map->ptr;
 640       uint8_t *tiled_s8_map =
 641          iris_bo_map(map->dbg, res->bo, xfer->usage | MAP_RAW);
 642
 643       struct pipe_box box = xfer->box;
 644
 645       for (int s = 0; s < box.depth; s++) {
 646          unsigned x0_el, y0_el;
 647          get_image_offset_el(surf, xfer->level, box.z, &x0_el, &y0_el);
 648
 649          for (uint32_t y = 0; y < box.height; y++) {
 650             for (uint32_t x = 0; x < box.width; x++) {
 651                ptrdiff_t offset = s8_offset(surf->row_pitch_B,
 652                                             x0_el + box.x + x,
 653                                             y0_el + box.y + y,
 654                                             has_swizzling);
 655                untiled_s8_map[s * xfer->layer_stride + y * xfer->stride + x] =
 656                   tiled_s8_map[offset];
 657             }
 658          }
 659
 660          box.z++;
 661       }
 662    }
 663
 664    map->unmap = iris_unmap_s8;
 665 }
 666
 667 /* Compute extent parameters for use with tiled_memcpy functions.
 668  * xs are in units of bytes and ys are in units of strides.
 669  */
 670 static inline void
 671 tile_extents(struct isl_surf *surf,
 672              const struct pipe_box *box,
 673              unsigned level,
 674              unsigned *x1_B, unsigned *x2_B,
 675              unsigned *y1_el, unsigned *y2_el)
 676 {
 677    const struct isl_format_layout *fmtl = isl_format_get_layout(surf->format);
 678    const unsigned cpp = fmtl->bpb / 8;
 679
 680    assert(box->x % fmtl->bw == 0);
 681    assert(box->y % fmtl->bh == 0);
 682
 683    unsigned x0_el, y0_el;
 684    get_image_offset_el(surf, level, box->z, &x0_el, &y0_el);
 685
 686    *x1_B = (box->x / fmtl->bw + x0_el) * cpp;
 687    *y1_el = box->y / fmtl->bh + y0_el;
 688    *x2_B = (DIV_ROUND_UP(box->x + box->width, fmtl->bw) + x0_el) * cpp;
 689    *y2_el = DIV_ROUND_UP(box->y + box->height, fmtl->bh) + y0_el;
 690 }
 691
 692 static void
 693 iris_unmap_tiled_memcpy(struct iris_transfer *map)
 694 {
 695    struct pipe_transfer *xfer = &map->base;
 696    struct pipe_box box = xfer->box;
 697    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 698    struct isl_surf *surf = &res->surf;
 699
 700    const bool has_swizzling = false;
 701
 702    if (xfer->usage & PIPE_TRANSFER_WRITE) {
 703       char *dst = iris_bo_map(map->dbg, res->bo, xfer->usage | MAP_RAW);
 704
 705       for (int s = 0; s < box.depth; s++) {
 706          unsigned x1, x2, y1, y2;
 707          tile_extents(surf, &box, xfer->level, &x1, &x2, &y1, &y2);
 708
 709          void *ptr = map->ptr + s * xfer->layer_stride;
 710
 711          isl_memcpy_linear_to_tiled(x1, x2, y1, y2, dst, ptr,
 712                                     surf->row_pitch_B, xfer->stride,
 713                                     has_swizzling, surf->tiling, ISL_MEMCPY);
 714          box.z++;
 715       }
 716    }
 717    os_free_aligned(map->buffer);
 718    map->buffer = map->ptr = NULL;
 719 }
 720
 721 static void
 722 iris_map_tiled_memcpy(struct iris_transfer *map)
 723 {
 724    struct pipe_transfer *xfer = &map->base;
 725    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 726    struct isl_surf *surf = &res->surf;
 727
 728    xfer->stride = ALIGN(surf->row_pitch_B, 16);
 729    xfer->layer_stride = xfer->stride * xfer->box.height;
 730
 731    unsigned x1, x2, y1, y2;
 732    tile_extents(surf, &xfer->box, xfer->level, &x1, &x2, &y1, &y2);
 733
 734    /* The tiling and detiling functions require that the linear buffer has
 735     * a 16-byte alignment (that is, its `x0` is 16-byte aligned).  Here we
 736     * over-allocate the linear buffer to get the proper alignment.
 737     */
 738    map->buffer =
 739       os_malloc_aligned(xfer->layer_stride * xfer->box.depth, 16);
 740    assert(map->buffer);
 741    map->ptr = (char *)map->buffer + (x1 & 0xf);
 742
 743    const bool has_swizzling = false;
 744
 745    // XXX: PIPE_TRANSFER_READ?
 746    if (!(xfer->usage & PIPE_TRANSFER_DISCARD_RANGE)) {
 747       char *src = iris_bo_map(map->dbg, res->bo, xfer->usage | MAP_RAW);
 748
 749       struct pipe_box box = xfer->box;
 750
 751       for (int s = 0; s < box.depth; s++) {
 752          unsigned x1, x2, y1, y2;
 753          tile_extents(surf, &box, xfer->level, &x1, &x2, &y1, &y2);
 754
 755          /* When transferring cubes, box.depth is counted in cubes, but
 756           * box.z is counted in faces.  We want to transfer only the
 757           * specified face, but for all array elements.  So, use 's'
 758           * (the zero-based slice count) rather than box.z.
 759           */
 760          void *ptr = map->ptr + s * xfer->layer_stride;
 761
 762          isl_memcpy_tiled_to_linear(x1, x2, y1, y2, ptr, src, xfer->stride,
 763                                     surf->row_pitch_B, has_swizzling,
 764                                     surf->tiling, ISL_MEMCPY);
 765          box.z++;
 766       }
 767    }
 768
 769    map->unmap = iris_unmap_tiled_memcpy;
 770 }
 771
 772 static void
 773 iris_map_direct(struct iris_transfer *map)
 774 {
 775    struct pipe_transfer *xfer = &map->base;
 776    struct pipe_box *box = &xfer->box;
 777    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 778    struct isl_surf *surf = &res->surf;
 779    const struct isl_format_layout *fmtl = isl_format_get_layout(surf->format);
 780    const unsigned cpp = fmtl->bpb / 8;
 781    unsigned x0_el, y0_el;
 782
 783    void *ptr = iris_bo_map(map->dbg, res->bo, xfer->usage);
 784
 785    if (res->base.target == PIPE_BUFFER) {
 786       xfer->stride = 0;
 787       xfer->layer_stride = 0;
 788
 789       map->ptr = ptr + box->x;
 790    } else {
 791       get_image_offset_el(surf, xfer->level, box->z, &x0_el, &y0_el);
 792
 793       xfer->stride = isl_surf_get_row_pitch_B(surf);
 794       xfer->layer_stride = isl_surf_get_array_pitch(surf);
 795
 796       map->ptr = ptr + (y0_el + box->y) * xfer->stride + (x0_el + box->x) * cpp;
 797    }
 798 }
 799
 800 static void *
 801 iris_transfer_map(struct pipe_context *ctx,
 802                   struct pipe_resource *resource,
 803                   unsigned level,
 804                   enum pipe_transfer_usage usage,
 805                   const struct pipe_box *box,
 806                   struct pipe_transfer **ptransfer)
 807 {
 808    struct iris_context *ice = (struct iris_context *)ctx;
 809    struct iris_resource *res = (struct iris_resource *)resource;
 810    struct isl_surf *surf = &res->surf;
 811
 812    if (surf->tiling != ISL_TILING_LINEAR &&
 813        (usage & PIPE_TRANSFER_MAP_DIRECTLY))
 814       return NULL;
 815
 816    if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED) &&
 817        iris_batch_references(&ice->render_batch, res->bo)) {
 818       iris_batch_flush(&ice->render_batch);
 819    }
 820
 821    if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED) &&
 822        iris_batch_references(&ice->compute_batch, res->bo)) {
 823       iris_batch_flush(&ice->compute_batch);
 824    }
 825
 826    if ((usage & PIPE_TRANSFER_DONTBLOCK) && iris_bo_busy(res->bo))
 827       return NULL;
 828
 829    struct iris_transfer *map = slab_alloc(&ice->transfer_pool);
 830    struct pipe_transfer *xfer = &map->base;
 831
 832    // PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
 833    // PIPE_TRANSFER_DISCARD_RANGE
 834
 835    if (!map)
 836       return NULL;
 837
 838    memset(map, 0, sizeof(*map));
 839    map->dbg = &ice->dbg;
 840
 841    pipe_resource_reference(&xfer->resource, resource);
 842    xfer->level = level;
 843    xfer->usage = usage;
 844    xfer->box = *box;
 845    *ptransfer = xfer;
 846
 847    xfer->usage &= (PIPE_TRANSFER_READ |
 848                    PIPE_TRANSFER_WRITE |
 849                    PIPE_TRANSFER_UNSYNCHRONIZED |
 850                    PIPE_TRANSFER_PERSISTENT |
 851                    PIPE_TRANSFER_COHERENT |
 852                    PIPE_TRANSFER_DISCARD_RANGE);
 853
 854    if (surf->tiling == ISL_TILING_W) {
 855       // XXX: just teach iris_map_tiled_memcpy about W tiling...
 856       iris_map_s8(map);
 857    } else if (surf->tiling != ISL_TILING_LINEAR) {
 858       iris_map_tiled_memcpy(map);
 859    } else {
 860       iris_map_direct(map);
 861    }
 862
 863    return map->ptr;
 864 }
 865
 866 static void
 867 iris_transfer_unmap(struct pipe_context *ctx, struct pipe_transfer *xfer)
 868 {
 869    struct iris_context *ice = (struct iris_context *)ctx;
 870    struct iris_transfer *map = (void *) xfer;
 871    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 872    struct isl_surf *surf = &res->surf;
 873
 874    if (map->unmap)
 875       map->unmap(map);
 876
 877    /* XXX: big ol' hack!  need to re-emit UBOs.  want bind_history? */
 878    if (surf->tiling == ISL_TILING_LINEAR) {
 879       ice->state.dirty |= IRIS_DIRTY_CONSTANTS_VS  | IRIS_DIRTY_BINDINGS_VS
 880                        |  IRIS_DIRTY_CONSTANTS_TCS | IRIS_DIRTY_BINDINGS_TCS
 881                        |  IRIS_DIRTY_CONSTANTS_TES | IRIS_DIRTY_BINDINGS_TES
 882                        |  IRIS_DIRTY_CONSTANTS_GS  | IRIS_DIRTY_BINDINGS_GS
 883                        |  IRIS_DIRTY_CONSTANTS_FS  | IRIS_DIRTY_BINDINGS_FS;
 884    }
 885
 886    pipe_resource_reference(&xfer->resource, NULL);
 887    slab_free(&ice->transfer_pool, map);
 888 }
 889
 890 static void
 891 iris_flush_resource(struct pipe_context *ctx, struct pipe_resource *resource)
 892 {
 893 }
 894
 895 static enum pipe_format
 896 iris_resource_get_internal_format(struct pipe_resource *p_res)
 897 {
 898    struct iris_resource *res = (void *) p_res;
 899    return res->internal_format;
 900 }
 901
 902 static const struct u_transfer_vtbl transfer_vtbl = {
 903    .resource_create       = iris_resource_create,
 904    .resource_destroy      = iris_resource_destroy,
 905    .transfer_map          = iris_transfer_map,
 906    .transfer_unmap        = iris_transfer_unmap,
 907    .transfer_flush_region = u_default_transfer_flush_region,
 908    .get_internal_format   = iris_resource_get_internal_format,
 909    .set_stencil           = iris_resource_set_separate_stencil,
 910    .get_stencil           = iris_resource_get_separate_stencil,
 911 };
 912
 913 void
 914 iris_init_screen_resource_functions(struct pipe_screen *pscreen)
 915 {
 916    pscreen->resource_create_with_modifiers =
 917       iris_resource_create_with_modifiers;
 918    pscreen->resource_create = u_transfer_helper_resource_create;
 919    pscreen->resource_from_user_memory = iris_resource_from_user_memory;
 920    pscreen->resource_from_handle = iris_resource_from_handle;
 921    pscreen->resource_get_handle = iris_resource_get_handle;
 922    pscreen->resource_destroy = u_transfer_helper_resource_destroy;
 923    pscreen->transfer_helper =
 924       u_transfer_helper_create(&transfer_vtbl, true, true, false, true);
 925 }
 926
 927 void
 928 iris_init_resource_functions(struct pipe_context *ctx)
 929 {
 930    ctx->flush_resource = iris_flush_resource;
 931    ctx->transfer_map = u_transfer_helper_transfer_map;
 932    ctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
 933    ctx->transfer_unmap = u_transfer_helper_transfer_unmap;
 934    ctx->buffer_subdata = u_default_buffer_subdata;
 935    ctx->texture_subdata = u_default_texture_subdata;
 936 }