src/gallium/drivers/iris/iris_resource.c

   1 /*
   2  * Copyright © 2017 Intel Corporation
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included
  12  * in all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  15  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  18  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  19  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  20  * DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 /**
  24  * @file iris_resource.c
  25  *
  26  * Resources are images, buffers, and other objects used by the GPU.
  27  *
  28  * XXX: explain resources
  29  */
  30
  31 #include <stdio.h>
  32 #include <errno.h>
  33 #include "pipe/p_defines.h"
  34 #include "pipe/p_state.h"
  35 #include "pipe/p_context.h"
  36 #include "pipe/p_screen.h"
  37 #include "util/os_memory.h"
  38 #include "util/u_cpu_detect.h"
  39 #include "util/u_inlines.h"
  40 #include "util/u_format.h"
  41 #include "util/u_transfer.h"
  42 #include "util/u_transfer_helper.h"
  43 #include "util/u_upload_mgr.h"
  44 #include "util/ralloc.h"
  45 #include "iris_batch.h"
  46 #include "iris_context.h"
  47 #include "iris_resource.h"
  48 #include "iris_screen.h"
  49 #include "intel/common/gen_debug.h"
  50 #include "isl/isl.h"
  51 #include "drm-uapi/drm_fourcc.h"
  52 #include "drm-uapi/i915_drm.h"
  53
  54 enum modifier_priority {
  55    MODIFIER_PRIORITY_INVALID = 0,
  56    MODIFIER_PRIORITY_LINEAR,
  57    MODIFIER_PRIORITY_X,
  58    MODIFIER_PRIORITY_Y,
  59    MODIFIER_PRIORITY_Y_CCS,
  60 };
  61
  62 static const uint64_t priority_to_modifier[] = {
  63    [MODIFIER_PRIORITY_INVALID] = DRM_FORMAT_MOD_INVALID,
  64    [MODIFIER_PRIORITY_LINEAR] = DRM_FORMAT_MOD_LINEAR,
  65    [MODIFIER_PRIORITY_X] = I915_FORMAT_MOD_X_TILED,
  66    [MODIFIER_PRIORITY_Y] = I915_FORMAT_MOD_Y_TILED,
  67    [MODIFIER_PRIORITY_Y_CCS] = I915_FORMAT_MOD_Y_TILED_CCS,
  68 };
  69
  70 static bool
  71 modifier_is_supported(const struct gen_device_info *devinfo,
  72                       uint64_t modifier)
  73 {
  74    /* XXX: do something real */
  75    switch (modifier) {
  76    case I915_FORMAT_MOD_Y_TILED:
  77    case I915_FORMAT_MOD_X_TILED:
  78    case DRM_FORMAT_MOD_LINEAR:
  79       return true;
  80    case I915_FORMAT_MOD_Y_TILED_CCS:
  81    case DRM_FORMAT_MOD_INVALID:
  82    default:
  83       return false;
  84    }
  85 }
  86
  87 static uint64_t
  88 select_best_modifier(struct gen_device_info *devinfo,
  89                      const uint64_t *modifiers,
  90                      int count)
  91 {
  92    enum modifier_priority prio = MODIFIER_PRIORITY_INVALID;
  93
  94    for (int i = 0; i < count; i++) {
  95       if (!modifier_is_supported(devinfo, modifiers[i]))
  96          continue;
  97
  98       switch (modifiers[i]) {
  99       case I915_FORMAT_MOD_Y_TILED_CCS:
 100          prio = MAX2(prio, MODIFIER_PRIORITY_Y_CCS);
 101          break;
 102       case I915_FORMAT_MOD_Y_TILED:
 103          prio = MAX2(prio, MODIFIER_PRIORITY_Y);
 104          break;
 105       case I915_FORMAT_MOD_X_TILED:
 106          prio = MAX2(prio, MODIFIER_PRIORITY_X);
 107          break;
 108       case DRM_FORMAT_MOD_LINEAR:
 109          prio = MAX2(prio, MODIFIER_PRIORITY_LINEAR);
 110          break;
 111       case DRM_FORMAT_MOD_INVALID:
 112       default:
 113          break;
 114       }
 115    }
 116
 117    return priority_to_modifier[prio];
 118 }
 119
 120 static enum isl_surf_dim
 121 target_to_isl_surf_dim(enum pipe_texture_target target)
 122 {
 123    switch (target) {
 124    case PIPE_BUFFER:
 125    case PIPE_TEXTURE_1D:
 126    case PIPE_TEXTURE_1D_ARRAY:
 127       return ISL_SURF_DIM_1D;
 128    case PIPE_TEXTURE_2D:
 129    case PIPE_TEXTURE_CUBE:
 130    case PIPE_TEXTURE_RECT:
 131    case PIPE_TEXTURE_2D_ARRAY:
 132    case PIPE_TEXTURE_CUBE_ARRAY:
 133       return ISL_SURF_DIM_2D;
 134    case PIPE_TEXTURE_3D:
 135       return ISL_SURF_DIM_3D;
 136    case PIPE_MAX_TEXTURE_TYPES:
 137       break;
 138    }
 139    unreachable("invalid texture type");
 140 }
 141
 142 static isl_surf_usage_flags_t
 143 pipe_bind_to_isl_usage(unsigned bindings)
 144 {
 145    isl_surf_usage_flags_t usage = 0;
 146
 147    if (bindings & PIPE_BIND_RENDER_TARGET)
 148       usage |= ISL_SURF_USAGE_RENDER_TARGET_BIT;
 149
 150    if (bindings & PIPE_BIND_SAMPLER_VIEW)
 151       usage |= ISL_SURF_USAGE_TEXTURE_BIT;
 152
 153    if (bindings & (PIPE_BIND_SHADER_IMAGE | PIPE_BIND_SHADER_BUFFER))
 154       usage |= ISL_SURF_USAGE_STORAGE_BIT;
 155
 156    if (bindings & PIPE_BIND_DISPLAY_TARGET)
 157       usage |= ISL_SURF_USAGE_DISPLAY_BIT;
 158
 159    return usage;
 160 }
 161
 162 struct pipe_resource *
 163 iris_resource_get_separate_stencil(struct pipe_resource *p_res)
 164 {
 165    /* For packed depth-stencil, we treat depth as the primary resource
 166     * and store S8 as the "second plane" resource.
 167     */
 168    return p_res->next;
 169 }
 170
 171 static void
 172 iris_resource_set_separate_stencil(struct pipe_resource *p_res,
 173                                    struct pipe_resource *stencil)
 174 {
 175    assert(util_format_has_depth(util_format_description(p_res->format)));
 176    pipe_resource_reference(&p_res->next, stencil);
 177 }
 178
 179 void
 180 iris_get_depth_stencil_resources(struct pipe_resource *res,
 181                                  struct iris_resource **out_z,
 182                                  struct iris_resource **out_s)
 183 {
 184    if (!res) {
 185       *out_z = NULL;
 186       *out_s = NULL;
 187       return;
 188    }
 189
 190    const struct util_format_description *desc =
 191       util_format_description(res->format);
 192
 193    if (util_format_has_depth(desc)) {
 194       *out_z = (void *) res;
 195       *out_s = (void *) iris_resource_get_separate_stencil(res);
 196    } else {
 197       assert(util_format_has_stencil(desc));
 198       *out_z = NULL;
 199       *out_s = (void *) res;
 200    }
 201 }
 202
 203 static void
 204 iris_resource_destroy(struct pipe_screen *screen,
 205                       struct pipe_resource *resource)
 206 {
 207    struct iris_resource *res = (struct iris_resource *)resource;
 208
 209    iris_bo_unreference(res->bo);
 210    free(res);
 211 }
 212
 213 static struct iris_resource *
 214 iris_alloc_resource(struct pipe_screen *pscreen,
 215                     const struct pipe_resource *templ)
 216 {
 217    struct iris_resource *res = calloc(1, sizeof(struct iris_resource));
 218    if (!res)
 219       return NULL;
 220
 221    res->base = *templ;
 222    res->base.screen = pscreen;
 223    pipe_reference_init(&res->base.reference, 1);
 224
 225    return res;
 226 }
 227
 228 static struct pipe_resource *
 229 iris_resource_create_with_modifiers(struct pipe_screen *pscreen,
 230                                     const struct pipe_resource *templ,
 231                                     const uint64_t *modifiers,
 232                                     int modifiers_count)
 233 {
 234    struct iris_screen *screen = (struct iris_screen *)pscreen;
 235    struct gen_device_info *devinfo = &screen->devinfo;
 236    struct iris_resource *res = iris_alloc_resource(pscreen, templ);
 237    const struct util_format_description *format_desc =
 238       util_format_description(templ->format);
 239
 240    if (!res)
 241       return NULL;
 242
 243    const bool has_depth = util_format_has_depth(format_desc);
 244    uint64_t modifier =
 245       select_best_modifier(devinfo, modifiers, modifiers_count);
 246
 247    isl_tiling_flags_t tiling_flags = ISL_TILING_ANY_MASK;
 248
 249    if (modifier != DRM_FORMAT_MOD_INVALID) {
 250       const struct isl_drm_modifier_info *mod_info =
 251          isl_drm_modifier_get_info(modifier);
 252
 253       tiling_flags = 1 << mod_info->tiling;
 254    } else {
 255       if (modifiers_count > 0) {
 256          fprintf(stderr, "Unsupported modifier, resource creation failed.\n");
 257          return NULL;
 258       }
 259
 260       /* No modifiers - we can select our own tiling. */
 261
 262       if (has_depth) {
 263          /* Depth must be Y-tiled */
 264          tiling_flags = ISL_TILING_Y0_BIT;
 265       } else if (templ->format == PIPE_FORMAT_S8_UINT) {
 266          /* Stencil must be W-tiled */
 267          tiling_flags = ISL_TILING_W_BIT;
 268       } else if (templ->target == PIPE_BUFFER ||
 269                  templ->target == PIPE_TEXTURE_1D ||
 270                  templ->target == PIPE_TEXTURE_1D_ARRAY) {
 271          /* Use linear for buffers and 1D textures */
 272          tiling_flags = ISL_TILING_LINEAR_BIT;
 273       }
 274
 275       /* Use linear for staging buffers */
 276       if (templ->usage == PIPE_USAGE_STAGING ||
 277           templ->bind & (PIPE_BIND_LINEAR | PIPE_BIND_CURSOR) )
 278          tiling_flags = ISL_TILING_LINEAR_BIT;
 279    }
 280
 281    isl_surf_usage_flags_t usage = pipe_bind_to_isl_usage(templ->bind);
 282
 283    if (templ->target == PIPE_TEXTURE_CUBE ||
 284        templ->target == PIPE_TEXTURE_CUBE_ARRAY)
 285       usage |= ISL_SURF_USAGE_CUBE_BIT;
 286
 287    if (templ->usage != PIPE_USAGE_STAGING) {
 288       if (templ->format == PIPE_FORMAT_S8_UINT)
 289          usage |= ISL_SURF_USAGE_STENCIL_BIT;
 290       else if (has_depth)
 291          usage |= ISL_SURF_USAGE_DEPTH_BIT;
 292    }
 293
 294    enum pipe_format pfmt = templ->format;
 295    res->internal_format = pfmt;
 296
 297    /* Should be handled by u_transfer_helper */
 298    assert(!util_format_is_depth_and_stencil(pfmt));
 299
 300    struct iris_format_info fmt = iris_format_for_usage(devinfo, pfmt, usage);
 301    assert(fmt.fmt != ISL_FORMAT_UNSUPPORTED);
 302
 303    UNUSED const bool isl_surf_created_successfully =
 304       isl_surf_init(&screen->isl_dev, &res->surf,
 305                     .dim = target_to_isl_surf_dim(templ->target),
 306                     .format = fmt.fmt,
 307                     .width = templ->width0,
 308                     .height = templ->height0,
 309                     .depth = templ->depth0,
 310                     .levels = templ->last_level + 1,
 311                     .array_len = templ->array_size,
 312                     .samples = MAX2(templ->nr_samples, 1),
 313                     .min_alignment_B = 0,
 314                     .row_pitch_B = 0,
 315                     .usage = usage,
 316                     .tiling_flags = tiling_flags);
 317    assert(isl_surf_created_successfully);
 318
 319    enum iris_memory_zone memzone = IRIS_MEMZONE_OTHER;
 320    const char *name = templ->target == PIPE_BUFFER ? "buffer" : "miptree";
 321    if (templ->flags & IRIS_RESOURCE_FLAG_SHADER_MEMZONE) {
 322       memzone = IRIS_MEMZONE_SHADER;
 323       name = "shader kernels";
 324    } else if (templ->flags & IRIS_RESOURCE_FLAG_SURFACE_MEMZONE) {
 325       memzone = IRIS_MEMZONE_SURFACE;
 326       name = "surface state";
 327    } else if (templ->flags & IRIS_RESOURCE_FLAG_DYNAMIC_MEMZONE) {
 328       memzone = IRIS_MEMZONE_DYNAMIC;
 329       name = "dynamic state";
 330    }
 331
 332    res->bo = iris_bo_alloc_tiled(screen->bufmgr, name, res->surf.size_B,
 333                                  memzone,
 334                                  isl_tiling_to_i915_tiling(res->surf.tiling),
 335                                  res->surf.row_pitch_B, 0);
 336    if (!res->bo)
 337       goto fail;
 338
 339    return &res->base;
 340
 341 fail:
 342    iris_resource_destroy(pscreen, &res->base);
 343    return NULL;
 344 }
 345
 346 static struct pipe_resource *
 347 iris_resource_create(struct pipe_screen *pscreen,
 348                      const struct pipe_resource *templ)
 349 {
 350    return iris_resource_create_with_modifiers(pscreen, templ, NULL, 0);
 351 }
 352
 353 static uint64_t
 354 tiling_to_modifier(uint32_t tiling)
 355 {
 356    static const uint64_t map[] = {
 357       [I915_TILING_NONE]   = DRM_FORMAT_MOD_LINEAR,
 358       [I915_TILING_X]      = I915_FORMAT_MOD_X_TILED,
 359       [I915_TILING_Y]      = I915_FORMAT_MOD_Y_TILED,
 360    };
 361
 362    assert(tiling < ARRAY_SIZE(map));
 363
 364    return map[tiling];
 365 }
 366
 367 static struct pipe_resource *
 368 iris_resource_from_user_memory(struct pipe_screen *pscreen,
 369                                const struct pipe_resource *templ,
 370                                void *user_memory)
 371 {
 372    struct iris_screen *screen = (struct iris_screen *)pscreen;
 373    struct gen_device_info *devinfo = &screen->devinfo;
 374    struct iris_bufmgr *bufmgr = screen->bufmgr;
 375    struct iris_resource *res = iris_alloc_resource(pscreen, templ);
 376    if (!res)
 377       return NULL;
 378
 379    res->bo = iris_bo_create_userptr(bufmgr, "user",
 380                                     user_memory, templ->width0,
 381                                     IRIS_MEMZONE_OTHER);
 382    if (!res->bo) {
 383       free(res);
 384       return NULL;
 385    }
 386
 387    res->internal_format = templ->format;
 388
 389    // XXX: usage...
 390    isl_surf_usage_flags_t isl_usage = 0;
 391
 392    const struct iris_format_info fmt =
 393       iris_format_for_usage(devinfo, templ->format, isl_usage);
 394
 395    isl_surf_init(&screen->isl_dev, &res->surf,
 396                  .dim = target_to_isl_surf_dim(templ->target),
 397                  .format = fmt.fmt,
 398                  .width = templ->width0,
 399                  .height = templ->height0,
 400                  .depth = templ->depth0,
 401                  .levels = templ->last_level + 1,
 402                  .array_len = templ->array_size,
 403                  .samples = MAX2(templ->nr_samples, 1),
 404                  .min_alignment_B = 0,
 405                  .row_pitch_B = 0,
 406                  .usage = isl_usage,
 407                  .tiling_flags = 1 << ISL_TILING_LINEAR);
 408
 409    assert(res->bo->tiling_mode == isl_tiling_to_i915_tiling(res->surf.tiling));
 410
 411    return &res->base;
 412 }
 413
 414 static struct pipe_resource *
 415 iris_resource_from_handle(struct pipe_screen *pscreen,
 416                           const struct pipe_resource *templ,
 417                           struct winsys_handle *whandle,
 418                           unsigned usage)
 419 {
 420    struct iris_screen *screen = (struct iris_screen *)pscreen;
 421    struct gen_device_info *devinfo = &screen->devinfo;
 422    struct iris_bufmgr *bufmgr = screen->bufmgr;
 423    struct iris_resource *res = iris_alloc_resource(pscreen, templ);
 424    if (!res)
 425       return NULL;
 426
 427    if (whandle->offset != 0) {
 428       dbg_printf("Attempt to import unsupported winsys offset %u\n",
 429                  whandle->offset);
 430       goto fail;
 431    }
 432
 433    switch (whandle->type) {
 434    case WINSYS_HANDLE_TYPE_FD:
 435       res->bo = iris_bo_import_dmabuf(bufmgr, whandle->handle);
 436       break;
 437    case WINSYS_HANDLE_TYPE_SHARED:
 438       res->bo = iris_bo_gem_create_from_name(bufmgr, "winsys image",
 439                                              whandle->handle);
 440       break;
 441    default:
 442       unreachable("invalid winsys handle type");
 443    }
 444    if (!res->bo)
 445       return NULL;
 446
 447    uint64_t modifier = whandle->modifier;
 448    if (modifier == DRM_FORMAT_MOD_INVALID) {
 449           modifier = tiling_to_modifier(res->bo->tiling_mode);
 450    }
 451    const struct isl_drm_modifier_info *mod_info =
 452       isl_drm_modifier_get_info(modifier);
 453    assert(mod_info);
 454
 455    // XXX: usage...
 456    isl_surf_usage_flags_t isl_usage = ISL_SURF_USAGE_DISPLAY_BIT;
 457
 458    const struct iris_format_info fmt =
 459       iris_format_for_usage(devinfo, templ->format, isl_usage);
 460
 461    isl_surf_init(&screen->isl_dev, &res->surf,
 462                  .dim = target_to_isl_surf_dim(templ->target),
 463                  .format = fmt.fmt,
 464                  .width = templ->width0,
 465                  .height = templ->height0,
 466                  .depth = templ->depth0,
 467                  .levels = templ->last_level + 1,
 468                  .array_len = templ->array_size,
 469                  .samples = MAX2(templ->nr_samples, 1),
 470                  .min_alignment_B = 0,
 471                  .row_pitch_B = 0,
 472                  .usage = isl_usage,
 473                  .tiling_flags = 1 << mod_info->tiling);
 474
 475    assert(res->bo->tiling_mode == isl_tiling_to_i915_tiling(res->surf.tiling));
 476
 477    return &res->base;
 478
 479 fail:
 480    iris_resource_destroy(pscreen, &res->base);
 481    return NULL;
 482 }
 483
 484 static boolean
 485 iris_resource_get_handle(struct pipe_screen *pscreen,
 486                          struct pipe_context *ctx,
 487                          struct pipe_resource *resource,
 488                          struct winsys_handle *whandle,
 489                          unsigned usage)
 490 {
 491    struct iris_resource *res = (struct iris_resource *)resource;
 492
 493    whandle->stride = res->surf.row_pitch_B;
 494    whandle->modifier = tiling_to_modifier(res->bo->tiling_mode);
 495
 496    switch (whandle->type) {
 497    case WINSYS_HANDLE_TYPE_SHARED:
 498       return iris_bo_flink(res->bo, &whandle->handle) == 0;
 499    case WINSYS_HANDLE_TYPE_KMS:
 500       whandle->handle = iris_bo_export_gem_handle(res->bo);
 501       return true;
 502    case WINSYS_HANDLE_TYPE_FD:
 503       return iris_bo_export_dmabuf(res->bo, (int *) &whandle->handle) == 0;
 504    }
 505
 506    return false;
 507 }
 508
 509 static void
 510 get_image_offset_el(struct isl_surf *surf, unsigned level, unsigned z,
 511                     unsigned *out_x0_el, unsigned *out_y0_el)
 512 {
 513    if (surf->dim == ISL_SURF_DIM_3D) {
 514       isl_surf_get_image_offset_el(surf, level, 0, z, out_x0_el, out_y0_el);
 515    } else {
 516       isl_surf_get_image_offset_el(surf, level, z, 0, out_x0_el, out_y0_el);
 517    }
 518 }
 519
 520 /**
 521  * Get pointer offset into stencil buffer.
 522  *
 523  * The stencil buffer is W tiled. Since the GTT is incapable of W fencing, we
 524  * must decode the tile's layout in software.
 525  *
 526  * See
 527  *   - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.2.1 W-Major Tile
 528  *     Format.
 529  *   - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.3 Tiling Algorithm
 530  *
 531  * Even though the returned offset is always positive, the return type is
 532  * signed due to
 533  *    commit e8b1c6d6f55f5be3bef25084fdd8b6127517e137
 534  *    mesa: Fix return type of  _mesa_get_format_bytes() (#37351)
 535  */
 536 static intptr_t
 537 s8_offset(uint32_t stride, uint32_t x, uint32_t y, bool swizzled)
 538 {
 539    uint32_t tile_size = 4096;
 540    uint32_t tile_width = 64;
 541    uint32_t tile_height = 64;
 542    uint32_t row_size = 64 * stride / 2; /* Two rows are interleaved. */
 543
 544    uint32_t tile_x = x / tile_width;
 545    uint32_t tile_y = y / tile_height;
 546
 547    /* The byte's address relative to the tile's base addres. */
 548    uint32_t byte_x = x % tile_width;
 549    uint32_t byte_y = y % tile_height;
 550
 551    uintptr_t u = tile_y * row_size
 552                + tile_x * tile_size
 553                + 512 * (byte_x / 8)
 554                +  64 * (byte_y / 8)
 555                +  32 * ((byte_y / 4) % 2)
 556                +  16 * ((byte_x / 4) % 2)
 557                +   8 * ((byte_y / 2) % 2)
 558                +   4 * ((byte_x / 2) % 2)
 559                +   2 * (byte_y % 2)
 560                +   1 * (byte_x % 2);
 561
 562    if (swizzled) {
 563       /* adjust for bit6 swizzling */
 564       if (((byte_x / 8) % 2) == 1) {
 565          if (((byte_y / 8) % 2) == 0) {
 566             u += 64;
 567          } else {
 568             u -= 64;
 569          }
 570       }
 571    }
 572
 573    return u;
 574 }
 575
 576 static void
 577 iris_unmap_s8(struct iris_transfer *map)
 578 {
 579    struct pipe_transfer *xfer = &map->base;
 580    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 581    struct isl_surf *surf = &res->surf;
 582    const bool has_swizzling = false;
 583
 584    if (xfer->usage & PIPE_TRANSFER_WRITE) {
 585       uint8_t *untiled_s8_map = map->ptr;
 586       uint8_t *tiled_s8_map =
 587          iris_bo_map(map->dbg, res->bo, xfer->usage | MAP_RAW);
 588
 589       struct pipe_box box = xfer->box;
 590
 591       for (int s = 0; s < box.depth; s++) {
 592          unsigned x0_el, y0_el;
 593          get_image_offset_el(surf, xfer->level, box.z, &x0_el, &y0_el);
 594
 595          for (uint32_t y = 0; y < box.height; y++) {
 596             for (uint32_t x = 0; x < box.width; x++) {
 597                ptrdiff_t offset = s8_offset(surf->row_pitch_B,
 598                                             x0_el + box.x + x,
 599                                             y0_el + box.y + y,
 600                                             has_swizzling);
 601                tiled_s8_map[offset] =
 602                   untiled_s8_map[s * xfer->layer_stride + y * xfer->stride + x];
 603             }
 604          }
 605
 606          box.z++;
 607       }
 608    }
 609
 610    free(map->buffer);
 611 }
 612
 613 static void
 614 iris_map_s8(struct iris_transfer *map)
 615 {
 616    struct pipe_transfer *xfer = &map->base;
 617    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 618    struct isl_surf *surf = &res->surf;
 619
 620    xfer->stride = surf->row_pitch_B;
 621    xfer->layer_stride = xfer->stride * xfer->box.height;
 622
 623    /* The tiling and detiling functions require that the linear buffer has
 624     * a 16-byte alignment (that is, its `x0` is 16-byte aligned).  Here we
 625     * over-allocate the linear buffer to get the proper alignment.
 626     */
 627    map->buffer = map->ptr = malloc(xfer->layer_stride * xfer->box.depth);
 628    assert(map->buffer);
 629
 630    const bool has_swizzling = false;
 631
 632    /* One of either READ_BIT or WRITE_BIT or both is set.  READ_BIT implies no
 633     * INVALIDATE_RANGE_BIT.  WRITE_BIT needs the original values read in unless
 634     * invalidate is set, since we'll be writing the whole rectangle from our
 635     * temporary buffer back out.
 636     */
 637    if (!(xfer->usage & PIPE_TRANSFER_DISCARD_RANGE)) {
 638       uint8_t *untiled_s8_map = map->ptr;
 639       uint8_t *tiled_s8_map =
 640          iris_bo_map(map->dbg, res->bo, xfer->usage | MAP_RAW);
 641
 642       struct pipe_box box = xfer->box;
 643
 644       for (int s = 0; s < box.depth; s++) {
 645          unsigned x0_el, y0_el;
 646          get_image_offset_el(surf, xfer->level, box.z, &x0_el, &y0_el);
 647
 648          for (uint32_t y = 0; y < box.height; y++) {
 649             for (uint32_t x = 0; x < box.width; x++) {
 650                ptrdiff_t offset = s8_offset(surf->row_pitch_B,
 651                                             x0_el + box.x + x,
 652                                             y0_el + box.y + y,
 653                                             has_swizzling);
 654                untiled_s8_map[s * xfer->layer_stride + y * xfer->stride + x] =
 655                   tiled_s8_map[offset];
 656             }
 657          }
 658
 659          box.z++;
 660       }
 661    }
 662
 663    map->unmap = iris_unmap_s8;
 664 }
 665
 666 /* Compute extent parameters for use with tiled_memcpy functions.
 667  * xs are in units of bytes and ys are in units of strides.
 668  */
 669 static inline void
 670 tile_extents(struct isl_surf *surf,
 671              const struct pipe_box *box,
 672              unsigned level,
 673              unsigned *x1_B, unsigned *x2_B,
 674              unsigned *y1_el, unsigned *y2_el)
 675 {
 676    const struct isl_format_layout *fmtl = isl_format_get_layout(surf->format);
 677    const unsigned cpp = fmtl->bpb / 8;
 678
 679    assert(box->x % fmtl->bw == 0);
 680    assert(box->y % fmtl->bh == 0);
 681
 682    unsigned x0_el, y0_el;
 683    get_image_offset_el(surf, level, box->z, &x0_el, &y0_el);
 684
 685    *x1_B = (box->x / fmtl->bw + x0_el) * cpp;
 686    *y1_el = box->y / fmtl->bh + y0_el;
 687    *x2_B = (DIV_ROUND_UP(box->x + box->width, fmtl->bw) + x0_el) * cpp;
 688    *y2_el = DIV_ROUND_UP(box->y + box->height, fmtl->bh) + y0_el;
 689 }
 690
 691 static void
 692 iris_unmap_tiled_memcpy(struct iris_transfer *map)
 693 {
 694    struct pipe_transfer *xfer = &map->base;
 695    struct pipe_box box = xfer->box;
 696    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 697    struct isl_surf *surf = &res->surf;
 698
 699    const bool has_swizzling = false;
 700
 701    if (xfer->usage & PIPE_TRANSFER_WRITE) {
 702       char *dst = iris_bo_map(map->dbg, res->bo, xfer->usage | MAP_RAW);
 703
 704       for (int s = 0; s < box.depth; s++) {
 705          unsigned x1, x2, y1, y2;
 706          tile_extents(surf, &box, xfer->level, &x1, &x2, &y1, &y2);
 707
 708          void *ptr = map->ptr + s * xfer->layer_stride;
 709
 710          isl_memcpy_linear_to_tiled(x1, x2, y1, y2, dst, ptr,
 711                                     surf->row_pitch_B, xfer->stride,
 712                                     has_swizzling, surf->tiling, ISL_MEMCPY);
 713          box.z++;
 714       }
 715    }
 716    os_free_aligned(map->buffer);
 717    map->buffer = map->ptr = NULL;
 718 }
 719
 720 static void
 721 iris_map_tiled_memcpy(struct iris_transfer *map)
 722 {
 723    struct pipe_transfer *xfer = &map->base;
 724    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 725    struct isl_surf *surf = &res->surf;
 726
 727    xfer->stride = ALIGN(surf->row_pitch_B, 16);
 728    xfer->layer_stride = xfer->stride * xfer->box.height;
 729
 730    unsigned x1, x2, y1, y2;
 731    tile_extents(surf, &xfer->box, xfer->level, &x1, &x2, &y1, &y2);
 732
 733    /* The tiling and detiling functions require that the linear buffer has
 734     * a 16-byte alignment (that is, its `x0` is 16-byte aligned).  Here we
 735     * over-allocate the linear buffer to get the proper alignment.
 736     */
 737    map->buffer =
 738       os_malloc_aligned(xfer->layer_stride * xfer->box.depth, 16);
 739    assert(map->buffer);
 740    map->ptr = (char *)map->buffer + (x1 & 0xf);
 741
 742    const bool has_swizzling = false;
 743
 744    // XXX: PIPE_TRANSFER_READ?
 745    if (!(xfer->usage & PIPE_TRANSFER_DISCARD_RANGE)) {
 746       char *src = iris_bo_map(map->dbg, res->bo, xfer->usage | MAP_RAW);
 747
 748       struct pipe_box box = xfer->box;
 749
 750       for (int s = 0; s < box.depth; s++) {
 751          unsigned x1, x2, y1, y2;
 752          tile_extents(surf, &box, xfer->level, &x1, &x2, &y1, &y2);
 753
 754          /* When transferring cubes, box.depth is counted in cubes, but
 755           * box.z is counted in faces.  We want to transfer only the
 756           * specified face, but for all array elements.  So, use 's'
 757           * (the zero-based slice count) rather than box.z.
 758           */
 759          void *ptr = map->ptr + s * xfer->layer_stride;
 760
 761          isl_memcpy_tiled_to_linear(x1, x2, y1, y2, ptr, src, xfer->stride,
 762                                     surf->row_pitch_B, has_swizzling,
 763                                     surf->tiling, ISL_MEMCPY);
 764          box.z++;
 765       }
 766    }
 767
 768    map->unmap = iris_unmap_tiled_memcpy;
 769 }
 770
 771 static void
 772 iris_map_direct(struct iris_transfer *map)
 773 {
 774    struct pipe_transfer *xfer = &map->base;
 775    struct pipe_box *box = &xfer->box;
 776    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 777
 778    void *ptr = iris_bo_map(map->dbg, res->bo, xfer->usage);
 779
 780    if (res->base.target == PIPE_BUFFER) {
 781       xfer->stride = 0;
 782       xfer->layer_stride = 0;
 783
 784       map->ptr = ptr + box->x;
 785    } else {
 786       struct isl_surf *surf = &res->surf;
 787       const struct isl_format_layout *fmtl =
 788          isl_format_get_layout(surf->format);
 789       const unsigned cpp = fmtl->bpb / 8;
 790       unsigned x0_el, y0_el;
 791
 792       get_image_offset_el(surf, xfer->level, box->z, &x0_el, &y0_el);
 793
 794       xfer->stride = isl_surf_get_row_pitch_B(surf);
 795       xfer->layer_stride = isl_surf_get_array_pitch(surf);
 796
 797       map->ptr = ptr + (y0_el + box->y) * xfer->stride + (x0_el + box->x) * cpp;
 798    }
 799 }
 800
 801 static void *
 802 iris_transfer_map(struct pipe_context *ctx,
 803                   struct pipe_resource *resource,
 804                   unsigned level,
 805                   enum pipe_transfer_usage usage,
 806                   const struct pipe_box *box,
 807                   struct pipe_transfer **ptransfer)
 808 {
 809    struct iris_context *ice = (struct iris_context *)ctx;
 810    struct iris_resource *res = (struct iris_resource *)resource;
 811    struct isl_surf *surf = &res->surf;
 812
 813    if (surf->tiling != ISL_TILING_LINEAR &&
 814        (usage & PIPE_TRANSFER_MAP_DIRECTLY))
 815       return NULL;
 816
 817    if (!(usage & PIPE_TRANSFER_UNSYNCHRONIZED)) {
 818       for (int i = 0; i < IRIS_BATCH_COUNT; i++) {
 819          if (iris_batch_references(&ice->batches[i], res->bo))
 820             iris_batch_flush(&ice->batches[i]);
 821       }
 822    }
 823
 824    if ((usage & PIPE_TRANSFER_DONTBLOCK) && iris_bo_busy(res->bo))
 825       return NULL;
 826
 827    struct iris_transfer *map = slab_alloc(&ice->transfer_pool);
 828    struct pipe_transfer *xfer = &map->base;
 829
 830    // PIPE_TRANSFER_DISCARD_WHOLE_RESOURCE
 831    // PIPE_TRANSFER_DISCARD_RANGE
 832
 833    if (!map)
 834       return NULL;
 835
 836    memset(map, 0, sizeof(*map));
 837    map->dbg = &ice->dbg;
 838
 839    pipe_resource_reference(&xfer->resource, resource);
 840    xfer->level = level;
 841    xfer->usage = usage;
 842    xfer->box = *box;
 843    *ptransfer = xfer;
 844
 845    xfer->usage &= (PIPE_TRANSFER_READ |
 846                    PIPE_TRANSFER_WRITE |
 847                    PIPE_TRANSFER_UNSYNCHRONIZED |
 848                    PIPE_TRANSFER_PERSISTENT |
 849                    PIPE_TRANSFER_COHERENT |
 850                    PIPE_TRANSFER_DISCARD_RANGE);
 851
 852    if (surf->tiling == ISL_TILING_W) {
 853       // XXX: just teach iris_map_tiled_memcpy about W tiling...
 854       iris_map_s8(map);
 855    } else if (surf->tiling != ISL_TILING_LINEAR) {
 856       iris_map_tiled_memcpy(map);
 857    } else {
 858       iris_map_direct(map);
 859    }
 860
 861    return map->ptr;
 862 }
 863
 864 static void
 865 iris_transfer_flush_region(struct pipe_context *ctx,
 866                            struct pipe_transfer *xfer,
 867                            const struct pipe_box *box)
 868 {
 869    struct iris_context *ice = (struct iris_context *)ctx;
 870    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 871
 872
 873    // XXX: don't emit flushes in both engines...? we may also need to flush
 874    // even if there isn't a draw yet - may still be stale data in caches...
 875    for (int i = 0; i < IRIS_BATCH_COUNT; i++) {
 876       if (ice->batches[i].contains_draw) {
 877          iris_batch_maybe_flush(&ice->batches[i], 24);
 878          iris_flush_and_dirty_for_history(ice, &ice->batches[i], res);
 879       }
 880    }
 881 }
 882
 883 static void
 884 iris_transfer_unmap(struct pipe_context *ctx, struct pipe_transfer *xfer)
 885 {
 886    struct iris_context *ice = (struct iris_context *)ctx;
 887    struct iris_transfer *map = (void *) xfer;
 888    struct iris_resource *res = (struct iris_resource *) xfer->resource;
 889
 890    if (map->unmap)
 891       map->unmap(map);
 892
 893    // XXX: don't emit flushes in both engines...?
 894    for (int i = 0; i < IRIS_BATCH_COUNT; i++) {
 895       if (ice->batches[i].contains_draw) {
 896          iris_batch_maybe_flush(&ice->batches[i], 24);
 897          iris_flush_and_dirty_for_history(ice, &ice->batches[i], res);
 898       }
 899    }
 900
 901    pipe_resource_reference(&xfer->resource, NULL);
 902    slab_free(&ice->transfer_pool, map);
 903 }
 904
 905 static void
 906 iris_flush_resource(struct pipe_context *ctx, struct pipe_resource *resource)
 907 {
 908 }
 909
 910 void
 911 iris_flush_and_dirty_for_history(struct iris_context *ice,
 912                                  struct iris_batch *batch,
 913                                  struct iris_resource *res)
 914 {
 915    if (res->base.target != PIPE_BUFFER)
 916       return;
 917
 918    unsigned flush = PIPE_CONTROL_CS_STALL;
 919    uint64_t dirty = 0ull;
 920
 921    if (res->bind_history & PIPE_BIND_CONSTANT_BUFFER) {
 922       flush |= PIPE_CONTROL_CONST_CACHE_INVALIDATE |
 923                PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 924       dirty |= IRIS_DIRTY_CONSTANTS_VS |
 925                IRIS_DIRTY_CONSTANTS_TCS |
 926                IRIS_DIRTY_CONSTANTS_TES |
 927                IRIS_DIRTY_CONSTANTS_GS |
 928                IRIS_DIRTY_CONSTANTS_FS |
 929                IRIS_DIRTY_CONSTANTS_CS |
 930                IRIS_ALL_DIRTY_BINDINGS;
 931    }
 932
 933    if (res->bind_history & PIPE_BIND_SAMPLER_VIEW)
 934       flush |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
 935
 936    if (res->bind_history & (PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_INDEX_BUFFER))
 937       flush |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
 938
 939    if (res->bind_history & (PIPE_BIND_SHADER_BUFFER | PIPE_BIND_SHADER_IMAGE))
 940       flush |= PIPE_CONTROL_DATA_CACHE_FLUSH;
 941
 942    iris_emit_pipe_control_flush(batch, flush);
 943
 944    ice->state.dirty |= dirty;
 945 }
 946
 947 static enum pipe_format
 948 iris_resource_get_internal_format(struct pipe_resource *p_res)
 949 {
 950    struct iris_resource *res = (void *) p_res;
 951    return res->internal_format;
 952 }
 953
 954 static const struct u_transfer_vtbl transfer_vtbl = {
 955    .resource_create       = iris_resource_create,
 956    .resource_destroy      = iris_resource_destroy,
 957    .transfer_map          = iris_transfer_map,
 958    .transfer_unmap        = iris_transfer_unmap,
 959    .transfer_flush_region = iris_transfer_flush_region,
 960    .get_internal_format   = iris_resource_get_internal_format,
 961    .set_stencil           = iris_resource_set_separate_stencil,
 962    .get_stencil           = iris_resource_get_separate_stencil,
 963 };
 964
 965 void
 966 iris_init_screen_resource_functions(struct pipe_screen *pscreen)
 967 {
 968    pscreen->resource_create_with_modifiers =
 969       iris_resource_create_with_modifiers;
 970    pscreen->resource_create = u_transfer_helper_resource_create;
 971    pscreen->resource_from_user_memory = iris_resource_from_user_memory;
 972    pscreen->resource_from_handle = iris_resource_from_handle;
 973    pscreen->resource_get_handle = iris_resource_get_handle;
 974    pscreen->resource_destroy = u_transfer_helper_resource_destroy;
 975    pscreen->transfer_helper =
 976       u_transfer_helper_create(&transfer_vtbl, true, true, false, true);
 977 }
 978
 979 void
 980 iris_init_resource_functions(struct pipe_context *ctx)
 981 {
 982    ctx->flush_resource = iris_flush_resource;
 983    ctx->transfer_map = u_transfer_helper_transfer_map;
 984    ctx->transfer_flush_region = u_transfer_helper_transfer_flush_region;
 985    ctx->transfer_unmap = u_transfer_helper_transfer_unmap;
 986    ctx->buffer_subdata = u_default_buffer_subdata;
 987    ctx->texture_subdata = u_default_texture_subdata;
 988 }