+ for (int i = 0; i < IRIS_BATCH_COUNT; i++)
+ busy |= iris_batch_references(&ice->batches[i], res->bo);
+
+ return busy;
+}
+
+static void
+iris_invalidate_resource(struct pipe_context *ctx,
+ struct pipe_resource *resource)
+{
+ struct iris_screen *screen = (void *) ctx->screen;
+ struct iris_context *ice = (void *) ctx;
+ struct iris_resource *res = (void *) resource;
+
+ if (resource->target != PIPE_BUFFER)
+ return;
+
+ if (!resource_is_busy(ice, res)) {
+ /* The resource is idle, so just mark that it contains no data and
+ * keep using the same underlying buffer object.
+ */
+ util_range_set_empty(&res->valid_buffer_range);
+ return;
+ }
+
+ /* Otherwise, try and replace the backing storage with a new BO. */
+
+ /* We can't reallocate memory we didn't allocate in the first place. */
+ if (res->bo->userptr)
+ return;
+
+ // XXX: We should support this.
+ if (res->bind_history & PIPE_BIND_STREAM_OUTPUT)
+ return;
+
+ struct iris_bo *old_bo = res->bo;
+ struct iris_bo *new_bo =
+ iris_bo_alloc(screen->bufmgr, res->bo->name, resource->width0,
+ iris_memzone_for_address(old_bo->gtt_offset));
+ if (!new_bo)
+ return;
+
+ /* Swap out the backing storage */
+ res->bo = new_bo;
+
+ /* Rebind the buffer, replacing any state referring to the old BO's
+ * address, and marking state dirty so it's reemitted.
+ */
+ ice->vtbl.rebind_buffer(ice, res, old_bo->gtt_offset);
+
+ util_range_set_empty(&res->valid_buffer_range);
+
+ iris_bo_unreference(old_bo);
+}
+
+static void
+iris_flush_staging_region(struct pipe_transfer *xfer,
+ const struct pipe_box *flush_box)
+{
+ if (!(xfer->usage & PIPE_TRANSFER_WRITE))
+ return;
+
+ struct iris_transfer *map = (void *) xfer;
+
+ struct pipe_box src_box = *flush_box;
+
+ /* Account for extra alignment padding in staging buffer */
+ if (xfer->resource->target == PIPE_BUFFER)
+ src_box.x += xfer->box.x % IRIS_MAP_BUFFER_ALIGNMENT;
+
+ struct pipe_box dst_box = (struct pipe_box) {
+ .x = xfer->box.x + flush_box->x,
+ .y = xfer->box.y + flush_box->y,
+ .z = xfer->box.z + flush_box->z,
+ .width = flush_box->width,
+ .height = flush_box->height,
+ .depth = flush_box->depth,
+ };
+
+ iris_copy_region(map->blorp, map->batch, xfer->resource, xfer->level,
+ dst_box.x, dst_box.y, dst_box.z, map->staging, 0,
+ &src_box);
+}
+
+static void
+iris_unmap_copy_region(struct iris_transfer *map)
+{
+ iris_resource_destroy(map->staging->screen, map->staging);
+
+ map->ptr = NULL;
+}
+
+static void
+iris_map_copy_region(struct iris_transfer *map)
+{
+ struct pipe_screen *pscreen = &map->batch->screen->base;
+ struct pipe_transfer *xfer = &map->base;
+ struct pipe_box *box = &xfer->box;
+ struct iris_resource *res = (void *) xfer->resource;
+
+ unsigned extra = xfer->resource->target == PIPE_BUFFER ?
+ box->x % IRIS_MAP_BUFFER_ALIGNMENT : 0;
+
+ struct pipe_resource templ = (struct pipe_resource) {
+ .usage = PIPE_USAGE_STAGING,
+ .width0 = box->width + extra,
+ .height0 = box->height,
+ .depth0 = 1,
+ .nr_samples = xfer->resource->nr_samples,
+ .nr_storage_samples = xfer->resource->nr_storage_samples,
+ .array_size = box->depth,
+ };
+
+ if (xfer->resource->target == PIPE_BUFFER)
+ templ.target = PIPE_BUFFER;
+ else if (templ.array_size > 1)
+ templ.target = PIPE_TEXTURE_2D_ARRAY;
+ else
+ templ.target = PIPE_TEXTURE_2D;
+
+ /* Depth, stencil, and ASTC can't be linear surfaces, so we can't use
+ * xfer->resource->format directly. Pick a bpb compatible format so
+ * resource creation will succeed; blorp_copy will override it anyway.
+ */
+ switch (util_format_get_blocksizebits(res->internal_format)) {
+ case 8: templ.format = PIPE_FORMAT_R8_UINT; break;
+ case 16: templ.format = PIPE_FORMAT_R8G8_UINT; break;
+ case 24: templ.format = PIPE_FORMAT_R8G8B8_UINT; break;
+ case 32: templ.format = PIPE_FORMAT_R8G8B8A8_UINT; break;
+ case 48: templ.format = PIPE_FORMAT_R16G16B16_UINT; break;
+ case 64: templ.format = PIPE_FORMAT_R16G16B16A16_UINT; break;
+ case 96: templ.format = PIPE_FORMAT_R32G32B32_UINT; break;
+ case 128: templ.format = PIPE_FORMAT_R32G32B32A32_UINT; break;
+ default: unreachable("Invalid bpb");
+ }
+
+ map->staging = iris_resource_create(pscreen, &templ);
+ assert(map->staging);
+
+ if (templ.target != PIPE_BUFFER) {
+ struct isl_surf *surf = &((struct iris_resource *) map->staging)->surf;
+ xfer->stride = isl_surf_get_row_pitch_B(surf);
+ xfer->layer_stride = isl_surf_get_array_pitch(surf);
+ }
+
+ if (!(xfer->usage & PIPE_TRANSFER_DISCARD_RANGE)) {
+ iris_copy_region(map->blorp, map->batch, map->staging, 0, extra, 0, 0,
+ xfer->resource, xfer->level, box);
+ /* Ensure writes to the staging BO land before we map it below. */
+ iris_emit_pipe_control_flush(map->batch,
+ PIPE_CONTROL_RENDER_TARGET_FLUSH |
+ PIPE_CONTROL_CS_STALL);
+ }
+
+ struct iris_bo *staging_bo = iris_resource_bo(map->staging);
+
+ if (iris_batch_references(map->batch, staging_bo))
+ iris_batch_flush(map->batch);
+
+ map->ptr =
+ iris_bo_map(map->dbg, staging_bo, xfer->usage & MAP_FLAGS) + extra;
+
+ map->unmap = iris_unmap_copy_region;
+}
+
+static void
+get_image_offset_el(const struct isl_surf *surf, unsigned level, unsigned z,
+ unsigned *out_x0_el, unsigned *out_y0_el)
+{
+ if (surf->dim == ISL_SURF_DIM_3D) {
+ isl_surf_get_image_offset_el(surf, level, 0, z, out_x0_el, out_y0_el);
+ } else {
+ isl_surf_get_image_offset_el(surf, level, z, 0, out_x0_el, out_y0_el);
+ }
+}
+
+/**
+ * Get pointer offset into stencil buffer.
+ *
+ * The stencil buffer is W tiled. Since the GTT is incapable of W fencing, we
+ * must decode the tile's layout in software.
+ *
+ * See
+ * - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.2.1 W-Major Tile
+ * Format.
+ * - PRM, 2011 Sandy Bridge, Volume 1, Part 2, Section 4.5.3 Tiling Algorithm
+ *
+ * Even though the returned offset is always positive, the return type is
+ * signed due to
+ * commit e8b1c6d6f55f5be3bef25084fdd8b6127517e137
+ * mesa: Fix return type of _mesa_get_format_bytes() (#37351)
+ */
+static intptr_t
+s8_offset(uint32_t stride, uint32_t x, uint32_t y, bool swizzled)
+{
+ uint32_t tile_size = 4096;
+ uint32_t tile_width = 64;
+ uint32_t tile_height = 64;
+ uint32_t row_size = 64 * stride / 2; /* Two rows are interleaved. */
+
+ uint32_t tile_x = x / tile_width;
+ uint32_t tile_y = y / tile_height;
+
+ /* The byte's address relative to the tile's base addres. */
+ uint32_t byte_x = x % tile_width;
+ uint32_t byte_y = y % tile_height;
+
+ uintptr_t u = tile_y * row_size
+ + tile_x * tile_size
+ + 512 * (byte_x / 8)
+ + 64 * (byte_y / 8)
+ + 32 * ((byte_y / 4) % 2)
+ + 16 * ((byte_x / 4) % 2)
+ + 8 * ((byte_y / 2) % 2)
+ + 4 * ((byte_x / 2) % 2)
+ + 2 * (byte_y % 2)
+ + 1 * (byte_x % 2);
+
+ if (swizzled) {
+ /* adjust for bit6 swizzling */
+ if (((byte_x / 8) % 2) == 1) {
+ if (((byte_y / 8) % 2) == 0) {
+ u += 64;
+ } else {
+ u -= 64;
+ }
+ }
+ }
+
+ return u;
+}
+
+static void
+iris_unmap_s8(struct iris_transfer *map)
+{
+ struct pipe_transfer *xfer = &map->base;
+ const struct pipe_box *box = &xfer->box;
+ struct iris_resource *res = (struct iris_resource *) xfer->resource;
+ struct isl_surf *surf = &res->surf;
+ const bool has_swizzling = false;
+
+ if (xfer->usage & PIPE_TRANSFER_WRITE) {
+ uint8_t *untiled_s8_map = map->ptr;
+ uint8_t *tiled_s8_map =
+ iris_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS);
+
+ for (int s = 0; s < box->depth; s++) {
+ unsigned x0_el, y0_el;
+ get_image_offset_el(surf, xfer->level, box->z + s, &x0_el, &y0_el);
+
+ for (uint32_t y = 0; y < box->height; y++) {
+ for (uint32_t x = 0; x < box->width; x++) {
+ ptrdiff_t offset = s8_offset(surf->row_pitch_B,
+ x0_el + box->x + x,
+ y0_el + box->y + y,
+ has_swizzling);
+ tiled_s8_map[offset] =
+ untiled_s8_map[s * xfer->layer_stride + y * xfer->stride + x];
+ }
+ }
+ }
+ }
+
+ free(map->buffer);
+}
+
+static void
+iris_map_s8(struct iris_transfer *map)
+{
+ struct pipe_transfer *xfer = &map->base;
+ const struct pipe_box *box = &xfer->box;
+ struct iris_resource *res = (struct iris_resource *) xfer->resource;
+ struct isl_surf *surf = &res->surf;
+
+ xfer->stride = surf->row_pitch_B;
+ xfer->layer_stride = xfer->stride * box->height;
+
+ /* The tiling and detiling functions require that the linear buffer has
+ * a 16-byte alignment (that is, its `x0` is 16-byte aligned). Here we
+ * over-allocate the linear buffer to get the proper alignment.
+ */
+ map->buffer = map->ptr = malloc(xfer->layer_stride * box->depth);
+ assert(map->buffer);
+
+ const bool has_swizzling = false;
+
+ /* One of either READ_BIT or WRITE_BIT or both is set. READ_BIT implies no
+ * INVALIDATE_RANGE_BIT. WRITE_BIT needs the original values read in unless
+ * invalidate is set, since we'll be writing the whole rectangle from our
+ * temporary buffer back out.
+ */
+ if (!(xfer->usage & PIPE_TRANSFER_DISCARD_RANGE)) {
+ uint8_t *untiled_s8_map = map->ptr;
+ uint8_t *tiled_s8_map =
+ iris_bo_map(map->dbg, res->bo, (xfer->usage | MAP_RAW) & MAP_FLAGS);
+
+ for (int s = 0; s < box->depth; s++) {
+ unsigned x0_el, y0_el;
+ get_image_offset_el(surf, xfer->level, box->z + s, &x0_el, &y0_el);
+
+ for (uint32_t y = 0; y < box->height; y++) {
+ for (uint32_t x = 0; x < box->width; x++) {
+ ptrdiff_t offset = s8_offset(surf->row_pitch_B,
+ x0_el + box->x + x,
+ y0_el + box->y + y,
+ has_swizzling);
+ untiled_s8_map[s * xfer->layer_stride + y * xfer->stride + x] =
+ tiled_s8_map[offset];
+ }
+ }
+ }
+ }
+
+ map->unmap = iris_unmap_s8;
+}