* clipping 4 * 2 = 8 > 5 in the src.
*/
+ if (*src_x0 == *src_x1 || *src_y0 == *src_y1
+ || *dst_x0 == *dst_x1 || *dst_y0 == *dst_y1)
+ return true;
+
float scale_x = (float) (*src_x1 - *src_x0) / (*dst_x1 - *dst_x0);
float scale_y = (float) (*src_y1 - *src_y0) / (*dst_y1 - *dst_y0);
clip_coordinates(mirror_y, src_y1, dst_y1, dst_y0,
clip_dst_y1, clip_dst_y0, scale_y, false);
- return false;
+ /* Check for invalid bounds
+ * Can't blit for 0-dimensions
+ */
+ return *src_x0 == *src_x1 || *src_y0 == *src_y1
+ || *dst_x0 == *dst_x1 || *dst_y0 == *dst_y1;
}
void
-iris_blorp_surf_for_resource(struct blorp_surf *surf,
+iris_blorp_surf_for_resource(struct iris_vtable *vtbl,
+ struct blorp_surf *surf,
struct pipe_resource *p_res,
enum isl_aux_usage aux_usage,
unsigned level,
{
struct iris_resource *res = (void *) p_res;
- if (aux_usage == ISL_AUX_USAGE_HIZ &&
+ assert(!iris_resource_unfinished_aux_import(res));
+
+ if (isl_aux_usage_has_hiz(aux_usage) &&
!iris_resource_level_has_hiz(res, level))
aux_usage = ISL_AUX_USAGE_NONE;
.surf = &res->surf,
.addr = (struct blorp_address) {
.buffer = res->bo,
- .offset = 0, // XXX: ???
+ .offset = res->offset,
.reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0,
- .mocs = I915_MOCS_CACHED, // XXX: BDW MOCS, PTE MOCS
+ .mocs = vtbl->mocs(res->bo),
},
.aux_usage = aux_usage,
};
.buffer = res->aux.bo,
.offset = res->aux.offset,
.reloc_flags = is_render_target ? EXEC_OBJECT_WRITE : 0,
- .mocs = I915_MOCS_CACHED,
+ .mocs = vtbl->mocs(res->bo),
+ };
+ surf->clear_color =
+ iris_resource_get_clear_color(res, NULL, NULL);
+ surf->clear_color_addr = (struct blorp_address) {
+ .buffer = res->aux.clear_color_bo,
+ .offset = res->aux.clear_color_offset,
+ .reloc_flags = 0,
+ .mocs = vtbl->mocs(res->aux.clear_color_bo),
};
}
// XXX: ASTC
}
+static bool
+is_astc(enum isl_format format)
+{
+ return format != ISL_FORMAT_UNSUPPORTED &&
+ isl_format_get_layout(format)->txc == ISL_TXC_ASTC;
+}
+
+static void
+tex_cache_flush_hack(struct iris_batch *batch,
+ enum isl_format view_format,
+ enum isl_format surf_format)
+{
+ const struct gen_device_info *devinfo = &batch->screen->devinfo;
+
+ /* The WaSamplerCacheFlushBetweenRedescribedSurfaceReads workaround says:
+ *
+ * "Currently Sampler assumes that a surface would not have two
+ * different format associate with it. It will not properly cache
+ * the different views in the MT cache, causing a data corruption."
+ *
+ * We may need to handle this for texture views in general someday, but
+ * for now we handle it here, as it hurts copies and blits particularly
+ * badly because they ofter reinterpret formats.
+ *
+ * If the BO hasn't been referenced yet this batch, we assume that the
+ * texture cache doesn't contain any relevant data nor need flushing.
+ *
+ * Icelake (Gen11+) claims to fix this issue, but seems to still have
+ * issues with ASTC formats.
+ */
+ bool need_flush = devinfo->gen >= 11 ?
+ is_astc(surf_format) != is_astc(view_format) :
+ view_format != surf_format;
+ if (!need_flush)
+ return;
+
+ const char *reason =
+ "workaround: WaSamplerCacheFlushBetweenRedescribedSurfaceReads";
+
+ iris_emit_pipe_control_flush(batch, reason, PIPE_CONTROL_CS_STALL);
+ iris_emit_pipe_control_flush(batch, reason,
+ PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
+}
+
/**
* The pipe->blit() driver hook.
*
enum isl_aux_usage src_aux_usage =
iris_resource_texture_aux_usage(ice, src_res, src_fmt.fmt, 0);
- if (src_aux_usage == ISL_AUX_USAGE_HIZ)
+ if (iris_resource_level_has_hiz(src_res, info->src.level))
src_aux_usage = ISL_AUX_USAGE_NONE;
bool src_clear_supported = src_aux_usage != ISL_AUX_USAGE_NONE &&
bool dst_clear_supported = dst_aux_usage != ISL_AUX_USAGE_NONE;
struct blorp_surf src_surf, dst_surf;
- iris_blorp_surf_for_resource(&src_surf, info->src.resource,
+ iris_blorp_surf_for_resource(&ice->vtbl, &src_surf, info->src.resource,
src_aux_usage, info->src.level, false);
- iris_blorp_surf_for_resource(&dst_surf, info->dst.resource,
+ iris_blorp_surf_for_resource(&ice->vtbl, &dst_surf, info->dst.resource,
dst_aux_usage, info->dst.level, true);
iris_resource_prepare_access(ice, batch, dst_res, info->dst.level, 1,
filter = BLORP_FILTER_NEAREST;
}
+ if (iris_batch_references(batch, src_res->bo))
+ tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format);
+
+ if (dst_res->base.target == PIPE_BUFFER)
+ util_range_add(&dst_res->base, &dst_res->valid_buffer_range, dst_x0, dst_x1);
+
struct blorp_batch blorp_batch;
blorp_batch_init(&ice->blorp, &blorp_batch, batch, blorp_flags);
unsigned main_mask;
- if (info->dst.format == PIPE_FORMAT_S8_UINT)
- main_mask = PIPE_MASK_S;
- else if (util_format_is_depth_or_stencil(info->dst.format))
+ if (util_format_is_depth_or_stencil(info->dst.format))
main_mask = PIPE_MASK_Z;
else
main_mask = PIPE_MASK_RGBA;
&src_surf, info->src.level, info->src.box.z + slice,
src_fmt.fmt, src_fmt.swizzle,
&dst_surf, info->dst.level, info->dst.box.z + slice,
- dst_fmt.fmt, ISL_SWIZZLE_IDENTITY,
+ dst_fmt.fmt, dst_fmt.swizzle,
src_x0, src_y0, src_x1, src_y1,
dst_x0, dst_y0, dst_x1, dst_y1,
filter, mirror_x, mirror_y);
}
if ((info->mask & PIPE_MASK_S) &&
- util_format_is_depth_and_stencil(info->dst.format) &&
+ util_format_has_stencil(util_format_description(info->dst.format)) &&
util_format_has_stencil(util_format_description(info->src.format))) {
struct iris_resource *src_res, *dst_res, *junk;
iris_get_depth_stencil_resources(info->src.resource, &junk, &src_res);
iris_get_depth_stencil_resources(info->dst.resource, &junk, &dst_res);
- iris_blorp_surf_for_resource(&src_surf, &src_res->base,
+ iris_blorp_surf_for_resource(&ice->vtbl, &src_surf, &src_res->base,
ISL_AUX_USAGE_NONE, info->src.level, false);
- iris_blorp_surf_for_resource(&dst_surf, &dst_res->base,
+ iris_blorp_surf_for_resource(&ice->vtbl, &dst_surf, &dst_res->base,
ISL_AUX_USAGE_NONE, info->dst.level, true);
for (int slice = 0; slice < info->dst.box.depth; slice++) {
blorp_batch_finish(&blorp_batch);
+ tex_cache_flush_hack(batch, src_fmt.fmt, src_res->surf.format);
+
iris_resource_finish_write(ice, dst_res, info->dst.level, info->dst.box.z,
info->dst.box.depth, dst_aux_usage);
iris_flush_and_dirty_for_history(ice, batch, (struct iris_resource *)
- info->dst.resource);
+ info->dst.resource,
+ PIPE_CONTROL_RENDER_TARGET_FLUSH,
+ "cache history: post-blit");
}
static void
}
/**
- * The pipe->resource_copy_region() driver hook.
+ * Perform a GPU-based raw memory copy between compatible view classes.
*
- * This implements ARB_copy_image semantics - a raw memory copy between
- * compatible view classes.
+ * Does not perform any flushing - the new data may still be left in the
+ * render cache, and old data may remain in other caches.
+ *
+ * Wraps blorp_copy() and blorp_buffer_copy().
*/
-static void
-iris_resource_copy_region(struct pipe_context *ctx,
- struct pipe_resource *dst,
- unsigned dst_level,
- unsigned dstx, unsigned dsty, unsigned dstz,
- struct pipe_resource *src,
- unsigned src_level,
- const struct pipe_box *src_box)
+void
+iris_copy_region(struct blorp_context *blorp,
+ struct iris_batch *batch,
+ struct pipe_resource *dst,
+ unsigned dst_level,
+ unsigned dstx, unsigned dsty, unsigned dstz,
+ struct pipe_resource *src,
+ unsigned src_level,
+ const struct pipe_box *src_box)
{
- struct iris_screen *screen = (void *) ctx->screen;
- const struct gen_device_info *devinfo = &screen->devinfo;
struct blorp_batch blorp_batch;
- struct iris_context *ice = (void *) ctx;
- struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
+ struct iris_context *ice = blorp->driver_ctx;
+ struct iris_screen *screen = (void *) ice->ctx.screen;
+ const struct gen_device_info *devinfo = &screen->devinfo;
struct iris_resource *src_res = (void *) src;
struct iris_resource *dst_res = (void *) dst;
get_copy_region_aux_settings(devinfo, dst_res, &dst_aux_usage,
&dst_clear_supported);
+ if (iris_batch_references(batch, src_res->bo))
+ tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format);
+
+ if (dst->target == PIPE_BUFFER)
+ util_range_add(&dst_res->base, &dst_res->valid_buffer_range, dstx, dstx + src_box->width);
+
if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
struct blorp_address src_addr = {
.buffer = iris_resource_bo(src), .offset = src_box->x,
};
struct blorp_address dst_addr = {
.buffer = iris_resource_bo(dst), .offset = dstx,
+ .reloc_flags = EXEC_OBJECT_WRITE,
};
iris_batch_maybe_flush(batch, 1500);
blorp_batch_init(&ice->blorp, &blorp_batch, batch, 0);
blorp_buffer_copy(&blorp_batch, src_addr, dst_addr, src_box->width);
blorp_batch_finish(&blorp_batch);
-
- iris_flush_and_dirty_for_history(ice, batch,
- (struct iris_resource *) dst);
} else {
// XXX: what about one surface being a buffer and not the other?
struct blorp_surf src_surf, dst_surf;
- iris_blorp_surf_for_resource(&src_surf, src, src_aux_usage,
+ iris_blorp_surf_for_resource(&ice->vtbl, &src_surf, src, src_aux_usage,
src_level, false);
- iris_blorp_surf_for_resource(&dst_surf, dst, dst_aux_usage,
+ iris_blorp_surf_for_resource(&ice->vtbl, &dst_surf, dst, dst_aux_usage,
dst_level, true);
iris_resource_prepare_access(ice, batch, src_res, src_level, 1,
iris_resource_finish_write(ice, dst_res, dst_level, dstz,
src_box->depth, dst_aux_usage);
+ }
+
+ tex_cache_flush_hack(batch, ISL_FORMAT_UNSUPPORTED, src_res->surf.format);
+}
+static struct iris_batch *
+get_preferred_batch(struct iris_context *ice, struct iris_bo *bo)
+{
+ /* If the compute batch is already using this buffer, we'd prefer to
+ * continue queueing in the compute batch.
+ */
+ if (iris_batch_references(&ice->batches[IRIS_BATCH_COMPUTE], bo))
+ return &ice->batches[IRIS_BATCH_COMPUTE];
+
+ /* Otherwise default to the render batch. */
+ return &ice->batches[IRIS_BATCH_RENDER];
+}
+
+
+/**
+ * The pipe->resource_copy_region() driver hook.
+ *
+ * This implements ARB_copy_image semantics - a raw memory copy between
+ * compatible view classes.
+ */
+static void
+iris_resource_copy_region(struct pipe_context *ctx,
+ struct pipe_resource *dst,
+ unsigned dst_level,
+ unsigned dstx, unsigned dsty, unsigned dstz,
+ struct pipe_resource *src,
+ unsigned src_level,
+ const struct pipe_box *src_box)
+{
+ struct iris_context *ice = (void *) ctx;
+ struct iris_batch *batch = &ice->batches[IRIS_BATCH_RENDER];
+
+ /* Use MI_COPY_MEM_MEM for tiny (<= 16 byte, % 4) buffer copies. */
+ if (src->target == PIPE_BUFFER && dst->target == PIPE_BUFFER &&
+ (src_box->width % 4 == 0) && src_box->width <= 16) {
+ struct iris_bo *dst_bo = iris_resource_bo(dst);
+ batch = get_preferred_batch(ice, dst_bo);
+ iris_batch_maybe_flush(batch, 24 + 5 * (src_box->width / 4));
+ iris_emit_pipe_control_flush(batch,
+ "stall for MI_COPY_MEM_MEM copy_region",
+ PIPE_CONTROL_CS_STALL);
+ ice->vtbl.copy_mem_mem(batch, dst_bo, dstx, iris_resource_bo(src),
+ src_box->x, src_box->width);
+ return;
}
+
+ iris_copy_region(&ice->blorp, batch, dst, dst_level, dstx, dsty, dstz,
+ src, src_level, src_box);
+
+ if (util_format_is_depth_and_stencil(dst->format) &&
+ util_format_has_stencil(util_format_description(src->format))) {
+ struct iris_resource *junk, *s_src_res, *s_dst_res;
+ iris_get_depth_stencil_resources(src, &junk, &s_src_res);
+ iris_get_depth_stencil_resources(dst, &junk, &s_dst_res);
+
+ iris_copy_region(&ice->blorp, batch, &s_dst_res->base, dst_level, dstx,
+ dsty, dstz, &s_src_res->base, src_level, src_box);
+ }
+
+ iris_flush_and_dirty_for_history(ice, batch, (struct iris_resource *) dst,
+ PIPE_CONTROL_RENDER_TARGET_FLUSH,
+ "cache history: post copy_region");
}
void
projects / mesa.git / blobdiff