2 * Copyright © 2017 Intel Corporation
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included
12 * in all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
19 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
20 * DEALINGS IN THE SOFTWARE.
24 #include "pipe/p_defines.h"
25 #include "pipe/p_state.h"
26 #include "pipe/p_context.h"
27 #include "pipe/p_screen.h"
28 #include "util/format/u_format.h"
29 #include "util/u_inlines.h"
30 #include "util/ralloc.h"
31 #include "intel/blorp/blorp.h"
32 #include "iris_context.h"
33 #include "iris_resource.h"
34 #include "iris_screen.h"
37 * Helper function for handling mirror image blits.
39 * If coord0 > coord1, swap them and return "true" (mirrored).
42 apply_mirror(float *coord0
, float *coord1
)
44 if (*coord0
> *coord1
) {
54 * Compute the number of pixels to clip for each side of a rect
56 * \param x0 The rect's left coordinate
57 * \param y0 The rect's bottom coordinate
58 * \param x1 The rect's right coordinate
59 * \param y1 The rect's top coordinate
60 * \param min_x The clipping region's left coordinate
61 * \param min_y The clipping region's bottom coordinate
62 * \param max_x The clipping region's right coordinate
63 * \param max_y The clipping region's top coordinate
64 * \param clipped_x0 The number of pixels to clip from the left side
65 * \param clipped_y0 The number of pixels to clip from the bottom side
66 * \param clipped_x1 The number of pixels to clip from the right side
67 * \param clipped_y1 The number of pixels to clip from the top side
69 * \return false if we clip everything away, true otherwise
72 compute_pixels_clipped(float x0
, float y0
, float x1
, float y1
,
73 float min_x
, float min_y
, float max_x
, float max_y
,
74 float *clipped_x0
, float *clipped_y0
,
75 float *clipped_x1
, float *clipped_y1
)
77 /* If we are going to clip everything away, stop. */
78 if (!(min_x
<= max_x
&&
90 *clipped_x0
= min_x
- x0
;
94 *clipped_x1
= x1
- max_x
;
99 *clipped_y0
= min_y
- y0
;
103 *clipped_y1
= y1
- max_y
;
111 * Clips a coordinate (left, right, top or bottom) for the src or dst rect
112 * (whichever requires the largest clip) and adjusts the coordinate
113 * for the other rect accordingly.
115 * \param mirror true if mirroring is required
116 * \param src the source rect coordinate (for example src_x0)
117 * \param dst0 the dst rect coordinate (for example dst_x0)
118 * \param dst1 the opposite dst rect coordinate (for example dst_x1)
119 * \param clipped_dst0 number of pixels to clip from the dst coordinate
120 * \param clipped_dst1 number of pixels to clip from the opposite dst coordinate
121 * \param scale the src vs dst scale involved for that coordinate
122 * \param is_left_or_bottom true if we are clipping the left or bottom sides
126 clip_coordinates(bool mirror
,
127 float *src
, float *dst0
, float *dst1
,
131 bool is_left_or_bottom
)
133 /* When clipping we need to add or subtract pixels from the original
134 * coordinates depending on whether we are acting on the left/bottom
135 * or right/top sides of the rect respectively. We assume we have to
136 * add them in the code below, and multiply by -1 when we should
139 int mult
= is_left_or_bottom
? 1 : -1;
142 *dst0
+= clipped_dst0
* mult
;
143 *src
+= clipped_dst0
* scale
* mult
;
145 *dst1
-= clipped_dst1
* mult
;
146 *src
+= clipped_dst1
* scale
* mult
;
151 * Apply a scissor rectangle to blit coordinates.
153 * Returns true if the blit was entirely scissored away.
156 apply_blit_scissor(const struct pipe_scissor_state
*scissor
,
157 float *src_x0
, float *src_y0
,
158 float *src_x1
, float *src_y1
,
159 float *dst_x0
, float *dst_y0
,
160 float *dst_x1
, float *dst_y1
,
161 bool mirror_x
, bool mirror_y
)
163 float clip_dst_x0
, clip_dst_x1
, clip_dst_y0
, clip_dst_y1
;
165 /* Compute number of pixels to scissor away. */
166 if (!compute_pixels_clipped(*dst_x0
, *dst_y0
, *dst_x1
, *dst_y1
,
167 scissor
->minx
, scissor
->miny
,
168 scissor
->maxx
, scissor
->maxy
,
169 &clip_dst_x0
, &clip_dst_y0
,
170 &clip_dst_x1
, &clip_dst_y1
))
173 // XXX: comments assume source clipping, which we don't do
175 /* When clipping any of the two rects we need to adjust the coordinates
176 * in the other rect considering the scaling factor involved. To obtain
177 * the best precision we want to make sure that we only clip once per
178 * side to avoid accumulating errors due to the scaling adjustment.
180 * For example, if src_x0 and dst_x0 need both to be clipped we want to
181 * avoid the situation where we clip src_x0 first, then adjust dst_x0
182 * accordingly but then we realize that the resulting dst_x0 still needs
183 * to be clipped, so we clip dst_x0 and adjust src_x0 again. Because we are
184 * applying scaling factors to adjust the coordinates in each clipping
185 * pass we lose some precision and that can affect the results of the
186 * blorp blit operation slightly. What we want to do here is detect the
187 * rect that we should clip first for each side so that when we adjust
188 * the other rect we ensure the resulting coordinate does not need to be
191 * The code below implements this by comparing the number of pixels that
192 * we need to clip for each side of both rects considering the scales
193 * involved. For example, clip_src_x0 represents the number of pixels
194 * to be clipped for the src rect's left side, so if clip_src_x0 = 5,
195 * clip_dst_x0 = 4 and scale_x = 2 it means that we are clipping more
196 * from the dst rect so we should clip dst_x0 only and adjust src_x0.
197 * This is because clipping 4 pixels in the dst is equivalent to
198 * clipping 4 * 2 = 8 > 5 in the src.
201 if (*src_x0
== *src_x1
|| *src_y0
== *src_y1
202 || *dst_x0
== *dst_x1
|| *dst_y0
== *dst_y1
)
205 float scale_x
= (float) (*src_x1
- *src_x0
) / (*dst_x1
- *dst_x0
);
206 float scale_y
= (float) (*src_y1
- *src_y0
) / (*dst_y1
- *dst_y0
);
209 clip_coordinates(mirror_x
, src_x0
, dst_x0
, dst_x1
,
210 clip_dst_x0
, clip_dst_x1
, scale_x
, true);
212 /* Clip right side */
213 clip_coordinates(mirror_x
, src_x1
, dst_x1
, dst_x0
,
214 clip_dst_x1
, clip_dst_x0
, scale_x
, false);
216 /* Clip bottom side */
217 clip_coordinates(mirror_y
, src_y0
, dst_y0
, dst_y1
,
218 clip_dst_y0
, clip_dst_y1
, scale_y
, true);
221 clip_coordinates(mirror_y
, src_y1
, dst_y1
, dst_y0
,
222 clip_dst_y1
, clip_dst_y0
, scale_y
, false);
224 /* Check for invalid bounds
225 * Can't blit for 0-dimensions
227 return *src_x0
== *src_x1
|| *src_y0
== *src_y1
228 || *dst_x0
== *dst_x1
|| *dst_y0
== *dst_y1
;
232 iris_blorp_surf_for_resource(struct isl_device
*isl_dev
,
233 struct blorp_surf
*surf
,
234 struct pipe_resource
*p_res
,
235 enum isl_aux_usage aux_usage
,
237 bool is_render_target
)
239 struct iris_resource
*res
= (void *) p_res
;
241 assert(!iris_resource_unfinished_aux_import(res
));
243 if (isl_aux_usage_has_hiz(aux_usage
) &&
244 !iris_resource_level_has_hiz(res
, level
))
245 aux_usage
= ISL_AUX_USAGE_NONE
;
247 *surf
= (struct blorp_surf
) {
249 .addr
= (struct blorp_address
) {
251 .offset
= res
->offset
,
252 .reloc_flags
= is_render_target
? EXEC_OBJECT_WRITE
: 0,
253 .mocs
= iris_mocs(res
->bo
, isl_dev
),
255 .aux_usage
= aux_usage
,
258 if (aux_usage
!= ISL_AUX_USAGE_NONE
) {
259 surf
->aux_surf
= &res
->aux
.surf
;
260 surf
->aux_addr
= (struct blorp_address
) {
261 .buffer
= res
->aux
.bo
,
262 .offset
= res
->aux
.offset
,
263 .reloc_flags
= is_render_target
? EXEC_OBJECT_WRITE
: 0,
264 .mocs
= iris_mocs(res
->bo
, isl_dev
),
267 iris_resource_get_clear_color(res
, NULL
, NULL
);
268 surf
->clear_color_addr
= (struct blorp_address
) {
269 .buffer
= res
->aux
.clear_color_bo
,
270 .offset
= res
->aux
.clear_color_offset
,
272 .mocs
= iris_mocs(res
->aux
.clear_color_bo
, isl_dev
),
278 is_astc(enum isl_format format
)
280 return format
!= ISL_FORMAT_UNSUPPORTED
&&
281 isl_format_get_layout(format
)->txc
== ISL_TXC_ASTC
;
285 tex_cache_flush_hack(struct iris_batch
*batch
,
286 enum isl_format view_format
,
287 enum isl_format surf_format
)
289 const struct gen_device_info
*devinfo
= &batch
->screen
->devinfo
;
291 /* The WaSamplerCacheFlushBetweenRedescribedSurfaceReads workaround says:
293 * "Currently Sampler assumes that a surface would not have two
294 * different format associate with it. It will not properly cache
295 * the different views in the MT cache, causing a data corruption."
297 * We may need to handle this for texture views in general someday, but
298 * for now we handle it here, as it hurts copies and blits particularly
299 * badly because they ofter reinterpret formats.
301 * If the BO hasn't been referenced yet this batch, we assume that the
302 * texture cache doesn't contain any relevant data nor need flushing.
304 * Icelake (Gen11+) claims to fix this issue, but seems to still have
305 * issues with ASTC formats.
307 bool need_flush
= devinfo
->gen
>= 11 ?
308 is_astc(surf_format
) != is_astc(view_format
) :
309 view_format
!= surf_format
;
314 "workaround: WaSamplerCacheFlushBetweenRedescribedSurfaceReads";
316 iris_emit_pipe_control_flush(batch
, reason
, PIPE_CONTROL_CS_STALL
);
317 iris_emit_pipe_control_flush(batch
, reason
,
318 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE
);
321 static enum isl_aux_usage
322 iris_resource_blorp_write_aux_usage(struct iris_context
*ice
,
323 struct iris_resource
*res
,
324 enum isl_format render_format
)
326 if (res
->surf
.usage
& (ISL_SURF_USAGE_DEPTH_BIT
|
327 ISL_SURF_USAGE_STENCIL_BIT
)) {
328 assert(render_format
== res
->surf
.format
);
329 return res
->aux
.usage
;
331 return iris_resource_render_aux_usage(ice
, res
, render_format
, false);
336 * The pipe->blit() driver hook.
338 * This performs a blit between two surfaces, which copies data but may
339 * also perform format conversion, scaling, flipping, and so on.
342 iris_blit(struct pipe_context
*ctx
, const struct pipe_blit_info
*info
)
344 struct iris_context
*ice
= (void *) ctx
;
345 struct iris_screen
*screen
= (struct iris_screen
*)ctx
->screen
;
346 const struct gen_device_info
*devinfo
= &screen
->devinfo
;
347 struct iris_batch
*batch
= &ice
->batches
[IRIS_BATCH_RENDER
];
348 enum blorp_batch_flags blorp_flags
= 0;
349 struct iris_resource
*src_res
= (void *) info
->src
.resource
;
350 struct iris_resource
*dst_res
= (void *) info
->dst
.resource
;
352 /* We don't support color masking. */
353 assert((info
->mask
& PIPE_MASK_RGBA
) == PIPE_MASK_RGBA
||
354 (info
->mask
& PIPE_MASK_RGBA
) == 0);
356 if (info
->render_condition_enable
) {
357 if (ice
->state
.predicate
== IRIS_PREDICATE_STATE_DONT_RENDER
)
360 if (ice
->state
.predicate
== IRIS_PREDICATE_STATE_USE_BIT
)
361 blorp_flags
|= BLORP_BATCH_PREDICATE_ENABLE
;
364 if (iris_resource_unfinished_aux_import(src_res
))
365 iris_resource_finish_aux_import(ctx
->screen
, src_res
);
366 if (iris_resource_unfinished_aux_import(dst_res
))
367 iris_resource_finish_aux_import(ctx
->screen
, dst_res
);
369 struct iris_format_info src_fmt
=
370 iris_format_for_usage(devinfo
, info
->src
.format
,
371 ISL_SURF_USAGE_TEXTURE_BIT
);
372 enum isl_aux_usage src_aux_usage
=
373 iris_resource_texture_aux_usage(ice
, src_res
, src_fmt
.fmt
);
375 if (iris_resource_level_has_hiz(src_res
, info
->src
.level
))
376 assert(src_res
->surf
.format
== src_fmt
.fmt
);
378 bool src_clear_supported
= isl_aux_usage_has_fast_clears(src_aux_usage
) &&
379 src_res
->surf
.format
== src_fmt
.fmt
;
381 iris_resource_prepare_access(ice
, src_res
, info
->src
.level
, 1,
382 info
->src
.box
.z
, info
->src
.box
.depth
,
383 src_aux_usage
, src_clear_supported
);
384 iris_emit_buffer_barrier_for(batch
, src_res
->bo
, IRIS_DOMAIN_OTHER_READ
);
386 struct iris_format_info dst_fmt
=
387 iris_format_for_usage(devinfo
, info
->dst
.format
,
388 ISL_SURF_USAGE_RENDER_TARGET_BIT
);
389 enum isl_aux_usage dst_aux_usage
=
390 iris_resource_blorp_write_aux_usage(ice
, dst_res
, dst_fmt
.fmt
);
391 bool dst_clear_supported
= isl_aux_usage_has_fast_clears(dst_aux_usage
);
393 struct blorp_surf src_surf
, dst_surf
;
394 iris_blorp_surf_for_resource(&screen
->isl_dev
, &src_surf
,
395 info
->src
.resource
, src_aux_usage
,
396 info
->src
.level
, false);
397 iris_blorp_surf_for_resource(&screen
->isl_dev
, &dst_surf
,
398 info
->dst
.resource
, dst_aux_usage
,
399 info
->dst
.level
, true);
401 iris_resource_prepare_access(ice
, dst_res
, info
->dst
.level
, 1,
402 info
->dst
.box
.z
, info
->dst
.box
.depth
,
403 dst_aux_usage
, dst_clear_supported
);
404 iris_emit_buffer_barrier_for(batch
, dst_res
->bo
, IRIS_DOMAIN_RENDER_WRITE
);
406 float src_x0
= info
->src
.box
.x
;
407 float src_x1
= info
->src
.box
.x
+ info
->src
.box
.width
;
408 float src_y0
= info
->src
.box
.y
;
409 float src_y1
= info
->src
.box
.y
+ info
->src
.box
.height
;
410 float dst_x0
= info
->dst
.box
.x
;
411 float dst_x1
= info
->dst
.box
.x
+ info
->dst
.box
.width
;
412 float dst_y0
= info
->dst
.box
.y
;
413 float dst_y1
= info
->dst
.box
.y
+ info
->dst
.box
.height
;
414 bool mirror_x
= apply_mirror(&src_x0
, &src_x1
);
415 bool mirror_y
= apply_mirror(&src_y0
, &src_y1
);
416 enum blorp_filter filter
;
418 if (info
->scissor_enable
) {
419 bool noop
= apply_blit_scissor(&info
->scissor
,
420 &src_x0
, &src_y0
, &src_x1
, &src_y1
,
421 &dst_x0
, &dst_y0
, &dst_x1
, &dst_y1
,
427 if (abs(info
->dst
.box
.width
) == abs(info
->src
.box
.width
) &&
428 abs(info
->dst
.box
.height
) == abs(info
->src
.box
.height
)) {
429 if (src_surf
.surf
->samples
> 1 && dst_surf
.surf
->samples
<= 1) {
430 /* The OpenGL ES 3.2 specification, section 16.2.1, says:
432 * "If the read framebuffer is multisampled (its effective
433 * value of SAMPLE_BUFFERS is one) and the draw framebuffer
434 * is not (its value of SAMPLE_BUFFERS is zero), the samples
435 * corresponding to each pixel location in the source are
436 * converted to a single sample before being written to the
437 * destination. The filter parameter is ignored. If the
438 * source formats are integer types or stencil values, a
439 * single sample’s value is selected for each pixel. If the
440 * source formats are floating-point or normalized types,
441 * the sample values for each pixel are resolved in an
442 * implementation-dependent manner. If the source formats
443 * are depth values, sample values are resolved in an
444 * implementation-dependent manner where the result will be
445 * between the minimum and maximum depth values in the pixel."
447 * When selecting a single sample, we always choose sample 0.
449 if (util_format_is_depth_or_stencil(info
->src
.format
) ||
450 util_format_is_pure_integer(info
->src
.format
)) {
451 filter
= BLORP_FILTER_SAMPLE_0
;
453 filter
= BLORP_FILTER_AVERAGE
;
456 /* The OpenGL 4.6 specification, section 18.3.1, says:
458 * "If the source and destination dimensions are identical,
459 * no filtering is applied."
461 * Using BLORP_FILTER_NONE will also handle the upsample case by
462 * replicating the one value in the source to all values in the
465 filter
= BLORP_FILTER_NONE
;
467 } else if (info
->filter
== PIPE_TEX_FILTER_LINEAR
) {
468 filter
= BLORP_FILTER_BILINEAR
;
470 filter
= BLORP_FILTER_NEAREST
;
473 if (iris_batch_references(batch
, src_res
->bo
))
474 tex_cache_flush_hack(batch
, src_fmt
.fmt
, src_res
->surf
.format
);
476 if (dst_res
->base
.target
== PIPE_BUFFER
)
477 util_range_add(&dst_res
->base
, &dst_res
->valid_buffer_range
, dst_x0
, dst_x1
);
479 struct blorp_batch blorp_batch
;
480 blorp_batch_init(&ice
->blorp
, &blorp_batch
, batch
, blorp_flags
);
483 if (util_format_is_depth_or_stencil(info
->dst
.format
))
484 main_mask
= PIPE_MASK_Z
;
486 main_mask
= PIPE_MASK_RGBA
;
488 if (info
->mask
& main_mask
) {
489 for (int slice
= 0; slice
< info
->dst
.box
.depth
; slice
++) {
490 iris_batch_maybe_flush(batch
, 1500);
491 iris_batch_sync_region_start(batch
);
493 blorp_blit(&blorp_batch
,
494 &src_surf
, info
->src
.level
, info
->src
.box
.z
+ slice
,
495 src_fmt
.fmt
, src_fmt
.swizzle
,
496 &dst_surf
, info
->dst
.level
, info
->dst
.box
.z
+ slice
,
497 dst_fmt
.fmt
, dst_fmt
.swizzle
,
498 src_x0
, src_y0
, src_x1
, src_y1
,
499 dst_x0
, dst_y0
, dst_x1
, dst_y1
,
500 filter
, mirror_x
, mirror_y
);
502 iris_batch_sync_region_end(batch
);
506 struct iris_resource
*stc_dst
= NULL
;
507 enum isl_aux_usage stc_dst_aux_usage
= ISL_AUX_USAGE_NONE
;
508 if ((info
->mask
& PIPE_MASK_S
) &&
509 util_format_has_stencil(util_format_description(info
->dst
.format
)) &&
510 util_format_has_stencil(util_format_description(info
->src
.format
))) {
511 struct iris_resource
*src_res
, *junk
;
512 struct blorp_surf src_surf
, dst_surf
;
513 iris_get_depth_stencil_resources(info
->src
.resource
, &junk
, &src_res
);
514 iris_get_depth_stencil_resources(info
->dst
.resource
, &junk
, &stc_dst
);
516 struct iris_format_info src_fmt
=
517 iris_format_for_usage(devinfo
, src_res
->base
.format
,
518 ISL_SURF_USAGE_TEXTURE_BIT
);
519 enum isl_aux_usage stc_src_aux_usage
=
520 iris_resource_texture_aux_usage(ice
, src_res
, src_fmt
.fmt
);
522 struct iris_format_info dst_fmt
=
523 iris_format_for_usage(devinfo
, stc_dst
->base
.format
,
524 ISL_SURF_USAGE_RENDER_TARGET_BIT
);
526 iris_resource_blorp_write_aux_usage(ice
, stc_dst
, dst_fmt
.fmt
);
528 iris_resource_prepare_access(ice
, src_res
, info
->src
.level
, 1,
529 info
->src
.box
.z
, info
->src
.box
.depth
,
530 stc_src_aux_usage
, false);
531 iris_emit_buffer_barrier_for(batch
, src_res
->bo
, IRIS_DOMAIN_OTHER_READ
);
532 iris_resource_prepare_access(ice
, stc_dst
, info
->dst
.level
, 1,
533 info
->dst
.box
.z
, info
->dst
.box
.depth
,
534 stc_dst_aux_usage
, false);
535 iris_emit_buffer_barrier_for(batch
, stc_dst
->bo
, IRIS_DOMAIN_RENDER_WRITE
);
536 iris_blorp_surf_for_resource(&screen
->isl_dev
, &src_surf
,
537 &src_res
->base
, stc_src_aux_usage
,
538 info
->src
.level
, false);
539 iris_blorp_surf_for_resource(&screen
->isl_dev
, &dst_surf
,
540 &stc_dst
->base
, stc_dst_aux_usage
,
541 info
->dst
.level
, true);
543 for (int slice
= 0; slice
< info
->dst
.box
.depth
; slice
++) {
544 iris_batch_maybe_flush(batch
, 1500);
545 iris_batch_sync_region_start(batch
);
547 blorp_blit(&blorp_batch
,
548 &src_surf
, info
->src
.level
, info
->src
.box
.z
+ slice
,
549 ISL_FORMAT_R8_UINT
, ISL_SWIZZLE_IDENTITY
,
550 &dst_surf
, info
->dst
.level
, info
->dst
.box
.z
+ slice
,
551 ISL_FORMAT_R8_UINT
, ISL_SWIZZLE_IDENTITY
,
552 src_x0
, src_y0
, src_x1
, src_y1
,
553 dst_x0
, dst_y0
, dst_x1
, dst_y1
,
554 filter
, mirror_x
, mirror_y
);
556 iris_batch_sync_region_end(batch
);
560 blorp_batch_finish(&blorp_batch
);
562 tex_cache_flush_hack(batch
, src_fmt
.fmt
, src_res
->surf
.format
);
564 if (info
->mask
& main_mask
) {
565 iris_resource_finish_write(ice
, dst_res
, info
->dst
.level
, info
->dst
.box
.z
,
566 info
->dst
.box
.depth
, dst_aux_usage
);
570 iris_resource_finish_write(ice
, stc_dst
, info
->dst
.level
, info
->dst
.box
.z
,
571 info
->dst
.box
.depth
, stc_dst_aux_usage
);
574 iris_flush_and_dirty_for_history(ice
, batch
, (struct iris_resource
*)
576 PIPE_CONTROL_RENDER_TARGET_FLUSH
,
577 "cache history: post-blit");
581 get_copy_region_aux_settings(struct iris_context
*ice
,
582 struct iris_resource
*res
,
583 enum isl_aux_usage
*out_aux_usage
,
584 bool *out_clear_supported
,
585 bool is_render_target
)
587 struct iris_screen
*screen
= (void *) ice
->ctx
.screen
;
588 const struct gen_device_info
*devinfo
= &screen
->devinfo
;
590 switch (res
->aux
.usage
) {
591 case ISL_AUX_USAGE_HIZ
:
592 case ISL_AUX_USAGE_HIZ_CCS
:
593 case ISL_AUX_USAGE_HIZ_CCS_WT
:
594 if (is_render_target
) {
595 *out_aux_usage
= res
->aux
.usage
;
597 *out_aux_usage
= iris_resource_texture_aux_usage(ice
, res
,
600 *out_clear_supported
= (*out_aux_usage
!= ISL_AUX_USAGE_NONE
);
602 case ISL_AUX_USAGE_MCS
:
603 case ISL_AUX_USAGE_MCS_CCS
:
604 case ISL_AUX_USAGE_CCS_E
:
605 case ISL_AUX_USAGE_GEN12_CCS_E
:
606 *out_aux_usage
= res
->aux
.usage
;
607 /* Prior to Gen9, fast-clear only supported 0/1 clear colors. Since
608 * we're going to re-interpret the format as an integer format possibly
609 * with a different number of components, we can't handle clear colors
612 *out_clear_supported
= devinfo
->gen
>= 9;
614 case ISL_AUX_USAGE_STC_CCS
:
615 *out_aux_usage
= res
->aux
.usage
;
616 *out_clear_supported
= false;
619 *out_aux_usage
= ISL_AUX_USAGE_NONE
;
620 *out_clear_supported
= false;
626 * Perform a GPU-based raw memory copy between compatible view classes.
628 * Does not perform any flushing - the new data may still be left in the
629 * render cache, and old data may remain in other caches.
631 * Wraps blorp_copy() and blorp_buffer_copy().
634 iris_copy_region(struct blorp_context
*blorp
,
635 struct iris_batch
*batch
,
636 struct pipe_resource
*dst
,
638 unsigned dstx
, unsigned dsty
, unsigned dstz
,
639 struct pipe_resource
*src
,
641 const struct pipe_box
*src_box
)
643 struct blorp_batch blorp_batch
;
644 struct iris_context
*ice
= blorp
->driver_ctx
;
645 struct iris_screen
*screen
= (void *) ice
->ctx
.screen
;
646 struct iris_resource
*src_res
= (void *) src
;
647 struct iris_resource
*dst_res
= (void *) dst
;
649 enum isl_aux_usage src_aux_usage
, dst_aux_usage
;
650 bool src_clear_supported
, dst_clear_supported
;
651 get_copy_region_aux_settings(ice
, src_res
, &src_aux_usage
,
652 &src_clear_supported
, false);
653 get_copy_region_aux_settings(ice
, dst_res
, &dst_aux_usage
,
654 &dst_clear_supported
, true);
656 if (iris_batch_references(batch
, src_res
->bo
))
657 tex_cache_flush_hack(batch
, ISL_FORMAT_UNSUPPORTED
, src_res
->surf
.format
);
659 if (dst
->target
== PIPE_BUFFER
)
660 util_range_add(&dst_res
->base
, &dst_res
->valid_buffer_range
, dstx
, dstx
+ src_box
->width
);
662 if (dst
->target
== PIPE_BUFFER
&& src
->target
== PIPE_BUFFER
) {
663 struct blorp_address src_addr
= {
664 .buffer
= iris_resource_bo(src
), .offset
= src_box
->x
,
666 struct blorp_address dst_addr
= {
667 .buffer
= iris_resource_bo(dst
), .offset
= dstx
,
668 .reloc_flags
= EXEC_OBJECT_WRITE
,
671 iris_emit_buffer_barrier_for(batch
, iris_resource_bo(src
),
672 IRIS_DOMAIN_OTHER_READ
);
673 iris_emit_buffer_barrier_for(batch
, iris_resource_bo(dst
),
674 IRIS_DOMAIN_RENDER_WRITE
);
676 iris_batch_maybe_flush(batch
, 1500);
678 iris_batch_sync_region_start(batch
);
679 blorp_batch_init(&ice
->blorp
, &blorp_batch
, batch
, 0);
680 blorp_buffer_copy(&blorp_batch
, src_addr
, dst_addr
, src_box
->width
);
681 blorp_batch_finish(&blorp_batch
);
682 iris_batch_sync_region_end(batch
);
684 // XXX: what about one surface being a buffer and not the other?
686 struct blorp_surf src_surf
, dst_surf
;
687 iris_blorp_surf_for_resource(&screen
->isl_dev
, &src_surf
,
688 src
, src_aux_usage
, src_level
, false);
689 iris_blorp_surf_for_resource(&screen
->isl_dev
, &dst_surf
,
690 dst
, dst_aux_usage
, dst_level
, true);
692 iris_resource_prepare_access(ice
, src_res
, src_level
, 1,
693 src_box
->z
, src_box
->depth
,
694 src_aux_usage
, src_clear_supported
);
695 iris_resource_prepare_access(ice
, dst_res
, dst_level
, 1,
696 dstz
, src_box
->depth
,
697 dst_aux_usage
, dst_clear_supported
);
699 iris_emit_buffer_barrier_for(batch
, iris_resource_bo(src
),
700 IRIS_DOMAIN_OTHER_READ
);
701 iris_emit_buffer_barrier_for(batch
, iris_resource_bo(dst
),
702 IRIS_DOMAIN_RENDER_WRITE
);
704 blorp_batch_init(&ice
->blorp
, &blorp_batch
, batch
, 0);
706 for (int slice
= 0; slice
< src_box
->depth
; slice
++) {
707 iris_batch_maybe_flush(batch
, 1500);
709 iris_batch_sync_region_start(batch
);
710 blorp_copy(&blorp_batch
, &src_surf
, src_level
, src_box
->z
+ slice
,
711 &dst_surf
, dst_level
, dstz
+ slice
,
712 src_box
->x
, src_box
->y
, dstx
, dsty
,
713 src_box
->width
, src_box
->height
);
714 iris_batch_sync_region_end(batch
);
716 blorp_batch_finish(&blorp_batch
);
718 iris_resource_finish_write(ice
, dst_res
, dst_level
, dstz
,
719 src_box
->depth
, dst_aux_usage
);
722 tex_cache_flush_hack(batch
, ISL_FORMAT_UNSUPPORTED
, src_res
->surf
.format
);
725 static struct iris_batch
*
726 get_preferred_batch(struct iris_context
*ice
, struct iris_bo
*bo
)
728 /* If the compute batch is already using this buffer, we'd prefer to
729 * continue queueing in the compute batch.
731 if (iris_batch_references(&ice
->batches
[IRIS_BATCH_COMPUTE
], bo
))
732 return &ice
->batches
[IRIS_BATCH_COMPUTE
];
734 /* Otherwise default to the render batch. */
735 return &ice
->batches
[IRIS_BATCH_RENDER
];
740 * The pipe->resource_copy_region() driver hook.
742 * This implements ARB_copy_image semantics - a raw memory copy between
743 * compatible view classes.
746 iris_resource_copy_region(struct pipe_context
*ctx
,
747 struct pipe_resource
*p_dst
,
749 unsigned dstx
, unsigned dsty
, unsigned dstz
,
750 struct pipe_resource
*p_src
,
752 const struct pipe_box
*src_box
)
754 struct iris_context
*ice
= (void *) ctx
;
755 struct iris_screen
*screen
= (void *) ctx
->screen
;
756 struct iris_batch
*batch
= &ice
->batches
[IRIS_BATCH_RENDER
];
757 struct iris_resource
*src
= (void *) p_src
;
758 struct iris_resource
*dst
= (void *) p_dst
;
760 if (iris_resource_unfinished_aux_import(src
))
761 iris_resource_finish_aux_import(ctx
->screen
, src
);
762 if (iris_resource_unfinished_aux_import(dst
))
763 iris_resource_finish_aux_import(ctx
->screen
, dst
);
765 /* Use MI_COPY_MEM_MEM for tiny (<= 16 byte, % 4) buffer copies. */
766 if (p_src
->target
== PIPE_BUFFER
&& p_dst
->target
== PIPE_BUFFER
&&
767 (src_box
->width
% 4 == 0) && src_box
->width
<= 16) {
768 struct iris_bo
*dst_bo
= iris_resource_bo(p_dst
);
769 batch
= get_preferred_batch(ice
, dst_bo
);
770 iris_batch_maybe_flush(batch
, 24 + 5 * (src_box
->width
/ 4));
771 iris_emit_pipe_control_flush(batch
,
772 "stall for MI_COPY_MEM_MEM copy_region",
773 PIPE_CONTROL_CS_STALL
);
774 screen
->vtbl
.copy_mem_mem(batch
, dst_bo
, dstx
, iris_resource_bo(p_src
),
775 src_box
->x
, src_box
->width
);
779 iris_copy_region(&ice
->blorp
, batch
, p_dst
, dst_level
, dstx
, dsty
, dstz
,
780 p_src
, src_level
, src_box
);
782 if (util_format_is_depth_and_stencil(p_dst
->format
) &&
783 util_format_has_stencil(util_format_description(p_src
->format
))) {
784 struct iris_resource
*junk
, *s_src_res
, *s_dst_res
;
785 iris_get_depth_stencil_resources(p_src
, &junk
, &s_src_res
);
786 iris_get_depth_stencil_resources(p_dst
, &junk
, &s_dst_res
);
788 iris_copy_region(&ice
->blorp
, batch
, &s_dst_res
->base
, dst_level
, dstx
,
789 dsty
, dstz
, &s_src_res
->base
, src_level
, src_box
);
792 iris_flush_and_dirty_for_history(ice
, batch
, dst
,
793 PIPE_CONTROL_RENDER_TARGET_FLUSH
,
794 "cache history: post copy_region");
798 iris_init_blit_functions(struct pipe_context
*ctx
)
800 ctx
->blit
= iris_blit
;
801 ctx
->resource_copy_region
= iris_resource_copy_region
;