projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
radeonsi: only decompress the required ZS planes from si_blit
[mesa.git] / src / gallium / drivers / radeonsi / si_blit.c
1 /*
2 * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "si_pipe.h"
25 #include "util/u_format.h"
26 #include "util/u_surface.h"
27
28 enum si_blitter_op /* bitmask */
29 {
30 SI_SAVE_TEXTURES = 1,
31 SI_SAVE_FRAMEBUFFER = 2,
32 SI_SAVE_FRAGMENT_STATE = 4,
33 SI_DISABLE_RENDER_COND = 8,
34
35 SI_CLEAR = SI_SAVE_FRAGMENT_STATE,
36
37 SI_CLEAR_SURFACE = SI_SAVE_FRAMEBUFFER | SI_SAVE_FRAGMENT_STATE,
38
39 SI_COPY = SI_SAVE_FRAMEBUFFER | SI_SAVE_TEXTURES |
40 SI_SAVE_FRAGMENT_STATE | SI_DISABLE_RENDER_COND,
41
42 SI_BLIT = SI_SAVE_FRAMEBUFFER | SI_SAVE_TEXTURES |
43 SI_SAVE_FRAGMENT_STATE,
44
45 SI_DECOMPRESS = SI_SAVE_FRAMEBUFFER | SI_SAVE_FRAGMENT_STATE |
46 SI_DISABLE_RENDER_COND,
47
48 SI_COLOR_RESOLVE = SI_SAVE_FRAMEBUFFER | SI_SAVE_FRAGMENT_STATE
49 };
50
51 static void si_blitter_begin(struct pipe_context *ctx, enum si_blitter_op op)
52 {
53 struct si_context *sctx = (struct si_context *)ctx;
54
55 util_blitter_save_vertex_buffer_slot(sctx->blitter, sctx->vertex_buffer);
56 util_blitter_save_vertex_elements(sctx->blitter, sctx->vertex_elements);
57 util_blitter_save_vertex_shader(sctx->blitter, sctx->vs_shader.cso);
58 util_blitter_save_tessctrl_shader(sctx->blitter, sctx->tcs_shader.cso);
59 util_blitter_save_tesseval_shader(sctx->blitter, sctx->tes_shader.cso);
60 util_blitter_save_geometry_shader(sctx->blitter, sctx->gs_shader.cso);
61 util_blitter_save_so_targets(sctx->blitter, sctx->b.streamout.num_targets,
62 (struct pipe_stream_output_target**)sctx->b.streamout.targets);
63 util_blitter_save_rasterizer(sctx->blitter, sctx->queued.named.rasterizer);
64
65 if (op & SI_SAVE_FRAGMENT_STATE) {
66 util_blitter_save_blend(sctx->blitter, sctx->queued.named.blend);
67 util_blitter_save_depth_stencil_alpha(sctx->blitter, sctx->queued.named.dsa);
68 util_blitter_save_stencil_ref(sctx->blitter, &sctx->stencil_ref.state);
69 util_blitter_save_fragment_shader(sctx->blitter, sctx->ps_shader.cso);
70 util_blitter_save_sample_mask(sctx->blitter, sctx->sample_mask.sample_mask);
71 util_blitter_save_viewport(sctx->blitter, &sctx->b.viewports.states[0]);
72 util_blitter_save_scissor(sctx->blitter, &sctx->b.scissors.states[0]);
73 }
74
75 if (op & SI_SAVE_FRAMEBUFFER)
76 util_blitter_save_framebuffer(sctx->blitter, &sctx->framebuffer.state);
77
78 if (op & SI_SAVE_TEXTURES) {
79 util_blitter_save_fragment_sampler_states(
80 sctx->blitter, 2,
81 sctx->samplers[PIPE_SHADER_FRAGMENT].views.sampler_states);
82
83 util_blitter_save_fragment_sampler_views(sctx->blitter, 2,
84 sctx->samplers[PIPE_SHADER_FRAGMENT].views.views);
85 }
86
87 if (op & SI_DISABLE_RENDER_COND)
88 sctx->b.render_cond_force_off = true;
89 }
90
91 static void si_blitter_end(struct pipe_context *ctx)
92 {
93 struct si_context *sctx = (struct si_context *)ctx;
94
95 sctx->b.render_cond_force_off = false;
96 }
97
98 static unsigned u_max_sample(struct pipe_resource *r)
99 {
100 return r->nr_samples ? r->nr_samples - 1 : 0;
101 }
102
103 static void si_blit_decompress_depth(struct pipe_context *ctx,
104 struct r600_texture *texture,
105 struct r600_texture *staging,
106 unsigned first_level, unsigned last_level,
107 unsigned first_layer, unsigned last_layer,
108 unsigned first_sample, unsigned last_sample)
109 {
110 struct si_context *sctx = (struct si_context *)ctx;
111 unsigned layer, level, sample, checked_last_layer, max_layer, max_sample;
112 float depth = 1.0f;
113 const struct util_format_description *desc;
114 struct r600_texture *flushed_depth_texture = staging ?
115 staging : texture->flushed_depth_texture;
116
117 if (!staging && !texture->dirty_level_mask)
118 return;
119
120 max_sample = u_max_sample(&texture->resource.b.b);
121
122 desc = util_format_description(flushed_depth_texture->resource.b.b.format);
123
124 if (util_format_has_depth(desc))
125 sctx->dbcb_depth_copy_enabled = true;
126 if (util_format_has_stencil(desc))
127 sctx->dbcb_stencil_copy_enabled = true;
128
129 assert(sctx->dbcb_depth_copy_enabled || sctx->dbcb_stencil_copy_enabled);
130
131 for (level = first_level; level <= last_level; level++) {
132 if (!staging && !(texture->dirty_level_mask & (1 << level)))
133 continue;
134
135 /* The smaller the mipmap level, the less layers there are
136 * as far as 3D textures are concerned. */
137 max_layer = util_max_layer(&texture->resource.b.b, level);
138 checked_last_layer = MIN2(last_layer, max_layer);
139
140 for (layer = first_layer; layer <= checked_last_layer; layer++) {
141 for (sample = first_sample; sample <= last_sample; sample++) {
142 struct pipe_surface *zsurf, *cbsurf, surf_tmpl;
143
144 sctx->dbcb_copy_sample = sample;
145 si_mark_atom_dirty(sctx, &sctx->db_render_state);
146
147 surf_tmpl.format = texture->resource.b.b.format;
148 surf_tmpl.u.tex.level = level;
149 surf_tmpl.u.tex.first_layer = layer;
150 surf_tmpl.u.tex.last_layer = layer;
151
152 zsurf = ctx->create_surface(ctx, &texture->resource.b.b, &surf_tmpl);
153
154 surf_tmpl.format = flushed_depth_texture->resource.b.b.format;
155 cbsurf = ctx->create_surface(ctx,
156 (struct pipe_resource*)flushed_depth_texture, &surf_tmpl);
157
158 si_blitter_begin(ctx, SI_DECOMPRESS);
159 util_blitter_custom_depth_stencil(sctx->blitter, zsurf, cbsurf, 1 << sample,
160 sctx->custom_dsa_flush, depth);
161 si_blitter_end(ctx);
162
163 pipe_surface_reference(&zsurf, NULL);
164 pipe_surface_reference(&cbsurf, NULL);
165 }
166 }
167
168 /* The texture will always be dirty if some layers aren't flushed.
169 * I don't think this case can occur though. */
170 if (!staging &&
171 first_layer == 0 && last_layer == max_layer &&
172 first_sample == 0 && last_sample == max_sample) {
173 texture->dirty_level_mask &= ~(1 << level);
174 }
175 }
176
177 sctx->dbcb_depth_copy_enabled = false;
178 sctx->dbcb_stencil_copy_enabled = false;
179 si_mark_atom_dirty(sctx, &sctx->db_render_state);
180 }
181
182 /* Helper function for si_blit_decompress_zs_in_place.
183 */
184 static void
185 si_blit_decompress_zs_planes_in_place(struct si_context *sctx,
186 struct r600_texture *texture,
187 unsigned planes, unsigned level_mask,
188 unsigned first_layer, unsigned last_layer)
189 {
190 struct pipe_surface *zsurf, surf_tmpl = {{0}};
191 unsigned layer, max_layer, checked_last_layer;
192 unsigned fully_decompressed_mask = 0;
193
194 if (!level_mask)
195 return;
196
197 if (planes & PIPE_MASK_S)
198 sctx->db_flush_stencil_inplace = true;
199 if (planes & PIPE_MASK_Z)
200 sctx->db_flush_depth_inplace = true;
201 si_mark_atom_dirty(sctx, &sctx->db_render_state);
202
203 surf_tmpl.format = texture->resource.b.b.format;
204
205 while (level_mask) {
206 unsigned level = u_bit_scan(&level_mask);
207
208 surf_tmpl.u.tex.level = level;
209
210 /* The smaller the mipmap level, the less layers there are
211 * as far as 3D textures are concerned. */
212 max_layer = util_max_layer(&texture->resource.b.b, level);
213 checked_last_layer = MIN2(last_layer, max_layer);
214
215 for (layer = first_layer; layer <= checked_last_layer; layer++) {
216 surf_tmpl.u.tex.first_layer = layer;
217 surf_tmpl.u.tex.last_layer = layer;
218
219 zsurf = sctx->b.b.create_surface(&sctx->b.b, &texture->resource.b.b, &surf_tmpl);
220
221 si_blitter_begin(&sctx->b.b, SI_DECOMPRESS);
222 util_blitter_custom_depth_stencil(sctx->blitter, zsurf, NULL, ~0,
223 sctx->custom_dsa_flush,
224 1.0f);
225 si_blitter_end(&sctx->b.b);
226
227 pipe_surface_reference(&zsurf, NULL);
228 }
229
230 /* The texture will always be dirty if some layers aren't flushed.
231 * I don't think this case occurs often though. */
232 if (first_layer == 0 && last_layer == max_layer) {
233 fully_decompressed_mask |= 1u << level;
234 }
235 }
236
237 if (planes & PIPE_MASK_Z)
238 texture->dirty_level_mask &= ~fully_decompressed_mask;
239 if (planes & PIPE_MASK_S)
240 texture->stencil_dirty_level_mask &= ~fully_decompressed_mask;
241
242 sctx->db_flush_depth_inplace = false;
243 sctx->db_flush_stencil_inplace = false;
244 si_mark_atom_dirty(sctx, &sctx->db_render_state);
245 }
246
247 /* Decompress Z and/or S planes in place, depending on mask.
248 */
249 static void
250 si_blit_decompress_zs_in_place(struct si_context *sctx,
251 struct r600_texture *texture,
252 unsigned planes,
253 unsigned first_level, unsigned last_level,
254 unsigned first_layer, unsigned last_layer)
255 {
256 unsigned level_mask =
257 u_bit_consecutive(first_level, last_level - first_level + 1);
258 unsigned cur_level_mask;
259
260 /* First, do combined Z & S decompresses for levels that need it. */
261 if (planes == (PIPE_MASK_Z | PIPE_MASK_S)) {
262 cur_level_mask =
263 level_mask &
264 texture->dirty_level_mask &
265 texture->stencil_dirty_level_mask;
266 si_blit_decompress_zs_planes_in_place(
267 sctx, texture, PIPE_MASK_Z | PIPE_MASK_S,
268 cur_level_mask,
269 first_layer, last_layer);
270 level_mask &= ~cur_level_mask;
271 }
272
273 /* Now do separate Z and S decompresses. */
274 if (planes & PIPE_MASK_Z) {
275 cur_level_mask = level_mask & texture->dirty_level_mask;
276 si_blit_decompress_zs_planes_in_place(
277 sctx, texture, PIPE_MASK_Z,
278 cur_level_mask,
279 first_layer, last_layer);
280 level_mask &= ~cur_level_mask;
281 }
282
283 if (planes & PIPE_MASK_S) {
284 cur_level_mask = level_mask & texture->stencil_dirty_level_mask;
285 si_blit_decompress_zs_planes_in_place(
286 sctx, texture, PIPE_MASK_S,
287 cur_level_mask,
288 first_layer, last_layer);
289 }
290 }
291
292 static void
293 si_flush_depth_textures(struct si_context *sctx,
294 struct si_textures_info *textures)
295 {
296 unsigned i;
297 unsigned mask = textures->depth_texture_mask;
298
299 while (mask) {
300 struct pipe_sampler_view *view;
301 struct si_sampler_view *sview;
302 struct r600_texture *tex;
303
304 i = u_bit_scan(&mask);
305
306 view = textures->views.views[i];
307 assert(view);
308 sview = (struct si_sampler_view*)view;
309
310 tex = (struct r600_texture *)view->texture;
311 assert(tex->is_depth && !tex->is_flushing_texture);
312
313 si_blit_decompress_zs_in_place(sctx, tex,
314 sview->is_stencil_sampler ? PIPE_MASK_S
315 : PIPE_MASK_Z,
316 view->u.tex.first_level, view->u.tex.last_level,
317 0, util_max_layer(&tex->resource.b.b, view->u.tex.first_level));
318 }
319 }
320
321 static void si_blit_decompress_color(struct pipe_context *ctx,
322 struct r600_texture *rtex,
323 unsigned first_level, unsigned last_level,
324 unsigned first_layer, unsigned last_layer,
325 bool need_dcc_decompress)
326 {
327 struct si_context *sctx = (struct si_context *)ctx;
328 unsigned layer, level, checked_last_layer, max_layer;
329
330 if (!rtex->dirty_level_mask && !need_dcc_decompress)
331 return;
332
333 for (level = first_level; level <= last_level; level++) {
334 void* custom_blend;
335
336 if (!(rtex->dirty_level_mask & (1 << level)) && !need_dcc_decompress)
337 continue;
338
339 if (rtex->dcc_offset && need_dcc_decompress) {
340 custom_blend = sctx->custom_blend_dcc_decompress;
341 } else if (rtex->fmask.size) {
342 custom_blend = sctx->custom_blend_decompress;
343 } else {
344 custom_blend = sctx->custom_blend_fastclear;
345 }
346
347 /* The smaller the mipmap level, the less layers there are
348 * as far as 3D textures are concerned. */
349 max_layer = util_max_layer(&rtex->resource.b.b, level);
350 checked_last_layer = MIN2(last_layer, max_layer);
351
352 for (layer = first_layer; layer <= checked_last_layer; layer++) {
353 struct pipe_surface *cbsurf, surf_tmpl;
354
355 surf_tmpl.format = rtex->resource.b.b.format;
356 surf_tmpl.u.tex.level = level;
357 surf_tmpl.u.tex.first_layer = layer;
358 surf_tmpl.u.tex.last_layer = layer;
359 cbsurf = ctx->create_surface(ctx, &rtex->resource.b.b, &surf_tmpl);
360
361 si_blitter_begin(ctx, SI_DECOMPRESS);
362 util_blitter_custom_color(sctx->blitter, cbsurf, custom_blend);
363 si_blitter_end(ctx);
364
365 pipe_surface_reference(&cbsurf, NULL);
366 }
367
368 /* The texture will always be dirty if some layers aren't flushed.
369 * I don't think this case occurs often though. */
370 if (first_layer == 0 && last_layer == max_layer) {
371 rtex->dirty_level_mask &= ~(1 << level);
372 }
373 }
374 }
375
376 static void
377 si_decompress_sampler_color_textures(struct si_context *sctx,
378 struct si_textures_info *textures)
379 {
380 unsigned i;
381 unsigned mask = textures->compressed_colortex_mask;
382
383 while (mask) {
384 struct pipe_sampler_view *view;
385 struct r600_texture *tex;
386
387 i = u_bit_scan(&mask);
388
389 view = textures->views.views[i];
390 assert(view);
391
392 tex = (struct r600_texture *)view->texture;
393 assert(tex->cmask.size || tex->fmask.size || tex->dcc_offset);
394
395 si_blit_decompress_color(&sctx->b.b, tex,
396 view->u.tex.first_level, view->u.tex.last_level,
397 0, util_max_layer(&tex->resource.b.b, view->u.tex.first_level),
398 false);
399 }
400 }
401
402 static void
403 si_decompress_image_color_textures(struct si_context *sctx,
404 struct si_images_info *images)
405 {
406 unsigned i;
407 unsigned mask = images->compressed_colortex_mask;
408
409 while (mask) {
410 const struct pipe_image_view *view;
411 struct r600_texture *tex;
412
413 i = u_bit_scan(&mask);
414
415 view = &images->views[i];
416 assert(view->resource->target != PIPE_BUFFER);
417
418 tex = (struct r600_texture *)view->resource;
419 if (!tex->cmask.size && !tex->fmask.size && !tex->dcc_offset)
420 continue;
421
422 si_blit_decompress_color(&sctx->b.b, tex,
423 view->u.tex.level, view->u.tex.level,
424 0, util_max_layer(&tex->resource.b.b, view->u.tex.level),
425 false);
426 }
427 }
428
429 static void si_decompress_textures(struct si_context *sctx, int shader_start,
430 int shader_end)
431 {
432 unsigned compressed_colortex_counter;
433
434 if (sctx->blitter->running)
435 return;
436
437 /* Update the compressed_colortex_mask if necessary. */
438 compressed_colortex_counter = p_atomic_read(&sctx->screen->b.compressed_colortex_counter);
439 if (compressed_colortex_counter != sctx->b.last_compressed_colortex_counter) {
440 sctx->b.last_compressed_colortex_counter = compressed_colortex_counter;
441 si_update_compressed_colortex_masks(sctx);
442 }
443
444 /* Flush depth textures which need to be flushed. */
445 for (int i = shader_start; i < shader_end; i++) {
446 if (sctx->samplers[i].depth_texture_mask) {
447 si_flush_depth_textures(sctx, &sctx->samplers[i]);
448 }
449 if (sctx->samplers[i].compressed_colortex_mask) {
450 si_decompress_sampler_color_textures(sctx, &sctx->samplers[i]);
451 }
452 if (sctx->images[i].compressed_colortex_mask) {
453 si_decompress_image_color_textures(sctx, &sctx->images[i]);
454 }
455 }
456 }
457
458 void si_decompress_graphics_textures(struct si_context *sctx)
459 {
460 si_decompress_textures(sctx, 0, SI_NUM_GRAPHICS_SHADERS);
461 }
462
463 void si_decompress_compute_textures(struct si_context *sctx)
464 {
465 si_decompress_textures(sctx, SI_NUM_GRAPHICS_SHADERS, SI_NUM_SHADERS);
466 }
467
468 static void si_clear(struct pipe_context *ctx, unsigned buffers,
469 const union pipe_color_union *color,
470 double depth, unsigned stencil)
471 {
472 struct si_context *sctx = (struct si_context *)ctx;
473 struct pipe_framebuffer_state *fb = &sctx->framebuffer.state;
474 struct pipe_surface *zsbuf = fb->zsbuf;
475 struct r600_texture *zstex =
476 zsbuf ? (struct r600_texture*)zsbuf->texture : NULL;
477
478 if (buffers & PIPE_CLEAR_COLOR) {
479 evergreen_do_fast_color_clear(&sctx->b, fb,
480 &sctx->framebuffer.atom, &buffers,
481 &sctx->framebuffer.dirty_cbufs,
482 color);
483 if (!buffers)
484 return; /* all buffers have been fast cleared */
485 }
486
487 if (buffers & PIPE_CLEAR_COLOR) {
488 int i;
489
490 /* These buffers cannot use fast clear, make sure to disable expansion. */
491 for (i = 0; i < fb->nr_cbufs; i++) {
492 struct r600_texture *tex;
493
494 /* If not clearing this buffer, skip. */
495 if (!(buffers & (PIPE_CLEAR_COLOR0 << i)))
496 continue;
497
498 if (!fb->cbufs[i])
499 continue;
500
501 tex = (struct r600_texture *)fb->cbufs[i]->texture;
502 if (tex->fmask.size == 0)
503 tex->dirty_level_mask &= ~(1 << fb->cbufs[i]->u.tex.level);
504 }
505 }
506
507 if (zstex && zstex->htile_buffer &&
508 zsbuf->u.tex.level == 0 &&
509 zsbuf->u.tex.first_layer == 0 &&
510 zsbuf->u.tex.last_layer == util_max_layer(&zstex->resource.b.b, 0)) {
511 if (buffers & PIPE_CLEAR_DEPTH) {
512 /* Need to disable EXPCLEAR temporarily if clearing
513 * to a new value. */
514 if (zstex->depth_cleared && zstex->depth_clear_value != depth) {
515 sctx->db_depth_disable_expclear = true;
516 }
517
518 zstex->depth_clear_value = depth;
519 sctx->framebuffer.dirty_zsbuf = true;
520 si_mark_atom_dirty(sctx, &sctx->framebuffer.atom); /* updates DB_DEPTH_CLEAR */
521 sctx->db_depth_clear = true;
522 si_mark_atom_dirty(sctx, &sctx->db_render_state);
523 }
524
525 if (buffers & PIPE_CLEAR_STENCIL) {
526 stencil &= 0xff;
527
528 /* Need to disable EXPCLEAR temporarily if clearing
529 * to a new value. */
530 if (zstex->stencil_cleared && zstex->stencil_clear_value != stencil) {
531 sctx->db_stencil_disable_expclear = true;
532 }
533
534 zstex->stencil_clear_value = stencil;
535 sctx->framebuffer.dirty_zsbuf = true;
536 si_mark_atom_dirty(sctx, &sctx->framebuffer.atom); /* updates DB_STENCIL_CLEAR */
537 sctx->db_stencil_clear = true;
538 si_mark_atom_dirty(sctx, &sctx->db_render_state);
539 }
540 }
541
542 si_blitter_begin(ctx, SI_CLEAR);
543 util_blitter_clear(sctx->blitter, fb->width, fb->height,
544 util_framebuffer_get_num_layers(fb),
545 buffers, color, depth, stencil);
546 si_blitter_end(ctx);
547
548 if (sctx->db_depth_clear) {
549 sctx->db_depth_clear = false;
550 sctx->db_depth_disable_expclear = false;
551 zstex->depth_cleared = true;
552 si_mark_atom_dirty(sctx, &sctx->db_render_state);
553 }
554
555 if (sctx->db_stencil_clear) {
556 sctx->db_stencil_clear = false;
557 sctx->db_stencil_disable_expclear = false;
558 zstex->stencil_cleared = true;
559 si_mark_atom_dirty(sctx, &sctx->db_render_state);
560 }
561 }
562
563 static void si_clear_render_target(struct pipe_context *ctx,
564 struct pipe_surface *dst,
565 const union pipe_color_union *color,
566 unsigned dstx, unsigned dsty,
567 unsigned width, unsigned height)
568 {
569 struct si_context *sctx = (struct si_context *)ctx;
570
571 si_blitter_begin(ctx, SI_CLEAR_SURFACE);
572 util_blitter_clear_render_target(sctx->blitter, dst, color,
573 dstx, dsty, width, height);
574 si_blitter_end(ctx);
575 }
576
577 static void si_clear_depth_stencil(struct pipe_context *ctx,
578 struct pipe_surface *dst,
579 unsigned clear_flags,
580 double depth,
581 unsigned stencil,
582 unsigned dstx, unsigned dsty,
583 unsigned width, unsigned height)
584 {
585 struct si_context *sctx = (struct si_context *)ctx;
586
587 si_blitter_begin(ctx, SI_CLEAR_SURFACE);
588 util_blitter_clear_depth_stencil(sctx->blitter, dst, clear_flags, depth, stencil,
589 dstx, dsty, width, height);
590 si_blitter_end(ctx);
591 }
592
593 /* Helper for decompressing a portion of a color or depth resource before
594 * blitting if any decompression is needed.
595 * The driver doesn't decompress resources automatically while u_blitter is
596 * rendering. */
597 static void si_decompress_subresource(struct pipe_context *ctx,
598 struct pipe_resource *tex,
599 unsigned planes, unsigned level,
600 unsigned first_layer, unsigned last_layer)
601 {
602 struct si_context *sctx = (struct si_context *)ctx;
603 struct r600_texture *rtex = (struct r600_texture*)tex;
604
605 if (rtex->is_depth && !rtex->is_flushing_texture) {
606 planes &= PIPE_MASK_Z | PIPE_MASK_S;
607
608 if (!(rtex->surface.flags & RADEON_SURF_SBUFFER))
609 planes &= ~PIPE_MASK_S;
610
611 si_blit_decompress_zs_in_place(sctx, rtex, planes,
612 level, level,
613 first_layer, last_layer);
614 } else if (rtex->fmask.size || rtex->cmask.size || rtex->dcc_offset) {
615 si_blit_decompress_color(ctx, rtex, level, level,
616 first_layer, last_layer, false);
617 }
618 }
619
620 struct texture_orig_info {
621 unsigned format;
622 unsigned width0;
623 unsigned height0;
624 unsigned npix_x;
625 unsigned npix_y;
626 unsigned npix0_x;
627 unsigned npix0_y;
628 };
629
630 void si_resource_copy_region(struct pipe_context *ctx,
631 struct pipe_resource *dst,
632 unsigned dst_level,
633 unsigned dstx, unsigned dsty, unsigned dstz,
634 struct pipe_resource *src,
635 unsigned src_level,
636 const struct pipe_box *src_box)
637 {
638 struct si_context *sctx = (struct si_context *)ctx;
639 struct pipe_surface *dst_view, dst_templ;
640 struct pipe_sampler_view src_templ, *src_view;
641 unsigned dst_width, dst_height, src_width0, src_height0;
642 unsigned src_force_level = 0;
643 struct pipe_box sbox, dstbox;
644
645 /* Handle buffers first. */
646 if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
647 si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width, false);
648 return;
649 }
650
651 assert(u_max_sample(dst) == u_max_sample(src));
652
653 /* The driver doesn't decompress resources automatically while
654 * u_blitter is rendering. */
655 si_decompress_subresource(ctx, src, PIPE_MASK_RGBAZS, src_level,
656 src_box->z, src_box->z + src_box->depth - 1);
657
658 dst_width = u_minify(dst->width0, dst_level);
659 dst_height = u_minify(dst->height0, dst_level);
660 src_width0 = src->width0;
661 src_height0 = src->height0;
662
663 util_blitter_default_dst_texture(&dst_templ, dst, dst_level, dstz);
664 util_blitter_default_src_texture(&src_templ, src, src_level);
665
666 if (util_format_is_compressed(src->format) ||
667 util_format_is_compressed(dst->format)) {
668 unsigned blocksize = util_format_get_blocksize(src->format);
669
670 if (blocksize == 8)
671 src_templ.format = PIPE_FORMAT_R16G16B16A16_UINT; /* 64-bit block */
672 else
673 src_templ.format = PIPE_FORMAT_R32G32B32A32_UINT; /* 128-bit block */
674 dst_templ.format = src_templ.format;
675
676 dst_width = util_format_get_nblocksx(dst->format, dst_width);
677 dst_height = util_format_get_nblocksy(dst->format, dst_height);
678 src_width0 = util_format_get_nblocksx(src->format, src_width0);
679 src_height0 = util_format_get_nblocksy(src->format, src_height0);
680
681 dstx = util_format_get_nblocksx(dst->format, dstx);
682 dsty = util_format_get_nblocksy(dst->format, dsty);
683
684 sbox.x = util_format_get_nblocksx(src->format, src_box->x);
685 sbox.y = util_format_get_nblocksy(src->format, src_box->y);
686 sbox.z = src_box->z;
687 sbox.width = util_format_get_nblocksx(src->format, src_box->width);
688 sbox.height = util_format_get_nblocksy(src->format, src_box->height);
689 sbox.depth = src_box->depth;
690 src_box = &sbox;
691
692 src_force_level = src_level;
693 } else if (!util_blitter_is_copy_supported(sctx->blitter, dst, src) ||
694 /* also *8_SNORM has precision issues, use UNORM instead */
695 util_format_is_snorm8(src->format)) {
696 if (util_format_is_subsampled_422(src->format)) {
697 src_templ.format = PIPE_FORMAT_R8G8B8A8_UINT;
698 dst_templ.format = PIPE_FORMAT_R8G8B8A8_UINT;
699
700 dst_width = util_format_get_nblocksx(dst->format, dst_width);
701 src_width0 = util_format_get_nblocksx(src->format, src_width0);
702
703 dstx = util_format_get_nblocksx(dst->format, dstx);
704
705 sbox = *src_box;
706 sbox.x = util_format_get_nblocksx(src->format, src_box->x);
707 sbox.width = util_format_get_nblocksx(src->format, src_box->width);
708 src_box = &sbox;
709 } else {
710 unsigned blocksize = util_format_get_blocksize(src->format);
711
712 switch (blocksize) {
713 case 1:
714 dst_templ.format = PIPE_FORMAT_R8_UNORM;
715 src_templ.format = PIPE_FORMAT_R8_UNORM;
716 break;
717 case 2:
718 dst_templ.format = PIPE_FORMAT_R8G8_UNORM;
719 src_templ.format = PIPE_FORMAT_R8G8_UNORM;
720 break;
721 case 4:
722 dst_templ.format = PIPE_FORMAT_R8G8B8A8_UNORM;
723 src_templ.format = PIPE_FORMAT_R8G8B8A8_UNORM;
724 break;
725 case 8:
726 dst_templ.format = PIPE_FORMAT_R16G16B16A16_UINT;
727 src_templ.format = PIPE_FORMAT_R16G16B16A16_UINT;
728 break;
729 case 16:
730 dst_templ.format = PIPE_FORMAT_R32G32B32A32_UINT;
731 src_templ.format = PIPE_FORMAT_R32G32B32A32_UINT;
732 break;
733 default:
734 fprintf(stderr, "Unhandled format %s with blocksize %u\n",
735 util_format_short_name(src->format), blocksize);
736 assert(0);
737 }
738 }
739 }
740
741 /* Initialize the surface. */
742 dst_view = r600_create_surface_custom(ctx, dst, &dst_templ,
743 dst_width, dst_height);
744
745 /* Initialize the sampler view. */
746 src_view = si_create_sampler_view_custom(ctx, src, &src_templ,
747 src_width0, src_height0,
748 src_force_level);
749
750 u_box_3d(dstx, dsty, dstz, abs(src_box->width), abs(src_box->height),
751 abs(src_box->depth), &dstbox);
752
753 /* Copy. */
754 si_blitter_begin(ctx, SI_COPY);
755 util_blitter_blit_generic(sctx->blitter, dst_view, &dstbox,
756 src_view, src_box, src_width0, src_height0,
757 PIPE_MASK_RGBAZS, PIPE_TEX_FILTER_NEAREST, NULL,
758 FALSE);
759 si_blitter_end(ctx);
760
761 pipe_surface_reference(&dst_view, NULL);
762 pipe_sampler_view_reference(&src_view, NULL);
763 }
764
765 /* For MSAA integer resolving to work, we change the format to NORM using this function. */
766 static enum pipe_format int_to_norm_format(enum pipe_format format)
767 {
768 switch (format) {
769 #define REPLACE_FORMAT_SIGN(format,sign) \
770 case PIPE_FORMAT_##format##_##sign##INT: \
771 return PIPE_FORMAT_##format##_##sign##NORM
772 #define REPLACE_FORMAT(format) \
773 REPLACE_FORMAT_SIGN(format, U); \
774 REPLACE_FORMAT_SIGN(format, S)
775
776 REPLACE_FORMAT_SIGN(B10G10R10A2, U);
777 REPLACE_FORMAT(R8);
778 REPLACE_FORMAT(R8G8);
779 REPLACE_FORMAT(R8G8B8X8);
780 REPLACE_FORMAT(R8G8B8A8);
781 REPLACE_FORMAT(A8);
782 REPLACE_FORMAT(I8);
783 REPLACE_FORMAT(L8);
784 REPLACE_FORMAT(L8A8);
785 REPLACE_FORMAT(R16);
786 REPLACE_FORMAT(R16G16);
787 REPLACE_FORMAT(R16G16B16X16);
788 REPLACE_FORMAT(R16G16B16A16);
789 REPLACE_FORMAT(A16);
790 REPLACE_FORMAT(I16);
791 REPLACE_FORMAT(L16);
792 REPLACE_FORMAT(L16A16);
793
794 #undef REPLACE_FORMAT
795 #undef REPLACE_FORMAT_SIGN
796 default:
797 return format;
798 }
799 }
800
801 static bool do_hardware_msaa_resolve(struct pipe_context *ctx,
802 const struct pipe_blit_info *info)
803 {
804 struct si_context *sctx = (struct si_context*)ctx;
805 struct r600_texture *dst = (struct r600_texture*)info->dst.resource;
806 unsigned dst_width = u_minify(info->dst.resource->width0, info->dst.level);
807 unsigned dst_height = u_minify(info->dst.resource->height0, info->dst.level);
808 enum pipe_format format = int_to_norm_format(info->dst.format);
809 unsigned sample_mask = ~0;
810
811 /* Hardware MSAA resolve doesn't work if SPI format = NORM16_ABGR and
812 * the format is R16G16. Use R16A16, which does work.
813 */
814 if (format == PIPE_FORMAT_R16G16_UNORM)
815 format = PIPE_FORMAT_R16A16_UNORM;
816 if (format == PIPE_FORMAT_R16G16_SNORM)
817 format = PIPE_FORMAT_R16A16_SNORM;
818
819 if (info->src.resource->nr_samples > 1 &&
820 info->dst.resource->nr_samples <= 1 &&
821 util_max_layer(info->src.resource, 0) == 0 &&
822 util_max_layer(info->dst.resource, info->dst.level) == 0 &&
823 info->dst.format == info->src.format &&
824 !util_format_is_pure_integer(format) &&
825 !util_format_is_depth_or_stencil(format) &&
826 !info->scissor_enable &&
827 (info->mask & PIPE_MASK_RGBA) == PIPE_MASK_RGBA &&
828 dst_width == info->src.resource->width0 &&
829 dst_height == info->src.resource->height0 &&
830 info->dst.box.x == 0 &&
831 info->dst.box.y == 0 &&
832 info->dst.box.width == dst_width &&
833 info->dst.box.height == dst_height &&
834 info->dst.box.depth == 1 &&
835 info->src.box.x == 0 &&
836 info->src.box.y == 0 &&
837 info->src.box.width == dst_width &&
838 info->src.box.height == dst_height &&
839 info->src.box.depth == 1 &&
840 dst->surface.level[info->dst.level].mode >= RADEON_SURF_MODE_1D &&
841 !(dst->surface.flags & RADEON_SURF_SCANOUT) &&
842 (!dst->cmask.size || !dst->dirty_level_mask) && /* dst cannot be fast-cleared */
843 !dst->dcc_offset) {
844 si_blitter_begin(ctx, SI_COLOR_RESOLVE |
845 (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
846 util_blitter_custom_resolve_color(sctx->blitter,
847 info->dst.resource, info->dst.level,
848 info->dst.box.z,
849 info->src.resource, info->src.box.z,
850 sample_mask, sctx->custom_blend_resolve,
851 format);
852 si_blitter_end(ctx);
853 return true;
854 }
855 return false;
856 }
857
858 static void si_blit(struct pipe_context *ctx,
859 const struct pipe_blit_info *info)
860 {
861 struct si_context *sctx = (struct si_context*)ctx;
862
863 if (do_hardware_msaa_resolve(ctx, info)) {
864 return;
865 }
866
867 assert(util_blitter_is_blit_supported(sctx->blitter, info));
868
869 /* The driver doesn't decompress resources automatically while
870 * u_blitter is rendering. */
871 si_decompress_subresource(ctx, info->src.resource, info->mask,
872 info->src.level,
873 info->src.box.z,
874 info->src.box.z + info->src.box.depth - 1);
875
876 if (sctx->screen->b.debug_flags & DBG_FORCE_DMA &&
877 util_try_blit_via_copy_region(ctx, info))
878 return;
879
880 si_blitter_begin(ctx, SI_BLIT |
881 (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
882 util_blitter_blit(sctx->blitter, info);
883 si_blitter_end(ctx);
884 }
885
886 static void si_flush_resource(struct pipe_context *ctx,
887 struct pipe_resource *res)
888 {
889 struct r600_texture *rtex = (struct r600_texture*)res;
890
891 assert(res->target != PIPE_BUFFER);
892
893 if (!rtex->is_depth && (rtex->cmask.size || rtex->dcc_offset)) {
894 si_blit_decompress_color(ctx, rtex, 0, res->last_level,
895 0, util_max_layer(res, 0), false);
896 }
897 }
898
899 static void si_decompress_dcc(struct pipe_context *ctx,
900 struct r600_texture *rtex)
901 {
902 if (!rtex->dcc_offset)
903 return;
904
905 si_blit_decompress_color(ctx, rtex, 0, rtex->resource.b.b.last_level,
906 0, util_max_layer(&rtex->resource.b.b, 0),
907 true);
908 }
909
910 static void si_pipe_clear_buffer(struct pipe_context *ctx,
911 struct pipe_resource *dst,
912 unsigned offset, unsigned size,
913 const void *clear_value_ptr,
914 int clear_value_size)
915 {
916 struct si_context *sctx = (struct si_context*)ctx;
917 uint32_t dword_value;
918 unsigned i;
919
920 assert(offset % clear_value_size == 0);
921 assert(size % clear_value_size == 0);
922
923 if (clear_value_size > 4) {
924 const uint32_t *u32 = clear_value_ptr;
925 bool clear_dword_duplicated = true;
926
927 /* See if we can lower large fills to dword fills. */
928 for (i = 1; i < clear_value_size / 4; i++)
929 if (u32[0] != u32[i]) {
930 clear_dword_duplicated = false;
931 break;
932 }
933
934 if (!clear_dword_duplicated) {
935 /* Use transform feedback for 64-bit, 96-bit, and
936 * 128-bit fills.
937 */
938 union pipe_color_union clear_value;
939
940 memcpy(&clear_value, clear_value_ptr, clear_value_size);
941 si_blitter_begin(ctx, SI_DISABLE_RENDER_COND);
942 util_blitter_clear_buffer(sctx->blitter, dst, offset,
943 size, clear_value_size / 4,
944 &clear_value);
945 si_blitter_end(ctx);
946 return;
947 }
948 }
949
950 /* Expand the clear value to a dword. */
951 switch (clear_value_size) {
952 case 1:
953 dword_value = *(uint8_t*)clear_value_ptr;
954 dword_value |= (dword_value << 8) |
955 (dword_value << 16) |
956 (dword_value << 24);
957 break;
958 case 2:
959 dword_value = *(uint16_t*)clear_value_ptr;
960 dword_value |= dword_value << 16;
961 break;
962 default:
963 dword_value = *(uint32_t*)clear_value_ptr;
964 }
965
966 sctx->b.clear_buffer(ctx, dst, offset, size, dword_value, false);
967 }
968
969 void si_init_blit_functions(struct si_context *sctx)
970 {
971 sctx->b.b.clear = si_clear;
972 sctx->b.b.clear_buffer = si_pipe_clear_buffer;
973 sctx->b.b.clear_render_target = si_clear_render_target;
974 sctx->b.b.clear_depth_stencil = si_clear_depth_stencil;
975 sctx->b.b.resource_copy_region = si_resource_copy_region;
976 sctx->b.b.blit = si_blit;
977 sctx->b.b.flush_resource = si_flush_resource;
978 sctx->b.blit_decompress_depth = si_blit_decompress_depth;
979 sctx->b.decompress_dcc = si_decompress_dcc;
980 }