projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
radeonsi: clarify the MSAA resolve limitation with scanout
[mesa.git] / src / gallium / drivers / radeonsi / si_blit.c
1 /*
2 * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "si_pipe.h"
25 #include "util/u_format.h"
26 #include "util/u_surface.h"
27
28 enum si_blitter_op /* bitmask */
29 {
30 SI_SAVE_TEXTURES = 1,
31 SI_SAVE_FRAMEBUFFER = 2,
32 SI_SAVE_FRAGMENT_STATE = 4,
33 SI_DISABLE_RENDER_COND = 8,
34
35 SI_CLEAR = SI_SAVE_FRAGMENT_STATE,
36
37 SI_CLEAR_SURFACE = SI_SAVE_FRAMEBUFFER | SI_SAVE_FRAGMENT_STATE,
38
39 SI_COPY = SI_SAVE_FRAMEBUFFER | SI_SAVE_TEXTURES |
40 SI_SAVE_FRAGMENT_STATE | SI_DISABLE_RENDER_COND,
41
42 SI_BLIT = SI_SAVE_FRAMEBUFFER | SI_SAVE_TEXTURES |
43 SI_SAVE_FRAGMENT_STATE,
44
45 SI_DECOMPRESS = SI_SAVE_FRAMEBUFFER | SI_SAVE_FRAGMENT_STATE |
46 SI_DISABLE_RENDER_COND,
47
48 SI_COLOR_RESOLVE = SI_SAVE_FRAMEBUFFER | SI_SAVE_FRAGMENT_STATE
49 };
50
51 static void si_blitter_begin(struct pipe_context *ctx, enum si_blitter_op op)
52 {
53 struct si_context *sctx = (struct si_context *)ctx;
54
55 util_blitter_save_vertex_buffer_slot(sctx->blitter, sctx->vertex_buffer);
56 util_blitter_save_vertex_elements(sctx->blitter, sctx->vertex_elements);
57 util_blitter_save_vertex_shader(sctx->blitter, sctx->vs_shader.cso);
58 util_blitter_save_tessctrl_shader(sctx->blitter, sctx->tcs_shader.cso);
59 util_blitter_save_tesseval_shader(sctx->blitter, sctx->tes_shader.cso);
60 util_blitter_save_geometry_shader(sctx->blitter, sctx->gs_shader.cso);
61 util_blitter_save_so_targets(sctx->blitter, sctx->b.streamout.num_targets,
62 (struct pipe_stream_output_target**)sctx->b.streamout.targets);
63 util_blitter_save_rasterizer(sctx->blitter, sctx->queued.named.rasterizer);
64
65 if (op & SI_SAVE_FRAGMENT_STATE) {
66 util_blitter_save_blend(sctx->blitter, sctx->queued.named.blend);
67 util_blitter_save_depth_stencil_alpha(sctx->blitter, sctx->queued.named.dsa);
68 util_blitter_save_stencil_ref(sctx->blitter, &sctx->stencil_ref.state);
69 util_blitter_save_fragment_shader(sctx->blitter, sctx->ps_shader.cso);
70 util_blitter_save_sample_mask(sctx->blitter, sctx->sample_mask.sample_mask);
71 util_blitter_save_viewport(sctx->blitter, &sctx->b.viewports.states[0]);
72 util_blitter_save_scissor(sctx->blitter, &sctx->b.scissors.states[0]);
73 }
74
75 if (op & SI_SAVE_FRAMEBUFFER)
76 util_blitter_save_framebuffer(sctx->blitter, &sctx->framebuffer.state);
77
78 if (op & SI_SAVE_TEXTURES) {
79 util_blitter_save_fragment_sampler_states(
80 sctx->blitter, 2,
81 sctx->samplers[PIPE_SHADER_FRAGMENT].views.sampler_states);
82
83 util_blitter_save_fragment_sampler_views(sctx->blitter, 2,
84 sctx->samplers[PIPE_SHADER_FRAGMENT].views.views);
85 }
86
87 if (op & SI_DISABLE_RENDER_COND)
88 sctx->b.render_cond_force_off = true;
89 }
90
91 static void si_blitter_end(struct pipe_context *ctx)
92 {
93 struct si_context *sctx = (struct si_context *)ctx;
94
95 sctx->b.render_cond_force_off = false;
96 }
97
98 static unsigned u_max_sample(struct pipe_resource *r)
99 {
100 return r->nr_samples ? r->nr_samples - 1 : 0;
101 }
102
103 static void si_blit_decompress_depth(struct pipe_context *ctx,
104 struct r600_texture *texture,
105 struct r600_texture *staging,
106 unsigned first_level, unsigned last_level,
107 unsigned first_layer, unsigned last_layer,
108 unsigned first_sample, unsigned last_sample)
109 {
110 struct si_context *sctx = (struct si_context *)ctx;
111 unsigned layer, level, sample, checked_last_layer, max_layer;
112 float depth = 1.0f;
113 const struct util_format_description *desc;
114
115 assert(staging != NULL && "use si_blit_decompress_zs_in_place instead");
116
117 desc = util_format_description(staging->resource.b.b.format);
118
119 if (util_format_has_depth(desc))
120 sctx->dbcb_depth_copy_enabled = true;
121 if (util_format_has_stencil(desc))
122 sctx->dbcb_stencil_copy_enabled = true;
123
124 assert(sctx->dbcb_depth_copy_enabled || sctx->dbcb_stencil_copy_enabled);
125
126 for (level = first_level; level <= last_level; level++) {
127 /* The smaller the mipmap level, the less layers there are
128 * as far as 3D textures are concerned. */
129 max_layer = util_max_layer(&texture->resource.b.b, level);
130 checked_last_layer = MIN2(last_layer, max_layer);
131
132 for (layer = first_layer; layer <= checked_last_layer; layer++) {
133 for (sample = first_sample; sample <= last_sample; sample++) {
134 struct pipe_surface *zsurf, *cbsurf, surf_tmpl;
135
136 sctx->dbcb_copy_sample = sample;
137 si_mark_atom_dirty(sctx, &sctx->db_render_state);
138
139 surf_tmpl.format = texture->resource.b.b.format;
140 surf_tmpl.u.tex.level = level;
141 surf_tmpl.u.tex.first_layer = layer;
142 surf_tmpl.u.tex.last_layer = layer;
143
144 zsurf = ctx->create_surface(ctx, &texture->resource.b.b, &surf_tmpl);
145
146 surf_tmpl.format = staging->resource.b.b.format;
147 cbsurf = ctx->create_surface(ctx,
148 (struct pipe_resource*)staging, &surf_tmpl);
149
150 si_blitter_begin(ctx, SI_DECOMPRESS);
151 util_blitter_custom_depth_stencil(sctx->blitter, zsurf, cbsurf, 1 << sample,
152 sctx->custom_dsa_flush, depth);
153 si_blitter_end(ctx);
154
155 pipe_surface_reference(&zsurf, NULL);
156 pipe_surface_reference(&cbsurf, NULL);
157 }
158 }
159 }
160
161 sctx->dbcb_depth_copy_enabled = false;
162 sctx->dbcb_stencil_copy_enabled = false;
163 si_mark_atom_dirty(sctx, &sctx->db_render_state);
164 }
165
166 /* Helper function for si_blit_decompress_zs_in_place.
167 */
168 static void
169 si_blit_decompress_zs_planes_in_place(struct si_context *sctx,
170 struct r600_texture *texture,
171 unsigned planes, unsigned level_mask,
172 unsigned first_layer, unsigned last_layer)
173 {
174 struct pipe_surface *zsurf, surf_tmpl = {{0}};
175 unsigned layer, max_layer, checked_last_layer;
176 unsigned fully_decompressed_mask = 0;
177
178 if (!level_mask)
179 return;
180
181 if (planes & PIPE_MASK_S)
182 sctx->db_flush_stencil_inplace = true;
183 if (planes & PIPE_MASK_Z)
184 sctx->db_flush_depth_inplace = true;
185 si_mark_atom_dirty(sctx, &sctx->db_render_state);
186
187 surf_tmpl.format = texture->resource.b.b.format;
188
189 while (level_mask) {
190 unsigned level = u_bit_scan(&level_mask);
191
192 surf_tmpl.u.tex.level = level;
193
194 /* The smaller the mipmap level, the less layers there are
195 * as far as 3D textures are concerned. */
196 max_layer = util_max_layer(&texture->resource.b.b, level);
197 checked_last_layer = MIN2(last_layer, max_layer);
198
199 for (layer = first_layer; layer <= checked_last_layer; layer++) {
200 surf_tmpl.u.tex.first_layer = layer;
201 surf_tmpl.u.tex.last_layer = layer;
202
203 zsurf = sctx->b.b.create_surface(&sctx->b.b, &texture->resource.b.b, &surf_tmpl);
204
205 si_blitter_begin(&sctx->b.b, SI_DECOMPRESS);
206 util_blitter_custom_depth_stencil(sctx->blitter, zsurf, NULL, ~0,
207 sctx->custom_dsa_flush,
208 1.0f);
209 si_blitter_end(&sctx->b.b);
210
211 pipe_surface_reference(&zsurf, NULL);
212 }
213
214 /* The texture will always be dirty if some layers aren't flushed.
215 * I don't think this case occurs often though. */
216 if (first_layer == 0 && last_layer == max_layer) {
217 fully_decompressed_mask |= 1u << level;
218 }
219 }
220
221 if (planes & PIPE_MASK_Z)
222 texture->dirty_level_mask &= ~fully_decompressed_mask;
223 if (planes & PIPE_MASK_S)
224 texture->stencil_dirty_level_mask &= ~fully_decompressed_mask;
225
226 sctx->db_flush_depth_inplace = false;
227 sctx->db_flush_stencil_inplace = false;
228 si_mark_atom_dirty(sctx, &sctx->db_render_state);
229 }
230
231 /* Decompress Z and/or S planes in place, depending on mask.
232 */
233 static void
234 si_blit_decompress_zs_in_place(struct si_context *sctx,
235 struct r600_texture *texture,
236 unsigned planes,
237 unsigned first_level, unsigned last_level,
238 unsigned first_layer, unsigned last_layer)
239 {
240 unsigned level_mask =
241 u_bit_consecutive(first_level, last_level - first_level + 1);
242 unsigned cur_level_mask;
243
244 /* First, do combined Z & S decompresses for levels that need it. */
245 if (planes == (PIPE_MASK_Z | PIPE_MASK_S)) {
246 cur_level_mask =
247 level_mask &
248 texture->dirty_level_mask &
249 texture->stencil_dirty_level_mask;
250 si_blit_decompress_zs_planes_in_place(
251 sctx, texture, PIPE_MASK_Z | PIPE_MASK_S,
252 cur_level_mask,
253 first_layer, last_layer);
254 level_mask &= ~cur_level_mask;
255 }
256
257 /* Now do separate Z and S decompresses. */
258 if (planes & PIPE_MASK_Z) {
259 cur_level_mask = level_mask & texture->dirty_level_mask;
260 si_blit_decompress_zs_planes_in_place(
261 sctx, texture, PIPE_MASK_Z,
262 cur_level_mask,
263 first_layer, last_layer);
264 level_mask &= ~cur_level_mask;
265 }
266
267 if (planes & PIPE_MASK_S) {
268 cur_level_mask = level_mask & texture->stencil_dirty_level_mask;
269 si_blit_decompress_zs_planes_in_place(
270 sctx, texture, PIPE_MASK_S,
271 cur_level_mask,
272 first_layer, last_layer);
273 }
274 }
275
276 static void
277 si_flush_depth_textures(struct si_context *sctx,
278 struct si_textures_info *textures)
279 {
280 unsigned i;
281 unsigned mask = textures->depth_texture_mask;
282
283 while (mask) {
284 struct pipe_sampler_view *view;
285 struct si_sampler_view *sview;
286 struct r600_texture *tex;
287
288 i = u_bit_scan(&mask);
289
290 view = textures->views.views[i];
291 assert(view);
292 sview = (struct si_sampler_view*)view;
293
294 tex = (struct r600_texture *)view->texture;
295 assert(tex->is_depth && !tex->is_flushing_texture);
296
297 si_blit_decompress_zs_in_place(sctx, tex,
298 sview->is_stencil_sampler ? PIPE_MASK_S
299 : PIPE_MASK_Z,
300 view->u.tex.first_level, view->u.tex.last_level,
301 0, util_max_layer(&tex->resource.b.b, view->u.tex.first_level));
302 }
303 }
304
305 static void si_blit_decompress_color(struct pipe_context *ctx,
306 struct r600_texture *rtex,
307 unsigned first_level, unsigned last_level,
308 unsigned first_layer, unsigned last_layer,
309 bool need_dcc_decompress)
310 {
311 struct si_context *sctx = (struct si_context *)ctx;
312 void* custom_blend;
313 unsigned layer, checked_last_layer, max_layer;
314 unsigned level_mask =
315 u_bit_consecutive(first_level, last_level - first_level + 1);
316
317 if (!need_dcc_decompress)
318 level_mask &= rtex->dirty_level_mask;
319 if (!level_mask)
320 return;
321
322 if (rtex->dcc_offset && need_dcc_decompress) {
323 custom_blend = sctx->custom_blend_dcc_decompress;
324
325 /* disable levels without DCC */
326 for (int i = first_level; i <= last_level; i++) {
327 if (!rtex->dcc_offset ||
328 !rtex->surface.level[i].dcc_enabled)
329 level_mask &= ~(1 << i);
330 }
331 } else if (rtex->fmask.size) {
332 custom_blend = sctx->custom_blend_decompress;
333 } else {
334 custom_blend = sctx->custom_blend_fastclear;
335 }
336
337 while (level_mask) {
338 unsigned level = u_bit_scan(&level_mask);
339
340 /* The smaller the mipmap level, the less layers there are
341 * as far as 3D textures are concerned. */
342 max_layer = util_max_layer(&rtex->resource.b.b, level);
343 checked_last_layer = MIN2(last_layer, max_layer);
344
345 for (layer = first_layer; layer <= checked_last_layer; layer++) {
346 struct pipe_surface *cbsurf, surf_tmpl;
347
348 surf_tmpl.format = rtex->resource.b.b.format;
349 surf_tmpl.u.tex.level = level;
350 surf_tmpl.u.tex.first_layer = layer;
351 surf_tmpl.u.tex.last_layer = layer;
352 cbsurf = ctx->create_surface(ctx, &rtex->resource.b.b, &surf_tmpl);
353
354 si_blitter_begin(ctx, SI_DECOMPRESS);
355 util_blitter_custom_color(sctx->blitter, cbsurf, custom_blend);
356 si_blitter_end(ctx);
357
358 pipe_surface_reference(&cbsurf, NULL);
359 }
360
361 /* The texture will always be dirty if some layers aren't flushed.
362 * I don't think this case occurs often though. */
363 if (first_layer == 0 && last_layer == max_layer) {
364 rtex->dirty_level_mask &= ~(1 << level);
365 }
366 }
367 }
368
369 static void
370 si_decompress_sampler_color_textures(struct si_context *sctx,
371 struct si_textures_info *textures)
372 {
373 unsigned i;
374 unsigned mask = textures->compressed_colortex_mask;
375
376 while (mask) {
377 struct pipe_sampler_view *view;
378 struct r600_texture *tex;
379
380 i = u_bit_scan(&mask);
381
382 view = textures->views.views[i];
383 assert(view);
384
385 tex = (struct r600_texture *)view->texture;
386 assert(tex->cmask.size || tex->fmask.size || tex->dcc_offset);
387
388 si_blit_decompress_color(&sctx->b.b, tex,
389 view->u.tex.first_level, view->u.tex.last_level,
390 0, util_max_layer(&tex->resource.b.b, view->u.tex.first_level),
391 false);
392 }
393 }
394
395 static void
396 si_decompress_image_color_textures(struct si_context *sctx,
397 struct si_images_info *images)
398 {
399 unsigned i;
400 unsigned mask = images->compressed_colortex_mask;
401
402 while (mask) {
403 const struct pipe_image_view *view;
404 struct r600_texture *tex;
405
406 i = u_bit_scan(&mask);
407
408 view = &images->views[i];
409 assert(view->resource->target != PIPE_BUFFER);
410
411 tex = (struct r600_texture *)view->resource;
412 if (!tex->cmask.size && !tex->fmask.size && !tex->dcc_offset)
413 continue;
414
415 si_blit_decompress_color(&sctx->b.b, tex,
416 view->u.tex.level, view->u.tex.level,
417 0, util_max_layer(&tex->resource.b.b, view->u.tex.level),
418 false);
419 }
420 }
421
422 static void si_check_render_feedback_textures(struct si_context *sctx,
423 struct si_textures_info *textures)
424 {
425 uint32_t mask = textures->views.enabled_mask;
426
427 while (mask) {
428 const struct pipe_sampler_view *view;
429 struct r600_texture *tex;
430 bool render_feedback = false;
431
432 unsigned i = u_bit_scan(&mask);
433
434 view = textures->views.views[i];
435 if(view->texture->target == PIPE_BUFFER)
436 continue;
437
438 tex = (struct r600_texture *)view->texture;
439 if (!tex->dcc_offset)
440 continue;
441
442 for (unsigned j = 0; j < sctx->framebuffer.state.nr_cbufs; ++j) {
443 struct r600_surface * surf;
444
445 if (!sctx->framebuffer.state.cbufs[j])
446 continue;
447
448 surf = (struct r600_surface*)sctx->framebuffer.state.cbufs[j];
449
450 if (tex == (struct r600_texture*)surf->base.texture &&
451 surf->base.u.tex.level >= view->u.tex.first_level &&
452 surf->base.u.tex.level <= view->u.tex.last_level &&
453 surf->base.u.tex.first_layer <= view->u.tex.last_layer &&
454 surf->base.u.tex.last_layer >= view->u.tex.first_layer)
455 render_feedback = true;
456 }
457
458 if (render_feedback) {
459 struct si_screen *screen = sctx->screen;
460 r600_texture_disable_dcc(&screen->b, tex);
461 }
462 }
463 }
464
465 static void si_check_render_feedback_images(struct si_context *sctx,
466 struct si_images_info *images)
467 {
468 uint32_t mask = images->enabled_mask;
469
470 while (mask) {
471 const struct pipe_image_view *view;
472 struct r600_texture *tex;
473 bool render_feedback = false;
474
475 unsigned i = u_bit_scan(&mask);
476
477 view = &images->views[i];
478 if (view->resource->target == PIPE_BUFFER)
479 continue;
480
481 tex = (struct r600_texture *)view->resource;
482 if (!tex->dcc_offset)
483 continue;
484
485 for (unsigned j = 0; j < sctx->framebuffer.state.nr_cbufs; ++j) {
486 struct r600_surface * surf;
487
488 if (!sctx->framebuffer.state.cbufs[j])
489 continue;
490
491 surf = (struct r600_surface*)sctx->framebuffer.state.cbufs[j];
492
493 if (tex == (struct r600_texture*)surf->base.texture &&
494 surf->base.u.tex.level == view->u.tex.level &&
495 surf->base.u.tex.first_layer <= view->u.tex.last_layer &&
496 surf->base.u.tex.last_layer >= view->u.tex.first_layer)
497 render_feedback = true;
498 }
499
500 if (render_feedback) {
501 struct si_screen *screen = sctx->screen;
502 r600_texture_disable_dcc(&screen->b, tex);
503 }
504 }
505 }
506
507 static void si_check_render_feedback(struct si_context *sctx)
508 {
509
510 if (!sctx->need_check_render_feedback)
511 return;
512
513 for (int i = 0; i < SI_NUM_SHADERS; ++i) {
514 si_check_render_feedback_images(sctx, &sctx->images[i]);
515 si_check_render_feedback_textures(sctx, &sctx->samplers[i]);
516 }
517 sctx->need_check_render_feedback = false;
518 }
519
520 static void si_decompress_textures(struct si_context *sctx, int shader_start,
521 int shader_end)
522 {
523 unsigned compressed_colortex_counter;
524
525 if (sctx->blitter->running)
526 return;
527
528 /* Update the compressed_colortex_mask if necessary. */
529 compressed_colortex_counter = p_atomic_read(&sctx->screen->b.compressed_colortex_counter);
530 if (compressed_colortex_counter != sctx->b.last_compressed_colortex_counter) {
531 sctx->b.last_compressed_colortex_counter = compressed_colortex_counter;
532 si_update_compressed_colortex_masks(sctx);
533 }
534
535 /* Flush depth textures which need to be flushed. */
536 for (int i = shader_start; i < shader_end; i++) {
537 if (sctx->samplers[i].depth_texture_mask) {
538 si_flush_depth_textures(sctx, &sctx->samplers[i]);
539 }
540 if (sctx->samplers[i].compressed_colortex_mask) {
541 si_decompress_sampler_color_textures(sctx, &sctx->samplers[i]);
542 }
543 if (sctx->images[i].compressed_colortex_mask) {
544 si_decompress_image_color_textures(sctx, &sctx->images[i]);
545 }
546 }
547
548 si_check_render_feedback(sctx);
549 }
550
551 void si_decompress_graphics_textures(struct si_context *sctx)
552 {
553 si_decompress_textures(sctx, 0, SI_NUM_GRAPHICS_SHADERS);
554 }
555
556 void si_decompress_compute_textures(struct si_context *sctx)
557 {
558 si_decompress_textures(sctx, SI_NUM_GRAPHICS_SHADERS, SI_NUM_SHADERS);
559 }
560
561 static void si_clear(struct pipe_context *ctx, unsigned buffers,
562 const union pipe_color_union *color,
563 double depth, unsigned stencil)
564 {
565 struct si_context *sctx = (struct si_context *)ctx;
566 struct pipe_framebuffer_state *fb = &sctx->framebuffer.state;
567 struct pipe_surface *zsbuf = fb->zsbuf;
568 struct r600_texture *zstex =
569 zsbuf ? (struct r600_texture*)zsbuf->texture : NULL;
570
571 if (buffers & PIPE_CLEAR_COLOR) {
572 evergreen_do_fast_color_clear(&sctx->b, fb,
573 &sctx->framebuffer.atom, &buffers,
574 &sctx->framebuffer.dirty_cbufs,
575 color);
576 if (!buffers)
577 return; /* all buffers have been fast cleared */
578 }
579
580 if (buffers & PIPE_CLEAR_COLOR) {
581 int i;
582
583 /* These buffers cannot use fast clear, make sure to disable expansion. */
584 for (i = 0; i < fb->nr_cbufs; i++) {
585 struct r600_texture *tex;
586
587 /* If not clearing this buffer, skip. */
588 if (!(buffers & (PIPE_CLEAR_COLOR0 << i)))
589 continue;
590
591 if (!fb->cbufs[i])
592 continue;
593
594 tex = (struct r600_texture *)fb->cbufs[i]->texture;
595 if (tex->fmask.size == 0)
596 tex->dirty_level_mask &= ~(1 << fb->cbufs[i]->u.tex.level);
597 }
598 }
599
600 if (zstex && zstex->htile_buffer &&
601 zsbuf->u.tex.level == 0 &&
602 zsbuf->u.tex.first_layer == 0 &&
603 zsbuf->u.tex.last_layer == util_max_layer(&zstex->resource.b.b, 0)) {
604 if (buffers & PIPE_CLEAR_DEPTH) {
605 /* Need to disable EXPCLEAR temporarily if clearing
606 * to a new value. */
607 if (!zstex->depth_cleared || zstex->depth_clear_value != depth) {
608 sctx->db_depth_disable_expclear = true;
609 }
610
611 zstex->depth_clear_value = depth;
612 sctx->framebuffer.dirty_zsbuf = true;
613 si_mark_atom_dirty(sctx, &sctx->framebuffer.atom); /* updates DB_DEPTH_CLEAR */
614 sctx->db_depth_clear = true;
615 si_mark_atom_dirty(sctx, &sctx->db_render_state);
616 }
617
618 if (buffers & PIPE_CLEAR_STENCIL) {
619 stencil &= 0xff;
620
621 /* Need to disable EXPCLEAR temporarily if clearing
622 * to a new value. */
623 if (!zstex->stencil_cleared || zstex->stencil_clear_value != stencil) {
624 sctx->db_stencil_disable_expclear = true;
625 }
626
627 zstex->stencil_clear_value = stencil;
628 sctx->framebuffer.dirty_zsbuf = true;
629 si_mark_atom_dirty(sctx, &sctx->framebuffer.atom); /* updates DB_STENCIL_CLEAR */
630 sctx->db_stencil_clear = true;
631 si_mark_atom_dirty(sctx, &sctx->db_render_state);
632 }
633 }
634
635 si_blitter_begin(ctx, SI_CLEAR);
636 util_blitter_clear(sctx->blitter, fb->width, fb->height,
637 util_framebuffer_get_num_layers(fb),
638 buffers, color, depth, stencil);
639 si_blitter_end(ctx);
640
641 if (sctx->db_depth_clear) {
642 sctx->db_depth_clear = false;
643 sctx->db_depth_disable_expclear = false;
644 zstex->depth_cleared = true;
645 si_mark_atom_dirty(sctx, &sctx->db_render_state);
646 }
647
648 if (sctx->db_stencil_clear) {
649 sctx->db_stencil_clear = false;
650 sctx->db_stencil_disable_expclear = false;
651 zstex->stencil_cleared = true;
652 si_mark_atom_dirty(sctx, &sctx->db_render_state);
653 }
654 }
655
656 static void si_clear_render_target(struct pipe_context *ctx,
657 struct pipe_surface *dst,
658 const union pipe_color_union *color,
659 unsigned dstx, unsigned dsty,
660 unsigned width, unsigned height)
661 {
662 struct si_context *sctx = (struct si_context *)ctx;
663
664 si_blitter_begin(ctx, SI_CLEAR_SURFACE);
665 util_blitter_clear_render_target(sctx->blitter, dst, color,
666 dstx, dsty, width, height);
667 si_blitter_end(ctx);
668 }
669
670 static void si_clear_depth_stencil(struct pipe_context *ctx,
671 struct pipe_surface *dst,
672 unsigned clear_flags,
673 double depth,
674 unsigned stencil,
675 unsigned dstx, unsigned dsty,
676 unsigned width, unsigned height)
677 {
678 struct si_context *sctx = (struct si_context *)ctx;
679
680 si_blitter_begin(ctx, SI_CLEAR_SURFACE);
681 util_blitter_clear_depth_stencil(sctx->blitter, dst, clear_flags, depth, stencil,
682 dstx, dsty, width, height);
683 si_blitter_end(ctx);
684 }
685
686 /* Helper for decompressing a portion of a color or depth resource before
687 * blitting if any decompression is needed.
688 * The driver doesn't decompress resources automatically while u_blitter is
689 * rendering. */
690 static void si_decompress_subresource(struct pipe_context *ctx,
691 struct pipe_resource *tex,
692 unsigned planes, unsigned level,
693 unsigned first_layer, unsigned last_layer)
694 {
695 struct si_context *sctx = (struct si_context *)ctx;
696 struct r600_texture *rtex = (struct r600_texture*)tex;
697
698 if (rtex->is_depth && !rtex->is_flushing_texture) {
699 planes &= PIPE_MASK_Z | PIPE_MASK_S;
700
701 if (!(rtex->surface.flags & RADEON_SURF_SBUFFER))
702 planes &= ~PIPE_MASK_S;
703
704 si_blit_decompress_zs_in_place(sctx, rtex, planes,
705 level, level,
706 first_layer, last_layer);
707 } else if (rtex->fmask.size || rtex->cmask.size || rtex->dcc_offset) {
708 si_blit_decompress_color(ctx, rtex, level, level,
709 first_layer, last_layer, false);
710 }
711 }
712
713 struct texture_orig_info {
714 unsigned format;
715 unsigned width0;
716 unsigned height0;
717 unsigned npix_x;
718 unsigned npix_y;
719 unsigned npix0_x;
720 unsigned npix0_y;
721 };
722
723 void si_resource_copy_region(struct pipe_context *ctx,
724 struct pipe_resource *dst,
725 unsigned dst_level,
726 unsigned dstx, unsigned dsty, unsigned dstz,
727 struct pipe_resource *src,
728 unsigned src_level,
729 const struct pipe_box *src_box)
730 {
731 struct si_context *sctx = (struct si_context *)ctx;
732 struct pipe_surface *dst_view, dst_templ;
733 struct pipe_sampler_view src_templ, *src_view;
734 unsigned dst_width, dst_height, src_width0, src_height0;
735 unsigned src_force_level = 0;
736 struct pipe_box sbox, dstbox;
737
738 /* Handle buffers first. */
739 if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
740 si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width);
741 return;
742 }
743
744 assert(u_max_sample(dst) == u_max_sample(src));
745
746 /* The driver doesn't decompress resources automatically while
747 * u_blitter is rendering. */
748 si_decompress_subresource(ctx, src, PIPE_MASK_RGBAZS, src_level,
749 src_box->z, src_box->z + src_box->depth - 1);
750
751 dst_width = u_minify(dst->width0, dst_level);
752 dst_height = u_minify(dst->height0, dst_level);
753 src_width0 = src->width0;
754 src_height0 = src->height0;
755
756 util_blitter_default_dst_texture(&dst_templ, dst, dst_level, dstz);
757 util_blitter_default_src_texture(&src_templ, src, src_level);
758
759 if (util_format_is_compressed(src->format) ||
760 util_format_is_compressed(dst->format)) {
761 unsigned blocksize = util_format_get_blocksize(src->format);
762
763 if (blocksize == 8)
764 src_templ.format = PIPE_FORMAT_R16G16B16A16_UINT; /* 64-bit block */
765 else
766 src_templ.format = PIPE_FORMAT_R32G32B32A32_UINT; /* 128-bit block */
767 dst_templ.format = src_templ.format;
768
769 dst_width = util_format_get_nblocksx(dst->format, dst_width);
770 dst_height = util_format_get_nblocksy(dst->format, dst_height);
771 src_width0 = util_format_get_nblocksx(src->format, src_width0);
772 src_height0 = util_format_get_nblocksy(src->format, src_height0);
773
774 dstx = util_format_get_nblocksx(dst->format, dstx);
775 dsty = util_format_get_nblocksy(dst->format, dsty);
776
777 sbox.x = util_format_get_nblocksx(src->format, src_box->x);
778 sbox.y = util_format_get_nblocksy(src->format, src_box->y);
779 sbox.z = src_box->z;
780 sbox.width = util_format_get_nblocksx(src->format, src_box->width);
781 sbox.height = util_format_get_nblocksy(src->format, src_box->height);
782 sbox.depth = src_box->depth;
783 src_box = &sbox;
784
785 src_force_level = src_level;
786 } else if (!util_blitter_is_copy_supported(sctx->blitter, dst, src) ||
787 /* also *8_SNORM has precision issues, use UNORM instead */
788 util_format_is_snorm8(src->format)) {
789 if (util_format_is_subsampled_422(src->format)) {
790 src_templ.format = PIPE_FORMAT_R8G8B8A8_UINT;
791 dst_templ.format = PIPE_FORMAT_R8G8B8A8_UINT;
792
793 dst_width = util_format_get_nblocksx(dst->format, dst_width);
794 src_width0 = util_format_get_nblocksx(src->format, src_width0);
795
796 dstx = util_format_get_nblocksx(dst->format, dstx);
797
798 sbox = *src_box;
799 sbox.x = util_format_get_nblocksx(src->format, src_box->x);
800 sbox.width = util_format_get_nblocksx(src->format, src_box->width);
801 src_box = &sbox;
802 } else {
803 unsigned blocksize = util_format_get_blocksize(src->format);
804
805 switch (blocksize) {
806 case 1:
807 dst_templ.format = PIPE_FORMAT_R8_UNORM;
808 src_templ.format = PIPE_FORMAT_R8_UNORM;
809 break;
810 case 2:
811 dst_templ.format = PIPE_FORMAT_R8G8_UNORM;
812 src_templ.format = PIPE_FORMAT_R8G8_UNORM;
813 break;
814 case 4:
815 dst_templ.format = PIPE_FORMAT_R8G8B8A8_UNORM;
816 src_templ.format = PIPE_FORMAT_R8G8B8A8_UNORM;
817 break;
818 case 8:
819 dst_templ.format = PIPE_FORMAT_R16G16B16A16_UINT;
820 src_templ.format = PIPE_FORMAT_R16G16B16A16_UINT;
821 break;
822 case 16:
823 dst_templ.format = PIPE_FORMAT_R32G32B32A32_UINT;
824 src_templ.format = PIPE_FORMAT_R32G32B32A32_UINT;
825 break;
826 default:
827 fprintf(stderr, "Unhandled format %s with blocksize %u\n",
828 util_format_short_name(src->format), blocksize);
829 assert(0);
830 }
831 }
832 }
833
834 /* Initialize the surface. */
835 dst_view = r600_create_surface_custom(ctx, dst, &dst_templ,
836 dst_width, dst_height);
837
838 /* Initialize the sampler view. */
839 src_view = si_create_sampler_view_custom(ctx, src, &src_templ,
840 src_width0, src_height0,
841 src_force_level);
842
843 u_box_3d(dstx, dsty, dstz, abs(src_box->width), abs(src_box->height),
844 abs(src_box->depth), &dstbox);
845
846 /* Copy. */
847 si_blitter_begin(ctx, SI_COPY);
848 util_blitter_blit_generic(sctx->blitter, dst_view, &dstbox,
849 src_view, src_box, src_width0, src_height0,
850 PIPE_MASK_RGBAZS, PIPE_TEX_FILTER_NEAREST, NULL,
851 FALSE);
852 si_blitter_end(ctx);
853
854 pipe_surface_reference(&dst_view, NULL);
855 pipe_sampler_view_reference(&src_view, NULL);
856 }
857
858 static bool do_hardware_msaa_resolve(struct pipe_context *ctx,
859 const struct pipe_blit_info *info)
860 {
861 struct si_context *sctx = (struct si_context*)ctx;
862 struct r600_texture *src = (struct r600_texture*)info->src.resource;
863 struct r600_texture *dst = (struct r600_texture*)info->dst.resource;
864 unsigned dst_width = u_minify(info->dst.resource->width0, info->dst.level);
865 unsigned dst_height = u_minify(info->dst.resource->height0, info->dst.level);
866 enum pipe_format format = info->src.format;
867 unsigned sample_mask = ~0;
868 struct pipe_resource *tmp, templ;
869 struct pipe_blit_info blit;
870
871 /* Check basic requirements for hw resolve. */
872 if (!(info->src.resource->nr_samples > 1 &&
873 info->dst.resource->nr_samples <= 1 &&
874 !util_format_is_pure_integer(format) &&
875 !util_format_is_depth_or_stencil(format) &&
876 util_max_layer(info->src.resource, 0) == 0))
877 return false;
878
879 /* Hardware MSAA resolve doesn't work if SPI format = NORM16_ABGR and
880 * the format is R16G16. Use R16A16, which does work.
881 */
882 if (format == PIPE_FORMAT_R16G16_UNORM)
883 format = PIPE_FORMAT_R16A16_UNORM;
884 if (format == PIPE_FORMAT_R16G16_SNORM)
885 format = PIPE_FORMAT_R16A16_SNORM;
886
887 /* Check the remaining requirements for hw resolve. */
888 if (util_max_layer(info->dst.resource, info->dst.level) == 0 &&
889 !info->scissor_enable &&
890 (info->mask & PIPE_MASK_RGBA) == PIPE_MASK_RGBA &&
891 util_is_format_compatible(util_format_description(info->src.format),
892 util_format_description(info->dst.format)) &&
893 dst_width == info->src.resource->width0 &&
894 dst_height == info->src.resource->height0 &&
895 info->dst.box.x == 0 &&
896 info->dst.box.y == 0 &&
897 info->dst.box.width == dst_width &&
898 info->dst.box.height == dst_height &&
899 info->dst.box.depth == 1 &&
900 info->src.box.x == 0 &&
901 info->src.box.y == 0 &&
902 info->src.box.width == dst_width &&
903 info->src.box.height == dst_height &&
904 info->src.box.depth == 1 &&
905 dst->surface.level[info->dst.level].mode >= RADEON_SURF_MODE_1D &&
906 src->surface.micro_tile_mode == dst->surface.micro_tile_mode &&
907 (!dst->cmask.size || !dst->dirty_level_mask)) { /* dst cannot be fast-cleared */
908 /* Resolving into a surface with DCC is unsupported. Since
909 * it's being overwritten anyway, clear it to uncompressed.
910 * This is still the fastest codepath even with this clear.
911 */
912 if (dst->dcc_offset &&
913 dst->surface.level[info->dst.level].dcc_enabled) {
914 vi_dcc_clear_level(&sctx->b, dst, info->dst.level,
915 0xFFFFFFFF);
916 dst->dirty_level_mask &= ~(1 << info->dst.level);
917 }
918
919 /* Resolve directly from src to dst. */
920 si_blitter_begin(ctx, SI_COLOR_RESOLVE |
921 (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
922 util_blitter_custom_resolve_color(sctx->blitter,
923 info->dst.resource, info->dst.level,
924 info->dst.box.z,
925 info->src.resource, info->src.box.z,
926 sample_mask, sctx->custom_blend_resolve,
927 format);
928 si_blitter_end(ctx);
929 return true;
930 }
931
932 /* Shader-based resolve is VERY SLOW. Instead, resolve into
933 * a temporary texture and blit.
934 */
935 memset(&templ, 0, sizeof(templ));
936 templ.target = PIPE_TEXTURE_2D;
937 templ.format = info->src.resource->format;
938 templ.width0 = info->src.resource->width0;
939 templ.height0 = info->src.resource->height0;
940 templ.depth0 = 1;
941 templ.array_size = 1;
942 templ.usage = PIPE_USAGE_DEFAULT;
943 templ.flags = R600_RESOURCE_FLAG_FORCE_TILING |
944 R600_RESOURCE_FLAG_DISABLE_DCC;
945
946 tmp = ctx->screen->resource_create(ctx->screen, &templ);
947 if (!tmp)
948 return false;
949
950 assert(src->surface.micro_tile_mode ==
951 ((struct r600_texture*)tmp)->surface.micro_tile_mode);
952
953 /* resolve */
954 si_blitter_begin(ctx, SI_COLOR_RESOLVE |
955 (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
956 util_blitter_custom_resolve_color(sctx->blitter, tmp, 0, 0,
957 info->src.resource, info->src.box.z,
958 sample_mask, sctx->custom_blend_resolve,
959 format);
960 si_blitter_end(ctx);
961
962 /* blit */
963 blit = *info;
964 blit.src.resource = tmp;
965 blit.src.box.z = 0;
966
967 si_blitter_begin(ctx, SI_BLIT |
968 (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
969 util_blitter_blit(sctx->blitter, &blit);
970 si_blitter_end(ctx);
971
972 pipe_resource_reference(&tmp, NULL);
973 return true;
974 }
975
976 static void si_blit(struct pipe_context *ctx,
977 const struct pipe_blit_info *info)
978 {
979 struct si_context *sctx = (struct si_context*)ctx;
980
981 if (do_hardware_msaa_resolve(ctx, info)) {
982 return;
983 }
984
985 assert(util_blitter_is_blit_supported(sctx->blitter, info));
986
987 /* The driver doesn't decompress resources automatically while
988 * u_blitter is rendering. */
989 si_decompress_subresource(ctx, info->src.resource, info->mask,
990 info->src.level,
991 info->src.box.z,
992 info->src.box.z + info->src.box.depth - 1);
993
994 if (sctx->screen->b.debug_flags & DBG_FORCE_DMA &&
995 util_try_blit_via_copy_region(ctx, info))
996 return;
997
998 si_blitter_begin(ctx, SI_BLIT |
999 (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
1000 util_blitter_blit(sctx->blitter, info);
1001 si_blitter_end(ctx);
1002 }
1003
1004 static void si_flush_resource(struct pipe_context *ctx,
1005 struct pipe_resource *res)
1006 {
1007 struct r600_texture *rtex = (struct r600_texture*)res;
1008
1009 assert(res->target != PIPE_BUFFER);
1010
1011 if (!rtex->is_depth && (rtex->cmask.size || rtex->dcc_offset)) {
1012 si_blit_decompress_color(ctx, rtex, 0, res->last_level,
1013 0, util_max_layer(res, 0), false);
1014 }
1015 }
1016
1017 static void si_decompress_dcc(struct pipe_context *ctx,
1018 struct r600_texture *rtex)
1019 {
1020 if (!rtex->dcc_offset)
1021 return;
1022
1023 si_blit_decompress_color(ctx, rtex, 0, rtex->resource.b.b.last_level,
1024 0, util_max_layer(&rtex->resource.b.b, 0),
1025 true);
1026 }
1027
1028 static void si_pipe_clear_buffer(struct pipe_context *ctx,
1029 struct pipe_resource *dst,
1030 unsigned offset, unsigned size,
1031 const void *clear_value_ptr,
1032 int clear_value_size)
1033 {
1034 struct si_context *sctx = (struct si_context*)ctx;
1035 uint32_t dword_value;
1036 unsigned i;
1037
1038 assert(offset % clear_value_size == 0);
1039 assert(size % clear_value_size == 0);
1040
1041 if (clear_value_size > 4) {
1042 const uint32_t *u32 = clear_value_ptr;
1043 bool clear_dword_duplicated = true;
1044
1045 /* See if we can lower large fills to dword fills. */
1046 for (i = 1; i < clear_value_size / 4; i++)
1047 if (u32[0] != u32[i]) {
1048 clear_dword_duplicated = false;
1049 break;
1050 }
1051
1052 if (!clear_dword_duplicated) {
1053 /* Use transform feedback for 64-bit, 96-bit, and
1054 * 128-bit fills.
1055 */
1056 union pipe_color_union clear_value;
1057
1058 memcpy(&clear_value, clear_value_ptr, clear_value_size);
1059 si_blitter_begin(ctx, SI_DISABLE_RENDER_COND);
1060 util_blitter_clear_buffer(sctx->blitter, dst, offset,
1061 size, clear_value_size / 4,
1062 &clear_value);
1063 si_blitter_end(ctx);
1064 return;
1065 }
1066 }
1067
1068 /* Expand the clear value to a dword. */
1069 switch (clear_value_size) {
1070 case 1:
1071 dword_value = *(uint8_t*)clear_value_ptr;
1072 dword_value |= (dword_value << 8) |
1073 (dword_value << 16) |
1074 (dword_value << 24);
1075 break;
1076 case 2:
1077 dword_value = *(uint16_t*)clear_value_ptr;
1078 dword_value |= dword_value << 16;
1079 break;
1080 default:
1081 dword_value = *(uint32_t*)clear_value_ptr;
1082 }
1083
1084 sctx->b.clear_buffer(ctx, dst, offset, size, dword_value,
1085 R600_COHERENCY_SHADER);
1086 }
1087
1088 void si_init_blit_functions(struct si_context *sctx)
1089 {
1090 sctx->b.b.clear = si_clear;
1091 sctx->b.b.clear_buffer = si_pipe_clear_buffer;
1092 sctx->b.b.clear_render_target = si_clear_render_target;
1093 sctx->b.b.clear_depth_stencil = si_clear_depth_stencil;
1094 sctx->b.b.resource_copy_region = si_resource_copy_region;
1095 sctx->b.b.blit = si_blit;
1096 sctx->b.b.flush_resource = si_flush_resource;
1097 sctx->b.blit_decompress_depth = si_blit_decompress_depth;
1098 sctx->b.decompress_dcc = si_decompress_dcc;
1099 }