projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
radeonsi: boolean -> bool, TRUE -> true, FALSE -> false
[mesa.git] / src / gallium / drivers / radeonsi / si_blit.c
1 /*
2 * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "si_pipe.h"
25 #include "sid.h"
26 #include "util/u_format.h"
27 #include "util/u_surface.h"
28
29 enum si_blitter_op /* bitmask */
30 {
31 SI_SAVE_TEXTURES = 1,
32 SI_SAVE_FRAMEBUFFER = 2,
33 SI_SAVE_FRAGMENT_STATE = 4,
34 SI_DISABLE_RENDER_COND = 8,
35
36 SI_CLEAR = SI_SAVE_FRAGMENT_STATE,
37
38 SI_CLEAR_SURFACE = SI_SAVE_FRAMEBUFFER | SI_SAVE_FRAGMENT_STATE,
39
40 SI_COPY = SI_SAVE_FRAMEBUFFER | SI_SAVE_TEXTURES |
41 SI_SAVE_FRAGMENT_STATE | SI_DISABLE_RENDER_COND,
42
43 SI_BLIT = SI_SAVE_FRAMEBUFFER | SI_SAVE_TEXTURES |
44 SI_SAVE_FRAGMENT_STATE,
45
46 SI_DECOMPRESS = SI_SAVE_FRAMEBUFFER | SI_SAVE_FRAGMENT_STATE |
47 SI_DISABLE_RENDER_COND,
48
49 SI_COLOR_RESOLVE = SI_SAVE_FRAMEBUFFER | SI_SAVE_FRAGMENT_STATE
50 };
51
52 static void si_blitter_begin(struct pipe_context *ctx, enum si_blitter_op op)
53 {
54 struct si_context *sctx = (struct si_context *)ctx;
55
56 util_blitter_save_vertex_buffer_slot(sctx->blitter, sctx->vertex_buffer);
57 util_blitter_save_vertex_elements(sctx->blitter, sctx->vertex_elements);
58 util_blitter_save_vertex_shader(sctx->blitter, sctx->vs_shader.cso);
59 util_blitter_save_tessctrl_shader(sctx->blitter, sctx->tcs_shader.cso);
60 util_blitter_save_tesseval_shader(sctx->blitter, sctx->tes_shader.cso);
61 util_blitter_save_geometry_shader(sctx->blitter, sctx->gs_shader.cso);
62 util_blitter_save_so_targets(sctx->blitter, sctx->b.streamout.num_targets,
63 (struct pipe_stream_output_target**)sctx->b.streamout.targets);
64 util_blitter_save_rasterizer(sctx->blitter, sctx->queued.named.rasterizer);
65
66 if (op & SI_SAVE_FRAGMENT_STATE) {
67 util_blitter_save_blend(sctx->blitter, sctx->queued.named.blend);
68 util_blitter_save_depth_stencil_alpha(sctx->blitter, sctx->queued.named.dsa);
69 util_blitter_save_stencil_ref(sctx->blitter, &sctx->stencil_ref.state);
70 util_blitter_save_fragment_shader(sctx->blitter, sctx->ps_shader.cso);
71 util_blitter_save_sample_mask(sctx->blitter, sctx->sample_mask.sample_mask);
72 util_blitter_save_viewport(sctx->blitter, &sctx->b.viewports.states[0]);
73 util_blitter_save_scissor(sctx->blitter, &sctx->b.scissors.states[0]);
74 }
75
76 if (op & SI_SAVE_FRAMEBUFFER)
77 util_blitter_save_framebuffer(sctx->blitter, &sctx->framebuffer.state);
78
79 if (op & SI_SAVE_TEXTURES) {
80 util_blitter_save_fragment_sampler_states(
81 sctx->blitter, 2,
82 sctx->samplers[PIPE_SHADER_FRAGMENT].views.sampler_states);
83
84 util_blitter_save_fragment_sampler_views(sctx->blitter, 2,
85 sctx->samplers[PIPE_SHADER_FRAGMENT].views.views);
86 }
87
88 if (op & SI_DISABLE_RENDER_COND)
89 sctx->b.render_cond_force_off = true;
90 }
91
92 static void si_blitter_end(struct pipe_context *ctx)
93 {
94 struct si_context *sctx = (struct si_context *)ctx;
95
96 sctx->b.render_cond_force_off = false;
97 }
98
99 static unsigned u_max_sample(struct pipe_resource *r)
100 {
101 return r->nr_samples ? r->nr_samples - 1 : 0;
102 }
103
104 static void si_blit_decompress_depth(struct pipe_context *ctx,
105 struct r600_texture *texture,
106 struct r600_texture *staging,
107 unsigned first_level, unsigned last_level,
108 unsigned first_layer, unsigned last_layer,
109 unsigned first_sample, unsigned last_sample)
110 {
111 struct si_context *sctx = (struct si_context *)ctx;
112 unsigned layer, level, sample, checked_last_layer, max_layer;
113 float depth = 1.0f;
114 const struct util_format_description *desc;
115
116 assert(staging != NULL && "use si_blit_decompress_zs_in_place instead");
117
118 desc = util_format_description(staging->resource.b.b.format);
119
120 if (util_format_has_depth(desc))
121 sctx->dbcb_depth_copy_enabled = true;
122 if (util_format_has_stencil(desc))
123 sctx->dbcb_stencil_copy_enabled = true;
124
125 assert(sctx->dbcb_depth_copy_enabled || sctx->dbcb_stencil_copy_enabled);
126
127 for (level = first_level; level <= last_level; level++) {
128 /* The smaller the mipmap level, the less layers there are
129 * as far as 3D textures are concerned. */
130 max_layer = util_max_layer(&texture->resource.b.b, level);
131 checked_last_layer = MIN2(last_layer, max_layer);
132
133 for (layer = first_layer; layer <= checked_last_layer; layer++) {
134 for (sample = first_sample; sample <= last_sample; sample++) {
135 struct pipe_surface *zsurf, *cbsurf, surf_tmpl;
136
137 sctx->dbcb_copy_sample = sample;
138 si_mark_atom_dirty(sctx, &sctx->db_render_state);
139
140 surf_tmpl.format = texture->resource.b.b.format;
141 surf_tmpl.u.tex.level = level;
142 surf_tmpl.u.tex.first_layer = layer;
143 surf_tmpl.u.tex.last_layer = layer;
144
145 zsurf = ctx->create_surface(ctx, &texture->resource.b.b, &surf_tmpl);
146
147 surf_tmpl.format = staging->resource.b.b.format;
148 cbsurf = ctx->create_surface(ctx,
149 (struct pipe_resource*)staging, &surf_tmpl);
150
151 si_blitter_begin(ctx, SI_DECOMPRESS);
152 util_blitter_custom_depth_stencil(sctx->blitter, zsurf, cbsurf, 1 << sample,
153 sctx->custom_dsa_flush, depth);
154 si_blitter_end(ctx);
155
156 pipe_surface_reference(&zsurf, NULL);
157 pipe_surface_reference(&cbsurf, NULL);
158 }
159 }
160 }
161
162 sctx->dbcb_depth_copy_enabled = false;
163 sctx->dbcb_stencil_copy_enabled = false;
164 si_mark_atom_dirty(sctx, &sctx->db_render_state);
165 }
166
167 /* Helper function for si_blit_decompress_zs_in_place.
168 */
169 static void
170 si_blit_decompress_zs_planes_in_place(struct si_context *sctx,
171 struct r600_texture *texture,
172 unsigned planes, unsigned level_mask,
173 unsigned first_layer, unsigned last_layer)
174 {
175 struct pipe_surface *zsurf, surf_tmpl = {{0}};
176 unsigned layer, max_layer, checked_last_layer;
177 unsigned fully_decompressed_mask = 0;
178
179 if (!level_mask)
180 return;
181
182 if (planes & PIPE_MASK_S)
183 sctx->db_flush_stencil_inplace = true;
184 if (planes & PIPE_MASK_Z)
185 sctx->db_flush_depth_inplace = true;
186 si_mark_atom_dirty(sctx, &sctx->db_render_state);
187
188 surf_tmpl.format = texture->resource.b.b.format;
189
190 while (level_mask) {
191 unsigned level = u_bit_scan(&level_mask);
192
193 surf_tmpl.u.tex.level = level;
194
195 /* The smaller the mipmap level, the less layers there are
196 * as far as 3D textures are concerned. */
197 max_layer = util_max_layer(&texture->resource.b.b, level);
198 checked_last_layer = MIN2(last_layer, max_layer);
199
200 for (layer = first_layer; layer <= checked_last_layer; layer++) {
201 surf_tmpl.u.tex.first_layer = layer;
202 surf_tmpl.u.tex.last_layer = layer;
203
204 zsurf = sctx->b.b.create_surface(&sctx->b.b, &texture->resource.b.b, &surf_tmpl);
205
206 si_blitter_begin(&sctx->b.b, SI_DECOMPRESS);
207 util_blitter_custom_depth_stencil(sctx->blitter, zsurf, NULL, ~0,
208 sctx->custom_dsa_flush,
209 1.0f);
210 si_blitter_end(&sctx->b.b);
211
212 pipe_surface_reference(&zsurf, NULL);
213 }
214
215 /* The texture will always be dirty if some layers aren't flushed.
216 * I don't think this case occurs often though. */
217 if (first_layer == 0 && last_layer == max_layer) {
218 fully_decompressed_mask |= 1u << level;
219 }
220 }
221
222 if (planes & PIPE_MASK_Z)
223 texture->dirty_level_mask &= ~fully_decompressed_mask;
224 if (planes & PIPE_MASK_S)
225 texture->stencil_dirty_level_mask &= ~fully_decompressed_mask;
226
227 sctx->db_flush_depth_inplace = false;
228 sctx->db_flush_stencil_inplace = false;
229 si_mark_atom_dirty(sctx, &sctx->db_render_state);
230 }
231
232 /* Decompress Z and/or S planes in place, depending on mask.
233 */
234 static void
235 si_blit_decompress_zs_in_place(struct si_context *sctx,
236 struct r600_texture *texture,
237 unsigned planes,
238 unsigned first_level, unsigned last_level,
239 unsigned first_layer, unsigned last_layer)
240 {
241 unsigned level_mask =
242 u_bit_consecutive(first_level, last_level - first_level + 1);
243 unsigned cur_level_mask;
244
245 /* First, do combined Z & S decompresses for levels that need it. */
246 if (planes == (PIPE_MASK_Z | PIPE_MASK_S)) {
247 cur_level_mask =
248 level_mask &
249 texture->dirty_level_mask &
250 texture->stencil_dirty_level_mask;
251 si_blit_decompress_zs_planes_in_place(
252 sctx, texture, PIPE_MASK_Z | PIPE_MASK_S,
253 cur_level_mask,
254 first_layer, last_layer);
255 level_mask &= ~cur_level_mask;
256 }
257
258 /* Now do separate Z and S decompresses. */
259 if (planes & PIPE_MASK_Z) {
260 cur_level_mask = level_mask & texture->dirty_level_mask;
261 si_blit_decompress_zs_planes_in_place(
262 sctx, texture, PIPE_MASK_Z,
263 cur_level_mask,
264 first_layer, last_layer);
265 level_mask &= ~cur_level_mask;
266 }
267
268 if (planes & PIPE_MASK_S) {
269 cur_level_mask = level_mask & texture->stencil_dirty_level_mask;
270 si_blit_decompress_zs_planes_in_place(
271 sctx, texture, PIPE_MASK_S,
272 cur_level_mask,
273 first_layer, last_layer);
274 }
275 }
276
277 static void
278 si_flush_depth_textures(struct si_context *sctx,
279 struct si_textures_info *textures)
280 {
281 unsigned i;
282 unsigned mask = textures->depth_texture_mask;
283
284 while (mask) {
285 struct pipe_sampler_view *view;
286 struct si_sampler_view *sview;
287 struct r600_texture *tex;
288
289 i = u_bit_scan(&mask);
290
291 view = textures->views.views[i];
292 assert(view);
293 sview = (struct si_sampler_view*)view;
294
295 tex = (struct r600_texture *)view->texture;
296 assert(tex->is_depth && !tex->is_flushing_texture);
297
298 si_blit_decompress_zs_in_place(sctx, tex,
299 sview->is_stencil_sampler ? PIPE_MASK_S
300 : PIPE_MASK_Z,
301 view->u.tex.first_level, view->u.tex.last_level,
302 0, util_max_layer(&tex->resource.b.b, view->u.tex.first_level));
303 }
304 }
305
306 static void si_blit_decompress_color(struct pipe_context *ctx,
307 struct r600_texture *rtex,
308 unsigned first_level, unsigned last_level,
309 unsigned first_layer, unsigned last_layer,
310 bool need_dcc_decompress)
311 {
312 struct si_context *sctx = (struct si_context *)ctx;
313 void* custom_blend;
314 unsigned layer, checked_last_layer, max_layer;
315 unsigned level_mask =
316 u_bit_consecutive(first_level, last_level - first_level + 1);
317
318 if (!need_dcc_decompress)
319 level_mask &= rtex->dirty_level_mask;
320 if (!level_mask)
321 return;
322
323 if (rtex->dcc_offset && need_dcc_decompress) {
324 custom_blend = sctx->custom_blend_dcc_decompress;
325
326 /* disable levels without DCC */
327 for (int i = first_level; i <= last_level; i++) {
328 if (!rtex->dcc_offset ||
329 !rtex->surface.level[i].dcc_enabled)
330 level_mask &= ~(1 << i);
331 }
332 } else if (rtex->fmask.size) {
333 custom_blend = sctx->custom_blend_decompress;
334 } else {
335 custom_blend = sctx->custom_blend_fastclear;
336 }
337
338 while (level_mask) {
339 unsigned level = u_bit_scan(&level_mask);
340
341 /* The smaller the mipmap level, the less layers there are
342 * as far as 3D textures are concerned. */
343 max_layer = util_max_layer(&rtex->resource.b.b, level);
344 checked_last_layer = MIN2(last_layer, max_layer);
345
346 for (layer = first_layer; layer <= checked_last_layer; layer++) {
347 struct pipe_surface *cbsurf, surf_tmpl;
348
349 surf_tmpl.format = rtex->resource.b.b.format;
350 surf_tmpl.u.tex.level = level;
351 surf_tmpl.u.tex.first_layer = layer;
352 surf_tmpl.u.tex.last_layer = layer;
353 cbsurf = ctx->create_surface(ctx, &rtex->resource.b.b, &surf_tmpl);
354
355 si_blitter_begin(ctx, SI_DECOMPRESS);
356 util_blitter_custom_color(sctx->blitter, cbsurf, custom_blend);
357 si_blitter_end(ctx);
358
359 pipe_surface_reference(&cbsurf, NULL);
360 }
361
362 /* The texture will always be dirty if some layers aren't flushed.
363 * I don't think this case occurs often though. */
364 if (first_layer == 0 && last_layer == max_layer) {
365 rtex->dirty_level_mask &= ~(1 << level);
366 }
367 }
368 }
369
370 static void
371 si_decompress_sampler_color_textures(struct si_context *sctx,
372 struct si_textures_info *textures)
373 {
374 unsigned i;
375 unsigned mask = textures->compressed_colortex_mask;
376
377 while (mask) {
378 struct pipe_sampler_view *view;
379 struct r600_texture *tex;
380
381 i = u_bit_scan(&mask);
382
383 view = textures->views.views[i];
384 assert(view);
385
386 tex = (struct r600_texture *)view->texture;
387 assert(tex->cmask.size || tex->fmask.size || tex->dcc_offset);
388
389 si_blit_decompress_color(&sctx->b.b, tex,
390 view->u.tex.first_level, view->u.tex.last_level,
391 0, util_max_layer(&tex->resource.b.b, view->u.tex.first_level),
392 false);
393 }
394 }
395
396 static void
397 si_decompress_image_color_textures(struct si_context *sctx,
398 struct si_images_info *images)
399 {
400 unsigned i;
401 unsigned mask = images->compressed_colortex_mask;
402
403 while (mask) {
404 const struct pipe_image_view *view;
405 struct r600_texture *tex;
406
407 i = u_bit_scan(&mask);
408
409 view = &images->views[i];
410 assert(view->resource->target != PIPE_BUFFER);
411
412 tex = (struct r600_texture *)view->resource;
413 if (!tex->cmask.size && !tex->fmask.size && !tex->dcc_offset)
414 continue;
415
416 si_blit_decompress_color(&sctx->b.b, tex,
417 view->u.tex.level, view->u.tex.level,
418 0, util_max_layer(&tex->resource.b.b, view->u.tex.level),
419 false);
420 }
421 }
422
423 static void si_check_render_feedback_textures(struct si_context *sctx,
424 struct si_textures_info *textures)
425 {
426 uint32_t mask = textures->views.enabled_mask;
427
428 while (mask) {
429 const struct pipe_sampler_view *view;
430 struct r600_texture *tex;
431 bool render_feedback = false;
432
433 unsigned i = u_bit_scan(&mask);
434
435 view = textures->views.views[i];
436 if(view->texture->target == PIPE_BUFFER)
437 continue;
438
439 tex = (struct r600_texture *)view->texture;
440 if (!tex->dcc_offset)
441 continue;
442
443 for (unsigned j = 0; j < sctx->framebuffer.state.nr_cbufs; ++j) {
444 struct r600_surface * surf;
445
446 if (!sctx->framebuffer.state.cbufs[j])
447 continue;
448
449 surf = (struct r600_surface*)sctx->framebuffer.state.cbufs[j];
450
451 if (tex == (struct r600_texture*)surf->base.texture &&
452 surf->base.u.tex.level >= view->u.tex.first_level &&
453 surf->base.u.tex.level <= view->u.tex.last_level &&
454 surf->base.u.tex.first_layer <= view->u.tex.last_layer &&
455 surf->base.u.tex.last_layer >= view->u.tex.first_layer)
456 render_feedback = true;
457 }
458
459 if (render_feedback) {
460 struct si_screen *screen = sctx->screen;
461 r600_texture_disable_dcc(&screen->b, tex);
462 }
463 }
464 }
465
466 static void si_check_render_feedback_images(struct si_context *sctx,
467 struct si_images_info *images)
468 {
469 uint32_t mask = images->enabled_mask;
470
471 while (mask) {
472 const struct pipe_image_view *view;
473 struct r600_texture *tex;
474 bool render_feedback = false;
475
476 unsigned i = u_bit_scan(&mask);
477
478 view = &images->views[i];
479 if (view->resource->target == PIPE_BUFFER)
480 continue;
481
482 tex = (struct r600_texture *)view->resource;
483 if (!tex->dcc_offset)
484 continue;
485
486 for (unsigned j = 0; j < sctx->framebuffer.state.nr_cbufs; ++j) {
487 struct r600_surface * surf;
488
489 if (!sctx->framebuffer.state.cbufs[j])
490 continue;
491
492 surf = (struct r600_surface*)sctx->framebuffer.state.cbufs[j];
493
494 if (tex == (struct r600_texture*)surf->base.texture &&
495 surf->base.u.tex.level == view->u.tex.level &&
496 surf->base.u.tex.first_layer <= view->u.tex.last_layer &&
497 surf->base.u.tex.last_layer >= view->u.tex.first_layer)
498 render_feedback = true;
499 }
500
501 if (render_feedback) {
502 struct si_screen *screen = sctx->screen;
503 r600_texture_disable_dcc(&screen->b, tex);
504 }
505 }
506 }
507
508 static void si_check_render_feedback(struct si_context *sctx)
509 {
510
511 if (!sctx->need_check_render_feedback)
512 return;
513
514 for (int i = 0; i < SI_NUM_SHADERS; ++i) {
515 si_check_render_feedback_images(sctx, &sctx->images[i]);
516 si_check_render_feedback_textures(sctx, &sctx->samplers[i]);
517 }
518 sctx->need_check_render_feedback = false;
519 }
520
521 static void si_decompress_textures(struct si_context *sctx, int shader_start,
522 int shader_end)
523 {
524 unsigned compressed_colortex_counter;
525
526 if (sctx->blitter->running)
527 return;
528
529 /* Update the compressed_colortex_mask if necessary. */
530 compressed_colortex_counter = p_atomic_read(&sctx->screen->b.compressed_colortex_counter);
531 if (compressed_colortex_counter != sctx->b.last_compressed_colortex_counter) {
532 sctx->b.last_compressed_colortex_counter = compressed_colortex_counter;
533 si_update_compressed_colortex_masks(sctx);
534 }
535
536 /* Flush depth textures which need to be flushed. */
537 for (int i = shader_start; i < shader_end; i++) {
538 if (sctx->samplers[i].depth_texture_mask) {
539 si_flush_depth_textures(sctx, &sctx->samplers[i]);
540 }
541 if (sctx->samplers[i].compressed_colortex_mask) {
542 si_decompress_sampler_color_textures(sctx, &sctx->samplers[i]);
543 }
544 if (sctx->images[i].compressed_colortex_mask) {
545 si_decompress_image_color_textures(sctx, &sctx->images[i]);
546 }
547 }
548
549 si_check_render_feedback(sctx);
550 }
551
552 void si_decompress_graphics_textures(struct si_context *sctx)
553 {
554 si_decompress_textures(sctx, 0, SI_NUM_GRAPHICS_SHADERS);
555 }
556
557 void si_decompress_compute_textures(struct si_context *sctx)
558 {
559 si_decompress_textures(sctx, SI_NUM_GRAPHICS_SHADERS, SI_NUM_SHADERS);
560 }
561
562 static void si_clear(struct pipe_context *ctx, unsigned buffers,
563 const union pipe_color_union *color,
564 double depth, unsigned stencil)
565 {
566 struct si_context *sctx = (struct si_context *)ctx;
567 struct pipe_framebuffer_state *fb = &sctx->framebuffer.state;
568 struct pipe_surface *zsbuf = fb->zsbuf;
569 struct r600_texture *zstex =
570 zsbuf ? (struct r600_texture*)zsbuf->texture : NULL;
571
572 if (buffers & PIPE_CLEAR_COLOR) {
573 evergreen_do_fast_color_clear(&sctx->b, fb,
574 &sctx->framebuffer.atom, &buffers,
575 &sctx->framebuffer.dirty_cbufs,
576 color);
577 if (!buffers)
578 return; /* all buffers have been fast cleared */
579 }
580
581 if (buffers & PIPE_CLEAR_COLOR) {
582 int i;
583
584 /* These buffers cannot use fast clear, make sure to disable expansion. */
585 for (i = 0; i < fb->nr_cbufs; i++) {
586 struct r600_texture *tex;
587
588 /* If not clearing this buffer, skip. */
589 if (!(buffers & (PIPE_CLEAR_COLOR0 << i)))
590 continue;
591
592 if (!fb->cbufs[i])
593 continue;
594
595 tex = (struct r600_texture *)fb->cbufs[i]->texture;
596 if (tex->fmask.size == 0)
597 tex->dirty_level_mask &= ~(1 << fb->cbufs[i]->u.tex.level);
598 }
599 }
600
601 if (zstex && zstex->htile_buffer &&
602 zsbuf->u.tex.level == 0 &&
603 zsbuf->u.tex.first_layer == 0 &&
604 zsbuf->u.tex.last_layer == util_max_layer(&zstex->resource.b.b, 0)) {
605 if (buffers & PIPE_CLEAR_DEPTH) {
606 /* Need to disable EXPCLEAR temporarily if clearing
607 * to a new value. */
608 if (!zstex->depth_cleared || zstex->depth_clear_value != depth) {
609 sctx->db_depth_disable_expclear = true;
610 }
611
612 zstex->depth_clear_value = depth;
613 sctx->framebuffer.dirty_zsbuf = true;
614 si_mark_atom_dirty(sctx, &sctx->framebuffer.atom); /* updates DB_DEPTH_CLEAR */
615 sctx->db_depth_clear = true;
616 si_mark_atom_dirty(sctx, &sctx->db_render_state);
617 }
618
619 if (buffers & PIPE_CLEAR_STENCIL) {
620 stencil &= 0xff;
621
622 /* Need to disable EXPCLEAR temporarily if clearing
623 * to a new value. */
624 if (!zstex->stencil_cleared || zstex->stencil_clear_value != stencil) {
625 sctx->db_stencil_disable_expclear = true;
626 }
627
628 zstex->stencil_clear_value = stencil;
629 sctx->framebuffer.dirty_zsbuf = true;
630 si_mark_atom_dirty(sctx, &sctx->framebuffer.atom); /* updates DB_STENCIL_CLEAR */
631 sctx->db_stencil_clear = true;
632 si_mark_atom_dirty(sctx, &sctx->db_render_state);
633 }
634 }
635
636 si_blitter_begin(ctx, SI_CLEAR);
637 util_blitter_clear(sctx->blitter, fb->width, fb->height,
638 util_framebuffer_get_num_layers(fb),
639 buffers, color, depth, stencil);
640 si_blitter_end(ctx);
641
642 if (sctx->db_depth_clear) {
643 sctx->db_depth_clear = false;
644 sctx->db_depth_disable_expclear = false;
645 zstex->depth_cleared = true;
646 si_mark_atom_dirty(sctx, &sctx->db_render_state);
647 }
648
649 if (sctx->db_stencil_clear) {
650 sctx->db_stencil_clear = false;
651 sctx->db_stencil_disable_expclear = false;
652 zstex->stencil_cleared = true;
653 si_mark_atom_dirty(sctx, &sctx->db_render_state);
654 }
655 }
656
657 static void si_clear_render_target(struct pipe_context *ctx,
658 struct pipe_surface *dst,
659 const union pipe_color_union *color,
660 unsigned dstx, unsigned dsty,
661 unsigned width, unsigned height)
662 {
663 struct si_context *sctx = (struct si_context *)ctx;
664
665 si_blitter_begin(ctx, SI_CLEAR_SURFACE);
666 util_blitter_clear_render_target(sctx->blitter, dst, color,
667 dstx, dsty, width, height);
668 si_blitter_end(ctx);
669 }
670
671 static void si_clear_depth_stencil(struct pipe_context *ctx,
672 struct pipe_surface *dst,
673 unsigned clear_flags,
674 double depth,
675 unsigned stencil,
676 unsigned dstx, unsigned dsty,
677 unsigned width, unsigned height)
678 {
679 struct si_context *sctx = (struct si_context *)ctx;
680
681 si_blitter_begin(ctx, SI_CLEAR_SURFACE);
682 util_blitter_clear_depth_stencil(sctx->blitter, dst, clear_flags, depth, stencil,
683 dstx, dsty, width, height);
684 si_blitter_end(ctx);
685 }
686
687 /* Helper for decompressing a portion of a color or depth resource before
688 * blitting if any decompression is needed.
689 * The driver doesn't decompress resources automatically while u_blitter is
690 * rendering. */
691 static void si_decompress_subresource(struct pipe_context *ctx,
692 struct pipe_resource *tex,
693 unsigned planes, unsigned level,
694 unsigned first_layer, unsigned last_layer)
695 {
696 struct si_context *sctx = (struct si_context *)ctx;
697 struct r600_texture *rtex = (struct r600_texture*)tex;
698
699 if (rtex->is_depth && !rtex->is_flushing_texture) {
700 planes &= PIPE_MASK_Z | PIPE_MASK_S;
701
702 if (!(rtex->surface.flags & RADEON_SURF_SBUFFER))
703 planes &= ~PIPE_MASK_S;
704
705 si_blit_decompress_zs_in_place(sctx, rtex, planes,
706 level, level,
707 first_layer, last_layer);
708 } else if (rtex->fmask.size || rtex->cmask.size || rtex->dcc_offset) {
709 si_blit_decompress_color(ctx, rtex, level, level,
710 first_layer, last_layer, false);
711 }
712 }
713
714 struct texture_orig_info {
715 unsigned format;
716 unsigned width0;
717 unsigned height0;
718 unsigned npix_x;
719 unsigned npix_y;
720 unsigned npix0_x;
721 unsigned npix0_y;
722 };
723
724 void si_resource_copy_region(struct pipe_context *ctx,
725 struct pipe_resource *dst,
726 unsigned dst_level,
727 unsigned dstx, unsigned dsty, unsigned dstz,
728 struct pipe_resource *src,
729 unsigned src_level,
730 const struct pipe_box *src_box)
731 {
732 struct si_context *sctx = (struct si_context *)ctx;
733 struct pipe_surface *dst_view, dst_templ;
734 struct pipe_sampler_view src_templ, *src_view;
735 unsigned dst_width, dst_height, src_width0, src_height0;
736 unsigned src_force_level = 0;
737 struct pipe_box sbox, dstbox;
738
739 /* Handle buffers first. */
740 if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
741 si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width);
742 return;
743 }
744
745 assert(u_max_sample(dst) == u_max_sample(src));
746
747 /* The driver doesn't decompress resources automatically while
748 * u_blitter is rendering. */
749 si_decompress_subresource(ctx, src, PIPE_MASK_RGBAZS, src_level,
750 src_box->z, src_box->z + src_box->depth - 1);
751
752 dst_width = u_minify(dst->width0, dst_level);
753 dst_height = u_minify(dst->height0, dst_level);
754 src_width0 = src->width0;
755 src_height0 = src->height0;
756
757 util_blitter_default_dst_texture(&dst_templ, dst, dst_level, dstz);
758 util_blitter_default_src_texture(&src_templ, src, src_level);
759
760 if (util_format_is_compressed(src->format) ||
761 util_format_is_compressed(dst->format)) {
762 unsigned blocksize = util_format_get_blocksize(src->format);
763
764 if (blocksize == 8)
765 src_templ.format = PIPE_FORMAT_R16G16B16A16_UINT; /* 64-bit block */
766 else
767 src_templ.format = PIPE_FORMAT_R32G32B32A32_UINT; /* 128-bit block */
768 dst_templ.format = src_templ.format;
769
770 dst_width = util_format_get_nblocksx(dst->format, dst_width);
771 dst_height = util_format_get_nblocksy(dst->format, dst_height);
772 src_width0 = util_format_get_nblocksx(src->format, src_width0);
773 src_height0 = util_format_get_nblocksy(src->format, src_height0);
774
775 dstx = util_format_get_nblocksx(dst->format, dstx);
776 dsty = util_format_get_nblocksy(dst->format, dsty);
777
778 sbox.x = util_format_get_nblocksx(src->format, src_box->x);
779 sbox.y = util_format_get_nblocksy(src->format, src_box->y);
780 sbox.z = src_box->z;
781 sbox.width = util_format_get_nblocksx(src->format, src_box->width);
782 sbox.height = util_format_get_nblocksy(src->format, src_box->height);
783 sbox.depth = src_box->depth;
784 src_box = &sbox;
785
786 src_force_level = src_level;
787 } else if (!util_blitter_is_copy_supported(sctx->blitter, dst, src) ||
788 /* also *8_SNORM has precision issues, use UNORM instead */
789 util_format_is_snorm8(src->format)) {
790 if (util_format_is_subsampled_422(src->format)) {
791 src_templ.format = PIPE_FORMAT_R8G8B8A8_UINT;
792 dst_templ.format = PIPE_FORMAT_R8G8B8A8_UINT;
793
794 dst_width = util_format_get_nblocksx(dst->format, dst_width);
795 src_width0 = util_format_get_nblocksx(src->format, src_width0);
796
797 dstx = util_format_get_nblocksx(dst->format, dstx);
798
799 sbox = *src_box;
800 sbox.x = util_format_get_nblocksx(src->format, src_box->x);
801 sbox.width = util_format_get_nblocksx(src->format, src_box->width);
802 src_box = &sbox;
803 } else {
804 unsigned blocksize = util_format_get_blocksize(src->format);
805
806 switch (blocksize) {
807 case 1:
808 dst_templ.format = PIPE_FORMAT_R8_UNORM;
809 src_templ.format = PIPE_FORMAT_R8_UNORM;
810 break;
811 case 2:
812 dst_templ.format = PIPE_FORMAT_R8G8_UNORM;
813 src_templ.format = PIPE_FORMAT_R8G8_UNORM;
814 break;
815 case 4:
816 dst_templ.format = PIPE_FORMAT_R8G8B8A8_UNORM;
817 src_templ.format = PIPE_FORMAT_R8G8B8A8_UNORM;
818 break;
819 case 8:
820 dst_templ.format = PIPE_FORMAT_R16G16B16A16_UINT;
821 src_templ.format = PIPE_FORMAT_R16G16B16A16_UINT;
822 break;
823 case 16:
824 dst_templ.format = PIPE_FORMAT_R32G32B32A32_UINT;
825 src_templ.format = PIPE_FORMAT_R32G32B32A32_UINT;
826 break;
827 default:
828 fprintf(stderr, "Unhandled format %s with blocksize %u\n",
829 util_format_short_name(src->format), blocksize);
830 assert(0);
831 }
832 }
833 }
834
835 /* Initialize the surface. */
836 dst_view = r600_create_surface_custom(ctx, dst, &dst_templ,
837 dst_width, dst_height);
838
839 /* Initialize the sampler view. */
840 src_view = si_create_sampler_view_custom(ctx, src, &src_templ,
841 src_width0, src_height0,
842 src_force_level);
843
844 u_box_3d(dstx, dsty, dstz, abs(src_box->width), abs(src_box->height),
845 abs(src_box->depth), &dstbox);
846
847 /* Copy. */
848 si_blitter_begin(ctx, SI_COPY);
849 util_blitter_blit_generic(sctx->blitter, dst_view, &dstbox,
850 src_view, src_box, src_width0, src_height0,
851 PIPE_MASK_RGBAZS, PIPE_TEX_FILTER_NEAREST, NULL,
852 false);
853 si_blitter_end(ctx);
854
855 pipe_surface_reference(&dst_view, NULL);
856 pipe_sampler_view_reference(&src_view, NULL);
857 }
858
859 static bool do_hardware_msaa_resolve(struct pipe_context *ctx,
860 const struct pipe_blit_info *info)
861 {
862 struct si_context *sctx = (struct si_context*)ctx;
863 struct r600_texture *src = (struct r600_texture*)info->src.resource;
864 struct r600_texture *dst = (struct r600_texture*)info->dst.resource;
865 unsigned dst_width = u_minify(info->dst.resource->width0, info->dst.level);
866 unsigned dst_height = u_minify(info->dst.resource->height0, info->dst.level);
867 enum pipe_format format = info->src.format;
868 unsigned sample_mask = ~0;
869 struct pipe_resource *tmp, templ;
870 struct pipe_blit_info blit;
871
872 /* Check basic requirements for hw resolve. */
873 if (!(info->src.resource->nr_samples > 1 &&
874 info->dst.resource->nr_samples <= 1 &&
875 !util_format_is_pure_integer(format) &&
876 !util_format_is_depth_or_stencil(format) &&
877 util_max_layer(info->src.resource, 0) == 0))
878 return false;
879
880 /* Hardware MSAA resolve doesn't work if SPI format = NORM16_ABGR and
881 * the format is R16G16. Use R16A16, which does work.
882 */
883 if (format == PIPE_FORMAT_R16G16_UNORM)
884 format = PIPE_FORMAT_R16A16_UNORM;
885 if (format == PIPE_FORMAT_R16G16_SNORM)
886 format = PIPE_FORMAT_R16A16_SNORM;
887
888 /* Check the remaining requirements for hw resolve. */
889 if (util_max_layer(info->dst.resource, info->dst.level) == 0 &&
890 !info->scissor_enable &&
891 (info->mask & PIPE_MASK_RGBA) == PIPE_MASK_RGBA &&
892 util_is_format_compatible(util_format_description(info->src.format),
893 util_format_description(info->dst.format)) &&
894 dst_width == info->src.resource->width0 &&
895 dst_height == info->src.resource->height0 &&
896 info->dst.box.x == 0 &&
897 info->dst.box.y == 0 &&
898 info->dst.box.width == dst_width &&
899 info->dst.box.height == dst_height &&
900 info->dst.box.depth == 1 &&
901 info->src.box.x == 0 &&
902 info->src.box.y == 0 &&
903 info->src.box.width == dst_width &&
904 info->src.box.height == dst_height &&
905 info->src.box.depth == 1 &&
906 dst->surface.level[info->dst.level].mode >= RADEON_SURF_MODE_1D &&
907 (!dst->cmask.size || !dst->dirty_level_mask)) { /* dst cannot be fast-cleared */
908 /* Check the last constraint. */
909 if (src->surface.micro_tile_mode != dst->surface.micro_tile_mode) {
910 /* The next fast clear will switch to this mode to
911 * get direct hw resolve next time if the mode is
912 * different now.
913 */
914 src->last_msaa_resolve_target_micro_mode =
915 dst->surface.micro_tile_mode;
916 goto resolve_to_temp;
917 }
918
919 /* Resolving into a surface with DCC is unsupported. Since
920 * it's being overwritten anyway, clear it to uncompressed.
921 * This is still the fastest codepath even with this clear.
922 */
923 if (dst->dcc_offset &&
924 dst->surface.level[info->dst.level].dcc_enabled) {
925 vi_dcc_clear_level(&sctx->b, dst, info->dst.level,
926 0xFFFFFFFF);
927 dst->dirty_level_mask &= ~(1 << info->dst.level);
928 }
929
930 /* Resolve directly from src to dst. */
931 si_blitter_begin(ctx, SI_COLOR_RESOLVE |
932 (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
933 util_blitter_custom_resolve_color(sctx->blitter,
934 info->dst.resource, info->dst.level,
935 info->dst.box.z,
936 info->src.resource, info->src.box.z,
937 sample_mask, sctx->custom_blend_resolve,
938 format);
939 si_blitter_end(ctx);
940 return true;
941 }
942
943 resolve_to_temp:
944 /* Shader-based resolve is VERY SLOW. Instead, resolve into
945 * a temporary texture and blit.
946 */
947 memset(&templ, 0, sizeof(templ));
948 templ.target = PIPE_TEXTURE_2D;
949 templ.format = info->src.resource->format;
950 templ.width0 = info->src.resource->width0;
951 templ.height0 = info->src.resource->height0;
952 templ.depth0 = 1;
953 templ.array_size = 1;
954 templ.usage = PIPE_USAGE_DEFAULT;
955 templ.flags = R600_RESOURCE_FLAG_FORCE_TILING |
956 R600_RESOURCE_FLAG_DISABLE_DCC;
957
958 /* The src and dst microtile modes must be the same. */
959 if (src->surface.micro_tile_mode == V_009910_ADDR_SURF_DISPLAY_MICRO_TILING)
960 templ.bind = PIPE_BIND_SCANOUT;
961 else
962 templ.bind = 0;
963
964 tmp = ctx->screen->resource_create(ctx->screen, &templ);
965 if (!tmp)
966 return false;
967
968 assert(src->surface.micro_tile_mode ==
969 ((struct r600_texture*)tmp)->surface.micro_tile_mode);
970
971 /* resolve */
972 si_blitter_begin(ctx, SI_COLOR_RESOLVE |
973 (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
974 util_blitter_custom_resolve_color(sctx->blitter, tmp, 0, 0,
975 info->src.resource, info->src.box.z,
976 sample_mask, sctx->custom_blend_resolve,
977 format);
978 si_blitter_end(ctx);
979
980 /* blit */
981 blit = *info;
982 blit.src.resource = tmp;
983 blit.src.box.z = 0;
984
985 si_blitter_begin(ctx, SI_BLIT |
986 (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
987 util_blitter_blit(sctx->blitter, &blit);
988 si_blitter_end(ctx);
989
990 pipe_resource_reference(&tmp, NULL);
991 return true;
992 }
993
994 static void si_blit(struct pipe_context *ctx,
995 const struct pipe_blit_info *info)
996 {
997 struct si_context *sctx = (struct si_context*)ctx;
998
999 if (do_hardware_msaa_resolve(ctx, info)) {
1000 return;
1001 }
1002
1003 assert(util_blitter_is_blit_supported(sctx->blitter, info));
1004
1005 /* The driver doesn't decompress resources automatically while
1006 * u_blitter is rendering. */
1007 si_decompress_subresource(ctx, info->src.resource, info->mask,
1008 info->src.level,
1009 info->src.box.z,
1010 info->src.box.z + info->src.box.depth - 1);
1011
1012 if (sctx->screen->b.debug_flags & DBG_FORCE_DMA &&
1013 util_try_blit_via_copy_region(ctx, info))
1014 return;
1015
1016 si_blitter_begin(ctx, SI_BLIT |
1017 (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
1018 util_blitter_blit(sctx->blitter, info);
1019 si_blitter_end(ctx);
1020 }
1021
1022 static boolean si_generate_mipmap(struct pipe_context *ctx,
1023 struct pipe_resource *tex,
1024 enum pipe_format format,
1025 unsigned base_level, unsigned last_level,
1026 unsigned first_layer, unsigned last_layer)
1027 {
1028 struct si_context *sctx = (struct si_context*)ctx;
1029 struct r600_texture *rtex = (struct r600_texture *)tex;
1030
1031 if (!util_blitter_is_copy_supported(sctx->blitter, tex, tex))
1032 return false;
1033
1034 /* The driver doesn't decompress resources automatically while
1035 * u_blitter is rendering. */
1036 si_decompress_subresource(ctx, tex, PIPE_MASK_RGBAZS,
1037 base_level, first_layer, last_layer);
1038
1039 /* Clear dirty_level_mask for the levels that will be overwritten. */
1040 assert(base_level < last_level);
1041 rtex->dirty_level_mask &= ~u_bit_consecutive(base_level + 1,
1042 last_level - base_level);
1043
1044 si_blitter_begin(ctx, SI_BLIT | SI_DISABLE_RENDER_COND);
1045 util_blitter_generate_mipmap(sctx->blitter, tex, format,
1046 base_level, last_level,
1047 first_layer, last_layer);
1048 si_blitter_end(ctx);
1049 return true;
1050 }
1051
1052 static void si_flush_resource(struct pipe_context *ctx,
1053 struct pipe_resource *res)
1054 {
1055 struct r600_texture *rtex = (struct r600_texture*)res;
1056
1057 assert(res->target != PIPE_BUFFER);
1058
1059 if (!rtex->is_depth && (rtex->cmask.size || rtex->dcc_offset)) {
1060 si_blit_decompress_color(ctx, rtex, 0, res->last_level,
1061 0, util_max_layer(res, 0), false);
1062 }
1063 }
1064
1065 static void si_decompress_dcc(struct pipe_context *ctx,
1066 struct r600_texture *rtex)
1067 {
1068 if (!rtex->dcc_offset)
1069 return;
1070
1071 si_blit_decompress_color(ctx, rtex, 0, rtex->resource.b.b.last_level,
1072 0, util_max_layer(&rtex->resource.b.b, 0),
1073 true);
1074 }
1075
1076 static void si_pipe_clear_buffer(struct pipe_context *ctx,
1077 struct pipe_resource *dst,
1078 unsigned offset, unsigned size,
1079 const void *clear_value_ptr,
1080 int clear_value_size)
1081 {
1082 struct si_context *sctx = (struct si_context*)ctx;
1083 uint32_t dword_value;
1084 unsigned i;
1085
1086 assert(offset % clear_value_size == 0);
1087 assert(size % clear_value_size == 0);
1088
1089 if (clear_value_size > 4) {
1090 const uint32_t *u32 = clear_value_ptr;
1091 bool clear_dword_duplicated = true;
1092
1093 /* See if we can lower large fills to dword fills. */
1094 for (i = 1; i < clear_value_size / 4; i++)
1095 if (u32[0] != u32[i]) {
1096 clear_dword_duplicated = false;
1097 break;
1098 }
1099
1100 if (!clear_dword_duplicated) {
1101 /* Use transform feedback for 64-bit, 96-bit, and
1102 * 128-bit fills.
1103 */
1104 union pipe_color_union clear_value;
1105
1106 memcpy(&clear_value, clear_value_ptr, clear_value_size);
1107 si_blitter_begin(ctx, SI_DISABLE_RENDER_COND);
1108 util_blitter_clear_buffer(sctx->blitter, dst, offset,
1109 size, clear_value_size / 4,
1110 &clear_value);
1111 si_blitter_end(ctx);
1112 return;
1113 }
1114 }
1115
1116 /* Expand the clear value to a dword. */
1117 switch (clear_value_size) {
1118 case 1:
1119 dword_value = *(uint8_t*)clear_value_ptr;
1120 dword_value |= (dword_value << 8) |
1121 (dword_value << 16) |
1122 (dword_value << 24);
1123 break;
1124 case 2:
1125 dword_value = *(uint16_t*)clear_value_ptr;
1126 dword_value |= dword_value << 16;
1127 break;
1128 default:
1129 dword_value = *(uint32_t*)clear_value_ptr;
1130 }
1131
1132 sctx->b.clear_buffer(ctx, dst, offset, size, dword_value,
1133 R600_COHERENCY_SHADER);
1134 }
1135
1136 void si_init_blit_functions(struct si_context *sctx)
1137 {
1138 sctx->b.b.clear = si_clear;
1139 sctx->b.b.clear_buffer = si_pipe_clear_buffer;
1140 sctx->b.b.clear_render_target = si_clear_render_target;
1141 sctx->b.b.clear_depth_stencil = si_clear_depth_stencil;
1142 sctx->b.b.resource_copy_region = si_resource_copy_region;
1143 sctx->b.b.blit = si_blit;
1144 sctx->b.b.flush_resource = si_flush_resource;
1145 sctx->b.b.generate_mipmap = si_generate_mipmap;
1146 sctx->b.blit_decompress_depth = si_blit_decompress_depth;
1147 sctx->b.decompress_dcc = si_decompress_dcc;
1148 }