projects / mesa.git / blob
? search:


716a52231aa2400a2e3d11cdfedb184aa52a02db
[mesa.git] / src / gallium / drivers / radeonsi / si_blit.c
1 /*
2 * Copyright 2010 Jerome Glisse <glisse@freedesktop.org>
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "si_pipe.h"
25 #include "util/u_format.h"
26 #include "util/u_surface.h"
27
28 enum si_blitter_op /* bitmask */
29 {
30 SI_SAVE_TEXTURES = 1,
31 SI_SAVE_FRAMEBUFFER = 2,
32 SI_SAVE_FRAGMENT_STATE = 4,
33 SI_DISABLE_RENDER_COND = 8,
34
35 SI_CLEAR = SI_SAVE_FRAGMENT_STATE,
36
37 SI_CLEAR_SURFACE = SI_SAVE_FRAMEBUFFER | SI_SAVE_FRAGMENT_STATE,
38
39 SI_COPY = SI_SAVE_FRAMEBUFFER | SI_SAVE_TEXTURES |
40 SI_SAVE_FRAGMENT_STATE | SI_DISABLE_RENDER_COND,
41
42 SI_BLIT = SI_SAVE_FRAMEBUFFER | SI_SAVE_TEXTURES |
43 SI_SAVE_FRAGMENT_STATE,
44
45 SI_DECOMPRESS = SI_SAVE_FRAMEBUFFER | SI_SAVE_FRAGMENT_STATE |
46 SI_DISABLE_RENDER_COND,
47
48 SI_COLOR_RESOLVE = SI_SAVE_FRAMEBUFFER | SI_SAVE_FRAGMENT_STATE
49 };
50
51 static void si_blitter_begin(struct pipe_context *ctx, enum si_blitter_op op)
52 {
53 struct si_context *sctx = (struct si_context *)ctx;
54
55 util_blitter_save_vertex_buffer_slot(sctx->blitter, sctx->vertex_buffer);
56 util_blitter_save_vertex_elements(sctx->blitter, sctx->vertex_elements);
57 util_blitter_save_vertex_shader(sctx->blitter, sctx->vs_shader.cso);
58 util_blitter_save_tessctrl_shader(sctx->blitter, sctx->tcs_shader.cso);
59 util_blitter_save_tesseval_shader(sctx->blitter, sctx->tes_shader.cso);
60 util_blitter_save_geometry_shader(sctx->blitter, sctx->gs_shader.cso);
61 util_blitter_save_so_targets(sctx->blitter, sctx->b.streamout.num_targets,
62 (struct pipe_stream_output_target**)sctx->b.streamout.targets);
63 util_blitter_save_rasterizer(sctx->blitter, sctx->queued.named.rasterizer);
64
65 if (op & SI_SAVE_FRAGMENT_STATE) {
66 util_blitter_save_blend(sctx->blitter, sctx->queued.named.blend);
67 util_blitter_save_depth_stencil_alpha(sctx->blitter, sctx->queued.named.dsa);
68 util_blitter_save_stencil_ref(sctx->blitter, &sctx->stencil_ref.state);
69 util_blitter_save_fragment_shader(sctx->blitter, sctx->ps_shader.cso);
70 util_blitter_save_sample_mask(sctx->blitter, sctx->sample_mask.sample_mask);
71 util_blitter_save_viewport(sctx->blitter, &sctx->b.viewports.states[0]);
72 util_blitter_save_scissor(sctx->blitter, &sctx->b.scissors.states[0]);
73 }
74
75 if (op & SI_SAVE_FRAMEBUFFER)
76 util_blitter_save_framebuffer(sctx->blitter, &sctx->framebuffer.state);
77
78 if (op & SI_SAVE_TEXTURES) {
79 util_blitter_save_fragment_sampler_states(
80 sctx->blitter, 2,
81 sctx->samplers[PIPE_SHADER_FRAGMENT].views.sampler_states);
82
83 util_blitter_save_fragment_sampler_views(sctx->blitter, 2,
84 sctx->samplers[PIPE_SHADER_FRAGMENT].views.views);
85 }
86
87 if (op & SI_DISABLE_RENDER_COND)
88 sctx->b.render_cond_force_off = true;
89 }
90
91 static void si_blitter_end(struct pipe_context *ctx)
92 {
93 struct si_context *sctx = (struct si_context *)ctx;
94
95 sctx->b.render_cond_force_off = false;
96 }
97
98 static unsigned u_max_sample(struct pipe_resource *r)
99 {
100 return r->nr_samples ? r->nr_samples - 1 : 0;
101 }
102
103 static void si_blit_decompress_depth(struct pipe_context *ctx,
104 struct r600_texture *texture,
105 struct r600_texture *staging,
106 unsigned first_level, unsigned last_level,
107 unsigned first_layer, unsigned last_layer,
108 unsigned first_sample, unsigned last_sample)
109 {
110 struct si_context *sctx = (struct si_context *)ctx;
111 unsigned layer, level, sample, checked_last_layer, max_layer;
112 float depth = 1.0f;
113 const struct util_format_description *desc;
114
115 assert(staging != NULL && "use si_blit_decompress_zs_in_place instead");
116
117 desc = util_format_description(staging->resource.b.b.format);
118
119 if (util_format_has_depth(desc))
120 sctx->dbcb_depth_copy_enabled = true;
121 if (util_format_has_stencil(desc))
122 sctx->dbcb_stencil_copy_enabled = true;
123
124 assert(sctx->dbcb_depth_copy_enabled || sctx->dbcb_stencil_copy_enabled);
125
126 for (level = first_level; level <= last_level; level++) {
127 /* The smaller the mipmap level, the less layers there are
128 * as far as 3D textures are concerned. */
129 max_layer = util_max_layer(&texture->resource.b.b, level);
130 checked_last_layer = MIN2(last_layer, max_layer);
131
132 for (layer = first_layer; layer <= checked_last_layer; layer++) {
133 for (sample = first_sample; sample <= last_sample; sample++) {
134 struct pipe_surface *zsurf, *cbsurf, surf_tmpl;
135
136 sctx->dbcb_copy_sample = sample;
137 si_mark_atom_dirty(sctx, &sctx->db_render_state);
138
139 surf_tmpl.format = texture->resource.b.b.format;
140 surf_tmpl.u.tex.level = level;
141 surf_tmpl.u.tex.first_layer = layer;
142 surf_tmpl.u.tex.last_layer = layer;
143
144 zsurf = ctx->create_surface(ctx, &texture->resource.b.b, &surf_tmpl);
145
146 surf_tmpl.format = staging->resource.b.b.format;
147 cbsurf = ctx->create_surface(ctx,
148 (struct pipe_resource*)staging, &surf_tmpl);
149
150 si_blitter_begin(ctx, SI_DECOMPRESS);
151 util_blitter_custom_depth_stencil(sctx->blitter, zsurf, cbsurf, 1 << sample,
152 sctx->custom_dsa_flush, depth);
153 si_blitter_end(ctx);
154
155 pipe_surface_reference(&zsurf, NULL);
156 pipe_surface_reference(&cbsurf, NULL);
157 }
158 }
159 }
160
161 sctx->dbcb_depth_copy_enabled = false;
162 sctx->dbcb_stencil_copy_enabled = false;
163 si_mark_atom_dirty(sctx, &sctx->db_render_state);
164 }
165
166 /* Helper function for si_blit_decompress_zs_in_place.
167 */
168 static void
169 si_blit_decompress_zs_planes_in_place(struct si_context *sctx,
170 struct r600_texture *texture,
171 unsigned planes, unsigned level_mask,
172 unsigned first_layer, unsigned last_layer)
173 {
174 struct pipe_surface *zsurf, surf_tmpl = {{0}};
175 unsigned layer, max_layer, checked_last_layer;
176 unsigned fully_decompressed_mask = 0;
177
178 if (!level_mask)
179 return;
180
181 if (planes & PIPE_MASK_S)
182 sctx->db_flush_stencil_inplace = true;
183 if (planes & PIPE_MASK_Z)
184 sctx->db_flush_depth_inplace = true;
185 si_mark_atom_dirty(sctx, &sctx->db_render_state);
186
187 surf_tmpl.format = texture->resource.b.b.format;
188
189 while (level_mask) {
190 unsigned level = u_bit_scan(&level_mask);
191
192 surf_tmpl.u.tex.level = level;
193
194 /* The smaller the mipmap level, the less layers there are
195 * as far as 3D textures are concerned. */
196 max_layer = util_max_layer(&texture->resource.b.b, level);
197 checked_last_layer = MIN2(last_layer, max_layer);
198
199 for (layer = first_layer; layer <= checked_last_layer; layer++) {
200 surf_tmpl.u.tex.first_layer = layer;
201 surf_tmpl.u.tex.last_layer = layer;
202
203 zsurf = sctx->b.b.create_surface(&sctx->b.b, &texture->resource.b.b, &surf_tmpl);
204
205 si_blitter_begin(&sctx->b.b, SI_DECOMPRESS);
206 util_blitter_custom_depth_stencil(sctx->blitter, zsurf, NULL, ~0,
207 sctx->custom_dsa_flush,
208 1.0f);
209 si_blitter_end(&sctx->b.b);
210
211 pipe_surface_reference(&zsurf, NULL);
212 }
213
214 /* The texture will always be dirty if some layers aren't flushed.
215 * I don't think this case occurs often though. */
216 if (first_layer == 0 && last_layer == max_layer) {
217 fully_decompressed_mask |= 1u << level;
218 }
219 }
220
221 if (planes & PIPE_MASK_Z)
222 texture->dirty_level_mask &= ~fully_decompressed_mask;
223 if (planes & PIPE_MASK_S)
224 texture->stencil_dirty_level_mask &= ~fully_decompressed_mask;
225
226 sctx->db_flush_depth_inplace = false;
227 sctx->db_flush_stencil_inplace = false;
228 si_mark_atom_dirty(sctx, &sctx->db_render_state);
229 }
230
231 /* Decompress Z and/or S planes in place, depending on mask.
232 */
233 static void
234 si_blit_decompress_zs_in_place(struct si_context *sctx,
235 struct r600_texture *texture,
236 unsigned planes,
237 unsigned first_level, unsigned last_level,
238 unsigned first_layer, unsigned last_layer)
239 {
240 unsigned level_mask =
241 u_bit_consecutive(first_level, last_level - first_level + 1);
242 unsigned cur_level_mask;
243
244 /* First, do combined Z & S decompresses for levels that need it. */
245 if (planes == (PIPE_MASK_Z | PIPE_MASK_S)) {
246 cur_level_mask =
247 level_mask &
248 texture->dirty_level_mask &
249 texture->stencil_dirty_level_mask;
250 si_blit_decompress_zs_planes_in_place(
251 sctx, texture, PIPE_MASK_Z | PIPE_MASK_S,
252 cur_level_mask,
253 first_layer, last_layer);
254 level_mask &= ~cur_level_mask;
255 }
256
257 /* Now do separate Z and S decompresses. */
258 if (planes & PIPE_MASK_Z) {
259 cur_level_mask = level_mask & texture->dirty_level_mask;
260 si_blit_decompress_zs_planes_in_place(
261 sctx, texture, PIPE_MASK_Z,
262 cur_level_mask,
263 first_layer, last_layer);
264 level_mask &= ~cur_level_mask;
265 }
266
267 if (planes & PIPE_MASK_S) {
268 cur_level_mask = level_mask & texture->stencil_dirty_level_mask;
269 si_blit_decompress_zs_planes_in_place(
270 sctx, texture, PIPE_MASK_S,
271 cur_level_mask,
272 first_layer, last_layer);
273 }
274 }
275
276 static void
277 si_flush_depth_textures(struct si_context *sctx,
278 struct si_textures_info *textures)
279 {
280 unsigned i;
281 unsigned mask = textures->depth_texture_mask;
282
283 while (mask) {
284 struct pipe_sampler_view *view;
285 struct si_sampler_view *sview;
286 struct r600_texture *tex;
287
288 i = u_bit_scan(&mask);
289
290 view = textures->views.views[i];
291 assert(view);
292 sview = (struct si_sampler_view*)view;
293
294 tex = (struct r600_texture *)view->texture;
295 assert(tex->is_depth && !tex->is_flushing_texture);
296
297 si_blit_decompress_zs_in_place(sctx, tex,
298 sview->is_stencil_sampler ? PIPE_MASK_S
299 : PIPE_MASK_Z,
300 view->u.tex.first_level, view->u.tex.last_level,
301 0, util_max_layer(&tex->resource.b.b, view->u.tex.first_level));
302 }
303 }
304
305 static void si_blit_decompress_color(struct pipe_context *ctx,
306 struct r600_texture *rtex,
307 unsigned first_level, unsigned last_level,
308 unsigned first_layer, unsigned last_layer,
309 bool need_dcc_decompress)
310 {
311 struct si_context *sctx = (struct si_context *)ctx;
312 void* custom_blend;
313 unsigned layer, checked_last_layer, max_layer;
314 unsigned level_mask =
315 u_bit_consecutive(first_level, last_level - first_level + 1);
316
317 if (!need_dcc_decompress)
318 level_mask &= rtex->dirty_level_mask;
319 if (!level_mask)
320 return;
321
322 if (rtex->dcc_offset && need_dcc_decompress) {
323 custom_blend = sctx->custom_blend_dcc_decompress;
324 } else if (rtex->fmask.size) {
325 custom_blend = sctx->custom_blend_decompress;
326 } else {
327 custom_blend = sctx->custom_blend_fastclear;
328 }
329
330 while (level_mask) {
331 unsigned level = u_bit_scan(&level_mask);
332
333 /* The smaller the mipmap level, the less layers there are
334 * as far as 3D textures are concerned. */
335 max_layer = util_max_layer(&rtex->resource.b.b, level);
336 checked_last_layer = MIN2(last_layer, max_layer);
337
338 for (layer = first_layer; layer <= checked_last_layer; layer++) {
339 struct pipe_surface *cbsurf, surf_tmpl;
340
341 surf_tmpl.format = rtex->resource.b.b.format;
342 surf_tmpl.u.tex.level = level;
343 surf_tmpl.u.tex.first_layer = layer;
344 surf_tmpl.u.tex.last_layer = layer;
345 cbsurf = ctx->create_surface(ctx, &rtex->resource.b.b, &surf_tmpl);
346
347 si_blitter_begin(ctx, SI_DECOMPRESS);
348 util_blitter_custom_color(sctx->blitter, cbsurf, custom_blend);
349 si_blitter_end(ctx);
350
351 pipe_surface_reference(&cbsurf, NULL);
352 }
353
354 /* The texture will always be dirty if some layers aren't flushed.
355 * I don't think this case occurs often though. */
356 if (first_layer == 0 && last_layer == max_layer) {
357 rtex->dirty_level_mask &= ~(1 << level);
358 }
359 }
360 }
361
362 static void
363 si_decompress_sampler_color_textures(struct si_context *sctx,
364 struct si_textures_info *textures)
365 {
366 unsigned i;
367 unsigned mask = textures->compressed_colortex_mask;
368
369 while (mask) {
370 struct pipe_sampler_view *view;
371 struct r600_texture *tex;
372
373 i = u_bit_scan(&mask);
374
375 view = textures->views.views[i];
376 assert(view);
377
378 tex = (struct r600_texture *)view->texture;
379 assert(tex->cmask.size || tex->fmask.size || tex->dcc_offset);
380
381 si_blit_decompress_color(&sctx->b.b, tex,
382 view->u.tex.first_level, view->u.tex.last_level,
383 0, util_max_layer(&tex->resource.b.b, view->u.tex.first_level),
384 false);
385 }
386 }
387
388 static void
389 si_decompress_image_color_textures(struct si_context *sctx,
390 struct si_images_info *images)
391 {
392 unsigned i;
393 unsigned mask = images->compressed_colortex_mask;
394
395 while (mask) {
396 const struct pipe_image_view *view;
397 struct r600_texture *tex;
398
399 i = u_bit_scan(&mask);
400
401 view = &images->views[i];
402 assert(view->resource->target != PIPE_BUFFER);
403
404 tex = (struct r600_texture *)view->resource;
405 if (!tex->cmask.size && !tex->fmask.size && !tex->dcc_offset)
406 continue;
407
408 si_blit_decompress_color(&sctx->b.b, tex,
409 view->u.tex.level, view->u.tex.level,
410 0, util_max_layer(&tex->resource.b.b, view->u.tex.level),
411 false);
412 }
413 }
414
415 static void si_decompress_textures(struct si_context *sctx, int shader_start,
416 int shader_end)
417 {
418 unsigned compressed_colortex_counter;
419
420 if (sctx->blitter->running)
421 return;
422
423 /* Update the compressed_colortex_mask if necessary. */
424 compressed_colortex_counter = p_atomic_read(&sctx->screen->b.compressed_colortex_counter);
425 if (compressed_colortex_counter != sctx->b.last_compressed_colortex_counter) {
426 sctx->b.last_compressed_colortex_counter = compressed_colortex_counter;
427 si_update_compressed_colortex_masks(sctx);
428 }
429
430 /* Flush depth textures which need to be flushed. */
431 for (int i = shader_start; i < shader_end; i++) {
432 if (sctx->samplers[i].depth_texture_mask) {
433 si_flush_depth_textures(sctx, &sctx->samplers[i]);
434 }
435 if (sctx->samplers[i].compressed_colortex_mask) {
436 si_decompress_sampler_color_textures(sctx, &sctx->samplers[i]);
437 }
438 if (sctx->images[i].compressed_colortex_mask) {
439 si_decompress_image_color_textures(sctx, &sctx->images[i]);
440 }
441 }
442 }
443
444 void si_decompress_graphics_textures(struct si_context *sctx)
445 {
446 si_decompress_textures(sctx, 0, SI_NUM_GRAPHICS_SHADERS);
447 }
448
449 void si_decompress_compute_textures(struct si_context *sctx)
450 {
451 si_decompress_textures(sctx, SI_NUM_GRAPHICS_SHADERS, SI_NUM_SHADERS);
452 }
453
454 static void si_clear(struct pipe_context *ctx, unsigned buffers,
455 const union pipe_color_union *color,
456 double depth, unsigned stencil)
457 {
458 struct si_context *sctx = (struct si_context *)ctx;
459 struct pipe_framebuffer_state *fb = &sctx->framebuffer.state;
460 struct pipe_surface *zsbuf = fb->zsbuf;
461 struct r600_texture *zstex =
462 zsbuf ? (struct r600_texture*)zsbuf->texture : NULL;
463
464 if (buffers & PIPE_CLEAR_COLOR) {
465 evergreen_do_fast_color_clear(&sctx->b, fb,
466 &sctx->framebuffer.atom, &buffers,
467 &sctx->framebuffer.dirty_cbufs,
468 color);
469 if (!buffers)
470 return; /* all buffers have been fast cleared */
471 }
472
473 if (buffers & PIPE_CLEAR_COLOR) {
474 int i;
475
476 /* These buffers cannot use fast clear, make sure to disable expansion. */
477 for (i = 0; i < fb->nr_cbufs; i++) {
478 struct r600_texture *tex;
479
480 /* If not clearing this buffer, skip. */
481 if (!(buffers & (PIPE_CLEAR_COLOR0 << i)))
482 continue;
483
484 if (!fb->cbufs[i])
485 continue;
486
487 tex = (struct r600_texture *)fb->cbufs[i]->texture;
488 if (tex->fmask.size == 0)
489 tex->dirty_level_mask &= ~(1 << fb->cbufs[i]->u.tex.level);
490 }
491 }
492
493 if (zstex && zstex->htile_buffer &&
494 zsbuf->u.tex.level == 0 &&
495 zsbuf->u.tex.first_layer == 0 &&
496 zsbuf->u.tex.last_layer == util_max_layer(&zstex->resource.b.b, 0)) {
497 if (buffers & PIPE_CLEAR_DEPTH) {
498 /* Need to disable EXPCLEAR temporarily if clearing
499 * to a new value. */
500 if (!zstex->depth_cleared || zstex->depth_clear_value != depth) {
501 sctx->db_depth_disable_expclear = true;
502 }
503
504 zstex->depth_clear_value = depth;
505 sctx->framebuffer.dirty_zsbuf = true;
506 si_mark_atom_dirty(sctx, &sctx->framebuffer.atom); /* updates DB_DEPTH_CLEAR */
507 sctx->db_depth_clear = true;
508 si_mark_atom_dirty(sctx, &sctx->db_render_state);
509 }
510
511 if (buffers & PIPE_CLEAR_STENCIL) {
512 stencil &= 0xff;
513
514 /* Need to disable EXPCLEAR temporarily if clearing
515 * to a new value. */
516 if (!zstex->stencil_cleared || zstex->stencil_clear_value != stencil) {
517 sctx->db_stencil_disable_expclear = true;
518 }
519
520 zstex->stencil_clear_value = stencil;
521 sctx->framebuffer.dirty_zsbuf = true;
522 si_mark_atom_dirty(sctx, &sctx->framebuffer.atom); /* updates DB_STENCIL_CLEAR */
523 sctx->db_stencil_clear = true;
524 si_mark_atom_dirty(sctx, &sctx->db_render_state);
525 }
526 }
527
528 si_blitter_begin(ctx, SI_CLEAR);
529 util_blitter_clear(sctx->blitter, fb->width, fb->height,
530 util_framebuffer_get_num_layers(fb),
531 buffers, color, depth, stencil);
532 si_blitter_end(ctx);
533
534 if (sctx->db_depth_clear) {
535 sctx->db_depth_clear = false;
536 sctx->db_depth_disable_expclear = false;
537 zstex->depth_cleared = true;
538 si_mark_atom_dirty(sctx, &sctx->db_render_state);
539 }
540
541 if (sctx->db_stencil_clear) {
542 sctx->db_stencil_clear = false;
543 sctx->db_stencil_disable_expclear = false;
544 zstex->stencil_cleared = true;
545 si_mark_atom_dirty(sctx, &sctx->db_render_state);
546 }
547 }
548
549 static void si_clear_render_target(struct pipe_context *ctx,
550 struct pipe_surface *dst,
551 const union pipe_color_union *color,
552 unsigned dstx, unsigned dsty,
553 unsigned width, unsigned height)
554 {
555 struct si_context *sctx = (struct si_context *)ctx;
556
557 si_blitter_begin(ctx, SI_CLEAR_SURFACE);
558 util_blitter_clear_render_target(sctx->blitter, dst, color,
559 dstx, dsty, width, height);
560 si_blitter_end(ctx);
561 }
562
563 static void si_clear_depth_stencil(struct pipe_context *ctx,
564 struct pipe_surface *dst,
565 unsigned clear_flags,
566 double depth,
567 unsigned stencil,
568 unsigned dstx, unsigned dsty,
569 unsigned width, unsigned height)
570 {
571 struct si_context *sctx = (struct si_context *)ctx;
572
573 si_blitter_begin(ctx, SI_CLEAR_SURFACE);
574 util_blitter_clear_depth_stencil(sctx->blitter, dst, clear_flags, depth, stencil,
575 dstx, dsty, width, height);
576 si_blitter_end(ctx);
577 }
578
579 /* Helper for decompressing a portion of a color or depth resource before
580 * blitting if any decompression is needed.
581 * The driver doesn't decompress resources automatically while u_blitter is
582 * rendering. */
583 static void si_decompress_subresource(struct pipe_context *ctx,
584 struct pipe_resource *tex,
585 unsigned planes, unsigned level,
586 unsigned first_layer, unsigned last_layer)
587 {
588 struct si_context *sctx = (struct si_context *)ctx;
589 struct r600_texture *rtex = (struct r600_texture*)tex;
590
591 if (rtex->is_depth && !rtex->is_flushing_texture) {
592 planes &= PIPE_MASK_Z | PIPE_MASK_S;
593
594 if (!(rtex->surface.flags & RADEON_SURF_SBUFFER))
595 planes &= ~PIPE_MASK_S;
596
597 si_blit_decompress_zs_in_place(sctx, rtex, planes,
598 level, level,
599 first_layer, last_layer);
600 } else if (rtex->fmask.size || rtex->cmask.size || rtex->dcc_offset) {
601 si_blit_decompress_color(ctx, rtex, level, level,
602 first_layer, last_layer, false);
603 }
604 }
605
606 struct texture_orig_info {
607 unsigned format;
608 unsigned width0;
609 unsigned height0;
610 unsigned npix_x;
611 unsigned npix_y;
612 unsigned npix0_x;
613 unsigned npix0_y;
614 };
615
616 void si_resource_copy_region(struct pipe_context *ctx,
617 struct pipe_resource *dst,
618 unsigned dst_level,
619 unsigned dstx, unsigned dsty, unsigned dstz,
620 struct pipe_resource *src,
621 unsigned src_level,
622 const struct pipe_box *src_box)
623 {
624 struct si_context *sctx = (struct si_context *)ctx;
625 struct pipe_surface *dst_view, dst_templ;
626 struct pipe_sampler_view src_templ, *src_view;
627 unsigned dst_width, dst_height, src_width0, src_height0;
628 unsigned src_force_level = 0;
629 struct pipe_box sbox, dstbox;
630
631 /* Handle buffers first. */
632 if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
633 si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width);
634 return;
635 }
636
637 assert(u_max_sample(dst) == u_max_sample(src));
638
639 /* The driver doesn't decompress resources automatically while
640 * u_blitter is rendering. */
641 si_decompress_subresource(ctx, src, PIPE_MASK_RGBAZS, src_level,
642 src_box->z, src_box->z + src_box->depth - 1);
643
644 dst_width = u_minify(dst->width0, dst_level);
645 dst_height = u_minify(dst->height0, dst_level);
646 src_width0 = src->width0;
647 src_height0 = src->height0;
648
649 util_blitter_default_dst_texture(&dst_templ, dst, dst_level, dstz);
650 util_blitter_default_src_texture(&src_templ, src, src_level);
651
652 if (util_format_is_compressed(src->format) ||
653 util_format_is_compressed(dst->format)) {
654 unsigned blocksize = util_format_get_blocksize(src->format);
655
656 if (blocksize == 8)
657 src_templ.format = PIPE_FORMAT_R16G16B16A16_UINT; /* 64-bit block */
658 else
659 src_templ.format = PIPE_FORMAT_R32G32B32A32_UINT; /* 128-bit block */
660 dst_templ.format = src_templ.format;
661
662 dst_width = util_format_get_nblocksx(dst->format, dst_width);
663 dst_height = util_format_get_nblocksy(dst->format, dst_height);
664 src_width0 = util_format_get_nblocksx(src->format, src_width0);
665 src_height0 = util_format_get_nblocksy(src->format, src_height0);
666
667 dstx = util_format_get_nblocksx(dst->format, dstx);
668 dsty = util_format_get_nblocksy(dst->format, dsty);
669
670 sbox.x = util_format_get_nblocksx(src->format, src_box->x);
671 sbox.y = util_format_get_nblocksy(src->format, src_box->y);
672 sbox.z = src_box->z;
673 sbox.width = util_format_get_nblocksx(src->format, src_box->width);
674 sbox.height = util_format_get_nblocksy(src->format, src_box->height);
675 sbox.depth = src_box->depth;
676 src_box = &sbox;
677
678 src_force_level = src_level;
679 } else if (!util_blitter_is_copy_supported(sctx->blitter, dst, src) ||
680 /* also *8_SNORM has precision issues, use UNORM instead */
681 util_format_is_snorm8(src->format)) {
682 if (util_format_is_subsampled_422(src->format)) {
683 src_templ.format = PIPE_FORMAT_R8G8B8A8_UINT;
684 dst_templ.format = PIPE_FORMAT_R8G8B8A8_UINT;
685
686 dst_width = util_format_get_nblocksx(dst->format, dst_width);
687 src_width0 = util_format_get_nblocksx(src->format, src_width0);
688
689 dstx = util_format_get_nblocksx(dst->format, dstx);
690
691 sbox = *src_box;
692 sbox.x = util_format_get_nblocksx(src->format, src_box->x);
693 sbox.width = util_format_get_nblocksx(src->format, src_box->width);
694 src_box = &sbox;
695 } else {
696 unsigned blocksize = util_format_get_blocksize(src->format);
697
698 switch (blocksize) {
699 case 1:
700 dst_templ.format = PIPE_FORMAT_R8_UNORM;
701 src_templ.format = PIPE_FORMAT_R8_UNORM;
702 break;
703 case 2:
704 dst_templ.format = PIPE_FORMAT_R8G8_UNORM;
705 src_templ.format = PIPE_FORMAT_R8G8_UNORM;
706 break;
707 case 4:
708 dst_templ.format = PIPE_FORMAT_R8G8B8A8_UNORM;
709 src_templ.format = PIPE_FORMAT_R8G8B8A8_UNORM;
710 break;
711 case 8:
712 dst_templ.format = PIPE_FORMAT_R16G16B16A16_UINT;
713 src_templ.format = PIPE_FORMAT_R16G16B16A16_UINT;
714 break;
715 case 16:
716 dst_templ.format = PIPE_FORMAT_R32G32B32A32_UINT;
717 src_templ.format = PIPE_FORMAT_R32G32B32A32_UINT;
718 break;
719 default:
720 fprintf(stderr, "Unhandled format %s with blocksize %u\n",
721 util_format_short_name(src->format), blocksize);
722 assert(0);
723 }
724 }
725 }
726
727 /* Initialize the surface. */
728 dst_view = r600_create_surface_custom(ctx, dst, &dst_templ,
729 dst_width, dst_height);
730
731 /* Initialize the sampler view. */
732 src_view = si_create_sampler_view_custom(ctx, src, &src_templ,
733 src_width0, src_height0,
734 src_force_level);
735
736 u_box_3d(dstx, dsty, dstz, abs(src_box->width), abs(src_box->height),
737 abs(src_box->depth), &dstbox);
738
739 /* Copy. */
740 si_blitter_begin(ctx, SI_COPY);
741 util_blitter_blit_generic(sctx->blitter, dst_view, &dstbox,
742 src_view, src_box, src_width0, src_height0,
743 PIPE_MASK_RGBAZS, PIPE_TEX_FILTER_NEAREST, NULL,
744 FALSE);
745 si_blitter_end(ctx);
746
747 pipe_surface_reference(&dst_view, NULL);
748 pipe_sampler_view_reference(&src_view, NULL);
749 }
750
751 /* For MSAA integer resolving to work, we change the format to NORM using this function. */
752 static enum pipe_format int_to_norm_format(enum pipe_format format)
753 {
754 switch (format) {
755 #define REPLACE_FORMAT_SIGN(format,sign) \
756 case PIPE_FORMAT_##format##_##sign##INT: \
757 return PIPE_FORMAT_##format##_##sign##NORM
758 #define REPLACE_FORMAT(format) \
759 REPLACE_FORMAT_SIGN(format, U); \
760 REPLACE_FORMAT_SIGN(format, S)
761
762 REPLACE_FORMAT_SIGN(B10G10R10A2, U);
763 REPLACE_FORMAT(R8);
764 REPLACE_FORMAT(R8G8);
765 REPLACE_FORMAT(R8G8B8X8);
766 REPLACE_FORMAT(R8G8B8A8);
767 REPLACE_FORMAT(A8);
768 REPLACE_FORMAT(I8);
769 REPLACE_FORMAT(L8);
770 REPLACE_FORMAT(L8A8);
771 REPLACE_FORMAT(R16);
772 REPLACE_FORMAT(R16G16);
773 REPLACE_FORMAT(R16G16B16X16);
774 REPLACE_FORMAT(R16G16B16A16);
775 REPLACE_FORMAT(A16);
776 REPLACE_FORMAT(I16);
777 REPLACE_FORMAT(L16);
778 REPLACE_FORMAT(L16A16);
779
780 #undef REPLACE_FORMAT
781 #undef REPLACE_FORMAT_SIGN
782 default:
783 return format;
784 }
785 }
786
787 static bool do_hardware_msaa_resolve(struct pipe_context *ctx,
788 const struct pipe_blit_info *info)
789 {
790 struct si_context *sctx = (struct si_context*)ctx;
791 struct r600_texture *dst = (struct r600_texture*)info->dst.resource;
792 unsigned dst_width = u_minify(info->dst.resource->width0, info->dst.level);
793 unsigned dst_height = u_minify(info->dst.resource->height0, info->dst.level);
794 enum pipe_format format = int_to_norm_format(info->dst.format);
795 unsigned sample_mask = ~0;
796
797 /* Hardware MSAA resolve doesn't work if SPI format = NORM16_ABGR and
798 * the format is R16G16. Use R16A16, which does work.
799 */
800 if (format == PIPE_FORMAT_R16G16_UNORM)
801 format = PIPE_FORMAT_R16A16_UNORM;
802 if (format == PIPE_FORMAT_R16G16_SNORM)
803 format = PIPE_FORMAT_R16A16_SNORM;
804
805 if (info->src.resource->nr_samples > 1 &&
806 info->dst.resource->nr_samples <= 1 &&
807 util_max_layer(info->src.resource, 0) == 0 &&
808 util_max_layer(info->dst.resource, info->dst.level) == 0 &&
809 util_is_format_compatible(util_format_description(info->src.format),
810 util_format_description(info->dst.format)) &&
811 !util_format_is_pure_integer(format) &&
812 !util_format_is_depth_or_stencil(format) &&
813 !info->scissor_enable &&
814 (info->mask & PIPE_MASK_RGBA) == PIPE_MASK_RGBA &&
815 dst_width == info->src.resource->width0 &&
816 dst_height == info->src.resource->height0 &&
817 info->dst.box.x == 0 &&
818 info->dst.box.y == 0 &&
819 info->dst.box.width == dst_width &&
820 info->dst.box.height == dst_height &&
821 info->dst.box.depth == 1 &&
822 info->src.box.x == 0 &&
823 info->src.box.y == 0 &&
824 info->src.box.width == dst_width &&
825 info->src.box.height == dst_height &&
826 info->src.box.depth == 1 &&
827 dst->surface.level[info->dst.level].mode >= RADEON_SURF_MODE_1D &&
828 !(dst->surface.flags & RADEON_SURF_SCANOUT) &&
829 (!dst->cmask.size || !dst->dirty_level_mask) && /* dst cannot be fast-cleared */
830 !dst->dcc_offset) {
831 si_blitter_begin(ctx, SI_COLOR_RESOLVE |
832 (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
833 util_blitter_custom_resolve_color(sctx->blitter,
834 info->dst.resource, info->dst.level,
835 info->dst.box.z,
836 info->src.resource, info->src.box.z,
837 sample_mask, sctx->custom_blend_resolve,
838 format);
839 si_blitter_end(ctx);
840 return true;
841 }
842 return false;
843 }
844
845 static void si_blit(struct pipe_context *ctx,
846 const struct pipe_blit_info *info)
847 {
848 struct si_context *sctx = (struct si_context*)ctx;
849
850 if (do_hardware_msaa_resolve(ctx, info)) {
851 return;
852 }
853
854 assert(util_blitter_is_blit_supported(sctx->blitter, info));
855
856 /* The driver doesn't decompress resources automatically while
857 * u_blitter is rendering. */
858 si_decompress_subresource(ctx, info->src.resource, info->mask,
859 info->src.level,
860 info->src.box.z,
861 info->src.box.z + info->src.box.depth - 1);
862
863 if (sctx->screen->b.debug_flags & DBG_FORCE_DMA &&
864 util_try_blit_via_copy_region(ctx, info))
865 return;
866
867 si_blitter_begin(ctx, SI_BLIT |
868 (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
869 util_blitter_blit(sctx->blitter, info);
870 si_blitter_end(ctx);
871 }
872
873 static void si_flush_resource(struct pipe_context *ctx,
874 struct pipe_resource *res)
875 {
876 struct r600_texture *rtex = (struct r600_texture*)res;
877
878 assert(res->target != PIPE_BUFFER);
879
880 if (!rtex->is_depth && (rtex->cmask.size || rtex->dcc_offset)) {
881 si_blit_decompress_color(ctx, rtex, 0, res->last_level,
882 0, util_max_layer(res, 0), false);
883 }
884 }
885
886 static void si_decompress_dcc(struct pipe_context *ctx,
887 struct r600_texture *rtex)
888 {
889 if (!rtex->dcc_offset)
890 return;
891
892 si_blit_decompress_color(ctx, rtex, 0, rtex->resource.b.b.last_level,
893 0, util_max_layer(&rtex->resource.b.b, 0),
894 true);
895 }
896
897 static void si_pipe_clear_buffer(struct pipe_context *ctx,
898 struct pipe_resource *dst,
899 unsigned offset, unsigned size,
900 const void *clear_value_ptr,
901 int clear_value_size)
902 {
903 struct si_context *sctx = (struct si_context*)ctx;
904 uint32_t dword_value;
905 unsigned i;
906
907 assert(offset % clear_value_size == 0);
908 assert(size % clear_value_size == 0);
909
910 if (clear_value_size > 4) {
911 const uint32_t *u32 = clear_value_ptr;
912 bool clear_dword_duplicated = true;
913
914 /* See if we can lower large fills to dword fills. */
915 for (i = 1; i < clear_value_size / 4; i++)
916 if (u32[0] != u32[i]) {
917 clear_dword_duplicated = false;
918 break;
919 }
920
921 if (!clear_dword_duplicated) {
922 /* Use transform feedback for 64-bit, 96-bit, and
923 * 128-bit fills.
924 */
925 union pipe_color_union clear_value;
926
927 memcpy(&clear_value, clear_value_ptr, clear_value_size);
928 si_blitter_begin(ctx, SI_DISABLE_RENDER_COND);
929 util_blitter_clear_buffer(sctx->blitter, dst, offset,
930 size, clear_value_size / 4,
931 &clear_value);
932 si_blitter_end(ctx);
933 return;
934 }
935 }
936
937 /* Expand the clear value to a dword. */
938 switch (clear_value_size) {
939 case 1:
940 dword_value = *(uint8_t*)clear_value_ptr;
941 dword_value |= (dword_value << 8) |
942 (dword_value << 16) |
943 (dword_value << 24);
944 break;
945 case 2:
946 dword_value = *(uint16_t*)clear_value_ptr;
947 dword_value |= dword_value << 16;
948 break;
949 default:
950 dword_value = *(uint32_t*)clear_value_ptr;
951 }
952
953 sctx->b.clear_buffer(ctx, dst, offset, size, dword_value,
954 R600_COHERENCY_SHADER);
955 }
956
957 void si_init_blit_functions(struct si_context *sctx)
958 {
959 sctx->b.b.clear = si_clear;
960 sctx->b.b.clear_buffer = si_pipe_clear_buffer;
961 sctx->b.b.clear_render_target = si_clear_render_target;
962 sctx->b.b.clear_depth_stencil = si_clear_depth_stencil;
963 sctx->b.b.resource_copy_region = si_resource_copy_region;
964 sctx->b.b.blit = si_blit;
965 sctx->b.b.flush_resource = si_flush_resource;
966 sctx->b.blit_decompress_depth = si_blit_decompress_depth;
967 sctx->b.decompress_dcc = si_decompress_dcc;
968 }