X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fdrivers%2Fradeonsi%2Fsi_blit.c;h=68ed4cc5002e2744a01bccd48852d80e4e561e0e;hb=abc226cf41574454c79477c217e60e8ff1fddfad;hp=a88a5e22ca4c563453db5aff9c7178ddd4d636e5;hpb=5f1cddde78aa93ea1272c50a93e479cb76144af7;p=mesa.git diff --git a/src/gallium/drivers/radeonsi/si_blit.c b/src/gallium/drivers/radeonsi/si_blit.c index a88a5e22ca4..68ed4cc5002 100644 --- a/src/gallium/drivers/radeonsi/si_blit.c +++ b/src/gallium/drivers/radeonsi/si_blit.c @@ -57,8 +57,12 @@ void si_blitter_begin(struct si_context *sctx, enum si_blitter_op op) util_blitter_save_depth_stencil_alpha(sctx->blitter, sctx->queued.named.dsa); util_blitter_save_stencil_ref(sctx->blitter, &sctx->stencil_ref.state); util_blitter_save_fragment_shader(sctx->blitter, sctx->ps_shader.cso); - util_blitter_save_sample_mask(sctx->blitter, sctx->sample_mask.sample_mask); - util_blitter_save_scissor(sctx->blitter, &sctx->scissors.states[0]); + util_blitter_save_sample_mask(sctx->blitter, sctx->sample_mask); + util_blitter_save_scissor(sctx->blitter, &sctx->scissors[0]); + util_blitter_save_window_rectangles(sctx->blitter, + sctx->window_rectangles_include, + sctx->num_window_rectangles, + sctx->window_rectangles); } if (op & SI_SAVE_FRAMEBUFFER) @@ -74,18 +78,28 @@ void si_blitter_begin(struct si_context *sctx, enum si_blitter_op op) } if (op & SI_DISABLE_RENDER_COND) - sctx->b.render_cond_force_off = true; + sctx->render_cond_force_off = true; + + if (sctx->screen->dpbb_allowed) { + sctx->dpbb_force_off = true; + si_mark_atom_dirty(sctx, &sctx->atoms.s.dpbb_state); + } } void si_blitter_end(struct si_context *sctx) { - sctx->b.render_cond_force_off = false; + if (sctx->screen->dpbb_allowed) { + sctx->dpbb_force_off = false; + si_mark_atom_dirty(sctx, &sctx->atoms.s.dpbb_state); + } + + sctx->render_cond_force_off = false; /* Restore shader pointers because the VS blit shader changed all * non-global VS user SGPRs. */ sctx->shader_pointers_dirty |= SI_DESCS_SHADER_MASK(VERTEX); sctx->vertex_buffer_pointer_dirty = sctx->vb_descriptors_buffer != NULL; - si_mark_atom_dirty(sctx, &sctx->shader_pointers.atom); + si_mark_atom_dirty(sctx, &sctx->atoms.s.shader_pointers); } static unsigned u_max_sample(struct pipe_resource *r) @@ -95,8 +109,8 @@ static unsigned u_max_sample(struct pipe_resource *r) static unsigned si_blit_dbcb_copy(struct si_context *sctx, - struct r600_texture *src, - struct r600_texture *dst, + struct si_texture *src, + struct si_texture *dst, unsigned planes, unsigned level_mask, unsigned first_layer, unsigned last_layer, unsigned first_sample, unsigned last_sample) @@ -109,7 +123,7 @@ si_blit_dbcb_copy(struct si_context *sctx, sctx->dbcb_depth_copy_enabled = true; if (planes & PIPE_MASK_S) sctx->dbcb_stencil_copy_enabled = true; - si_mark_atom_dirty(sctx, &sctx->db_render_state); + si_mark_atom_dirty(sctx, &sctx->atoms.s.db_render_state); assert(sctx->dbcb_depth_copy_enabled || sctx->dbcb_stencil_copy_enabled); @@ -120,7 +134,7 @@ si_blit_dbcb_copy(struct si_context *sctx, /* The smaller the mipmap level, the less layers there are * as far as 3D textures are concerned. */ - max_layer = util_max_layer(&src->resource.b.b, level); + max_layer = util_max_layer(&src->buffer.b.b, level); checked_last_layer = MIN2(last_layer, max_layer); surf_tmpl.u.tex.level = level; @@ -128,19 +142,19 @@ si_blit_dbcb_copy(struct si_context *sctx, for (layer = first_layer; layer <= checked_last_layer; layer++) { struct pipe_surface *zsurf, *cbsurf; - surf_tmpl.format = src->resource.b.b.format; + surf_tmpl.format = src->buffer.b.b.format; surf_tmpl.u.tex.first_layer = layer; surf_tmpl.u.tex.last_layer = layer; - zsurf = sctx->b.b.create_surface(&sctx->b.b, &src->resource.b.b, &surf_tmpl); + zsurf = sctx->b.create_surface(&sctx->b, &src->buffer.b.b, &surf_tmpl); - surf_tmpl.format = dst->resource.b.b.format; - cbsurf = sctx->b.b.create_surface(&sctx->b.b, &dst->resource.b.b, &surf_tmpl); + surf_tmpl.format = dst->buffer.b.b.format; + cbsurf = sctx->b.create_surface(&sctx->b, &dst->buffer.b.b, &surf_tmpl); for (sample = first_sample; sample <= last_sample; sample++) { if (sample != sctx->dbcb_copy_sample) { sctx->dbcb_copy_sample = sample; - si_mark_atom_dirty(sctx, &sctx->db_render_state); + si_mark_atom_dirty(sctx, &sctx->atoms.s.db_render_state); } si_blitter_begin(sctx, SI_DECOMPRESS); @@ -154,48 +168,23 @@ si_blit_dbcb_copy(struct si_context *sctx, } if (first_layer == 0 && last_layer >= max_layer && - first_sample == 0 && last_sample >= u_max_sample(&src->resource.b.b)) + first_sample == 0 && last_sample >= u_max_sample(&src->buffer.b.b)) fully_copied_levels |= 1u << level; } sctx->decompression_enabled = false; sctx->dbcb_depth_copy_enabled = false; sctx->dbcb_stencil_copy_enabled = false; - si_mark_atom_dirty(sctx, &sctx->db_render_state); + si_mark_atom_dirty(sctx, &sctx->atoms.s.db_render_state); return fully_copied_levels; } -void si_blit_decompress_depth(struct pipe_context *ctx, - struct r600_texture *texture, - struct r600_texture *staging, - unsigned first_level, unsigned last_level, - unsigned first_layer, unsigned last_layer, - unsigned first_sample, unsigned last_sample) -{ - const struct util_format_description *desc; - unsigned planes = 0; - - assert(staging != NULL && "use si_blit_decompress_zs_in_place instead"); - - desc = util_format_description(staging->resource.b.b.format); - - if (util_format_has_depth(desc)) - planes |= PIPE_MASK_Z; - if (util_format_has_stencil(desc)) - planes |= PIPE_MASK_S; - - si_blit_dbcb_copy( - (struct si_context *)ctx, texture, staging, planes, - u_bit_consecutive(first_level, last_level - first_level + 1), - first_layer, last_layer, first_sample, last_sample); -} - /* Helper function for si_blit_decompress_zs_in_place. */ static void si_blit_decompress_zs_planes_in_place(struct si_context *sctx, - struct r600_texture *texture, + struct si_texture *texture, unsigned planes, unsigned level_mask, unsigned first_layer, unsigned last_layer) { @@ -210,9 +199,9 @@ si_blit_decompress_zs_planes_in_place(struct si_context *sctx, sctx->db_flush_stencil_inplace = true; if (planes & PIPE_MASK_Z) sctx->db_flush_depth_inplace = true; - si_mark_atom_dirty(sctx, &sctx->db_render_state); + si_mark_atom_dirty(sctx, &sctx->atoms.s.db_render_state); - surf_tmpl.format = texture->resource.b.b.format; + surf_tmpl.format = texture->buffer.b.b.format; sctx->decompression_enabled = true; @@ -223,14 +212,14 @@ si_blit_decompress_zs_planes_in_place(struct si_context *sctx, /* The smaller the mipmap level, the less layers there are * as far as 3D textures are concerned. */ - max_layer = util_max_layer(&texture->resource.b.b, level); + max_layer = util_max_layer(&texture->buffer.b.b, level); checked_last_layer = MIN2(last_layer, max_layer); for (layer = first_layer; layer <= checked_last_layer; layer++) { surf_tmpl.u.tex.first_layer = layer; surf_tmpl.u.tex.last_layer = layer; - zsurf = sctx->b.b.create_surface(&sctx->b.b, &texture->resource.b.b, &surf_tmpl); + zsurf = sctx->b.create_surface(&sctx->b, &texture->buffer.b.b, &surf_tmpl); si_blitter_begin(sctx, SI_DECOMPRESS); util_blitter_custom_depth_stencil(sctx->blitter, zsurf, NULL, ~0, @@ -256,7 +245,7 @@ si_blit_decompress_zs_planes_in_place(struct si_context *sctx, sctx->decompression_enabled = false; sctx->db_flush_depth_inplace = false; sctx->db_flush_stencil_inplace = false; - si_mark_atom_dirty(sctx, &sctx->db_render_state); + si_mark_atom_dirty(sctx, &sctx->atoms.s.db_render_state); } /* Helper function of si_flush_depth_texture: decompress the given levels @@ -264,7 +253,7 @@ si_blit_decompress_zs_planes_in_place(struct si_context *sctx, */ static void si_blit_decompress_zs_in_place(struct si_context *sctx, - struct r600_texture *texture, + struct si_texture *texture, unsigned levels_z, unsigned levels_s, unsigned first_layer, unsigned last_layer) { @@ -298,7 +287,7 @@ si_blit_decompress_zs_in_place(struct si_context *sctx, static void si_decompress_depth(struct si_context *sctx, - struct r600_texture *tex, + struct si_texture *tex, unsigned required_planes, unsigned first_level, unsigned last_level, unsigned first_layer, unsigned last_layer) @@ -330,8 +319,8 @@ si_decompress_depth(struct si_context *sctx, } } - if (unlikely(sctx->b.log)) - u_log_printf(sctx->b.log, + if (unlikely(sctx->log)) + u_log_printf(sctx->log, "\n------------------------------------------------\n" "Decompress Depth (levels %u - %u, levels Z: 0x%x S: 0x%x)\n\n", first_level, last_level, levels_z, levels_s); @@ -341,14 +330,14 @@ si_decompress_depth(struct si_context *sctx, */ if (copy_planes && (tex->flushed_depth_texture || - si_init_flushed_depth_texture(&sctx->b.b, &tex->resource.b.b, NULL))) { - struct r600_texture *dst = tex->flushed_depth_texture; + si_init_flushed_depth_texture(&sctx->b, &tex->buffer.b.b))) { + struct si_texture *dst = tex->flushed_depth_texture; unsigned fully_copied_levels; unsigned levels = 0; assert(tex->flushed_depth_texture); - if (util_format_is_depth_and_stencil(dst->resource.b.b.format)) + if (util_format_is_depth_and_stencil(dst->buffer.b.b.format)) copy_planes = PIPE_MASK_Z | PIPE_MASK_S; if (copy_planes & PIPE_MASK_Z) { @@ -363,7 +352,7 @@ si_decompress_depth(struct si_context *sctx, fully_copied_levels = si_blit_dbcb_copy( sctx, tex, dst, copy_planes, levels, first_layer, last_layer, - 0, u_max_sample(&tex->resource.b.b)); + 0, u_max_sample(&tex->buffer.b.b)); if (copy_planes & PIPE_MASK_Z) tex->dirty_level_mask &= ~fully_copied_levels; @@ -372,8 +361,9 @@ si_decompress_depth(struct si_context *sctx, } if (inplace_planes) { - bool has_htile = si_htile_enabled(tex, first_level); - bool tc_compat_htile = vi_tc_compat_htile_enabled(tex, first_level); + bool has_htile = si_htile_enabled(tex, first_level, inplace_planes); + bool tc_compat_htile = vi_tc_compat_htile_enabled(tex, first_level, + inplace_planes); /* Don't decompress if there is no HTILE or when HTILE is * TC-compatible. */ @@ -398,16 +388,16 @@ si_decompress_depth(struct si_context *sctx, /* Only in-place decompression needs to flush DB caches, or * when we don't decompress but TC-compatible planes are dirty. */ - si_make_DB_shader_coherent(sctx, tex->resource.b.b.nr_samples, + si_make_DB_shader_coherent(sctx, tex->buffer.b.b.nr_samples, inplace_planes & PIPE_MASK_S, tc_compat_htile); } /* set_framebuffer_state takes care of coherency for single-sample. * The DB->CB copy uses CB for the final writes. */ - if (copy_planes && tex->resource.b.b.nr_samples > 1) - si_make_CB_shader_coherent(sctx, tex->resource.b.b.nr_samples, - false); + if (copy_planes && tex->buffer.b.b.nr_samples > 1) + si_make_CB_shader_coherent(sctx, tex->buffer.b.b.nr_samples, + false, true /* no DCC */); } static void @@ -420,7 +410,7 @@ si_decompress_sampler_depth_textures(struct si_context *sctx, while (mask) { struct pipe_sampler_view *view; struct si_sampler_view *sview; - struct r600_texture *tex; + struct si_texture *tex; i = u_bit_scan(&mask); @@ -428,21 +418,21 @@ si_decompress_sampler_depth_textures(struct si_context *sctx, assert(view); sview = (struct si_sampler_view*)view; - tex = (struct r600_texture *)view->texture; + tex = (struct si_texture *)view->texture; assert(tex->db_compatible); si_decompress_depth(sctx, tex, sview->is_stencil_sampler ? PIPE_MASK_S : PIPE_MASK_Z, view->u.tex.first_level, view->u.tex.last_level, - 0, util_max_layer(&tex->resource.b.b, view->u.tex.first_level)); + 0, util_max_layer(&tex->buffer.b.b, view->u.tex.first_level)); } } static void si_blit_decompress_color(struct si_context *sctx, - struct r600_texture *rtex, - unsigned first_level, unsigned last_level, - unsigned first_layer, unsigned last_layer, - bool need_dcc_decompress) + struct si_texture *tex, + unsigned first_level, unsigned last_level, + unsigned first_layer, unsigned last_layer, + bool need_dcc_decompress) { void* custom_blend; unsigned layer, checked_last_layer, max_layer; @@ -450,12 +440,12 @@ static void si_blit_decompress_color(struct si_context *sctx, u_bit_consecutive(first_level, last_level - first_level + 1); if (!need_dcc_decompress) - level_mask &= rtex->dirty_level_mask; + level_mask &= tex->dirty_level_mask; if (!level_mask) return; - if (unlikely(sctx->b.log)) - u_log_printf(sctx->b.log, + if (unlikely(sctx->log)) + u_log_printf(sctx->log, "\n------------------------------------------------\n" "Decompress Color (levels %u - %u, mask 0x%x)\n\n", first_level, last_level, level_mask); @@ -463,14 +453,14 @@ static void si_blit_decompress_color(struct si_context *sctx, if (need_dcc_decompress) { custom_blend = sctx->custom_blend_dcc_decompress; - assert(rtex->dcc_offset); + assert(tex->dcc_offset); /* disable levels without DCC */ for (int i = first_level; i <= last_level; i++) { - if (!vi_dcc_enabled(rtex, i)) + if (!vi_dcc_enabled(tex, i)) level_mask &= ~(1 << i); } - } else if (rtex->fmask.size) { + } else if (tex->surface.fmask_size) { custom_blend = sctx->custom_blend_fmask_decompress; } else { custom_blend = sctx->custom_blend_eliminate_fastclear; @@ -483,22 +473,22 @@ static void si_blit_decompress_color(struct si_context *sctx, /* The smaller the mipmap level, the less layers there are * as far as 3D textures are concerned. */ - max_layer = util_max_layer(&rtex->resource.b.b, level); + max_layer = util_max_layer(&tex->buffer.b.b, level); checked_last_layer = MIN2(last_layer, max_layer); for (layer = first_layer; layer <= checked_last_layer; layer++) { struct pipe_surface *cbsurf, surf_tmpl; - surf_tmpl.format = rtex->resource.b.b.format; + surf_tmpl.format = tex->buffer.b.b.format; surf_tmpl.u.tex.level = level; surf_tmpl.u.tex.first_layer = layer; surf_tmpl.u.tex.last_layer = layer; - cbsurf = sctx->b.b.create_surface(&sctx->b.b, &rtex->resource.b.b, &surf_tmpl); + cbsurf = sctx->b.create_surface(&sctx->b, &tex->buffer.b.b, &surf_tmpl); /* Required before and after FMASK and DCC_DECOMPRESS. */ if (custom_blend == sctx->custom_blend_fmask_decompress || custom_blend == sctx->custom_blend_dcc_decompress) - sctx->b.flags |= SI_CONTEXT_FLUSH_AND_INV_CB; + sctx->flags |= SI_CONTEXT_FLUSH_AND_INV_CB; si_blitter_begin(sctx, SI_DECOMPRESS); util_blitter_custom_color(sctx->blitter, cbsurf, custom_blend); @@ -506,7 +496,7 @@ static void si_blit_decompress_color(struct si_context *sctx, if (custom_blend == sctx->custom_blend_fmask_decompress || custom_blend == sctx->custom_blend_dcc_decompress) - sctx->b.flags |= SI_CONTEXT_FLUSH_AND_INV_CB; + sctx->flags |= SI_CONTEXT_FLUSH_AND_INV_CB; pipe_surface_reference(&cbsurf, NULL); } @@ -514,25 +504,26 @@ static void si_blit_decompress_color(struct si_context *sctx, /* The texture will always be dirty if some layers aren't flushed. * I don't think this case occurs often though. */ if (first_layer == 0 && last_layer >= max_layer) { - rtex->dirty_level_mask &= ~(1 << level); + tex->dirty_level_mask &= ~(1 << level); } } sctx->decompression_enabled = false; - si_make_CB_shader_coherent(sctx, rtex->resource.b.b.nr_samples, - vi_dcc_enabled(rtex, first_level)); + si_make_CB_shader_coherent(sctx, tex->buffer.b.b.nr_samples, + vi_dcc_enabled(tex, first_level), + tex->surface.u.gfx9.dcc.pipe_aligned); } static void -si_decompress_color_texture(struct si_context *sctx, struct r600_texture *tex, +si_decompress_color_texture(struct si_context *sctx, struct si_texture *tex, unsigned first_level, unsigned last_level) { /* CMASK or DCC can be discarded and we can still end up here. */ - if (!tex->cmask.size && !tex->fmask.size && !tex->dcc_offset) + if (!tex->cmask_buffer && !tex->surface.fmask_size && !tex->dcc_offset) return; si_blit_decompress_color(sctx, tex, first_level, last_level, 0, - util_max_layer(&tex->resource.b.b, first_level), + util_max_layer(&tex->buffer.b.b, first_level), false); } @@ -545,14 +536,14 @@ si_decompress_sampler_color_textures(struct si_context *sctx, while (mask) { struct pipe_sampler_view *view; - struct r600_texture *tex; + struct si_texture *tex; i = u_bit_scan(&mask); view = textures->views[i]; assert(view); - tex = (struct r600_texture *)view->texture; + tex = (struct si_texture *)view->texture; si_decompress_color_texture(sctx, tex, view->u.tex.first_level, view->u.tex.last_level); @@ -568,14 +559,14 @@ si_decompress_image_color_textures(struct si_context *sctx, while (mask) { const struct pipe_image_view *view; - struct r600_texture *tex; + struct si_texture *tex; i = u_bit_scan(&mask); view = &images->views[i]; assert(view->resource->target != PIPE_BUFFER); - tex = (struct r600_texture *)view->resource; + tex = (struct si_texture *)view->resource; si_decompress_color_texture(sctx, tex, view->u.tex.level, view->u.tex.level); @@ -583,7 +574,7 @@ si_decompress_image_color_textures(struct si_context *sctx, } static void si_check_render_feedback_texture(struct si_context *sctx, - struct r600_texture *tex, + struct si_texture *tex, unsigned first_level, unsigned last_level, unsigned first_layer, @@ -595,14 +586,14 @@ static void si_check_render_feedback_texture(struct si_context *sctx, return; for (unsigned j = 0; j < sctx->framebuffer.state.nr_cbufs; ++j) { - struct r600_surface * surf; + struct si_surface * surf; if (!sctx->framebuffer.state.cbufs[j]) continue; - surf = (struct r600_surface*)sctx->framebuffer.state.cbufs[j]; + surf = (struct si_surface*)sctx->framebuffer.state.cbufs[j]; - if (tex == (struct r600_texture *)surf->base.texture && + if (tex == (struct si_texture *)surf->base.texture && surf->base.u.tex.level >= first_level && surf->base.u.tex.level <= last_level && surf->base.u.tex.first_layer <= last_layer && @@ -623,7 +614,7 @@ static void si_check_render_feedback_textures(struct si_context *sctx, while (mask) { const struct pipe_sampler_view *view; - struct r600_texture *tex; + struct si_texture *tex; unsigned i = u_bit_scan(&mask); @@ -631,7 +622,7 @@ static void si_check_render_feedback_textures(struct si_context *sctx, if(view->texture->target == PIPE_BUFFER) continue; - tex = (struct r600_texture *)view->texture; + tex = (struct si_texture *)view->texture; si_check_render_feedback_texture(sctx, tex, view->u.tex.first_level, @@ -648,7 +639,7 @@ static void si_check_render_feedback_images(struct si_context *sctx, while (mask) { const struct pipe_image_view *view; - struct r600_texture *tex; + struct si_texture *tex; unsigned i = u_bit_scan(&mask); @@ -656,7 +647,7 @@ static void si_check_render_feedback_images(struct si_context *sctx, if (view->resource->target == PIPE_BUFFER) continue; - tex = (struct r600_texture *)view->resource; + tex = (struct si_texture *)view->resource; si_check_render_feedback_texture(sctx, tex, view->u.tex.level, @@ -671,13 +662,13 @@ static void si_check_render_feedback_resident_textures(struct si_context *sctx) util_dynarray_foreach(&sctx->resident_tex_handles, struct si_texture_handle *, tex_handle) { struct pipe_sampler_view *view; - struct r600_texture *tex; + struct si_texture *tex; view = (*tex_handle)->view; if (view->texture->target == PIPE_BUFFER) continue; - tex = (struct r600_texture *)view->texture; + tex = (struct si_texture *)view->texture; si_check_render_feedback_texture(sctx, tex, view->u.tex.first_level, @@ -692,13 +683,13 @@ static void si_check_render_feedback_resident_images(struct si_context *sctx) util_dynarray_foreach(&sctx->resident_img_handles, struct si_image_handle *, img_handle) { struct pipe_image_view *view; - struct r600_texture *tex; + struct si_texture *tex; view = &(*img_handle)->view; if (view->resource->target == PIPE_BUFFER) continue; - tex = (struct r600_texture *)view->resource; + tex = (struct si_texture *)view->resource; si_check_render_feedback_texture(sctx, tex, view->u.tex.level, @@ -710,15 +701,15 @@ static void si_check_render_feedback_resident_images(struct si_context *sctx) static void si_check_render_feedback(struct si_context *sctx) { + if (!sctx->need_check_render_feedback) + return; + /* There is no render feedback if color writes are disabled. * (e.g. a pixel shader with image stores) */ if (!si_get_total_colormask(sctx)) return; - if (!sctx->need_check_render_feedback) - return; - for (int i = 0; i < SI_NUM_SHADERS; ++i) { si_check_render_feedback_images(sctx, &sctx->images[i]); si_check_render_feedback_textures(sctx, &sctx->samplers[i]); @@ -735,7 +726,7 @@ static void si_decompress_resident_textures(struct si_context *sctx) util_dynarray_foreach(&sctx->resident_tex_needs_color_decompress, struct si_texture_handle *, tex_handle) { struct pipe_sampler_view *view = (*tex_handle)->view; - struct r600_texture *tex = (struct r600_texture *)view->texture; + struct si_texture *tex = (struct si_texture *)view->texture; si_decompress_color_texture(sctx, tex, view->u.tex.first_level, view->u.tex.last_level); @@ -745,12 +736,12 @@ static void si_decompress_resident_textures(struct si_context *sctx) struct si_texture_handle *, tex_handle) { struct pipe_sampler_view *view = (*tex_handle)->view; struct si_sampler_view *sview = (struct si_sampler_view *)view; - struct r600_texture *tex = (struct r600_texture *)view->texture; + struct si_texture *tex = (struct si_texture *)view->texture; si_decompress_depth(sctx, tex, sview->is_stencil_sampler ? PIPE_MASK_S : PIPE_MASK_Z, view->u.tex.first_level, view->u.tex.last_level, - 0, util_max_layer(&tex->resource.b.b, view->u.tex.first_level)); + 0, util_max_layer(&tex->buffer.b.b, view->u.tex.first_level)); } } @@ -759,7 +750,7 @@ static void si_decompress_resident_images(struct si_context *sctx) util_dynarray_foreach(&sctx->resident_img_needs_color_decompress, struct si_image_handle *, img_handle) { struct pipe_image_view *view = &(*img_handle)->view; - struct r600_texture *tex = (struct r600_texture *)view->resource; + struct si_texture *tex = (struct si_texture *)view->resource; si_decompress_color_texture(sctx, tex, view->u.tex.level, view->u.tex.level); @@ -775,8 +766,8 @@ void si_decompress_textures(struct si_context *sctx, unsigned shader_mask) /* Update the compressed_colortex_mask if necessary. */ compressed_colortex_counter = p_atomic_read(&sctx->screen->compressed_colortex_counter); - if (compressed_colortex_counter != sctx->b.last_compressed_colortex_counter) { - sctx->b.last_compressed_colortex_counter = compressed_colortex_counter; + if (compressed_colortex_counter != sctx->last_compressed_colortex_counter) { + sctx->last_compressed_colortex_counter = compressed_colortex_counter; si_update_needs_color_decompress_masks(sctx); } @@ -801,22 +792,22 @@ void si_decompress_textures(struct si_context *sctx, unsigned shader_mask) si_decompress_resident_textures(sctx); if (sctx->uses_bindless_images) si_decompress_resident_images(sctx); + + if (sctx->ps_uses_fbfetch) { + struct pipe_surface *cb0 = sctx->framebuffer.state.cbufs[0]; + si_decompress_color_texture(sctx, + (struct si_texture*)cb0->texture, + cb0->u.tex.first_layer, + cb0->u.tex.last_layer); + } + + si_check_render_feedback(sctx); } else if (shader_mask & (1 << PIPE_SHADER_COMPUTE)) { - if (sctx->cs_shader_state.program->uses_bindless_samplers) + if (sctx->cs_shader_state.program->sel.info.uses_bindless_samplers) si_decompress_resident_textures(sctx); - if (sctx->cs_shader_state.program->uses_bindless_images) + if (sctx->cs_shader_state.program->sel.info.uses_bindless_images) si_decompress_resident_images(sctx); } - - if (sctx->ps_uses_fbfetch) { - struct pipe_surface *cb0 = sctx->framebuffer.state.cbufs[0]; - si_decompress_color_texture(sctx, - (struct r600_texture*)cb0->texture, - cb0->u.tex.first_layer, - cb0->u.tex.last_layer); - } - - si_check_render_feedback(sctx); } /* Helper for decompressing a portion of a color or depth resource before @@ -829,12 +820,12 @@ static void si_decompress_subresource(struct pipe_context *ctx, unsigned first_layer, unsigned last_layer) { struct si_context *sctx = (struct si_context *)ctx; - struct r600_texture *rtex = (struct r600_texture*)tex; + struct si_texture *stex = (struct si_texture*)tex; - if (rtex->db_compatible) { + if (stex->db_compatible) { planes &= PIPE_MASK_Z | PIPE_MASK_S; - if (!rtex->surface.has_stencil) + if (!stex->surface.has_stencil) planes &= ~PIPE_MASK_S; /* If we've rendered into the framebuffer and it's a blitting @@ -846,10 +837,10 @@ static void si_decompress_subresource(struct pipe_context *ctx, sctx->framebuffer.state.zsbuf->texture == tex) si_update_fb_dirtiness_after_rendering(sctx); - si_decompress_depth(sctx, rtex, planes, + si_decompress_depth(sctx, stex, planes, level, level, first_layer, last_layer); - } else if (rtex->fmask.size || rtex->cmask.size || rtex->dcc_offset) { + } else if (stex->surface.fmask_size || stex->cmask_buffer || stex->dcc_offset) { /* If we've rendered into the framebuffer and it's a blitting * source, make sure the decompression pass is invoked * by dirtying the framebuffer. @@ -863,7 +854,7 @@ static void si_decompress_subresource(struct pipe_context *ctx, } } - si_blit_decompress_color(sctx, rtex, level, level, + si_blit_decompress_color(sctx, stex, level, level, first_layer, last_layer, false); } } @@ -887,7 +878,8 @@ void si_resource_copy_region(struct pipe_context *ctx, const struct pipe_box *src_box) { struct si_context *sctx = (struct si_context *)ctx; - struct r600_texture *rsrc = (struct r600_texture*)src; + struct si_texture *ssrc = (struct si_texture*)src; + struct si_texture *sdst = (struct si_texture*)dst; struct pipe_surface *dst_view, dst_templ; struct pipe_sampler_view src_templ, *src_view; unsigned dst_width, dst_height, src_width0, src_height0; @@ -896,7 +888,18 @@ void si_resource_copy_region(struct pipe_context *ctx, /* Handle buffers first. */ if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) { - si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width, 0); + si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width); + return; + } + + if (!util_format_is_compressed(src->format) && + !util_format_is_compressed(dst->format) && + !util_format_is_depth_or_stencil(src->format) && + src->nr_samples <= 1 && + !sdst->dcc_offset && + !(dst->target != src->target && + (src->target == PIPE_TEXTURE_1D_ARRAY || dst->target == PIPE_TEXTURE_1D_ARRAY))) { + si_compute_copy_image(sctx, dst, dst_level, src, src_level, dstx, dsty, dstz, src_box); return; } @@ -919,7 +922,7 @@ void si_resource_copy_region(struct pipe_context *ctx, if (util_format_is_compressed(src->format) || util_format_is_compressed(dst->format)) { - unsigned blocksize = rsrc->surface.bpe; + unsigned blocksize = ssrc->surface.bpe; if (blocksize == 8) src_templ.format = PIPE_FORMAT_R16G16B16A16_UINT; /* 64-bit block */ @@ -962,7 +965,7 @@ void si_resource_copy_region(struct pipe_context *ctx, sbox.width = util_format_get_nblocksx(src->format, src_box->width); src_box = &sbox; } else { - unsigned blocksize = rsrc->surface.bpe; + unsigned blocksize = ssrc->surface.bpe; switch (blocksize) { case 1: @@ -998,36 +1001,8 @@ void si_resource_copy_region(struct pipe_context *ctx, * Note that some chips avoid this issue by using SDMA. */ if (util_format_is_snorm8(dst_templ.format)) { - switch (dst_templ.format) { - case PIPE_FORMAT_R8_SNORM: - dst_templ.format = src_templ.format = PIPE_FORMAT_R8_SINT; - break; - case PIPE_FORMAT_R8G8_SNORM: - dst_templ.format = src_templ.format = PIPE_FORMAT_R8G8_SINT; - break; - case PIPE_FORMAT_R8G8B8X8_SNORM: - dst_templ.format = src_templ.format = PIPE_FORMAT_R8G8B8X8_SINT; - break; - case PIPE_FORMAT_R8G8B8A8_SNORM: - /* There are no SINT variants for ABGR and XBGR, so we have to use RGBA. */ - case PIPE_FORMAT_A8B8G8R8_SNORM: - case PIPE_FORMAT_X8B8G8R8_SNORM: - dst_templ.format = src_templ.format = PIPE_FORMAT_R8G8B8A8_SINT; - break; - case PIPE_FORMAT_A8_SNORM: - dst_templ.format = src_templ.format = PIPE_FORMAT_A8_SINT; - break; - case PIPE_FORMAT_L8_SNORM: - dst_templ.format = src_templ.format = PIPE_FORMAT_L8_SINT; - break; - case PIPE_FORMAT_L8A8_SNORM: - dst_templ.format = src_templ.format = PIPE_FORMAT_L8A8_SINT; - break; - case PIPE_FORMAT_I8_SNORM: - dst_templ.format = src_templ.format = PIPE_FORMAT_I8_SINT; - break; - default:; /* fall through */ - } + dst_templ.format = src_templ.format = + util_format_snorm8_to_sint8(dst_templ.format); } vi_disable_dcc_if_incompatible_format(sctx, dst, dst_level, @@ -1067,7 +1042,7 @@ static void si_do_CB_resolve(struct si_context *sctx, enum pipe_format format) { /* Required before and after CB_RESOLVE. */ - sctx->b.flags |= SI_CONTEXT_FLUSH_AND_INV_CB; + sctx->flags |= SI_CONTEXT_FLUSH_AND_INV_CB; si_blitter_begin(sctx, SI_COLOR_RESOLVE | (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND)); @@ -1078,16 +1053,16 @@ static void si_do_CB_resolve(struct si_context *sctx, si_blitter_end(sctx); /* Flush caches for possible texturing. */ - si_make_CB_shader_coherent(sctx, 1, false); + si_make_CB_shader_coherent(sctx, 1, false, true /* no DCC */); } static bool do_hardware_msaa_resolve(struct pipe_context *ctx, const struct pipe_blit_info *info) { struct si_context *sctx = (struct si_context*)ctx; - struct r600_texture *src = (struct r600_texture*)info->src.resource; - struct r600_texture *dst = (struct r600_texture*)info->dst.resource; - MAYBE_UNUSED struct r600_texture *rtmp; + struct si_texture *src = (struct si_texture*)info->src.resource; + struct si_texture *dst = (struct si_texture*)info->dst.resource; + ASSERTED struct si_texture *stmp; unsigned dst_width = u_minify(info->dst.resource->width0, info->dst.level); unsigned dst_height = u_minify(info->dst.resource->height0, info->dst.level); enum pipe_format format = info->src.format; @@ -1129,12 +1104,18 @@ static bool do_hardware_msaa_resolve(struct pipe_context *ctx, info->src.box.height == dst_height && info->src.box.depth == 1 && !dst->surface.is_linear && - (!dst->cmask.size || !dst->dirty_level_mask)) { /* dst cannot be fast-cleared */ + (!dst->cmask_buffer || !dst->dirty_level_mask)) { /* dst cannot be fast-cleared */ /* Check the last constraint. */ if (src->surface.micro_tile_mode != dst->surface.micro_tile_mode) { /* The next fast clear will switch to this mode to * get direct hw resolve next time if the mode is * different now. + * + * TODO-GFX10: This does not work in GFX10 because MSAA + * is restricted to 64KB_R_X and 64KB_Z_X swizzle modes. + * In some cases we could change the swizzle of the + * destination texture instead, but the more general + * solution is to implement compute shader resolve. */ src->last_msaa_resolve_target_micro_mode = dst->surface.micro_tile_mode; @@ -1146,13 +1127,10 @@ static bool do_hardware_msaa_resolve(struct pipe_context *ctx, * This is still the fastest codepath even with this clear. */ if (vi_dcc_enabled(dst, info->dst.level)) { - /* TODO: Implement per-level DCC clears for GFX9. */ - if (sctx->b.chip_class >= GFX9 && - info->dst.resource->last_level != 0) + if (!vi_dcc_clear_level(sctx, dst, info->dst.level, + DCC_UNCOMPRESSED)) goto resolve_to_temp; - vi_dcc_clear_level(sctx, dst, info->dst.level, - 0xFFFFFFFF); dst->dirty_level_mask &= ~(1 << info->dst.level); } @@ -1174,7 +1152,7 @@ resolve_to_temp: templ.depth0 = 1; templ.array_size = 1; templ.usage = PIPE_USAGE_DEFAULT; - templ.flags = SI_RESOURCE_FLAG_FORCE_TILING | + templ.flags = SI_RESOURCE_FLAG_FORCE_MSAA_TILING | SI_RESOURCE_FLAG_DISABLE_DCC; /* The src and dst microtile modes must be the same. */ @@ -1186,10 +1164,10 @@ resolve_to_temp: tmp = ctx->screen->resource_create(ctx->screen, &templ); if (!tmp) return false; - rtmp = (struct r600_texture*)tmp; + stmp = (struct si_texture*)tmp; - assert(!rtmp->surface.is_linear); - assert(src->surface.micro_tile_mode == rtmp->surface.micro_tile_mode); + assert(!stmp->surface.is_linear); + assert(src->surface.micro_tile_mode == stmp->surface.micro_tile_mode); /* resolve */ si_do_CB_resolve(sctx, info, tmp, 0, 0, format); @@ -1212,7 +1190,7 @@ static void si_blit(struct pipe_context *ctx, const struct pipe_blit_info *info) { struct si_context *sctx = (struct si_context*)ctx; - struct r600_texture *rdst = (struct r600_texture *)info->dst.resource; + struct si_texture *dst = (struct si_texture *)info->dst.resource; if (do_hardware_msaa_resolve(ctx, info)) { return; @@ -1224,10 +1202,10 @@ static void si_blit(struct pipe_context *ctx, * resource_copy_region can't do this yet, because dma_copy calls it * on failure (recursion). */ - if (rdst->surface.is_linear && - sctx->b.dma_copy && + if (dst->surface.is_linear && + sctx->dma_copy && util_can_blit_via_copy_region(info, false)) { - sctx->b.dma_copy(ctx, info->dst.resource, info->dst.level, + sctx->dma_copy(ctx, info->dst.resource, info->dst.level, info->dst.box.x, info->dst.box.y, info->dst.box.z, info->src.resource, info->src.level, @@ -1245,7 +1223,7 @@ static void si_blit(struct pipe_context *ctx, vi_disable_dcc_if_incompatible_format(sctx, info->dst.resource, info->dst.level, info->dst.format); - si_decompress_subresource(ctx, info->src.resource, info->mask, + si_decompress_subresource(ctx, info->src.resource, PIPE_MASK_RGBAZS, info->src.level, info->src.box.z, info->src.box.z + info->src.box.depth - 1); @@ -1260,14 +1238,14 @@ static void si_blit(struct pipe_context *ctx, si_blitter_end(sctx); } -static boolean si_generate_mipmap(struct pipe_context *ctx, - struct pipe_resource *tex, - enum pipe_format format, - unsigned base_level, unsigned last_level, - unsigned first_layer, unsigned last_layer) +static bool si_generate_mipmap(struct pipe_context *ctx, + struct pipe_resource *tex, + enum pipe_format format, + unsigned base_level, unsigned last_level, + unsigned first_layer, unsigned last_layer) { struct si_context *sctx = (struct si_context*)ctx; - struct r600_texture *rtex = (struct r600_texture *)tex; + struct si_texture *stex = (struct si_texture *)tex; if (!util_blitter_is_copy_supported(sctx->blitter, tex, tex)) return false; @@ -1281,10 +1259,10 @@ static boolean si_generate_mipmap(struct pipe_context *ctx, /* Clear dirty_level_mask for the levels that will be overwritten. */ assert(base_level < last_level); - rtex->dirty_level_mask &= ~u_bit_consecutive(base_level + 1, + stex->dirty_level_mask &= ~u_bit_consecutive(base_level + 1, last_level - base_level); - sctx->generate_mipmap_for_depth = rtex->is_depth; + sctx->generate_mipmap_for_depth = stex->is_depth; si_blitter_begin(sctx, SI_BLIT | SI_DISABLE_RENDER_COND); util_blitter_generate_mipmap(sctx->blitter, tex, format, @@ -1300,43 +1278,76 @@ static void si_flush_resource(struct pipe_context *ctx, struct pipe_resource *res) { struct si_context *sctx = (struct si_context*)ctx; - struct r600_texture *rtex = (struct r600_texture*)res; + struct si_texture *tex = (struct si_texture*)res; assert(res->target != PIPE_BUFFER); - assert(!rtex->dcc_separate_buffer || rtex->dcc_gather_statistics); + assert(!tex->dcc_separate_buffer || tex->dcc_gather_statistics); /* st/dri calls flush twice per frame (not a bug), this prevents double * decompression. */ - if (rtex->dcc_separate_buffer && !rtex->separate_dcc_dirty) + if (tex->dcc_separate_buffer && !tex->separate_dcc_dirty) return; - if (!rtex->is_depth && (rtex->cmask.size || rtex->dcc_offset)) { - si_blit_decompress_color(sctx, rtex, 0, res->last_level, + if (!tex->is_depth && (tex->cmask_buffer || tex->dcc_offset)) { + si_blit_decompress_color(sctx, tex, 0, res->last_level, 0, util_max_layer(res, 0), - rtex->dcc_separate_buffer != NULL); + tex->dcc_separate_buffer != NULL); + + if (tex->display_dcc_offset) + si_retile_dcc(sctx, tex); } /* Always do the analysis even if DCC is disabled at the moment. */ - if (rtex->dcc_gather_statistics && rtex->separate_dcc_dirty) { - rtex->separate_dcc_dirty = false; - vi_separate_dcc_process_and_reset_stats(ctx, rtex); + if (tex->dcc_gather_statistics) { + bool separate_dcc_dirty = tex->separate_dcc_dirty; + + /* If the color buffer hasn't been unbound and fast clear hasn't + * been used, separate_dcc_dirty is false, but there may have been + * new rendering. Check if the color buffer is bound and assume + * it's dirty. + * + * Note that DRI2 never unbinds window colorbuffers, which means + * the DCC pipeline statistics query would never be re-set and would + * keep adding new results until all free memory is exhausted if we + * didn't do this. + */ + if (!separate_dcc_dirty) { + for (unsigned i = 0; i < sctx->framebuffer.state.nr_cbufs; i++) { + if (sctx->framebuffer.state.cbufs[i] && + sctx->framebuffer.state.cbufs[i]->texture == res) { + separate_dcc_dirty = true; + break; + } + } + } + + if (separate_dcc_dirty) { + tex->separate_dcc_dirty = false; + vi_separate_dcc_process_and_reset_stats(ctx, tex); + } } } -void si_decompress_dcc(struct si_context *sctx, struct r600_texture *rtex) +void si_decompress_dcc(struct si_context *sctx, struct si_texture *tex) { - if (!rtex->dcc_offset) + /* If graphics is disabled, we can't decompress DCC, but it shouldn't + * be compressed either. The caller should simply discard it. + */ + if (!tex->dcc_offset || !sctx->has_graphics) return; - si_blit_decompress_color(sctx, rtex, 0, rtex->resource.b.b.last_level, - 0, util_max_layer(&rtex->resource.b.b, 0), + si_blit_decompress_color(sctx, tex, 0, tex->buffer.b.b.last_level, + 0, util_max_layer(&tex->buffer.b.b, 0), true); } void si_init_blit_functions(struct si_context *sctx) { - sctx->b.b.resource_copy_region = si_resource_copy_region; - sctx->b.b.blit = si_blit; - sctx->b.b.flush_resource = si_flush_resource; - sctx->b.b.generate_mipmap = si_generate_mipmap; + sctx->b.resource_copy_region = si_resource_copy_region; + + if (sctx->has_graphics) { + sctx->b.blit = si_blit; + sctx->b.flush_resource = si_flush_resource; + sctx->b.generate_mipmap = si_generate_mipmap; + } }