X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fdrivers%2Fradeonsi%2Fsi_blit.c;h=fe17f7392424de904e4f7ffee2e40e827267ddf5;hb=c66a550385b4937b2aaba8484aeaa41cf77399b7;hp=54da7a20203fbda2b31196fec620f34a77a1c269;hpb=12f88ba32a14ea79134f4e995a55149f078a2f27;p=mesa.git

diff --git a/src/gallium/drivers/radeonsi/si_blit.c b/src/gallium/drivers/radeonsi/si_blit.c
index 54da7a20203..fe17f739242 100644
--- a/src/gallium/drivers/radeonsi/si_blit.c
+++ b/src/gallium/drivers/radeonsi/si_blit.c
@@ -22,6 +22,7 @@
  */
 
 #include "si_pipe.h"
+#include "sid.h"
 #include "util/u_format.h"
 #include "util/u_surface.h"
 
@@ -100,116 +101,133 @@ static unsigned u_max_sample(struct pipe_resource *r)
 	return r->nr_samples ? r->nr_samples - 1 : 0;
 }
 
-static void si_blit_decompress_depth(struct pipe_context *ctx,
-				     struct r600_texture *texture,
-				     struct r600_texture *staging,
-				     unsigned first_level, unsigned last_level,
-				     unsigned first_layer, unsigned last_layer,
-				     unsigned first_sample, unsigned last_sample)
+static unsigned
+si_blit_dbcb_copy(struct si_context *sctx,
+		  struct r600_texture *src,
+		  struct r600_texture *dst,
+		  unsigned planes, unsigned level_mask,
+		  unsigned first_layer, unsigned last_layer,
+		  unsigned first_sample, unsigned last_sample)
 {
-	struct si_context *sctx = (struct si_context *)ctx;
-	unsigned layer, level, sample, checked_last_layer, max_layer, max_sample;
-	float depth = 1.0f;
-	const struct util_format_description *desc;
-	struct r600_texture *flushed_depth_texture = staging ?
-			staging : texture->flushed_depth_texture;
-
-	if (!staging && !texture->dirty_level_mask)
-		return;
-
-	max_sample = u_max_sample(&texture->resource.b.b);
-
-	desc = util_format_description(flushed_depth_texture->resource.b.b.format);
+	struct pipe_surface surf_tmpl = {{0}};
+	unsigned layer, sample, checked_last_layer, max_layer;
+	unsigned fully_copied_levels = 0;
 
-	if (util_format_has_depth(desc))
+	if (planes & PIPE_MASK_Z)
 		sctx->dbcb_depth_copy_enabled = true;
-	if (util_format_has_stencil(desc))
+	if (planes & PIPE_MASK_S)
 		sctx->dbcb_stencil_copy_enabled = true;
+	si_mark_atom_dirty(sctx, &sctx->db_render_state);
 
 	assert(sctx->dbcb_depth_copy_enabled || sctx->dbcb_stencil_copy_enabled);
 
-	for (level = first_level; level <= last_level; level++) {
-		if (!staging && !(texture->dirty_level_mask & (1 << level)))
-			continue;
+	while (level_mask) {
+		unsigned level = u_bit_scan(&level_mask);
 
 		/* The smaller the mipmap level, the less layers there are
 		 * as far as 3D textures are concerned. */
-		max_layer = util_max_layer(&texture->resource.b.b, level);
-		checked_last_layer = last_layer < max_layer ? last_layer : max_layer;
+		max_layer = util_max_layer(&src->resource.b.b, level);
+		checked_last_layer = MIN2(last_layer, max_layer);
+
+		surf_tmpl.u.tex.level = level;
 
 		for (layer = first_layer; layer <= checked_last_layer; layer++) {
-			for (sample = first_sample; sample <= last_sample; sample++) {
-				struct pipe_surface *zsurf, *cbsurf, surf_tmpl;
+			struct pipe_surface *zsurf, *cbsurf;
 
-				sctx->dbcb_copy_sample = sample;
-				si_mark_atom_dirty(sctx, &sctx->db_render_state);
+			surf_tmpl.format = src->resource.b.b.format;
+			surf_tmpl.u.tex.first_layer = layer;
+			surf_tmpl.u.tex.last_layer = layer;
 
-				surf_tmpl.format = texture->resource.b.b.format;
-				surf_tmpl.u.tex.level = level;
-				surf_tmpl.u.tex.first_layer = layer;
-				surf_tmpl.u.tex.last_layer = layer;
+			zsurf = sctx->b.b.create_surface(&sctx->b.b, &src->resource.b.b, &surf_tmpl);
 
-				zsurf = ctx->create_surface(ctx, &texture->resource.b.b, &surf_tmpl);
+			surf_tmpl.format = dst->resource.b.b.format;
+			cbsurf = sctx->b.b.create_surface(&sctx->b.b, &dst->resource.b.b, &surf_tmpl);
 
-				surf_tmpl.format = flushed_depth_texture->resource.b.b.format;
-				cbsurf = ctx->create_surface(ctx,
-						(struct pipe_resource*)flushed_depth_texture, &surf_tmpl);
+			for (sample = first_sample; sample <= last_sample; sample++) {
+				if (sample != sctx->dbcb_copy_sample) {
+					sctx->dbcb_copy_sample = sample;
+					si_mark_atom_dirty(sctx, &sctx->db_render_state);
+				}
 
-				si_blitter_begin(ctx, SI_DECOMPRESS);
+				si_blitter_begin(&sctx->b.b, SI_DECOMPRESS);
 				util_blitter_custom_depth_stencil(sctx->blitter, zsurf, cbsurf, 1 << sample,
-								  sctx->custom_dsa_flush, depth);
-				si_blitter_end(ctx);
-
-				pipe_surface_reference(&zsurf, NULL);
-				pipe_surface_reference(&cbsurf, NULL);
+								  sctx->custom_dsa_flush, 1.0f);
+				si_blitter_end(&sctx->b.b);
 			}
-		}
 
-		/* The texture will always be dirty if some layers aren't flushed.
-		 * I don't think this case can occur though. */
-		if (!staging &&
-		    first_layer == 0 && last_layer == max_layer &&
-		    first_sample == 0 && last_sample == max_sample) {
-			texture->dirty_level_mask &= ~(1 << level);
+			pipe_surface_reference(&zsurf, NULL);
+			pipe_surface_reference(&cbsurf, NULL);
 		}
+
+		if (first_layer == 0 && last_layer >= max_layer &&
+		    first_sample == 0 && last_sample >= u_max_sample(&src->resource.b.b))
+			fully_copied_levels |= 1u << level;
 	}
 
 	sctx->dbcb_depth_copy_enabled = false;
 	sctx->dbcb_stencil_copy_enabled = false;
 	si_mark_atom_dirty(sctx, &sctx->db_render_state);
+
+	return fully_copied_levels;
 }
 
-static void si_blit_decompress_depth_in_place(struct si_context *sctx,
-                                              struct r600_texture *texture,
-					      bool is_stencil_sampler,
-                                              unsigned first_level, unsigned last_level,
-                                              unsigned first_layer, unsigned last_layer)
+static void si_blit_decompress_depth(struct pipe_context *ctx,
+				     struct r600_texture *texture,
+				     struct r600_texture *staging,
+				     unsigned first_level, unsigned last_level,
+				     unsigned first_layer, unsigned last_layer,
+				     unsigned first_sample, unsigned last_sample)
+{
+	const struct util_format_description *desc;
+	unsigned planes = 0;
+
+	assert(staging != NULL && "use si_blit_decompress_zs_in_place instead");
+
+	desc = util_format_description(staging->resource.b.b.format);
+
+	if (util_format_has_depth(desc))
+		planes |= PIPE_MASK_Z;
+	if (util_format_has_stencil(desc))
+		planes |= PIPE_MASK_S;
+
+	si_blit_dbcb_copy(
+		(struct si_context *)ctx, texture, staging, planes,
+		u_bit_consecutive(first_level, last_level - first_level + 1),
+		first_layer, last_layer, first_sample, last_sample);
+}
+
+/* Helper function for si_blit_decompress_zs_in_place.
+ */
+static void
+si_blit_decompress_zs_planes_in_place(struct si_context *sctx,
+				      struct r600_texture *texture,
+				      unsigned planes, unsigned level_mask,
+				      unsigned first_layer, unsigned last_layer)
 {
 	struct pipe_surface *zsurf, surf_tmpl = {{0}};
-	unsigned layer, max_layer, checked_last_layer, level;
-	unsigned *dirty_level_mask;
+	unsigned layer, max_layer, checked_last_layer;
+	unsigned fully_decompressed_mask = 0;
+
+	if (!level_mask)
+		return;
 
-	if (is_stencil_sampler) {
+	if (planes & PIPE_MASK_S)
 		sctx->db_flush_stencil_inplace = true;
-		dirty_level_mask = &texture->stencil_dirty_level_mask;
-	} else {
+	if (planes & PIPE_MASK_Z)
 		sctx->db_flush_depth_inplace = true;
-		dirty_level_mask = &texture->dirty_level_mask;
-	}
 	si_mark_atom_dirty(sctx, &sctx->db_render_state);
 
 	surf_tmpl.format = texture->resource.b.b.format;
 
-	for (level = first_level; level <= last_level; level++) {
-		if (!(*dirty_level_mask & (1 << level)))
-			continue;
+	while (level_mask) {
+		unsigned level = u_bit_scan(&level_mask);
 
 		surf_tmpl.u.tex.level = level;
 
 		/* The smaller the mipmap level, the less layers there are
 		 * as far as 3D textures are concerned. */
 		max_layer = util_max_layer(&texture->resource.b.b, level);
-		checked_last_layer = last_layer < max_layer ? last_layer : max_layer;
+		checked_last_layer = MIN2(last_layer, max_layer);
 
 		for (layer = first_layer; layer <= checked_last_layer; layer++) {
 			surf_tmpl.u.tex.first_layer = layer;
@@ -228,41 +246,163 @@ static void si_blit_decompress_depth_in_place(struct si_context *sctx,
 
 		/* The texture will always be dirty if some layers aren't flushed.
 		 * I don't think this case occurs often though. */
-		if (first_layer == 0 && last_layer == max_layer) {
-			*dirty_level_mask &= ~(1 << level);
+		if (first_layer == 0 && last_layer >= max_layer) {
+			fully_decompressed_mask |= 1u << level;
 		}
 	}
 
+	if (planes & PIPE_MASK_Z)
+		texture->dirty_level_mask &= ~fully_decompressed_mask;
+	if (planes & PIPE_MASK_S)
+		texture->stencil_dirty_level_mask &= ~fully_decompressed_mask;
+
 	sctx->db_flush_depth_inplace = false;
 	sctx->db_flush_stencil_inplace = false;
 	si_mark_atom_dirty(sctx, &sctx->db_render_state);
 }
 
+/* Helper function of si_flush_depth_texture: decompress the given levels
+ * of Z and/or S planes in place.
+ */
+static void
+si_blit_decompress_zs_in_place(struct si_context *sctx,
+			       struct r600_texture *texture,
+			       unsigned levels_z, unsigned levels_s,
+			       unsigned first_layer, unsigned last_layer)
+{
+	unsigned both = levels_z & levels_s;
+
+	/* First, do combined Z & S decompresses for levels that need it. */
+	if (both) {
+		si_blit_decompress_zs_planes_in_place(
+				sctx, texture, PIPE_MASK_Z | PIPE_MASK_S,
+				both,
+				first_layer, last_layer);
+		levels_z &= ~both;
+		levels_s &= ~both;
+	}
+
+	/* Now do separate Z and S decompresses. */
+	if (levels_z) {
+		si_blit_decompress_zs_planes_in_place(
+				sctx, texture, PIPE_MASK_Z,
+				levels_z,
+				first_layer, last_layer);
+	}
+
+	if (levels_s) {
+		si_blit_decompress_zs_planes_in_place(
+				sctx, texture, PIPE_MASK_S,
+				levels_s,
+				first_layer, last_layer);
+	}
+}
+
+static void
+si_flush_depth_texture(struct si_context *sctx,
+		       struct r600_texture *tex,
+		       unsigned required_planes,
+		       unsigned first_level, unsigned last_level,
+		       unsigned first_layer, unsigned last_layer)
+{
+	unsigned inplace_planes = 0;
+	unsigned copy_planes = 0;
+	unsigned level_mask = u_bit_consecutive(first_level, last_level - first_level + 1);
+	unsigned levels_z = 0;
+	unsigned levels_s = 0;
+
+	if (required_planes & PIPE_MASK_Z) {
+		levels_z = level_mask & tex->dirty_level_mask;
+
+		if (levels_z) {
+			if (r600_can_sample_zs(tex, false))
+				inplace_planes |= PIPE_MASK_Z;
+			else
+				copy_planes |= PIPE_MASK_Z;
+		}
+	}
+	if (required_planes & PIPE_MASK_S) {
+		levels_s = level_mask & tex->stencil_dirty_level_mask;
+
+		if (levels_s) {
+			if (r600_can_sample_zs(tex, true))
+				inplace_planes |= PIPE_MASK_S;
+			else
+				copy_planes |= PIPE_MASK_S;
+		}
+	}
+
+	assert(!tex->tc_compatible_htile || levels_z == 0);
+
+	/* We may have to allocate the flushed texture here when called from
+	 * si_decompress_subresource.
+	 */
+	if (copy_planes &&
+	    (tex->flushed_depth_texture ||
+	     r600_init_flushed_depth_texture(&sctx->b.b, &tex->resource.b.b, NULL))) {
+		struct r600_texture *dst = tex->flushed_depth_texture;
+		unsigned fully_copied_levels;
+		unsigned levels = 0;
+
+		assert(tex->flushed_depth_texture);
+
+		if (util_format_is_depth_and_stencil(dst->resource.b.b.format))
+			copy_planes = PIPE_MASK_Z | PIPE_MASK_S;
+
+		if (copy_planes & PIPE_MASK_Z) {
+			levels |= levels_z;
+			levels_z = 0;
+		}
+		if (copy_planes & PIPE_MASK_S) {
+			levels |= levels_s;
+			levels_s = 0;
+		}
+
+		fully_copied_levels = si_blit_dbcb_copy(
+			sctx, tex, dst, copy_planes, levels,
+			first_layer, last_layer,
+			0, u_max_sample(&tex->resource.b.b));
+
+		if (copy_planes & PIPE_MASK_Z)
+			tex->dirty_level_mask &= ~fully_copied_levels;
+		if (copy_planes & PIPE_MASK_S)
+			tex->stencil_dirty_level_mask &= ~fully_copied_levels;
+	}
+
+	if (inplace_planes) {
+		si_blit_decompress_zs_in_place(
+			sctx, tex,
+			levels_z, levels_s,
+			first_layer, last_layer);
+	}
+}
+
 static void
 si_flush_depth_textures(struct si_context *sctx,
 			struct si_textures_info *textures)
 {
 	unsigned i;
-	uint64_t mask = textures->depth_texture_mask;
+	unsigned mask = textures->depth_texture_mask;
 
 	while (mask) {
 		struct pipe_sampler_view *view;
 		struct si_sampler_view *sview;
 		struct r600_texture *tex;
 
-		i = u_bit_scan64(&mask);
+		i = u_bit_scan(&mask);
 
 		view = textures->views.views[i];
 		assert(view);
 		sview = (struct si_sampler_view*)view;
 
 		tex = (struct r600_texture *)view->texture;
-		assert(tex->is_depth && !tex->is_flushing_texture);
+		assert(tex->db_compatible);
 
-		si_blit_decompress_depth_in_place(sctx, tex,
-						  sview->is_stencil_sampler,
-						  view->u.tex.first_level, view->u.tex.last_level,
-						  0, util_max_layer(&tex->resource.b.b, view->u.tex.first_level));
+		si_flush_depth_texture(
+				sctx, tex,
+				sview->is_stencil_sampler ? PIPE_MASK_S : PIPE_MASK_Z,
+				view->u.tex.first_level, view->u.tex.last_level,
+				0, util_max_layer(&tex->resource.b.b, view->u.tex.first_level));
 	}
 }
 
@@ -273,29 +413,38 @@ static void si_blit_decompress_color(struct pipe_context *ctx,
 		bool need_dcc_decompress)
 {
 	struct si_context *sctx = (struct si_context *)ctx;
-	unsigned layer, level, checked_last_layer, max_layer;
-
-	if (!rtex->dirty_level_mask && !need_dcc_decompress)
+	void* custom_blend;
+	unsigned layer, checked_last_layer, max_layer;
+	unsigned level_mask =
+		u_bit_consecutive(first_level, last_level - first_level + 1);
+
+	if (!need_dcc_decompress)
+		level_mask &= rtex->dirty_level_mask;
+	if (!level_mask)
 		return;
 
-	for (level = first_level; level <= last_level; level++) {
-		void* custom_blend;
+	if (rtex->dcc_offset && need_dcc_decompress) {
+		custom_blend = sctx->custom_blend_dcc_decompress;
 
-		if (!(rtex->dirty_level_mask & (1 << level)) && !need_dcc_decompress)
-			continue;
-
-		if (rtex->dcc_offset && need_dcc_decompress) {
-			custom_blend = sctx->custom_blend_dcc_decompress;
-		} else if (rtex->fmask.size) {
-			custom_blend = sctx->custom_blend_decompress;
-		} else {
-			custom_blend = sctx->custom_blend_fastclear;
+		/* disable levels without DCC */
+		for (int i = first_level; i <= last_level; i++) {
+			if (!rtex->dcc_offset ||
+			    i >= rtex->surface.num_dcc_levels)
+				level_mask &= ~(1 << i);
 		}
+	} else if (rtex->fmask.size) {
+		custom_blend = sctx->custom_blend_decompress;
+	} else {
+		custom_blend = sctx->custom_blend_fastclear;
+	}
+
+	while (level_mask) {
+		unsigned level = u_bit_scan(&level_mask);
 
 		/* The smaller the mipmap level, the less layers there are
 		 * as far as 3D textures are concerned. */
 		max_layer = util_max_layer(&rtex->resource.b.b, level);
-		checked_last_layer = last_layer < max_layer ? last_layer : max_layer;
+		checked_last_layer = MIN2(last_layer, max_layer);
 
 		for (layer = first_layer; layer <= checked_last_layer; layer++) {
 			struct pipe_surface *cbsurf, surf_tmpl;
@@ -315,7 +464,7 @@ static void si_blit_decompress_color(struct pipe_context *ctx,
 
 		/* The texture will always be dirty if some layers aren't flushed.
 		 * I don't think this case occurs often though. */
-		if (first_layer == 0 && last_layer == max_layer) {
+		if (first_layer == 0 && last_layer >= max_layer) {
 			rtex->dirty_level_mask &= ~(1 << level);
 		}
 	}
@@ -326,13 +475,13 @@ si_decompress_sampler_color_textures(struct si_context *sctx,
 				     struct si_textures_info *textures)
 {
 	unsigned i;
-	uint64_t mask = textures->compressed_colortex_mask;
+	unsigned mask = textures->compressed_colortex_mask;
 
 	while (mask) {
 		struct pipe_sampler_view *view;
 		struct r600_texture *tex;
 
-		i = u_bit_scan64(&mask);
+		i = u_bit_scan(&mask);
 
 		view = textures->views.views[i];
 		assert(view);
@@ -352,13 +501,13 @@ si_decompress_image_color_textures(struct si_context *sctx,
 				   struct si_images_info *images)
 {
 	unsigned i;
-	uint64_t mask = images->compressed_colortex_mask;
+	unsigned mask = images->compressed_colortex_mask;
 
 	while (mask) {
 		const struct pipe_image_view *view;
 		struct r600_texture *tex;
 
-		i = u_bit_scan64(&mask);
+		i = u_bit_scan(&mask);
 
 		view = &images->views[i];
 		assert(view->resource->target != PIPE_BUFFER);
@@ -374,7 +523,102 @@ si_decompress_image_color_textures(struct si_context *sctx,
 	}
 }
 
-void si_decompress_textures(struct si_context *sctx)
+static void si_check_render_feedback_textures(struct si_context *sctx,
+                                              struct si_textures_info *textures)
+{
+	uint32_t mask = textures->views.enabled_mask;
+
+	while (mask) {
+		const struct pipe_sampler_view *view;
+		struct r600_texture *tex;
+		bool render_feedback = false;
+
+		unsigned i = u_bit_scan(&mask);
+
+		view = textures->views.views[i];
+		if(view->texture->target == PIPE_BUFFER)
+			continue;
+
+		tex = (struct r600_texture *)view->texture;
+		if (!tex->dcc_offset)
+			continue;
+
+		for (unsigned j = 0; j < sctx->framebuffer.state.nr_cbufs; ++j) {
+			struct r600_surface * surf;
+
+			if (!sctx->framebuffer.state.cbufs[j])
+				continue;
+
+			surf = (struct r600_surface*)sctx->framebuffer.state.cbufs[j];
+
+			if (tex == (struct r600_texture*)surf->base.texture &&
+			    surf->base.u.tex.level >= view->u.tex.first_level &&
+			    surf->base.u.tex.level <= view->u.tex.last_level &&
+			    surf->base.u.tex.first_layer <= view->u.tex.last_layer &&
+			    surf->base.u.tex.last_layer >= view->u.tex.first_layer)
+				render_feedback = true;
+		}
+
+		if (render_feedback)
+			r600_texture_disable_dcc(&sctx->b, tex);
+	}
+}
+
+static void si_check_render_feedback_images(struct si_context *sctx,
+                                            struct si_images_info *images)
+{
+	uint32_t mask = images->enabled_mask;
+
+	while (mask) {
+		const struct pipe_image_view *view;
+		struct r600_texture *tex;
+		bool render_feedback = false;
+
+		unsigned i = u_bit_scan(&mask);
+
+		view = &images->views[i];
+		if (view->resource->target == PIPE_BUFFER)
+			continue;
+
+		tex = (struct r600_texture *)view->resource;
+		if (!tex->dcc_offset)
+			continue;
+
+		for (unsigned j = 0; j < sctx->framebuffer.state.nr_cbufs; ++j) {
+			struct r600_surface * surf;
+
+			if (!sctx->framebuffer.state.cbufs[j])
+				continue;
+
+			surf = (struct r600_surface*)sctx->framebuffer.state.cbufs[j];
+
+			if (tex == (struct r600_texture*)surf->base.texture &&
+			    surf->base.u.tex.level == view->u.tex.level &&
+			    surf->base.u.tex.first_layer <= view->u.tex.last_layer &&
+			    surf->base.u.tex.last_layer >= view->u.tex.first_layer)
+				render_feedback = true;
+		}
+
+		if (render_feedback)
+			r600_texture_disable_dcc(&sctx->b, tex);
+	}
+}
+
+static void si_check_render_feedback(struct si_context *sctx)
+{
+
+	if (!sctx->need_check_render_feedback)
+		return;
+
+	for (int i = 0; i < SI_NUM_SHADERS; ++i) {
+		si_check_render_feedback_images(sctx, &sctx->images[i]);
+		si_check_render_feedback_textures(sctx, &sctx->samplers[i]);
+	}
+	sctx->need_check_render_feedback = false;
+}
+
+static void si_decompress_textures(struct si_context *sctx, int shader_start,
+                                   int shader_end)
 {
 	unsigned compressed_colortex_counter;
 
@@ -389,7 +633,7 @@ void si_decompress_textures(struct si_context *sctx)
 	}
 
 	/* Flush depth textures which need to be flushed. */
-	for (int i = 0; i < SI_NUM_SHADERS; i++) {
+	for (int i = shader_start; i < shader_end; i++) {
 		if (sctx->samplers[i].depth_texture_mask) {
 			si_flush_depth_textures(sctx, &sctx->samplers[i]);
 		}
@@ -400,6 +644,18 @@ void si_decompress_textures(struct si_context *sctx)
 			si_decompress_image_color_textures(sctx, &sctx->images[i]);
 		}
 	}
+
+	si_check_render_feedback(sctx);
+}
+
+void si_decompress_graphics_textures(struct si_context *sctx)
+{
+	si_decompress_textures(sctx, 0, SI_NUM_GRAPHICS_SHADERS);
+}
+
+void si_decompress_compute_textures(struct si_context *sctx)
+{
+	si_decompress_textures(sctx, SI_NUM_GRAPHICS_SHADERS, SI_NUM_SHADERS);
 }
 
 static void si_clear(struct pipe_context *ctx, unsigned buffers,
@@ -445,10 +701,13 @@ static void si_clear(struct pipe_context *ctx, unsigned buffers,
 	    zsbuf->u.tex.level == 0 &&
 	    zsbuf->u.tex.first_layer == 0 &&
 	    zsbuf->u.tex.last_layer == util_max_layer(&zstex->resource.b.b, 0)) {
-		if (buffers & PIPE_CLEAR_DEPTH) {
+		/* TC-compatible HTILE only supports depth clears to 0 or 1. */
+		if (buffers & PIPE_CLEAR_DEPTH &&
+		    (!zstex->tc_compatible_htile ||
+		     depth == 0 || depth == 1)) {
 			/* Need to disable EXPCLEAR temporarily if clearing
 			 * to a new value. */
-			if (zstex->depth_cleared && zstex->depth_clear_value != depth) {
+			if (!zstex->depth_cleared || zstex->depth_clear_value != depth) {
 				sctx->db_depth_disable_expclear = true;
 			}
 
@@ -459,12 +718,14 @@ static void si_clear(struct pipe_context *ctx, unsigned buffers,
 			si_mark_atom_dirty(sctx, &sctx->db_render_state);
 		}
 
-		if (buffers & PIPE_CLEAR_STENCIL) {
+		/* TC-compatible HTILE only supports stencil clears to 0. */
+		if (buffers & PIPE_CLEAR_STENCIL &&
+		    (!zstex->tc_compatible_htile || stencil == 0)) {
 			stencil &= 0xff;
 
 			/* Need to disable EXPCLEAR temporarily if clearing
 			 * to a new value. */
-			if (zstex->stencil_cleared && zstex->stencil_clear_value != stencil) {
+			if (!zstex->stencil_cleared || zstex->stencil_clear_value != stencil) {
 				sctx->db_stencil_disable_expclear = true;
 			}
 
@@ -501,11 +762,13 @@ static void si_clear_render_target(struct pipe_context *ctx,
 				   struct pipe_surface *dst,
 				   const union pipe_color_union *color,
 				   unsigned dstx, unsigned dsty,
-				   unsigned width, unsigned height)
+				   unsigned width, unsigned height,
+				   bool render_condition_enabled)
 {
 	struct si_context *sctx = (struct si_context *)ctx;
 
-	si_blitter_begin(ctx, SI_CLEAR_SURFACE);
+	si_blitter_begin(ctx, SI_CLEAR_SURFACE |
+			 (render_condition_enabled ? 0 : SI_DISABLE_RENDER_COND));
 	util_blitter_clear_render_target(sctx->blitter, dst, color,
 					 dstx, dsty, width, height);
 	si_blitter_end(ctx);
@@ -517,11 +780,13 @@ static void si_clear_depth_stencil(struct pipe_context *ctx,
 				   double depth,
 				   unsigned stencil,
 				   unsigned dstx, unsigned dsty,
-				   unsigned width, unsigned height)
+				   unsigned width, unsigned height,
+				   bool render_condition_enabled)
 {
 	struct si_context *sctx = (struct si_context *)ctx;
 
-	si_blitter_begin(ctx, SI_CLEAR_SURFACE);
+	si_blitter_begin(ctx, SI_CLEAR_SURFACE |
+			 (render_condition_enabled ? 0 : SI_DISABLE_RENDER_COND));
 	util_blitter_clear_depth_stencil(sctx->blitter, dst, clear_flags, depth, stencil,
 					 dstx, dsty, width, height);
 	si_blitter_end(ctx);
@@ -533,20 +798,21 @@ static void si_clear_depth_stencil(struct pipe_context *ctx,
  * rendering. */
 static void si_decompress_subresource(struct pipe_context *ctx,
 				      struct pipe_resource *tex,
-				      unsigned level,
+				      unsigned planes, unsigned level,
 				      unsigned first_layer, unsigned last_layer)
 {
 	struct si_context *sctx = (struct si_context *)ctx;
 	struct r600_texture *rtex = (struct r600_texture*)tex;
 
-	if (rtex->is_depth && !rtex->is_flushing_texture) {
-		si_blit_decompress_depth_in_place(sctx, rtex, false,
-						  level, level,
-						  first_layer, last_layer);
-		if (rtex->surface.flags & RADEON_SURF_SBUFFER)
-			si_blit_decompress_depth_in_place(sctx, rtex, true,
-							  level, level,
-							  first_layer, last_layer);
+	if (rtex->db_compatible) {
+		planes &= PIPE_MASK_Z | PIPE_MASK_S;
+
+		if (!(rtex->surface.flags & RADEON_SURF_SBUFFER))
+			planes &= ~PIPE_MASK_S;
+
+		si_flush_depth_texture(sctx, rtex, planes,
+				       level, level,
+				       first_layer, last_layer);
 	} else if (rtex->fmask.size || rtex->cmask.size || rtex->dcc_offset) {
 		si_blit_decompress_color(ctx, rtex, level, level,
 					 first_layer, last_layer, false);
@@ -572,6 +838,7 @@ void si_resource_copy_region(struct pipe_context *ctx,
 			     const struct pipe_box *src_box)
 {
 	struct si_context *sctx = (struct si_context *)ctx;
+	struct r600_texture *rsrc = (struct r600_texture*)src;
 	struct pipe_surface *dst_view, dst_templ;
 	struct pipe_sampler_view src_templ, *src_view;
 	unsigned dst_width, dst_height, src_width0, src_height0;
@@ -580,7 +847,7 @@ void si_resource_copy_region(struct pipe_context *ctx,
 
 	/* Handle buffers first. */
 	if (dst->target == PIPE_BUFFER && src->target == PIPE_BUFFER) {
-		si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width, false);
+		si_copy_buffer(sctx, dst, src, dstx, src_box->x, src_box->width);
 		return;
 	}
 
@@ -588,7 +855,7 @@ void si_resource_copy_region(struct pipe_context *ctx,
 
 	/* The driver doesn't decompress resources automatically while
 	 * u_blitter is rendering. */
-	si_decompress_subresource(ctx, src, src_level,
+	si_decompress_subresource(ctx, src, PIPE_MASK_RGBAZS, src_level,
 				  src_box->z, src_box->z + src_box->depth - 1);
 
 	dst_width = u_minify(dst->width0, dst_level);
@@ -601,7 +868,7 @@ void si_resource_copy_region(struct pipe_context *ctx,
 
 	if (util_format_is_compressed(src->format) ||
 	    util_format_is_compressed(dst->format)) {
-		unsigned blocksize = util_format_get_blocksize(src->format);
+		unsigned blocksize = rsrc->surface.bpe;
 
 		if (blocksize == 8)
 			src_templ.format = PIPE_FORMAT_R16G16B16A16_UINT; /* 64-bit block */
@@ -643,7 +910,7 @@ void si_resource_copy_region(struct pipe_context *ctx,
 			sbox.width = util_format_get_nblocksx(src->format, src_box->width);
 			src_box = &sbox;
 		} else {
-			unsigned blocksize = util_format_get_blocksize(src->format);
+			unsigned blocksize = rsrc->surface.bpe;
 
 			switch (blocksize) {
 			case 1:
@@ -691,58 +958,33 @@ void si_resource_copy_region(struct pipe_context *ctx,
 	util_blitter_blit_generic(sctx->blitter, dst_view, &dstbox,
 				  src_view, src_box, src_width0, src_height0,
 				  PIPE_MASK_RGBAZS, PIPE_TEX_FILTER_NEAREST, NULL,
-				  FALSE);
+				  false);
 	si_blitter_end(ctx);
 
 	pipe_surface_reference(&dst_view, NULL);
 	pipe_sampler_view_reference(&src_view, NULL);
 }
 
-/* For MSAA integer resolving to work, we change the format to NORM using this function. */
-static enum pipe_format int_to_norm_format(enum pipe_format format)
-{
-	switch (format) {
-#define REPLACE_FORMAT_SIGN(format,sign) \
-	case PIPE_FORMAT_##format##_##sign##INT: \
-		return PIPE_FORMAT_##format##_##sign##NORM
-#define REPLACE_FORMAT(format) \
-		REPLACE_FORMAT_SIGN(format, U); \
-		REPLACE_FORMAT_SIGN(format, S)
-
-	REPLACE_FORMAT_SIGN(B10G10R10A2, U);
-	REPLACE_FORMAT(R8);
-	REPLACE_FORMAT(R8G8);
-	REPLACE_FORMAT(R8G8B8X8);
-	REPLACE_FORMAT(R8G8B8A8);
-	REPLACE_FORMAT(A8);
-	REPLACE_FORMAT(I8);
-	REPLACE_FORMAT(L8);
-	REPLACE_FORMAT(L8A8);
-	REPLACE_FORMAT(R16);
-	REPLACE_FORMAT(R16G16);
-	REPLACE_FORMAT(R16G16B16X16);
-	REPLACE_FORMAT(R16G16B16A16);
-	REPLACE_FORMAT(A16);
-	REPLACE_FORMAT(I16);
-	REPLACE_FORMAT(L16);
-	REPLACE_FORMAT(L16A16);
-
-#undef REPLACE_FORMAT
-#undef REPLACE_FORMAT_SIGN
-	default:
-		return format;
-	}
-}
-
 static bool do_hardware_msaa_resolve(struct pipe_context *ctx,
 				     const struct pipe_blit_info *info)
 {
 	struct si_context *sctx = (struct si_context*)ctx;
+	struct r600_texture *src = (struct r600_texture*)info->src.resource;
 	struct r600_texture *dst = (struct r600_texture*)info->dst.resource;
 	unsigned dst_width = u_minify(info->dst.resource->width0, info->dst.level);
 	unsigned dst_height = u_minify(info->dst.resource->height0, info->dst.level);
-	enum pipe_format format = int_to_norm_format(info->dst.format);
+	enum pipe_format format = info->src.format;
 	unsigned sample_mask = ~0;
+	struct pipe_resource *tmp, templ;
+	struct pipe_blit_info blit;
+
+	/* Check basic requirements for hw resolve. */
+	if (!(info->src.resource->nr_samples > 1 &&
+	      info->dst.resource->nr_samples <= 1 &&
+	      !util_format_is_pure_integer(format) &&
+	      !util_format_is_depth_or_stencil(format) &&
+	      util_max_layer(info->src.resource, 0) == 0))
+		return false;
 
 	/* Hardware MSAA resolve doesn't work if SPI format = NORM16_ABGR and
 	 * the format is R16G16. Use R16A16, which does work.
@@ -752,15 +994,12 @@ static bool do_hardware_msaa_resolve(struct pipe_context *ctx,
 	if (format == PIPE_FORMAT_R16G16_SNORM)
 		format = PIPE_FORMAT_R16A16_SNORM;
 
-	if (info->src.resource->nr_samples > 1 &&
-	    info->dst.resource->nr_samples <= 1 &&
-	    util_max_layer(info->src.resource, 0) == 0 &&
-	    util_max_layer(info->dst.resource, info->dst.level) == 0 &&
-	    info->dst.format == info->src.format &&
-	    !util_format_is_pure_integer(format) &&
-	    !util_format_is_depth_or_stencil(format) &&
+	/* Check the remaining requirements for hw resolve. */
+	if (util_max_layer(info->dst.resource, info->dst.level) == 0 &&
 	    !info->scissor_enable &&
 	    (info->mask & PIPE_MASK_RGBA) == PIPE_MASK_RGBA &&
+	    util_is_format_compatible(util_format_description(info->src.format),
+				      util_format_description(info->dst.format)) &&
 	    dst_width == info->src.resource->width0 &&
 	    dst_height == info->src.resource->height0 &&
 	    info->dst.box.x == 0 &&
@@ -774,9 +1013,30 @@ static bool do_hardware_msaa_resolve(struct pipe_context *ctx,
 	    info->src.box.height == dst_height &&
 	    info->src.box.depth == 1 &&
 	    dst->surface.level[info->dst.level].mode >= RADEON_SURF_MODE_1D &&
-	    !(dst->surface.flags & RADEON_SURF_SCANOUT) &&
-	    (!dst->cmask.size || !dst->dirty_level_mask) && /* dst cannot be fast-cleared */
-	    !dst->dcc_offset) {
+	    (!dst->cmask.size || !dst->dirty_level_mask)) { /* dst cannot be fast-cleared */
+		/* Check the last constraint. */
+		if (src->surface.micro_tile_mode != dst->surface.micro_tile_mode) {
+			/* The next fast clear will switch to this mode to
+			 * get direct hw resolve next time if the mode is
+			 * different now.
+			 */
+			src->last_msaa_resolve_target_micro_mode =
+				dst->surface.micro_tile_mode;
+			goto resolve_to_temp;
+		}
+
+		/* Resolving into a surface with DCC is unsupported. Since
+		 * it's being overwritten anyway, clear it to uncompressed.
+		 * This is still the fastest codepath even with this clear.
+		 */
+		if (dst->dcc_offset &&
+		    info->dst.level < dst->surface.num_dcc_levels) {
+			vi_dcc_clear_level(&sctx->b, dst, info->dst.level,
+					   0xFFFFFFFF);
+			dst->dirty_level_mask &= ~(1 << info->dst.level);
+		}
+
+		/* Resolve directly from src to dst. */
 		si_blitter_begin(ctx, SI_COLOR_RESOLVE |
 				 (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
 		util_blitter_custom_resolve_color(sctx->blitter,
@@ -788,23 +1048,98 @@ static bool do_hardware_msaa_resolve(struct pipe_context *ctx,
 		si_blitter_end(ctx);
 		return true;
 	}
-	return false;
+
+resolve_to_temp:
+	/* Shader-based resolve is VERY SLOW. Instead, resolve into
+	 * a temporary texture and blit.
+	 */
+	memset(&templ, 0, sizeof(templ));
+	templ.target = PIPE_TEXTURE_2D;
+	templ.format = info->src.resource->format;
+	templ.width0 = info->src.resource->width0;
+	templ.height0 = info->src.resource->height0;
+	templ.depth0 = 1;
+	templ.array_size = 1;
+	templ.usage = PIPE_USAGE_DEFAULT;
+	templ.flags = R600_RESOURCE_FLAG_FORCE_TILING |
+		      R600_RESOURCE_FLAG_DISABLE_DCC;
+
+	/* The src and dst microtile modes must be the same. */
+	if (src->surface.micro_tile_mode == V_009910_ADDR_SURF_DISPLAY_MICRO_TILING)
+		templ.bind = PIPE_BIND_SCANOUT;
+	else
+		templ.bind = 0;
+
+	tmp = ctx->screen->resource_create(ctx->screen, &templ);
+	if (!tmp)
+		return false;
+
+	assert(src->surface.micro_tile_mode ==
+	       ((struct r600_texture*)tmp)->surface.micro_tile_mode);
+
+	/* resolve */
+	si_blitter_begin(ctx, SI_COLOR_RESOLVE |
+			 (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
+	util_blitter_custom_resolve_color(sctx->blitter, tmp, 0, 0,
+					  info->src.resource, info->src.box.z,
+					  sample_mask, sctx->custom_blend_resolve,
+					  format);
+	si_blitter_end(ctx);
+
+	/* blit */
+	blit = *info;
+	blit.src.resource = tmp;
+	blit.src.box.z = 0;
+
+	si_blitter_begin(ctx, SI_BLIT |
+			 (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
+	util_blitter_blit(sctx->blitter, &blit);
+	si_blitter_end(ctx);
+
+	pipe_resource_reference(&tmp, NULL);
+	return true;
 }
 
 static void si_blit(struct pipe_context *ctx,
 		    const struct pipe_blit_info *info)
 {
 	struct si_context *sctx = (struct si_context*)ctx;
+	struct r600_texture *rdst = (struct r600_texture *)info->dst.resource;
 
 	if (do_hardware_msaa_resolve(ctx, info)) {
 		return;
 	}
 
+	/* Using SDMA for copying to a linear texture in GTT is much faster.
+	 * This improves DRI PRIME performance.
+	 *
+	 * resource_copy_region can't do this yet, because dma_copy calls it
+	 * on failure (recursion).
+	 */
+	if (rdst->surface.level[info->dst.level].mode ==
+	    RADEON_SURF_MODE_LINEAR_ALIGNED &&
+	    sctx->b.dma_copy &&
+	    util_can_blit_via_copy_region(info, false)) {
+		sctx->b.dma_copy(ctx, info->dst.resource, info->dst.level,
+				 info->dst.box.x, info->dst.box.y,
+				 info->dst.box.z,
+				 info->src.resource, info->src.level,
+				 &info->src.box);
+		return;
+	}
+
 	assert(util_blitter_is_blit_supported(sctx->blitter, info));
 
 	/* The driver doesn't decompress resources automatically while
 	 * u_blitter is rendering. */
-	si_decompress_subresource(ctx, info->src.resource, info->src.level,
+	vi_dcc_disable_if_incompatible_format(&sctx->b, info->src.resource,
+					      info->src.level,
+					      info->src.format);
+	vi_dcc_disable_if_incompatible_format(&sctx->b, info->dst.resource,
+					      info->dst.level,
+					      info->dst.format);
+	si_decompress_subresource(ctx, info->src.resource, info->mask,
+				  info->src.level,
 				  info->src.box.z,
 				  info->src.box.z + info->src.box.depth - 1);
 
@@ -818,16 +1153,61 @@ static void si_blit(struct pipe_context *ctx,
 	si_blitter_end(ctx);
 }
 
+static boolean si_generate_mipmap(struct pipe_context *ctx,
+				  struct pipe_resource *tex,
+				  enum pipe_format format,
+				  unsigned base_level, unsigned last_level,
+				  unsigned first_layer, unsigned last_layer)
+{
+	struct si_context *sctx = (struct si_context*)ctx;
+	struct r600_texture *rtex = (struct r600_texture *)tex;
+
+	if (!util_blitter_is_copy_supported(sctx->blitter, tex, tex))
+		return false;
+
+	/* The driver doesn't decompress resources automatically while
+	 * u_blitter is rendering. */
+	vi_dcc_disable_if_incompatible_format(&sctx->b, tex, base_level,
+					      format);
+	si_decompress_subresource(ctx, tex, PIPE_MASK_RGBAZS,
+				  base_level, first_layer, last_layer);
+
+	/* Clear dirty_level_mask for the levels that will be overwritten. */
+	assert(base_level < last_level);
+	rtex->dirty_level_mask &= ~u_bit_consecutive(base_level + 1,
+						     last_level - base_level);
+
+	si_blitter_begin(ctx, SI_BLIT | SI_DISABLE_RENDER_COND);
+	util_blitter_generate_mipmap(sctx->blitter, tex, format,
+				     base_level, last_level,
+				     first_layer, last_layer);
+	si_blitter_end(ctx);
+	return true;
+}
+
 static void si_flush_resource(struct pipe_context *ctx,
 			      struct pipe_resource *res)
 {
 	struct r600_texture *rtex = (struct r600_texture*)res;
 
 	assert(res->target != PIPE_BUFFER);
+	assert(!rtex->dcc_separate_buffer || rtex->dcc_gather_statistics);
+
+	/* st/dri calls flush twice per frame (not a bug), this prevents double
+	 * decompression. */
+	if (rtex->dcc_separate_buffer && !rtex->separate_dcc_dirty)
+		return;
 
 	if (!rtex->is_depth && (rtex->cmask.size || rtex->dcc_offset)) {
 		si_blit_decompress_color(ctx, rtex, 0, res->last_level,
-					 0, util_max_layer(res, 0), false);
+					 0, util_max_layer(res, 0),
+					 rtex->dcc_separate_buffer != NULL);
+	}
+
+	/* Always do the analysis even if DCC is disabled at the moment. */
+	if (rtex->dcc_gather_statistics && rtex->separate_dcc_dirty) {
+		rtex->separate_dcc_dirty = false;
+		vi_separate_dcc_process_and_reset_stats(ctx, rtex);
 	}
 }
 
@@ -898,7 +1278,8 @@ static void si_pipe_clear_buffer(struct pipe_context *ctx,
 		dword_value = *(uint32_t*)clear_value_ptr;
 	}
 
-	sctx->b.clear_buffer(ctx, dst, offset, size, dword_value, false);
+	sctx->b.clear_buffer(ctx, dst, offset, size, dword_value,
+			     R600_COHERENCY_SHADER);
 }
 
 void si_init_blit_functions(struct si_context *sctx)
@@ -910,6 +1291,7 @@ void si_init_blit_functions(struct si_context *sctx)
 	sctx->b.b.resource_copy_region = si_resource_copy_region;
 	sctx->b.b.blit = si_blit;
 	sctx->b.b.flush_resource = si_flush_resource;
+	sctx->b.b.generate_mipmap = si_generate_mipmap;
 	sctx->b.blit_decompress_depth = si_blit_decompress_depth;
 	sctx->b.decompress_dcc = si_decompress_dcc;
 }