radeonsi: apply a multi-wave workgroup SPI bug workaround to affected CIK chips
[mesa.git] / src / gallium / drivers / radeonsi / si_blit.c
index dee3c2f32d33481af09ebdc1bf0310e368a3c297..f5f49c1380ca395d71dbb74178f39df0e5409731 100644 (file)
@@ -100,67 +100,99 @@ static unsigned u_max_sample(struct pipe_resource *r)
        return r->nr_samples ? r->nr_samples - 1 : 0;
 }
 
-static void si_blit_decompress_depth(struct pipe_context *ctx,
-                                    struct r600_texture *texture,
-                                    struct r600_texture *staging,
-                                    unsigned first_level, unsigned last_level,
-                                    unsigned first_layer, unsigned last_layer,
-                                    unsigned first_sample, unsigned last_sample)
+static unsigned
+si_blit_dbcb_copy(struct si_context *sctx,
+                 struct r600_texture *src,
+                 struct r600_texture *dst,
+                 unsigned planes, unsigned level_mask,
+                 unsigned first_layer, unsigned last_layer,
+                 unsigned first_sample, unsigned last_sample)
 {
-       struct si_context *sctx = (struct si_context *)ctx;
-       unsigned layer, level, sample, checked_last_layer, max_layer;
-       float depth = 1.0f;
-       const struct util_format_description *desc;
-
-       assert(staging != NULL && "use si_blit_decompress_zs_in_place instead");
+       struct pipe_surface surf_tmpl = {{0}};
+       unsigned layer, sample, checked_last_layer, max_layer;
+       unsigned fully_copied_levels = 0;
 
-       desc = util_format_description(staging->resource.b.b.format);
-
-       if (util_format_has_depth(desc))
+       if (planes & PIPE_MASK_Z)
                sctx->dbcb_depth_copy_enabled = true;
-       if (util_format_has_stencil(desc))
+       if (planes & PIPE_MASK_S)
                sctx->dbcb_stencil_copy_enabled = true;
+       si_mark_atom_dirty(sctx, &sctx->db_render_state);
 
        assert(sctx->dbcb_depth_copy_enabled || sctx->dbcb_stencil_copy_enabled);
 
-       for (level = first_level; level <= last_level; level++) {
+       while (level_mask) {
+               unsigned level = u_bit_scan(&level_mask);
+
                /* The smaller the mipmap level, the less layers there are
                 * as far as 3D textures are concerned. */
-               max_layer = util_max_layer(&texture->resource.b.b, level);
+               max_layer = util_max_layer(&src->resource.b.b, level);
                checked_last_layer = MIN2(last_layer, max_layer);
 
+               surf_tmpl.u.tex.level = level;
+
                for (layer = first_layer; layer <= checked_last_layer; layer++) {
-                       for (sample = first_sample; sample <= last_sample; sample++) {
-                               struct pipe_surface *zsurf, *cbsurf, surf_tmpl;
+                       struct pipe_surface *zsurf, *cbsurf;
 
-                               sctx->dbcb_copy_sample = sample;
-                               si_mark_atom_dirty(sctx, &sctx->db_render_state);
+                       surf_tmpl.format = src->resource.b.b.format;
+                       surf_tmpl.u.tex.first_layer = layer;
+                       surf_tmpl.u.tex.last_layer = layer;
 
-                               surf_tmpl.format = texture->resource.b.b.format;
-                               surf_tmpl.u.tex.level = level;
-                               surf_tmpl.u.tex.first_layer = layer;
-                               surf_tmpl.u.tex.last_layer = layer;
+                       zsurf = sctx->b.b.create_surface(&sctx->b.b, &src->resource.b.b, &surf_tmpl);
 
-                               zsurf = ctx->create_surface(ctx, &texture->resource.b.b, &surf_tmpl);
+                       surf_tmpl.format = dst->resource.b.b.format;
+                       cbsurf = sctx->b.b.create_surface(&sctx->b.b, &dst->resource.b.b, &surf_tmpl);
 
-                               surf_tmpl.format = staging->resource.b.b.format;
-                               cbsurf = ctx->create_surface(ctx,
-                                               (struct pipe_resource*)staging, &surf_tmpl);
+                       for (sample = first_sample; sample <= last_sample; sample++) {
+                               if (sample != sctx->dbcb_copy_sample) {
+                                       sctx->dbcb_copy_sample = sample;
+                                       si_mark_atom_dirty(sctx, &sctx->db_render_state);
+                               }
 
-                               si_blitter_begin(ctx, SI_DECOMPRESS);
+                               si_blitter_begin(&sctx->b.b, SI_DECOMPRESS);
                                util_blitter_custom_depth_stencil(sctx->blitter, zsurf, cbsurf, 1 << sample,
-                                                                 sctx->custom_dsa_flush, depth);
-                               si_blitter_end(ctx);
-
-                               pipe_surface_reference(&zsurf, NULL);
-                               pipe_surface_reference(&cbsurf, NULL);
+                                                                 sctx->custom_dsa_flush, 1.0f);
+                               si_blitter_end(&sctx->b.b);
                        }
+
+                       pipe_surface_reference(&zsurf, NULL);
+                       pipe_surface_reference(&cbsurf, NULL);
                }
+
+               if (first_layer == 0 && last_layer >= max_layer &&
+                   first_sample == 0 && last_sample >= u_max_sample(&src->resource.b.b))
+                       fully_copied_levels |= 1u << level;
        }
 
        sctx->dbcb_depth_copy_enabled = false;
        sctx->dbcb_stencil_copy_enabled = false;
        si_mark_atom_dirty(sctx, &sctx->db_render_state);
+
+       return fully_copied_levels;
+}
+
+static void si_blit_decompress_depth(struct pipe_context *ctx,
+                                    struct r600_texture *texture,
+                                    struct r600_texture *staging,
+                                    unsigned first_level, unsigned last_level,
+                                    unsigned first_layer, unsigned last_layer,
+                                    unsigned first_sample, unsigned last_sample)
+{
+       const struct util_format_description *desc;
+       unsigned planes = 0;
+
+       assert(staging != NULL && "use si_blit_decompress_zs_in_place instead");
+
+       desc = util_format_description(staging->resource.b.b.format);
+
+       if (util_format_has_depth(desc))
+               planes |= PIPE_MASK_Z;
+       if (util_format_has_stencil(desc))
+               planes |= PIPE_MASK_S;
+
+       si_blit_dbcb_copy(
+               (struct si_context *)ctx, texture, staging, planes,
+               u_bit_consecutive(first_level, last_level - first_level + 1),
+               first_layer, last_layer, first_sample, last_sample);
 }
 
 /* Helper function for si_blit_decompress_zs_in_place.
@@ -213,7 +245,7 @@ si_blit_decompress_zs_planes_in_place(struct si_context *sctx,
 
                /* The texture will always be dirty if some layers aren't flushed.
                 * I don't think this case occurs often though. */
-               if (first_layer == 0 && last_layer == max_layer) {
+               if (first_layer == 0 && last_layer >= max_layer) {
                        fully_decompressed_mask |= 1u << level;
                }
        }
@@ -228,51 +260,122 @@ si_blit_decompress_zs_planes_in_place(struct si_context *sctx,
        si_mark_atom_dirty(sctx, &sctx->db_render_state);
 }
 
-/* Decompress Z and/or S planes in place, depending on mask.
+/* Helper function of si_flush_depth_texture: decompress the given levels
+ * of Z and/or S planes in place.
  */
 static void
 si_blit_decompress_zs_in_place(struct si_context *sctx,
                               struct r600_texture *texture,
-                              unsigned planes,
-                              unsigned first_level, unsigned last_level,
+                              unsigned levels_z, unsigned levels_s,
                               unsigned first_layer, unsigned last_layer)
 {
-       unsigned level_mask =
-               u_bit_consecutive(first_level, last_level - first_level + 1);
-       unsigned cur_level_mask;
+       unsigned both = levels_z & levels_s;
 
        /* First, do combined Z & S decompresses for levels that need it. */
-       if (planes == (PIPE_MASK_Z | PIPE_MASK_S)) {
-               cur_level_mask =
-                       level_mask &
-                       texture->dirty_level_mask &
-                       texture->stencil_dirty_level_mask;
+       if (both) {
                si_blit_decompress_zs_planes_in_place(
                                sctx, texture, PIPE_MASK_Z | PIPE_MASK_S,
-                               cur_level_mask,
+                               both,
                                first_layer, last_layer);
-               level_mask &= ~cur_level_mask;
+               levels_z &= ~both;
+               levels_s &= ~both;
        }
 
        /* Now do separate Z and S decompresses. */
-       if (planes & PIPE_MASK_Z) {
-               cur_level_mask = level_mask & texture->dirty_level_mask;
+       if (levels_z) {
                si_blit_decompress_zs_planes_in_place(
                                sctx, texture, PIPE_MASK_Z,
-                               cur_level_mask,
+                               levels_z,
                                first_layer, last_layer);
-               level_mask &= ~cur_level_mask;
        }
 
-       if (planes & PIPE_MASK_S) {
-               cur_level_mask = level_mask & texture->stencil_dirty_level_mask;
+       if (levels_s) {
                si_blit_decompress_zs_planes_in_place(
                                sctx, texture, PIPE_MASK_S,
-                               cur_level_mask,
+                               levels_s,
                                first_layer, last_layer);
        }
 }
 
+static void
+si_flush_depth_texture(struct si_context *sctx,
+                      struct r600_texture *tex,
+                      unsigned required_planes,
+                      unsigned first_level, unsigned last_level,
+                      unsigned first_layer, unsigned last_layer)
+{
+       unsigned inplace_planes = 0;
+       unsigned copy_planes = 0;
+       unsigned level_mask = u_bit_consecutive(first_level, last_level - first_level + 1);
+       unsigned levels_z = 0;
+       unsigned levels_s = 0;
+
+       if (required_planes & PIPE_MASK_Z) {
+               levels_z = level_mask & tex->dirty_level_mask;
+
+               if (levels_z) {
+                       if (r600_can_sample_zs(tex, false))
+                               inplace_planes |= PIPE_MASK_Z;
+                       else
+                               copy_planes |= PIPE_MASK_Z;
+               }
+       }
+       if (required_planes & PIPE_MASK_S) {
+               levels_s = level_mask & tex->stencil_dirty_level_mask;
+
+               if (levels_s) {
+                       if (r600_can_sample_zs(tex, true))
+                               inplace_planes |= PIPE_MASK_S;
+                       else
+                               copy_planes |= PIPE_MASK_S;
+               }
+       }
+
+       assert(!tex->tc_compatible_htile || levels_z == 0);
+
+       /* We may have to allocate the flushed texture here when called from
+        * si_decompress_subresource.
+        */
+       if (copy_planes &&
+           (tex->flushed_depth_texture ||
+            r600_init_flushed_depth_texture(&sctx->b.b, &tex->resource.b.b, NULL))) {
+               struct r600_texture *dst = tex->flushed_depth_texture;
+               unsigned fully_copied_levels;
+               unsigned levels = 0;
+
+               assert(tex->flushed_depth_texture);
+
+               if (util_format_is_depth_and_stencil(dst->resource.b.b.format))
+                       copy_planes = PIPE_MASK_Z | PIPE_MASK_S;
+
+               if (copy_planes & PIPE_MASK_Z) {
+                       levels |= levels_z;
+                       levels_z = 0;
+               }
+               if (copy_planes & PIPE_MASK_S) {
+                       levels |= levels_s;
+                       levels_s = 0;
+               }
+
+               fully_copied_levels = si_blit_dbcb_copy(
+                       sctx, tex, dst, copy_planes, levels,
+                       first_layer, last_layer,
+                       0, u_max_sample(&tex->resource.b.b));
+
+               if (copy_planes & PIPE_MASK_Z)
+                       tex->dirty_level_mask &= ~fully_copied_levels;
+               if (copy_planes & PIPE_MASK_S)
+                       tex->stencil_dirty_level_mask &= ~fully_copied_levels;
+       }
+
+       if (inplace_planes) {
+               si_blit_decompress_zs_in_place(
+                       sctx, tex,
+                       levels_z, levels_s,
+                       first_layer, last_layer);
+       }
+}
+
 static void
 si_flush_depth_textures(struct si_context *sctx,
                        struct si_textures_info *textures)
@@ -292,13 +395,13 @@ si_flush_depth_textures(struct si_context *sctx,
                sview = (struct si_sampler_view*)view;
 
                tex = (struct r600_texture *)view->texture;
-               assert(tex->is_depth && !tex->is_flushing_texture);
+               assert(tex->db_compatible);
 
-               si_blit_decompress_zs_in_place(sctx, tex,
-                                              sview->is_stencil_sampler ? PIPE_MASK_S
-                                                                        : PIPE_MASK_Z,
-                                              view->u.tex.first_level, view->u.tex.last_level,
-                                              0, util_max_layer(&tex->resource.b.b, view->u.tex.first_level));
+               si_flush_depth_texture(
+                               sctx, tex,
+                               sview->is_stencil_sampler ? PIPE_MASK_S : PIPE_MASK_Z,
+                               view->u.tex.first_level, view->u.tex.last_level,
+                               0, util_max_layer(&tex->resource.b.b, view->u.tex.first_level));
        }
 }
 
@@ -325,7 +428,7 @@ static void si_blit_decompress_color(struct pipe_context *ctx,
                /* disable levels without DCC */
                for (int i = first_level; i <= last_level; i++) {
                        if (!rtex->dcc_offset ||
-                           !rtex->surface.level[i].dcc_enabled)
+                           i >= rtex->surface.num_dcc_levels)
                                level_mask &= ~(1 << i);
                }
        } else if (rtex->fmask.size) {
@@ -360,7 +463,7 @@ static void si_blit_decompress_color(struct pipe_context *ctx,
 
                /* The texture will always be dirty if some layers aren't flushed.
                 * I don't think this case occurs often though. */
-               if (first_layer == 0 && last_layer == max_layer) {
+               if (first_layer == 0 && last_layer >= max_layer) {
                        rtex->dirty_level_mask &= ~(1 << level);
                }
        }
@@ -383,7 +486,9 @@ si_decompress_sampler_color_textures(struct si_context *sctx,
                assert(view);
 
                tex = (struct r600_texture *)view->texture;
-               assert(tex->cmask.size || tex->fmask.size || tex->dcc_offset);
+               /* CMASK or DCC can be discarded and we can still end up here. */
+               if (!tex->cmask.size && !tex->fmask.size && !tex->dcc_offset)
+                       continue;
 
                si_blit_decompress_color(&sctx->b.b, tex,
                                         view->u.tex.first_level, view->u.tex.last_level,
@@ -455,10 +560,8 @@ static void si_check_render_feedback_textures(struct si_context *sctx,
                                render_feedback = true;
                }
 
-               if (render_feedback) {
-                       struct si_screen *screen = sctx->screen;
-                       r600_texture_disable_dcc(&screen->b, tex);
-               }
+               if (render_feedback)
+                       r600_texture_disable_dcc(&sctx->b, tex);
        }
 }
 
@@ -497,10 +600,8 @@ static void si_check_render_feedback_images(struct si_context *sctx,
                                render_feedback = true;
                }
 
-               if (render_feedback) {
-                       struct si_screen *screen = sctx->screen;
-                       r600_texture_disable_dcc(&screen->b, tex);
-               }
+               if (render_feedback)
+                       r600_texture_disable_dcc(&sctx->b, tex);
        }
 }
 
@@ -601,7 +702,10 @@ static void si_clear(struct pipe_context *ctx, unsigned buffers,
            zsbuf->u.tex.level == 0 &&
            zsbuf->u.tex.first_layer == 0 &&
            zsbuf->u.tex.last_layer == util_max_layer(&zstex->resource.b.b, 0)) {
-               if (buffers & PIPE_CLEAR_DEPTH) {
+               /* TC-compatible HTILE only supports depth clears to 0 or 1. */
+               if (buffers & PIPE_CLEAR_DEPTH &&
+                   (!zstex->tc_compatible_htile ||
+                    depth == 0 || depth == 1)) {
                        /* Need to disable EXPCLEAR temporarily if clearing
                         * to a new value. */
                        if (!zstex->depth_cleared || zstex->depth_clear_value != depth) {
@@ -615,7 +719,9 @@ static void si_clear(struct pipe_context *ctx, unsigned buffers,
                        si_mark_atom_dirty(sctx, &sctx->db_render_state);
                }
 
-               if (buffers & PIPE_CLEAR_STENCIL) {
+               /* TC-compatible HTILE only supports stencil clears to 0. */
+               if (buffers & PIPE_CLEAR_STENCIL &&
+                   (!zstex->tc_compatible_htile || stencil == 0)) {
                        stencil &= 0xff;
 
                        /* Need to disable EXPCLEAR temporarily if clearing
@@ -657,11 +763,13 @@ static void si_clear_render_target(struct pipe_context *ctx,
                                   struct pipe_surface *dst,
                                   const union pipe_color_union *color,
                                   unsigned dstx, unsigned dsty,
-                                  unsigned width, unsigned height)
+                                  unsigned width, unsigned height,
+                                  bool render_condition_enabled)
 {
        struct si_context *sctx = (struct si_context *)ctx;
 
-       si_blitter_begin(ctx, SI_CLEAR_SURFACE);
+       si_blitter_begin(ctx, SI_CLEAR_SURFACE |
+                        (render_condition_enabled ? 0 : SI_DISABLE_RENDER_COND));
        util_blitter_clear_render_target(sctx->blitter, dst, color,
                                         dstx, dsty, width, height);
        si_blitter_end(ctx);
@@ -673,11 +781,13 @@ static void si_clear_depth_stencil(struct pipe_context *ctx,
                                   double depth,
                                   unsigned stencil,
                                   unsigned dstx, unsigned dsty,
-                                  unsigned width, unsigned height)
+                                  unsigned width, unsigned height,
+                                  bool render_condition_enabled)
 {
        struct si_context *sctx = (struct si_context *)ctx;
 
-       si_blitter_begin(ctx, SI_CLEAR_SURFACE);
+       si_blitter_begin(ctx, SI_CLEAR_SURFACE |
+                        (render_condition_enabled ? 0 : SI_DISABLE_RENDER_COND));
        util_blitter_clear_depth_stencil(sctx->blitter, dst, clear_flags, depth, stencil,
                                         dstx, dsty, width, height);
        si_blitter_end(ctx);
@@ -695,15 +805,15 @@ static void si_decompress_subresource(struct pipe_context *ctx,
        struct si_context *sctx = (struct si_context *)ctx;
        struct r600_texture *rtex = (struct r600_texture*)tex;
 
-       if (rtex->is_depth && !rtex->is_flushing_texture) {
+       if (rtex->db_compatible) {
                planes &= PIPE_MASK_Z | PIPE_MASK_S;
 
                if (!(rtex->surface.flags & RADEON_SURF_SBUFFER))
                        planes &= ~PIPE_MASK_S;
 
-               si_blit_decompress_zs_in_place(sctx, rtex, planes,
-                                              level, level,
-                                              first_layer, last_layer);
+               si_flush_depth_texture(sctx, rtex, planes,
+                                      level, level,
+                                      first_layer, last_layer);
        } else if (rtex->fmask.size || rtex->cmask.size || rtex->dcc_offset) {
                si_blit_decompress_color(ctx, rtex, level, level,
                                         first_layer, last_layer, false);
@@ -729,6 +839,7 @@ void si_resource_copy_region(struct pipe_context *ctx,
                             const struct pipe_box *src_box)
 {
        struct si_context *sctx = (struct si_context *)ctx;
+       struct r600_texture *rsrc = (struct r600_texture*)src;
        struct pipe_surface *dst_view, dst_templ;
        struct pipe_sampler_view src_templ, *src_view;
        unsigned dst_width, dst_height, src_width0, src_height0;
@@ -758,7 +869,7 @@ void si_resource_copy_region(struct pipe_context *ctx,
 
        if (util_format_is_compressed(src->format) ||
            util_format_is_compressed(dst->format)) {
-               unsigned blocksize = util_format_get_blocksize(src->format);
+               unsigned blocksize = rsrc->surface.bpe;
 
                if (blocksize == 8)
                        src_templ.format = PIPE_FORMAT_R16G16B16A16_UINT; /* 64-bit block */
@@ -800,7 +911,7 @@ void si_resource_copy_region(struct pipe_context *ctx,
                        sbox.width = util_format_get_nblocksx(src->format, src_box->width);
                        src_box = &sbox;
                } else {
-                       unsigned blocksize = util_format_get_blocksize(src->format);
+                       unsigned blocksize = rsrc->surface.bpe;
 
                        switch (blocksize) {
                        case 1:
@@ -848,7 +959,7 @@ void si_resource_copy_region(struct pipe_context *ctx,
        util_blitter_blit_generic(sctx->blitter, dst_view, &dstbox,
                                  src_view, src_box, src_width0, src_height0,
                                  PIPE_MASK_RGBAZS, PIPE_TEX_FILTER_NEAREST, NULL,
-                                 FALSE);
+                                 false);
        si_blitter_end(ctx);
 
        pipe_surface_reference(&dst_view, NULL);
@@ -859,7 +970,9 @@ static bool do_hardware_msaa_resolve(struct pipe_context *ctx,
                                     const struct pipe_blit_info *info)
 {
        struct si_context *sctx = (struct si_context*)ctx;
+       struct r600_texture *src = (struct r600_texture*)info->src.resource;
        struct r600_texture *dst = (struct r600_texture*)info->dst.resource;
+       struct r600_texture *rtmp;
        unsigned dst_width = u_minify(info->dst.resource->width0, info->dst.level);
        unsigned dst_height = u_minify(info->dst.resource->height0, info->dst.level);
        enum pipe_format format = info->src.format;
@@ -901,10 +1014,31 @@ static bool do_hardware_msaa_resolve(struct pipe_context *ctx,
            info->src.box.width == dst_width &&
            info->src.box.height == dst_height &&
            info->src.box.depth == 1 &&
-           dst->surface.level[info->dst.level].mode >= RADEON_SURF_MODE_1D &&
-           !(dst->surface.flags & RADEON_SURF_SCANOUT) &&
-           (!dst->cmask.size || !dst->dirty_level_mask) && /* dst cannot be fast-cleared */
-           !dst->dcc_offset) {
+           !dst->surface.is_linear &&
+           (!dst->cmask.size || !dst->dirty_level_mask)) { /* dst cannot be fast-cleared */
+               /* Check the last constraint. */
+               if (src->surface.micro_tile_mode != dst->surface.micro_tile_mode) {
+                       /* The next fast clear will switch to this mode to
+                        * get direct hw resolve next time if the mode is
+                        * different now.
+                        */
+                       src->last_msaa_resolve_target_micro_mode =
+                               dst->surface.micro_tile_mode;
+                       goto resolve_to_temp;
+               }
+
+               /* Resolving into a surface with DCC is unsupported. Since
+                * it's being overwritten anyway, clear it to uncompressed.
+                * This is still the fastest codepath even with this clear.
+                */
+               if (dst->dcc_offset &&
+                   info->dst.level < dst->surface.num_dcc_levels) {
+                       vi_dcc_clear_level(&sctx->b, dst, info->dst.level,
+                                          0xFFFFFFFF);
+                       dst->dirty_level_mask &= ~(1 << info->dst.level);
+               }
+
+               /* Resolve directly from src to dst. */
                si_blitter_begin(ctx, SI_COLOR_RESOLVE |
                                 (info->render_condition_enable ? 0 : SI_DISABLE_RENDER_COND));
                util_blitter_custom_resolve_color(sctx->blitter,
@@ -917,6 +1051,7 @@ static bool do_hardware_msaa_resolve(struct pipe_context *ctx,
                return true;
        }
 
+resolve_to_temp:
        /* Shader-based resolve is VERY SLOW. Instead, resolve into
         * a temporary texture and blit.
         */
@@ -931,9 +1066,19 @@ static bool do_hardware_msaa_resolve(struct pipe_context *ctx,
        templ.flags = R600_RESOURCE_FLAG_FORCE_TILING |
                      R600_RESOURCE_FLAG_DISABLE_DCC;
 
+       /* The src and dst microtile modes must be the same. */
+       if (src->surface.micro_tile_mode == RADEON_MICRO_MODE_DISPLAY)
+               templ.bind = PIPE_BIND_SCANOUT;
+       else
+               templ.bind = 0;
+
        tmp = ctx->screen->resource_create(ctx->screen, &templ);
        if (!tmp)
                return false;
+       rtmp = (struct r600_texture*)tmp;
+
+       assert(!rtmp->surface.is_linear);
+       assert(src->surface.micro_tile_mode == rtmp->surface.micro_tile_mode);
 
        /* resolve */
        si_blitter_begin(ctx, SI_COLOR_RESOLVE |
@@ -962,15 +1107,39 @@ static void si_blit(struct pipe_context *ctx,
                    const struct pipe_blit_info *info)
 {
        struct si_context *sctx = (struct si_context*)ctx;
+       struct r600_texture *rdst = (struct r600_texture *)info->dst.resource;
 
        if (do_hardware_msaa_resolve(ctx, info)) {
                return;
        }
 
+       /* Using SDMA for copying to a linear texture in GTT is much faster.
+        * This improves DRI PRIME performance.
+        *
+        * resource_copy_region can't do this yet, because dma_copy calls it
+        * on failure (recursion).
+        */
+       if (rdst->surface.is_linear &&
+           sctx->b.dma_copy &&
+           util_can_blit_via_copy_region(info, false)) {
+               sctx->b.dma_copy(ctx, info->dst.resource, info->dst.level,
+                                info->dst.box.x, info->dst.box.y,
+                                info->dst.box.z,
+                                info->src.resource, info->src.level,
+                                &info->src.box);
+               return;
+       }
+
        assert(util_blitter_is_blit_supported(sctx->blitter, info));
 
        /* The driver doesn't decompress resources automatically while
         * u_blitter is rendering. */
+       vi_dcc_disable_if_incompatible_format(&sctx->b, info->src.resource,
+                                             info->src.level,
+                                             info->src.format);
+       vi_dcc_disable_if_incompatible_format(&sctx->b, info->dst.resource,
+                                             info->dst.level,
+                                             info->dst.format);
        si_decompress_subresource(ctx, info->src.resource, info->mask,
                                  info->src.level,
                                  info->src.box.z,
@@ -986,16 +1155,61 @@ static void si_blit(struct pipe_context *ctx,
        si_blitter_end(ctx);
 }
 
+static boolean si_generate_mipmap(struct pipe_context *ctx,
+                                 struct pipe_resource *tex,
+                                 enum pipe_format format,
+                                 unsigned base_level, unsigned last_level,
+                                 unsigned first_layer, unsigned last_layer)
+{
+       struct si_context *sctx = (struct si_context*)ctx;
+       struct r600_texture *rtex = (struct r600_texture *)tex;
+
+       if (!util_blitter_is_copy_supported(sctx->blitter, tex, tex))
+               return false;
+
+       /* The driver doesn't decompress resources automatically while
+        * u_blitter is rendering. */
+       vi_dcc_disable_if_incompatible_format(&sctx->b, tex, base_level,
+                                             format);
+       si_decompress_subresource(ctx, tex, PIPE_MASK_RGBAZS,
+                                 base_level, first_layer, last_layer);
+
+       /* Clear dirty_level_mask for the levels that will be overwritten. */
+       assert(base_level < last_level);
+       rtex->dirty_level_mask &= ~u_bit_consecutive(base_level + 1,
+                                                    last_level - base_level);
+
+       si_blitter_begin(ctx, SI_BLIT | SI_DISABLE_RENDER_COND);
+       util_blitter_generate_mipmap(sctx->blitter, tex, format,
+                                    base_level, last_level,
+                                    first_layer, last_layer);
+       si_blitter_end(ctx);
+       return true;
+}
+
 static void si_flush_resource(struct pipe_context *ctx,
                              struct pipe_resource *res)
 {
        struct r600_texture *rtex = (struct r600_texture*)res;
 
        assert(res->target != PIPE_BUFFER);
+       assert(!rtex->dcc_separate_buffer || rtex->dcc_gather_statistics);
+
+       /* st/dri calls flush twice per frame (not a bug), this prevents double
+        * decompression. */
+       if (rtex->dcc_separate_buffer && !rtex->separate_dcc_dirty)
+               return;
 
        if (!rtex->is_depth && (rtex->cmask.size || rtex->dcc_offset)) {
                si_blit_decompress_color(ctx, rtex, 0, res->last_level,
-                                        0, util_max_layer(res, 0), false);
+                                        0, util_max_layer(res, 0),
+                                        rtex->dcc_separate_buffer != NULL);
+       }
+
+       /* Always do the analysis even if DCC is disabled at the moment. */
+       if (rtex->dcc_gather_statistics && rtex->separate_dcc_dirty) {
+               rtex->separate_dcc_dirty = false;
+               vi_separate_dcc_process_and_reset_stats(ctx, rtex);
        }
 }
 
@@ -1079,6 +1293,7 @@ void si_init_blit_functions(struct si_context *sctx)
        sctx->b.b.resource_copy_region = si_resource_copy_region;
        sctx->b.b.blit = si_blit;
        sctx->b.b.flush_resource = si_flush_resource;
+       sctx->b.b.generate_mipmap = si_generate_mipmap;
        sctx->b.blit_decompress_depth = si_blit_decompress_depth;
        sctx->b.decompress_dcc = si_decompress_dcc;
 }